+2 -1

Documentation/networking/xfrm_device.rst

reviewed

··· 169 169 170 170 hand the packet to napi_gro_receive() as usual 171 171 172 172 - In ESN mode, xdo_dev_state_advance_esn() is called from xfrm_replay_advance_esn(). 172 172 + In ESN mode, xdo_dev_state_advance_esn() is called from 173 173 + xfrm_replay_advance_esn() for RX, and xfrm_replay_overflow_offload_esn for TX. 173 174 Driver will check packet seq number and update HW ESN state machine if needed. 174 175 175 176 Packet offload mode:

+3

drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c

reviewed

··· 6562 6562 { 6563 6563 struct adapter *adap = netdev2adap(x->xso.dev); 6564 6564 6565 6565 + if (x->xso.dir != XFRM_DEV_OFFLOAD_IN) 6566 6566 + return; 6567 6567 + 6565 6568 if (!mutex_trylock(&uld_mutex)) { 6566 6569 dev_dbg(adap->pdev_dev, 6567 6570 "crypto uld critical resource is under use\n");

+15 -22

drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c

reviewed

··· 94 94 u32 esn, esn_msb; 95 95 u8 overlap; 96 96 97 97 - switch (x->xso.type) { 98 98 - case XFRM_DEV_OFFLOAD_PACKET: 99 99 - switch (x->xso.dir) { 100 100 - case XFRM_DEV_OFFLOAD_IN: 101 101 - esn = x->replay_esn->seq; 102 102 - esn_msb = x->replay_esn->seq_hi; 103 103 - break; 104 104 - case XFRM_DEV_OFFLOAD_OUT: 105 105 - esn = x->replay_esn->oseq; 106 106 - esn_msb = x->replay_esn->oseq_hi; 107 107 - break; 108 108 - default: 109 109 - WARN_ON(true); 110 110 - return false; 111 111 - } 112 112 - break; 113 113 - case XFRM_DEV_OFFLOAD_CRYPTO: 114 114 - /* Already parsed by XFRM core */ 97 97 + switch (x->xso.dir) { 98 98 + case XFRM_DEV_OFFLOAD_IN: 115 99 esn = x->replay_esn->seq; 100 100 + esn_msb = x->replay_esn->seq_hi; 101 101 + break; 102 102 + case XFRM_DEV_OFFLOAD_OUT: 103 103 + esn = x->replay_esn->oseq; 104 104 + esn_msb = x->replay_esn->oseq_hi; 116 105 break; 117 106 default: 118 107 WARN_ON(true); ··· 110 121 111 122 overlap = sa_entry->esn_state.overlap; 112 123 113 113 - if (esn >= x->replay_esn->replay_window) 114 114 - seq_bottom = esn - x->replay_esn->replay_window + 1; 124 124 + if (!x->replay_esn->replay_window) { 125 125 + seq_bottom = esn; 126 126 + } else { 127 127 + if (esn >= x->replay_esn->replay_window) 128 128 + seq_bottom = esn - x->replay_esn->replay_window + 1; 115 129 116 116 - if (x->xso.type == XFRM_DEV_OFFLOAD_CRYPTO) 117 117 - esn_msb = xfrm_replay_seqhi(x, htonl(seq_bottom)); 130 130 + if (x->xso.type == XFRM_DEV_OFFLOAD_CRYPTO) 131 131 + esn_msb = xfrm_replay_seqhi(x, htonl(seq_bottom)); 132 132 + } 118 133 119 134 if (sa_entry->esn_state.esn_msb) 120 135 sa_entry->esn_state.esn = esn;

+44

include/net/xfrm.h

reviewed

··· 38 38 #define XFRM_PROTO_COMP 108 39 39 #define XFRM_PROTO_IPIP 4 40 40 #define XFRM_PROTO_IPV6 41 41 41 + #define XFRM_PROTO_IPTFS IPPROTO_AGGFRAG 41 42 #define XFRM_PROTO_ROUTING IPPROTO_ROUTING 42 43 #define XFRM_PROTO_DSTOPTS IPPROTO_DSTOPTS 43 44 ··· 214 213 u16 family; 215 214 xfrm_address_t saddr; 216 215 int header_len; 216 216 + int enc_hdr_len; 217 217 int trailer_len; 218 218 u32 extra_flags; 219 219 struct xfrm_mark smark; ··· 305 303 * interpreted by xfrm_type methods. */ 306 304 void *data; 307 305 u8 dir; 306 306 + 307 307 + const struct xfrm_mode_cbs *mode_cbs; 308 308 + void *mode_data; 308 309 }; 309 310 310 311 static inline struct net *xs_net(struct xfrm_state *x) ··· 464 459 465 460 int xfrm_register_type_offload(const struct xfrm_type_offload *type, unsigned short family); 466 461 void xfrm_unregister_type_offload(const struct xfrm_type_offload *type, unsigned short family); 462 462 + 463 463 + /** 464 464 + * struct xfrm_mode_cbs - XFRM mode callbacks 465 465 + * @owner: module owner or NULL 466 466 + * @init_state: Add/init mode specific state in `xfrm_state *x` 467 467 + * @clone_state: Copy mode specific values from `orig` to new state `x` 468 468 + * @destroy_state: Cleanup mode specific state from `xfrm_state *x` 469 469 + * @user_init: Process mode specific netlink attributes from user 470 470 + * @copy_to_user: Add netlink attributes to `attrs` based on state in `x` 471 471 + * @sa_len: Return space required to store mode specific netlink attributes 472 472 + * @get_inner_mtu: Return avail payload space after removing encap overhead 473 473 + * @input: Process received packet from SA using mode 474 474 + * @output: Output given packet using mode 475 475 + * @prepare_output: Add mode specific encapsulation to packet in skb. On return 476 476 + * `transport_header` should point at ESP header, `network_header` should 477 477 + * point at outer IP header and `mac_header` should opint at the 478 478 + * protocol/nexthdr field of the outer IP. 479 479 + * 480 480 + * One should examine and understand the specific uses of these callbacks in 481 481 + * xfrm for further detail on how and when these functions are called. RTSL. 482 482 + */ 483 483 + struct xfrm_mode_cbs { 484 484 + struct module *owner; 485 485 + int (*init_state)(struct xfrm_state *x); 486 486 + int (*clone_state)(struct xfrm_state *x, struct xfrm_state *orig); 487 487 + void (*destroy_state)(struct xfrm_state *x); 488 488 + int (*user_init)(struct net *net, struct xfrm_state *x, 489 489 + struct nlattr **attrs, 490 490 + struct netlink_ext_ack *extack); 491 491 + int (*copy_to_user)(struct xfrm_state *x, struct sk_buff *skb); 492 492 + unsigned int (*sa_len)(const struct xfrm_state *x); 493 493 + u32 (*get_inner_mtu)(struct xfrm_state *x, int outer_mtu); 494 494 + int (*input)(struct xfrm_state *x, struct sk_buff *skb); 495 495 + int (*output)(struct net *net, struct sock *sk, struct sk_buff *skb); 496 496 + int (*prepare_output)(struct xfrm_state *x, struct sk_buff *skb); 497 497 + }; 498 498 + 499 499 + int xfrm_register_mode_cbs(u8 mode, const struct xfrm_mode_cbs *mode_cbs); 500 500 + void xfrm_unregister_mode_cbs(u8 mode); 467 501 468 502 static inline int xfrm_af2proto(unsigned int family) 469 503 {

+2

include/uapi/linux/in.h

reviewed

··· 79 79 #define IPPROTO_MPLS IPPROTO_MPLS 80 80 IPPROTO_ETHERNET = 143, /* Ethernet-within-IPv6 Encapsulation */ 81 81 #define IPPROTO_ETHERNET IPPROTO_ETHERNET 82 82 + IPPROTO_AGGFRAG = 144, /* AGGFRAG in ESP (RFC 9347) */ 83 83 + #define IPPROTO_AGGFRAG IPPROTO_AGGFRAG 82 84 IPPROTO_RAW = 255, /* Raw IP packets */ 83 85 #define IPPROTO_RAW IPPROTO_RAW 84 86 IPPROTO_SMC = 256, /* Shared Memory Communications */

+16

include/uapi/linux/ip.h

reviewed

··· 137 137 __u8 reserved; 138 138 }; 139 139 140 140 + struct ip_iptfs_hdr { 141 141 + __u8 subtype; /* 0*: basic, 1: CC */ 142 142 + __u8 flags; 143 143 + __be16 block_offset; 144 144 + }; 145 145 + 146 146 + struct ip_iptfs_cc_hdr { 147 147 + __u8 subtype; /* 0: basic, 1*: CC */ 148 148 + __u8 flags; 149 149 + __be16 block_offset; 150 150 + __be32 loss_rate; 151 151 + __be64 rtt_adelay_xdelay; 152 152 + __be32 tval; 153 153 + __be32 techo; 154 154 + }; 155 155 + 140 156 /* index values for the variables in ipv4_devconf */ 141 157 enum 142 158 {

+2 -1

include/uapi/linux/ipsec.h

reviewed

··· 14 14 IPSEC_MODE_ANY = 0, /* We do not support this for SA */ 15 15 IPSEC_MODE_TRANSPORT = 1, 16 16 IPSEC_MODE_TUNNEL = 2, 17 17 - IPSEC_MODE_BEET = 3 17 17 + IPSEC_MODE_BEET = 3, 18 18 + IPSEC_MODE_IPTFS = 4 18 19 }; 19 20 20 21 enum {

+2

include/uapi/linux/snmp.h

reviewed

··· 339 339 LINUX_MIB_XFRMACQUIREERROR, /* XfrmAcquireError */ 340 340 LINUX_MIB_XFRMOUTSTATEDIRERROR, /* XfrmOutStateDirError */ 341 341 LINUX_MIB_XFRMINSTATEDIRERROR, /* XfrmInStateDirError */ 342 342 + LINUX_MIB_XFRMINIPTFSERROR, /* XfrmInIptfsError */ 343 343 + LINUX_MIB_XFRMOUTNOQSPACE, /* XfrmOutNoQueueSpace */ 342 344 __LINUX_MIB_XFRMMAX 343 345 }; 344 346

+8 -1

include/uapi/linux/xfrm.h

reviewed

··· 158 158 #define XFRM_MODE_ROUTEOPTIMIZATION 2 159 159 #define XFRM_MODE_IN_TRIGGER 3 160 160 #define XFRM_MODE_BEET 4 161 161 - #define XFRM_MODE_MAX 5 161 161 + #define XFRM_MODE_IPTFS 5 162 162 + #define XFRM_MODE_MAX 6 162 163 163 164 /* Netlink configuration messages. */ 164 165 enum { ··· 324 323 XFRMA_SA_DIR, /* __u8 */ 325 324 XFRMA_NAT_KEEPALIVE_INTERVAL, /* __u32 in seconds for NAT keepalive */ 326 325 XFRMA_SA_PCPU, /* __u32 */ 326 326 + XFRMA_IPTFS_DROP_TIME, /* __u32 in: usec to wait for next seq */ 327 327 + XFRMA_IPTFS_REORDER_WINDOW, /* __u16 in: reorder window size (pkts) */ 328 328 + XFRMA_IPTFS_DONT_FRAG, /* out: don't use fragmentation */ 329 329 + XFRMA_IPTFS_INIT_DELAY, /* __u32 out: initial packet wait delay (usec) */ 330 330 + XFRMA_IPTFS_MAX_QSIZE, /* __u32 out: max ingress queue size (octets) */ 331 331 + XFRMA_IPTFS_PKT_SIZE, /* __u32 out: size of outer packet, 0 for PMTU */ 327 332 __XFRMA_MAX 328 333 329 334 #define XFRMA_OUTPUT_MARK XFRMA_SET_MARK /* Compatibility */

+2 -1

net/ipv4/esp4.c

reviewed

··· 816 816 } 817 817 818 818 skb_pull_rcsum(skb, hlen); 819 819 - if (x->props.mode == XFRM_MODE_TUNNEL) 819 819 + if (x->props.mode == XFRM_MODE_TUNNEL || 820 820 + x->props.mode == XFRM_MODE_IPTFS) 820 821 skb_reset_transport_header(skb); 821 822 else 822 823 skb_set_transport_header(skb, -ihl);

+2 -1

net/ipv6/esp6.c

reviewed

··· 859 859 skb_postpull_rcsum(skb, skb_network_header(skb), 860 860 skb_network_header_len(skb)); 861 861 skb_pull_rcsum(skb, hlen); 862 862 - if (x->props.mode == XFRM_MODE_TUNNEL) 862 862 + if (x->props.mode == XFRM_MODE_TUNNEL || 863 863 + x->props.mode == XFRM_MODE_IPTFS) 863 864 skb_reset_transport_header(skb); 864 865 else 865 866 skb_set_transport_header(skb, -hdr_len);

+2 -1

net/netfilter/nft_xfrm.c

reviewed

··· 112 112 return true; 113 113 } 114 114 115 115 - return mode == XFRM_MODE_BEET || mode == XFRM_MODE_TUNNEL; 115 115 + return mode == XFRM_MODE_BEET || mode == XFRM_MODE_TUNNEL || 116 116 + mode == XFRM_MODE_IPTFS; 116 117 } 117 118 118 119 static void nft_xfrm_state_get_key(const struct nft_xfrm *priv,

+16

net/xfrm/Kconfig

reviewed

··· 135 135 136 136 If unsure, say N. 137 137 138 138 + config XFRM_IPTFS 139 139 + tristate "IPsec IP-TFS/AGGFRAG (RFC 9347) encapsulation support" 140 140 + depends on XFRM 141 141 + help 142 142 + Information on the IP-TFS/AGGFRAG encapsulation can be found 143 143 + in RFC 9347. This feature supports demand driven (i.e., 144 144 + non-constant send rate) IP-TFS to take advantage of the 145 145 + AGGFRAG ESP payload encapsulation. This payload type 146 146 + supports aggregation and fragmentation of the inner IP 147 147 + packet stream which in turn yields higher small-packet 148 148 + bandwidth as well as reducing MTU/PMTU issues. Congestion 149 149 + control is unimplementated as the send rate is demand driven 150 150 + rather than constant. 151 151 + 152 152 + If unsure, say N. 153 153 + 138 154 config XFRM_ESPINTCP 139 155 bool 140 156

+1

net/xfrm/Makefile

reviewed

··· 21 21 obj-$(CONFIG_XFRM_USER_COMPAT) += xfrm_compat.o 22 22 obj-$(CONFIG_XFRM_IPCOMP) += xfrm_ipcomp.o 23 23 obj-$(CONFIG_XFRM_INTERFACE) += xfrm_interface.o 24 24 + obj-$(CONFIG_XFRM_IPTFS) += xfrm_iptfs.o 24 25 obj-$(CONFIG_XFRM_ESPINTCP) += espintcp.o 25 26 obj-$(CONFIG_DEBUG_INFO_BTF) += xfrm_state_bpf.o

+218

net/xfrm/trace_iptfs.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 2 + /* xfrm_trace_iptfs.h 3 3 + * 4 4 + * August 12 2023, Christian Hopps <chopps@labn.net> 5 5 + * 6 6 + * Copyright (c) 2023, LabN Consulting, L.L.C. 7 7 + */ 8 8 + 9 9 + #undef TRACE_SYSTEM 10 10 + #define TRACE_SYSTEM iptfs 11 11 + 12 12 + #if !defined(_TRACE_IPTFS_H) || defined(TRACE_HEADER_MULTI_READ) 13 13 + #define _TRACE_IPTFS_H 14 14 + 15 15 + #include <linux/kernel.h> 16 16 + #include <linux/skbuff.h> 17 17 + #include <linux/tracepoint.h> 18 18 + #include <net/ip.h> 19 19 + 20 20 + struct xfrm_iptfs_data; 21 21 + 22 22 + TRACE_EVENT(iptfs_egress_recv, 23 23 + TP_PROTO(struct sk_buff *skb, struct xfrm_iptfs_data *xtfs, u16 blkoff), 24 24 + TP_ARGS(skb, xtfs, blkoff), 25 25 + TP_STRUCT__entry(__field(struct sk_buff *, skb) 26 26 + __field(void *, head) 27 27 + __field(void *, head_pg_addr) 28 28 + __field(void *, pg0addr) 29 29 + __field(u32, skb_len) 30 30 + __field(u32, data_len) 31 31 + __field(u32, headroom) 32 32 + __field(u32, tailroom) 33 33 + __field(u32, tail) 34 34 + __field(u32, end) 35 35 + __field(u32, pg0off) 36 36 + __field(u8, head_frag) 37 37 + __field(u8, frag_list) 38 38 + __field(u8, nr_frags) 39 39 + __field(u16, blkoff)), 40 40 + TP_fast_assign(__entry->skb = skb; 41 41 + __entry->head = skb->head; 42 42 + __entry->skb_len = skb->len; 43 43 + __entry->data_len = skb->data_len; 44 44 + __entry->headroom = skb_headroom(skb); 45 45 + __entry->tailroom = skb_tailroom(skb); 46 46 + __entry->tail = (u32)skb->tail; 47 47 + __entry->end = (u32)skb->end; 48 48 + __entry->head_frag = skb->head_frag; 49 49 + __entry->frag_list = (bool)skb_shinfo(skb)->frag_list; 50 50 + __entry->nr_frags = skb_shinfo(skb)->nr_frags; 51 51 + __entry->blkoff = blkoff; 52 52 + __entry->head_pg_addr = page_address(virt_to_head_page(skb->head)); 53 53 + __entry->pg0addr = (__entry->nr_frags 54 54 + ? page_address(netmem_to_page(skb_shinfo(skb)->frags[0].netmem)) 55 55 + : NULL); 56 56 + __entry->pg0off = (__entry->nr_frags 57 57 + ? skb_shinfo(skb)->frags[0].offset 58 58 + : 0); 59 59 + ), 60 60 + TP_printk("EGRESS: skb=%p len=%u data_len=%u headroom=%u head_frag=%u frag_list=%u nr_frags=%u blkoff=%u\n\t\ttailroom=%u tail=%u end=%u head=%p hdpgaddr=%p pg0->addr=%p pg0->data=%p pg0->off=%u", 61 61 + __entry->skb, __entry->skb_len, __entry->data_len, __entry->headroom, 62 62 + __entry->head_frag, __entry->frag_list, __entry->nr_frags, __entry->blkoff, 63 63 + __entry->tailroom, __entry->tail, __entry->end, __entry->head, 64 64 + __entry->head_pg_addr, __entry->pg0addr, __entry->pg0addr + __entry->pg0off, 65 65 + __entry->pg0off) 66 66 + ) 67 67 + 68 68 + DECLARE_EVENT_CLASS(iptfs_ingress_preq_event, 69 69 + TP_PROTO(struct sk_buff *skb, struct xfrm_iptfs_data *xtfs, 70 70 + u32 pmtu, u8 was_gso), 71 71 + TP_ARGS(skb, xtfs, pmtu, was_gso), 72 72 + TP_STRUCT__entry(__field(struct sk_buff *, skb) 73 73 + __field(u32, skb_len) 74 74 + __field(u32, data_len) 75 75 + __field(u32, pmtu) 76 76 + __field(u32, queue_size) 77 77 + __field(u32, proto_seq) 78 78 + __field(u8, proto) 79 79 + __field(u8, was_gso) 80 80 + ), 81 81 + TP_fast_assign(__entry->skb = skb; 82 82 + __entry->skb_len = skb->len; 83 83 + __entry->data_len = skb->data_len; 84 84 + __entry->queue_size = 85 85 + xtfs->cfg.max_queue_size - xtfs->queue_size; 86 86 + __entry->proto = __trace_ip_proto(ip_hdr(skb)); 87 87 + __entry->proto_seq = __trace_ip_proto_seq(ip_hdr(skb)); 88 88 + __entry->pmtu = pmtu; 89 89 + __entry->was_gso = was_gso; 90 90 + ), 91 91 + TP_printk("INGRPREQ: skb=%p len=%u data_len=%u qsize=%u proto=%u proto_seq=%u pmtu=%u was_gso=%u", 92 92 + __entry->skb, __entry->skb_len, __entry->data_len, 93 93 + __entry->queue_size, __entry->proto, __entry->proto_seq, 94 94 + __entry->pmtu, __entry->was_gso)); 95 95 + 96 96 + DEFINE_EVENT(iptfs_ingress_preq_event, iptfs_enqueue, 97 97 + TP_PROTO(struct sk_buff *skb, struct xfrm_iptfs_data *xtfs, u32 pmtu, u8 was_gso), 98 98 + TP_ARGS(skb, xtfs, pmtu, was_gso)); 99 99 + 100 100 + DEFINE_EVENT(iptfs_ingress_preq_event, iptfs_no_queue_space, 101 101 + TP_PROTO(struct sk_buff *skb, struct xfrm_iptfs_data *xtfs, u32 pmtu, u8 was_gso), 102 102 + TP_ARGS(skb, xtfs, pmtu, was_gso)); 103 103 + 104 104 + DEFINE_EVENT(iptfs_ingress_preq_event, iptfs_too_big, 105 105 + TP_PROTO(struct sk_buff *skb, struct xfrm_iptfs_data *xtfs, u32 pmtu, u8 was_gso), 106 106 + TP_ARGS(skb, xtfs, pmtu, was_gso)); 107 107 + 108 108 + DECLARE_EVENT_CLASS(iptfs_ingress_postq_event, 109 109 + TP_PROTO(struct sk_buff *skb, u32 mtu, u16 blkoff, struct iphdr *iph), 110 110 + TP_ARGS(skb, mtu, blkoff, iph), 111 111 + TP_STRUCT__entry(__field(struct sk_buff *, skb) 112 112 + __field(u32, skb_len) 113 113 + __field(u32, data_len) 114 114 + __field(u32, mtu) 115 115 + __field(u32, proto_seq) 116 116 + __field(u16, blkoff) 117 117 + __field(u8, proto)), 118 118 + TP_fast_assign(__entry->skb = skb; 119 119 + __entry->skb_len = skb->len; 120 120 + __entry->data_len = skb->data_len; 121 121 + __entry->mtu = mtu; 122 122 + __entry->blkoff = blkoff; 123 123 + __entry->proto = iph ? __trace_ip_proto(iph) : 0; 124 124 + __entry->proto_seq = iph ? __trace_ip_proto_seq(iph) : 0; 125 125 + ), 126 126 + TP_printk("INGRPSTQ: skb=%p len=%u data_len=%u mtu=%u blkoff=%u proto=%u proto_seq=%u", 127 127 + __entry->skb, __entry->skb_len, __entry->data_len, __entry->mtu, 128 128 + __entry->blkoff, __entry->proto, __entry->proto_seq)); 129 129 + 130 130 + DEFINE_EVENT(iptfs_ingress_postq_event, iptfs_first_dequeue, 131 131 + TP_PROTO(struct sk_buff *skb, u32 mtu, u16 blkoff, 132 132 + struct iphdr *iph), 133 133 + TP_ARGS(skb, mtu, blkoff, iph)); 134 134 + 135 135 + DEFINE_EVENT(iptfs_ingress_postq_event, iptfs_first_fragmenting, 136 136 + TP_PROTO(struct sk_buff *skb, u32 mtu, u16 blkoff, 137 137 + struct iphdr *iph), 138 138 + TP_ARGS(skb, mtu, blkoff, iph)); 139 139 + 140 140 + DEFINE_EVENT(iptfs_ingress_postq_event, iptfs_first_final_fragment, 141 141 + TP_PROTO(struct sk_buff *skb, u32 mtu, u16 blkoff, 142 142 + struct iphdr *iph), 143 143 + TP_ARGS(skb, mtu, blkoff, iph)); 144 144 + 145 145 + DEFINE_EVENT(iptfs_ingress_postq_event, iptfs_first_toobig, 146 146 + TP_PROTO(struct sk_buff *skb, u32 mtu, u16 blkoff, 147 147 + struct iphdr *iph), 148 148 + TP_ARGS(skb, mtu, blkoff, iph)); 149 149 + 150 150 + TRACE_EVENT(iptfs_ingress_nth_peek, 151 151 + TP_PROTO(struct sk_buff *skb, u32 remaining), 152 152 + TP_ARGS(skb, remaining), 153 153 + TP_STRUCT__entry(__field(struct sk_buff *, skb) 154 154 + __field(u32, skb_len) 155 155 + __field(u32, remaining)), 156 156 + TP_fast_assign(__entry->skb = skb; 157 157 + __entry->skb_len = skb->len; 158 158 + __entry->remaining = remaining; 159 159 + ), 160 160 + TP_printk("INGRPSTQ: NTHPEEK: skb=%p len=%u remaining=%u", 161 161 + __entry->skb, __entry->skb_len, __entry->remaining)); 162 162 + 163 163 + TRACE_EVENT(iptfs_ingress_nth_add, TP_PROTO(struct sk_buff *skb, u8 share_ok), 164 164 + TP_ARGS(skb, share_ok), 165 165 + TP_STRUCT__entry(__field(struct sk_buff *, skb) 166 166 + __field(u32, skb_len) 167 167 + __field(u32, data_len) 168 168 + __field(u8, share_ok) 169 169 + __field(u8, head_frag) 170 170 + __field(u8, pp_recycle) 171 171 + __field(u8, cloned) 172 172 + __field(u8, shared) 173 173 + __field(u8, nr_frags) 174 174 + __field(u8, frag_list) 175 175 + ), 176 176 + TP_fast_assign(__entry->skb = skb; 177 177 + __entry->skb_len = skb->len; 178 178 + __entry->data_len = skb->data_len; 179 179 + __entry->share_ok = share_ok; 180 180 + __entry->head_frag = skb->head_frag; 181 181 + __entry->pp_recycle = skb->pp_recycle; 182 182 + __entry->cloned = skb_cloned(skb); 183 183 + __entry->shared = skb_shared(skb); 184 184 + __entry->nr_frags = skb_shinfo(skb)->nr_frags; 185 185 + __entry->frag_list = (bool)skb_shinfo(skb)->frag_list; 186 186 + ), 187 187 + TP_printk("INGRPSTQ: NTHADD: skb=%p len=%u data_len=%u share_ok=%u head_frag=%u pp_recycle=%u cloned=%u shared=%u nr_frags=%u frag_list=%u", 188 188 + __entry->skb, __entry->skb_len, __entry->data_len, __entry->share_ok, 189 189 + __entry->head_frag, __entry->pp_recycle, __entry->cloned, __entry->shared, 190 190 + __entry->nr_frags, __entry->frag_list)); 191 191 + 192 192 + DECLARE_EVENT_CLASS(iptfs_timer_event, 193 193 + TP_PROTO(struct xfrm_iptfs_data *xtfs, u64 time_val), 194 194 + TP_ARGS(xtfs, time_val), 195 195 + TP_STRUCT__entry(__field(u64, time_val) 196 196 + __field(u64, set_time)), 197 197 + TP_fast_assign(__entry->time_val = time_val; 198 198 + __entry->set_time = xtfs->iptfs_settime; 199 199 + ), 200 200 + TP_printk("TIMER: set_time=%llu time_val=%llu", 201 201 + __entry->set_time, __entry->time_val)); 202 202 + 203 203 + DEFINE_EVENT(iptfs_timer_event, iptfs_timer_start, 204 204 + TP_PROTO(struct xfrm_iptfs_data *xtfs, u64 time_val), 205 205 + TP_ARGS(xtfs, time_val)); 206 206 + 207 207 + DEFINE_EVENT(iptfs_timer_event, iptfs_timer_expire, 208 208 + TP_PROTO(struct xfrm_iptfs_data *xtfs, u64 time_val), 209 209 + TP_ARGS(xtfs, time_val)); 210 210 + 211 211 + #endif /* _TRACE_IPTFS_H */ 212 212 + 213 213 + /* This part must be outside protection */ 214 214 + #undef TRACE_INCLUDE_PATH 215 215 + #define TRACE_INCLUDE_PATH ../../net/xfrm 216 216 + #undef TRACE_INCLUDE_FILE 217 217 + #define TRACE_INCLUDE_FILE trace_iptfs 218 218 + #include <trace/define_trace.h>

+8 -2

net/xfrm/xfrm_compat.c

reviewed

··· 284 284 case XFRMA_SA_DIR: 285 285 case XFRMA_NAT_KEEPALIVE_INTERVAL: 286 286 case XFRMA_SA_PCPU: 287 287 + case XFRMA_IPTFS_DROP_TIME: 288 288 + case XFRMA_IPTFS_REORDER_WINDOW: 289 289 + case XFRMA_IPTFS_DONT_FRAG: 290 290 + case XFRMA_IPTFS_INIT_DELAY: 291 291 + case XFRMA_IPTFS_MAX_QSIZE: 292 292 + case XFRMA_IPTFS_PKT_SIZE: 287 293 return xfrm_nla_cpy(dst, src, nla_len(src)); 288 294 default: 289 289 - BUILD_BUG_ON(XFRMA_MAX != XFRMA_SA_PCPU); 295 295 + BUILD_BUG_ON(XFRMA_MAX != XFRMA_IPTFS_PKT_SIZE); 290 296 pr_warn_once("unsupported nla_type %d\n", src->nla_type); 291 297 return -EOPNOTSUPP; 292 298 } ··· 447 441 int err; 448 442 449 443 if (type > XFRMA_MAX) { 450 450 - BUILD_BUG_ON(XFRMA_MAX != XFRMA_SA_PCPU); 444 444 + BUILD_BUG_ON(XFRMA_MAX != XFRMA_IPTFS_PKT_SIZE); 451 445 NL_SET_ERR_MSG(extack, "Bad attribute"); 452 446 return -EOPNOTSUPP; 453 447 }

+3 -1

net/xfrm/xfrm_device.c

reviewed

··· 42 42 skb->transport_header = skb->network_header + hsize; 43 43 44 44 skb_reset_mac_len(skb); 45 45 - pskb_pull(skb, skb->mac_len + x->props.header_len); 45 45 + pskb_pull(skb, 46 46 + skb->mac_len + x->props.header_len - x->props.enc_hdr_len); 46 47 } 47 48 48 49 static void __xfrm_mode_beet_prep(struct xfrm_state *x, struct sk_buff *skb, ··· 69 68 static void xfrm_outer_mode_prep(struct xfrm_state *x, struct sk_buff *skb) 70 69 { 71 70 switch (x->outer_mode.encap) { 71 71 + case XFRM_MODE_IPTFS: 72 72 case XFRM_MODE_TUNNEL: 73 73 if (x->outer_mode.family == AF_INET) 74 74 return __xfrm_mode_tunnel_prep(x, skb,

+16 -2

net/xfrm/xfrm_input.c

reviewed

··· 446 446 WARN_ON_ONCE(1); 447 447 break; 448 448 default: 449 449 + if (x->mode_cbs && x->mode_cbs->input) 450 450 + return x->mode_cbs->input(x, skb); 451 451 + 449 452 WARN_ON_ONCE(1); 450 453 break; 451 454 } ··· 456 453 return -EOPNOTSUPP; 457 454 } 458 455 456 456 + /* NOTE: encap_type - In addition to the normal (non-negative) values for 457 457 + * encap_type, a negative value of -1 or -2 can be used to resume/restart this 458 458 + * function after a previous invocation early terminated for async operation. 459 459 + */ 459 460 int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) 460 461 { 461 462 const struct xfrm_state_afinfo *afinfo; ··· 495 488 } 496 489 497 490 family = x->props.family; 491 491 + 492 492 + /* An encap_type of -2 indicates reconstructed inner packet */ 493 493 + if (encap_type == -2) 494 494 + goto resume_decapped; 498 495 499 496 /* An encap_type of -1 indicates async resumption. */ 500 497 if (encap_type == -1) { ··· 690 679 691 680 XFRM_MODE_SKB_CB(skb)->protocol = nexthdr; 692 681 693 693 - if (xfrm_inner_mode_input(x, skb)) { 682 682 + err = xfrm_inner_mode_input(x, skb); 683 683 + if (err == -EINPROGRESS) 684 684 + return 0; 685 685 + else if (err) { 694 686 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR); 695 687 goto drop; 696 688 } 697 697 - 689 689 + resume_decapped: 698 690 if (x->outer_mode.flags & XFRM_MODE_FLAG_TUNNEL) { 699 691 decaps = 1; 700 692 break;

+2764

net/xfrm/xfrm_iptfs.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* xfrm_iptfs: IPTFS encapsulation support 3 3 + * 4 4 + * April 21 2022, Christian Hopps <chopps@labn.net> 5 5 + * 6 6 + * Copyright (c) 2022, LabN Consulting, L.L.C. 7 7 + * 8 8 + */ 9 9 + 10 10 + #include <linux/kernel.h> 11 11 + #include <linux/icmpv6.h> 12 12 + #include <linux/skbuff_ref.h> 13 13 + #include <net/gro.h> 14 14 + #include <net/icmp.h> 15 15 + #include <net/ip6_route.h> 16 16 + #include <net/inet_ecn.h> 17 17 + #include <net/xfrm.h> 18 18 + 19 19 + #include <crypto/aead.h> 20 20 + 21 21 + #include "xfrm_inout.h" 22 22 + #include "trace_iptfs.h" 23 23 + 24 24 + /* IPTFS encap (header) values. */ 25 25 + #define IPTFS_SUBTYPE_BASIC 0 26 26 + #define IPTFS_SUBTYPE_CC 1 27 27 + 28 28 + /* ----------------------------------------------- */ 29 29 + /* IP-TFS default SA values (tunnel egress/dir-in) */ 30 30 + /* ----------------------------------------------- */ 31 31 + 32 32 + /** 33 33 + * define IPTFS_DEFAULT_DROP_TIME_USECS - default drop time 34 34 + * 35 35 + * The default IPTFS drop time in microseconds. The drop time is the amount of 36 36 + * time before a missing out-of-order IPTFS tunnel packet is considered lost. 37 37 + * See also the reorder window. 38 38 + * 39 39 + * Default 1s. 40 40 + */ 41 41 + #define IPTFS_DEFAULT_DROP_TIME_USECS 1000000 42 42 + 43 43 + /** 44 44 + * define IPTFS_DEFAULT_REORDER_WINDOW - default reorder window size 45 45 + * 46 46 + * The default IPTFS reorder window size. The reorder window size dictates the 47 47 + * maximum number of IPTFS tunnel packets in a sequence that may arrive out of 48 48 + * order. 49 49 + * 50 50 + * Default 3. (tcp folks suggested) 51 51 + */ 52 52 + #define IPTFS_DEFAULT_REORDER_WINDOW 3 53 53 + 54 54 + /* ------------------------------------------------ */ 55 55 + /* IPTFS default SA values (tunnel ingress/dir-out) */ 56 56 + /* ------------------------------------------------ */ 57 57 + 58 58 + /** 59 59 + * define IPTFS_DEFAULT_INIT_DELAY_USECS - default initial output delay 60 60 + * 61 61 + * The initial output delay is the amount of time prior to servicing the output 62 62 + * queue after queueing the first packet on said queue. This applies anytime the 63 63 + * output queue was previously empty. 64 64 + * 65 65 + * Default 0. 66 66 + */ 67 67 + #define IPTFS_DEFAULT_INIT_DELAY_USECS 0 68 68 + 69 69 + /** 70 70 + * define IPTFS_DEFAULT_MAX_QUEUE_SIZE - default max output queue size. 71 71 + * 72 72 + * The default IPTFS max output queue size in octets. The output queue is where 73 73 + * received packets destined for output over an IPTFS tunnel are stored prior to 74 74 + * being output in aggregated/fragmented form over the IPTFS tunnel. 75 75 + * 76 76 + * Default 1M. 77 77 + */ 78 78 + #define IPTFS_DEFAULT_MAX_QUEUE_SIZE (1024 * 10240) 79 79 + 80 80 + /* Assumed: skb->head is cache aligned. 81 81 + * 82 82 + * L2 Header resv: Arrange for cacheline to start at skb->data - 16 to keep the 83 83 + * to-be-pushed L2 header in the same cacheline as resulting `skb->data` (i.e., 84 84 + * the L3 header). If cacheline size is > 64 then skb->data + pushed L2 will all 85 85 + * be in a single cacheline if we simply reserve 64 bytes. 86 86 + * 87 87 + * L3 Header resv: For L3+L2 headers (i.e., skb->data points at the IPTFS payload) 88 88 + * we want `skb->data` to be cacheline aligned and all pushed L2L3 headers will 89 89 + * be in their own cacheline[s]. 128 works for cachelins up to 128 bytes, for 90 90 + * any larger cacheline sizes the pushed headers will simply share the cacheline 91 91 + * with the start of the IPTFS payload (skb->data). 92 92 + */ 93 93 + #define XFRM_IPTFS_MIN_L3HEADROOM 128 94 94 + #define XFRM_IPTFS_MIN_L2HEADROOM (L1_CACHE_BYTES > 64 ? 64 : 64 + 16) 95 95 + 96 96 + /* Min to try to share outer iptfs skb data vs copying into new skb */ 97 97 + #define IPTFS_PKT_SHARE_MIN 129 98 98 + 99 99 + #define NSECS_IN_USEC 1000 100 100 + 101 101 + #define IPTFS_HRTIMER_MODE HRTIMER_MODE_REL_SOFT 102 102 + 103 103 + /** 104 104 + * struct xfrm_iptfs_config - configuration for the IPTFS tunnel. 105 105 + * @pkt_size: size of the outer IP packet. 0 to use interface and MTU discovery, 106 106 + * otherwise the user specified value. 107 107 + * @max_queue_size: The maximum number of octets allowed to be queued to be sent 108 108 + * over the IPTFS SA. The queue size is measured as the size of all the 109 109 + * packets enqueued. 110 110 + * @reorder_win_size: the number slots in the reorder window, thus the number of 111 111 + * packets that may arrive out of order. 112 112 + * @dont_frag: true to inhibit fragmenting across IPTFS outer packets. 113 113 + */ 114 114 + struct xfrm_iptfs_config { 115 115 + u32 pkt_size; /* outer_packet_size or 0 */ 116 116 + u32 max_queue_size; /* octets */ 117 117 + u16 reorder_win_size; 118 118 + u8 dont_frag : 1; 119 119 + }; 120 120 + 121 121 + struct skb_wseq { 122 122 + struct sk_buff *skb; 123 123 + u64 drop_time; 124 124 + }; 125 125 + 126 126 + /** 127 127 + * struct xfrm_iptfs_data - mode specific xfrm state. 128 128 + * @cfg: IPTFS tunnel config. 129 129 + * @x: owning SA (xfrm_state). 130 130 + * @queue: queued user packets to send. 131 131 + * @queue_size: number of octets on queue (sum of packet sizes). 132 132 + * @ecn_queue_size: octets above with ECN mark. 133 133 + * @init_delay_ns: nanoseconds to wait to send initial IPTFS packet. 134 134 + * @iptfs_timer: output timer. 135 135 + * @iptfs_settime: time the output timer was set. 136 136 + * @payload_mtu: max payload size. 137 137 + * @w_seq_set: true after first seq received. 138 138 + * @w_wantseq: waiting for this seq number as next to process (in order). 139 139 + * @w_saved: the saved buf array (reorder window). 140 140 + * @w_savedlen: the saved len (not size). 141 141 + * @drop_lock: lock to protect reorder queue. 142 142 + * @drop_timer: timer for considering next packet lost. 143 143 + * @drop_time_ns: timer intervan in nanoseconds. 144 144 + * @ra_newskb: new pkt being reassembled. 145 145 + * @ra_wantseq: expected next sequence for reassembly. 146 146 + * @ra_runt: last pkt bytes from very end of last skb. 147 147 + * @ra_runtlen: size of ra_runt. 148 148 + */ 149 149 + struct xfrm_iptfs_data { 150 150 + struct xfrm_iptfs_config cfg; 151 151 + 152 152 + /* Ingress User Input */ 153 153 + struct xfrm_state *x; /* owning state */ 154 154 + struct sk_buff_head queue; /* output queue */ 155 155 + 156 156 + u32 queue_size; /* octets */ 157 157 + u32 ecn_queue_size; /* octets above which ECN mark */ 158 158 + u64 init_delay_ns; /* nanoseconds */ 159 159 + struct hrtimer iptfs_timer; /* output timer */ 160 160 + time64_t iptfs_settime; /* time timer was set */ 161 161 + u32 payload_mtu; /* max payload size */ 162 162 + 163 163 + /* Tunnel input reordering */ 164 164 + bool w_seq_set; /* true after first seq received */ 165 165 + u64 w_wantseq; /* expected next sequence */ 166 166 + struct skb_wseq *w_saved; /* the saved buf array */ 167 167 + u32 w_savedlen; /* the saved len (not size) */ 168 168 + spinlock_t drop_lock; 169 169 + struct hrtimer drop_timer; 170 170 + u64 drop_time_ns; 171 171 + 172 172 + /* Tunnel input reassembly */ 173 173 + struct sk_buff *ra_newskb; /* new pkt being reassembled */ 174 174 + u64 ra_wantseq; /* expected next sequence */ 175 175 + u8 ra_runt[6]; /* last pkt bytes from last skb */ 176 176 + u8 ra_runtlen; /* count of ra_runt */ 177 177 + }; 178 178 + 179 179 + static u32 __iptfs_get_inner_mtu(struct xfrm_state *x, int outer_mtu); 180 180 + static enum hrtimer_restart iptfs_delay_timer(struct hrtimer *me); 181 181 + static enum hrtimer_restart iptfs_drop_timer(struct hrtimer *me); 182 182 + 183 183 + /* ================= */ 184 184 + /* Utility Functions */ 185 185 + /* ================= */ 186 186 + 187 187 + #ifdef TRACEPOINTS_ENABLED 188 188 + static u32 __trace_ip_proto(struct iphdr *iph) 189 189 + { 190 190 + if (iph->version == 4) 191 191 + return iph->protocol; 192 192 + return ((struct ipv6hdr *)iph)->nexthdr; 193 193 + } 194 194 + 195 195 + static u32 __trace_ip_proto_seq(struct iphdr *iph) 196 196 + { 197 197 + void *nexthdr; 198 198 + u32 protocol = 0; 199 199 + 200 200 + if (iph->version == 4) { 201 201 + nexthdr = (void *)(iph + 1); 202 202 + protocol = iph->protocol; 203 203 + } else if (iph->version == 6) { 204 204 + nexthdr = (void *)(((struct ipv6hdr *)(iph)) + 1); 205 205 + protocol = ((struct ipv6hdr *)(iph))->nexthdr; 206 206 + } 207 207 + switch (protocol) { 208 208 + case IPPROTO_ICMP: 209 209 + return ntohs(((struct icmphdr *)nexthdr)->un.echo.sequence); 210 210 + case IPPROTO_ICMPV6: 211 211 + return ntohs(((struct icmp6hdr *)nexthdr)->icmp6_sequence); 212 212 + case IPPROTO_TCP: 213 213 + return ntohl(((struct tcphdr *)nexthdr)->seq); 214 214 + case IPPROTO_UDP: 215 215 + return ntohs(((struct udphdr *)nexthdr)->source); 216 216 + default: 217 217 + return 0; 218 218 + } 219 219 + } 220 220 + #endif /*TRACEPOINTS_ENABLED*/ 221 221 + 222 222 + static u64 __esp_seq(struct sk_buff *skb) 223 223 + { 224 224 + u64 seq = ntohl(XFRM_SKB_CB(skb)->seq.input.low); 225 225 + 226 226 + return seq | (u64)ntohl(XFRM_SKB_CB(skb)->seq.input.hi) << 32; 227 227 + } 228 228 + 229 229 + /* ======================= */ 230 230 + /* IPTFS SK_BUFF Functions */ 231 231 + /* ======================= */ 232 232 + 233 233 + /** 234 234 + * iptfs_alloc_skb() - Allocate a new `skb`. 235 235 + * @tpl: the skb to copy required meta-data from. 236 236 + * @len: the linear length of the head data, zero is fine. 237 237 + * @l3resv: true if skb reserve needs to support pushing L3 headers 238 238 + * 239 239 + * A new `skb` is allocated and required meta-data is copied from `tpl`, the 240 240 + * head data is sized to `len` + reserved space set according to the @l3resv 241 241 + * boolean. 242 242 + * 243 243 + * When @l3resv is false, resv is XFRM_IPTFS_MIN_L2HEADROOM which arranges for 244 244 + * `skb->data - 16` which is a good guess for good cache alignment (placing the 245 245 + * to be pushed L2 header at the start of a cacheline. 246 246 + * 247 247 + * Otherwise, @l3resv is true and resv is set to the correct reserved space for 248 248 + * dst->dev plus the calculated L3 overhead for the xfrm dst or 249 249 + * XFRM_IPTFS_MIN_L3HEADROOM whichever is larger. This is then cache aligned so 250 250 + * that all the headers will commonly fall in a cacheline when possible. 251 251 + * 252 252 + * l3resv=true is used on tunnel ingress (tx), because we need to reserve for 253 253 + * the new IPTFS packet (i.e., L2+L3 headers). On tunnel egress (rx) the data 254 254 + * being copied into the skb includes the user L3 headers already so we only 255 255 + * need to reserve for L2. 256 256 + * 257 257 + * Return: the new skb or NULL. 258 258 + */ 259 259 + static struct sk_buff *iptfs_alloc_skb(struct sk_buff *tpl, u32 len, bool l3resv) 260 260 + { 261 261 + struct sk_buff *skb; 262 262 + u32 resv; 263 263 + 264 264 + if (!l3resv) { 265 265 + resv = XFRM_IPTFS_MIN_L2HEADROOM; 266 266 + } else { 267 267 + struct dst_entry *dst = skb_dst(tpl); 268 268 + 269 269 + resv = LL_RESERVED_SPACE(dst->dev) + dst->header_len; 270 270 + resv = max(resv, XFRM_IPTFS_MIN_L3HEADROOM); 271 271 + resv = L1_CACHE_ALIGN(resv); 272 272 + } 273 273 + 274 274 + skb = alloc_skb(len + resv, GFP_ATOMIC | __GFP_NOWARN); 275 275 + if (!skb) 276 276 + return NULL; 277 277 + 278 278 + skb_reserve(skb, resv); 279 279 + 280 280 + if (!l3resv) { 281 281 + /* xfrm_input resume needs dev and xfrm ext from tunnel pkt */ 282 282 + skb->dev = tpl->dev; 283 283 + __skb_ext_copy(skb, tpl); 284 284 + } 285 285 + 286 286 + /* dropped by xfrm_input, used by xfrm_output */ 287 287 + skb_dst_copy(skb, tpl); 288 288 + 289 289 + return skb; 290 290 + } 291 291 + 292 292 + /** 293 293 + * iptfs_skb_head_to_frag() - initialize a skb_frag_t based on skb head data 294 294 + * @skb: skb with the head data 295 295 + * @frag: frag to initialize 296 296 + */ 297 297 + static void iptfs_skb_head_to_frag(const struct sk_buff *skb, skb_frag_t *frag) 298 298 + { 299 299 + struct page *page = virt_to_head_page(skb->data); 300 300 + unsigned char *addr = (unsigned char *)page_address(page); 301 301 + 302 302 + skb_frag_fill_page_desc(frag, page, skb->data - addr, skb_headlen(skb)); 303 303 + } 304 304 + 305 305 + /** 306 306 + * struct iptfs_skb_frag_walk - use to track a walk through fragments 307 307 + * @fragi: current fragment index 308 308 + * @past: length of data in fragments before @fragi 309 309 + * @total: length of data in all fragments 310 310 + * @nr_frags: number of fragments present in array 311 311 + * @initial_offset: the value passed in to skb_prepare_frag_walk() 312 312 + * @frags: the page fragments inc. room for head page 313 313 + * @pp_recycle: copy of skb->pp_recycle 314 314 + */ 315 315 + struct iptfs_skb_frag_walk { 316 316 + u32 fragi; 317 317 + u32 past; 318 318 + u32 total; 319 319 + u32 nr_frags; 320 320 + u32 initial_offset; 321 321 + skb_frag_t frags[MAX_SKB_FRAGS + 1]; 322 322 + bool pp_recycle; 323 323 + }; 324 324 + 325 325 + /** 326 326 + * iptfs_skb_prepare_frag_walk() - initialize a frag walk over an skb. 327 327 + * @skb: the skb to walk. 328 328 + * @initial_offset: start the walk @initial_offset into the skb. 329 329 + * @walk: the walk to initialize 330 330 + * 331 331 + * Future calls to skb_add_frags() will expect the @offset value to be at 332 332 + * least @initial_offset large. 333 333 + */ 334 334 + static void iptfs_skb_prepare_frag_walk(struct sk_buff *skb, u32 initial_offset, 335 335 + struct iptfs_skb_frag_walk *walk) 336 336 + { 337 337 + struct skb_shared_info *shinfo = skb_shinfo(skb); 338 338 + skb_frag_t *frag, *from; 339 339 + u32 i; 340 340 + 341 341 + walk->initial_offset = initial_offset; 342 342 + walk->fragi = 0; 343 343 + walk->past = 0; 344 344 + walk->total = 0; 345 345 + walk->nr_frags = 0; 346 346 + walk->pp_recycle = skb->pp_recycle; 347 347 + 348 348 + if (skb->head_frag) { 349 349 + if (initial_offset >= skb_headlen(skb)) { 350 350 + initial_offset -= skb_headlen(skb); 351 351 + } else { 352 352 + frag = &walk->frags[walk->nr_frags++]; 353 353 + iptfs_skb_head_to_frag(skb, frag); 354 354 + frag->offset += initial_offset; 355 355 + frag->len -= initial_offset; 356 356 + walk->total += frag->len; 357 357 + initial_offset = 0; 358 358 + } 359 359 + } else { 360 360 + initial_offset -= skb_headlen(skb); 361 361 + } 362 362 + 363 363 + for (i = 0; i < shinfo->nr_frags; i++) { 364 364 + from = &shinfo->frags[i]; 365 365 + if (initial_offset >= from->len) { 366 366 + initial_offset -= from->len; 367 367 + continue; 368 368 + } 369 369 + frag = &walk->frags[walk->nr_frags++]; 370 370 + *frag = *from; 371 371 + if (initial_offset) { 372 372 + frag->offset += initial_offset; 373 373 + frag->len -= initial_offset; 374 374 + initial_offset = 0; 375 375 + } 376 376 + walk->total += frag->len; 377 377 + } 378 378 + } 379 379 + 380 380 + static u32 iptfs_skb_reset_frag_walk(struct iptfs_skb_frag_walk *walk, 381 381 + u32 offset) 382 382 + { 383 383 + /* Adjust offset to refer to internal walk values */ 384 384 + offset -= walk->initial_offset; 385 385 + 386 386 + /* Get to the correct fragment for offset */ 387 387 + while (offset < walk->past) { 388 388 + walk->past -= walk->frags[--walk->fragi].len; 389 389 + if (offset >= walk->past) 390 390 + break; 391 391 + } 392 392 + while (offset >= walk->past + walk->frags[walk->fragi].len) 393 393 + walk->past += walk->frags[walk->fragi++].len; 394 394 + 395 395 + /* offset now relative to this current frag */ 396 396 + offset -= walk->past; 397 397 + return offset; 398 398 + } 399 399 + 400 400 + /** 401 401 + * iptfs_skb_can_add_frags() - check if ok to add frags from walk to skb 402 402 + * @skb: skb to check for adding frags to 403 403 + * @walk: the walk that will be used as source for frags. 404 404 + * @offset: offset from beginning of original skb to start from. 405 405 + * @len: amount of data to add frag references to in @skb. 406 406 + * 407 407 + * Return: true if ok to add frags. 408 408 + */ 409 409 + static bool iptfs_skb_can_add_frags(const struct sk_buff *skb, 410 410 + struct iptfs_skb_frag_walk *walk, 411 411 + u32 offset, u32 len) 412 412 + { 413 413 + struct skb_shared_info *shinfo = skb_shinfo(skb); 414 414 + u32 fragi, nr_frags, fraglen; 415 415 + 416 416 + if (skb_has_frag_list(skb) || skb->pp_recycle != walk->pp_recycle) 417 417 + return false; 418 418 + 419 419 + /* Make offset relative to current frag after setting that */ 420 420 + offset = iptfs_skb_reset_frag_walk(walk, offset); 421 421 + 422 422 + /* Verify we have array space for the fragments we need to add */ 423 423 + fragi = walk->fragi; 424 424 + nr_frags = shinfo->nr_frags; 425 425 + while (len && fragi < walk->nr_frags) { 426 426 + skb_frag_t *frag = &walk->frags[fragi]; 427 427 + 428 428 + fraglen = frag->len; 429 429 + if (offset) { 430 430 + fraglen -= offset; 431 431 + offset = 0; 432 432 + } 433 433 + if (++nr_frags > MAX_SKB_FRAGS) 434 434 + return false; 435 435 + if (len <= fraglen) 436 436 + return true; 437 437 + len -= fraglen; 438 438 + fragi++; 439 439 + } 440 440 + /* We may not copy all @len but what we have will fit. */ 441 441 + return true; 442 442 + } 443 443 + 444 444 + /** 445 445 + * iptfs_skb_add_frags() - add a range of fragment references into an skb 446 446 + * @skb: skb to add references into 447 447 + * @walk: the walk to add referenced fragments from. 448 448 + * @offset: offset from beginning of original skb to start from. 449 449 + * @len: amount of data to add frag references to in @skb. 450 450 + * 451 451 + * iptfs_skb_can_add_frags() should be called before this function to verify 452 452 + * that the destination @skb is compatible with the walk and has space in the 453 453 + * array for the to be added frag references. 454 454 + * 455 455 + * Return: The number of bytes not added to @skb b/c we reached the end of the 456 456 + * walk before adding all of @len. 457 457 + */ 458 458 + static int iptfs_skb_add_frags(struct sk_buff *skb, 459 459 + struct iptfs_skb_frag_walk *walk, u32 offset, 460 460 + u32 len) 461 461 + { 462 462 + struct skb_shared_info *shinfo = skb_shinfo(skb); 463 463 + u32 fraglen; 464 464 + 465 465 + if (!walk->nr_frags || offset >= walk->total + walk->initial_offset) 466 466 + return len; 467 467 + 468 468 + /* make offset relative to current frag after setting that */ 469 469 + offset = iptfs_skb_reset_frag_walk(walk, offset); 470 470 + 471 471 + while (len && walk->fragi < walk->nr_frags) { 472 472 + skb_frag_t *frag = &walk->frags[walk->fragi]; 473 473 + skb_frag_t *tofrag = &shinfo->frags[shinfo->nr_frags]; 474 474 + 475 475 + *tofrag = *frag; 476 476 + if (offset) { 477 477 + tofrag->offset += offset; 478 478 + tofrag->len -= offset; 479 479 + offset = 0; 480 480 + } 481 481 + __skb_frag_ref(tofrag); 482 482 + shinfo->nr_frags++; 483 483 + 484 484 + /* see if we are done */ 485 485 + fraglen = tofrag->len; 486 486 + if (len < fraglen) { 487 487 + tofrag->len = len; 488 488 + skb->len += len; 489 489 + skb->data_len += len; 490 490 + return 0; 491 491 + } 492 492 + /* advance to next source fragment */ 493 493 + len -= fraglen; /* careful, use dst bv_len */ 494 494 + skb->len += fraglen; /* careful, " " " */ 495 495 + skb->data_len += fraglen; /* careful, " " " */ 496 496 + walk->past += frag->len; /* careful, use src bv_len */ 497 497 + walk->fragi++; 498 498 + } 499 499 + return len; 500 500 + } 501 501 + 502 502 + /* ================================== */ 503 503 + /* IPTFS Trace Event Definitions */ 504 504 + /* ================================== */ 505 505 + 506 506 + #define CREATE_TRACE_POINTS 507 507 + #include "trace_iptfs.h" 508 508 + 509 509 + /* ================================== */ 510 510 + /* IPTFS Receiving (egress) Functions */ 511 511 + /* ================================== */ 512 512 + 513 513 + /** 514 514 + * iptfs_pskb_add_frags() - Create and add frags into a new sk_buff. 515 515 + * @tpl: template to create new skb from. 516 516 + * @walk: The source for fragments to add. 517 517 + * @off: The offset into @walk to add frags from, also used with @st and 518 518 + * @copy_len. 519 519 + * @len: The length of data to add covering frags from @walk into @skb. 520 520 + * This must be <= @skblen. 521 521 + * @st: The sequence state to copy from into the new head skb. 522 522 + * @copy_len: Copy @copy_len bytes from @st at offset @off into the new skb 523 523 + * linear space. 524 524 + * 525 525 + * Create a new sk_buff `skb` using the template @tpl. Copy @copy_len bytes from 526 526 + * @st into the new skb linear space, and then add shared fragments from the 527 527 + * frag walk for the remaining @len of data (i.e., @len - @copy_len bytes). 528 528 + * 529 529 + * Return: The newly allocated sk_buff `skb` or NULL if an error occurs. 530 530 + */ 531 531 + static struct sk_buff * 532 532 + iptfs_pskb_add_frags(struct sk_buff *tpl, struct iptfs_skb_frag_walk *walk, 533 533 + u32 off, u32 len, struct skb_seq_state *st, u32 copy_len) 534 534 + { 535 535 + struct sk_buff *skb; 536 536 + 537 537 + skb = iptfs_alloc_skb(tpl, copy_len, false); 538 538 + if (!skb) 539 539 + return NULL; 540 540 + 541 541 + /* this should not normally be happening */ 542 542 + if (!iptfs_skb_can_add_frags(skb, walk, off + copy_len, 543 543 + len - copy_len)) { 544 544 + kfree_skb(skb); 545 545 + return NULL; 546 546 + } 547 547 + 548 548 + if (copy_len && 549 549 + skb_copy_seq_read(st, off, skb_put(skb, copy_len), copy_len)) { 550 550 + XFRM_INC_STATS(dev_net(st->root_skb->dev), 551 551 + LINUX_MIB_XFRMINERROR); 552 552 + kfree_skb(skb); 553 553 + return NULL; 554 554 + } 555 555 + 556 556 + iptfs_skb_add_frags(skb, walk, off + copy_len, len - copy_len); 557 557 + return skb; 558 558 + } 559 559 + 560 560 + /** 561 561 + * iptfs_pskb_extract_seq() - Create and load data into a new sk_buff. 562 562 + * @skblen: the total data size for `skb`. 563 563 + * @st: The source for the rest of the data to copy into `skb`. 564 564 + * @off: The offset into @st to copy data from. 565 565 + * @len: The length of data to copy from @st into `skb`. This must be <= 566 566 + * @skblen. 567 567 + * 568 568 + * Create a new sk_buff `skb` with @skblen of packet data space. If non-zero, 569 569 + * copy @rlen bytes of @runt into `skb`. Then using seq functions copy @len 570 570 + * bytes from @st into `skb` starting from @off. 571 571 + * 572 572 + * It is an error for @len to be greater than the amount of data left in @st. 573 573 + * 574 574 + * Return: The newly allocated sk_buff `skb` or NULL if an error occurs. 575 575 + */ 576 576 + static struct sk_buff * 577 577 + iptfs_pskb_extract_seq(u32 skblen, struct skb_seq_state *st, u32 off, int len) 578 578 + { 579 579 + struct sk_buff *skb = iptfs_alloc_skb(st->root_skb, skblen, false); 580 580 + 581 581 + if (!skb) 582 582 + return NULL; 583 583 + if (skb_copy_seq_read(st, off, skb_put(skb, len), len)) { 584 584 + XFRM_INC_STATS(dev_net(st->root_skb->dev), LINUX_MIB_XFRMINERROR); 585 585 + kfree_skb(skb); 586 586 + return NULL; 587 587 + } 588 588 + return skb; 589 589 + } 590 590 + 591 591 + /** 592 592 + * iptfs_input_save_runt() - save data in xtfs runt space. 593 593 + * @xtfs: xtfs state 594 594 + * @seq: the current sequence 595 595 + * @buf: packet data 596 596 + * @len: length of packet data 597 597 + * 598 598 + * Save the small (`len`) start of a fragmented packet in `buf` in the xtfs data 599 599 + * runt space. 600 600 + */ 601 601 + static void iptfs_input_save_runt(struct xfrm_iptfs_data *xtfs, u64 seq, 602 602 + u8 *buf, int len) 603 603 + { 604 604 + memcpy(xtfs->ra_runt, buf, len); 605 605 + 606 606 + xtfs->ra_runtlen = len; 607 607 + xtfs->ra_wantseq = seq + 1; 608 608 + } 609 609 + 610 610 + /** 611 611 + * __iptfs_iphlen() - return the v4/v6 header length using packet data. 612 612 + * @data: pointer at octet with version nibble 613 613 + * 614 614 + * The version data has been checked to be valid (i.e., either 4 or 6). 615 615 + * 616 616 + * Return: the IP header size based on the IP version. 617 617 + */ 618 618 + static u32 __iptfs_iphlen(u8 *data) 619 619 + { 620 620 + struct iphdr *iph = (struct iphdr *)data; 621 621 + 622 622 + if (iph->version == 0x4) 623 623 + return sizeof(*iph); 624 624 + return sizeof(struct ipv6hdr); 625 625 + } 626 626 + 627 627 + /** 628 628 + * __iptfs_iplen() - return the v4/v6 length using packet data. 629 629 + * @data: pointer to ip (v4/v6) packet header 630 630 + * 631 631 + * Grab the IPv4 or IPv6 length value in the start of the inner packet header 632 632 + * pointed to by `data`. Assumes data len is enough for the length field only. 633 633 + * 634 634 + * The version data has been checked to be valid (i.e., either 4 or 6). 635 635 + * 636 636 + * Return: the length value. 637 637 + */ 638 638 + static u32 __iptfs_iplen(u8 *data) 639 639 + { 640 640 + struct iphdr *iph = (struct iphdr *)data; 641 641 + 642 642 + if (iph->version == 0x4) 643 643 + return ntohs(iph->tot_len); 644 644 + return ntohs(((struct ipv6hdr *)iph)->payload_len) + 645 645 + sizeof(struct ipv6hdr); 646 646 + } 647 647 + 648 648 + /** 649 649 + * iptfs_complete_inner_skb() - finish preparing the inner packet for gro recv. 650 650 + * @x: xfrm state 651 651 + * @skb: the inner packet 652 652 + * 653 653 + * Finish the standard xfrm processing on the inner packet prior to sending back 654 654 + * through gro_cells_receive. We do this separately b/c we are building a list 655 655 + * of packets in the hopes that one day a list will be taken by 656 656 + * xfrm_input. 657 657 + */ 658 658 + static void iptfs_complete_inner_skb(struct xfrm_state *x, struct sk_buff *skb) 659 659 + { 660 660 + skb_reset_network_header(skb); 661 661 + 662 662 + /* The packet is going back through gro_cells_receive no need to 663 663 + * set this. 664 664 + */ 665 665 + skb_reset_transport_header(skb); 666 666 + 667 667 + /* Packet already has checksum value set. */ 668 668 + skb->ip_summed = CHECKSUM_NONE; 669 669 + 670 670 + /* Our skb will contain the header data copied when this outer packet 671 671 + * which contained the start of this inner packet. This is true 672 672 + * when we allocate a new skb as well as when we reuse the existing skb. 673 673 + */ 674 674 + if (ip_hdr(skb)->version == 0x4) { 675 675 + struct iphdr *iph = ip_hdr(skb); 676 676 + 677 677 + if (x->props.flags & XFRM_STATE_DECAP_DSCP) 678 678 + ipv4_copy_dscp(XFRM_MODE_SKB_CB(skb)->tos, iph); 679 679 + if (!(x->props.flags & XFRM_STATE_NOECN)) 680 680 + if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos)) 681 681 + IP_ECN_set_ce(iph); 682 682 + 683 683 + skb->protocol = htons(ETH_P_IP); 684 684 + } else { 685 685 + struct ipv6hdr *iph = ipv6_hdr(skb); 686 686 + 687 687 + if (x->props.flags & XFRM_STATE_DECAP_DSCP) 688 688 + ipv6_copy_dscp(XFRM_MODE_SKB_CB(skb)->tos, iph); 689 689 + if (!(x->props.flags & XFRM_STATE_NOECN)) 690 690 + if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos)) 691 691 + IP6_ECN_set_ce(skb, iph); 692 692 + 693 693 + skb->protocol = htons(ETH_P_IPV6); 694 694 + } 695 695 + } 696 696 + 697 697 + static void __iptfs_reassem_done(struct xfrm_iptfs_data *xtfs, bool free) 698 698 + { 699 699 + assert_spin_locked(&xtfs->drop_lock); 700 700 + 701 701 + /* We don't care if it works locking takes care of things */ 702 702 + hrtimer_try_to_cancel(&xtfs->drop_timer); 703 703 + if (free) 704 704 + kfree_skb(xtfs->ra_newskb); 705 705 + xtfs->ra_newskb = NULL; 706 706 + } 707 707 + 708 708 + /** 709 709 + * iptfs_reassem_abort() - In-progress packet is aborted free the state. 710 710 + * @xtfs: xtfs state 711 711 + */ 712 712 + static void iptfs_reassem_abort(struct xfrm_iptfs_data *xtfs) 713 713 + { 714 714 + __iptfs_reassem_done(xtfs, true); 715 715 + } 716 716 + 717 717 + /** 718 718 + * iptfs_reassem_done() - In-progress packet is complete, clear the state. 719 719 + * @xtfs: xtfs state 720 720 + */ 721 721 + static void iptfs_reassem_done(struct xfrm_iptfs_data *xtfs) 722 722 + { 723 723 + __iptfs_reassem_done(xtfs, false); 724 724 + } 725 725 + 726 726 + /** 727 727 + * iptfs_reassem_cont() - Continue the reassembly of an inner packets. 728 728 + * @xtfs: xtfs state 729 729 + * @seq: sequence of current packet 730 730 + * @st: seq read stat for current packet 731 731 + * @skb: current packet 732 732 + * @data: offset into sequential packet data 733 733 + * @blkoff: packet blkoff value 734 734 + * @list: list of skbs to enqueue completed packet on 735 735 + * 736 736 + * Process an IPTFS payload that has a non-zero `blkoff` or when we are 737 737 + * expecting the continuation b/c we have a runt or in-progress packet. 738 738 + * 739 739 + * Return: the new data offset to continue processing from. 740 740 + */ 741 741 + static u32 iptfs_reassem_cont(struct xfrm_iptfs_data *xtfs, u64 seq, 742 742 + struct skb_seq_state *st, struct sk_buff *skb, 743 743 + u32 data, u32 blkoff, struct list_head *list) 744 744 + { 745 745 + struct iptfs_skb_frag_walk _fragwalk; 746 746 + struct iptfs_skb_frag_walk *fragwalk = NULL; 747 747 + struct sk_buff *newskb = xtfs->ra_newskb; 748 748 + u32 remaining = skb->len - data; 749 749 + u32 runtlen = xtfs->ra_runtlen; 750 750 + u32 copylen, fraglen, ipremain, iphlen, iphremain, rrem; 751 751 + 752 752 + /* Handle packet fragment we aren't expecting */ 753 753 + if (!runtlen && !xtfs->ra_newskb) 754 754 + return data + min(blkoff, remaining); 755 755 + 756 756 + /* Important to remember that input to this function is an ordered 757 757 + * packet stream (unless the user disabled the reorder window). Thus if 758 758 + * we are waiting for, and expecting the next packet so we can continue 759 759 + * assembly, a newer sequence number indicates older ones are not coming 760 760 + * (or if they do should be ignored). Technically we can receive older 761 761 + * ones when the reorder window is disabled; however, the user should 762 762 + * have disabled fragmentation in this case, and regardless we don't 763 763 + * deal with it. 764 764 + * 765 765 + * blkoff could be zero if the stream is messed up (or it's an all pad 766 766 + * insertion) be careful to handle that case in each of the below 767 767 + */ 768 768 + 769 769 + /* Too old case: This can happen when the reorder window is disabled so 770 770 + * ordering isn't actually guaranteed. 771 771 + */ 772 772 + if (seq < xtfs->ra_wantseq) 773 773 + return data + remaining; 774 774 + 775 775 + /* Too new case: We missed what we wanted cleanup. */ 776 776 + if (seq > xtfs->ra_wantseq) { 777 777 + XFRM_INC_STATS(xs_net(xtfs->x), LINUX_MIB_XFRMINIPTFSERROR); 778 778 + goto abandon; 779 779 + } 780 780 + 781 781 + if (blkoff == 0) { 782 782 + if ((*skb->data & 0xF0) != 0) { 783 783 + XFRM_INC_STATS(xs_net(xtfs->x), 784 784 + LINUX_MIB_XFRMINIPTFSERROR); 785 785 + goto abandon; 786 786 + } 787 787 + /* Handle all pad case, advance expected sequence number. 788 788 + * (RFC 9347 S2.2.3) 789 789 + */ 790 790 + xtfs->ra_wantseq++; 791 791 + /* will end parsing */ 792 792 + return data + remaining; 793 793 + } 794 794 + 795 795 + if (runtlen) { 796 796 + /* Regardless of what happens we're done with the runt */ 797 797 + xtfs->ra_runtlen = 0; 798 798 + 799 799 + /* The start of this inner packet was at the very end of the last 800 800 + * iptfs payload which didn't include enough for the ip header 801 801 + * length field. We must have *at least* that now. 802 802 + */ 803 803 + rrem = sizeof(xtfs->ra_runt) - runtlen; 804 804 + if (remaining < rrem || blkoff < rrem) { 805 805 + XFRM_INC_STATS(xs_net(xtfs->x), 806 806 + LINUX_MIB_XFRMINIPTFSERROR); 807 807 + goto abandon; 808 808 + } 809 809 + 810 810 + /* fill in the runt data */ 811 811 + if (skb_copy_seq_read(st, data, &xtfs->ra_runt[runtlen], 812 812 + rrem)) { 813 813 + XFRM_INC_STATS(xs_net(xtfs->x), 814 814 + LINUX_MIB_XFRMINBUFFERERROR); 815 815 + goto abandon; 816 816 + } 817 817 + 818 818 + /* We have enough data to get the ip length value now, 819 819 + * allocate an in progress skb 820 820 + */ 821 821 + ipremain = __iptfs_iplen(xtfs->ra_runt); 822 822 + if (ipremain < sizeof(xtfs->ra_runt)) { 823 823 + /* length has to be at least runtsize large */ 824 824 + XFRM_INC_STATS(xs_net(xtfs->x), 825 825 + LINUX_MIB_XFRMINIPTFSERROR); 826 826 + goto abandon; 827 827 + } 828 828 + 829 829 + /* For the runt case we don't attempt sharing currently. NOTE: 830 830 + * Currently, this IPTFS implementation will not create runts. 831 831 + */ 832 832 + 833 833 + newskb = iptfs_alloc_skb(skb, ipremain, false); 834 834 + if (!newskb) { 835 835 + XFRM_INC_STATS(xs_net(xtfs->x), LINUX_MIB_XFRMINERROR); 836 836 + goto abandon; 837 837 + } 838 838 + xtfs->ra_newskb = newskb; 839 839 + 840 840 + /* Copy the runt data into the buffer, but leave data 841 841 + * pointers the same as normal non-runt case. The extra `rrem` 842 842 + * recopied bytes are basically cacheline free. Allows using 843 843 + * same logic below to complete. 844 844 + */ 845 845 + memcpy(skb_put(newskb, runtlen), xtfs->ra_runt, 846 846 + sizeof(xtfs->ra_runt)); 847 847 + } 848 848 + 849 849 + /* Continue reassembling the packet */ 850 850 + ipremain = __iptfs_iplen(newskb->data); 851 851 + iphlen = __iptfs_iphlen(newskb->data); 852 852 + 853 853 + ipremain -= newskb->len; 854 854 + if (blkoff < ipremain) { 855 855 + /* Corrupt data, we don't have enough to complete the packet */ 856 856 + XFRM_INC_STATS(xs_net(xtfs->x), LINUX_MIB_XFRMINIPTFSERROR); 857 857 + goto abandon; 858 858 + } 859 859 + 860 860 + /* We want the IP header in linear space */ 861 861 + if (newskb->len < iphlen) { 862 862 + iphremain = iphlen - newskb->len; 863 863 + if (blkoff < iphremain) { 864 864 + XFRM_INC_STATS(xs_net(xtfs->x), 865 865 + LINUX_MIB_XFRMINIPTFSERROR); 866 866 + goto abandon; 867 867 + } 868 868 + fraglen = min(blkoff, remaining); 869 869 + copylen = min(fraglen, iphremain); 870 870 + if (skb_copy_seq_read(st, data, skb_put(newskb, copylen), 871 871 + copylen)) { 872 872 + XFRM_INC_STATS(xs_net(xtfs->x), 873 873 + LINUX_MIB_XFRMINBUFFERERROR); 874 874 + goto abandon; 875 875 + } 876 876 + /* this is a silly condition that might occur anyway */ 877 877 + if (copylen < iphremain) { 878 878 + xtfs->ra_wantseq++; 879 879 + return data + fraglen; 880 880 + } 881 881 + /* update data and things derived from it */ 882 882 + data += copylen; 883 883 + blkoff -= copylen; 884 884 + remaining -= copylen; 885 885 + ipremain -= copylen; 886 886 + } 887 887 + 888 888 + fraglen = min(blkoff, remaining); 889 889 + copylen = min(fraglen, ipremain); 890 890 + 891 891 + /* If we may have the opportunity to share prepare a fragwalk. */ 892 892 + if (!skb_has_frag_list(skb) && !skb_has_frag_list(newskb) && 893 893 + (skb->head_frag || skb->len == skb->data_len) && 894 894 + skb->pp_recycle == newskb->pp_recycle) { 895 895 + fragwalk = &_fragwalk; 896 896 + iptfs_skb_prepare_frag_walk(skb, data, fragwalk); 897 897 + } 898 898 + 899 899 + /* Try share then copy. */ 900 900 + if (fragwalk && 901 901 + iptfs_skb_can_add_frags(newskb, fragwalk, data, copylen)) { 902 902 + iptfs_skb_add_frags(newskb, fragwalk, data, copylen); 903 903 + } else { 904 904 + /* copy fragment data into newskb */ 905 905 + if (skb_copy_seq_read(st, data, skb_put(newskb, copylen), 906 906 + copylen)) { 907 907 + XFRM_INC_STATS(xs_net(xtfs->x), 908 908 + LINUX_MIB_XFRMINBUFFERERROR); 909 909 + goto abandon; 910 910 + } 911 911 + } 912 912 + 913 913 + if (copylen < ipremain) { 914 914 + xtfs->ra_wantseq++; 915 915 + } else { 916 916 + /* We are done with packet reassembly! */ 917 917 + iptfs_reassem_done(xtfs); 918 918 + iptfs_complete_inner_skb(xtfs->x, newskb); 919 919 + list_add_tail(&newskb->list, list); 920 920 + } 921 921 + 922 922 + /* will continue on to new data block or end */ 923 923 + return data + fraglen; 924 924 + 925 925 + abandon: 926 926 + if (xtfs->ra_newskb) { 927 927 + iptfs_reassem_abort(xtfs); 928 928 + } else { 929 929 + xtfs->ra_runtlen = 0; 930 930 + xtfs->ra_wantseq = 0; 931 931 + } 932 932 + /* skip past fragment, maybe to end */ 933 933 + return data + min(blkoff, remaining); 934 934 + } 935 935 + 936 936 + static bool __input_process_payload(struct xfrm_state *x, u32 data, 937 937 + struct skb_seq_state *skbseq, 938 938 + struct list_head *sublist) 939 939 + { 940 940 + u8 hbytes[sizeof(struct ipv6hdr)]; 941 941 + struct iptfs_skb_frag_walk _fragwalk; 942 942 + struct iptfs_skb_frag_walk *fragwalk = NULL; 943 943 + struct sk_buff *defer, *first_skb, *next, *skb; 944 944 + const unsigned char *old_mac; 945 945 + struct xfrm_iptfs_data *xtfs; 946 946 + struct iphdr *iph; 947 947 + struct net *net; 948 948 + u32 first_iplen, iphlen, iplen, remaining, tail; 949 949 + u32 capturelen; 950 950 + u64 seq; 951 951 + 952 952 + xtfs = x->mode_data; 953 953 + net = xs_net(x); 954 954 + skb = skbseq->root_skb; 955 955 + first_skb = NULL; 956 956 + defer = NULL; 957 957 + 958 958 + seq = __esp_seq(skb); 959 959 + 960 960 + /* Save the old mac header if set */ 961 961 + old_mac = skb_mac_header_was_set(skb) ? skb_mac_header(skb) : NULL; 962 962 + 963 963 + /* New packets */ 964 964 + 965 965 + tail = skb->len; 966 966 + while (data < tail) { 967 967 + __be16 protocol = 0; 968 968 + 969 969 + /* Gather information on the next data block. 970 970 + * `data` points to the start of the data block. 971 971 + */ 972 972 + remaining = tail - data; 973 973 + 974 974 + /* try and copy enough bytes to read length from ipv4/ipv6 */ 975 975 + iphlen = min_t(u32, remaining, 6); 976 976 + if (skb_copy_seq_read(skbseq, data, hbytes, iphlen)) { 977 977 + XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); 978 978 + goto done; 979 979 + } 980 980 + 981 981 + iph = (struct iphdr *)hbytes; 982 982 + if (iph->version == 0x4) { 983 983 + /* must have at least tot_len field present */ 984 984 + if (remaining < 4) { 985 985 + /* save the bytes we have, advance data and exit */ 986 986 + iptfs_input_save_runt(xtfs, seq, hbytes, 987 987 + remaining); 988 988 + data += remaining; 989 989 + break; 990 990 + } 991 991 + 992 992 + iplen = be16_to_cpu(iph->tot_len); 993 993 + iphlen = iph->ihl << 2; 994 994 + protocol = cpu_to_be16(ETH_P_IP); 995 995 + XFRM_MODE_SKB_CB(skbseq->root_skb)->tos = iph->tos; 996 996 + } else if (iph->version == 0x6) { 997 997 + /* must have at least payload_len field present */ 998 998 + if (remaining < 6) { 999 999 + /* save the bytes we have, advance data and exit */ 1000 1000 + iptfs_input_save_runt(xtfs, seq, hbytes, 1001 1001 + remaining); 1002 1002 + data += remaining; 1003 1003 + break; 1004 1004 + } 1005 1005 + 1006 1006 + iplen = be16_to_cpu(((struct ipv6hdr *)hbytes)->payload_len); 1007 1007 + iplen += sizeof(struct ipv6hdr); 1008 1008 + iphlen = sizeof(struct ipv6hdr); 1009 1009 + protocol = cpu_to_be16(ETH_P_IPV6); 1010 1010 + XFRM_MODE_SKB_CB(skbseq->root_skb)->tos = 1011 1011 + ipv6_get_dsfield((struct ipv6hdr *)iph); 1012 1012 + } else if (iph->version == 0x0) { 1013 1013 + /* pad */ 1014 1014 + data = tail; 1015 1015 + break; 1016 1016 + } else { 1017 1017 + XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); 1018 1018 + goto done; 1019 1019 + } 1020 1020 + 1021 1021 + if (unlikely(skbseq->stepped_offset)) { 1022 1022 + /* We need to reset our seq read, it can't backup at 1023 1023 + * this point. 1024 1024 + */ 1025 1025 + struct sk_buff *save = skbseq->root_skb; 1026 1026 + 1027 1027 + skb_abort_seq_read(skbseq); 1028 1028 + skb_prepare_seq_read(save, data, tail, skbseq); 1029 1029 + } 1030 1030 + 1031 1031 + if (first_skb) { 1032 1032 + skb = NULL; 1033 1033 + } else { 1034 1034 + first_skb = skb; 1035 1035 + first_iplen = iplen; 1036 1036 + fragwalk = NULL; 1037 1037 + 1038 1038 + /* We are going to skip over `data` bytes to reach the 1039 1039 + * start of the IP header of `iphlen` len for `iplen` 1040 1040 + * inner packet. 1041 1041 + */ 1042 1042 + 1043 1043 + if (skb_has_frag_list(skb)) { 1044 1044 + defer = skb; 1045 1045 + skb = NULL; 1046 1046 + } else if (data + iphlen <= skb_headlen(skb) && 1047 1047 + /* make sure our header is 32-bit aligned? */ 1048 1048 + /* ((uintptr_t)(skb->data + data) & 0x3) == 0 && */ 1049 1049 + skb_tailroom(skb) + tail - data >= iplen) { 1050 1050 + /* Reuse the received skb. 1051 1051 + * 1052 1052 + * We have enough headlen to pull past any 1053 1053 + * initial fragment data, leaving at least the 1054 1054 + * IP header in the linear buffer space. 1055 1055 + * 1056 1056 + * For linear buffer space we only require that 1057 1057 + * linear buffer space is large enough to 1058 1058 + * eventually hold the entire reassembled 1059 1059 + * packet (by including tailroom in the check). 1060 1060 + * 1061 1061 + * For non-linear tailroom is 0 and so we only 1062 1062 + * re-use if the entire packet is present 1063 1063 + * already. 1064 1064 + * 1065 1065 + * NOTE: there are many more options for 1066 1066 + * sharing, KISS for now. Also, this can produce 1067 1067 + * skb's with the IP header unaligned to 32 1068 1068 + * bits. If that ends up being a problem then a 1069 1069 + * check should be added to the conditional 1070 1070 + * above that the header lies on a 32-bit 1071 1071 + * boundary as well. 1072 1072 + */ 1073 1073 + skb_pull(skb, data); 1074 1074 + 1075 1075 + /* our range just changed */ 1076 1076 + data = 0; 1077 1077 + tail = skb->len; 1078 1078 + remaining = skb->len; 1079 1079 + 1080 1080 + skb->protocol = protocol; 1081 1081 + skb_mac_header_rebuild(skb); 1082 1082 + if (skb->mac_len) 1083 1083 + eth_hdr(skb)->h_proto = skb->protocol; 1084 1084 + 1085 1085 + /* all pointers could be changed now reset walk */ 1086 1086 + skb_abort_seq_read(skbseq); 1087 1087 + skb_prepare_seq_read(skb, data, tail, skbseq); 1088 1088 + } else if (skb->head_frag && 1089 1089 + /* We have the IP header right now */ 1090 1090 + remaining >= iphlen) { 1091 1091 + fragwalk = &_fragwalk; 1092 1092 + iptfs_skb_prepare_frag_walk(skb, data, fragwalk); 1093 1093 + defer = skb; 1094 1094 + skb = NULL; 1095 1095 + } else { 1096 1096 + /* We couldn't reuse the input skb so allocate a 1097 1097 + * new one. 1098 1098 + */ 1099 1099 + defer = skb; 1100 1100 + skb = NULL; 1101 1101 + } 1102 1102 + 1103 1103 + /* Don't trim `first_skb` until the end as we are 1104 1104 + * walking that data now. 1105 1105 + */ 1106 1106 + } 1107 1107 + 1108 1108 + capturelen = min(iplen, remaining); 1109 1109 + if (!skb) { 1110 1110 + if (!fragwalk || 1111 1111 + /* Large enough to be worth sharing */ 1112 1112 + iplen < IPTFS_PKT_SHARE_MIN || 1113 1113 + /* Have IP header + some data to share. */ 1114 1114 + capturelen <= iphlen || 1115 1115 + /* Try creating skb and adding frags */ 1116 1116 + !(skb = iptfs_pskb_add_frags(first_skb, fragwalk, 1117 1117 + data, capturelen, 1118 1118 + skbseq, iphlen))) { 1119 1119 + skb = iptfs_pskb_extract_seq(iplen, skbseq, data, capturelen); 1120 1120 + } 1121 1121 + if (!skb) { 1122 1122 + /* skip to next packet or done */ 1123 1123 + data += capturelen; 1124 1124 + continue; 1125 1125 + } 1126 1126 + 1127 1127 + skb->protocol = protocol; 1128 1128 + if (old_mac) { 1129 1129 + /* rebuild the mac header */ 1130 1130 + skb_set_mac_header(skb, -first_skb->mac_len); 1131 1131 + memcpy(skb_mac_header(skb), old_mac, first_skb->mac_len); 1132 1132 + eth_hdr(skb)->h_proto = skb->protocol; 1133 1133 + } 1134 1134 + } 1135 1135 + 1136 1136 + data += capturelen; 1137 1137 + 1138 1138 + if (skb->len < iplen) { 1139 1139 + /* Start reassembly */ 1140 1140 + spin_lock(&xtfs->drop_lock); 1141 1141 + 1142 1142 + xtfs->ra_newskb = skb; 1143 1143 + xtfs->ra_wantseq = seq + 1; 1144 1144 + if (!hrtimer_is_queued(&xtfs->drop_timer)) { 1145 1145 + /* softirq blocked lest the timer fire and interrupt us */ 1146 1146 + hrtimer_start(&xtfs->drop_timer, 1147 1147 + xtfs->drop_time_ns, 1148 1148 + IPTFS_HRTIMER_MODE); 1149 1149 + } 1150 1150 + 1151 1151 + spin_unlock(&xtfs->drop_lock); 1152 1152 + 1153 1153 + break; 1154 1154 + } 1155 1155 + 1156 1156 + iptfs_complete_inner_skb(x, skb); 1157 1157 + list_add_tail(&skb->list, sublist); 1158 1158 + } 1159 1159 + 1160 1160 + if (data != tail) 1161 1161 + /* this should not happen from the above code */ 1162 1162 + XFRM_INC_STATS(net, LINUX_MIB_XFRMINIPTFSERROR); 1163 1163 + 1164 1164 + if (first_skb && first_iplen && !defer && first_skb != xtfs->ra_newskb) { 1165 1165 + /* first_skb is queued b/c !defer and not partial */ 1166 1166 + if (pskb_trim(first_skb, first_iplen)) { 1167 1167 + /* error trimming */ 1168 1168 + list_del(&first_skb->list); 1169 1169 + defer = first_skb; 1170 1170 + } 1171 1171 + first_skb->ip_summed = CHECKSUM_NONE; 1172 1172 + } 1173 1173 + 1174 1174 + /* Send the packets! */ 1175 1175 + list_for_each_entry_safe(skb, next, sublist, list) { 1176 1176 + skb_list_del_init(skb); 1177 1177 + if (xfrm_input(skb, 0, 0, -2)) 1178 1178 + kfree_skb(skb); 1179 1179 + } 1180 1180 + done: 1181 1181 + skb = skbseq->root_skb; 1182 1182 + skb_abort_seq_read(skbseq); 1183 1183 + 1184 1184 + if (defer) { 1185 1185 + consume_skb(defer); 1186 1186 + } else if (!first_skb) { 1187 1187 + /* skb is the original passed in skb, but we didn't get far 1188 1188 + * enough to process it as the first_skb, if we had it would 1189 1189 + * either be save in ra_newskb, trimmed and sent on as an skb or 1190 1190 + * placed in defer to be freed. 1191 1191 + */ 1192 1192 + kfree_skb(skb); 1193 1193 + } 1194 1194 + return true; 1195 1195 + } 1196 1196 + 1197 1197 + /** 1198 1198 + * iptfs_input_ordered() - handle next in order IPTFS payload. 1199 1199 + * @x: xfrm state 1200 1200 + * @skb: current packet 1201 1201 + * 1202 1202 + * Process the IPTFS payload in `skb` and consume it afterwards. 1203 1203 + */ 1204 1204 + static void iptfs_input_ordered(struct xfrm_state *x, struct sk_buff *skb) 1205 1205 + { 1206 1206 + struct ip_iptfs_cc_hdr iptcch; 1207 1207 + struct skb_seq_state skbseq; 1208 1208 + struct list_head sublist; /* rename this it's just a list */ 1209 1209 + struct xfrm_iptfs_data *xtfs; 1210 1210 + struct ip_iptfs_hdr *ipth; 1211 1211 + struct net *net; 1212 1212 + u32 blkoff, data, remaining; 1213 1213 + bool consumed = false; 1214 1214 + u64 seq; 1215 1215 + 1216 1216 + xtfs = x->mode_data; 1217 1217 + net = xs_net(x); 1218 1218 + 1219 1219 + seq = __esp_seq(skb); 1220 1220 + 1221 1221 + /* Large enough to hold both types of header */ 1222 1222 + ipth = (struct ip_iptfs_hdr *)&iptcch; 1223 1223 + 1224 1224 + skb_prepare_seq_read(skb, 0, skb->len, &skbseq); 1225 1225 + 1226 1226 + /* Get the IPTFS header and validate it */ 1227 1227 + 1228 1228 + if (skb_copy_seq_read(&skbseq, 0, ipth, sizeof(*ipth))) { 1229 1229 + XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); 1230 1230 + goto done; 1231 1231 + } 1232 1232 + data = sizeof(*ipth); 1233 1233 + 1234 1234 + trace_iptfs_egress_recv(skb, xtfs, be16_to_cpu(ipth->block_offset)); 1235 1235 + 1236 1236 + /* Set data past the basic header */ 1237 1237 + if (ipth->subtype == IPTFS_SUBTYPE_CC) { 1238 1238 + /* Copy the rest of the CC header */ 1239 1239 + remaining = sizeof(iptcch) - sizeof(*ipth); 1240 1240 + if (skb_copy_seq_read(&skbseq, data, ipth + 1, remaining)) { 1241 1241 + XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); 1242 1242 + goto done; 1243 1243 + } 1244 1244 + data += remaining; 1245 1245 + } else if (ipth->subtype != IPTFS_SUBTYPE_BASIC) { 1246 1246 + XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); 1247 1247 + goto done; 1248 1248 + } 1249 1249 + 1250 1250 + if (ipth->flags != 0) { 1251 1251 + XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); 1252 1252 + goto done; 1253 1253 + } 1254 1254 + 1255 1255 + INIT_LIST_HEAD(&sublist); 1256 1256 + 1257 1257 + /* Handle fragment at start of payload, and/or waiting reassembly. */ 1258 1258 + 1259 1259 + blkoff = ntohs(ipth->block_offset); 1260 1260 + /* check before locking i.e., maybe */ 1261 1261 + if (blkoff || xtfs->ra_runtlen || xtfs->ra_newskb) { 1262 1262 + spin_lock(&xtfs->drop_lock); 1263 1263 + 1264 1264 + /* check again after lock */ 1265 1265 + if (blkoff || xtfs->ra_runtlen || xtfs->ra_newskb) { 1266 1266 + data = iptfs_reassem_cont(xtfs, seq, &skbseq, skb, data, 1267 1267 + blkoff, &sublist); 1268 1268 + } 1269 1269 + 1270 1270 + spin_unlock(&xtfs->drop_lock); 1271 1271 + } 1272 1272 + 1273 1273 + /* New packets */ 1274 1274 + consumed = __input_process_payload(x, data, &skbseq, &sublist); 1275 1275 + done: 1276 1276 + if (!consumed) { 1277 1277 + skb = skbseq.root_skb; 1278 1278 + skb_abort_seq_read(&skbseq); 1279 1279 + kfree_skb(skb); 1280 1280 + } 1281 1281 + } 1282 1282 + 1283 1283 + /* ------------------------------- */ 1284 1284 + /* Input (Egress) Re-ordering Code */ 1285 1285 + /* ------------------------------- */ 1286 1286 + 1287 1287 + static void __vec_shift(struct xfrm_iptfs_data *xtfs, u32 shift) 1288 1288 + { 1289 1289 + u32 savedlen = xtfs->w_savedlen; 1290 1290 + 1291 1291 + if (shift > savedlen) 1292 1292 + shift = savedlen; 1293 1293 + if (shift != savedlen) 1294 1294 + memcpy(xtfs->w_saved, xtfs->w_saved + shift, 1295 1295 + (savedlen - shift) * sizeof(*xtfs->w_saved)); 1296 1296 + memset(xtfs->w_saved + savedlen - shift, 0, 1297 1297 + shift * sizeof(*xtfs->w_saved)); 1298 1298 + xtfs->w_savedlen -= shift; 1299 1299 + } 1300 1300 + 1301 1301 + static void __reorder_past(struct xfrm_iptfs_data *xtfs, struct sk_buff *inskb, 1302 1302 + struct list_head *freelist) 1303 1303 + { 1304 1304 + list_add_tail(&inskb->list, freelist); 1305 1305 + } 1306 1306 + 1307 1307 + static u32 __reorder_drop(struct xfrm_iptfs_data *xtfs, struct list_head *list) 1308 1308 + 1309 1309 + { 1310 1310 + struct skb_wseq *s, *se; 1311 1311 + const u32 savedlen = xtfs->w_savedlen; 1312 1312 + time64_t now = ktime_get_raw_fast_ns(); 1313 1313 + u32 count = 0; 1314 1314 + u32 scount = 0; 1315 1315 + 1316 1316 + if (xtfs->w_saved[0].drop_time > now) 1317 1317 + goto set_timer; 1318 1318 + 1319 1319 + ++xtfs->w_wantseq; 1320 1320 + 1321 1321 + /* Keep flushing packets until we reach a drop time greater than now. */ 1322 1322 + s = xtfs->w_saved; 1323 1323 + se = s + savedlen; 1324 1324 + do { 1325 1325 + /* Walking past empty slots until we reach a packet */ 1326 1326 + for (; s < se && !s->skb; s++) { 1327 1327 + if (s->drop_time > now) 1328 1328 + goto outerdone; 1329 1329 + } 1330 1330 + /* Sending packets until we hit another empty slot. */ 1331 1331 + for (; s < se && s->skb; scount++, s++) 1332 1332 + list_add_tail(&s->skb->list, list); 1333 1333 + } while (s < se); 1334 1334 + outerdone: 1335 1335 + 1336 1336 + count = s - xtfs->w_saved; 1337 1337 + if (count) { 1338 1338 + xtfs->w_wantseq += count; 1339 1339 + 1340 1340 + /* Shift handled slots plus final empty slot into slot 0. */ 1341 1341 + __vec_shift(xtfs, count); 1342 1342 + } 1343 1343 + 1344 1344 + if (xtfs->w_savedlen) { 1345 1345 + set_timer: 1346 1346 + /* Drifting is OK */ 1347 1347 + hrtimer_start(&xtfs->drop_timer, 1348 1348 + xtfs->w_saved[0].drop_time - now, 1349 1349 + IPTFS_HRTIMER_MODE); 1350 1350 + } 1351 1351 + return scount; 1352 1352 + } 1353 1353 + 1354 1354 + static void __reorder_this(struct xfrm_iptfs_data *xtfs, struct sk_buff *inskb, 1355 1355 + struct list_head *list) 1356 1356 + { 1357 1357 + struct skb_wseq *s, *se; 1358 1358 + const u32 savedlen = xtfs->w_savedlen; 1359 1359 + u32 count = 0; 1360 1360 + 1361 1361 + /* Got what we wanted. */ 1362 1362 + list_add_tail(&inskb->list, list); 1363 1363 + ++xtfs->w_wantseq; 1364 1364 + if (!savedlen) 1365 1365 + return; 1366 1366 + 1367 1367 + /* Flush remaining consecutive packets. */ 1368 1368 + 1369 1369 + /* Keep sending until we hit another missed pkt. */ 1370 1370 + for (s = xtfs->w_saved, se = s + savedlen; s < se && s->skb; s++) 1371 1371 + list_add_tail(&s->skb->list, list); 1372 1372 + count = s - xtfs->w_saved; 1373 1373 + if (count) 1374 1374 + xtfs->w_wantseq += count; 1375 1375 + 1376 1376 + /* Shift handled slots plus final empty slot into slot 0. */ 1377 1377 + __vec_shift(xtfs, count + 1); 1378 1378 + } 1379 1379 + 1380 1380 + /* Set the slot's drop time and all the empty slots below it until reaching a 1381 1381 + * filled slot which will already be set. 1382 1382 + */ 1383 1383 + static void iptfs_set_window_drop_times(struct xfrm_iptfs_data *xtfs, int index) 1384 1384 + { 1385 1385 + const u32 savedlen = xtfs->w_savedlen; 1386 1386 + struct skb_wseq *s = xtfs->w_saved; 1387 1387 + time64_t drop_time; 1388 1388 + 1389 1389 + assert_spin_locked(&xtfs->drop_lock); 1390 1390 + 1391 1391 + if (savedlen > index + 1) { 1392 1392 + /* we are below another, our drop time and the timer are already set */ 1393 1393 + return; 1394 1394 + } 1395 1395 + /* we are the most future so get a new drop time. */ 1396 1396 + drop_time = ktime_get_raw_fast_ns(); 1397 1397 + drop_time += xtfs->drop_time_ns; 1398 1398 + 1399 1399 + /* Walk back through the array setting drop times as we go */ 1400 1400 + s[index].drop_time = drop_time; 1401 1401 + while (index-- > 0 && !s[index].skb) 1402 1402 + s[index].drop_time = drop_time; 1403 1403 + 1404 1404 + /* If we walked all the way back, schedule the drop timer if needed */ 1405 1405 + if (index == -1 && !hrtimer_is_queued(&xtfs->drop_timer)) 1406 1406 + hrtimer_start(&xtfs->drop_timer, xtfs->drop_time_ns, 1407 1407 + IPTFS_HRTIMER_MODE); 1408 1408 + } 1409 1409 + 1410 1410 + static void __reorder_future_fits(struct xfrm_iptfs_data *xtfs, 1411 1411 + struct sk_buff *inskb, 1412 1412 + struct list_head *freelist) 1413 1413 + { 1414 1414 + const u64 inseq = __esp_seq(inskb); 1415 1415 + const u64 wantseq = xtfs->w_wantseq; 1416 1416 + const u64 distance = inseq - wantseq; 1417 1417 + const u32 savedlen = xtfs->w_savedlen; 1418 1418 + const u32 index = distance - 1; 1419 1419 + 1420 1420 + /* Handle future sequence number received which fits in the window. 1421 1421 + * 1422 1422 + * We know we don't have the seq we want so we won't be able to flush 1423 1423 + * anything. 1424 1424 + */ 1425 1425 + 1426 1426 + /* slot count is 4, saved size is 3 savedlen is 2 1427 1427 + * 1428 1428 + * "window boundary" is based on the fixed window size 1429 1429 + * distance is also slot number 1430 1430 + * index is an array index (i.e., - 1 of slot) 1431 1431 + * : : - implicit NULL after array len 1432 1432 + * 1433 1433 + * +--------- used length (savedlen == 2) 1434 1434 + * | +----- array size (nslots - 1 == 3) 1435 1435 + * | | + window boundary (nslots == 4) 1436 1436 + * V V | V 1437 1437 + * | 1438 1438 + * 0 1 2 3 | slot number 1439 1439 + * --- 0 1 2 | array index 1440 1440 + * [-] [b] : :| array 1441 1441 + * 1442 1442 + * "2" "3" "4" *5*| seq numbers 1443 1443 + * 1444 1444 + * We receive seq number 5 1445 1445 + * distance == 3 [inseq(5) - w_wantseq(2)] 1446 1446 + * index == 2 [distance(6) - 1] 1447 1447 + */ 1448 1448 + 1449 1449 + if (xtfs->w_saved[index].skb) { 1450 1450 + /* a dup of a future */ 1451 1451 + list_add_tail(&inskb->list, freelist); 1452 1452 + return; 1453 1453 + } 1454 1454 + 1455 1455 + xtfs->w_saved[index].skb = inskb; 1456 1456 + xtfs->w_savedlen = max(savedlen, index + 1); 1457 1457 + iptfs_set_window_drop_times(xtfs, index); 1458 1458 + } 1459 1459 + 1460 1460 + static void __reorder_future_shifts(struct xfrm_iptfs_data *xtfs, 1461 1461 + struct sk_buff *inskb, 1462 1462 + struct list_head *list) 1463 1463 + { 1464 1464 + const u32 nslots = xtfs->cfg.reorder_win_size + 1; 1465 1465 + const u64 inseq = __esp_seq(inskb); 1466 1466 + u32 savedlen = xtfs->w_savedlen; 1467 1467 + u64 wantseq = xtfs->w_wantseq; 1468 1468 + struct skb_wseq *wnext; 1469 1469 + struct sk_buff *slot0; 1470 1470 + u32 beyond, shifting, slot; 1471 1471 + u64 distance; 1472 1472 + 1473 1473 + /* Handle future sequence number received. 1474 1474 + * 1475 1475 + * IMPORTANT: we are at least advancing w_wantseq (i.e., wantseq) by 1 1476 1476 + * b/c we are beyond the window boundary. 1477 1477 + * 1478 1478 + * We know we don't have the wantseq so that counts as a drop. 1479 1479 + */ 1480 1480 + 1481 1481 + /* example: slot count is 4, array size is 3 savedlen is 2, slot 0 is 1482 1482 + * the missing sequence number. 1483 1483 + * 1484 1484 + * the final slot at savedlen (index savedlen - 1) is always occupied. 1485 1485 + * 1486 1486 + * beyond is "beyond array size" not savedlen. 1487 1487 + * 1488 1488 + * +--------- array length (savedlen == 2) 1489 1489 + * | +----- array size (nslots - 1 == 3) 1490 1490 + * | | +- window boundary (nslots == 4) 1491 1491 + * V V | 1492 1492 + * | 1493 1493 + * 0 1 2 3 | slot number 1494 1494 + * --- 0 1 2 | array index 1495 1495 + * [b] [c] : :| array 1496 1496 + * | 1497 1497 + * "2" "3" "4" "5"|*6* seq numbers 1498 1498 + * 1499 1499 + * We receive seq number 6 1500 1500 + * distance == 4 [inseq(6) - w_wantseq(2)] 1501 1501 + * newslot == distance 1502 1502 + * index == 3 [distance(4) - 1] 1503 1503 + * beyond == 1 [newslot(4) - lastslot((nslots(4) - 1))] 1504 1504 + * shifting == 1 [min(savedlen(2), beyond(1)] 1505 1505 + * slot0_skb == [b], and should match w_wantseq 1506 1506 + * 1507 1507 + * +--- window boundary (nslots == 4) 1508 1508 + * 0 1 2 3 | 4 slot number 1509 1509 + * --- 0 1 2 | 3 array index 1510 1510 + * [b] : : : :| array 1511 1511 + * "2" "3" "4" "5" *6* seq numbers 1512 1512 + * 1513 1513 + * We receive seq number 6 1514 1514 + * distance == 4 [inseq(6) - w_wantseq(2)] 1515 1515 + * newslot == distance 1516 1516 + * index == 3 [distance(4) - 1] 1517 1517 + * beyond == 1 [newslot(4) - lastslot((nslots(4) - 1))] 1518 1518 + * shifting == 1 [min(savedlen(1), beyond(1)] 1519 1519 + * slot0_skb == [b] and should match w_wantseq 1520 1520 + * 1521 1521 + * +-- window boundary (nslots == 4) 1522 1522 + * 0 1 2 3 | 4 5 6 slot number 1523 1523 + * --- 0 1 2 | 3 4 5 array index 1524 1524 + * [-] [c] : :| array 1525 1525 + * "2" "3" "4" "5" "6" "7" *8* seq numbers 1526 1526 + * 1527 1527 + * savedlen = 2, beyond = 3 1528 1528 + * iter 1: slot0 == NULL, missed++, lastdrop = 2 (2+1-1), slot0 = [-] 1529 1529 + * iter 2: slot0 == NULL, missed++, lastdrop = 3 (2+2-1), slot0 = [c] 1530 1530 + * 2 < 3, extra = 1 (3-2), missed += extra, lastdrop = 4 (2+2+1-1) 1531 1531 + * 1532 1532 + * We receive seq number 8 1533 1533 + * distance == 6 [inseq(8) - w_wantseq(2)] 1534 1534 + * newslot == distance 1535 1535 + * index == 5 [distance(6) - 1] 1536 1536 + * beyond == 3 [newslot(6) - lastslot((nslots(4) - 1))] 1537 1537 + * shifting == 2 [min(savedlen(2), beyond(3)] 1538 1538 + * 1539 1539 + * slot0_skb == NULL changed from [b] when "savedlen < beyond" is true. 1540 1540 + */ 1541 1541 + 1542 1542 + /* Now send any packets that are being shifted out of saved, and account 1543 1543 + * for missing packets that are exiting the window as we shift it. 1544 1544 + */ 1545 1545 + 1546 1546 + distance = inseq - wantseq; 1547 1547 + beyond = distance - (nslots - 1); 1548 1548 + 1549 1549 + /* If savedlen > beyond we are shifting some, else all. */ 1550 1550 + shifting = min(savedlen, beyond); 1551 1551 + 1552 1552 + /* slot0 is the buf that just shifted out and into slot0 */ 1553 1553 + slot0 = NULL; 1554 1554 + wnext = xtfs->w_saved; 1555 1555 + for (slot = 1; slot <= shifting; slot++, wnext++) { 1556 1556 + /* handle what was in slot0 before we occupy it */ 1557 1557 + if (slot0) 1558 1558 + list_add_tail(&slot0->list, list); 1559 1559 + slot0 = wnext->skb; 1560 1560 + wnext->skb = NULL; 1561 1561 + } 1562 1562 + 1563 1563 + /* slot0 is now either NULL (in which case it's what we now are waiting 1564 1564 + * for, or a buf in which case we need to handle it like we received it; 1565 1565 + * however, we may be advancing past that buffer as well.. 1566 1566 + */ 1567 1567 + 1568 1568 + /* Handle case where we need to shift more than we had saved, slot0 will 1569 1569 + * be NULL iff savedlen is 0, otherwise slot0 will always be 1570 1570 + * non-NULL b/c we shifted the final element, which is always set if 1571 1571 + * there is any saved, into slot0. 1572 1572 + */ 1573 1573 + if (savedlen < beyond) { 1574 1574 + if (savedlen != 0) 1575 1575 + list_add_tail(&slot0->list, list); 1576 1576 + slot0 = NULL; 1577 1577 + /* slot0 has had an empty slot pushed into it */ 1578 1578 + } 1579 1579 + 1580 1580 + /* Remove the entries */ 1581 1581 + __vec_shift(xtfs, beyond); 1582 1582 + 1583 1583 + /* Advance want seq */ 1584 1584 + xtfs->w_wantseq += beyond; 1585 1585 + 1586 1586 + /* Process drops here when implementing congestion control */ 1587 1587 + 1588 1588 + /* We've shifted. plug the packet in at the end. */ 1589 1589 + xtfs->w_savedlen = nslots - 1; 1590 1590 + xtfs->w_saved[xtfs->w_savedlen - 1].skb = inskb; 1591 1591 + iptfs_set_window_drop_times(xtfs, xtfs->w_savedlen - 1); 1592 1592 + 1593 1593 + /* if we don't have a slot0 then we must wait for it */ 1594 1594 + if (!slot0) 1595 1595 + return; 1596 1596 + 1597 1597 + /* If slot0, seq must match new want seq */ 1598 1598 + 1599 1599 + /* slot0 is valid, treat like we received expected. */ 1600 1600 + __reorder_this(xtfs, slot0, list); 1601 1601 + } 1602 1602 + 1603 1603 + /* Receive a new packet into the reorder window. Return a list of ordered 1604 1604 + * packets from the window. 1605 1605 + */ 1606 1606 + static void iptfs_input_reorder(struct xfrm_iptfs_data *xtfs, 1607 1607 + struct sk_buff *inskb, struct list_head *list, 1608 1608 + struct list_head *freelist) 1609 1609 + { 1610 1610 + const u32 nslots = xtfs->cfg.reorder_win_size + 1; 1611 1611 + u64 inseq = __esp_seq(inskb); 1612 1612 + u64 wantseq; 1613 1613 + 1614 1614 + assert_spin_locked(&xtfs->drop_lock); 1615 1615 + 1616 1616 + if (unlikely(!xtfs->w_seq_set)) { 1617 1617 + xtfs->w_seq_set = true; 1618 1618 + xtfs->w_wantseq = inseq; 1619 1619 + } 1620 1620 + wantseq = xtfs->w_wantseq; 1621 1621 + 1622 1622 + if (likely(inseq == wantseq)) 1623 1623 + __reorder_this(xtfs, inskb, list); 1624 1624 + else if (inseq < wantseq) 1625 1625 + __reorder_past(xtfs, inskb, freelist); 1626 1626 + else if ((inseq - wantseq) < nslots) 1627 1627 + __reorder_future_fits(xtfs, inskb, freelist); 1628 1628 + else 1629 1629 + __reorder_future_shifts(xtfs, inskb, list); 1630 1630 + } 1631 1631 + 1632 1632 + /** 1633 1633 + * iptfs_drop_timer() - Handle drop timer expiry. 1634 1634 + * @me: the timer 1635 1635 + * 1636 1636 + * This is similar to our input function. 1637 1637 + * 1638 1638 + * The drop timer is set when we start an in progress reassembly, and also when 1639 1639 + * we save a future packet in the window saved array. 1640 1640 + * 1641 1641 + * NOTE packets in the save window are always newer WRT drop times as 1642 1642 + * they get further in the future. i.e. for: 1643 1643 + * 1644 1644 + * if slots (S0, S1, ... Sn) and `Dn` is the drop time for slot `Sn`, 1645 1645 + * then D(n-1) <= D(n). 1646 1646 + * 1647 1647 + * So, regardless of why the timer is firing we can always discard any inprogress 1648 1648 + * fragment; either it's the reassembly timer, or slot 0 is going to be 1649 1649 + * dropped as S0 must have the most recent drop time, and slot 0 holds the 1650 1650 + * continuation fragment of the in progress packet. 1651 1651 + * 1652 1652 + * Returns HRTIMER_NORESTART. 1653 1653 + */ 1654 1654 + static enum hrtimer_restart iptfs_drop_timer(struct hrtimer *me) 1655 1655 + { 1656 1656 + struct sk_buff *skb, *next; 1657 1657 + struct list_head list; 1658 1658 + struct xfrm_iptfs_data *xtfs; 1659 1659 + struct xfrm_state *x; 1660 1660 + u32 count; 1661 1661 + 1662 1662 + xtfs = container_of(me, typeof(*xtfs), drop_timer); 1663 1663 + x = xtfs->x; 1664 1664 + 1665 1665 + INIT_LIST_HEAD(&list); 1666 1666 + 1667 1667 + spin_lock(&xtfs->drop_lock); 1668 1668 + 1669 1669 + /* Drop any in progress packet */ 1670 1670 + skb = xtfs->ra_newskb; 1671 1671 + xtfs->ra_newskb = NULL; 1672 1672 + 1673 1673 + /* Now drop as many packets as we should from the reordering window 1674 1674 + * saved array 1675 1675 + */ 1676 1676 + count = xtfs->w_savedlen ? __reorder_drop(xtfs, &list) : 0; 1677 1677 + 1678 1678 + spin_unlock(&xtfs->drop_lock); 1679 1679 + 1680 1680 + if (skb) 1681 1681 + kfree_skb_reason(skb, SKB_DROP_REASON_FRAG_REASM_TIMEOUT); 1682 1682 + 1683 1683 + if (count) { 1684 1684 + list_for_each_entry_safe(skb, next, &list, list) { 1685 1685 + skb_list_del_init(skb); 1686 1686 + iptfs_input_ordered(x, skb); 1687 1687 + } 1688 1688 + } 1689 1689 + 1690 1690 + return HRTIMER_NORESTART; 1691 1691 + } 1692 1692 + 1693 1693 + /** 1694 1694 + * iptfs_input() - handle receipt of iptfs payload 1695 1695 + * @x: xfrm state 1696 1696 + * @skb: the packet 1697 1697 + * 1698 1698 + * We have an IPTFS payload order it if needed, then process newly in order 1699 1699 + * packets. 1700 1700 + * 1701 1701 + * Return: -EINPROGRESS to inform xfrm_input to stop processing the skb. 1702 1702 + */ 1703 1703 + static int iptfs_input(struct xfrm_state *x, struct sk_buff *skb) 1704 1704 + { 1705 1705 + struct list_head freelist, list; 1706 1706 + struct xfrm_iptfs_data *xtfs = x->mode_data; 1707 1707 + struct sk_buff *next; 1708 1708 + 1709 1709 + /* Fast path for no reorder window. */ 1710 1710 + if (xtfs->cfg.reorder_win_size == 0) { 1711 1711 + iptfs_input_ordered(x, skb); 1712 1712 + goto done; 1713 1713 + } 1714 1714 + 1715 1715 + /* Fetch list of in-order packets from the reordering window as well as 1716 1716 + * a list of buffers we need to now free. 1717 1717 + */ 1718 1718 + INIT_LIST_HEAD(&list); 1719 1719 + INIT_LIST_HEAD(&freelist); 1720 1720 + 1721 1721 + spin_lock(&xtfs->drop_lock); 1722 1722 + iptfs_input_reorder(xtfs, skb, &list, &freelist); 1723 1723 + spin_unlock(&xtfs->drop_lock); 1724 1724 + 1725 1725 + list_for_each_entry_safe(skb, next, &list, list) { 1726 1726 + skb_list_del_init(skb); 1727 1727 + iptfs_input_ordered(x, skb); 1728 1728 + } 1729 1729 + 1730 1730 + list_for_each_entry_safe(skb, next, &freelist, list) { 1731 1731 + skb_list_del_init(skb); 1732 1732 + kfree_skb(skb); 1733 1733 + } 1734 1734 + done: 1735 1735 + /* We always have dealt with the input SKB, either we are re-using it, 1736 1736 + * or we have freed it. Return EINPROGRESS so that xfrm_input stops 1737 1737 + * processing it. 1738 1738 + */ 1739 1739 + return -EINPROGRESS; 1740 1740 + } 1741 1741 + 1742 1742 + /* ================================= */ 1743 1743 + /* IPTFS Sending (ingress) Functions */ 1744 1744 + /* ================================= */ 1745 1745 + 1746 1746 + /* ------------------------- */ 1747 1747 + /* Enqueue to send functions */ 1748 1748 + /* ------------------------- */ 1749 1749 + 1750 1750 + /** 1751 1751 + * iptfs_enqueue() - enqueue packet if ok to send. 1752 1752 + * @xtfs: xtfs state 1753 1753 + * @skb: the packet 1754 1754 + * 1755 1755 + * Return: true if packet enqueued. 1756 1756 + */ 1757 1757 + static bool iptfs_enqueue(struct xfrm_iptfs_data *xtfs, struct sk_buff *skb) 1758 1758 + { 1759 1759 + u64 newsz = xtfs->queue_size + skb->len; 1760 1760 + struct iphdr *iph; 1761 1761 + 1762 1762 + assert_spin_locked(&xtfs->x->lock); 1763 1763 + 1764 1764 + if (newsz > xtfs->cfg.max_queue_size) 1765 1765 + return false; 1766 1766 + 1767 1767 + /* Set ECN CE if we are above our ECN queue threshold */ 1768 1768 + if (newsz > xtfs->ecn_queue_size) { 1769 1769 + iph = ip_hdr(skb); 1770 1770 + if (iph->version == 4) 1771 1771 + IP_ECN_set_ce(iph); 1772 1772 + else if (iph->version == 6) 1773 1773 + IP6_ECN_set_ce(skb, ipv6_hdr(skb)); 1774 1774 + } 1775 1775 + 1776 1776 + __skb_queue_tail(&xtfs->queue, skb); 1777 1777 + xtfs->queue_size += skb->len; 1778 1778 + return true; 1779 1779 + } 1780 1780 + 1781 1781 + static int iptfs_get_cur_pmtu(struct xfrm_state *x, struct xfrm_iptfs_data *xtfs, 1782 1782 + struct sk_buff *skb) 1783 1783 + { 1784 1784 + struct xfrm_dst *xdst = (struct xfrm_dst *)skb_dst(skb); 1785 1785 + u32 payload_mtu = xtfs->payload_mtu; 1786 1786 + u32 pmtu = __iptfs_get_inner_mtu(x, xdst->child_mtu_cached); 1787 1787 + 1788 1788 + if (payload_mtu && payload_mtu < pmtu) 1789 1789 + pmtu = payload_mtu; 1790 1790 + 1791 1791 + return pmtu; 1792 1792 + } 1793 1793 + 1794 1794 + static int iptfs_is_too_big(struct sock *sk, struct sk_buff *skb, u32 pmtu) 1795 1795 + { 1796 1796 + if (skb->len <= pmtu) 1797 1797 + return 0; 1798 1798 + 1799 1799 + /* We only send ICMP too big if the user has configured us as 1800 1800 + * dont-fragment. 1801 1801 + */ 1802 1802 + if (skb->dev) 1803 1803 + XFRM_INC_STATS(dev_net(skb->dev), LINUX_MIB_XFRMOUTERROR); 1804 1804 + 1805 1805 + if (sk) 1806 1806 + xfrm_local_error(skb, pmtu); 1807 1807 + else if (ip_hdr(skb)->version == 4) 1808 1808 + icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, htonl(pmtu)); 1809 1809 + else 1810 1810 + icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, pmtu); 1811 1811 + 1812 1812 + return 1; 1813 1813 + } 1814 1814 + 1815 1815 + /* IPv4/IPv6 packet ingress to IPTFS tunnel, arrange to send in IPTFS payload 1816 1816 + * (i.e., aggregating or fragmenting as appropriate). 1817 1817 + * This is set in dst->output for an SA. 1818 1818 + */ 1819 1819 + static int iptfs_output_collect(struct net *net, struct sock *sk, struct sk_buff *skb) 1820 1820 + { 1821 1821 + struct dst_entry *dst = skb_dst(skb); 1822 1822 + struct xfrm_state *x = dst->xfrm; 1823 1823 + struct xfrm_iptfs_data *xtfs = x->mode_data; 1824 1824 + struct sk_buff *segs, *nskb; 1825 1825 + u32 pmtu = 0; 1826 1826 + bool ok = true; 1827 1827 + bool was_gso; 1828 1828 + 1829 1829 + /* We have hooked into dst_entry->output which means we have skipped the 1830 1830 + * protocol specific netfilter (see xfrm4_output, xfrm6_output). 1831 1831 + * when our timer runs we will end up calling xfrm_output directly on 1832 1832 + * the encapsulated traffic. 1833 1833 + * 1834 1834 + * For both cases this is the NF_INET_POST_ROUTING hook which allows 1835 1835 + * changing the skb->dst entry which then may not be xfrm based anymore 1836 1836 + * in which case a REROUTED flag is set. and dst_output is called. 1837 1837 + * 1838 1838 + * For IPv6 we are also skipping fragmentation handling for local 1839 1839 + * sockets, which may or may not be good depending on our tunnel DF 1840 1840 + * setting. Normally with fragmentation supported we want to skip this 1841 1841 + * fragmentation. 1842 1842 + */ 1843 1843 + 1844 1844 + if (xtfs->cfg.dont_frag) 1845 1845 + pmtu = iptfs_get_cur_pmtu(x, xtfs, skb); 1846 1846 + 1847 1847 + /* Break apart GSO skbs. If the queue is nearing full then we want the 1848 1848 + * accounting and queuing to be based on the individual packets not on the 1849 1849 + * aggregate GSO buffer. 1850 1850 + */ 1851 1851 + was_gso = skb_is_gso(skb); 1852 1852 + if (!was_gso) { 1853 1853 + segs = skb; 1854 1854 + } else { 1855 1855 + segs = skb_gso_segment(skb, 0); 1856 1856 + if (IS_ERR_OR_NULL(segs)) { 1857 1857 + XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR); 1858 1858 + kfree_skb(skb); 1859 1859 + if (IS_ERR(segs)) 1860 1860 + return PTR_ERR(segs); 1861 1861 + return -EINVAL; 1862 1862 + } 1863 1863 + consume_skb(skb); 1864 1864 + skb = NULL; 1865 1865 + } 1866 1866 + 1867 1867 + /* We can be running on multiple cores and from the network softirq or 1868 1868 + * from user context depending on where the packet is coming from. 1869 1869 + */ 1870 1870 + spin_lock_bh(&x->lock); 1871 1871 + 1872 1872 + skb_list_walk_safe(segs, skb, nskb) { 1873 1873 + skb_mark_not_on_list(skb); 1874 1874 + 1875 1875 + /* Once we drop due to no queue space we continue to drop the 1876 1876 + * rest of the packets from that GRO. 1877 1877 + */ 1878 1878 + if (!ok) { 1879 1879 + nospace: 1880 1880 + trace_iptfs_no_queue_space(skb, xtfs, pmtu, was_gso); 1881 1881 + XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOQSPACE); 1882 1882 + kfree_skb_reason(skb, SKB_DROP_REASON_FULL_RING); 1883 1883 + continue; 1884 1884 + } 1885 1885 + 1886 1886 + /* If the user indicated no iptfs fragmenting check before 1887 1887 + * enqueue. 1888 1888 + */ 1889 1889 + if (xtfs->cfg.dont_frag && iptfs_is_too_big(sk, skb, pmtu)) { 1890 1890 + trace_iptfs_too_big(skb, xtfs, pmtu, was_gso); 1891 1891 + kfree_skb_reason(skb, SKB_DROP_REASON_PKT_TOO_BIG); 1892 1892 + continue; 1893 1893 + } 1894 1894 + 1895 1895 + /* Enqueue to send in tunnel */ 1896 1896 + ok = iptfs_enqueue(xtfs, skb); 1897 1897 + if (!ok) 1898 1898 + goto nospace; 1899 1899 + 1900 1900 + trace_iptfs_enqueue(skb, xtfs, pmtu, was_gso); 1901 1901 + } 1902 1902 + 1903 1903 + /* Start a delay timer if we don't have one yet */ 1904 1904 + if (!hrtimer_is_queued(&xtfs->iptfs_timer)) { 1905 1905 + hrtimer_start(&xtfs->iptfs_timer, xtfs->init_delay_ns, IPTFS_HRTIMER_MODE); 1906 1906 + xtfs->iptfs_settime = ktime_get_raw_fast_ns(); 1907 1907 + trace_iptfs_timer_start(xtfs, xtfs->init_delay_ns); 1908 1908 + } 1909 1909 + 1910 1910 + spin_unlock_bh(&x->lock); 1911 1911 + return 0; 1912 1912 + } 1913 1913 + 1914 1914 + /* -------------------------- */ 1915 1915 + /* Dequeue and send functions */ 1916 1916 + /* -------------------------- */ 1917 1917 + 1918 1918 + static void iptfs_output_prepare_skb(struct sk_buff *skb, u32 blkoff) 1919 1919 + { 1920 1920 + struct ip_iptfs_hdr *h; 1921 1921 + size_t hsz = sizeof(*h); 1922 1922 + 1923 1923 + /* now reset values to be pointing at the rest of the packets */ 1924 1924 + h = skb_push(skb, hsz); 1925 1925 + memset(h, 0, hsz); 1926 1926 + if (blkoff) 1927 1927 + h->block_offset = htons(blkoff); 1928 1928 + 1929 1929 + /* network_header current points at the inner IP packet 1930 1930 + * move it to the iptfs header 1931 1931 + */ 1932 1932 + skb->transport_header = skb->network_header; 1933 1933 + skb->network_header -= hsz; 1934 1934 + 1935 1935 + IPCB(skb)->flags |= IPSKB_XFRM_TUNNEL_SIZE; 1936 1936 + } 1937 1937 + 1938 1938 + /** 1939 1939 + * iptfs_copy_create_frag() - create an inner fragment skb. 1940 1940 + * @st: The source packet data. 1941 1941 + * @offset: offset in @st of the new fragment data. 1942 1942 + * @copy_len: the amount of data to copy from @st. 1943 1943 + * 1944 1944 + * Create a new skb holding a single IPTFS inner packet fragment. @copy_len must 1945 1945 + * not be greater than the max fragment size. 1946 1946 + * 1947 1947 + * Return: the new fragment skb or an ERR_PTR(). 1948 1948 + */ 1949 1949 + static struct sk_buff *iptfs_copy_create_frag(struct skb_seq_state *st, u32 offset, u32 copy_len) 1950 1950 + { 1951 1951 + struct sk_buff *src = st->root_skb; 1952 1952 + struct sk_buff *skb; 1953 1953 + int err; 1954 1954 + 1955 1955 + skb = iptfs_alloc_skb(src, copy_len, true); 1956 1956 + if (!skb) 1957 1957 + return ERR_PTR(-ENOMEM); 1958 1958 + 1959 1959 + /* Now copy `copy_len` data from src */ 1960 1960 + err = skb_copy_seq_read(st, offset, skb_put(skb, copy_len), copy_len); 1961 1961 + if (err) { 1962 1962 + kfree_skb(skb); 1963 1963 + return ERR_PTR(err); 1964 1964 + } 1965 1965 + 1966 1966 + return skb; 1967 1967 + } 1968 1968 + 1969 1969 + /** 1970 1970 + * iptfs_copy_create_frags() - create and send N-1 fragments of a larger skb. 1971 1971 + * @skbp: the source packet skb (IN), skb holding the last fragment in 1972 1972 + * the fragment stream (OUT). 1973 1973 + * @xtfs: IPTFS SA state. 1974 1974 + * @mtu: the max IPTFS fragment size. 1975 1975 + * 1976 1976 + * This function is responsible for fragmenting a larger inner packet into a 1977 1977 + * sequence of IPTFS payload packets. The last fragment is returned rather than 1978 1978 + * being sent so that the caller can append more inner packets (aggregation) if 1979 1979 + * there is room. 1980 1980 + * 1981 1981 + * Return: 0 on success or a negative error code on failure 1982 1982 + */ 1983 1983 + static int iptfs_copy_create_frags(struct sk_buff **skbp, struct xfrm_iptfs_data *xtfs, u32 mtu) 1984 1984 + { 1985 1985 + struct skb_seq_state skbseq; 1986 1986 + struct list_head sublist; 1987 1987 + struct sk_buff *skb = *skbp; 1988 1988 + struct sk_buff *nskb = *skbp; 1989 1989 + u32 copy_len, offset; 1990 1990 + u32 to_copy = skb->len - mtu; 1991 1991 + u32 blkoff = 0; 1992 1992 + int err = 0; 1993 1993 + 1994 1994 + INIT_LIST_HEAD(&sublist); 1995 1995 + 1996 1996 + skb_prepare_seq_read(skb, 0, skb->len, &skbseq); 1997 1997 + 1998 1998 + /* A trimmed `skb` will be sent as the first fragment, later. */ 1999 1999 + offset = mtu; 2000 2000 + to_copy = skb->len - offset; 2001 2001 + while (to_copy) { 2002 2002 + /* Send all but last fragment to allow agg. append */ 2003 2003 + trace_iptfs_first_fragmenting(nskb, mtu, to_copy, NULL); 2004 2004 + list_add_tail(&nskb->list, &sublist); 2005 2005 + 2006 2006 + /* FUTURE: if the packet has an odd/non-aligning length we could 2007 2007 + * send less data in the penultimate fragment so that the last 2008 2008 + * fragment then ends on an aligned boundary. 2009 2009 + */ 2010 2010 + copy_len = min(to_copy, mtu); 2011 2011 + nskb = iptfs_copy_create_frag(&skbseq, offset, copy_len); 2012 2012 + if (IS_ERR(nskb)) { 2013 2013 + XFRM_INC_STATS(xs_net(xtfs->x), LINUX_MIB_XFRMOUTERROR); 2014 2014 + skb_abort_seq_read(&skbseq); 2015 2015 + err = PTR_ERR(nskb); 2016 2016 + nskb = NULL; 2017 2017 + break; 2018 2018 + } 2019 2019 + iptfs_output_prepare_skb(nskb, to_copy); 2020 2020 + offset += copy_len; 2021 2021 + to_copy -= copy_len; 2022 2022 + blkoff = to_copy; 2023 2023 + } 2024 2024 + skb_abort_seq_read(&skbseq); 2025 2025 + 2026 2026 + /* return last fragment that will be unsent (or NULL) */ 2027 2027 + *skbp = nskb; 2028 2028 + if (nskb) 2029 2029 + trace_iptfs_first_final_fragment(nskb, mtu, blkoff, NULL); 2030 2030 + 2031 2031 + /* trim the original skb to MTU */ 2032 2032 + if (!err) 2033 2033 + err = pskb_trim(skb, mtu); 2034 2034 + 2035 2035 + if (err) { 2036 2036 + /* Free all frags. Don't bother sending a partial packet we will 2037 2037 + * never complete. 2038 2038 + */ 2039 2039 + kfree_skb(nskb); 2040 2040 + list_for_each_entry_safe(skb, nskb, &sublist, list) { 2041 2041 + skb_list_del_init(skb); 2042 2042 + kfree_skb(skb); 2043 2043 + } 2044 2044 + return err; 2045 2045 + } 2046 2046 + 2047 2047 + /* prepare the initial fragment with an iptfs header */ 2048 2048 + iptfs_output_prepare_skb(skb, 0); 2049 2049 + 2050 2050 + /* Send all but last fragment, if we fail to send a fragment then free 2051 2051 + * the rest -- no point in sending a packet that can't be reassembled. 2052 2052 + */ 2053 2053 + list_for_each_entry_safe(skb, nskb, &sublist, list) { 2054 2054 + skb_list_del_init(skb); 2055 2055 + if (!err) 2056 2056 + err = xfrm_output(NULL, skb); 2057 2057 + else 2058 2058 + kfree_skb(skb); 2059 2059 + } 2060 2060 + if (err) 2061 2061 + kfree_skb(*skbp); 2062 2062 + return err; 2063 2063 + } 2064 2064 + 2065 2065 + /** 2066 2066 + * iptfs_first_skb() - handle the first dequeued inner packet for output 2067 2067 + * @skbp: the source packet skb (IN), skb holding the last fragment in 2068 2068 + * the fragment stream (OUT). 2069 2069 + * @xtfs: IPTFS SA state. 2070 2070 + * @mtu: the max IPTFS fragment size. 2071 2071 + * 2072 2072 + * This function is responsible for fragmenting a larger inner packet into a 2073 2073 + * sequence of IPTFS payload packets. 2074 2074 + * 2075 2075 + * The last fragment is returned rather than being sent so that the caller can 2076 2076 + * append more inner packets (aggregation) if there is room. 2077 2077 + * 2078 2078 + * Return: 0 on success or a negative error code on failure 2079 2079 + */ 2080 2080 + static int iptfs_first_skb(struct sk_buff **skbp, struct xfrm_iptfs_data *xtfs, u32 mtu) 2081 2081 + { 2082 2082 + struct sk_buff *skb = *skbp; 2083 2083 + int err; 2084 2084 + 2085 2085 + /* Classic ESP skips the don't fragment ICMP error if DF is clear on 2086 2086 + * the inner packet or ignore_df is set. Otherwise it will send an ICMP 2087 2087 + * or local error if the inner packet won't fit it's MTU. 2088 2088 + * 2089 2089 + * With IPTFS we do not care about the inner packet DF bit. If the 2090 2090 + * tunnel is configured to "don't fragment" we error back if things 2091 2091 + * don't fit in our max packet size. Otherwise we iptfs-fragment as 2092 2092 + * normal. 2093 2093 + */ 2094 2094 + 2095 2095 + /* The opportunity for HW offload has ended */ 2096 2096 + if (skb->ip_summed == CHECKSUM_PARTIAL) { 2097 2097 + err = skb_checksum_help(skb); 2098 2098 + if (err) 2099 2099 + return err; 2100 2100 + } 2101 2101 + 2102 2102 + /* We've split gso up before queuing */ 2103 2103 + 2104 2104 + trace_iptfs_first_dequeue(skb, mtu, 0, ip_hdr(skb)); 2105 2105 + 2106 2106 + /* Consider the buffer Tx'd and no longer owned */ 2107 2107 + skb_orphan(skb); 2108 2108 + 2109 2109 + /* Simple case -- it fits. `mtu` accounted for all the overhead 2110 2110 + * including the basic IPTFS header. 2111 2111 + */ 2112 2112 + if (skb->len <= mtu) { 2113 2113 + iptfs_output_prepare_skb(skb, 0); 2114 2114 + return 0; 2115 2115 + } 2116 2116 + 2117 2117 + return iptfs_copy_create_frags(skbp, xtfs, mtu); 2118 2118 + } 2119 2119 + 2120 2120 + static struct sk_buff **iptfs_rehome_fraglist(struct sk_buff **nextp, struct sk_buff *child) 2121 2121 + { 2122 2122 + u32 fllen = 0; 2123 2123 + 2124 2124 + /* It might be possible to account for a frag list in addition to page 2125 2125 + * fragment if it's a valid state to be in. The page fragments size 2126 2126 + * should be kept as data_len so only the frag_list size is removed, 2127 2127 + * this must be done above as well. 2128 2128 + */ 2129 2129 + *nextp = skb_shinfo(child)->frag_list; 2130 2130 + while (*nextp) { 2131 2131 + fllen += (*nextp)->len; 2132 2132 + nextp = &(*nextp)->next; 2133 2133 + } 2134 2134 + skb_frag_list_init(child); 2135 2135 + child->len -= fllen; 2136 2136 + child->data_len -= fllen; 2137 2137 + 2138 2138 + return nextp; 2139 2139 + } 2140 2140 + 2141 2141 + static void iptfs_consume_frags(struct sk_buff *to, struct sk_buff *from) 2142 2142 + { 2143 2143 + struct skb_shared_info *fromi = skb_shinfo(from); 2144 2144 + struct skb_shared_info *toi = skb_shinfo(to); 2145 2145 + unsigned int new_truesize; 2146 2146 + 2147 2147 + /* If we have data in a head page, grab it */ 2148 2148 + if (!skb_headlen(from)) { 2149 2149 + new_truesize = SKB_TRUESIZE(skb_end_offset(from)); 2150 2150 + } else { 2151 2151 + iptfs_skb_head_to_frag(from, &toi->frags[toi->nr_frags]); 2152 2152 + skb_frag_ref(to, toi->nr_frags++); 2153 2153 + new_truesize = SKB_DATA_ALIGN(sizeof(struct sk_buff)); 2154 2154 + } 2155 2155 + 2156 2156 + /* Move any other page fragments rather than copy */ 2157 2157 + memcpy(&toi->frags[toi->nr_frags], fromi->frags, 2158 2158 + sizeof(fromi->frags[0]) * fromi->nr_frags); 2159 2159 + toi->nr_frags += fromi->nr_frags; 2160 2160 + fromi->nr_frags = 0; 2161 2161 + from->data_len = 0; 2162 2162 + from->len = 0; 2163 2163 + to->truesize += from->truesize - new_truesize; 2164 2164 + from->truesize = new_truesize; 2165 2165 + 2166 2166 + /* We are done with this SKB */ 2167 2167 + consume_skb(from); 2168 2168 + } 2169 2169 + 2170 2170 + static void iptfs_output_queued(struct xfrm_state *x, struct sk_buff_head *list) 2171 2171 + { 2172 2172 + struct xfrm_iptfs_data *xtfs = x->mode_data; 2173 2173 + struct sk_buff *skb, *skb2, **nextp; 2174 2174 + struct skb_shared_info *shi, *shi2; 2175 2175 + 2176 2176 + /* If we are fragmenting due to a large inner packet we will output all 2177 2177 + * the outer IPTFS packets required to contain the fragments of the 2178 2178 + * single large inner packet. These outer packets need to be sent 2179 2179 + * consecutively (ESP seq-wise). Since this output function is always 2180 2180 + * running from a timer we do not need a lock to provide this guarantee. 2181 2181 + * We will output our packets consecutively before the timer is allowed 2182 2182 + * to run again on some other CPU. 2183 2183 + */ 2184 2184 + 2185 2185 + while ((skb = __skb_dequeue(list))) { 2186 2186 + u32 mtu = iptfs_get_cur_pmtu(x, xtfs, skb); 2187 2187 + bool share_ok = true; 2188 2188 + int remaining; 2189 2189 + 2190 2190 + /* protocol comes to us cleared sometimes */ 2191 2191 + skb->protocol = x->outer_mode.family == AF_INET ? htons(ETH_P_IP) : 2192 2192 + htons(ETH_P_IPV6); 2193 2193 + 2194 2194 + if (skb->len > mtu && xtfs->cfg.dont_frag) { 2195 2195 + /* We handle this case before enqueueing so we are only 2196 2196 + * here b/c MTU changed after we enqueued before we 2197 2197 + * dequeued, just drop these. 2198 2198 + */ 2199 2199 + XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTERROR); 2200 2200 + 2201 2201 + trace_iptfs_first_toobig(skb, mtu, 0, ip_hdr(skb)); 2202 2202 + kfree_skb_reason(skb, SKB_DROP_REASON_PKT_TOO_BIG); 2203 2203 + continue; 2204 2204 + } 2205 2205 + 2206 2206 + /* Convert first inner packet into an outer IPTFS packet, 2207 2207 + * dealing with any fragmentation into multiple outer packets 2208 2208 + * if necessary. 2209 2209 + */ 2210 2210 + if (iptfs_first_skb(&skb, xtfs, mtu)) 2211 2211 + continue; 2212 2212 + 2213 2213 + /* If fragmentation was required the returned skb is the last 2214 2214 + * IPTFS fragment in the chain, and it's IPTFS header blkoff has 2215 2215 + * been set just past the end of the fragment data. 2216 2216 + * 2217 2217 + * In either case the space remaining to send more inner packet 2218 2218 + * data is `mtu` - (skb->len - sizeof iptfs header). This is b/c 2219 2219 + * the `mtu` value has the basic IPTFS header len accounted for, 2220 2220 + * and we added that header to the skb so it is a part of 2221 2221 + * skb->len, thus we subtract it from the skb length. 2222 2222 + */ 2223 2223 + remaining = mtu - (skb->len - sizeof(struct ip_iptfs_hdr)); 2224 2224 + 2225 2225 + /* Re-home (un-nest) nested fragment lists. We need to do this 2226 2226 + * b/c we will simply be appending any following aggregated 2227 2227 + * inner packets using the frag list. 2228 2228 + */ 2229 2229 + shi = skb_shinfo(skb); 2230 2230 + nextp = &shi->frag_list; 2231 2231 + while (*nextp) { 2232 2232 + if (skb_has_frag_list(*nextp)) 2233 2233 + nextp = iptfs_rehome_fraglist(&(*nextp)->next, *nextp); 2234 2234 + else 2235 2235 + nextp = &(*nextp)->next; 2236 2236 + } 2237 2237 + 2238 2238 + if (shi->frag_list || skb_cloned(skb) || skb_shared(skb)) 2239 2239 + share_ok = false; 2240 2240 + 2241 2241 + /* See if we have enough space to simply append. 2242 2242 + * 2243 2243 + * NOTE: Maybe do not append if we will be mis-aligned, 2244 2244 + * SW-based endpoints will probably have to copy in this 2245 2245 + * case. 2246 2246 + */ 2247 2247 + while ((skb2 = skb_peek(list))) { 2248 2248 + trace_iptfs_ingress_nth_peek(skb2, remaining); 2249 2249 + if (skb2->len > remaining) 2250 2250 + break; 2251 2251 + 2252 2252 + __skb_unlink(skb2, list); 2253 2253 + 2254 2254 + /* Consider the buffer Tx'd and no longer owned */ 2255 2255 + skb_orphan(skb); 2256 2256 + 2257 2257 + /* If we don't have a cksum in the packet we need to add 2258 2258 + * one before encapsulation. 2259 2259 + */ 2260 2260 + if (skb2->ip_summed == CHECKSUM_PARTIAL) { 2261 2261 + if (skb_checksum_help(skb2)) { 2262 2262 + XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTERROR); 2263 2263 + kfree_skb(skb2); 2264 2264 + continue; 2265 2265 + } 2266 2266 + } 2267 2267 + 2268 2268 + /* skb->pp_recycle is passed to __skb_flag_unref for all 2269 2269 + * frag pages so we can only share pages with skb's who 2270 2270 + * match ourselves. 2271 2271 + */ 2272 2272 + shi2 = skb_shinfo(skb2); 2273 2273 + if (share_ok && 2274 2274 + (shi2->frag_list || 2275 2275 + (!skb2->head_frag && skb_headlen(skb)) || 2276 2276 + skb->pp_recycle != skb2->pp_recycle || 2277 2277 + skb_zcopy(skb2) || 2278 2278 + (shi->nr_frags + shi2->nr_frags + 1 > MAX_SKB_FRAGS))) 2279 2279 + share_ok = false; 2280 2280 + 2281 2281 + /* Do accounting */ 2282 2282 + skb->data_len += skb2->len; 2283 2283 + skb->len += skb2->len; 2284 2284 + remaining -= skb2->len; 2285 2285 + 2286 2286 + trace_iptfs_ingress_nth_add(skb2, share_ok); 2287 2287 + 2288 2288 + if (share_ok) { 2289 2289 + iptfs_consume_frags(skb, skb2); 2290 2290 + } else { 2291 2291 + /* Append to the frag_list */ 2292 2292 + *nextp = skb2; 2293 2293 + nextp = &skb2->next; 2294 2294 + if (skb_has_frag_list(skb2)) 2295 2295 + nextp = iptfs_rehome_fraglist(nextp, 2296 2296 + skb2); 2297 2297 + skb->truesize += skb2->truesize; 2298 2298 + } 2299 2299 + } 2300 2300 + 2301 2301 + xfrm_output(NULL, skb); 2302 2302 + } 2303 2303 + } 2304 2304 + 2305 2305 + static enum hrtimer_restart iptfs_delay_timer(struct hrtimer *me) 2306 2306 + { 2307 2307 + struct sk_buff_head list; 2308 2308 + struct xfrm_iptfs_data *xtfs; 2309 2309 + struct xfrm_state *x; 2310 2310 + time64_t settime; 2311 2311 + 2312 2312 + xtfs = container_of(me, typeof(*xtfs), iptfs_timer); 2313 2313 + x = xtfs->x; 2314 2314 + 2315 2315 + /* Process all the queued packets 2316 2316 + * 2317 2317 + * softirq execution order: timer > tasklet > hrtimer 2318 2318 + * 2319 2319 + * Network rx will have run before us giving one last chance to queue 2320 2320 + * ingress packets for us to process and transmit. 2321 2321 + */ 2322 2322 + 2323 2323 + spin_lock(&x->lock); 2324 2324 + __skb_queue_head_init(&list); 2325 2325 + skb_queue_splice_init(&xtfs->queue, &list); 2326 2326 + xtfs->queue_size = 0; 2327 2327 + settime = xtfs->iptfs_settime; 2328 2328 + spin_unlock(&x->lock); 2329 2329 + 2330 2330 + /* After the above unlock, packets can begin queuing again, and the 2331 2331 + * timer can be set again, from another CPU either in softirq or user 2332 2332 + * context (not from this one since we are running at softirq level 2333 2333 + * already). 2334 2334 + */ 2335 2335 + 2336 2336 + trace_iptfs_timer_expire(xtfs, (unsigned long long)(ktime_get_raw_fast_ns() - settime)); 2337 2337 + 2338 2338 + iptfs_output_queued(x, &list); 2339 2339 + 2340 2340 + return HRTIMER_NORESTART; 2341 2341 + } 2342 2342 + 2343 2343 + /** 2344 2344 + * iptfs_encap_add_ipv4() - add outer encaps 2345 2345 + * @x: xfrm state 2346 2346 + * @skb: the packet 2347 2347 + * 2348 2348 + * This was originally taken from xfrm4_tunnel_encap_add. The reason for the 2349 2349 + * copy is that IP-TFS/AGGFRAG can have different functionality for how to set 2350 2350 + * the TOS/DSCP bits. Sets the protocol to a different value and doesn't do 2351 2351 + * anything with inner headers as they aren't pointing into a normal IP 2352 2352 + * singleton inner packet. 2353 2353 + * 2354 2354 + * Return: 0 on success or a negative error code on failure 2355 2355 + */ 2356 2356 + static int iptfs_encap_add_ipv4(struct xfrm_state *x, struct sk_buff *skb) 2357 2357 + { 2358 2358 + struct dst_entry *dst = skb_dst(skb); 2359 2359 + struct iphdr *top_iph; 2360 2360 + 2361 2361 + skb_reset_inner_network_header(skb); 2362 2362 + skb_reset_inner_transport_header(skb); 2363 2363 + 2364 2364 + skb_set_network_header(skb, -(x->props.header_len - x->props.enc_hdr_len)); 2365 2365 + skb->mac_header = skb->network_header + offsetof(struct iphdr, protocol); 2366 2366 + skb->transport_header = skb->network_header + sizeof(*top_iph); 2367 2367 + 2368 2368 + top_iph = ip_hdr(skb); 2369 2369 + top_iph->ihl = 5; 2370 2370 + top_iph->version = 4; 2371 2371 + top_iph->protocol = IPPROTO_AGGFRAG; 2372 2372 + 2373 2373 + /* As we have 0, fractional, 1 or N inner packets there's no obviously 2374 2374 + * correct DSCP mapping to inherit. ECN should be cleared per RFC9347 2375 2375 + * 3.1. 2376 2376 + */ 2377 2377 + top_iph->tos = 0; 2378 2378 + 2379 2379 + top_iph->frag_off = htons(IP_DF); 2380 2380 + top_iph->ttl = ip4_dst_hoplimit(xfrm_dst_child(dst)); 2381 2381 + top_iph->saddr = x->props.saddr.a4; 2382 2382 + top_iph->daddr = x->id.daddr.a4; 2383 2383 + ip_select_ident(dev_net(dst->dev), skb, NULL); 2384 2384 + 2385 2385 + return 0; 2386 2386 + } 2387 2387 + 2388 2388 + #if IS_ENABLED(CONFIG_IPV6) 2389 2389 + /** 2390 2390 + * iptfs_encap_add_ipv6() - add outer encaps 2391 2391 + * @x: xfrm state 2392 2392 + * @skb: the packet 2393 2393 + * 2394 2394 + * This was originally taken from xfrm6_tunnel_encap_add. The reason for the 2395 2395 + * copy is that IP-TFS/AGGFRAG can have different functionality for how to set 2396 2396 + * the flow label and TOS/DSCP bits. It also sets the protocol to a different 2397 2397 + * value and doesn't do anything with inner headers as they aren't pointing into 2398 2398 + * a normal IP singleton inner packet. 2399 2399 + * 2400 2400 + * Return: 0 on success or a negative error code on failure 2401 2401 + */ 2402 2402 + static int iptfs_encap_add_ipv6(struct xfrm_state *x, struct sk_buff *skb) 2403 2403 + { 2404 2404 + struct dst_entry *dst = skb_dst(skb); 2405 2405 + struct ipv6hdr *top_iph; 2406 2406 + int dsfield; 2407 2407 + 2408 2408 + skb_reset_inner_network_header(skb); 2409 2409 + skb_reset_inner_transport_header(skb); 2410 2410 + 2411 2411 + skb_set_network_header(skb, -x->props.header_len + x->props.enc_hdr_len); 2412 2412 + skb->mac_header = skb->network_header + offsetof(struct ipv6hdr, nexthdr); 2413 2413 + skb->transport_header = skb->network_header + sizeof(*top_iph); 2414 2414 + 2415 2415 + top_iph = ipv6_hdr(skb); 2416 2416 + top_iph->version = 6; 2417 2417 + top_iph->priority = 0; 2418 2418 + memset(top_iph->flow_lbl, 0, sizeof(top_iph->flow_lbl)); 2419 2419 + top_iph->nexthdr = IPPROTO_AGGFRAG; 2420 2420 + 2421 2421 + /* As we have 0, fractional, 1 or N inner packets there's no obviously 2422 2422 + * correct DSCP mapping to inherit. ECN should be cleared per RFC9347 2423 2423 + * 3.1. 2424 2424 + */ 2425 2425 + dsfield = 0; 2426 2426 + ipv6_change_dsfield(top_iph, 0, dsfield); 2427 2427 + 2428 2428 + top_iph->hop_limit = ip6_dst_hoplimit(xfrm_dst_child(dst)); 2429 2429 + top_iph->saddr = *(struct in6_addr *)&x->props.saddr; 2430 2430 + top_iph->daddr = *(struct in6_addr *)&x->id.daddr; 2431 2431 + 2432 2432 + return 0; 2433 2433 + } 2434 2434 + #endif 2435 2435 + 2436 2436 + /** 2437 2437 + * iptfs_prepare_output() - prepare the skb for output 2438 2438 + * @x: xfrm state 2439 2439 + * @skb: the packet 2440 2440 + * 2441 2441 + * Return: Error value, if 0 then skb values should be as follows: 2442 2442 + * - transport_header should point at ESP header 2443 2443 + * - network_header should point at Outer IP header 2444 2444 + * - mac_header should point at protocol/nexthdr of the outer IP 2445 2445 + */ 2446 2446 + static int iptfs_prepare_output(struct xfrm_state *x, struct sk_buff *skb) 2447 2447 + { 2448 2448 + if (x->outer_mode.family == AF_INET) 2449 2449 + return iptfs_encap_add_ipv4(x, skb); 2450 2450 + if (x->outer_mode.family == AF_INET6) { 2451 2451 + #if IS_ENABLED(CONFIG_IPV6) 2452 2452 + return iptfs_encap_add_ipv6(x, skb); 2453 2453 + #else 2454 2454 + return -EAFNOSUPPORT; 2455 2455 + #endif 2456 2456 + } 2457 2457 + return -EOPNOTSUPP; 2458 2458 + } 2459 2459 + 2460 2460 + /* ========================== */ 2461 2461 + /* State Management Functions */ 2462 2462 + /* ========================== */ 2463 2463 + 2464 2464 + /** 2465 2465 + * __iptfs_get_inner_mtu() - return inner MTU with no fragmentation. 2466 2466 + * @x: xfrm state. 2467 2467 + * @outer_mtu: the outer mtu 2468 2468 + * 2469 2469 + * Return: Correct MTU taking in to account the encap overhead. 2470 2470 + */ 2471 2471 + static u32 __iptfs_get_inner_mtu(struct xfrm_state *x, int outer_mtu) 2472 2472 + { 2473 2473 + struct crypto_aead *aead; 2474 2474 + u32 blksize; 2475 2475 + 2476 2476 + aead = x->data; 2477 2477 + blksize = ALIGN(crypto_aead_blocksize(aead), 4); 2478 2478 + return ((outer_mtu - x->props.header_len - crypto_aead_authsize(aead)) & 2479 2479 + ~(blksize - 1)) - 2; 2480 2480 + } 2481 2481 + 2482 2482 + /** 2483 2483 + * iptfs_get_inner_mtu() - return the inner MTU for an IPTFS xfrm. 2484 2484 + * @x: xfrm state. 2485 2485 + * @outer_mtu: Outer MTU for the encapsulated packet. 2486 2486 + * 2487 2487 + * Return: Correct MTU taking in to account the encap overhead. 2488 2488 + */ 2489 2489 + static u32 iptfs_get_inner_mtu(struct xfrm_state *x, int outer_mtu) 2490 2490 + { 2491 2491 + struct xfrm_iptfs_data *xtfs = x->mode_data; 2492 2492 + 2493 2493 + /* If not dont-frag we have no MTU */ 2494 2494 + if (!xtfs->cfg.dont_frag) 2495 2495 + return x->outer_mode.family == AF_INET ? IP_MAX_MTU : IP6_MAX_MTU; 2496 2496 + return __iptfs_get_inner_mtu(x, outer_mtu); 2497 2497 + } 2498 2498 + 2499 2499 + /** 2500 2500 + * iptfs_user_init() - initialize the SA with IPTFS options from netlink. 2501 2501 + * @net: the net data 2502 2502 + * @x: xfrm state 2503 2503 + * @attrs: netlink attributes 2504 2504 + * @extack: extack return data 2505 2505 + * 2506 2506 + * Return: 0 on success or a negative error code on failure 2507 2507 + */ 2508 2508 + static int iptfs_user_init(struct net *net, struct xfrm_state *x, 2509 2509 + struct nlattr **attrs, 2510 2510 + struct netlink_ext_ack *extack) 2511 2511 + { 2512 2512 + struct xfrm_iptfs_data *xtfs = x->mode_data; 2513 2513 + struct xfrm_iptfs_config *xc; 2514 2514 + u64 q; 2515 2515 + 2516 2516 + xc = &xtfs->cfg; 2517 2517 + xc->max_queue_size = IPTFS_DEFAULT_MAX_QUEUE_SIZE; 2518 2518 + xc->reorder_win_size = IPTFS_DEFAULT_REORDER_WINDOW; 2519 2519 + xtfs->drop_time_ns = IPTFS_DEFAULT_DROP_TIME_USECS * NSECS_IN_USEC; 2520 2520 + xtfs->init_delay_ns = IPTFS_DEFAULT_INIT_DELAY_USECS * NSECS_IN_USEC; 2521 2521 + 2522 2522 + if (attrs[XFRMA_IPTFS_DONT_FRAG]) 2523 2523 + xc->dont_frag = true; 2524 2524 + if (attrs[XFRMA_IPTFS_REORDER_WINDOW]) 2525 2525 + xc->reorder_win_size = 2526 2526 + nla_get_u16(attrs[XFRMA_IPTFS_REORDER_WINDOW]); 2527 2527 + /* saved array is for saving 1..N seq nums from wantseq */ 2528 2528 + if (xc->reorder_win_size) { 2529 2529 + xtfs->w_saved = kcalloc(xc->reorder_win_size, 2530 2530 + sizeof(*xtfs->w_saved), GFP_KERNEL); 2531 2531 + if (!xtfs->w_saved) { 2532 2532 + NL_SET_ERR_MSG(extack, "Cannot alloc reorder window"); 2533 2533 + return -ENOMEM; 2534 2534 + } 2535 2535 + } 2536 2536 + if (attrs[XFRMA_IPTFS_PKT_SIZE]) { 2537 2537 + xc->pkt_size = nla_get_u32(attrs[XFRMA_IPTFS_PKT_SIZE]); 2538 2538 + if (!xc->pkt_size) { 2539 2539 + xtfs->payload_mtu = 0; 2540 2540 + } else if (xc->pkt_size > x->props.header_len) { 2541 2541 + xtfs->payload_mtu = xc->pkt_size - x->props.header_len; 2542 2542 + } else { 2543 2543 + NL_SET_ERR_MSG(extack, 2544 2544 + "Packet size must be 0 or greater than IPTFS/ESP header length"); 2545 2545 + return -EINVAL; 2546 2546 + } 2547 2547 + } 2548 2548 + if (attrs[XFRMA_IPTFS_MAX_QSIZE]) 2549 2549 + xc->max_queue_size = nla_get_u32(attrs[XFRMA_IPTFS_MAX_QSIZE]); 2550 2550 + if (attrs[XFRMA_IPTFS_DROP_TIME]) 2551 2551 + xtfs->drop_time_ns = 2552 2552 + (u64)nla_get_u32(attrs[XFRMA_IPTFS_DROP_TIME]) * 2553 2553 + NSECS_IN_USEC; 2554 2554 + if (attrs[XFRMA_IPTFS_INIT_DELAY]) 2555 2555 + xtfs->init_delay_ns = 2556 2556 + (u64)nla_get_u32(attrs[XFRMA_IPTFS_INIT_DELAY]) * NSECS_IN_USEC; 2557 2557 + 2558 2558 + q = (u64)xc->max_queue_size * 95; 2559 2559 + do_div(q, 100); 2560 2560 + xtfs->ecn_queue_size = (u32)q; 2561 2561 + 2562 2562 + return 0; 2563 2563 + } 2564 2564 + 2565 2565 + static unsigned int iptfs_sa_len(const struct xfrm_state *x) 2566 2566 + { 2567 2567 + struct xfrm_iptfs_data *xtfs = x->mode_data; 2568 2568 + struct xfrm_iptfs_config *xc = &xtfs->cfg; 2569 2569 + unsigned int l = 0; 2570 2570 + 2571 2571 + if (x->dir == XFRM_SA_DIR_IN) { 2572 2572 + l += nla_total_size(sizeof(u32)); /* drop time usec */ 2573 2573 + l += nla_total_size(sizeof(xc->reorder_win_size)); 2574 2574 + } else { 2575 2575 + if (xc->dont_frag) 2576 2576 + l += nla_total_size(0); /* dont-frag flag */ 2577 2577 + l += nla_total_size(sizeof(u32)); /* init delay usec */ 2578 2578 + l += nla_total_size(sizeof(xc->max_queue_size)); 2579 2579 + l += nla_total_size(sizeof(xc->pkt_size)); 2580 2580 + } 2581 2581 + 2582 2582 + return l; 2583 2583 + } 2584 2584 + 2585 2585 + static int iptfs_copy_to_user(struct xfrm_state *x, struct sk_buff *skb) 2586 2586 + { 2587 2587 + struct xfrm_iptfs_data *xtfs = x->mode_data; 2588 2588 + struct xfrm_iptfs_config *xc = &xtfs->cfg; 2589 2589 + int ret = 0; 2590 2590 + u64 q; 2591 2591 + 2592 2592 + if (x->dir == XFRM_SA_DIR_IN) { 2593 2593 + q = xtfs->drop_time_ns; 2594 2594 + do_div(q, NSECS_IN_USEC); 2595 2595 + ret = nla_put_u32(skb, XFRMA_IPTFS_DROP_TIME, q); 2596 2596 + if (ret) 2597 2597 + return ret; 2598 2598 + 2599 2599 + ret = nla_put_u16(skb, XFRMA_IPTFS_REORDER_WINDOW, 2600 2600 + xc->reorder_win_size); 2601 2601 + } else { 2602 2602 + if (xc->dont_frag) { 2603 2603 + ret = nla_put_flag(skb, XFRMA_IPTFS_DONT_FRAG); 2604 2604 + if (ret) 2605 2605 + return ret; 2606 2606 + } 2607 2607 + 2608 2608 + q = xtfs->init_delay_ns; 2609 2609 + do_div(q, NSECS_IN_USEC); 2610 2610 + ret = nla_put_u32(skb, XFRMA_IPTFS_INIT_DELAY, q); 2611 2611 + if (ret) 2612 2612 + return ret; 2613 2613 + 2614 2614 + ret = nla_put_u32(skb, XFRMA_IPTFS_MAX_QSIZE, xc->max_queue_size); 2615 2615 + if (ret) 2616 2616 + return ret; 2617 2617 + 2618 2618 + ret = nla_put_u32(skb, XFRMA_IPTFS_PKT_SIZE, xc->pkt_size); 2619 2619 + } 2620 2620 + 2621 2621 + return ret; 2622 2622 + } 2623 2623 + 2624 2624 + static void __iptfs_init_state(struct xfrm_state *x, 2625 2625 + struct xfrm_iptfs_data *xtfs) 2626 2626 + { 2627 2627 + __skb_queue_head_init(&xtfs->queue); 2628 2628 + hrtimer_init(&xtfs->iptfs_timer, CLOCK_MONOTONIC, IPTFS_HRTIMER_MODE); 2629 2629 + xtfs->iptfs_timer.function = iptfs_delay_timer; 2630 2630 + 2631 2631 + spin_lock_init(&xtfs->drop_lock); 2632 2632 + hrtimer_init(&xtfs->drop_timer, CLOCK_MONOTONIC, IPTFS_HRTIMER_MODE); 2633 2633 + xtfs->drop_timer.function = iptfs_drop_timer; 2634 2634 + 2635 2635 + /* Modify type (esp) adjustment values */ 2636 2636 + 2637 2637 + if (x->props.family == AF_INET) 2638 2638 + x->props.header_len += sizeof(struct iphdr) + sizeof(struct ip_iptfs_hdr); 2639 2639 + else if (x->props.family == AF_INET6) 2640 2640 + x->props.header_len += sizeof(struct ipv6hdr) + sizeof(struct ip_iptfs_hdr); 2641 2641 + x->props.enc_hdr_len = sizeof(struct ip_iptfs_hdr); 2642 2642 + 2643 2643 + /* Always keep a module reference when x->mode_data is set */ 2644 2644 + __module_get(x->mode_cbs->owner); 2645 2645 + 2646 2646 + x->mode_data = xtfs; 2647 2647 + xtfs->x = x; 2648 2648 + } 2649 2649 + 2650 2650 + static int iptfs_clone_state(struct xfrm_state *x, struct xfrm_state *orig) 2651 2651 + { 2652 2652 + struct xfrm_iptfs_data *xtfs; 2653 2653 + 2654 2654 + xtfs = kmemdup(orig->mode_data, sizeof(*xtfs), GFP_KERNEL); 2655 2655 + if (!xtfs) 2656 2656 + return -ENOMEM; 2657 2657 + 2658 2658 + x->mode_data = xtfs; 2659 2659 + xtfs->x = x; 2660 2660 + 2661 2661 + xtfs->ra_newskb = NULL; 2662 2662 + if (xtfs->cfg.reorder_win_size) { 2663 2663 + xtfs->w_saved = kcalloc(xtfs->cfg.reorder_win_size, 2664 2664 + sizeof(*xtfs->w_saved), GFP_KERNEL); 2665 2665 + if (!xtfs->w_saved) { 2666 2666 + kfree_sensitive(xtfs); 2667 2667 + return -ENOMEM; 2668 2668 + } 2669 2669 + } 2670 2670 + 2671 2671 + return 0; 2672 2672 + } 2673 2673 + 2674 2674 + static int iptfs_init_state(struct xfrm_state *x) 2675 2675 + { 2676 2676 + struct xfrm_iptfs_data *xtfs; 2677 2677 + 2678 2678 + if (x->mode_data) { 2679 2679 + /* We have arrived here from xfrm_state_clone() */ 2680 2680 + xtfs = x->mode_data; 2681 2681 + } else { 2682 2682 + xtfs = kzalloc(sizeof(*xtfs), GFP_KERNEL); 2683 2683 + if (!xtfs) 2684 2684 + return -ENOMEM; 2685 2685 + } 2686 2686 + 2687 2687 + __iptfs_init_state(x, xtfs); 2688 2688 + 2689 2689 + return 0; 2690 2690 + } 2691 2691 + 2692 2692 + static void iptfs_destroy_state(struct xfrm_state *x) 2693 2693 + { 2694 2694 + struct xfrm_iptfs_data *xtfs = x->mode_data; 2695 2695 + struct sk_buff_head list; 2696 2696 + struct skb_wseq *s, *se; 2697 2697 + struct sk_buff *skb; 2698 2698 + 2699 2699 + if (!xtfs) 2700 2700 + return; 2701 2701 + 2702 2702 + spin_lock_bh(&xtfs->x->lock); 2703 2703 + hrtimer_cancel(&xtfs->iptfs_timer); 2704 2704 + __skb_queue_head_init(&list); 2705 2705 + skb_queue_splice_init(&xtfs->queue, &list); 2706 2706 + spin_unlock_bh(&xtfs->x->lock); 2707 2707 + 2708 2708 + while ((skb = __skb_dequeue(&list))) 2709 2709 + kfree_skb(skb); 2710 2710 + 2711 2711 + spin_lock_bh(&xtfs->drop_lock); 2712 2712 + hrtimer_cancel(&xtfs->drop_timer); 2713 2713 + spin_unlock_bh(&xtfs->drop_lock); 2714 2714 + 2715 2715 + if (xtfs->ra_newskb) 2716 2716 + kfree_skb(xtfs->ra_newskb); 2717 2717 + 2718 2718 + for (s = xtfs->w_saved, se = s + xtfs->w_savedlen; s < se; s++) { 2719 2719 + if (s->skb) 2720 2720 + kfree_skb(s->skb); 2721 2721 + } 2722 2722 + 2723 2723 + kfree_sensitive(xtfs->w_saved); 2724 2724 + kfree_sensitive(xtfs); 2725 2725 + 2726 2726 + module_put(x->mode_cbs->owner); 2727 2727 + } 2728 2728 + 2729 2729 + static const struct xfrm_mode_cbs iptfs_mode_cbs = { 2730 2730 + .owner = THIS_MODULE, 2731 2731 + .init_state = iptfs_init_state, 2732 2732 + .clone_state = iptfs_clone_state, 2733 2733 + .destroy_state = iptfs_destroy_state, 2734 2734 + .user_init = iptfs_user_init, 2735 2735 + .copy_to_user = iptfs_copy_to_user, 2736 2736 + .sa_len = iptfs_sa_len, 2737 2737 + .get_inner_mtu = iptfs_get_inner_mtu, 2738 2738 + .input = iptfs_input, 2739 2739 + .output = iptfs_output_collect, 2740 2740 + .prepare_output = iptfs_prepare_output, 2741 2741 + }; 2742 2742 + 2743 2743 + static int __init xfrm_iptfs_init(void) 2744 2744 + { 2745 2745 + int err; 2746 2746 + 2747 2747 + pr_info("xfrm_iptfs: IPsec IP-TFS tunnel mode module\n"); 2748 2748 + 2749 2749 + err = xfrm_register_mode_cbs(XFRM_MODE_IPTFS, &iptfs_mode_cbs); 2750 2750 + if (err < 0) 2751 2751 + pr_info("%s: can't register IP-TFS\n", __func__); 2752 2752 + 2753 2753 + return err; 2754 2754 + } 2755 2755 + 2756 2756 + static void __exit xfrm_iptfs_fini(void) 2757 2757 + { 2758 2758 + xfrm_unregister_mode_cbs(XFRM_MODE_IPTFS); 2759 2759 + } 2760 2760 + 2761 2761 + module_init(xfrm_iptfs_init); 2762 2762 + module_exit(xfrm_iptfs_fini); 2763 2763 + MODULE_LICENSE("GPL"); 2764 2764 + MODULE_DESCRIPTION("IP-TFS support for xfrm ipsec tunnels");

+6

net/xfrm/xfrm_output.c

reviewed

··· 472 472 WARN_ON_ONCE(1); 473 473 break; 474 474 default: 475 475 + if (x->mode_cbs && x->mode_cbs->prepare_output) 476 476 + return x->mode_cbs->prepare_output(x, skb); 475 477 WARN_ON_ONCE(1); 476 478 break; 477 479 } ··· 675 673 break; 676 674 } 677 675 676 676 + return; 677 677 + } 678 678 + if (x->outer_mode.encap == XFRM_MODE_IPTFS) { 679 679 + xo->inner_ipproto = IPPROTO_AGGFRAG; 678 680 return; 679 681 } 680 682

+17 -9

net/xfrm/xfrm_policy.c

reviewed

··· 2497 2497 struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i]; 2498 2498 2499 2499 if (tmpl->mode == XFRM_MODE_TUNNEL || 2500 2500 + tmpl->mode == XFRM_MODE_IPTFS || 2500 2501 tmpl->mode == XFRM_MODE_BEET) { 2501 2502 remote = &tmpl->id.daddr; 2502 2503 local = &tmpl->saddr; ··· 2749 2748 2750 2749 dst1->input = dst_discard; 2751 2750 2752 2752 - rcu_read_lock(); 2753 2753 - afinfo = xfrm_state_afinfo_get_rcu(inner_mode->family); 2754 2754 - if (likely(afinfo)) 2755 2755 - dst1->output = afinfo->output; 2756 2756 - else 2757 2757 - dst1->output = dst_discard_out; 2758 2758 - rcu_read_unlock(); 2751 2751 + if (xfrm[i]->mode_cbs && xfrm[i]->mode_cbs->output) { 2752 2752 + dst1->output = xfrm[i]->mode_cbs->output; 2753 2753 + } else { 2754 2754 + rcu_read_lock(); 2755 2755 + afinfo = xfrm_state_afinfo_get_rcu(inner_mode->family); 2756 2756 + if (likely(afinfo)) 2757 2757 + dst1->output = afinfo->output; 2758 2758 + else 2759 2759 + dst1->output = dst_discard_out; 2760 2760 + rcu_read_unlock(); 2761 2761 + } 2759 2762 2760 2763 xdst_prev = xdst; 2761 2764 ··· 3295 3290 ok: 3296 3291 xfrm_pols_put(pols, drop_pols); 3297 3292 if (dst && dst->xfrm && 3298 3298 - dst->xfrm->props.mode == XFRM_MODE_TUNNEL) 3293 3293 + (dst->xfrm->props.mode == XFRM_MODE_TUNNEL || 3294 3294 + dst->xfrm->props.mode == XFRM_MODE_IPTFS)) 3299 3295 dst->flags |= DST_XFRM_TUNNEL; 3300 3296 return dst; 3301 3297 ··· 4525 4519 switch (t->mode) { 4526 4520 case XFRM_MODE_TUNNEL: 4527 4521 case XFRM_MODE_BEET: 4522 4522 + case XFRM_MODE_IPTFS: 4528 4523 if (xfrm_addr_equal(&t->id.daddr, &m->old_daddr, 4529 4524 m->old_family) && 4530 4525 xfrm_addr_equal(&t->saddr, &m->old_saddr, ··· 4568 4561 continue; 4569 4562 n++; 4570 4563 if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL && 4571 4571 - pol->xfrm_vec[i].mode != XFRM_MODE_BEET) 4564 4564 + pol->xfrm_vec[i].mode != XFRM_MODE_BEET && 4565 4565 + pol->xfrm_vec[i].mode != XFRM_MODE_IPTFS) 4572 4566 continue; 4573 4567 /* update endpoints */ 4574 4568 memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr,

+2

net/xfrm/xfrm_proc.c

reviewed

··· 43 43 SNMP_MIB_ITEM("XfrmAcquireError", LINUX_MIB_XFRMACQUIREERROR), 44 44 SNMP_MIB_ITEM("XfrmOutStateDirError", LINUX_MIB_XFRMOUTSTATEDIRERROR), 45 45 SNMP_MIB_ITEM("XfrmInStateDirError", LINUX_MIB_XFRMINSTATEDIRERROR), 46 46 + SNMP_MIB_ITEM("XfrmInIptfsError", LINUX_MIB_XFRMINIPTFSERROR), 47 47 + SNMP_MIB_ITEM("XfrmOutNoQueueSpace", LINUX_MIB_XFRMOUTNOQSPACE), 46 48 SNMP_MIB_SENTINEL 47 49 }; 48 50

+1

net/xfrm/xfrm_replay.c

reviewed

··· 729 729 } 730 730 731 731 replay_esn->oseq = oseq; 732 732 + xfrm_dev_state_advance_esn(x); 732 733 733 734 if (xfrm_aevent_is_on(net)) 734 735 xfrm_replay_notify(x, XFRM_REPLAY_UPDATE);

+84

net/xfrm/xfrm_state.c

reviewed

··· 467 467 .flags = XFRM_MODE_FLAG_TUNNEL, 468 468 .family = AF_INET, 469 469 }, 470 470 + [XFRM_MODE_IPTFS] = { 471 471 + .encap = XFRM_MODE_IPTFS, 472 472 + .flags = XFRM_MODE_FLAG_TUNNEL, 473 473 + .family = AF_INET, 474 474 + }, 470 475 }; 471 476 472 477 static const struct xfrm_mode xfrm6_mode_map[XFRM_MODE_MAX] = { ··· 490 485 }, 491 486 [XFRM_MODE_TUNNEL] = { 492 487 .encap = XFRM_MODE_TUNNEL, 488 488 + .flags = XFRM_MODE_FLAG_TUNNEL, 489 489 + .family = AF_INET6, 490 490 + }, 491 491 + [XFRM_MODE_IPTFS] = { 492 492 + .encap = XFRM_MODE_IPTFS, 493 493 .flags = XFRM_MODE_FLAG_TUNNEL, 494 494 .family = AF_INET6, 495 495 }, ··· 525 515 return NULL; 526 516 } 527 517 518 518 + static const struct xfrm_mode_cbs __rcu *xfrm_mode_cbs_map[XFRM_MODE_MAX]; 519 519 + static DEFINE_SPINLOCK(xfrm_mode_cbs_map_lock); 520 520 + 521 521 + int xfrm_register_mode_cbs(u8 mode, const struct xfrm_mode_cbs *mode_cbs) 522 522 + { 523 523 + if (mode >= XFRM_MODE_MAX) 524 524 + return -EINVAL; 525 525 + 526 526 + spin_lock_bh(&xfrm_mode_cbs_map_lock); 527 527 + rcu_assign_pointer(xfrm_mode_cbs_map[mode], mode_cbs); 528 528 + spin_unlock_bh(&xfrm_mode_cbs_map_lock); 529 529 + 530 530 + return 0; 531 531 + } 532 532 + EXPORT_SYMBOL(xfrm_register_mode_cbs); 533 533 + 534 534 + void xfrm_unregister_mode_cbs(u8 mode) 535 535 + { 536 536 + if (mode >= XFRM_MODE_MAX) 537 537 + return; 538 538 + 539 539 + spin_lock_bh(&xfrm_mode_cbs_map_lock); 540 540 + RCU_INIT_POINTER(xfrm_mode_cbs_map[mode], NULL); 541 541 + spin_unlock_bh(&xfrm_mode_cbs_map_lock); 542 542 + synchronize_rcu(); 543 543 + } 544 544 + EXPORT_SYMBOL(xfrm_unregister_mode_cbs); 545 545 + 546 546 + static const struct xfrm_mode_cbs *xfrm_get_mode_cbs(u8 mode) 547 547 + { 548 548 + const struct xfrm_mode_cbs *cbs; 549 549 + bool try_load = true; 550 550 + 551 551 + if (mode >= XFRM_MODE_MAX) 552 552 + return NULL; 553 553 + 554 554 + retry: 555 555 + rcu_read_lock(); 556 556 + 557 557 + cbs = rcu_dereference(xfrm_mode_cbs_map[mode]); 558 558 + if (cbs && !try_module_get(cbs->owner)) 559 559 + cbs = NULL; 560 560 + 561 561 + rcu_read_unlock(); 562 562 + 563 563 + if (mode == XFRM_MODE_IPTFS && !cbs && try_load) { 564 564 + request_module("xfrm-iptfs"); 565 565 + try_load = false; 566 566 + goto retry; 567 567 + } 568 568 + 569 569 + return cbs; 570 570 + } 571 571 + 528 572 void xfrm_state_free(struct xfrm_state *x) 529 573 { 530 574 kmem_cache_free(xfrm_state_cache, x); ··· 587 523 588 524 static void ___xfrm_state_destroy(struct xfrm_state *x) 589 525 { 526 526 + if (x->mode_cbs && x->mode_cbs->destroy_state) 527 527 + x->mode_cbs->destroy_state(x); 590 528 hrtimer_cancel(&x->mtimer); 591 529 del_timer_sync(&x->rtimer); 592 530 kfree(x->aead); ··· 748 682 x->replay_maxdiff = 0; 749 683 x->pcpu_num = UINT_MAX; 750 684 spin_lock_init(&x->lock); 685 685 + x->mode_data = NULL; 751 686 } 752 687 return x; 753 688 } ··· 2012 1945 x->new_mapping_sport = 0; 2013 1946 x->dir = orig->dir; 2014 1947 1948 1948 + x->mode_cbs = orig->mode_cbs; 1949 1949 + if (x->mode_cbs && x->mode_cbs->clone_state) { 1950 1950 + if (x->mode_cbs->clone_state(x, orig)) 1951 1951 + goto error; 1952 1952 + } 1953 1953 + 2015 1954 return x; 2016 1955 2017 1956 error: ··· 2344 2271 #endif 2345 2272 case XFRM_MODE_TUNNEL: 2346 2273 case XFRM_MODE_BEET: 2274 2274 + case XFRM_MODE_IPTFS: 2347 2275 return 4; 2348 2276 } 2349 2277 return 5; ··· 2371 2297 #endif 2372 2298 case XFRM_MODE_TUNNEL: 2373 2299 case XFRM_MODE_BEET: 2300 2300 + case XFRM_MODE_IPTFS: 2374 2301 return 3; 2375 2302 } 2376 2303 return 4; ··· 3061 2986 case XFRM_MODE_TUNNEL: 3062 2987 break; 3063 2988 default: 2989 2989 + if (x->mode_cbs && x->mode_cbs->get_inner_mtu) 2990 2990 + return x->mode_cbs->get_inner_mtu(x, mtu); 2991 2991 + 3064 2992 WARN_ON_ONCE(1); 3065 2993 break; 3066 2994 } ··· 3164 3086 } 3165 3087 } 3166 3088 3089 3089 + x->mode_cbs = xfrm_get_mode_cbs(x->props.mode); 3090 3090 + if (x->mode_cbs) { 3091 3091 + if (x->mode_cbs->init_state) 3092 3092 + err = x->mode_cbs->init_state(x); 3093 3093 + module_put(x->mode_cbs->owner); 3094 3094 + } 3167 3095 error: 3168 3096 return err; 3169 3097 }

+77

net/xfrm/xfrm_user.c

reviewed

··· 301 301 NL_SET_ERR_MSG(extack, "TFC padding can only be used in tunnel mode"); 302 302 goto out; 303 303 } 304 304 + if ((attrs[XFRMA_IPTFS_DROP_TIME] || 305 305 + attrs[XFRMA_IPTFS_REORDER_WINDOW] || 306 306 + attrs[XFRMA_IPTFS_DONT_FRAG] || 307 307 + attrs[XFRMA_IPTFS_INIT_DELAY] || 308 308 + attrs[XFRMA_IPTFS_MAX_QSIZE] || 309 309 + attrs[XFRMA_IPTFS_PKT_SIZE]) && 310 310 + p->mode != XFRM_MODE_IPTFS) { 311 311 + NL_SET_ERR_MSG(extack, "IP-TFS options can only be used in IP-TFS mode"); 312 312 + goto out; 313 313 + } 304 314 break; 305 315 306 316 case IPPROTO_COMP: ··· 383 373 case XFRM_MODE_ROUTEOPTIMIZATION: 384 374 case XFRM_MODE_BEET: 385 375 break; 376 376 + case XFRM_MODE_IPTFS: 377 377 + if (p->id.proto != IPPROTO_ESP) { 378 378 + NL_SET_ERR_MSG(extack, "IP-TFS mode only supported with ESP"); 379 379 + goto out; 380 380 + } 381 381 + if (sa_dir == 0) { 382 382 + NL_SET_ERR_MSG(extack, "IP-TFS mode requires in or out direction attribute"); 383 383 + goto out; 384 384 + } 385 385 + break; 386 386 387 387 default: 388 388 NL_SET_ERR_MSG(extack, "Unsupported mode"); ··· 441 421 goto out; 442 422 } 443 423 424 424 + if (attrs[XFRMA_IPTFS_DROP_TIME]) { 425 425 + NL_SET_ERR_MSG(extack, "IP-TFS drop time should not be set for output SA"); 426 426 + err = -EINVAL; 427 427 + goto out; 428 428 + } 429 429 + 430 430 + if (attrs[XFRMA_IPTFS_REORDER_WINDOW]) { 431 431 + NL_SET_ERR_MSG(extack, "IP-TFS reorder window should not be set for output SA"); 432 432 + err = -EINVAL; 433 433 + goto out; 434 434 + } 435 435 + 444 436 if (attrs[XFRMA_REPLAY_VAL]) { 445 437 struct xfrm_replay_state *replay; 446 438 ··· 489 457 goto out; 490 458 } 491 459 460 460 + } 461 461 + 462 462 + if (attrs[XFRMA_IPTFS_DONT_FRAG]) { 463 463 + NL_SET_ERR_MSG(extack, "IP-TFS don't fragment should not be set for input SA"); 464 464 + err = -EINVAL; 465 465 + goto out; 466 466 + } 467 467 + 468 468 + if (attrs[XFRMA_IPTFS_INIT_DELAY]) { 469 469 + NL_SET_ERR_MSG(extack, "IP-TFS initial delay should not be set for input SA"); 470 470 + err = -EINVAL; 471 471 + goto out; 472 472 + } 473 473 + 474 474 + if (attrs[XFRMA_IPTFS_MAX_QSIZE]) { 475 475 + NL_SET_ERR_MSG(extack, "IP-TFS max queue size should not be set for input SA"); 476 476 + err = -EINVAL; 477 477 + goto out; 478 478 + } 479 479 + 480 480 + if (attrs[XFRMA_IPTFS_PKT_SIZE]) { 481 481 + NL_SET_ERR_MSG(extack, "IP-TFS packet size should not be set for input SA"); 482 482 + err = -EINVAL; 483 483 + goto out; 492 484 } 493 485 } 494 486 ··· 942 886 goto error; 943 887 } 944 888 889 889 + if (x->mode_cbs && x->mode_cbs->user_init) { 890 890 + err = x->mode_cbs->user_init(net, x, attrs, extack); 891 891 + if (err) 892 892 + goto error; 893 893 + } 894 894 + 945 895 return x; 946 896 947 897 error: ··· 1363 1301 if (ret) 1364 1302 goto out; 1365 1303 } 1304 1304 + if (x->mode_cbs && x->mode_cbs->copy_to_user) 1305 1305 + ret = x->mode_cbs->copy_to_user(x, skb); 1306 1306 + if (ret) 1307 1307 + goto out; 1366 1308 if (x->mapping_maxage) { 1367 1309 ret = nla_put_u32(skb, XFRMA_MTIMER_THRESH, x->mapping_maxage); 1368 1310 if (ret) ··· 2023 1957 NL_SET_ERR_MSG(extack, "Mode in optional template not allowed in outbound policy"); 2024 1958 return -EINVAL; 2025 1959 } 1960 1960 + break; 1961 1961 + case XFRM_MODE_IPTFS: 2026 1962 break; 2027 1963 default: 2028 1964 if (ut[i].family != prev_family) { ··· 3288 3220 [XFRMA_SA_DIR] = NLA_POLICY_RANGE(NLA_U8, XFRM_SA_DIR_IN, XFRM_SA_DIR_OUT), 3289 3221 [XFRMA_NAT_KEEPALIVE_INTERVAL] = { .type = NLA_U32 }, 3290 3222 [XFRMA_SA_PCPU] = { .type = NLA_U32 }, 3223 3223 + [XFRMA_IPTFS_DROP_TIME] = { .type = NLA_U32 }, 3224 3224 + [XFRMA_IPTFS_REORDER_WINDOW] = { .type = NLA_U16 }, 3225 3225 + [XFRMA_IPTFS_DONT_FRAG] = { .type = NLA_FLAG }, 3226 3226 + [XFRMA_IPTFS_INIT_DELAY] = { .type = NLA_U32 }, 3227 3227 + [XFRMA_IPTFS_MAX_QSIZE] = { .type = NLA_U32 }, 3228 3228 + [XFRMA_IPTFS_PKT_SIZE] = { .type = NLA_U32 }, 3291 3229 }; 3292 3230 EXPORT_SYMBOL_GPL(xfrma_policy); 3293 3231 ··· 3627 3553 3628 3554 if (x->nat_keepalive_interval) 3629 3555 l += nla_total_size(sizeof(x->nat_keepalive_interval)); 3556 3556 + 3557 3557 + if (x->mode_cbs && x->mode_cbs->sa_len) 3558 3558 + l += x->mode_cbs->sa_len(x); 3630 3559 3631 3560 return l; 3632 3561 }