+55 -9

arch/s390/net/bpf_jit_comp.c

reviewed

··· 1209 1209 */ 1210 1210 case BPF_STX | BPF_ATOMIC | BPF_DW: 1211 1211 case BPF_STX | BPF_ATOMIC | BPF_W: 1212 1212 - if (insn->imm != BPF_ADD) { 1212 1212 + { 1213 1213 + bool is32 = BPF_SIZE(insn->code) == BPF_W; 1214 1214 + 1215 1215 + switch (insn->imm) { 1216 1216 + /* {op32|op64} {%w0|%src},%src,off(%dst) */ 1217 1217 + #define EMIT_ATOMIC(op32, op64) do { \ 1218 1218 + EMIT6_DISP_LH(0xeb000000, is32 ? (op32) : (op64), \ 1219 1219 + (insn->imm & BPF_FETCH) ? src_reg : REG_W0, \ 1220 1220 + src_reg, dst_reg, off); \ 1221 1221 + if (is32 && (insn->imm & BPF_FETCH)) \ 1222 1222 + EMIT_ZERO(src_reg); \ 1223 1223 + } while (0) 1224 1224 + case BPF_ADD: 1225 1225 + case BPF_ADD | BPF_FETCH: 1226 1226 + /* {laal|laalg} */ 1227 1227 + EMIT_ATOMIC(0x00fa, 0x00ea); 1228 1228 + break; 1229 1229 + case BPF_AND: 1230 1230 + case BPF_AND | BPF_FETCH: 1231 1231 + /* {lan|lang} */ 1232 1232 + EMIT_ATOMIC(0x00f4, 0x00e4); 1233 1233 + break; 1234 1234 + case BPF_OR: 1235 1235 + case BPF_OR | BPF_FETCH: 1236 1236 + /* {lao|laog} */ 1237 1237 + EMIT_ATOMIC(0x00f6, 0x00e6); 1238 1238 + break; 1239 1239 + case BPF_XOR: 1240 1240 + case BPF_XOR | BPF_FETCH: 1241 1241 + /* {lax|laxg} */ 1242 1242 + EMIT_ATOMIC(0x00f7, 0x00e7); 1243 1243 + break; 1244 1244 + #undef EMIT_ATOMIC 1245 1245 + case BPF_XCHG: 1246 1246 + /* {ly|lg} %w0,off(%dst) */ 1247 1247 + EMIT6_DISP_LH(0xe3000000, 1248 1248 + is32 ? 0x0058 : 0x0004, REG_W0, REG_0, 1249 1249 + dst_reg, off); 1250 1250 + /* 0: {csy|csg} %w0,%src,off(%dst) */ 1251 1251 + EMIT6_DISP_LH(0xeb000000, is32 ? 0x0014 : 0x0030, 1252 1252 + REG_W0, src_reg, dst_reg, off); 1253 1253 + /* brc 4,0b */ 1254 1254 + EMIT4_PCREL_RIC(0xa7040000, 4, jit->prg - 6); 1255 1255 + /* {llgfr|lgr} %src,%w0 */ 1256 1256 + EMIT4(is32 ? 0xb9160000 : 0xb9040000, src_reg, REG_W0); 1257 1257 + if (is32 && insn_is_zext(&insn[1])) 1258 1258 + insn_count = 2; 1259 1259 + break; 1260 1260 + case BPF_CMPXCHG: 1261 1261 + /* 0: {csy|csg} %b0,%src,off(%dst) */ 1262 1262 + EMIT6_DISP_LH(0xeb000000, is32 ? 0x0014 : 0x0030, 1263 1263 + BPF_REG_0, src_reg, dst_reg, off); 1264 1264 + break; 1265 1265 + default: 1213 1266 pr_err("Unknown atomic operation %02x\n", insn->imm); 1214 1267 return -1; 1215 1268 } 1216 1269 1217 1217 - /* *(u32/u64 *)(dst + off) += src 1218 1218 - * 1219 1219 - * BFW_W: laal %w0,%src,off(%dst) 1220 1220 - * BPF_DW: laalg %w0,%src,off(%dst) 1221 1221 - */ 1222 1222 - EMIT6_DISP_LH(0xeb000000, 1223 1223 - BPF_SIZE(insn->code) == BPF_W ? 0x00fa : 0x00ea, 1224 1224 - REG_W0, src_reg, dst_reg, off); 1225 1270 jit->seen |= SEEN_MEM; 1226 1271 break; 1272 1272 + } 1227 1273 /* 1228 1274 * BPF_LDX 1229 1275 */

+9 -12

drivers/net/ethernet/amazon/ena/ena_netdev.c

reviewed

··· 300 300 301 301 rc = ena_xdp_tx_map_frame(xdp_ring, tx_info, xdpf, &push_hdr, &push_len); 302 302 if (unlikely(rc)) 303 303 - goto error_drop_packet; 303 303 + return rc; 304 304 305 305 ena_tx_ctx.ena_bufs = tx_info->bufs; 306 306 ena_tx_ctx.push_header = push_hdr; ··· 330 330 error_unmap_dma: 331 331 ena_unmap_tx_buff(xdp_ring, tx_info); 332 332 tx_info->xdpf = NULL; 333 333 - error_drop_packet: 334 334 - xdp_return_frame(xdpf); 335 333 return rc; 336 334 } 337 335 ··· 337 339 struct xdp_frame **frames, u32 flags) 338 340 { 339 341 struct ena_adapter *adapter = netdev_priv(dev); 340 340 - int qid, i, err, drops = 0; 341 342 struct ena_ring *xdp_ring; 343 343 + int qid, i, nxmit = 0; 342 344 343 345 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 344 346 return -EINVAL; ··· 358 360 spin_lock(&xdp_ring->xdp_tx_lock); 359 361 360 362 for (i = 0; i < n; i++) { 361 361 - err = ena_xdp_xmit_frame(xdp_ring, dev, frames[i], 0); 362 362 - /* The descriptor is freed by ena_xdp_xmit_frame in case 363 363 - * of an error. 364 364 - */ 365 365 - if (err) 366 366 - drops++; 363 363 + if (ena_xdp_xmit_frame(xdp_ring, dev, frames[i], 0)) 364 364 + break; 365 365 + nxmit++; 367 366 } 368 367 369 368 /* Ring doorbell to make device aware of the packets */ ··· 373 378 spin_unlock(&xdp_ring->xdp_tx_lock); 374 379 375 380 /* Return number of packets sent */ 376 376 - return n - drops; 381 381 + return nxmit; 377 382 } 378 383 379 384 static int ena_xdp_execute(struct ena_ring *rx_ring, struct xdp_buff *xdp) ··· 410 415 /* The XDP queues are shared between XDP_TX and XDP_REDIRECT */ 411 416 spin_lock(&xdp_ring->xdp_tx_lock); 412 417 413 413 - ena_xdp_xmit_frame(xdp_ring, rx_ring->netdev, xdpf, XDP_XMIT_FLUSH); 418 418 + if (ena_xdp_xmit_frame(xdp_ring, rx_ring->netdev, xdpf, 419 419 + XDP_XMIT_FLUSH)) 420 420 + xdp_return_frame(xdpf); 414 421 415 422 spin_unlock(&xdp_ring->xdp_tx_lock); 416 423 xdp_stat = &rx_ring->rx_stats.xdp_tx;

+8 -12

drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c

reviewed

··· 217 217 struct pci_dev *pdev = bp->pdev; 218 218 struct bnxt_tx_ring_info *txr; 219 219 dma_addr_t mapping; 220 220 - int drops = 0; 220 220 + int nxmit = 0; 221 221 int ring; 222 222 int i; 223 223 ··· 233 233 struct xdp_frame *xdp = frames[i]; 234 234 235 235 if (!txr || !bnxt_tx_avail(bp, txr) || 236 236 - !(bp->bnapi[ring]->flags & BNXT_NAPI_FLAG_XDP)) { 237 237 - xdp_return_frame_rx_napi(xdp); 238 238 - drops++; 239 239 - continue; 240 240 - } 236 236 + !(bp->bnapi[ring]->flags & BNXT_NAPI_FLAG_XDP)) 237 237 + break; 241 238 242 239 mapping = dma_map_single(&pdev->dev, xdp->data, xdp->len, 243 240 DMA_TO_DEVICE); 244 241 245 245 - if (dma_mapping_error(&pdev->dev, mapping)) { 246 246 - xdp_return_frame_rx_napi(xdp); 247 247 - drops++; 248 248 - continue; 249 249 - } 242 242 + if (dma_mapping_error(&pdev->dev, mapping)) 243 243 + break; 244 244 + 250 245 __bnxt_xmit_xdp_redirect(bp, txr, mapping, xdp->len, xdp); 246 246 + nxmit++; 251 247 } 252 248 253 249 if (flags & XDP_XMIT_FLUSH) { ··· 252 256 bnxt_db_write(bp, &txr->tx_db, txr->tx_prod); 253 257 } 254 258 255 255 - return num_frames - drops; 259 259 + return nxmit; 256 260 } 257 261 258 262 /* Under rtnl_lock */

+5 -7

drivers/net/ethernet/freescale/dpaa/dpaa_eth.c

reviewed

··· 3081 3081 struct xdp_frame **frames, u32 flags) 3082 3082 { 3083 3083 struct xdp_frame *xdpf; 3084 3084 - int i, err, drops = 0; 3084 3084 + int i, nxmit = 0; 3085 3085 3086 3086 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 3087 3087 return -EINVAL; ··· 3091 3091 3092 3092 for (i = 0; i < n; i++) { 3093 3093 xdpf = frames[i]; 3094 3094 - err = dpaa_xdp_xmit_frame(net_dev, xdpf); 3095 3095 - if (err) { 3096 3096 - xdp_return_frame_rx_napi(xdpf); 3097 3097 - drops++; 3098 3098 - } 3094 3094 + if (dpaa_xdp_xmit_frame(net_dev, xdpf)) 3095 3095 + break; 3096 3096 + nxmit++; 3099 3097 } 3100 3098 3101 3101 - return n - drops; 3099 3099 + return nxmit; 3102 3100 } 3103 3101 3104 3102 static int dpaa_ts_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)

-2

drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c

reviewed

··· 2431 2431 percpu_stats->tx_packets += enqueued; 2432 2432 for (i = 0; i < enqueued; i++) 2433 2433 percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]); 2434 2434 - for (i = enqueued; i < n; i++) 2435 2435 - xdp_return_frame_rx_napi(frames[i]); 2436 2434 2437 2435 return enqueued; 2438 2436 }

+7 -8

drivers/net/ethernet/intel/i40e/i40e_txrx.c

reviewed

··· 3835 3835 * @frames: array of XDP buffer pointers 3836 3836 * @flags: XDP extra info 3837 3837 * 3838 3838 - * Returns number of frames successfully sent. Frames that fail are 3839 3839 - * free'ed via XDP return API. 3838 3838 + * Returns number of frames successfully sent. Failed frames 3839 3839 + * will be free'ed by XDP core. 3840 3840 * 3841 3841 * For error cases, a negative errno code is returned and no-frames 3842 3842 * are transmitted (caller must handle freeing frames). ··· 3849 3849 struct i40e_vsi *vsi = np->vsi; 3850 3850 struct i40e_pf *pf = vsi->back; 3851 3851 struct i40e_ring *xdp_ring; 3852 3852 - int drops = 0; 3852 3852 + int nxmit = 0; 3853 3853 int i; 3854 3854 3855 3855 if (test_bit(__I40E_VSI_DOWN, vsi->state)) ··· 3869 3869 int err; 3870 3870 3871 3871 err = i40e_xmit_xdp_ring(xdpf, xdp_ring); 3872 3872 - if (err != I40E_XDP_TX) { 3873 3873 - xdp_return_frame_rx_napi(xdpf); 3874 3874 - drops++; 3875 3875 - } 3872 3872 + if (err != I40E_XDP_TX) 3873 3873 + break; 3874 3874 + nxmit++; 3876 3875 } 3877 3876 3878 3877 if (unlikely(flags & XDP_XMIT_FLUSH)) 3879 3878 i40e_xdp_ring_update_tail(xdp_ring); 3880 3879 3881 3881 - return n - drops; 3880 3880 + return nxmit; 3882 3881 }

+7 -8

drivers/net/ethernet/intel/ice/ice_txrx.c

reviewed

··· 554 554 * @frames: XDP frames to be transmitted 555 555 * @flags: transmit flags 556 556 * 557 557 - * Returns number of frames successfully sent. Frames that fail are 558 558 - * free'ed via XDP return API. 557 557 + * Returns number of frames successfully sent. Failed frames 558 558 + * will be free'ed by XDP core. 559 559 * For error cases, a negative errno code is returned and no-frames 560 560 * are transmitted (caller must handle freeing frames). 561 561 */ ··· 567 567 unsigned int queue_index = smp_processor_id(); 568 568 struct ice_vsi *vsi = np->vsi; 569 569 struct ice_ring *xdp_ring; 570 570 - int drops = 0, i; 570 570 + int nxmit = 0, i; 571 571 572 572 if (test_bit(__ICE_DOWN, vsi->state)) 573 573 return -ENETDOWN; ··· 584 584 int err; 585 585 586 586 err = ice_xmit_xdp_ring(xdpf->data, xdpf->len, xdp_ring); 587 587 - if (err != ICE_XDP_TX) { 588 588 - xdp_return_frame_rx_napi(xdpf); 589 589 - drops++; 590 590 - } 587 587 + if (err != ICE_XDP_TX) 588 588 + break; 589 589 + nxmit++; 591 590 } 592 591 593 592 if (unlikely(flags & XDP_XMIT_FLUSH)) 594 593 ice_xdp_ring_update_tail(xdp_ring); 595 594 596 596 - return n - drops; 595 595 + return nxmit; 597 596 } 598 597 599 598 /**

+5 -6

drivers/net/ethernet/intel/igb/igb_main.c

reviewed

··· 2934 2934 int cpu = smp_processor_id(); 2935 2935 struct igb_ring *tx_ring; 2936 2936 struct netdev_queue *nq; 2937 2937 - int drops = 0; 2937 2937 + int nxmit = 0; 2938 2938 int i; 2939 2939 2940 2940 if (unlikely(test_bit(__IGB_DOWN, &adapter->state))) ··· 2961 2961 int err; 2962 2962 2963 2963 err = igb_xmit_xdp_ring(adapter, tx_ring, xdpf); 2964 2964 - if (err != IGB_XDP_TX) { 2965 2965 - xdp_return_frame_rx_napi(xdpf); 2966 2966 - drops++; 2967 2967 - } 2964 2964 + if (err != IGB_XDP_TX) 2965 2965 + break; 2966 2966 + nxmit++; 2968 2967 } 2969 2968 2970 2969 __netif_tx_unlock(nq); ··· 2971 2972 if (unlikely(flags & XDP_XMIT_FLUSH)) 2972 2973 igb_xdp_ring_update_tail(tx_ring); 2973 2974 2974 2974 - return n - drops; 2975 2975 + return nxmit; 2975 2976 } 2976 2977 2977 2978 static const struct net_device_ops igb_netdev_ops = {

+5 -6

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

reviewed

··· 10189 10189 { 10190 10190 struct ixgbe_adapter *adapter = netdev_priv(dev); 10191 10191 struct ixgbe_ring *ring; 10192 10192 - int drops = 0; 10192 10192 + int nxmit = 0; 10193 10193 int i; 10194 10194 10195 10195 if (unlikely(test_bit(__IXGBE_DOWN, &adapter->state))) ··· 10213 10213 int err; 10214 10214 10215 10215 err = ixgbe_xmit_xdp_ring(adapter, xdpf); 10216 10216 - if (err != IXGBE_XDP_TX) { 10217 10217 - xdp_return_frame_rx_napi(xdpf); 10218 10218 - drops++; 10219 10219 - } 10216 10216 + if (err != IXGBE_XDP_TX) 10217 10217 + break; 10218 10218 + nxmit++; 10220 10219 } 10221 10220 10222 10221 if (unlikely(flags & XDP_XMIT_FLUSH)) 10223 10222 ixgbe_xdp_ring_update_tail(ring); 10224 10223 10225 10225 - return n - drops; 10224 10224 + return nxmit; 10226 10225 } 10227 10226 10228 10227 static const struct net_device_ops ixgbe_netdev_ops = {

+6 -7

drivers/net/ethernet/marvell/mvneta.c

reviewed

··· 2137 2137 { 2138 2138 struct mvneta_port *pp = netdev_priv(dev); 2139 2139 struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats); 2140 2140 - int i, nxmit_byte = 0, nxmit = num_frame; 2140 2140 + int i, nxmit_byte = 0, nxmit = 0; 2141 2141 int cpu = smp_processor_id(); 2142 2142 struct mvneta_tx_queue *txq; 2143 2143 struct netdev_queue *nq; ··· 2155 2155 __netif_tx_lock(nq, cpu); 2156 2156 for (i = 0; i < num_frame; i++) { 2157 2157 ret = mvneta_xdp_submit_frame(pp, txq, frames[i], true); 2158 2158 - if (ret == MVNETA_XDP_TX) { 2159 2159 - nxmit_byte += frames[i]->len; 2160 2160 - } else { 2161 2161 - xdp_return_frame_rx_napi(frames[i]); 2162 2162 - nxmit--; 2163 2163 - } 2158 2158 + if (ret != MVNETA_XDP_TX) 2159 2159 + break; 2160 2160 + 2161 2161 + nxmit_byte += frames[i]->len; 2162 2162 + nxmit++; 2164 2163 } 2165 2164 2166 2165 if (unlikely(flags & XDP_XMIT_FLUSH))

+6 -7

drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c

reviewed

··· 3744 3744 struct xdp_frame **frames, u32 flags) 3745 3745 { 3746 3746 struct mvpp2_port *port = netdev_priv(dev); 3747 3747 - int i, nxmit_byte = 0, nxmit = num_frame; 3747 3747 + int i, nxmit_byte = 0, nxmit = 0; 3748 3748 struct mvpp2_pcpu_stats *stats; 3749 3749 u16 txq_id; 3750 3750 u32 ret; ··· 3762 3762 3763 3763 for (i = 0; i < num_frame; i++) { 3764 3764 ret = mvpp2_xdp_submit_frame(port, txq_id, frames[i], true); 3765 3765 - if (ret == MVPP2_XDP_TX) { 3766 3766 - nxmit_byte += frames[i]->len; 3767 3767 - } else { 3768 3768 - xdp_return_frame_rx_napi(frames[i]); 3769 3769 - nxmit--; 3770 3770 - } 3765 3765 + if (ret != MVPP2_XDP_TX) 3766 3766 + break; 3767 3767 + 3768 3768 + nxmit_byte += frames[i]->len; 3769 3769 + nxmit++; 3771 3770 } 3772 3771 3773 3772 if (likely(nxmit > 0))

+6 -9

drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c

reviewed

··· 500 500 { 501 501 struct mlx5e_priv *priv = netdev_priv(dev); 502 502 struct mlx5e_xdpsq *sq; 503 503 - int drops = 0; 503 503 + int nxmit = 0; 504 504 int sq_num; 505 505 int i; 506 506 ··· 529 529 xdptxd.dma_addr = dma_map_single(sq->pdev, xdptxd.data, 530 530 xdptxd.len, DMA_TO_DEVICE); 531 531 532 532 - if (unlikely(dma_mapping_error(sq->pdev, xdptxd.dma_addr))) { 533 533 - xdp_return_frame_rx_napi(xdpf); 534 534 - drops++; 535 535 - continue; 536 536 - } 532 532 + if (unlikely(dma_mapping_error(sq->pdev, xdptxd.dma_addr))) 533 533 + break; 537 534 538 535 xdpi.mode = MLX5E_XDP_XMIT_MODE_FRAME; 539 536 xdpi.frame.xdpf = xdpf; ··· 541 544 if (unlikely(!ret)) { 542 545 dma_unmap_single(sq->pdev, xdptxd.dma_addr, 543 546 xdptxd.len, DMA_TO_DEVICE); 544 544 - xdp_return_frame_rx_napi(xdpf); 545 545 - drops++; 547 547 + break; 546 548 } 549 549 + nxmit++; 547 550 } 548 551 549 552 if (flags & XDP_XMIT_FLUSH) { ··· 552 555 mlx5e_xmit_xdp_doorbell(sq); 553 556 } 554 557 555 555 - return n - drops; 558 558 + return nxmit; 556 559 } 557 560 558 561 void mlx5e_xdp_rx_poll_complete(struct mlx5e_rq *rq)

+7 -12

drivers/net/ethernet/qlogic/qede/qede_fp.c

reviewed

··· 345 345 struct qede_tx_queue *xdp_tx; 346 346 struct xdp_frame *xdpf; 347 347 dma_addr_t mapping; 348 348 - int i, drops = 0; 348 348 + int i, nxmit = 0; 349 349 u16 xdp_prod; 350 350 351 351 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) ··· 364 364 365 365 mapping = dma_map_single(dmadev, xdpf->data, xdpf->len, 366 366 DMA_TO_DEVICE); 367 367 - if (unlikely(dma_mapping_error(dmadev, mapping))) { 368 368 - xdp_return_frame_rx_napi(xdpf); 369 369 - drops++; 370 370 - 371 371 - continue; 372 372 - } 367 367 + if (unlikely(dma_mapping_error(dmadev, mapping))) 368 368 + break; 373 369 374 370 if (unlikely(qede_xdp_xmit(xdp_tx, mapping, 0, xdpf->len, 375 375 - NULL, xdpf))) { 376 376 - xdp_return_frame_rx_napi(xdpf); 377 377 - drops++; 378 378 - } 371 371 + NULL, xdpf))) 372 372 + break; 373 373 + nxmit++; 379 374 } 380 375 381 376 if (flags & XDP_XMIT_FLUSH) { ··· 382 387 383 388 spin_unlock(&xdp_tx->xdp_tx_lock); 384 389 385 385 - return n_frames - drops; 390 390 + return nxmit; 386 391 } 387 392 388 393 int qede_txq_has_work(struct qede_tx_queue *txq)

+1 -14

drivers/net/ethernet/sfc/tx.c

reviewed

··· 412 412 return NETDEV_TX_OK; 413 413 } 414 414 415 415 - static void efx_xdp_return_frames(int n, struct xdp_frame **xdpfs) 416 416 - { 417 417 - int i; 418 418 - 419 419 - for (i = 0; i < n; i++) 420 420 - xdp_return_frame_rx_napi(xdpfs[i]); 421 421 - } 422 422 - 423 415 /* Transmit a packet from an XDP buffer 424 416 * 425 417 * Returns number of packets sent on success, error code otherwise. ··· 484 492 if (flush && i > 0) 485 493 efx_nic_push_buffers(tx_queue); 486 494 487 487 - if (i == 0) 488 488 - return -EIO; 489 489 - 490 490 - efx_xdp_return_frames(n - i, xdpfs + i); 491 491 - 492 492 - return i; 495 495 + return i == 0 ? -EIO : i; 493 496 } 494 497 495 498 /* Initiate a packet transmission. We use one channel per CPU

+7 -9

drivers/net/ethernet/socionext/netsec.c

reviewed

··· 1760 1760 { 1761 1761 struct netsec_priv *priv = netdev_priv(ndev); 1762 1762 struct netsec_desc_ring *tx_ring = &priv->desc_ring[NETSEC_RING_TX]; 1763 1763 - int drops = 0; 1764 1764 - int i; 1763 1763 + int i, nxmit = 0; 1765 1764 1766 1765 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 1767 1766 return -EINVAL; ··· 1771 1772 int err; 1772 1773 1773 1774 err = netsec_xdp_queue_one(priv, xdpf, true); 1774 1774 - if (err != NETSEC_XDP_TX) { 1775 1775 - xdp_return_frame_rx_napi(xdpf); 1776 1776 - drops++; 1777 1777 - } else { 1778 1778 - tx_ring->xdp_xmit++; 1779 1779 - } 1775 1775 + if (err != NETSEC_XDP_TX) 1776 1776 + break; 1777 1777 + 1778 1778 + tx_ring->xdp_xmit++; 1779 1779 + nxmit++; 1780 1780 } 1781 1781 spin_unlock(&tx_ring->lock); 1782 1782 ··· 1784 1786 tx_ring->xdp_xmit = 0; 1785 1787 } 1786 1788 1787 1787 - return n - drops; 1789 1789 + return nxmit; 1788 1790 } 1789 1791 1790 1792 static int netsec_xdp_setup(struct netsec_priv *priv, struct bpf_prog *prog,

+6 -8

drivers/net/ethernet/ti/cpsw.c

reviewed

··· 1123 1123 struct cpsw_priv *priv = netdev_priv(ndev); 1124 1124 struct cpsw_common *cpsw = priv->cpsw; 1125 1125 struct xdp_frame *xdpf; 1126 1126 - int i, drops = 0, port; 1126 1126 + int i, nxmit = 0, port; 1127 1127 1128 1128 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 1129 1129 return -EINVAL; 1130 1130 1131 1131 for (i = 0; i < n; i++) { 1132 1132 xdpf = frames[i]; 1133 1133 - if (xdpf->len < CPSW_MIN_PACKET_SIZE) { 1134 1134 - xdp_return_frame_rx_napi(xdpf); 1135 1135 - drops++; 1136 1136 - continue; 1137 1137 - } 1133 1133 + if (xdpf->len < CPSW_MIN_PACKET_SIZE) 1134 1134 + break; 1138 1135 1139 1136 port = priv->emac_port + cpsw->data.dual_emac; 1140 1137 if (cpsw_xdp_tx_frame(priv, xdpf, NULL, port)) 1141 1141 - drops++; 1138 1138 + break; 1139 1139 + nxmit++; 1142 1140 } 1143 1141 1144 1144 - return n - drops; 1142 1142 + return nxmit; 1145 1143 } 1146 1144 1147 1145 #ifdef CONFIG_NET_POLL_CONTROLLER

+6 -8

drivers/net/ethernet/ti/cpsw_new.c

reviewed

··· 1093 1093 { 1094 1094 struct cpsw_priv *priv = netdev_priv(ndev); 1095 1095 struct xdp_frame *xdpf; 1096 1096 - int i, drops = 0; 1096 1096 + int i, nxmit = 0; 1097 1097 1098 1098 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 1099 1099 return -EINVAL; 1100 1100 1101 1101 for (i = 0; i < n; i++) { 1102 1102 xdpf = frames[i]; 1103 1103 - if (xdpf->len < CPSW_MIN_PACKET_SIZE) { 1104 1104 - xdp_return_frame_rx_napi(xdpf); 1105 1105 - drops++; 1106 1106 - continue; 1107 1107 - } 1103 1103 + if (xdpf->len < CPSW_MIN_PACKET_SIZE) 1104 1104 + break; 1108 1105 1109 1106 if (cpsw_xdp_tx_frame(priv, xdpf, NULL, priv->emac_port)) 1110 1110 - drops++; 1107 1107 + break; 1108 1108 + nxmit++; 1111 1109 } 1112 1110 1113 1113 - return n - drops; 1111 1111 + return nxmit; 1114 1112 } 1115 1113 1116 1114 static int cpsw_get_port_parent_id(struct net_device *ndev,

+4 -7

drivers/net/ethernet/ti/cpsw_priv.c

reviewed

··· 1305 1305 ret = cpdma_chan_submit_mapped(txch, cpsw_xdpf_to_handle(xdpf), 1306 1306 dma, xdpf->len, port); 1307 1307 } else { 1308 1308 - if (sizeof(*xmeta) > xdpf->headroom) { 1309 1309 - xdp_return_frame_rx_napi(xdpf); 1308 1308 + if (sizeof(*xmeta) > xdpf->headroom) 1310 1309 return -EINVAL; 1311 1311 - } 1312 1310 1313 1311 ret = cpdma_chan_submit(txch, cpsw_xdpf_to_handle(xdpf), 1314 1312 xdpf->data, xdpf->len, port); 1315 1313 } 1316 1314 1317 1317 - if (ret) { 1315 1315 + if (ret) 1318 1316 priv->ndev->stats.tx_dropped++; 1319 1319 - xdp_return_frame_rx_napi(xdpf); 1320 1320 - } 1321 1317 1322 1318 return ret; 1323 1319 } ··· 1349 1353 if (unlikely(!xdpf)) 1350 1354 goto drop; 1351 1355 1352 1352 - cpsw_xdp_tx_frame(priv, xdpf, page, port); 1356 1356 + if (cpsw_xdp_tx_frame(priv, xdpf, page, port)) 1357 1357 + xdp_return_frame_rx_napi(xdpf); 1353 1358 break; 1354 1359 case XDP_REDIRECT: 1355 1360 if (xdp_do_redirect(ndev, xdp, prog))

+9 -6

drivers/net/tun.c

reviewed

··· 1181 1181 struct tun_struct *tun = netdev_priv(dev); 1182 1182 struct tun_file *tfile; 1183 1183 u32 numqueues; 1184 1184 - int drops = 0; 1185 1185 - int cnt = n; 1184 1184 + int nxmit = 0; 1186 1185 int i; 1187 1186 1188 1187 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) ··· 1211 1212 1212 1213 if (__ptr_ring_produce(&tfile->tx_ring, frame)) { 1213 1214 atomic_long_inc(&dev->tx_dropped); 1214 1214 - xdp_return_frame_rx_napi(xdp); 1215 1215 - drops++; 1215 1215 + break; 1216 1216 } 1217 1217 + nxmit++; 1217 1218 } 1218 1219 spin_unlock(&tfile->tx_ring.producer_lock); 1219 1220 ··· 1221 1222 __tun_xdp_flush_tfile(tfile); 1222 1223 1223 1224 rcu_read_unlock(); 1224 1224 - return cnt - drops; 1225 1225 + return nxmit; 1225 1226 } 1226 1227 1227 1228 static int tun_xdp_tx(struct net_device *dev, struct xdp_buff *xdp) 1228 1229 { 1229 1230 struct xdp_frame *frame = xdp_convert_buff_to_frame(xdp); 1231 1231 + int nxmit; 1230 1232 1231 1233 if (unlikely(!frame)) 1232 1234 return -EOVERFLOW; 1233 1235 1234 1234 - return tun_xdp_xmit(dev, 1, &frame, XDP_XMIT_FLUSH); 1236 1236 + nxmit = tun_xdp_xmit(dev, 1, &frame, XDP_XMIT_FLUSH); 1237 1237 + if (!nxmit) 1238 1238 + xdp_return_frame_rx_napi(frame); 1239 1239 + return nxmit; 1235 1240 } 1236 1241 1237 1242 static const struct net_device_ops tap_netdev_ops = {

+15 -13

drivers/net/veth.c

reviewed

··· 433 433 u32 flags, bool ndo_xmit) 434 434 { 435 435 struct veth_priv *rcv_priv, *priv = netdev_priv(dev); 436 436 - int i, ret = -ENXIO, drops = 0; 436 436 + int i, ret = -ENXIO, nxmit = 0; 437 437 struct net_device *rcv; 438 438 unsigned int max_len; 439 439 struct veth_rq *rq; ··· 463 463 void *ptr = veth_xdp_to_ptr(frame); 464 464 465 465 if (unlikely(frame->len > max_len || 466 466 - __ptr_ring_produce(&rq->xdp_ring, ptr))) { 467 467 - xdp_return_frame_rx_napi(frame); 468 468 - drops++; 469 469 - } 466 466 + __ptr_ring_produce(&rq->xdp_ring, ptr))) 467 467 + break; 468 468 + nxmit++; 470 469 } 471 470 spin_unlock(&rq->xdp_ring.producer_lock); 472 471 473 472 if (flags & XDP_XMIT_FLUSH) 474 473 __veth_xdp_flush(rq); 475 474 476 476 - ret = n - drops; 475 475 + ret = nxmit; 477 476 if (ndo_xmit) { 478 477 u64_stats_update_begin(&rq->stats.syncp); 479 479 - rq->stats.vs.peer_tq_xdp_xmit += n - drops; 480 480 - rq->stats.vs.peer_tq_xdp_xmit_err += drops; 478 478 + rq->stats.vs.peer_tq_xdp_xmit += nxmit; 479 479 + rq->stats.vs.peer_tq_xdp_xmit_err += n - nxmit; 481 480 u64_stats_update_end(&rq->stats.syncp); 482 481 } 483 482 ··· 503 504 504 505 static void veth_xdp_flush_bq(struct veth_rq *rq, struct veth_xdp_tx_bq *bq) 505 506 { 506 506 - int sent, i, err = 0; 507 507 + int sent, i, err = 0, drops; 507 508 508 509 sent = veth_xdp_xmit(rq->dev, bq->count, bq->q, 0, false); 509 510 if (sent < 0) { 510 511 err = sent; 511 512 sent = 0; 512 512 - for (i = 0; i < bq->count; i++) 513 513 - xdp_return_frame(bq->q[i]); 514 513 } 515 515 - trace_xdp_bulk_tx(rq->dev, sent, bq->count - sent, err); 514 514 + 515 515 + for (i = sent; unlikely(i < bq->count); i++) 516 516 + xdp_return_frame(bq->q[i]); 517 517 + 518 518 + drops = bq->count - sent; 519 519 + trace_xdp_bulk_tx(rq->dev, sent, drops, err); 516 520 517 521 u64_stats_update_begin(&rq->stats.syncp); 518 522 rq->stats.vs.xdp_tx += sent; 519 519 - rq->stats.vs.xdp_tx_err += bq->count - sent; 523 523 + rq->stats.vs.xdp_tx_err += drops; 520 524 u64_stats_update_end(&rq->stats.syncp); 521 525 522 526 bq->count = 0;

+13 -12

drivers/net/virtio_net.c

reviewed

··· 531 531 unsigned int len; 532 532 int packets = 0; 533 533 int bytes = 0; 534 534 - int drops = 0; 534 534 + int nxmit = 0; 535 535 int kicks = 0; 536 536 - int ret, err; 537 536 void *ptr; 537 537 + int ret; 538 538 int i; 539 539 540 540 /* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this ··· 548 548 549 549 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) { 550 550 ret = -EINVAL; 551 551 - drops = n; 552 551 goto out; 553 552 } 554 553 ··· 570 571 for (i = 0; i < n; i++) { 571 572 struct xdp_frame *xdpf = frames[i]; 572 573 573 573 - err = __virtnet_xdp_xmit_one(vi, sq, xdpf); 574 574 - if (err) { 575 575 - xdp_return_frame_rx_napi(xdpf); 576 576 - drops++; 577 577 - } 574 574 + if (__virtnet_xdp_xmit_one(vi, sq, xdpf)) 575 575 + break; 576 576 + nxmit++; 578 577 } 579 579 - ret = n - drops; 578 578 + ret = nxmit; 580 579 581 580 if (flags & XDP_XMIT_FLUSH) { 582 581 if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) ··· 585 588 sq->stats.bytes += bytes; 586 589 sq->stats.packets += packets; 587 590 sq->stats.xdp_tx += n; 588 588 - sq->stats.xdp_tx_drops += drops; 591 591 + sq->stats.xdp_tx_drops += n - nxmit; 589 592 sq->stats.kicks += kicks; 590 593 u64_stats_update_end(&sq->stats.syncp); 591 594 ··· 739 742 if (unlikely(!xdpf)) 740 743 goto err_xdp; 741 744 err = virtnet_xdp_xmit(dev, 1, &xdpf, 0); 742 742 - if (unlikely(err < 0)) { 745 745 + if (unlikely(!err)) { 746 746 + xdp_return_frame_rx_napi(xdpf); 747 747 + } else if (unlikely(err < 0)) { 743 748 trace_xdp_exception(vi->dev, xdp_prog, act); 744 749 goto err_xdp; 745 750 } ··· 928 929 if (unlikely(!xdpf)) 929 930 goto err_xdp; 930 931 err = virtnet_xdp_xmit(dev, 1, &xdpf, 0); 931 931 - if (unlikely(err < 0)) { 932 932 + if (unlikely(!err)) { 933 933 + xdp_return_frame_rx_napi(xdpf); 934 934 + } else if (unlikely(err < 0)) { 932 935 trace_xdp_exception(vi->dev, xdp_prog, act); 933 936 if (unlikely(xdp_page != page)) 934 937 put_page(xdp_page);

+9 -9

drivers/net/xen-netfront.c

reviewed

··· 608 608 struct netfront_info *np = netdev_priv(dev); 609 609 struct netfront_queue *queue = NULL; 610 610 unsigned long irq_flags; 611 611 - int drops = 0; 612 612 - int i, err; 611 611 + int nxmit = 0; 612 612 + int i; 613 613 614 614 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 615 615 return -EINVAL; ··· 622 622 623 623 if (!xdpf) 624 624 continue; 625 625 - err = xennet_xdp_xmit_one(dev, queue, xdpf); 626 626 - if (err) { 627 627 - xdp_return_frame_rx_napi(xdpf); 628 628 - drops++; 629 629 - } 625 625 + if (xennet_xdp_xmit_one(dev, queue, xdpf)) 626 626 + break; 627 627 + nxmit++; 630 628 } 631 629 spin_unlock_irqrestore(&queue->tx_lock, irq_flags); 632 630 633 633 - return n - drops; 631 631 + return nxmit; 634 632 } 635 633 636 634 ··· 873 875 get_page(pdata); 874 876 xdpf = xdp_convert_buff_to_frame(xdp); 875 877 err = xennet_xdp_xmit(queue->info->netdev, 1, &xdpf, 0); 876 876 - if (unlikely(err < 0)) 878 878 + if (unlikely(!err)) 879 879 + xdp_return_frame_rx_napi(xdpf); 880 880 + else if (unlikely(err < 0)) 877 881 trace_xdp_exception(queue->info->netdev, prog, act); 878 882 break; 879 883 case XDP_REDIRECT:

+1

include/linux/btf.h

reviewed

··· 9 9 #include <uapi/linux/bpf.h> 10 10 11 11 #define BTF_TYPE_EMIT(type) ((void)(type *)0) 12 12 + #define BTF_TYPE_EMIT_ENUM(enum_val) ((void)enum_val) 12 13 13 14 struct btf; 14 15 struct btf_member;

+2 -2

kernel/bpf/bpf_lsm.c

reviewed

··· 67 67 68 68 BTF_ID_LIST_SINGLE(bpf_bprm_opts_set_btf_ids, struct, linux_binprm) 69 69 70 70 - const static struct bpf_func_proto bpf_bprm_opts_set_proto = { 70 70 + static const struct bpf_func_proto bpf_bprm_opts_set_proto = { 71 71 .func = bpf_bprm_opts_set, 72 72 .gpl_only = false, 73 73 .ret_type = RET_INTEGER, ··· 88 88 89 89 BTF_ID_LIST_SINGLE(bpf_ima_inode_hash_btf_ids, struct, inode) 90 90 91 91 - const static struct bpf_func_proto bpf_ima_inode_hash_proto = { 91 91 + static const struct bpf_func_proto bpf_ima_inode_hash_proto = { 92 92 .func = bpf_ima_inode_hash, 93 93 .gpl_only = false, 94 94 .ret_type = RET_INTEGER,

+1 -1

kernel/bpf/bpf_task_storage.c

reviewed

··· 20 20 21 21 DEFINE_BPF_STORAGE_CACHE(task_cache); 22 22 23 23 - DEFINE_PER_CPU(int, bpf_task_storage_busy); 23 23 + static DEFINE_PER_CPU(int, bpf_task_storage_busy); 24 24 25 25 static void bpf_task_storage_lock(void) 26 26 {

+12 -18

kernel/bpf/devmap.c

reviewed

··· 329 329 static void bq_xmit_all(struct xdp_dev_bulk_queue *bq, u32 flags) 330 330 { 331 331 struct net_device *dev = bq->dev; 332 332 - int sent = 0, drops = 0, err = 0; 332 332 + int sent = 0, err = 0; 333 333 int i; 334 334 335 335 if (unlikely(!bq->count)) ··· 343 343 344 344 sent = dev->netdev_ops->ndo_xdp_xmit(dev, bq->count, bq->q, flags); 345 345 if (sent < 0) { 346 346 + /* If ndo_xdp_xmit fails with an errno, no frames have 347 347 + * been xmit'ed. 348 348 + */ 346 349 err = sent; 347 350 sent = 0; 348 348 - goto error; 349 351 } 350 350 - drops = bq->count - sent; 351 351 - out: 352 352 - bq->count = 0; 353 352 354 354 - trace_xdp_devmap_xmit(bq->dev_rx, dev, sent, drops, err); 355 355 - bq->dev_rx = NULL; 356 356 - __list_del_clearprev(&bq->flush_node); 357 357 - return; 358 358 - error: 359 359 - /* If ndo_xdp_xmit fails with an errno, no frames have been 360 360 - * xmit'ed and it's our responsibility to them free all. 353 353 + /* If not all frames have been transmitted, it is our 354 354 + * responsibility to free them 361 355 */ 362 362 - for (i = 0; i < bq->count; i++) { 363 363 - struct xdp_frame *xdpf = bq->q[i]; 356 356 + for (i = sent; unlikely(i < bq->count); i++) 357 357 + xdp_return_frame_rx_napi(bq->q[i]); 364 358 365 365 - xdp_return_frame_rx_napi(xdpf); 366 366 - drops++; 367 367 - } 368 368 - goto out; 359 359 + trace_xdp_devmap_xmit(bq->dev_rx, dev, sent, bq->count - sent, err); 360 360 + bq->dev_rx = NULL; 361 361 + bq->count = 0; 362 362 + __list_del_clearprev(&bq->flush_node); 369 363 } 370 364 371 365 /* __dev_flush is called from xdp_do_flush() which _must_ be signaled

+1 -1

kernel/bpf/hashtab.c

reviewed

··· 31 31 /* 32 32 * The bucket lock has two protection scopes: 33 33 * 34 34 - * 1) Serializing concurrent operations from BPF programs on differrent 34 34 + * 1) Serializing concurrent operations from BPF programs on different 35 35 * CPUs 36 36 * 37 37 * 2) Serializing concurrent operations from BPF programs and sys_bpf()

-1

kernel/bpf/verifier.c

reviewed

··· 11912 11912 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { 11913 11913 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; 11914 11914 const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; 11915 11915 - struct bpf_insn insn_buf[16]; 11916 11915 struct bpf_insn *patch = &insn_buf[0]; 11917 11916 bool issrc, isneg; 11918 11917 u32 off_reg;

+6

net/core/filter.c

reviewed

··· 4729 4729 sk->sk_prot->keepalive(sk, valbool); 4730 4730 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool); 4731 4731 break; 4732 4732 + case SO_REUSEPORT: 4733 4733 + sk->sk_reuseport = valbool; 4734 4734 + break; 4732 4735 default: 4733 4736 ret = -EINVAL; 4734 4737 } ··· 4900 4897 break; 4901 4898 case SO_BINDTOIFINDEX: 4902 4899 *((int *)optval) = sk->sk_bound_dev_if; 4900 4900 + break; 4901 4901 + case SO_REUSEPORT: 4902 4902 + *((int *)optval) = sk->sk_reuseport; 4903 4903 break; 4904 4904 default: 4905 4905 goto err_clear;

+12

net/ipv4/tcp.c

reviewed

··· 267 267 #include <linux/slab.h> 268 268 #include <linux/errqueue.h> 269 269 #include <linux/static_key.h> 270 270 + #include <linux/btf.h> 270 271 271 272 #include <net/icmp.h> 272 273 #include <net/inet_common.h> ··· 2587 2586 BUILD_BUG_ON((int)BPF_TCP_CLOSING != (int)TCP_CLOSING); 2588 2587 BUILD_BUG_ON((int)BPF_TCP_NEW_SYN_RECV != (int)TCP_NEW_SYN_RECV); 2589 2588 BUILD_BUG_ON((int)BPF_TCP_MAX_STATES != (int)TCP_MAX_STATES); 2589 2589 + 2590 2590 + /* bpf uapi header bpf.h defines an anonymous enum with values 2591 2591 + * BPF_TCP_* used by bpf programs. Currently gcc built vmlinux 2592 2592 + * is able to emit this enum in DWARF due to the above BUILD_BUG_ON. 2593 2593 + * But clang built vmlinux does not have this enum in DWARF 2594 2594 + * since clang removes the above code before generating IR/debuginfo. 2595 2595 + * Let us explicitly emit the type debuginfo to ensure the 2596 2596 + * above-mentioned anonymous enum in the vmlinux DWARF and hence BTF 2597 2597 + * regardless of which compiler is used. 2598 2598 + */ 2599 2599 + BTF_TYPE_EMIT_ENUM(BPF_TCP_ESTABLISHED); 2590 2600 2591 2601 if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_STATE_CB_FLAG)) 2592 2602 tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state);

+1 -1

samples/bpf/do_hbm_test.sh

reviewed

··· 10 10 Usage() { 11 11 echo "Script for testing HBM (Host Bandwidth Manager) framework." 12 12 echo "It creates a cgroup to use for testing and load a BPF program to limit" 13 13 - echo "egress or ingress bandwidht. It then uses iperf3 or netperf to create" 13 13 + echo "egress or ingress bandwidth. It then uses iperf3 or netperf to create" 14 14 echo "loads. The output is the goodput in Mbps (unless -D was used)." 15 15 echo "" 16 16 echo "USAGE: $name [out] [-b=<prog>|--bpf=<prog>] [-c=<cc>|--cc=<cc>]"

+59 -19

tools/bpf/bpftool/Documentation/bpftool-gen.rst

reviewed

··· 14 14 15 15 *OPTIONS* := { { **-j** | **--json** } [{ **-p** | **--pretty** }] } 16 16 17 17 - *COMMAND* := { **skeleton** | **help** } 17 17 + *COMMAND* := { **object** | **skeleton** | **help** } 18 18 19 19 GEN COMMANDS 20 20 ============= 21 21 22 22 - | **bpftool** **gen skeleton** *FILE* 22 22 + | **bpftool** **gen object** *OUTPUT_FILE* *INPUT_FILE* [*INPUT_FILE*...] 23 23 + | **bpftool** **gen skeleton** *FILE* [**name** *OBJECT_NAME*] 23 24 | **bpftool** **gen help** 24 25 25 26 DESCRIPTION 26 27 =========== 28 28 + **bpftool gen object** *OUTPUT_FILE* *INPUT_FILE* [*INPUT_FILE*...] 29 29 + Statically link (combine) together one or more *INPUT_FILE*'s 30 30 + into a single resulting *OUTPUT_FILE*. All the files involved 31 31 + are BPF ELF object files. 32 32 + 33 33 + The rules of BPF static linking are mostly the same as for 34 34 + user-space object files, but in addition to combining data 35 35 + and instruction sections, .BTF and .BTF.ext (if present in 36 36 + any of the input files) data are combined together. .BTF 37 37 + data is deduplicated, so all the common types across 38 38 + *INPUT_FILE*'s will only be represented once in the resulting 39 39 + BTF information. 40 40 + 41 41 + BPF static linking allows to partition BPF source code into 42 42 + individually compiled files that are then linked into 43 43 + a single resulting BPF object file, which can be used to 44 44 + generated BPF skeleton (with **gen skeleton** command) or 45 45 + passed directly into **libbpf** (using **bpf_object__open()** 46 46 + family of APIs). 47 47 + 27 48 **bpftool gen skeleton** *FILE* 28 49 Generate BPF skeleton C header file for a given *FILE*. 29 50 ··· 96 75 specific maps, programs, etc. 97 76 98 77 As part of skeleton, few custom functions are generated. 99 99 - Each of them is prefixed with object name, derived from 100 100 - object file name. I.e., if BPF object file name is 101 101 - **example.o**, BPF object name will be **example**. The 102 102 - following custom functions are provided in such case: 78 78 + Each of them is prefixed with object name. Object name can 79 79 + either be derived from object file name, i.e., if BPF object 80 80 + file name is **example.o**, BPF object name will be 81 81 + **example**. Object name can be also specified explicitly 82 82 + through **name** *OBJECT_NAME* parameter. The following 83 83 + custom functions are provided (assuming **example** as 84 84 + the object name): 103 85 104 86 - **example__open** and **example__open_opts**. 105 87 These functions are used to instantiate skeleton. It ··· 154 130 155 131 EXAMPLES 156 132 ======== 157 157 - **$ cat example.c** 133 133 + **$ cat example1.bpf.c** 158 134 159 135 :: 160 136 161 137 #include <stdbool.h> 162 138 #include <linux/ptrace.h> 163 139 #include <linux/bpf.h> 164 164 - #include "bpf_helpers.h" 140 140 + #include <bpf/bpf_helpers.h> 165 141 166 142 const volatile int param1 = 42; 167 143 bool global_flag = true; 168 144 struct { int x; } data = {}; 169 169 - 170 170 - struct { 171 171 - __uint(type, BPF_MAP_TYPE_HASH); 172 172 - __uint(max_entries, 128); 173 173 - __type(key, int); 174 174 - __type(value, long); 175 175 - } my_map SEC(".maps"); 176 145 177 146 SEC("raw_tp/sys_enter") 178 147 int handle_sys_enter(struct pt_regs *ctx) ··· 178 161 return 0; 179 162 } 180 163 164 164 + **$ cat example2.bpf.c** 165 165 + 166 166 + :: 167 167 + 168 168 + #include <linux/ptrace.h> 169 169 + #include <linux/bpf.h> 170 170 + #include <bpf/bpf_helpers.h> 171 171 + 172 172 + struct { 173 173 + __uint(type, BPF_MAP_TYPE_HASH); 174 174 + __uint(max_entries, 128); 175 175 + __type(key, int); 176 176 + __type(value, long); 177 177 + } my_map SEC(".maps"); 178 178 + 181 179 SEC("raw_tp/sys_exit") 182 180 int handle_sys_exit(struct pt_regs *ctx) 183 181 { ··· 202 170 } 203 171 204 172 This is example BPF application with two BPF programs and a mix of BPF maps 205 205 - and global variables. 173 173 + and global variables. Source code is split across two source code files. 206 174 207 207 - **$ bpftool gen skeleton example.o** 175 175 + **$ clang -target bpf -g example1.bpf.c -o example1.bpf.o** 176 176 + **$ clang -target bpf -g example2.bpf.c -o example2.bpf.o** 177 177 + **$ bpftool gen object example.bpf.o example1.bpf.o example2.bpf.o** 178 178 + 179 179 + This set of commands compiles *example1.bpf.c* and *example2.bpf.c* 180 180 + individually and then statically links respective object files into the final 181 181 + BPF ELF object file *example.bpf.o*. 182 182 + 183 183 + **$ bpftool gen skeleton example.bpf.o name example | tee example.skel.h** 208 184 209 185 :: 210 186 ··· 267 227 268 228 #endif /* __EXAMPLE_SKEL_H__ */ 269 229 270 270 - **$ cat example_user.c** 230 230 + **$ cat example.c** 271 231 272 232 :: 273 233 ··· 310 270 return err; 311 271 } 312 272 313 313 - **# ./example_user** 273 273 + **# ./example** 314 274 315 275 :: 316 276

+15 -2

tools/bpf/bpftool/bash-completion/bpftool

reviewed

··· 981 981 ;; 982 982 gen) 983 983 case $command in 984 984 - skeleton) 984 984 + object) 985 985 _filedir 986 986 + return 0 987 987 + ;; 988 988 + skeleton) 989 989 + case $prev in 990 990 + $command) 991 991 + _filedir 992 992 + return 0 993 993 + ;; 994 994 + *) 995 995 + _bpftool_once_attr 'name' 996 996 + return 0 997 997 + ;; 998 998 + esac 986 999 ;; 987 1000 *) 988 1001 [[ $prev == $object ]] && \ 989 989 - COMPREPLY=( $( compgen -W 'skeleton help' -- "$cur" ) ) 1002 1002 + COMPREPLY=( $( compgen -W 'object skeleton help' -- "$cur" ) ) 990 1003 ;; 991 1004 esac 992 1005 ;;

+3

tools/bpf/bpftool/btf.c

reviewed

··· 546 546 NEXT_ARG(); 547 547 if (argc < 1) { 548 548 p_err("expecting value for 'format' option\n"); 549 549 + err = -EINVAL; 549 550 goto done; 550 551 } 551 552 if (strcmp(*argv, "c") == 0) { ··· 556 555 } else { 557 556 p_err("unrecognized format specifier: '%s', possible values: raw, c", 558 557 *argv); 558 558 + err = -EINVAL; 559 559 goto done; 560 560 } 561 561 NEXT_ARG(); 562 562 } else { 563 563 p_err("unrecognized option: '%s'", *argv); 564 564 + err = -EINVAL; 564 565 goto done; 565 566 } 566 567 }

+69 -3

tools/bpf/bpftool/gen.c

reviewed

··· 273 273 char header_guard[MAX_OBJ_NAME_LEN + sizeof("__SKEL_H__")]; 274 274 size_t i, map_cnt = 0, prog_cnt = 0, file_sz, mmap_sz; 275 275 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts); 276 276 - char obj_name[MAX_OBJ_NAME_LEN], *obj_data; 276 276 + char obj_name[MAX_OBJ_NAME_LEN] = "", *obj_data; 277 277 struct bpf_object *obj = NULL; 278 278 const char *file, *ident; 279 279 struct bpf_program *prog; ··· 287 287 return -1; 288 288 } 289 289 file = GET_ARG(); 290 290 + 291 291 + while (argc) { 292 292 + if (!REQ_ARGS(2)) 293 293 + return -1; 294 294 + 295 295 + if (is_prefix(*argv, "name")) { 296 296 + NEXT_ARG(); 297 297 + 298 298 + if (obj_name[0] != '\0') { 299 299 + p_err("object name already specified"); 300 300 + return -1; 301 301 + } 302 302 + 303 303 + strncpy(obj_name, *argv, MAX_OBJ_NAME_LEN - 1); 304 304 + obj_name[MAX_OBJ_NAME_LEN - 1] = '\0'; 305 305 + } else { 306 306 + p_err("unknown arg %s", *argv); 307 307 + return -1; 308 308 + } 309 309 + 310 310 + NEXT_ARG(); 311 311 + } 290 312 291 313 if (argc) { 292 314 p_err("extra unknown arguments"); ··· 332 310 p_err("failed to mmap() %s: %s", file, strerror(errno)); 333 311 goto out; 334 312 } 335 335 - get_obj_name(obj_name, file); 313 313 + if (obj_name[0] == '\0') 314 314 + get_obj_name(obj_name, file); 336 315 opts.object_name = obj_name; 337 316 obj = bpf_object__open_mem(obj_data, file_sz, &opts); 338 317 if (IS_ERR(obj)) { ··· 614 591 return err; 615 592 } 616 593 594 594 + static int do_object(int argc, char **argv) 595 595 + { 596 596 + struct bpf_linker *linker; 597 597 + const char *output_file, *file; 598 598 + int err = 0; 599 599 + 600 600 + if (!REQ_ARGS(2)) { 601 601 + usage(); 602 602 + return -1; 603 603 + } 604 604 + 605 605 + output_file = GET_ARG(); 606 606 + 607 607 + linker = bpf_linker__new(output_file, NULL); 608 608 + if (!linker) { 609 609 + p_err("failed to create BPF linker instance"); 610 610 + return -1; 611 611 + } 612 612 + 613 613 + while (argc) { 614 614 + file = GET_ARG(); 615 615 + 616 616 + err = bpf_linker__add_file(linker, file); 617 617 + if (err) { 618 618 + p_err("failed to link '%s': %s (%d)", file, strerror(err), err); 619 619 + goto out; 620 620 + } 621 621 + } 622 622 + 623 623 + err = bpf_linker__finalize(linker); 624 624 + if (err) { 625 625 + p_err("failed to finalize ELF file: %s (%d)", strerror(err), err); 626 626 + goto out; 627 627 + } 628 628 + 629 629 + err = 0; 630 630 + out: 631 631 + bpf_linker__free(linker); 632 632 + return err; 633 633 + } 634 634 + 617 635 static int do_help(int argc, char **argv) 618 636 { 619 637 if (json_output) { ··· 663 599 } 664 600 665 601 fprintf(stderr, 666 666 - "Usage: %1$s %2$s skeleton FILE\n" 602 602 + "Usage: %1$s %2$s object OUTPUT_FILE INPUT_FILE [INPUT_FILE...]\n" 603 603 + " %1$s %2$s skeleton FILE [name OBJECT_NAME]\n" 667 604 " %1$s %2$s help\n" 668 605 "\n" 669 606 " " HELP_SPEC_OPTIONS "\n" ··· 675 610 } 676 611 677 612 static const struct cmd cmds[] = { 613 613 + { "object", do_object }, 678 614 { "skeleton", do_skeleton }, 679 615 { "help", do_help }, 680 616 { 0 }

+1 -2

tools/bpf/bpftool/main.c

reviewed

··· 276 276 int n_argc; 277 277 FILE *fp; 278 278 char *cp; 279 279 - int err; 279 279 + int err = 0; 280 280 int i; 281 281 282 282 if (argc < 2) { ··· 370 370 } else { 371 371 if (!json_output) 372 372 printf("processed %d commands\n", lines); 373 373 - err = 0; 374 373 } 375 374 err_close: 376 375 if (fp != stdin)

+1 -1

tools/bpf/bpftool/map.c

reviewed

··· 100 100 void *value) 101 101 { 102 102 __u32 value_id; 103 103 - int ret; 103 103 + int ret = 0; 104 104 105 105 /* start of key-value pair */ 106 106 jsonw_start_object(d->jw);

+1 -1

tools/lib/bpf/Build

reviewed

··· 1 1 libbpf-y := libbpf.o bpf.o nlattr.o btf.o libbpf_errno.o str_error.o \ 2 2 netlink.o bpf_prog_linfo.o libbpf_probes.o xsk.o hashmap.o \ 3 3 - btf_dump.o ringbuf.o 3 3 + btf_dump.o ringbuf.o strset.o linker.o

-1

tools/lib/bpf/Makefile

reviewed

··· 228 228 $(call do_install,bpf.h,$(prefix)/include/bpf,644); \ 229 229 $(call do_install,libbpf.h,$(prefix)/include/bpf,644); \ 230 230 $(call do_install,btf.h,$(prefix)/include/bpf,644); \ 231 231 - $(call do_install,libbpf_util.h,$(prefix)/include/bpf,644); \ 232 231 $(call do_install,libbpf_common.h,$(prefix)/include/bpf,644); \ 233 232 $(call do_install,xsk.h,$(prefix)/include/bpf,644); \ 234 233 $(call do_install,bpf_helpers.h,$(prefix)/include/bpf,644); \

+18 -3

tools/lib/bpf/bpf_helpers.h

reviewed

··· 29 29 */ 30 30 #define SEC(NAME) __attribute__((section(NAME), used)) 31 31 32 32 - #ifndef __always_inline 32 32 + /* Avoid 'linux/stddef.h' definition of '__always_inline'. */ 33 33 + #undef __always_inline 33 34 #define __always_inline inline __attribute__((always_inline)) 34 34 - #endif 35 35 + 35 36 #ifndef __noinline 36 37 #define __noinline __attribute__((noinline)) 37 38 #endif ··· 40 39 #define __weak __attribute__((weak)) 41 40 #endif 42 41 42 42 + /* When utilizing vmlinux.h with BPF CO-RE, user BPF programs can't include 43 43 + * any system-level headers (such as stddef.h, linux/version.h, etc), and 44 44 + * commonly-used macros like NULL and KERNEL_VERSION aren't available through 45 45 + * vmlinux.h. This just adds unnecessary hurdles and forces users to re-define 46 46 + * them on their own. So as a convenience, provide such definitions here. 47 47 + */ 48 48 + #ifndef NULL 49 49 + #define NULL ((void *)0) 50 50 + #endif 51 51 + 52 52 + #ifndef KERNEL_VERSION 53 53 + #define KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + ((c) > 255 ? 255 : (c)) 54 54 + #endif 55 55 + 43 56 /* 44 44 - * Helper macro to manipulate data structures 57 57 + * Helper macros to manipulate data structures 45 58 */ 46 59 #ifndef offsetof 47 60 #define offsetof(TYPE, MEMBER) ((unsigned long)&((TYPE *)0)->MEMBER)

+341 -377

tools/lib/bpf/btf.c

reviewed

··· 21 21 #include "libbpf.h" 22 22 #include "libbpf_internal.h" 23 23 #include "hashmap.h" 24 24 + #include "strset.h" 24 25 25 26 #define BTF_MAX_NR_TYPES 0x7fffffffU 26 27 #define BTF_MAX_STR_OFFSET 0x7fffffffU ··· 68 67 * | | | 69 68 * hdr | | 70 69 * types_data----+ | 71 71 - * strs_data------------------+ 70 70 + * strset__data(strs_set)-----+ 72 71 * 73 72 * +----------+---------+-----------+ 74 73 * | Header | Types | Strings | ··· 106 105 */ 107 106 int start_str_off; 108 107 108 108 + /* only one of strs_data or strs_set can be non-NULL, depending on 109 109 + * whether BTF is in a modifiable state (strs_set is used) or not 110 110 + * (strs_data points inside raw_data) 111 111 + */ 109 112 void *strs_data; 110 110 - size_t strs_data_cap; /* used size stored in hdr->str_len */ 111 111 - 112 112 - /* lookup index for each unique string in strings section */ 113 113 - struct hashmap *strs_hash; 113 113 + /* a set of unique strings */ 114 114 + struct strset *strs_set; 114 115 /* whether strings are already deduplicated */ 115 116 bool strs_deduped; 116 116 - /* extra indirection layer to make strings hashmap work with stable 117 117 - * string offsets and ability to transparently choose between 118 118 - * btf->strs_data or btf_dedup->strs_data as a source of strings. 119 119 - * This is used for BTF strings dedup to transfer deduplicated strings 120 120 - * data back to struct btf without re-building strings index. 121 121 - */ 122 122 - void **strs_data_ptr; 123 117 124 118 /* BTF object FD, if loaded into kernel */ 125 119 int fd; ··· 138 142 * On success, memory pointer to the beginning of unused memory is returned. 139 143 * On error, NULL is returned. 140 144 */ 141 141 - void *btf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz, 142 142 - size_t cur_cnt, size_t max_cnt, size_t add_cnt) 145 145 + void *libbpf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz, 146 146 + size_t cur_cnt, size_t max_cnt, size_t add_cnt) 143 147 { 144 148 size_t new_cnt; 145 149 void *new_data; ··· 175 179 /* Ensure given dynamically allocated memory region has enough allocated space 176 180 * to accommodate *need_cnt* elements of size *elem_sz* bytes each 177 181 */ 178 178 - int btf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt) 182 182 + int libbpf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt) 179 183 { 180 184 void *p; 181 185 182 186 if (need_cnt <= *cap_cnt) 183 187 return 0; 184 188 185 185 - p = btf_add_mem(data, cap_cnt, elem_sz, *cap_cnt, SIZE_MAX, need_cnt - *cap_cnt); 189 189 + p = libbpf_add_mem(data, cap_cnt, elem_sz, *cap_cnt, SIZE_MAX, need_cnt - *cap_cnt); 186 190 if (!p) 187 191 return -ENOMEM; 188 192 ··· 193 197 { 194 198 __u32 *p; 195 199 196 196 - p = btf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32), 197 197 - btf->nr_types, BTF_MAX_NR_TYPES, 1); 200 200 + p = libbpf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32), 201 201 + btf->nr_types, BTF_MAX_NR_TYPES, 1); 198 202 if (!p) 199 203 return -ENOMEM; 200 204 ··· 431 435 } 432 436 433 437 /* internal helper returning non-const pointer to a type */ 434 434 - static struct btf_type *btf_type_by_id(struct btf *btf, __u32 type_id) 438 438 + struct btf_type *btf_type_by_id(struct btf *btf, __u32 type_id) 435 439 { 436 440 if (type_id == 0) 437 441 return &btf_void; ··· 734 738 */ 735 739 free(btf->hdr); 736 740 free(btf->types_data); 737 737 - free(btf->strs_data); 741 741 + strset__free(btf->strs_set); 738 742 } 739 743 free(btf->raw_data); 740 744 free(btf->raw_data_swapped); ··· 1242 1246 btf->fd = fd; 1243 1247 } 1244 1248 1249 1249 + static const void *btf_strs_data(const struct btf *btf) 1250 1250 + { 1251 1251 + return btf->strs_data ? btf->strs_data : strset__data(btf->strs_set); 1252 1252 + } 1253 1253 + 1245 1254 static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian) 1246 1255 { 1247 1256 struct btf_header *hdr = btf->hdr; ··· 1287 1286 } 1288 1287 p += hdr->type_len; 1289 1288 1290 1290 - memcpy(p, btf->strs_data, hdr->str_len); 1289 1289 + memcpy(p, btf_strs_data(btf), hdr->str_len); 1291 1290 p += hdr->str_len; 1292 1291 1293 1292 *size = data_sz; ··· 1321 1320 if (offset < btf->start_str_off) 1322 1321 return btf__str_by_offset(btf->base_btf, offset); 1323 1322 else if (offset - btf->start_str_off < btf->hdr->str_len) 1324 1324 - return btf->strs_data + (offset - btf->start_str_off); 1323 1323 + return btf_strs_data(btf) + (offset - btf->start_str_off); 1325 1324 else 1326 1325 return NULL; 1327 1326 } ··· 1475 1474 return 0; 1476 1475 } 1477 1476 1478 1478 - static size_t strs_hash_fn(const void *key, void *ctx) 1479 1479 - { 1480 1480 - const struct btf *btf = ctx; 1481 1481 - const char *strs = *btf->strs_data_ptr; 1482 1482 - const char *str = strs + (long)key; 1483 1483 - 1484 1484 - return str_hash(str); 1485 1485 - } 1486 1486 - 1487 1487 - static bool strs_hash_equal_fn(const void *key1, const void *key2, void *ctx) 1488 1488 - { 1489 1489 - const struct btf *btf = ctx; 1490 1490 - const char *strs = *btf->strs_data_ptr; 1491 1491 - const char *str1 = strs + (long)key1; 1492 1492 - const char *str2 = strs + (long)key2; 1493 1493 - 1494 1494 - return strcmp(str1, str2) == 0; 1495 1495 - } 1496 1496 - 1497 1477 static void btf_invalidate_raw_data(struct btf *btf) 1498 1478 { 1499 1479 if (btf->raw_data) { ··· 1493 1511 */ 1494 1512 static int btf_ensure_modifiable(struct btf *btf) 1495 1513 { 1496 1496 - void *hdr, *types, *strs, *strs_end, *s; 1497 1497 - struct hashmap *hash = NULL; 1498 1498 - long off; 1499 1499 - int err; 1514 1514 + void *hdr, *types; 1515 1515 + struct strset *set = NULL; 1516 1516 + int err = -ENOMEM; 1500 1517 1501 1518 if (btf_is_modifiable(btf)) { 1502 1519 /* any BTF modification invalidates raw_data */ ··· 1506 1525 /* split raw data into three memory regions */ 1507 1526 hdr = malloc(btf->hdr->hdr_len); 1508 1527 types = malloc(btf->hdr->type_len); 1509 1509 - strs = malloc(btf->hdr->str_len); 1510 1510 - if (!hdr || !types || !strs) 1528 1528 + if (!hdr || !types) 1511 1529 goto err_out; 1512 1530 1513 1531 memcpy(hdr, btf->hdr, btf->hdr->hdr_len); 1514 1532 memcpy(types, btf->types_data, btf->hdr->type_len); 1515 1515 - memcpy(strs, btf->strs_data, btf->hdr->str_len); 1516 1516 - 1517 1517 - /* make hashmap below use btf->strs_data as a source of strings */ 1518 1518 - btf->strs_data_ptr = &btf->strs_data; 1519 1533 1520 1534 /* build lookup index for all strings */ 1521 1521 - hash = hashmap__new(strs_hash_fn, strs_hash_equal_fn, btf); 1522 1522 - if (IS_ERR(hash)) { 1523 1523 - err = PTR_ERR(hash); 1524 1524 - hash = NULL; 1535 1535 + set = strset__new(BTF_MAX_STR_OFFSET, btf->strs_data, btf->hdr->str_len); 1536 1536 + if (IS_ERR(set)) { 1537 1537 + err = PTR_ERR(set); 1525 1538 goto err_out; 1526 1526 - } 1527 1527 - 1528 1528 - strs_end = strs + btf->hdr->str_len; 1529 1529 - for (off = 0, s = strs; s < strs_end; off += strlen(s) + 1, s = strs + off) { 1530 1530 - /* hashmap__add() returns EEXIST if string with the same 1531 1531 - * content already is in the hash map 1532 1532 - */ 1533 1533 - err = hashmap__add(hash, (void *)off, (void *)off); 1534 1534 - if (err == -EEXIST) 1535 1535 - continue; /* duplicate */ 1536 1536 - if (err) 1537 1537 - goto err_out; 1538 1539 } 1539 1540 1540 1541 /* only when everything was successful, update internal state */ 1541 1542 btf->hdr = hdr; 1542 1543 btf->types_data = types; 1543 1544 btf->types_data_cap = btf->hdr->type_len; 1544 1544 - btf->strs_data = strs; 1545 1545 - btf->strs_data_cap = btf->hdr->str_len; 1546 1546 - btf->strs_hash = hash; 1545 1545 + btf->strs_data = NULL; 1546 1546 + btf->strs_set = set; 1547 1547 /* if BTF was created from scratch, all strings are guaranteed to be 1548 1548 * unique and deduplicated 1549 1549 */ ··· 1539 1577 return 0; 1540 1578 1541 1579 err_out: 1542 1542 - hashmap__free(hash); 1580 1580 + strset__free(set); 1543 1581 free(hdr); 1544 1582 free(types); 1545 1545 - free(strs); 1546 1546 - return -ENOMEM; 1547 1547 - } 1548 1548 - 1549 1549 - static void *btf_add_str_mem(struct btf *btf, size_t add_sz) 1550 1550 - { 1551 1551 - return btf_add_mem(&btf->strs_data, &btf->strs_data_cap, 1, 1552 1552 - btf->hdr->str_len, BTF_MAX_STR_OFFSET, add_sz); 1583 1583 + return err; 1553 1584 } 1554 1585 1555 1586 /* Find an offset in BTF string section that corresponds to a given string *s*. ··· 1553 1598 */ 1554 1599 int btf__find_str(struct btf *btf, const char *s) 1555 1600 { 1556 1556 - long old_off, new_off, len; 1557 1557 - void *p; 1601 1601 + int off; 1558 1602 1559 1603 if (btf->base_btf) { 1560 1560 - int ret; 1561 1561 - 1562 1562 - ret = btf__find_str(btf->base_btf, s); 1563 1563 - if (ret != -ENOENT) 1564 1564 - return ret; 1604 1604 + off = btf__find_str(btf->base_btf, s); 1605 1605 + if (off != -ENOENT) 1606 1606 + return off; 1565 1607 } 1566 1608 1567 1609 /* BTF needs to be in a modifiable state to build string lookup index */ 1568 1610 if (btf_ensure_modifiable(btf)) 1569 1611 return -ENOMEM; 1570 1612 1571 1571 - /* see btf__add_str() for why we do this */ 1572 1572 - len = strlen(s) + 1; 1573 1573 - p = btf_add_str_mem(btf, len); 1574 1574 - if (!p) 1575 1575 - return -ENOMEM; 1613 1613 + off = strset__find_str(btf->strs_set, s); 1614 1614 + if (off < 0) 1615 1615 + return off; 1576 1616 1577 1577 - new_off = btf->hdr->str_len; 1578 1578 - memcpy(p, s, len); 1579 1579 - 1580 1580 - if (hashmap__find(btf->strs_hash, (void *)new_off, (void **)&old_off)) 1581 1581 - return btf->start_str_off + old_off; 1582 1582 - 1583 1583 - return -ENOENT; 1617 1617 + return btf->start_str_off + off; 1584 1618 } 1585 1619 1586 1620 /* Add a string s to the BTF string section. ··· 1579 1635 */ 1580 1636 int btf__add_str(struct btf *btf, const char *s) 1581 1637 { 1582 1582 - long old_off, new_off, len; 1583 1583 - void *p; 1584 1584 - int err; 1638 1638 + int off; 1585 1639 1586 1640 if (btf->base_btf) { 1587 1587 - int ret; 1588 1588 - 1589 1589 - ret = btf__find_str(btf->base_btf, s); 1590 1590 - if (ret != -ENOENT) 1591 1591 - return ret; 1641 1641 + off = btf__find_str(btf->base_btf, s); 1642 1642 + if (off != -ENOENT) 1643 1643 + return off; 1592 1644 } 1593 1645 1594 1646 if (btf_ensure_modifiable(btf)) 1595 1647 return -ENOMEM; 1596 1648 1597 1597 - /* Hashmap keys are always offsets within btf->strs_data, so to even 1598 1598 - * look up some string from the "outside", we need to first append it 1599 1599 - * at the end, so that it can be addressed with an offset. Luckily, 1600 1600 - * until btf->hdr->str_len is incremented, that string is just a piece 1601 1601 - * of garbage for the rest of BTF code, so no harm, no foul. On the 1602 1602 - * other hand, if the string is unique, it's already appended and 1603 1603 - * ready to be used, only a simple btf->hdr->str_len increment away. 1604 1604 - */ 1605 1605 - len = strlen(s) + 1; 1606 1606 - p = btf_add_str_mem(btf, len); 1607 1607 - if (!p) 1608 1608 - return -ENOMEM; 1649 1649 + off = strset__add_str(btf->strs_set, s); 1650 1650 + if (off < 0) 1651 1651 + return off; 1609 1652 1610 1610 - new_off = btf->hdr->str_len; 1611 1611 - memcpy(p, s, len); 1653 1653 + btf->hdr->str_len = strset__data_size(btf->strs_set); 1612 1654 1613 1613 - /* Now attempt to add the string, but only if the string with the same 1614 1614 - * contents doesn't exist already (HASHMAP_ADD strategy). If such 1615 1615 - * string exists, we'll get its offset in old_off (that's old_key). 1616 1616 - */ 1617 1617 - err = hashmap__insert(btf->strs_hash, (void *)new_off, (void *)new_off, 1618 1618 - HASHMAP_ADD, (const void **)&old_off, NULL); 1619 1619 - if (err == -EEXIST) 1620 1620 - return btf->start_str_off + old_off; /* duplicated string, return existing offset */ 1621 1621 - if (err) 1622 1622 - return err; 1623 1623 - 1624 1624 - btf->hdr->str_len += len; /* new unique string, adjust data length */ 1625 1625 - return btf->start_str_off + new_off; 1655 1655 + return btf->start_str_off + off; 1626 1656 } 1627 1657 1628 1658 static void *btf_add_type_mem(struct btf *btf, size_t add_sz) 1629 1659 { 1630 1630 - return btf_add_mem(&btf->types_data, &btf->types_data_cap, 1, 1631 1631 - btf->hdr->type_len, UINT_MAX, add_sz); 1660 1660 + return libbpf_add_mem(&btf->types_data, &btf->types_data_cap, 1, 1661 1661 + btf->hdr->type_len, UINT_MAX, add_sz); 1632 1662 } 1633 1663 1634 1664 static __u32 btf_type_info(int kind, int vlen, int kflag) ··· 1627 1709 btf->hdr->str_off += data_sz; 1628 1710 btf->nr_types++; 1629 1711 return btf->start_id + btf->nr_types - 1; 1712 1712 + } 1713 1713 + 1714 1714 + struct btf_pipe { 1715 1715 + const struct btf *src; 1716 1716 + struct btf *dst; 1717 1717 + }; 1718 1718 + 1719 1719 + static int btf_rewrite_str(__u32 *str_off, void *ctx) 1720 1720 + { 1721 1721 + struct btf_pipe *p = ctx; 1722 1722 + int off; 1723 1723 + 1724 1724 + if (!*str_off) /* nothing to do for empty strings */ 1725 1725 + return 0; 1726 1726 + 1727 1727 + off = btf__add_str(p->dst, btf__str_by_offset(p->src, *str_off)); 1728 1728 + if (off < 0) 1729 1729 + return off; 1730 1730 + 1731 1731 + *str_off = off; 1732 1732 + return 0; 1733 1733 + } 1734 1734 + 1735 1735 + int btf__add_type(struct btf *btf, const struct btf *src_btf, const struct btf_type *src_type) 1736 1736 + { 1737 1737 + struct btf_pipe p = { .src = src_btf, .dst = btf }; 1738 1738 + struct btf_type *t; 1739 1739 + int sz, err; 1740 1740 + 1741 1741 + sz = btf_type_size(src_type); 1742 1742 + if (sz < 0) 1743 1743 + return sz; 1744 1744 + 1745 1745 + /* deconstruct BTF, if necessary, and invalidate raw_data */ 1746 1746 + if (btf_ensure_modifiable(btf)) 1747 1747 + return -ENOMEM; 1748 1748 + 1749 1749 + t = btf_add_type_mem(btf, sz); 1750 1750 + if (!t) 1751 1751 + return -ENOMEM; 1752 1752 + 1753 1753 + memcpy(t, src_type, sz); 1754 1754 + 1755 1755 + err = btf_type_visit_str_offs(t, btf_rewrite_str, &p); 1756 1756 + if (err) 1757 1757 + return err; 1758 1758 + 1759 1759 + return btf_commit_type(btf, sz); 1630 1760 } 1631 1761 1632 1762 /* ··· 2982 3016 /* Various option modifying behavior of algorithm */ 2983 3017 struct btf_dedup_opts opts; 2984 3018 /* temporary strings deduplication state */ 2985 2985 - void *strs_data; 2986 2986 - size_t strs_cap; 2987 2987 - size_t strs_len; 2988 2988 - struct hashmap* strs_hash; 3019 3019 + struct strset *strs_set; 2989 3020 }; 2990 3021 2991 3022 static long hash_combine(long h, long value) ··· 3118 3155 return d; 3119 3156 } 3120 3157 3121 3121 - typedef int (*str_off_fn_t)(__u32 *str_off_ptr, void *ctx); 3122 3122 - 3123 3158 /* 3124 3159 * Iterate over all possible places in .BTF and .BTF.ext that can reference 3125 3160 * string and pass pointer to it to a provided callback `fn`. 3126 3161 */ 3127 3127 - static int btf_for_each_str_off(struct btf_dedup *d, str_off_fn_t fn, void *ctx) 3162 3162 + static int btf_for_each_str_off(struct btf_dedup *d, str_off_visit_fn fn, void *ctx) 3128 3163 { 3129 3129 - void *line_data_cur, *line_data_end; 3130 3130 - int i, j, r, rec_size; 3131 3131 - struct btf_type *t; 3164 3164 + int i, r; 3132 3165 3133 3166 for (i = 0; i < d->btf->nr_types; i++) { 3134 3134 - t = btf_type_by_id(d->btf, d->btf->start_id + i); 3135 3135 - r = fn(&t->name_off, ctx); 3167 3167 + struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i); 3168 3168 + 3169 3169 + r = btf_type_visit_str_offs(t, fn, ctx); 3136 3170 if (r) 3137 3171 return r; 3138 3138 - 3139 3139 - switch (btf_kind(t)) { 3140 3140 - case BTF_KIND_STRUCT: 3141 3141 - case BTF_KIND_UNION: { 3142 3142 - struct btf_member *m = btf_members(t); 3143 3143 - __u16 vlen = btf_vlen(t); 3144 3144 - 3145 3145 - for (j = 0; j < vlen; j++) { 3146 3146 - r = fn(&m->name_off, ctx); 3147 3147 - if (r) 3148 3148 - return r; 3149 3149 - m++; 3150 3150 - } 3151 3151 - break; 3152 3152 - } 3153 3153 - case BTF_KIND_ENUM: { 3154 3154 - struct btf_enum *m = btf_enum(t); 3155 3155 - __u16 vlen = btf_vlen(t); 3156 3156 - 3157 3157 - for (j = 0; j < vlen; j++) { 3158 3158 - r = fn(&m->name_off, ctx); 3159 3159 - if (r) 3160 3160 - return r; 3161 3161 - m++; 3162 3162 - } 3163 3163 - break; 3164 3164 - } 3165 3165 - case BTF_KIND_FUNC_PROTO: { 3166 3166 - struct btf_param *m = btf_params(t); 3167 3167 - __u16 vlen = btf_vlen(t); 3168 3168 - 3169 3169 - for (j = 0; j < vlen; j++) { 3170 3170 - r = fn(&m->name_off, ctx); 3171 3171 - if (r) 3172 3172 - return r; 3173 3173 - m++; 3174 3174 - } 3175 3175 - break; 3176 3176 - } 3177 3177 - default: 3178 3178 - break; 3179 3179 - } 3180 3172 } 3181 3173 3182 3174 if (!d->btf_ext) 3183 3175 return 0; 3184 3176 3185 3185 - line_data_cur = d->btf_ext->line_info.info; 3186 3186 - line_data_end = d->btf_ext->line_info.info + d->btf_ext->line_info.len; 3187 3187 - rec_size = d->btf_ext->line_info.rec_size; 3188 3188 - 3189 3189 - while (line_data_cur < line_data_end) { 3190 3190 - struct btf_ext_info_sec *sec = line_data_cur; 3191 3191 - struct bpf_line_info_min *line_info; 3192 3192 - __u32 num_info = sec->num_info; 3193 3193 - 3194 3194 - r = fn(&sec->sec_name_off, ctx); 3195 3195 - if (r) 3196 3196 - return r; 3197 3197 - 3198 3198 - line_data_cur += sizeof(struct btf_ext_info_sec); 3199 3199 - for (i = 0; i < num_info; i++) { 3200 3200 - line_info = line_data_cur; 3201 3201 - r = fn(&line_info->file_name_off, ctx); 3202 3202 - if (r) 3203 3203 - return r; 3204 3204 - r = fn(&line_info->line_off, ctx); 3205 3205 - if (r) 3206 3206 - return r; 3207 3207 - line_data_cur += rec_size; 3208 3208 - } 3209 3209 - } 3177 3177 + r = btf_ext_visit_str_offs(d->btf_ext, fn, ctx); 3178 3178 + if (r) 3179 3179 + return r; 3210 3180 3211 3181 return 0; 3212 3182 } ··· 3148 3252 { 3149 3253 struct btf_dedup *d = ctx; 3150 3254 __u32 str_off = *str_off_ptr; 3151 3151 - long old_off, new_off, len; 3152 3255 const char *s; 3153 3153 - void *p; 3154 3154 - int err; 3256 3256 + int off, err; 3155 3257 3156 3258 /* don't touch empty string or string in main BTF */ 3157 3259 if (str_off == 0 || str_off < d->btf->start_str_off) ··· 3166 3272 return err; 3167 3273 } 3168 3274 3169 3169 - len = strlen(s) + 1; 3275 3275 + off = strset__add_str(d->strs_set, s); 3276 3276 + if (off < 0) 3277 3277 + return off; 3170 3278 3171 3171 - new_off = d->strs_len; 3172 3172 - p = btf_add_mem(&d->strs_data, &d->strs_cap, 1, new_off, BTF_MAX_STR_OFFSET, len); 3173 3173 - if (!p) 3174 3174 - return -ENOMEM; 3175 3175 - 3176 3176 - memcpy(p, s, len); 3177 3177 - 3178 3178 - /* Now attempt to add the string, but only if the string with the same 3179 3179 - * contents doesn't exist already (HASHMAP_ADD strategy). If such 3180 3180 - * string exists, we'll get its offset in old_off (that's old_key). 3181 3181 - */ 3182 3182 - err = hashmap__insert(d->strs_hash, (void *)new_off, (void *)new_off, 3183 3183 - HASHMAP_ADD, (const void **)&old_off, NULL); 3184 3184 - if (err == -EEXIST) { 3185 3185 - *str_off_ptr = d->btf->start_str_off + old_off; 3186 3186 - } else if (err) { 3187 3187 - return err; 3188 3188 - } else { 3189 3189 - *str_off_ptr = d->btf->start_str_off + new_off; 3190 3190 - d->strs_len += len; 3191 3191 - } 3279 3279 + *str_off_ptr = d->btf->start_str_off + off; 3192 3280 return 0; 3193 3281 } 3194 3282 ··· 3187 3311 */ 3188 3312 static int btf_dedup_strings(struct btf_dedup *d) 3189 3313 { 3190 3190 - char *s; 3191 3314 int err; 3192 3315 3193 3316 if (d->btf->strs_deduped) 3194 3317 return 0; 3195 3318 3196 3196 - /* temporarily switch to use btf_dedup's strs_data for strings for hash 3197 3197 - * functions; later we'll just transfer hashmap to struct btf as is, 3198 3198 - * along the strs_data 3199 3199 - */ 3200 3200 - d->btf->strs_data_ptr = &d->strs_data; 3201 3201 - 3202 3202 - d->strs_hash = hashmap__new(strs_hash_fn, strs_hash_equal_fn, d->btf); 3203 3203 - if (IS_ERR(d->strs_hash)) { 3204 3204 - err = PTR_ERR(d->strs_hash); 3205 3205 - d->strs_hash = NULL; 3319 3319 + d->strs_set = strset__new(BTF_MAX_STR_OFFSET, NULL, 0); 3320 3320 + if (IS_ERR(d->strs_set)) { 3321 3321 + err = PTR_ERR(d->strs_set); 3206 3322 goto err_out; 3207 3323 } 3208 3324 3209 3325 if (!d->btf->base_btf) { 3210 3210 - s = btf_add_mem(&d->strs_data, &d->strs_cap, 1, d->strs_len, BTF_MAX_STR_OFFSET, 1); 3211 3211 - if (!s) 3212 3212 - return -ENOMEM; 3213 3213 - /* initial empty string */ 3214 3214 - s[0] = 0; 3215 3215 - d->strs_len = 1; 3216 3216 - 3217 3326 /* insert empty string; we won't be looking it up during strings 3218 3327 * dedup, but it's good to have it for generic BTF string lookups 3219 3328 */ 3220 3220 - err = hashmap__insert(d->strs_hash, (void *)0, (void *)0, 3221 3221 - HASHMAP_ADD, NULL, NULL); 3222 3222 - if (err) 3329 3329 + err = strset__add_str(d->strs_set, ""); 3330 3330 + if (err < 0) 3223 3331 goto err_out; 3224 3332 } 3225 3333 ··· 3213 3353 goto err_out; 3214 3354 3215 3355 /* replace BTF string data and hash with deduped ones */ 3216 3216 - free(d->btf->strs_data); 3217 3217 - hashmap__free(d->btf->strs_hash); 3218 3218 - d->btf->strs_data = d->strs_data; 3219 3219 - d->btf->strs_data_cap = d->strs_cap; 3220 3220 - d->btf->hdr->str_len = d->strs_len; 3221 3221 - d->btf->strs_hash = d->strs_hash; 3222 3222 - /* now point strs_data_ptr back to btf->strs_data */ 3223 3223 - d->btf->strs_data_ptr = &d->btf->strs_data; 3224 3224 - 3225 3225 - d->strs_data = d->strs_hash = NULL; 3226 3226 - d->strs_len = d->strs_cap = 0; 3356 3356 + strset__free(d->btf->strs_set); 3357 3357 + d->btf->hdr->str_len = strset__data_size(d->strs_set); 3358 3358 + d->btf->strs_set = d->strs_set; 3359 3359 + d->strs_set = NULL; 3227 3360 d->btf->strs_deduped = true; 3228 3361 return 0; 3229 3362 3230 3363 err_out: 3231 3231 - free(d->strs_data); 3232 3232 - hashmap__free(d->strs_hash); 3233 3233 - d->strs_data = d->strs_hash = NULL; 3234 3234 - d->strs_len = d->strs_cap = 0; 3235 3235 - 3236 3236 - /* restore strings pointer for existing d->btf->strs_hash back */ 3237 3237 - d->btf->strs_data_ptr = &d->strs_data; 3364 3364 + strset__free(d->strs_set); 3365 3365 + d->strs_set = NULL; 3238 3366 3239 3367 return err; 3240 3368 } ··· 4346 4498 * then mapping it to a deduplicated type ID, stored in btf_dedup->hypot_map, 4347 4499 * which is populated during compaction phase. 4348 4500 */ 4349 4349 - static int btf_dedup_remap_type_id(struct btf_dedup *d, __u32 type_id) 4501 4501 + static int btf_dedup_remap_type_id(__u32 *type_id, void *ctx) 4350 4502 { 4503 4503 + struct btf_dedup *d = ctx; 4351 4504 __u32 resolved_type_id, new_type_id; 4352 4505 4353 4353 - resolved_type_id = resolve_type_id(d, type_id); 4506 4506 + resolved_type_id = resolve_type_id(d, *type_id); 4354 4507 new_type_id = d->hypot_map[resolved_type_id]; 4355 4508 if (new_type_id > BTF_MAX_NR_TYPES) 4356 4509 return -EINVAL; 4357 4357 - return new_type_id; 4510 4510 + 4511 4511 + *type_id = new_type_id; 4512 4512 + return 0; 4358 4513 } 4359 4514 4360 4515 /* ··· 4370 4519 * referenced from any BTF type (e.g., struct fields, func proto args, etc) to 4371 4520 * their final deduped type IDs. 4372 4521 */ 4373 4373 - static int btf_dedup_remap_type(struct btf_dedup *d, __u32 type_id) 4374 4374 - { 4375 4375 - struct btf_type *t = btf_type_by_id(d->btf, type_id); 4376 4376 - int i, r; 4377 4377 - 4378 4378 - switch (btf_kind(t)) { 4379 4379 - case BTF_KIND_INT: 4380 4380 - case BTF_KIND_ENUM: 4381 4381 - case BTF_KIND_FLOAT: 4382 4382 - break; 4383 4383 - 4384 4384 - case BTF_KIND_FWD: 4385 4385 - case BTF_KIND_CONST: 4386 4386 - case BTF_KIND_VOLATILE: 4387 4387 - case BTF_KIND_RESTRICT: 4388 4388 - case BTF_KIND_PTR: 4389 4389 - case BTF_KIND_TYPEDEF: 4390 4390 - case BTF_KIND_FUNC: 4391 4391 - case BTF_KIND_VAR: 4392 4392 - r = btf_dedup_remap_type_id(d, t->type); 4393 4393 - if (r < 0) 4394 4394 - return r; 4395 4395 - t->type = r; 4396 4396 - break; 4397 4397 - 4398 4398 - case BTF_KIND_ARRAY: { 4399 4399 - struct btf_array *arr_info = btf_array(t); 4400 4400 - 4401 4401 - r = btf_dedup_remap_type_id(d, arr_info->type); 4402 4402 - if (r < 0) 4403 4403 - return r; 4404 4404 - arr_info->type = r; 4405 4405 - r = btf_dedup_remap_type_id(d, arr_info->index_type); 4406 4406 - if (r < 0) 4407 4407 - return r; 4408 4408 - arr_info->index_type = r; 4409 4409 - break; 4410 4410 - } 4411 4411 - 4412 4412 - case BTF_KIND_STRUCT: 4413 4413 - case BTF_KIND_UNION: { 4414 4414 - struct btf_member *member = btf_members(t); 4415 4415 - __u16 vlen = btf_vlen(t); 4416 4416 - 4417 4417 - for (i = 0; i < vlen; i++) { 4418 4418 - r = btf_dedup_remap_type_id(d, member->type); 4419 4419 - if (r < 0) 4420 4420 - return r; 4421 4421 - member->type = r; 4422 4422 - member++; 4423 4423 - } 4424 4424 - break; 4425 4425 - } 4426 4426 - 4427 4427 - case BTF_KIND_FUNC_PROTO: { 4428 4428 - struct btf_param *param = btf_params(t); 4429 4429 - __u16 vlen = btf_vlen(t); 4430 4430 - 4431 4431 - r = btf_dedup_remap_type_id(d, t->type); 4432 4432 - if (r < 0) 4433 4433 - return r; 4434 4434 - t->type = r; 4435 4435 - 4436 4436 - for (i = 0; i < vlen; i++) { 4437 4437 - r = btf_dedup_remap_type_id(d, param->type); 4438 4438 - if (r < 0) 4439 4439 - return r; 4440 4440 - param->type = r; 4441 4441 - param++; 4442 4442 - } 4443 4443 - break; 4444 4444 - } 4445 4445 - 4446 4446 - case BTF_KIND_DATASEC: { 4447 4447 - struct btf_var_secinfo *var = btf_var_secinfos(t); 4448 4448 - __u16 vlen = btf_vlen(t); 4449 4449 - 4450 4450 - for (i = 0; i < vlen; i++) { 4451 4451 - r = btf_dedup_remap_type_id(d, var->type); 4452 4452 - if (r < 0) 4453 4453 - return r; 4454 4454 - var->type = r; 4455 4455 - var++; 4456 4456 - } 4457 4457 - break; 4458 4458 - } 4459 4459 - 4460 4460 - default: 4461 4461 - return -EINVAL; 4462 4462 - } 4463 4463 - 4464 4464 - return 0; 4465 4465 - } 4466 4466 - 4467 4522 static int btf_dedup_remap_types(struct btf_dedup *d) 4468 4523 { 4469 4524 int i, r; 4470 4525 4471 4526 for (i = 0; i < d->btf->nr_types; i++) { 4472 4472 - r = btf_dedup_remap_type(d, d->btf->start_id + i); 4473 4473 - if (r < 0) 4527 4527 + struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i); 4528 4528 + 4529 4529 + r = btf_type_visit_type_ids(t, btf_dedup_remap_type_id, d); 4530 4530 + if (r) 4474 4531 return r; 4475 4532 } 4533 4533 + 4534 4534 + if (!d->btf_ext) 4535 4535 + return 0; 4536 4536 + 4537 4537 + r = btf_ext_visit_type_ids(d->btf_ext, btf_dedup_remap_type_id, d); 4538 4538 + if (r) 4539 4539 + return r; 4540 4540 + 4476 4541 return 0; 4477 4542 } 4478 4543 ··· 4441 4674 4442 4675 pr_warn("failed to find valid kernel BTF\n"); 4443 4676 return ERR_PTR(-ESRCH); 4677 4677 + } 4678 4678 + 4679 4679 + int btf_type_visit_type_ids(struct btf_type *t, type_id_visit_fn visit, void *ctx) 4680 4680 + { 4681 4681 + int i, n, err; 4682 4682 + 4683 4683 + switch (btf_kind(t)) { 4684 4684 + case BTF_KIND_INT: 4685 4685 + case BTF_KIND_FLOAT: 4686 4686 + case BTF_KIND_ENUM: 4687 4687 + return 0; 4688 4688 + 4689 4689 + case BTF_KIND_FWD: 4690 4690 + case BTF_KIND_CONST: 4691 4691 + case BTF_KIND_VOLATILE: 4692 4692 + case BTF_KIND_RESTRICT: 4693 4693 + case BTF_KIND_PTR: 4694 4694 + case BTF_KIND_TYPEDEF: 4695 4695 + case BTF_KIND_FUNC: 4696 4696 + case BTF_KIND_VAR: 4697 4697 + return visit(&t->type, ctx); 4698 4698 + 4699 4699 + case BTF_KIND_ARRAY: { 4700 4700 + struct btf_array *a = btf_array(t); 4701 4701 + 4702 4702 + err = visit(&a->type, ctx); 4703 4703 + err = err ?: visit(&a->index_type, ctx); 4704 4704 + return err; 4705 4705 + } 4706 4706 + 4707 4707 + case BTF_KIND_STRUCT: 4708 4708 + case BTF_KIND_UNION: { 4709 4709 + struct btf_member *m = btf_members(t); 4710 4710 + 4711 4711 + for (i = 0, n = btf_vlen(t); i < n; i++, m++) { 4712 4712 + err = visit(&m->type, ctx); 4713 4713 + if (err) 4714 4714 + return err; 4715 4715 + } 4716 4716 + return 0; 4717 4717 + } 4718 4718 + 4719 4719 + case BTF_KIND_FUNC_PROTO: { 4720 4720 + struct btf_param *m = btf_params(t); 4721 4721 + 4722 4722 + err = visit(&t->type, ctx); 4723 4723 + if (err) 4724 4724 + return err; 4725 4725 + for (i = 0, n = btf_vlen(t); i < n; i++, m++) { 4726 4726 + err = visit(&m->type, ctx); 4727 4727 + if (err) 4728 4728 + return err; 4729 4729 + } 4730 4730 + return 0; 4731 4731 + } 4732 4732 + 4733 4733 + case BTF_KIND_DATASEC: { 4734 4734 + struct btf_var_secinfo *m = btf_var_secinfos(t); 4735 4735 + 4736 4736 + for (i = 0, n = btf_vlen(t); i < n; i++, m++) { 4737 4737 + err = visit(&m->type, ctx); 4738 4738 + if (err) 4739 4739 + return err; 4740 4740 + } 4741 4741 + return 0; 4742 4742 + } 4743 4743 + 4744 4744 + default: 4745 4745 + return -EINVAL; 4746 4746 + } 4747 4747 + } 4748 4748 + 4749 4749 + int btf_type_visit_str_offs(struct btf_type *t, str_off_visit_fn visit, void *ctx) 4750 4750 + { 4751 4751 + int i, n, err; 4752 4752 + 4753 4753 + err = visit(&t->name_off, ctx); 4754 4754 + if (err) 4755 4755 + return err; 4756 4756 + 4757 4757 + switch (btf_kind(t)) { 4758 4758 + case BTF_KIND_STRUCT: 4759 4759 + case BTF_KIND_UNION: { 4760 4760 + struct btf_member *m = btf_members(t); 4761 4761 + 4762 4762 + for (i = 0, n = btf_vlen(t); i < n; i++, m++) { 4763 4763 + err = visit(&m->name_off, ctx); 4764 4764 + if (err) 4765 4765 + return err; 4766 4766 + } 4767 4767 + break; 4768 4768 + } 4769 4769 + case BTF_KIND_ENUM: { 4770 4770 + struct btf_enum *m = btf_enum(t); 4771 4771 + 4772 4772 + for (i = 0, n = btf_vlen(t); i < n; i++, m++) { 4773 4773 + err = visit(&m->name_off, ctx); 4774 4774 + if (err) 4775 4775 + return err; 4776 4776 + } 4777 4777 + break; 4778 4778 + } 4779 4779 + case BTF_KIND_FUNC_PROTO: { 4780 4780 + struct btf_param *m = btf_params(t); 4781 4781 + 4782 4782 + for (i = 0, n = btf_vlen(t); i < n; i++, m++) { 4783 4783 + err = visit(&m->name_off, ctx); 4784 4784 + if (err) 4785 4785 + return err; 4786 4786 + } 4787 4787 + break; 4788 4788 + } 4789 4789 + default: 4790 4790 + break; 4791 4791 + } 4792 4792 + 4793 4793 + return 0; 4794 4794 + } 4795 4795 + 4796 4796 + int btf_ext_visit_type_ids(struct btf_ext *btf_ext, type_id_visit_fn visit, void *ctx) 4797 4797 + { 4798 4798 + const struct btf_ext_info *seg; 4799 4799 + struct btf_ext_info_sec *sec; 4800 4800 + int i, err; 4801 4801 + 4802 4802 + seg = &btf_ext->func_info; 4803 4803 + for_each_btf_ext_sec(seg, sec) { 4804 4804 + struct bpf_func_info_min *rec; 4805 4805 + 4806 4806 + for_each_btf_ext_rec(seg, sec, i, rec) { 4807 4807 + err = visit(&rec->type_id, ctx); 4808 4808 + if (err < 0) 4809 4809 + return err; 4810 4810 + } 4811 4811 + } 4812 4812 + 4813 4813 + seg = &btf_ext->core_relo_info; 4814 4814 + for_each_btf_ext_sec(seg, sec) { 4815 4815 + struct bpf_core_relo *rec; 4816 4816 + 4817 4817 + for_each_btf_ext_rec(seg, sec, i, rec) { 4818 4818 + err = visit(&rec->type_id, ctx); 4819 4819 + if (err < 0) 4820 4820 + return err; 4821 4821 + } 4822 4822 + } 4823 4823 + 4824 4824 + return 0; 4825 4825 + } 4826 4826 + 4827 4827 + int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void *ctx) 4828 4828 + { 4829 4829 + const struct btf_ext_info *seg; 4830 4830 + struct btf_ext_info_sec *sec; 4831 4831 + int i, err; 4832 4832 + 4833 4833 + seg = &btf_ext->func_info; 4834 4834 + for_each_btf_ext_sec(seg, sec) { 4835 4835 + err = visit(&sec->sec_name_off, ctx); 4836 4836 + if (err) 4837 4837 + return err; 4838 4838 + } 4839 4839 + 4840 4840 + seg = &btf_ext->line_info; 4841 4841 + for_each_btf_ext_sec(seg, sec) { 4842 4842 + struct bpf_line_info_min *rec; 4843 4843 + 4844 4844 + err = visit(&sec->sec_name_off, ctx); 4845 4845 + if (err) 4846 4846 + return err; 4847 4847 + 4848 4848 + for_each_btf_ext_rec(seg, sec, i, rec) { 4849 4849 + err = visit(&rec->file_name_off, ctx); 4850 4850 + if (err) 4851 4851 + return err; 4852 4852 + err = visit(&rec->line_off, ctx); 4853 4853 + if (err) 4854 4854 + return err; 4855 4855 + } 4856 4856 + } 4857 4857 + 4858 4858 + seg = &btf_ext->core_relo_info; 4859 4859 + for_each_btf_ext_sec(seg, sec) { 4860 4860 + struct bpf_core_relo *rec; 4861 4861 + 4862 4862 + err = visit(&sec->sec_name_off, ctx); 4863 4863 + if (err) 4864 4864 + return err; 4865 4865 + 4866 4866 + for_each_btf_ext_rec(seg, sec, i, rec) { 4867 4867 + err = visit(&rec->access_str_off, ctx); 4868 4868 + if (err) 4869 4869 + return err; 4870 4870 + } 4871 4871 + } 4872 4872 + 4873 4873 + return 0; 4444 4874 }

+3

tools/lib/bpf/btf.h

reviewed

··· 93 93 94 94 LIBBPF_API int btf__find_str(struct btf *btf, const char *s); 95 95 LIBBPF_API int btf__add_str(struct btf *btf, const char *s); 96 96 + LIBBPF_API int btf__add_type(struct btf *btf, const struct btf *src_btf, 97 97 + const struct btf_type *src_type); 96 98 97 99 LIBBPF_API int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding); 98 100 LIBBPF_API int btf__add_float(struct btf *btf, const char *name, size_t byte_sz); ··· 176 174 int indent_level; 177 175 /* strip all the const/volatile/restrict mods */ 178 176 bool strip_mods; 177 177 + size_t :0; 179 178 }; 180 179 #define btf_dump_emit_type_decl_opts__last_field strip_mods 181 180

+4 -4

tools/lib/bpf/btf_dump.c

reviewed

··· 166 166 if (last_id <= d->last_id) 167 167 return 0; 168 168 169 169 - if (btf_ensure_mem((void **)&d->type_states, &d->type_states_cap, 170 170 - sizeof(*d->type_states), last_id + 1)) 169 169 + if (libbpf_ensure_mem((void **)&d->type_states, &d->type_states_cap, 170 170 + sizeof(*d->type_states), last_id + 1)) 171 171 return -ENOMEM; 172 172 - if (btf_ensure_mem((void **)&d->cached_names, &d->cached_names_cap, 173 173 - sizeof(*d->cached_names), last_id + 1)) 172 172 + if (libbpf_ensure_mem((void **)&d->cached_names, &d->cached_names_cap, 173 173 + sizeof(*d->cached_names), last_id + 1)) 174 174 return -ENOMEM; 175 175 176 176 if (d->last_id == 0) {

+3 -12

tools/lib/bpf/libbpf.c

reviewed

··· 55 55 #include "libbpf_internal.h" 56 56 #include "hashmap.h" 57 57 58 58 - #ifndef EM_BPF 59 59 - #define EM_BPF 247 60 60 - #endif 61 61 - 62 58 #ifndef BPF_FS_MAGIC 63 59 #define BPF_FS_MAGIC 0xcafe4a11 64 60 #endif ··· 1129 1133 obj->efile.obj_buf = NULL; 1130 1134 obj->efile.obj_buf_sz = 0; 1131 1135 } 1132 1132 - 1133 1133 - /* if libelf is old and doesn't support mmap(), fall back to read() */ 1134 1134 - #ifndef ELF_C_READ_MMAP 1135 1135 - #define ELF_C_READ_MMAP ELF_C_READ 1136 1136 - #endif 1137 1136 1138 1137 static int bpf_object__elf_init(struct bpf_object *obj) 1139 1138 { ··· 2799 2808 return true; 2800 2809 2801 2810 /* ignore .llvm_addrsig section as well */ 2802 2802 - if (hdr->sh_type == 0x6FFF4C03 /* SHT_LLVM_ADDRSIG */) 2811 2811 + if (hdr->sh_type == SHT_LLVM_ADDRSIG) 2803 2812 return true; 2804 2813 2805 2814 /* no subprograms will lead to an empty .text section, ignore it */ ··· 4859 4868 goto err_out; 4860 4869 } 4861 4870 4862 4862 - err = btf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap, 4863 4863 - sizeof(*obj->btf_modules), obj->btf_module_cnt + 1); 4871 4871 + err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap, 4872 4872 + sizeof(*obj->btf_modules), obj->btf_module_cnt + 1); 4864 4873 if (err) 4865 4874 goto err_out; 4866 4875

+14

tools/lib/bpf/libbpf.h

reviewed

··· 507 507 struct bpf_xdp_set_link_opts { 508 508 size_t sz; 509 509 int old_fd; 510 510 + size_t :0; 510 511 }; 511 512 #define bpf_xdp_set_link_opts__last_field old_fd 512 513 ··· 759 758 TRI_YES = 1, 760 759 TRI_MODULE = 2, 761 760 }; 761 761 + 762 762 + struct bpf_linker_opts { 763 763 + /* size of this struct, for forward/backward compatiblity */ 764 764 + size_t sz; 765 765 + }; 766 766 + #define bpf_linker_opts__last_field sz 767 767 + 768 768 + struct bpf_linker; 769 769 + 770 770 + LIBBPF_API struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts); 771 771 + LIBBPF_API int bpf_linker__add_file(struct bpf_linker *linker, const char *filename); 772 772 + LIBBPF_API int bpf_linker__finalize(struct bpf_linker *linker); 773 773 + LIBBPF_API void bpf_linker__free(struct bpf_linker *linker); 762 774 763 775 #ifdef __cplusplus 764 776 } /* extern "C" */

+5

tools/lib/bpf/libbpf.map

reviewed

··· 354 354 LIBBPF_0.4.0 { 355 355 global: 356 356 btf__add_float; 357 357 + btf__add_type; 358 358 + bpf_linker__add_file; 359 359 + bpf_linker__finalize; 360 360 + bpf_linker__free; 361 361 + bpf_linker__new; 357 362 } LIBBPF_0.3.0;

+35 -3

tools/lib/bpf/libbpf_internal.h

reviewed

··· 20 20 21 21 #include "libbpf.h" 22 22 23 23 + #ifndef EM_BPF 24 24 + #define EM_BPF 247 25 25 + #endif 26 26 + 27 27 + #ifndef R_BPF_64_64 28 28 + #define R_BPF_64_64 1 29 29 + #endif 30 30 + #ifndef R_BPF_64_32 31 31 + #define R_BPF_64_32 10 32 32 + #endif 33 33 + 34 34 + #ifndef SHT_LLVM_ADDRSIG 35 35 + #define SHT_LLVM_ADDRSIG 0x6FFF4C03 36 36 + #endif 37 37 + 38 38 + /* if libelf is old and doesn't support mmap(), fall back to read() */ 39 39 + #ifndef ELF_C_READ_MMAP 40 40 + #define ELF_C_READ_MMAP ELF_C_READ 41 41 + #endif 42 42 + 23 43 #define BTF_INFO_ENC(kind, kind_flag, vlen) \ 24 44 ((!!(kind_flag) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) 25 45 #define BTF_TYPE_ENC(name, info, size_or_type) (name), (info), (size_or_type) ··· 127 107 return realloc(ptr, total); 128 108 } 129 109 130 130 - void *btf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz, 131 131 - size_t cur_cnt, size_t max_cnt, size_t add_cnt); 132 132 - int btf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt); 110 110 + struct btf; 111 111 + struct btf_type; 112 112 + 113 113 + struct btf_type *btf_type_by_id(struct btf *btf, __u32 type_id); 114 114 + 115 115 + void *libbpf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz, 116 116 + size_t cur_cnt, size_t max_cnt, size_t add_cnt); 117 117 + int libbpf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt); 133 118 134 119 static inline bool libbpf_validate_opts(const char *opts, 135 120 size_t opts_sz, size_t user_sz, ··· 375 350 __u32 access_str_off; 376 351 enum bpf_core_relo_kind kind; 377 352 }; 353 353 + 354 354 + typedef int (*type_id_visit_fn)(__u32 *type_id, void *ctx); 355 355 + typedef int (*str_off_visit_fn)(__u32 *str_off, void *ctx); 356 356 + int btf_type_visit_type_ids(struct btf_type *t, type_id_visit_fn visit, void *ctx); 357 357 + int btf_type_visit_str_offs(struct btf_type *t, str_off_visit_fn visit, void *ctx); 358 358 + int btf_ext_visit_type_ids(struct btf_ext *btf_ext, type_id_visit_fn visit, void *ctx); 359 359 + int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void *ctx); 378 360 379 361 #endif /* __LIBBPF_LIBBPF_INTERNAL_H */

-75

tools/lib/bpf/libbpf_util.h

reviewed

··· 1 1 - /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ 2 2 - /* Copyright (c) 2019 Facebook */ 3 3 - 4 4 - #ifndef __LIBBPF_LIBBPF_UTIL_H 5 5 - #define __LIBBPF_LIBBPF_UTIL_H 6 6 - 7 7 - #include <stdbool.h> 8 8 - #include <linux/compiler.h> 9 9 - 10 10 - #ifdef __cplusplus 11 11 - extern "C" { 12 12 - #endif 13 13 - 14 14 - /* Use these barrier functions instead of smp_[rw]mb() when they are 15 15 - * used in a libbpf header file. That way they can be built into the 16 16 - * application that uses libbpf. 17 17 - */ 18 18 - #if defined(__i386__) || defined(__x86_64__) 19 19 - # define libbpf_smp_store_release(p, v) \ 20 20 - do { \ 21 21 - asm volatile("" : : : "memory"); \ 22 22 - WRITE_ONCE(*p, v); \ 23 23 - } while (0) 24 24 - # define libbpf_smp_load_acquire(p) \ 25 25 - ({ \ 26 26 - typeof(*p) ___p1 = READ_ONCE(*p); \ 27 27 - asm volatile("" : : : "memory"); \ 28 28 - ___p1; \ 29 29 - }) 30 30 - #elif defined(__aarch64__) 31 31 - # define libbpf_smp_store_release(p, v) \ 32 32 - asm volatile ("stlr %w1, %0" : "=Q" (*p) : "r" (v) : "memory") 33 33 - # define libbpf_smp_load_acquire(p) \ 34 34 - ({ \ 35 35 - typeof(*p) ___p1; \ 36 36 - asm volatile ("ldar %w0, %1" \ 37 37 - : "=r" (___p1) : "Q" (*p) : "memory"); \ 38 38 - ___p1; \ 39 39 - }) 40 40 - #elif defined(__riscv) 41 41 - # define libbpf_smp_store_release(p, v) \ 42 42 - do { \ 43 43 - asm volatile ("fence rw,w" : : : "memory"); \ 44 44 - WRITE_ONCE(*p, v); \ 45 45 - } while (0) 46 46 - # define libbpf_smp_load_acquire(p) \ 47 47 - ({ \ 48 48 - typeof(*p) ___p1 = READ_ONCE(*p); \ 49 49 - asm volatile ("fence r,rw" : : : "memory"); \ 50 50 - ___p1; \ 51 51 - }) 52 52 - #endif 53 53 - 54 54 - #ifndef libbpf_smp_store_release 55 55 - #define libbpf_smp_store_release(p, v) \ 56 56 - do { \ 57 57 - __sync_synchronize(); \ 58 58 - WRITE_ONCE(*p, v); \ 59 59 - } while (0) 60 60 - #endif 61 61 - 62 62 - #ifndef libbpf_smp_load_acquire 63 63 - #define libbpf_smp_load_acquire(p) \ 64 64 - ({ \ 65 65 - typeof(*p) ___p1 = READ_ONCE(*p); \ 66 66 - __sync_synchronize(); \ 67 67 - ___p1; \ 68 68 - }) 69 69 - #endif 70 70 - 71 71 - #ifdef __cplusplus 72 72 - } /* extern "C" */ 73 73 - #endif 74 74 - 75 75 - #endif

+1944

tools/lib/bpf/linker.c

reviewed

··· 1 1 + // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 2 + /* 3 3 + * BPF static linker 4 4 + * 5 5 + * Copyright (c) 2021 Facebook 6 6 + */ 7 7 + #include <stdbool.h> 8 8 + #include <stddef.h> 9 9 + #include <stdio.h> 10 10 + #include <stdlib.h> 11 11 + #include <string.h> 12 12 + #include <unistd.h> 13 13 + #include <errno.h> 14 14 + #include <linux/err.h> 15 15 + #include <linux/btf.h> 16 16 + #include <elf.h> 17 17 + #include <libelf.h> 18 18 + #include <gelf.h> 19 19 + #include <fcntl.h> 20 20 + #include "libbpf.h" 21 21 + #include "btf.h" 22 22 + #include "libbpf_internal.h" 23 23 + #include "strset.h" 24 24 + 25 25 + struct src_sec { 26 26 + const char *sec_name; 27 27 + /* positional (not necessarily ELF) index in an array of sections */ 28 28 + int id; 29 29 + /* positional (not necessarily ELF) index of a matching section in a final object file */ 30 30 + int dst_id; 31 31 + /* section data offset in a matching output section */ 32 32 + int dst_off; 33 33 + /* whether section is omitted from the final ELF file */ 34 34 + bool skipped; 35 35 + /* whether section is an ephemeral section, not mapped to an ELF section */ 36 36 + bool ephemeral; 37 37 + 38 38 + /* ELF info */ 39 39 + size_t sec_idx; 40 40 + Elf_Scn *scn; 41 41 + Elf64_Shdr *shdr; 42 42 + Elf_Data *data; 43 43 + 44 44 + /* corresponding BTF DATASEC type ID */ 45 45 + int sec_type_id; 46 46 + }; 47 47 + 48 48 + struct src_obj { 49 49 + const char *filename; 50 50 + int fd; 51 51 + Elf *elf; 52 52 + /* Section header strings section index */ 53 53 + size_t shstrs_sec_idx; 54 54 + /* SYMTAB section index */ 55 55 + size_t symtab_sec_idx; 56 56 + 57 57 + struct btf *btf; 58 58 + struct btf_ext *btf_ext; 59 59 + 60 60 + /* List of sections (including ephemeral). Slot zero is unused. */ 61 61 + struct src_sec *secs; 62 62 + int sec_cnt; 63 63 + 64 64 + /* mapping of symbol indices from src to dst ELF */ 65 65 + int *sym_map; 66 66 + /* mapping from the src BTF type IDs to dst ones */ 67 67 + int *btf_type_map; 68 68 + }; 69 69 + 70 70 + /* single .BTF.ext data section */ 71 71 + struct btf_ext_sec_data { 72 72 + size_t rec_cnt; 73 73 + __u32 rec_sz; 74 74 + void *recs; 75 75 + }; 76 76 + 77 77 + struct dst_sec { 78 78 + char *sec_name; 79 79 + /* positional (not necessarily ELF) index in an array of sections */ 80 80 + int id; 81 81 + 82 82 + /* ELF info */ 83 83 + size_t sec_idx; 84 84 + Elf_Scn *scn; 85 85 + Elf64_Shdr *shdr; 86 86 + Elf_Data *data; 87 87 + 88 88 + /* final output section size */ 89 89 + int sec_sz; 90 90 + /* final output contents of the section */ 91 91 + void *raw_data; 92 92 + 93 93 + /* corresponding STT_SECTION symbol index in SYMTAB */ 94 94 + int sec_sym_idx; 95 95 + 96 96 + /* section's DATASEC variable info, emitted on BTF finalization */ 97 97 + int sec_var_cnt; 98 98 + struct btf_var_secinfo *sec_vars; 99 99 + 100 100 + /* section's .BTF.ext data */ 101 101 + struct btf_ext_sec_data func_info; 102 102 + struct btf_ext_sec_data line_info; 103 103 + struct btf_ext_sec_data core_relo_info; 104 104 + }; 105 105 + 106 106 + struct bpf_linker { 107 107 + char *filename; 108 108 + int fd; 109 109 + Elf *elf; 110 110 + Elf64_Ehdr *elf_hdr; 111 111 + 112 112 + /* Output sections metadata */ 113 113 + struct dst_sec *secs; 114 114 + int sec_cnt; 115 115 + 116 116 + struct strset *strtab_strs; /* STRTAB unique strings */ 117 117 + size_t strtab_sec_idx; /* STRTAB section index */ 118 118 + size_t symtab_sec_idx; /* SYMTAB section index */ 119 119 + 120 120 + struct btf *btf; 121 121 + struct btf_ext *btf_ext; 122 122 + }; 123 123 + 124 124 + #define pr_warn_elf(fmt, ...) \ 125 125 + do { \ 126 126 + libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1)); \ 127 127 + } while (0) 128 128 + 129 129 + static int init_output_elf(struct bpf_linker *linker, const char *file); 130 130 + 131 131 + static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, struct src_obj *obj); 132 132 + static int linker_sanity_check_elf(struct src_obj *obj); 133 133 + static int linker_sanity_check_btf(struct src_obj *obj); 134 134 + static int linker_sanity_check_btf_ext(struct src_obj *obj); 135 135 + static int linker_fixup_btf(struct src_obj *obj); 136 136 + static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj); 137 137 + static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj); 138 138 + static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj); 139 139 + static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj); 140 140 + static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj); 141 141 + 142 142 + static int finalize_btf(struct bpf_linker *linker); 143 143 + static int finalize_btf_ext(struct bpf_linker *linker); 144 144 + 145 145 + void bpf_linker__free(struct bpf_linker *linker) 146 146 + { 147 147 + int i; 148 148 + 149 149 + if (!linker) 150 150 + return; 151 151 + 152 152 + free(linker->filename); 153 153 + 154 154 + if (linker->elf) 155 155 + elf_end(linker->elf); 156 156 + 157 157 + if (linker->fd >= 0) 158 158 + close(linker->fd); 159 159 + 160 160 + strset__free(linker->strtab_strs); 161 161 + 162 162 + btf__free(linker->btf); 163 163 + btf_ext__free(linker->btf_ext); 164 164 + 165 165 + for (i = 1; i < linker->sec_cnt; i++) { 166 166 + struct dst_sec *sec = &linker->secs[i]; 167 167 + 168 168 + free(sec->sec_name); 169 169 + free(sec->raw_data); 170 170 + free(sec->sec_vars); 171 171 + 172 172 + free(sec->func_info.recs); 173 173 + free(sec->line_info.recs); 174 174 + free(sec->core_relo_info.recs); 175 175 + } 176 176 + free(linker->secs); 177 177 + 178 178 + free(linker); 179 179 + } 180 180 + 181 181 + struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts) 182 182 + { 183 183 + struct bpf_linker *linker; 184 184 + int err; 185 185 + 186 186 + if (!OPTS_VALID(opts, bpf_linker_opts)) 187 187 + return NULL; 188 188 + 189 189 + if (elf_version(EV_CURRENT) == EV_NONE) { 190 190 + pr_warn_elf("libelf initialization failed"); 191 191 + return NULL; 192 192 + } 193 193 + 194 194 + linker = calloc(1, sizeof(*linker)); 195 195 + if (!linker) 196 196 + return NULL; 197 197 + 198 198 + linker->fd = -1; 199 199 + 200 200 + err = init_output_elf(linker, filename); 201 201 + if (err) 202 202 + goto err_out; 203 203 + 204 204 + return linker; 205 205 + 206 206 + err_out: 207 207 + bpf_linker__free(linker); 208 208 + return NULL; 209 209 + } 210 210 + 211 211 + static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name) 212 212 + { 213 213 + struct dst_sec *secs = linker->secs, *sec; 214 214 + size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2; 215 215 + 216 216 + secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs)); 217 217 + if (!secs) 218 218 + return NULL; 219 219 + 220 220 + /* zero out newly allocated memory */ 221 221 + memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs)); 222 222 + 223 223 + linker->secs = secs; 224 224 + linker->sec_cnt = new_cnt; 225 225 + 226 226 + sec = &linker->secs[new_cnt - 1]; 227 227 + sec->id = new_cnt - 1; 228 228 + sec->sec_name = strdup(sec_name); 229 229 + if (!sec->sec_name) 230 230 + return NULL; 231 231 + 232 232 + return sec; 233 233 + } 234 234 + 235 235 + static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx) 236 236 + { 237 237 + struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx]; 238 238 + Elf64_Sym *syms, *sym; 239 239 + size_t sym_cnt = symtab->sec_sz / sizeof(*sym); 240 240 + 241 241 + syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym)); 242 242 + if (!syms) 243 243 + return NULL; 244 244 + 245 245 + sym = &syms[sym_cnt]; 246 246 + memset(sym, 0, sizeof(*sym)); 247 247 + 248 248 + symtab->raw_data = syms; 249 249 + symtab->sec_sz += sizeof(*sym); 250 250 + symtab->shdr->sh_size += sizeof(*sym); 251 251 + symtab->data->d_size += sizeof(*sym); 252 252 + 253 253 + if (sym_idx) 254 254 + *sym_idx = sym_cnt; 255 255 + 256 256 + return sym; 257 257 + } 258 258 + 259 259 + static int init_output_elf(struct bpf_linker *linker, const char *file) 260 260 + { 261 261 + int err, str_off; 262 262 + Elf64_Sym *init_sym; 263 263 + struct dst_sec *sec; 264 264 + 265 265 + linker->filename = strdup(file); 266 266 + if (!linker->filename) 267 267 + return -ENOMEM; 268 268 + 269 269 + linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0644); 270 270 + if (linker->fd < 0) { 271 271 + err = -errno; 272 272 + pr_warn("failed to create '%s': %d\n", file, err); 273 273 + return err; 274 274 + } 275 275 + 276 276 + linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL); 277 277 + if (!linker->elf) { 278 278 + pr_warn_elf("failed to create ELF object"); 279 279 + return -EINVAL; 280 280 + } 281 281 + 282 282 + /* ELF header */ 283 283 + linker->elf_hdr = elf64_newehdr(linker->elf); 284 284 + if (!linker->elf_hdr){ 285 285 + pr_warn_elf("failed to create ELF header"); 286 286 + return -EINVAL; 287 287 + } 288 288 + 289 289 + linker->elf_hdr->e_machine = EM_BPF; 290 290 + linker->elf_hdr->e_type = ET_REL; 291 291 + #if __BYTE_ORDER == __LITTLE_ENDIAN 292 292 + linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB; 293 293 + #elif __BYTE_ORDER == __BIG_ENDIAN 294 294 + linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB; 295 295 + #else 296 296 + #error "Unknown __BYTE_ORDER" 297 297 + #endif 298 298 + 299 299 + /* STRTAB */ 300 300 + /* initialize strset with an empty string to conform to ELF */ 301 301 + linker->strtab_strs = strset__new(INT_MAX, "", sizeof("")); 302 302 + if (libbpf_get_error(linker->strtab_strs)) 303 303 + return libbpf_get_error(linker->strtab_strs); 304 304 + 305 305 + sec = add_dst_sec(linker, ".strtab"); 306 306 + if (!sec) 307 307 + return -ENOMEM; 308 308 + 309 309 + sec->scn = elf_newscn(linker->elf); 310 310 + if (!sec->scn) { 311 311 + pr_warn_elf("failed to create STRTAB section"); 312 312 + return -EINVAL; 313 313 + } 314 314 + 315 315 + sec->shdr = elf64_getshdr(sec->scn); 316 316 + if (!sec->shdr) 317 317 + return -EINVAL; 318 318 + 319 319 + sec->data = elf_newdata(sec->scn); 320 320 + if (!sec->data) { 321 321 + pr_warn_elf("failed to create STRTAB data"); 322 322 + return -EINVAL; 323 323 + } 324 324 + 325 325 + str_off = strset__add_str(linker->strtab_strs, sec->sec_name); 326 326 + if (str_off < 0) 327 327 + return str_off; 328 328 + 329 329 + sec->sec_idx = elf_ndxscn(sec->scn); 330 330 + linker->elf_hdr->e_shstrndx = sec->sec_idx; 331 331 + linker->strtab_sec_idx = sec->sec_idx; 332 332 + 333 333 + sec->shdr->sh_name = str_off; 334 334 + sec->shdr->sh_type = SHT_STRTAB; 335 335 + sec->shdr->sh_flags = SHF_STRINGS; 336 336 + sec->shdr->sh_offset = 0; 337 337 + sec->shdr->sh_link = 0; 338 338 + sec->shdr->sh_info = 0; 339 339 + sec->shdr->sh_addralign = 1; 340 340 + sec->shdr->sh_size = sec->sec_sz = 0; 341 341 + sec->shdr->sh_entsize = 0; 342 342 + 343 343 + /* SYMTAB */ 344 344 + sec = add_dst_sec(linker, ".symtab"); 345 345 + if (!sec) 346 346 + return -ENOMEM; 347 347 + 348 348 + sec->scn = elf_newscn(linker->elf); 349 349 + if (!sec->scn) { 350 350 + pr_warn_elf("failed to create SYMTAB section"); 351 351 + return -EINVAL; 352 352 + } 353 353 + 354 354 + sec->shdr = elf64_getshdr(sec->scn); 355 355 + if (!sec->shdr) 356 356 + return -EINVAL; 357 357 + 358 358 + sec->data = elf_newdata(sec->scn); 359 359 + if (!sec->data) { 360 360 + pr_warn_elf("failed to create SYMTAB data"); 361 361 + return -EINVAL; 362 362 + } 363 363 + 364 364 + str_off = strset__add_str(linker->strtab_strs, sec->sec_name); 365 365 + if (str_off < 0) 366 366 + return str_off; 367 367 + 368 368 + sec->sec_idx = elf_ndxscn(sec->scn); 369 369 + linker->symtab_sec_idx = sec->sec_idx; 370 370 + 371 371 + sec->shdr->sh_name = str_off; 372 372 + sec->shdr->sh_type = SHT_SYMTAB; 373 373 + sec->shdr->sh_flags = 0; 374 374 + sec->shdr->sh_offset = 0; 375 375 + sec->shdr->sh_link = linker->strtab_sec_idx; 376 376 + /* sh_info should be one greater than the index of the last local 377 377 + * symbol (i.e., binding is STB_LOCAL). But why and who cares? 378 378 + */ 379 379 + sec->shdr->sh_info = 0; 380 380 + sec->shdr->sh_addralign = 8; 381 381 + sec->shdr->sh_entsize = sizeof(Elf64_Sym); 382 382 + 383 383 + /* .BTF */ 384 384 + linker->btf = btf__new_empty(); 385 385 + err = libbpf_get_error(linker->btf); 386 386 + if (err) 387 387 + return err; 388 388 + 389 389 + /* add the special all-zero symbol */ 390 390 + init_sym = add_new_sym(linker, NULL); 391 391 + if (!init_sym) 392 392 + return -EINVAL; 393 393 + 394 394 + init_sym->st_name = 0; 395 395 + init_sym->st_info = 0; 396 396 + init_sym->st_other = 0; 397 397 + init_sym->st_shndx = SHN_UNDEF; 398 398 + init_sym->st_value = 0; 399 399 + init_sym->st_size = 0; 400 400 + 401 401 + return 0; 402 402 + } 403 403 + 404 404 + int bpf_linker__add_file(struct bpf_linker *linker, const char *filename) 405 405 + { 406 406 + struct src_obj obj = {}; 407 407 + int err = 0; 408 408 + 409 409 + if (!linker->elf) 410 410 + return -EINVAL; 411 411 + 412 412 + err = err ?: linker_load_obj_file(linker, filename, &obj); 413 413 + err = err ?: linker_append_sec_data(linker, &obj); 414 414 + err = err ?: linker_append_elf_syms(linker, &obj); 415 415 + err = err ?: linker_append_elf_relos(linker, &obj); 416 416 + err = err ?: linker_append_btf(linker, &obj); 417 417 + err = err ?: linker_append_btf_ext(linker, &obj); 418 418 + 419 419 + /* free up src_obj resources */ 420 420 + free(obj.btf_type_map); 421 421 + btf__free(obj.btf); 422 422 + btf_ext__free(obj.btf_ext); 423 423 + free(obj.secs); 424 424 + free(obj.sym_map); 425 425 + if (obj.elf) 426 426 + elf_end(obj.elf); 427 427 + if (obj.fd >= 0) 428 428 + close(obj.fd); 429 429 + 430 430 + return err; 431 431 + } 432 432 + 433 433 + static bool is_dwarf_sec_name(const char *name) 434 434 + { 435 435 + /* approximation, but the actual list is too long */ 436 436 + return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0; 437 437 + } 438 438 + 439 439 + static bool is_ignored_sec(struct src_sec *sec) 440 440 + { 441 441 + Elf64_Shdr *shdr = sec->shdr; 442 442 + const char *name = sec->sec_name; 443 443 + 444 444 + /* no special handling of .strtab */ 445 445 + if (shdr->sh_type == SHT_STRTAB) 446 446 + return true; 447 447 + 448 448 + /* ignore .llvm_addrsig section as well */ 449 449 + if (shdr->sh_type == SHT_LLVM_ADDRSIG) 450 450 + return true; 451 451 + 452 452 + /* no subprograms will lead to an empty .text section, ignore it */ 453 453 + if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 && 454 454 + strcmp(sec->sec_name, ".text") == 0) 455 455 + return true; 456 456 + 457 457 + /* DWARF sections */ 458 458 + if (is_dwarf_sec_name(sec->sec_name)) 459 459 + return true; 460 460 + 461 461 + if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) { 462 462 + name += sizeof(".rel") - 1; 463 463 + /* DWARF section relocations */ 464 464 + if (is_dwarf_sec_name(name)) 465 465 + return true; 466 466 + 467 467 + /* .BTF and .BTF.ext don't need relocations */ 468 468 + if (strcmp(name, BTF_ELF_SEC) == 0 || 469 469 + strcmp(name, BTF_EXT_ELF_SEC) == 0) 470 470 + return true; 471 471 + } 472 472 + 473 473 + return false; 474 474 + } 475 475 + 476 476 + static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name) 477 477 + { 478 478 + struct src_sec *secs = obj->secs, *sec; 479 479 + size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2; 480 480 + 481 481 + secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs)); 482 482 + if (!secs) 483 483 + return NULL; 484 484 + 485 485 + /* zero out newly allocated memory */ 486 486 + memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs)); 487 487 + 488 488 + obj->secs = secs; 489 489 + obj->sec_cnt = new_cnt; 490 490 + 491 491 + sec = &obj->secs[new_cnt - 1]; 492 492 + sec->id = new_cnt - 1; 493 493 + sec->sec_name = sec_name; 494 494 + 495 495 + return sec; 496 496 + } 497 497 + 498 498 + static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, struct src_obj *obj) 499 499 + { 500 500 + #if __BYTE_ORDER == __LITTLE_ENDIAN 501 501 + const int host_endianness = ELFDATA2LSB; 502 502 + #elif __BYTE_ORDER == __BIG_ENDIAN 503 503 + const int host_endianness = ELFDATA2MSB; 504 504 + #else 505 505 + #error "Unknown __BYTE_ORDER" 506 506 + #endif 507 507 + int err = 0; 508 508 + Elf_Scn *scn; 509 509 + Elf_Data *data; 510 510 + Elf64_Ehdr *ehdr; 511 511 + Elf64_Shdr *shdr; 512 512 + struct src_sec *sec; 513 513 + 514 514 + pr_debug("linker: adding object file '%s'...\n", filename); 515 515 + 516 516 + obj->filename = filename; 517 517 + 518 518 + obj->fd = open(filename, O_RDONLY); 519 519 + if (obj->fd < 0) { 520 520 + err = -errno; 521 521 + pr_warn("failed to open file '%s': %d\n", filename, err); 522 522 + return err; 523 523 + } 524 524 + obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL); 525 525 + if (!obj->elf) { 526 526 + err = -errno; 527 527 + pr_warn_elf("failed to parse ELF file '%s'", filename); 528 528 + return err; 529 529 + } 530 530 + 531 531 + /* Sanity check ELF file high-level properties */ 532 532 + ehdr = elf64_getehdr(obj->elf); 533 533 + if (!ehdr) { 534 534 + err = -errno; 535 535 + pr_warn_elf("failed to get ELF header for %s", filename); 536 536 + return err; 537 537 + } 538 538 + if (ehdr->e_ident[EI_DATA] != host_endianness) { 539 539 + err = -EOPNOTSUPP; 540 540 + pr_warn_elf("unsupported byte order of ELF file %s", filename); 541 541 + return err; 542 542 + } 543 543 + if (ehdr->e_type != ET_REL 544 544 + || ehdr->e_machine != EM_BPF 545 545 + || ehdr->e_ident[EI_CLASS] != ELFCLASS64) { 546 546 + err = -EOPNOTSUPP; 547 547 + pr_warn_elf("unsupported kind of ELF file %s", filename); 548 548 + return err; 549 549 + } 550 550 + 551 551 + if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) { 552 552 + err = -errno; 553 553 + pr_warn_elf("failed to get SHSTRTAB section index for %s", filename); 554 554 + return err; 555 555 + } 556 556 + 557 557 + scn = NULL; 558 558 + while ((scn = elf_nextscn(obj->elf, scn)) != NULL) { 559 559 + size_t sec_idx = elf_ndxscn(scn); 560 560 + const char *sec_name; 561 561 + 562 562 + shdr = elf64_getshdr(scn); 563 563 + if (!shdr) { 564 564 + err = -errno; 565 565 + pr_warn_elf("failed to get section #%zu header for %s", 566 566 + sec_idx, filename); 567 567 + return err; 568 568 + } 569 569 + 570 570 + sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name); 571 571 + if (!sec_name) { 572 572 + err = -errno; 573 573 + pr_warn_elf("failed to get section #%zu name for %s", 574 574 + sec_idx, filename); 575 575 + return err; 576 576 + } 577 577 + 578 578 + data = elf_getdata(scn, 0); 579 579 + if (!data) { 580 580 + err = -errno; 581 581 + pr_warn_elf("failed to get section #%zu (%s) data from %s", 582 582 + sec_idx, sec_name, filename); 583 583 + return err; 584 584 + } 585 585 + 586 586 + sec = add_src_sec(obj, sec_name); 587 587 + if (!sec) 588 588 + return -ENOMEM; 589 589 + 590 590 + sec->scn = scn; 591 591 + sec->shdr = shdr; 592 592 + sec->data = data; 593 593 + sec->sec_idx = elf_ndxscn(scn); 594 594 + 595 595 + if (is_ignored_sec(sec)) { 596 596 + sec->skipped = true; 597 597 + continue; 598 598 + } 599 599 + 600 600 + switch (shdr->sh_type) { 601 601 + case SHT_SYMTAB: 602 602 + if (obj->symtab_sec_idx) { 603 603 + err = -EOPNOTSUPP; 604 604 + pr_warn("multiple SYMTAB sections found, not supported\n"); 605 605 + return err; 606 606 + } 607 607 + obj->symtab_sec_idx = sec_idx; 608 608 + break; 609 609 + case SHT_STRTAB: 610 610 + /* we'll construct our own string table */ 611 611 + break; 612 612 + case SHT_PROGBITS: 613 613 + if (strcmp(sec_name, BTF_ELF_SEC) == 0) { 614 614 + obj->btf = btf__new(data->d_buf, shdr->sh_size); 615 615 + err = libbpf_get_error(obj->btf); 616 616 + if (err) { 617 617 + pr_warn("failed to parse .BTF from %s: %d\n", filename, err); 618 618 + return err; 619 619 + } 620 620 + sec->skipped = true; 621 621 + continue; 622 622 + } 623 623 + if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) { 624 624 + obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size); 625 625 + err = libbpf_get_error(obj->btf_ext); 626 626 + if (err) { 627 627 + pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err); 628 628 + return err; 629 629 + } 630 630 + sec->skipped = true; 631 631 + continue; 632 632 + } 633 633 + 634 634 + /* data & code */ 635 635 + break; 636 636 + case SHT_NOBITS: 637 637 + /* BSS */ 638 638 + break; 639 639 + case SHT_REL: 640 640 + /* relocations */ 641 641 + break; 642 642 + default: 643 643 + pr_warn("unrecognized section #%zu (%s) in %s\n", 644 644 + sec_idx, sec_name, filename); 645 645 + err = -EINVAL; 646 646 + return err; 647 647 + } 648 648 + } 649 649 + 650 650 + err = err ?: linker_sanity_check_elf(obj); 651 651 + err = err ?: linker_sanity_check_btf(obj); 652 652 + err = err ?: linker_sanity_check_btf_ext(obj); 653 653 + err = err ?: linker_fixup_btf(obj); 654 654 + 655 655 + return err; 656 656 + } 657 657 + 658 658 + static bool is_pow_of_2(size_t x) 659 659 + { 660 660 + return x && (x & (x - 1)) == 0; 661 661 + } 662 662 + 663 663 + static int linker_sanity_check_elf(struct src_obj *obj) 664 664 + { 665 665 + struct src_sec *sec, *link_sec; 666 666 + int i, j, n; 667 667 + 668 668 + if (!obj->symtab_sec_idx) { 669 669 + pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename); 670 670 + return -EINVAL; 671 671 + } 672 672 + if (!obj->shstrs_sec_idx) { 673 673 + pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename); 674 674 + return -EINVAL; 675 675 + } 676 676 + 677 677 + for (i = 1; i < obj->sec_cnt; i++) { 678 678 + sec = &obj->secs[i]; 679 679 + 680 680 + if (sec->sec_name[0] == '\0') { 681 681 + pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename); 682 682 + return -EINVAL; 683 683 + } 684 684 + 685 685 + if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) 686 686 + return -EINVAL; 687 687 + if (sec->shdr->sh_addralign != sec->data->d_align) 688 688 + return -EINVAL; 689 689 + 690 690 + if (sec->shdr->sh_size != sec->data->d_size) 691 691 + return -EINVAL; 692 692 + 693 693 + switch (sec->shdr->sh_type) { 694 694 + case SHT_SYMTAB: { 695 695 + Elf64_Sym *sym; 696 696 + 697 697 + if (sec->shdr->sh_entsize != sizeof(Elf64_Sym)) 698 698 + return -EINVAL; 699 699 + if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0) 700 700 + return -EINVAL; 701 701 + 702 702 + if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) { 703 703 + pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n", 704 704 + sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); 705 705 + return -EINVAL; 706 706 + } 707 707 + link_sec = &obj->secs[sec->shdr->sh_link]; 708 708 + if (link_sec->shdr->sh_type != SHT_STRTAB) { 709 709 + pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n", 710 710 + sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); 711 711 + return -EINVAL; 712 712 + } 713 713 + 714 714 + n = sec->shdr->sh_size / sec->shdr->sh_entsize; 715 715 + sym = sec->data->d_buf; 716 716 + for (j = 0; j < n; j++, sym++) { 717 717 + if (sym->st_shndx 718 718 + && sym->st_shndx < SHN_LORESERVE 719 719 + && sym->st_shndx >= obj->sec_cnt) { 720 720 + pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n", 721 721 + j, sec->sec_idx, (size_t)sym->st_shndx, obj->filename); 722 722 + return -EINVAL; 723 723 + } 724 724 + if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION) { 725 725 + if (sym->st_value != 0) 726 726 + return -EINVAL; 727 727 + } 728 728 + } 729 729 + break; 730 730 + } 731 731 + case SHT_STRTAB: 732 732 + break; 733 733 + case SHT_PROGBITS: 734 734 + if (sec->shdr->sh_flags & SHF_EXECINSTR) { 735 735 + if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) 736 736 + return -EINVAL; 737 737 + } 738 738 + break; 739 739 + case SHT_NOBITS: 740 740 + break; 741 741 + case SHT_REL: { 742 742 + Elf64_Rel *relo; 743 743 + struct src_sec *sym_sec; 744 744 + 745 745 + if (sec->shdr->sh_entsize != sizeof(Elf64_Rel)) 746 746 + return -EINVAL; 747 747 + if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0) 748 748 + return -EINVAL; 749 749 + 750 750 + /* SHT_REL's sh_link should point to SYMTAB */ 751 751 + if (sec->shdr->sh_link != obj->symtab_sec_idx) { 752 752 + pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n", 753 753 + sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); 754 754 + return -EINVAL; 755 755 + } 756 756 + 757 757 + /* SHT_REL's sh_info points to relocated section */ 758 758 + if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) { 759 759 + pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n", 760 760 + sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename); 761 761 + return -EINVAL; 762 762 + } 763 763 + link_sec = &obj->secs[sec->shdr->sh_info]; 764 764 + 765 765 + /* .rel<secname> -> <secname> pattern is followed */ 766 766 + if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0 767 767 + || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) { 768 768 + pr_warn("ELF relo section #%zu name has invalid name in %s\n", 769 769 + sec->sec_idx, obj->filename); 770 770 + return -EINVAL; 771 771 + } 772 772 + 773 773 + /* don't further validate relocations for ignored sections */ 774 774 + if (link_sec->skipped) 775 775 + break; 776 776 + 777 777 + /* relocatable section is data or instructions */ 778 778 + if (link_sec->shdr->sh_type != SHT_PROGBITS 779 779 + && link_sec->shdr->sh_type != SHT_NOBITS) { 780 780 + pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n", 781 781 + sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename); 782 782 + return -EINVAL; 783 783 + } 784 784 + 785 785 + /* check sanity of each relocation */ 786 786 + n = sec->shdr->sh_size / sec->shdr->sh_entsize; 787 787 + relo = sec->data->d_buf; 788 788 + sym_sec = &obj->secs[obj->symtab_sec_idx]; 789 789 + for (j = 0; j < n; j++, relo++) { 790 790 + size_t sym_idx = ELF64_R_SYM(relo->r_info); 791 791 + size_t sym_type = ELF64_R_TYPE(relo->r_info); 792 792 + 793 793 + if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32) { 794 794 + pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n", 795 795 + j, sec->sec_idx, sym_type, obj->filename); 796 796 + return -EINVAL; 797 797 + } 798 798 + 799 799 + if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) { 800 800 + pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n", 801 801 + j, sec->sec_idx, sym_idx, obj->filename); 802 802 + return -EINVAL; 803 803 + } 804 804 + 805 805 + if (link_sec->shdr->sh_flags & SHF_EXECINSTR) { 806 806 + if (relo->r_offset % sizeof(struct bpf_insn) != 0) { 807 807 + pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n", 808 808 + j, sec->sec_idx, sym_idx, obj->filename); 809 809 + return -EINVAL; 810 810 + } 811 811 + } 812 812 + } 813 813 + break; 814 814 + } 815 815 + case SHT_LLVM_ADDRSIG: 816 816 + break; 817 817 + default: 818 818 + pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n", 819 819 + sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename); 820 820 + return -EINVAL; 821 821 + } 822 822 + } 823 823 + 824 824 + return 0; 825 825 + } 826 826 + 827 827 + static int check_btf_type_id(__u32 *type_id, void *ctx) 828 828 + { 829 829 + struct btf *btf = ctx; 830 830 + 831 831 + if (*type_id > btf__get_nr_types(btf)) 832 832 + return -EINVAL; 833 833 + 834 834 + return 0; 835 835 + } 836 836 + 837 837 + static int check_btf_str_off(__u32 *str_off, void *ctx) 838 838 + { 839 839 + struct btf *btf = ctx; 840 840 + const char *s; 841 841 + 842 842 + s = btf__str_by_offset(btf, *str_off); 843 843 + 844 844 + if (!s) 845 845 + return -EINVAL; 846 846 + 847 847 + return 0; 848 848 + } 849 849 + 850 850 + static int linker_sanity_check_btf(struct src_obj *obj) 851 851 + { 852 852 + struct btf_type *t; 853 853 + int i, n, err = 0; 854 854 + 855 855 + if (!obj->btf) 856 856 + return 0; 857 857 + 858 858 + n = btf__get_nr_types(obj->btf); 859 859 + for (i = 1; i <= n; i++) { 860 860 + t = btf_type_by_id(obj->btf, i); 861 861 + 862 862 + err = err ?: btf_type_visit_type_ids(t, check_btf_type_id, obj->btf); 863 863 + err = err ?: btf_type_visit_str_offs(t, check_btf_str_off, obj->btf); 864 864 + if (err) 865 865 + return err; 866 866 + } 867 867 + 868 868 + return 0; 869 869 + } 870 870 + 871 871 + static int linker_sanity_check_btf_ext(struct src_obj *obj) 872 872 + { 873 873 + int err = 0; 874 874 + 875 875 + if (!obj->btf_ext) 876 876 + return 0; 877 877 + 878 878 + /* can't use .BTF.ext without .BTF */ 879 879 + if (!obj->btf) 880 880 + return -EINVAL; 881 881 + 882 882 + err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf); 883 883 + err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf); 884 884 + if (err) 885 885 + return err; 886 886 + 887 887 + return 0; 888 888 + } 889 889 + 890 890 + static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec) 891 891 + { 892 892 + Elf_Scn *scn; 893 893 + Elf_Data *data; 894 894 + Elf64_Shdr *shdr; 895 895 + int name_off; 896 896 + 897 897 + dst_sec->sec_sz = 0; 898 898 + dst_sec->sec_idx = 0; 899 899 + 900 900 + /* ephemeral sections are just thin section shells lacking most parts */ 901 901 + if (src_sec->ephemeral) 902 902 + return 0; 903 903 + 904 904 + scn = elf_newscn(linker->elf); 905 905 + if (!scn) 906 906 + return -1; 907 907 + data = elf_newdata(scn); 908 908 + if (!data) 909 909 + return -1; 910 910 + shdr = elf64_getshdr(scn); 911 911 + if (!shdr) 912 912 + return -1; 913 913 + 914 914 + dst_sec->scn = scn; 915 915 + dst_sec->shdr = shdr; 916 916 + dst_sec->data = data; 917 917 + dst_sec->sec_idx = elf_ndxscn(scn); 918 918 + 919 919 + name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name); 920 920 + if (name_off < 0) 921 921 + return name_off; 922 922 + 923 923 + shdr->sh_name = name_off; 924 924 + shdr->sh_type = src_sec->shdr->sh_type; 925 925 + shdr->sh_flags = src_sec->shdr->sh_flags; 926 926 + shdr->sh_size = 0; 927 927 + /* sh_link and sh_info have different meaning for different types of 928 928 + * sections, so we leave it up to the caller code to fill them in, if 929 929 + * necessary 930 930 + */ 931 931 + shdr->sh_link = 0; 932 932 + shdr->sh_info = 0; 933 933 + shdr->sh_addralign = src_sec->shdr->sh_addralign; 934 934 + shdr->sh_entsize = src_sec->shdr->sh_entsize; 935 935 + 936 936 + data->d_type = src_sec->data->d_type; 937 937 + data->d_size = 0; 938 938 + data->d_buf = NULL; 939 939 + data->d_align = src_sec->data->d_align; 940 940 + data->d_off = 0; 941 941 + 942 942 + return 0; 943 943 + } 944 944 + 945 945 + static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name) 946 946 + { 947 947 + struct dst_sec *sec; 948 948 + int i; 949 949 + 950 950 + for (i = 1; i < linker->sec_cnt; i++) { 951 951 + sec = &linker->secs[i]; 952 952 + 953 953 + if (strcmp(sec->sec_name, sec_name) == 0) 954 954 + return sec; 955 955 + } 956 956 + 957 957 + return NULL; 958 958 + } 959 959 + 960 960 + static bool secs_match(struct dst_sec *dst, struct src_sec *src) 961 961 + { 962 962 + if (dst->shdr->sh_type != src->shdr->sh_type) { 963 963 + pr_warn("sec %s types mismatch\n", dst->sec_name); 964 964 + return false; 965 965 + } 966 966 + if (dst->shdr->sh_flags != src->shdr->sh_flags) { 967 967 + pr_warn("sec %s flags mismatch\n", dst->sec_name); 968 968 + return false; 969 969 + } 970 970 + if (dst->shdr->sh_entsize != src->shdr->sh_entsize) { 971 971 + pr_warn("sec %s entsize mismatch\n", dst->sec_name); 972 972 + return false; 973 973 + } 974 974 + 975 975 + return true; 976 976 + } 977 977 + 978 978 + static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec) 979 979 + { 980 980 + if (dst_sec->sec_sz != src_sec->shdr->sh_size) 981 981 + return false; 982 982 + if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0) 983 983 + return false; 984 984 + return true; 985 985 + } 986 986 + 987 987 + static int extend_sec(struct dst_sec *dst, struct src_sec *src) 988 988 + { 989 989 + void *tmp; 990 990 + size_t dst_align = dst->shdr->sh_addralign; 991 991 + size_t src_align = src->shdr->sh_addralign; 992 992 + size_t dst_align_sz, dst_final_sz; 993 993 + 994 994 + if (dst_align == 0) 995 995 + dst_align = 1; 996 996 + if (dst_align < src_align) 997 997 + dst_align = src_align; 998 998 + 999 999 + dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align; 1000 1000 + 1001 1001 + /* no need to re-align final size */ 1002 1002 + dst_final_sz = dst_align_sz + src->shdr->sh_size; 1003 1003 + 1004 1004 + if (src->shdr->sh_type != SHT_NOBITS) { 1005 1005 + tmp = realloc(dst->raw_data, dst_final_sz); 1006 1006 + if (!tmp) 1007 1007 + return -ENOMEM; 1008 1008 + dst->raw_data = tmp; 1009 1009 + 1010 1010 + /* pad dst section, if it's alignment forced size increase */ 1011 1011 + memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz); 1012 1012 + /* now copy src data at a properly aligned offset */ 1013 1013 + memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size); 1014 1014 + } 1015 1015 + 1016 1016 + dst->sec_sz = dst_final_sz; 1017 1017 + dst->shdr->sh_size = dst_final_sz; 1018 1018 + dst->data->d_size = dst_final_sz; 1019 1019 + 1020 1020 + dst->shdr->sh_addralign = dst_align; 1021 1021 + dst->data->d_align = dst_align; 1022 1022 + 1023 1023 + src->dst_off = dst_align_sz; 1024 1024 + 1025 1025 + return 0; 1026 1026 + } 1027 1027 + 1028 1028 + static bool is_data_sec(struct src_sec *sec) 1029 1029 + { 1030 1030 + if (!sec || sec->skipped) 1031 1031 + return false; 1032 1032 + /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */ 1033 1033 + if (sec->ephemeral) 1034 1034 + return true; 1035 1035 + return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS; 1036 1036 + } 1037 1037 + 1038 1038 + static bool is_relo_sec(struct src_sec *sec) 1039 1039 + { 1040 1040 + if (!sec || sec->skipped || sec->ephemeral) 1041 1041 + return false; 1042 1042 + return sec->shdr->sh_type == SHT_REL; 1043 1043 + } 1044 1044 + 1045 1045 + static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj) 1046 1046 + { 1047 1047 + int i, err; 1048 1048 + 1049 1049 + for (i = 1; i < obj->sec_cnt; i++) { 1050 1050 + struct src_sec *src_sec; 1051 1051 + struct dst_sec *dst_sec; 1052 1052 + 1053 1053 + src_sec = &obj->secs[i]; 1054 1054 + if (!is_data_sec(src_sec)) 1055 1055 + continue; 1056 1056 + 1057 1057 + dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name); 1058 1058 + if (!dst_sec) { 1059 1059 + dst_sec = add_dst_sec(linker, src_sec->sec_name); 1060 1060 + if (!dst_sec) 1061 1061 + return -ENOMEM; 1062 1062 + err = init_sec(linker, dst_sec, src_sec); 1063 1063 + if (err) { 1064 1064 + pr_warn("failed to init section '%s'\n", src_sec->sec_name); 1065 1065 + return err; 1066 1066 + } 1067 1067 + } else { 1068 1068 + if (!secs_match(dst_sec, src_sec)) { 1069 1069 + pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name); 1070 1070 + return -1; 1071 1071 + } 1072 1072 + 1073 1073 + /* "license" and "version" sections are deduped */ 1074 1074 + if (strcmp(src_sec->sec_name, "license") == 0 1075 1075 + || strcmp(src_sec->sec_name, "version") == 0) { 1076 1076 + if (!sec_content_is_same(dst_sec, src_sec)) { 1077 1077 + pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name); 1078 1078 + return -EINVAL; 1079 1079 + } 1080 1080 + src_sec->skipped = true; 1081 1081 + src_sec->dst_id = dst_sec->id; 1082 1082 + continue; 1083 1083 + } 1084 1084 + } 1085 1085 + 1086 1086 + /* record mapped section index */ 1087 1087 + src_sec->dst_id = dst_sec->id; 1088 1088 + 1089 1089 + if (src_sec->ephemeral) 1090 1090 + continue; 1091 1091 + 1092 1092 + err = extend_sec(dst_sec, src_sec); 1093 1093 + if (err) 1094 1094 + return err; 1095 1095 + } 1096 1096 + 1097 1097 + return 0; 1098 1098 + } 1099 1099 + 1100 1100 + static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj) 1101 1101 + { 1102 1102 + struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx]; 1103 1103 + Elf64_Sym *sym = symtab->data->d_buf, *dst_sym; 1104 1104 + int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize; 1105 1105 + int str_sec_idx = symtab->shdr->sh_link; 1106 1106 + 1107 1107 + obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map)); 1108 1108 + if (!obj->sym_map) 1109 1109 + return -ENOMEM; 1110 1110 + 1111 1111 + for (i = 0; i < n; i++, sym++) { 1112 1112 + struct src_sec *src_sec = NULL; 1113 1113 + struct dst_sec *dst_sec = NULL; 1114 1114 + const char *sym_name; 1115 1115 + size_t dst_sym_idx; 1116 1116 + int name_off; 1117 1117 + 1118 1118 + /* we already have all-zero initial symbol */ 1119 1119 + if (sym->st_name == 0 && sym->st_info == 0 && 1120 1120 + sym->st_other == 0 && sym->st_shndx == SHN_UNDEF && 1121 1121 + sym->st_value == 0 && sym->st_size ==0) 1122 1122 + continue; 1123 1123 + 1124 1124 + sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name); 1125 1125 + if (!sym_name) { 1126 1126 + pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename); 1127 1127 + return -1; 1128 1128 + } 1129 1129 + 1130 1130 + if (sym->st_shndx && sym->st_shndx < SHN_LORESERVE) { 1131 1131 + src_sec = &obj->secs[sym->st_shndx]; 1132 1132 + if (src_sec->skipped) 1133 1133 + continue; 1134 1134 + dst_sec = &linker->secs[src_sec->dst_id]; 1135 1135 + 1136 1136 + /* allow only one STT_SECTION symbol per section */ 1137 1137 + if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && dst_sec->sec_sym_idx) { 1138 1138 + obj->sym_map[i] = dst_sec->sec_sym_idx; 1139 1139 + continue; 1140 1140 + } 1141 1141 + } 1142 1142 + 1143 1143 + name_off = strset__add_str(linker->strtab_strs, sym_name); 1144 1144 + if (name_off < 0) 1145 1145 + return name_off; 1146 1146 + 1147 1147 + dst_sym = add_new_sym(linker, &dst_sym_idx); 1148 1148 + if (!dst_sym) 1149 1149 + return -ENOMEM; 1150 1150 + 1151 1151 + dst_sym->st_name = name_off; 1152 1152 + dst_sym->st_info = sym->st_info; 1153 1153 + dst_sym->st_other = sym->st_other; 1154 1154 + dst_sym->st_shndx = src_sec ? dst_sec->sec_idx : sym->st_shndx; 1155 1155 + dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value; 1156 1156 + dst_sym->st_size = sym->st_size; 1157 1157 + 1158 1158 + obj->sym_map[i] = dst_sym_idx; 1159 1159 + 1160 1160 + if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && dst_sym) { 1161 1161 + dst_sec->sec_sym_idx = dst_sym_idx; 1162 1162 + dst_sym->st_value = 0; 1163 1163 + } 1164 1164 + 1165 1165 + } 1166 1166 + 1167 1167 + return 0; 1168 1168 + } 1169 1169 + 1170 1170 + static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj) 1171 1171 + { 1172 1172 + struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx]; 1173 1173 + struct dst_sec *dst_symtab = &linker->secs[linker->symtab_sec_idx]; 1174 1174 + int i, err; 1175 1175 + 1176 1176 + for (i = 1; i < obj->sec_cnt; i++) { 1177 1177 + struct src_sec *src_sec, *src_linked_sec; 1178 1178 + struct dst_sec *dst_sec, *dst_linked_sec; 1179 1179 + Elf64_Rel *src_rel, *dst_rel; 1180 1180 + int j, n; 1181 1181 + 1182 1182 + src_sec = &obj->secs[i]; 1183 1183 + if (!is_relo_sec(src_sec)) 1184 1184 + continue; 1185 1185 + 1186 1186 + /* shdr->sh_info points to relocatable section */ 1187 1187 + src_linked_sec = &obj->secs[src_sec->shdr->sh_info]; 1188 1188 + if (src_linked_sec->skipped) 1189 1189 + continue; 1190 1190 + 1191 1191 + dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name); 1192 1192 + if (!dst_sec) { 1193 1193 + dst_sec = add_dst_sec(linker, src_sec->sec_name); 1194 1194 + if (!dst_sec) 1195 1195 + return -ENOMEM; 1196 1196 + err = init_sec(linker, dst_sec, src_sec); 1197 1197 + if (err) { 1198 1198 + pr_warn("failed to init section '%s'\n", src_sec->sec_name); 1199 1199 + return err; 1200 1200 + } 1201 1201 + } else if (!secs_match(dst_sec, src_sec)) { 1202 1202 + pr_warn("Secs %s are not compatible\n", src_sec->sec_name); 1203 1203 + return -1; 1204 1204 + } 1205 1205 + 1206 1206 + /* shdr->sh_link points to SYMTAB */ 1207 1207 + dst_sec->shdr->sh_link = linker->symtab_sec_idx; 1208 1208 + 1209 1209 + /* shdr->sh_info points to relocated section */ 1210 1210 + dst_linked_sec = &linker->secs[src_linked_sec->dst_id]; 1211 1211 + dst_sec->shdr->sh_info = dst_linked_sec->sec_idx; 1212 1212 + 1213 1213 + src_sec->dst_id = dst_sec->id; 1214 1214 + err = extend_sec(dst_sec, src_sec); 1215 1215 + if (err) 1216 1216 + return err; 1217 1217 + 1218 1218 + src_rel = src_sec->data->d_buf; 1219 1219 + dst_rel = dst_sec->raw_data + src_sec->dst_off; 1220 1220 + n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize; 1221 1221 + for (j = 0; j < n; j++, src_rel++, dst_rel++) { 1222 1222 + size_t src_sym_idx = ELF64_R_SYM(src_rel->r_info); 1223 1223 + size_t sym_type = ELF64_R_TYPE(src_rel->r_info); 1224 1224 + Elf64_Sym *src_sym, *dst_sym; 1225 1225 + size_t dst_sym_idx; 1226 1226 + 1227 1227 + src_sym_idx = ELF64_R_SYM(src_rel->r_info); 1228 1228 + src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx; 1229 1229 + 1230 1230 + dst_sym_idx = obj->sym_map[src_sym_idx]; 1231 1231 + dst_sym = dst_symtab->raw_data + sizeof(*dst_sym) * dst_sym_idx; 1232 1232 + dst_rel->r_offset += src_linked_sec->dst_off; 1233 1233 + sym_type = ELF64_R_TYPE(src_rel->r_info); 1234 1234 + dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type); 1235 1235 + 1236 1236 + if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) { 1237 1237 + struct src_sec *sec = &obj->secs[src_sym->st_shndx]; 1238 1238 + struct bpf_insn *insn; 1239 1239 + 1240 1240 + if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) { 1241 1241 + /* calls to the very first static function inside 1242 1242 + * .text section at offset 0 will 1243 1243 + * reference section symbol, not the 1244 1244 + * function symbol. Fix that up, 1245 1245 + * otherwise it won't be possible to 1246 1246 + * relocate calls to two different 1247 1247 + * static functions with the same name 1248 1248 + * (rom two different object files) 1249 1249 + */ 1250 1250 + insn = dst_linked_sec->raw_data + dst_rel->r_offset; 1251 1251 + if (insn->code == (BPF_JMP | BPF_CALL)) 1252 1252 + insn->imm += sec->dst_off / sizeof(struct bpf_insn); 1253 1253 + else 1254 1254 + insn->imm += sec->dst_off; 1255 1255 + } else { 1256 1256 + pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n"); 1257 1257 + return -EINVAL; 1258 1258 + } 1259 1259 + } 1260 1260 + 1261 1261 + } 1262 1262 + } 1263 1263 + 1264 1264 + return 0; 1265 1265 + } 1266 1266 + 1267 1267 + static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name) 1268 1268 + { 1269 1269 + struct src_sec *sec; 1270 1270 + int i; 1271 1271 + 1272 1272 + for (i = 1; i < obj->sec_cnt; i++) { 1273 1273 + sec = &obj->secs[i]; 1274 1274 + 1275 1275 + if (strcmp(sec->sec_name, sec_name) == 0) 1276 1276 + return sec; 1277 1277 + } 1278 1278 + 1279 1279 + return NULL; 1280 1280 + } 1281 1281 + 1282 1282 + static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx, 1283 1283 + int sym_type, const char *sym_name) 1284 1284 + { 1285 1285 + struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx]; 1286 1286 + Elf64_Sym *sym = symtab->data->d_buf; 1287 1287 + int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize; 1288 1288 + int str_sec_idx = symtab->shdr->sh_link; 1289 1289 + const char *name; 1290 1290 + 1291 1291 + for (i = 0; i < n; i++, sym++) { 1292 1292 + if (sym->st_shndx != sec_idx) 1293 1293 + continue; 1294 1294 + if (ELF64_ST_TYPE(sym->st_info) != sym_type) 1295 1295 + continue; 1296 1296 + 1297 1297 + name = elf_strptr(obj->elf, str_sec_idx, sym->st_name); 1298 1298 + if (!name) 1299 1299 + return NULL; 1300 1300 + 1301 1301 + if (strcmp(sym_name, name) != 0) 1302 1302 + continue; 1303 1303 + 1304 1304 + return sym; 1305 1305 + } 1306 1306 + 1307 1307 + return NULL; 1308 1308 + } 1309 1309 + 1310 1310 + static int linker_fixup_btf(struct src_obj *obj) 1311 1311 + { 1312 1312 + const char *sec_name; 1313 1313 + struct src_sec *sec; 1314 1314 + int i, j, n, m; 1315 1315 + 1316 1316 + if (!obj->btf) 1317 1317 + return 0; 1318 1318 + 1319 1319 + n = btf__get_nr_types(obj->btf); 1320 1320 + for (i = 1; i <= n; i++) { 1321 1321 + struct btf_var_secinfo *vi; 1322 1322 + struct btf_type *t; 1323 1323 + 1324 1324 + t = btf_type_by_id(obj->btf, i); 1325 1325 + if (btf_kind(t) != BTF_KIND_DATASEC) 1326 1326 + continue; 1327 1327 + 1328 1328 + sec_name = btf__str_by_offset(obj->btf, t->name_off); 1329 1329 + sec = find_src_sec_by_name(obj, sec_name); 1330 1330 + if (sec) { 1331 1331 + /* record actual section size, unless ephemeral */ 1332 1332 + if (sec->shdr) 1333 1333 + t->size = sec->shdr->sh_size; 1334 1334 + } else { 1335 1335 + /* BTF can have some sections that are not represented 1336 1336 + * in ELF, e.g., .kconfig and .ksyms, which are used 1337 1337 + * for special extern variables. Here we'll 1338 1338 + * pre-create "section shells" for them to be able to 1339 1339 + * keep track of extra per-section metadata later 1340 1340 + * (e.g., BTF variables). 1341 1341 + */ 1342 1342 + sec = add_src_sec(obj, sec_name); 1343 1343 + if (!sec) 1344 1344 + return -ENOMEM; 1345 1345 + 1346 1346 + sec->ephemeral = true; 1347 1347 + sec->sec_idx = 0; /* will match UNDEF shndx in ELF */ 1348 1348 + } 1349 1349 + 1350 1350 + /* remember ELF section and its BTF type ID match */ 1351 1351 + sec->sec_type_id = i; 1352 1352 + 1353 1353 + /* fix up variable offsets */ 1354 1354 + vi = btf_var_secinfos(t); 1355 1355 + for (j = 0, m = btf_vlen(t); j < m; j++, vi++) { 1356 1356 + const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type); 1357 1357 + const char *var_name = btf__str_by_offset(obj->btf, vt->name_off); 1358 1358 + int var_linkage = btf_var(vt)->linkage; 1359 1359 + Elf64_Sym *sym; 1360 1360 + 1361 1361 + /* no need to patch up static or extern vars */ 1362 1362 + if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED) 1363 1363 + continue; 1364 1364 + 1365 1365 + sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name); 1366 1366 + if (!sym) { 1367 1367 + pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name); 1368 1368 + return -ENOENT; 1369 1369 + } 1370 1370 + 1371 1371 + vi->offset = sym->st_value; 1372 1372 + } 1373 1373 + } 1374 1374 + 1375 1375 + return 0; 1376 1376 + } 1377 1377 + 1378 1378 + static int remap_type_id(__u32 *type_id, void *ctx) 1379 1379 + { 1380 1380 + int *id_map = ctx; 1381 1381 + 1382 1382 + *type_id = id_map[*type_id]; 1383 1383 + 1384 1384 + return 0; 1385 1385 + } 1386 1386 + 1387 1387 + static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj) 1388 1388 + { 1389 1389 + const struct btf_type *t; 1390 1390 + int i, j, n, start_id, id; 1391 1391 + 1392 1392 + if (!obj->btf) 1393 1393 + return 0; 1394 1394 + 1395 1395 + start_id = btf__get_nr_types(linker->btf) + 1; 1396 1396 + n = btf__get_nr_types(obj->btf); 1397 1397 + 1398 1398 + obj->btf_type_map = calloc(n + 1, sizeof(int)); 1399 1399 + if (!obj->btf_type_map) 1400 1400 + return -ENOMEM; 1401 1401 + 1402 1402 + for (i = 1; i <= n; i++) { 1403 1403 + t = btf__type_by_id(obj->btf, i); 1404 1404 + 1405 1405 + /* DATASECs are handled specially below */ 1406 1406 + if (btf_kind(t) == BTF_KIND_DATASEC) 1407 1407 + continue; 1408 1408 + 1409 1409 + id = btf__add_type(linker->btf, obj->btf, t); 1410 1410 + if (id < 0) { 1411 1411 + pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename); 1412 1412 + return id; 1413 1413 + } 1414 1414 + 1415 1415 + obj->btf_type_map[i] = id; 1416 1416 + } 1417 1417 + 1418 1418 + /* remap all the types except DATASECs */ 1419 1419 + n = btf__get_nr_types(linker->btf); 1420 1420 + for (i = start_id; i <= n; i++) { 1421 1421 + struct btf_type *dst_t = btf_type_by_id(linker->btf, i); 1422 1422 + 1423 1423 + if (btf_type_visit_type_ids(dst_t, remap_type_id, obj->btf_type_map)) 1424 1424 + return -EINVAL; 1425 1425 + } 1426 1426 + 1427 1427 + /* append DATASEC info */ 1428 1428 + for (i = 1; i < obj->sec_cnt; i++) { 1429 1429 + struct src_sec *src_sec; 1430 1430 + struct dst_sec *dst_sec; 1431 1431 + const struct btf_var_secinfo *src_var; 1432 1432 + struct btf_var_secinfo *dst_var; 1433 1433 + 1434 1434 + src_sec = &obj->secs[i]; 1435 1435 + if (!src_sec->sec_type_id || src_sec->skipped) 1436 1436 + continue; 1437 1437 + dst_sec = &linker->secs[src_sec->dst_id]; 1438 1438 + 1439 1439 + t = btf__type_by_id(obj->btf, src_sec->sec_type_id); 1440 1440 + src_var = btf_var_secinfos(t); 1441 1441 + n = btf_vlen(t); 1442 1442 + for (j = 0; j < n; j++, src_var++) { 1443 1443 + void *sec_vars = dst_sec->sec_vars; 1444 1444 + 1445 1445 + sec_vars = libbpf_reallocarray(sec_vars, 1446 1446 + dst_sec->sec_var_cnt + 1, 1447 1447 + sizeof(*dst_sec->sec_vars)); 1448 1448 + if (!sec_vars) 1449 1449 + return -ENOMEM; 1450 1450 + 1451 1451 + dst_sec->sec_vars = sec_vars; 1452 1452 + dst_sec->sec_var_cnt++; 1453 1453 + 1454 1454 + dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1]; 1455 1455 + dst_var->type = obj->btf_type_map[src_var->type]; 1456 1456 + dst_var->size = src_var->size; 1457 1457 + dst_var->offset = src_sec->dst_off + src_var->offset; 1458 1458 + } 1459 1459 + } 1460 1460 + 1461 1461 + return 0; 1462 1462 + } 1463 1463 + 1464 1464 + static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec) 1465 1465 + { 1466 1466 + void *tmp; 1467 1467 + 1468 1468 + tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz); 1469 1469 + if (!tmp) 1470 1470 + return NULL; 1471 1471 + ext_data->recs = tmp; 1472 1472 + 1473 1473 + tmp += ext_data->rec_cnt * ext_data->rec_sz; 1474 1474 + memcpy(tmp, src_rec, ext_data->rec_sz); 1475 1475 + 1476 1476 + ext_data->rec_cnt++; 1477 1477 + 1478 1478 + return tmp; 1479 1479 + } 1480 1480 + 1481 1481 + static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj) 1482 1482 + { 1483 1483 + const struct btf_ext_info_sec *ext_sec; 1484 1484 + const char *sec_name, *s; 1485 1485 + struct src_sec *src_sec; 1486 1486 + struct dst_sec *dst_sec; 1487 1487 + int rec_sz, str_off, i; 1488 1488 + 1489 1489 + if (!obj->btf_ext) 1490 1490 + return 0; 1491 1491 + 1492 1492 + rec_sz = obj->btf_ext->func_info.rec_size; 1493 1493 + for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) { 1494 1494 + struct bpf_func_info_min *src_rec, *dst_rec; 1495 1495 + 1496 1496 + sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); 1497 1497 + src_sec = find_src_sec_by_name(obj, sec_name); 1498 1498 + if (!src_sec) { 1499 1499 + pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name); 1500 1500 + return -EINVAL; 1501 1501 + } 1502 1502 + dst_sec = &linker->secs[src_sec->dst_id]; 1503 1503 + 1504 1504 + if (dst_sec->func_info.rec_sz == 0) 1505 1505 + dst_sec->func_info.rec_sz = rec_sz; 1506 1506 + if (dst_sec->func_info.rec_sz != rec_sz) { 1507 1507 + pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name); 1508 1508 + return -EINVAL; 1509 1509 + } 1510 1510 + 1511 1511 + for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) { 1512 1512 + dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec); 1513 1513 + if (!dst_rec) 1514 1514 + return -ENOMEM; 1515 1515 + 1516 1516 + dst_rec->insn_off += src_sec->dst_off; 1517 1517 + dst_rec->type_id = obj->btf_type_map[dst_rec->type_id]; 1518 1518 + } 1519 1519 + } 1520 1520 + 1521 1521 + rec_sz = obj->btf_ext->line_info.rec_size; 1522 1522 + for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) { 1523 1523 + struct bpf_line_info_min *src_rec, *dst_rec; 1524 1524 + 1525 1525 + sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); 1526 1526 + src_sec = find_src_sec_by_name(obj, sec_name); 1527 1527 + if (!src_sec) { 1528 1528 + pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name); 1529 1529 + return -EINVAL; 1530 1530 + } 1531 1531 + dst_sec = &linker->secs[src_sec->dst_id]; 1532 1532 + 1533 1533 + if (dst_sec->line_info.rec_sz == 0) 1534 1534 + dst_sec->line_info.rec_sz = rec_sz; 1535 1535 + if (dst_sec->line_info.rec_sz != rec_sz) { 1536 1536 + pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name); 1537 1537 + return -EINVAL; 1538 1538 + } 1539 1539 + 1540 1540 + for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) { 1541 1541 + dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec); 1542 1542 + if (!dst_rec) 1543 1543 + return -ENOMEM; 1544 1544 + 1545 1545 + dst_rec->insn_off += src_sec->dst_off; 1546 1546 + 1547 1547 + s = btf__str_by_offset(obj->btf, src_rec->file_name_off); 1548 1548 + str_off = btf__add_str(linker->btf, s); 1549 1549 + if (str_off < 0) 1550 1550 + return -ENOMEM; 1551 1551 + dst_rec->file_name_off = str_off; 1552 1552 + 1553 1553 + s = btf__str_by_offset(obj->btf, src_rec->line_off); 1554 1554 + str_off = btf__add_str(linker->btf, s); 1555 1555 + if (str_off < 0) 1556 1556 + return -ENOMEM; 1557 1557 + dst_rec->line_off = str_off; 1558 1558 + 1559 1559 + /* dst_rec->line_col is fine */ 1560 1560 + } 1561 1561 + } 1562 1562 + 1563 1563 + rec_sz = obj->btf_ext->core_relo_info.rec_size; 1564 1564 + for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) { 1565 1565 + struct bpf_core_relo *src_rec, *dst_rec; 1566 1566 + 1567 1567 + sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); 1568 1568 + src_sec = find_src_sec_by_name(obj, sec_name); 1569 1569 + if (!src_sec) { 1570 1570 + pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name); 1571 1571 + return -EINVAL; 1572 1572 + } 1573 1573 + dst_sec = &linker->secs[src_sec->dst_id]; 1574 1574 + 1575 1575 + if (dst_sec->core_relo_info.rec_sz == 0) 1576 1576 + dst_sec->core_relo_info.rec_sz = rec_sz; 1577 1577 + if (dst_sec->core_relo_info.rec_sz != rec_sz) { 1578 1578 + pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name); 1579 1579 + return -EINVAL; 1580 1580 + } 1581 1581 + 1582 1582 + for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) { 1583 1583 + dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec); 1584 1584 + if (!dst_rec) 1585 1585 + return -ENOMEM; 1586 1586 + 1587 1587 + dst_rec->insn_off += src_sec->dst_off; 1588 1588 + dst_rec->type_id = obj->btf_type_map[dst_rec->type_id]; 1589 1589 + 1590 1590 + s = btf__str_by_offset(obj->btf, src_rec->access_str_off); 1591 1591 + str_off = btf__add_str(linker->btf, s); 1592 1592 + if (str_off < 0) 1593 1593 + return -ENOMEM; 1594 1594 + dst_rec->access_str_off = str_off; 1595 1595 + 1596 1596 + /* dst_rec->kind is fine */ 1597 1597 + } 1598 1598 + } 1599 1599 + 1600 1600 + return 0; 1601 1601 + } 1602 1602 + 1603 1603 + int bpf_linker__finalize(struct bpf_linker *linker) 1604 1604 + { 1605 1605 + struct dst_sec *sec; 1606 1606 + size_t strs_sz; 1607 1607 + const void *strs; 1608 1608 + int err, i; 1609 1609 + 1610 1610 + if (!linker->elf) 1611 1611 + return -EINVAL; 1612 1612 + 1613 1613 + err = finalize_btf(linker); 1614 1614 + if (err) 1615 1615 + return err; 1616 1616 + 1617 1617 + /* Finalize strings */ 1618 1618 + strs_sz = strset__data_size(linker->strtab_strs); 1619 1619 + strs = strset__data(linker->strtab_strs); 1620 1620 + 1621 1621 + sec = &linker->secs[linker->strtab_sec_idx]; 1622 1622 + sec->data->d_align = 1; 1623 1623 + sec->data->d_off = 0LL; 1624 1624 + sec->data->d_buf = (void *)strs; 1625 1625 + sec->data->d_type = ELF_T_BYTE; 1626 1626 + sec->data->d_size = strs_sz; 1627 1627 + sec->shdr->sh_size = strs_sz; 1628 1628 + 1629 1629 + for (i = 1; i < linker->sec_cnt; i++) { 1630 1630 + sec = &linker->secs[i]; 1631 1631 + 1632 1632 + /* STRTAB is handled specially above */ 1633 1633 + if (sec->sec_idx == linker->strtab_sec_idx) 1634 1634 + continue; 1635 1635 + 1636 1636 + /* special ephemeral sections (.ksyms, .kconfig, etc) */ 1637 1637 + if (!sec->scn) 1638 1638 + continue; 1639 1639 + 1640 1640 + sec->data->d_buf = sec->raw_data; 1641 1641 + } 1642 1642 + 1643 1643 + /* Finalize ELF layout */ 1644 1644 + if (elf_update(linker->elf, ELF_C_NULL) < 0) { 1645 1645 + err = -errno; 1646 1646 + pr_warn_elf("failed to finalize ELF layout"); 1647 1647 + return err; 1648 1648 + } 1649 1649 + 1650 1650 + /* Write out final ELF contents */ 1651 1651 + if (elf_update(linker->elf, ELF_C_WRITE) < 0) { 1652 1652 + err = -errno; 1653 1653 + pr_warn_elf("failed to write ELF contents"); 1654 1654 + return err; 1655 1655 + } 1656 1656 + 1657 1657 + elf_end(linker->elf); 1658 1658 + close(linker->fd); 1659 1659 + 1660 1660 + linker->elf = NULL; 1661 1661 + linker->fd = -1; 1662 1662 + 1663 1663 + return 0; 1664 1664 + } 1665 1665 + 1666 1666 + static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name, 1667 1667 + size_t align, const void *raw_data, size_t raw_sz) 1668 1668 + { 1669 1669 + Elf_Scn *scn; 1670 1670 + Elf_Data *data; 1671 1671 + Elf64_Shdr *shdr; 1672 1672 + int name_off; 1673 1673 + 1674 1674 + name_off = strset__add_str(linker->strtab_strs, sec_name); 1675 1675 + if (name_off < 0) 1676 1676 + return name_off; 1677 1677 + 1678 1678 + scn = elf_newscn(linker->elf); 1679 1679 + if (!scn) 1680 1680 + return -ENOMEM; 1681 1681 + data = elf_newdata(scn); 1682 1682 + if (!data) 1683 1683 + return -ENOMEM; 1684 1684 + shdr = elf64_getshdr(scn); 1685 1685 + if (!shdr) 1686 1686 + return -EINVAL; 1687 1687 + 1688 1688 + shdr->sh_name = name_off; 1689 1689 + shdr->sh_type = SHT_PROGBITS; 1690 1690 + shdr->sh_flags = 0; 1691 1691 + shdr->sh_size = raw_sz; 1692 1692 + shdr->sh_link = 0; 1693 1693 + shdr->sh_info = 0; 1694 1694 + shdr->sh_addralign = align; 1695 1695 + shdr->sh_entsize = 0; 1696 1696 + 1697 1697 + data->d_type = ELF_T_BYTE; 1698 1698 + data->d_size = raw_sz; 1699 1699 + data->d_buf = (void *)raw_data; 1700 1700 + data->d_align = align; 1701 1701 + data->d_off = 0; 1702 1702 + 1703 1703 + return 0; 1704 1704 + } 1705 1705 + 1706 1706 + static int finalize_btf(struct bpf_linker *linker) 1707 1707 + { 1708 1708 + struct btf *btf = linker->btf; 1709 1709 + const void *raw_data; 1710 1710 + int i, j, id, err; 1711 1711 + __u32 raw_sz; 1712 1712 + 1713 1713 + /* bail out if no BTF data was produced */ 1714 1714 + if (btf__get_nr_types(linker->btf) == 0) 1715 1715 + return 0; 1716 1716 + 1717 1717 + for (i = 1; i < linker->sec_cnt; i++) { 1718 1718 + struct dst_sec *sec = &linker->secs[i]; 1719 1719 + 1720 1720 + if (!sec->sec_var_cnt) 1721 1721 + continue; 1722 1722 + 1723 1723 + id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz); 1724 1724 + if (id < 0) { 1725 1725 + pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n", 1726 1726 + sec->sec_name, id); 1727 1727 + return id; 1728 1728 + } 1729 1729 + 1730 1730 + for (j = 0; j < sec->sec_var_cnt; j++) { 1731 1731 + struct btf_var_secinfo *vi = &sec->sec_vars[j]; 1732 1732 + 1733 1733 + if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size)) 1734 1734 + return -EINVAL; 1735 1735 + } 1736 1736 + } 1737 1737 + 1738 1738 + err = finalize_btf_ext(linker); 1739 1739 + if (err) { 1740 1740 + pr_warn(".BTF.ext generation failed: %d\n", err); 1741 1741 + return err; 1742 1742 + } 1743 1743 + 1744 1744 + err = btf__dedup(linker->btf, linker->btf_ext, NULL); 1745 1745 + if (err) { 1746 1746 + pr_warn("BTF dedup failed: %d\n", err); 1747 1747 + return err; 1748 1748 + } 1749 1749 + 1750 1750 + /* Emit .BTF section */ 1751 1751 + raw_data = btf__get_raw_data(linker->btf, &raw_sz); 1752 1752 + if (!raw_data) 1753 1753 + return -ENOMEM; 1754 1754 + 1755 1755 + err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz); 1756 1756 + if (err) { 1757 1757 + pr_warn("failed to write out .BTF ELF section: %d\n", err); 1758 1758 + return err; 1759 1759 + } 1760 1760 + 1761 1761 + /* Emit .BTF.ext section */ 1762 1762 + if (linker->btf_ext) { 1763 1763 + raw_data = btf_ext__get_raw_data(linker->btf_ext, &raw_sz); 1764 1764 + if (!raw_data) 1765 1765 + return -ENOMEM; 1766 1766 + 1767 1767 + err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz); 1768 1768 + if (err) { 1769 1769 + pr_warn("failed to write out .BTF.ext ELF section: %d\n", err); 1770 1770 + return err; 1771 1771 + } 1772 1772 + } 1773 1773 + 1774 1774 + return 0; 1775 1775 + } 1776 1776 + 1777 1777 + static int emit_btf_ext_data(struct bpf_linker *linker, void *output, 1778 1778 + const char *sec_name, struct btf_ext_sec_data *sec_data) 1779 1779 + { 1780 1780 + struct btf_ext_info_sec *sec_info; 1781 1781 + void *cur = output; 1782 1782 + int str_off; 1783 1783 + size_t sz; 1784 1784 + 1785 1785 + if (!sec_data->rec_cnt) 1786 1786 + return 0; 1787 1787 + 1788 1788 + str_off = btf__add_str(linker->btf, sec_name); 1789 1789 + if (str_off < 0) 1790 1790 + return -ENOMEM; 1791 1791 + 1792 1792 + sec_info = cur; 1793 1793 + sec_info->sec_name_off = str_off; 1794 1794 + sec_info->num_info = sec_data->rec_cnt; 1795 1795 + cur += sizeof(struct btf_ext_info_sec); 1796 1796 + 1797 1797 + sz = sec_data->rec_cnt * sec_data->rec_sz; 1798 1798 + memcpy(cur, sec_data->recs, sz); 1799 1799 + cur += sz; 1800 1800 + 1801 1801 + return cur - output; 1802 1802 + } 1803 1803 + 1804 1804 + static int finalize_btf_ext(struct bpf_linker *linker) 1805 1805 + { 1806 1806 + size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0; 1807 1807 + size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0; 1808 1808 + struct btf_ext_header *hdr; 1809 1809 + void *data, *cur; 1810 1810 + int i, err, sz; 1811 1811 + 1812 1812 + /* validate that all sections have the same .BTF.ext record sizes 1813 1813 + * and calculate total data size for each type of data (func info, 1814 1814 + * line info, core relos) 1815 1815 + */ 1816 1816 + for (i = 1; i < linker->sec_cnt; i++) { 1817 1817 + struct dst_sec *sec = &linker->secs[i]; 1818 1818 + 1819 1819 + if (sec->func_info.rec_cnt) { 1820 1820 + if (func_rec_sz == 0) 1821 1821 + func_rec_sz = sec->func_info.rec_sz; 1822 1822 + if (func_rec_sz != sec->func_info.rec_sz) { 1823 1823 + pr_warn("mismatch in func_info record size %zu != %u\n", 1824 1824 + func_rec_sz, sec->func_info.rec_sz); 1825 1825 + return -EINVAL; 1826 1826 + } 1827 1827 + 1828 1828 + funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt; 1829 1829 + } 1830 1830 + if (sec->line_info.rec_cnt) { 1831 1831 + if (line_rec_sz == 0) 1832 1832 + line_rec_sz = sec->line_info.rec_sz; 1833 1833 + if (line_rec_sz != sec->line_info.rec_sz) { 1834 1834 + pr_warn("mismatch in line_info record size %zu != %u\n", 1835 1835 + line_rec_sz, sec->line_info.rec_sz); 1836 1836 + return -EINVAL; 1837 1837 + } 1838 1838 + 1839 1839 + lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt; 1840 1840 + } 1841 1841 + if (sec->core_relo_info.rec_cnt) { 1842 1842 + if (core_relo_rec_sz == 0) 1843 1843 + core_relo_rec_sz = sec->core_relo_info.rec_sz; 1844 1844 + if (core_relo_rec_sz != sec->core_relo_info.rec_sz) { 1845 1845 + pr_warn("mismatch in core_relo_info record size %zu != %u\n", 1846 1846 + core_relo_rec_sz, sec->core_relo_info.rec_sz); 1847 1847 + return -EINVAL; 1848 1848 + } 1849 1849 + 1850 1850 + core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt; 1851 1851 + } 1852 1852 + } 1853 1853 + 1854 1854 + if (!funcs_sz && !lines_sz && !core_relos_sz) 1855 1855 + return 0; 1856 1856 + 1857 1857 + total_sz += sizeof(struct btf_ext_header); 1858 1858 + if (funcs_sz) { 1859 1859 + funcs_sz += sizeof(__u32); /* record size prefix */ 1860 1860 + total_sz += funcs_sz; 1861 1861 + } 1862 1862 + if (lines_sz) { 1863 1863 + lines_sz += sizeof(__u32); /* record size prefix */ 1864 1864 + total_sz += lines_sz; 1865 1865 + } 1866 1866 + if (core_relos_sz) { 1867 1867 + core_relos_sz += sizeof(__u32); /* record size prefix */ 1868 1868 + total_sz += core_relos_sz; 1869 1869 + } 1870 1870 + 1871 1871 + cur = data = calloc(1, total_sz); 1872 1872 + if (!data) 1873 1873 + return -ENOMEM; 1874 1874 + 1875 1875 + hdr = cur; 1876 1876 + hdr->magic = BTF_MAGIC; 1877 1877 + hdr->version = BTF_VERSION; 1878 1878 + hdr->flags = 0; 1879 1879 + hdr->hdr_len = sizeof(struct btf_ext_header); 1880 1880 + cur += sizeof(struct btf_ext_header); 1881 1881 + 1882 1882 + /* All offsets are in bytes relative to the end of this header */ 1883 1883 + hdr->func_info_off = 0; 1884 1884 + hdr->func_info_len = funcs_sz; 1885 1885 + hdr->line_info_off = funcs_sz; 1886 1886 + hdr->line_info_len = lines_sz; 1887 1887 + hdr->core_relo_off = funcs_sz + lines_sz;; 1888 1888 + hdr->core_relo_len = core_relos_sz; 1889 1889 + 1890 1890 + if (funcs_sz) { 1891 1891 + *(__u32 *)cur = func_rec_sz; 1892 1892 + cur += sizeof(__u32); 1893 1893 + 1894 1894 + for (i = 1; i < linker->sec_cnt; i++) { 1895 1895 + struct dst_sec *sec = &linker->secs[i]; 1896 1896 + 1897 1897 + sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info); 1898 1898 + if (sz < 0) 1899 1899 + return sz; 1900 1900 + 1901 1901 + cur += sz; 1902 1902 + } 1903 1903 + } 1904 1904 + 1905 1905 + if (lines_sz) { 1906 1906 + *(__u32 *)cur = line_rec_sz; 1907 1907 + cur += sizeof(__u32); 1908 1908 + 1909 1909 + for (i = 1; i < linker->sec_cnt; i++) { 1910 1910 + struct dst_sec *sec = &linker->secs[i]; 1911 1911 + 1912 1912 + sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info); 1913 1913 + if (sz < 0) 1914 1914 + return sz; 1915 1915 + 1916 1916 + cur += sz; 1917 1917 + } 1918 1918 + } 1919 1919 + 1920 1920 + if (core_relos_sz) { 1921 1921 + *(__u32 *)cur = core_relo_rec_sz; 1922 1922 + cur += sizeof(__u32); 1923 1923 + 1924 1924 + for (i = 1; i < linker->sec_cnt; i++) { 1925 1925 + struct dst_sec *sec = &linker->secs[i]; 1926 1926 + 1927 1927 + sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info); 1928 1928 + if (sz < 0) 1929 1929 + return sz; 1930 1930 + 1931 1931 + cur += sz; 1932 1932 + } 1933 1933 + } 1934 1934 + 1935 1935 + linker->btf_ext = btf_ext__new(data, total_sz); 1936 1936 + err = libbpf_get_error(linker->btf_ext); 1937 1937 + if (err) { 1938 1938 + linker->btf_ext = NULL; 1939 1939 + pr_warn("failed to parse final .BTF.ext data: %d\n", err); 1940 1940 + return err; 1941 1941 + } 1942 1942 + 1943 1943 + return 0; 1944 1944 + }

+176

tools/lib/bpf/strset.c

reviewed

··· 1 1 + // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 2 + /* Copyright (c) 2021 Facebook */ 3 3 + #include <stdint.h> 4 4 + #include <stdlib.h> 5 5 + #include <stdio.h> 6 6 + #include <errno.h> 7 7 + #include <linux/err.h> 8 8 + #include "hashmap.h" 9 9 + #include "libbpf_internal.h" 10 10 + #include "strset.h" 11 11 + 12 12 + struct strset { 13 13 + void *strs_data; 14 14 + size_t strs_data_len; 15 15 + size_t strs_data_cap; 16 16 + size_t strs_data_max_len; 17 17 + 18 18 + /* lookup index for each unique string in strings set */ 19 19 + struct hashmap *strs_hash; 20 20 + }; 21 21 + 22 22 + static size_t strset_hash_fn(const void *key, void *ctx) 23 23 + { 24 24 + const struct strset *s = ctx; 25 25 + const char *str = s->strs_data + (long)key; 26 26 + 27 27 + return str_hash(str); 28 28 + } 29 29 + 30 30 + static bool strset_equal_fn(const void *key1, const void *key2, void *ctx) 31 31 + { 32 32 + const struct strset *s = ctx; 33 33 + const char *str1 = s->strs_data + (long)key1; 34 34 + const char *str2 = s->strs_data + (long)key2; 35 35 + 36 36 + return strcmp(str1, str2) == 0; 37 37 + } 38 38 + 39 39 + struct strset *strset__new(size_t max_data_sz, const char *init_data, size_t init_data_sz) 40 40 + { 41 41 + struct strset *set = calloc(1, sizeof(*set)); 42 42 + struct hashmap *hash; 43 43 + int err = -ENOMEM; 44 44 + 45 45 + if (!set) 46 46 + return ERR_PTR(-ENOMEM); 47 47 + 48 48 + hash = hashmap__new(strset_hash_fn, strset_equal_fn, set); 49 49 + if (IS_ERR(hash)) 50 50 + goto err_out; 51 51 + 52 52 + set->strs_data_max_len = max_data_sz; 53 53 + set->strs_hash = hash; 54 54 + 55 55 + if (init_data) { 56 56 + long off; 57 57 + 58 58 + set->strs_data = malloc(init_data_sz); 59 59 + if (!set->strs_data) 60 60 + goto err_out; 61 61 + 62 62 + memcpy(set->strs_data, init_data, init_data_sz); 63 63 + set->strs_data_len = init_data_sz; 64 64 + set->strs_data_cap = init_data_sz; 65 65 + 66 66 + for (off = 0; off < set->strs_data_len; off += strlen(set->strs_data + off) + 1) { 67 67 + /* hashmap__add() returns EEXIST if string with the same 68 68 + * content already is in the hash map 69 69 + */ 70 70 + err = hashmap__add(hash, (void *)off, (void *)off); 71 71 + if (err == -EEXIST) 72 72 + continue; /* duplicate */ 73 73 + if (err) 74 74 + goto err_out; 75 75 + } 76 76 + } 77 77 + 78 78 + return set; 79 79 + err_out: 80 80 + strset__free(set); 81 81 + return ERR_PTR(err); 82 82 + } 83 83 + 84 84 + void strset__free(struct strset *set) 85 85 + { 86 86 + if (IS_ERR_OR_NULL(set)) 87 87 + return; 88 88 + 89 89 + hashmap__free(set->strs_hash); 90 90 + free(set->strs_data); 91 91 + } 92 92 + 93 93 + size_t strset__data_size(const struct strset *set) 94 94 + { 95 95 + return set->strs_data_len; 96 96 + } 97 97 + 98 98 + const char *strset__data(const struct strset *set) 99 99 + { 100 100 + return set->strs_data; 101 101 + } 102 102 + 103 103 + static void *strset_add_str_mem(struct strset *set, size_t add_sz) 104 104 + { 105 105 + return libbpf_add_mem(&set->strs_data, &set->strs_data_cap, 1, 106 106 + set->strs_data_len, set->strs_data_max_len, add_sz); 107 107 + } 108 108 + 109 109 + /* Find string offset that corresponds to a given string *s*. 110 110 + * Returns: 111 111 + * - >0 offset into string data, if string is found; 112 112 + * - -ENOENT, if string is not in the string data; 113 113 + * - <0, on any other error. 114 114 + */ 115 115 + int strset__find_str(struct strset *set, const char *s) 116 116 + { 117 117 + long old_off, new_off, len; 118 118 + void *p; 119 119 + 120 120 + /* see strset__add_str() for why we do this */ 121 121 + len = strlen(s) + 1; 122 122 + p = strset_add_str_mem(set, len); 123 123 + if (!p) 124 124 + return -ENOMEM; 125 125 + 126 126 + new_off = set->strs_data_len; 127 127 + memcpy(p, s, len); 128 128 + 129 129 + if (hashmap__find(set->strs_hash, (void *)new_off, (void **)&old_off)) 130 130 + return old_off; 131 131 + 132 132 + return -ENOENT; 133 133 + } 134 134 + 135 135 + /* Add a string s to the string data. If the string already exists, return its 136 136 + * offset within string data. 137 137 + * Returns: 138 138 + * - > 0 offset into string data, on success; 139 139 + * - < 0, on error. 140 140 + */ 141 141 + int strset__add_str(struct strset *set, const char *s) 142 142 + { 143 143 + long old_off, new_off, len; 144 144 + void *p; 145 145 + int err; 146 146 + 147 147 + /* Hashmap keys are always offsets within set->strs_data, so to even 148 148 + * look up some string from the "outside", we need to first append it 149 149 + * at the end, so that it can be addressed with an offset. Luckily, 150 150 + * until set->strs_data_len is incremented, that string is just a piece 151 151 + * of garbage for the rest of the code, so no harm, no foul. On the 152 152 + * other hand, if the string is unique, it's already appended and 153 153 + * ready to be used, only a simple set->strs_data_len increment away. 154 154 + */ 155 155 + len = strlen(s) + 1; 156 156 + p = strset_add_str_mem(set, len); 157 157 + if (!p) 158 158 + return -ENOMEM; 159 159 + 160 160 + new_off = set->strs_data_len; 161 161 + memcpy(p, s, len); 162 162 + 163 163 + /* Now attempt to add the string, but only if the string with the same 164 164 + * contents doesn't exist already (HASHMAP_ADD strategy). If such 165 165 + * string exists, we'll get its offset in old_off (that's old_key). 166 166 + */ 167 167 + err = hashmap__insert(set->strs_hash, (void *)new_off, (void *)new_off, 168 168 + HASHMAP_ADD, (const void **)&old_off, NULL); 169 169 + if (err == -EEXIST) 170 170 + return old_off; /* duplicated string, return existing offset */ 171 171 + if (err) 172 172 + return err; 173 173 + 174 174 + set->strs_data_len += len; /* new unique string, adjust data length */ 175 175 + return new_off; 176 176 + }

+21

tools/lib/bpf/strset.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ 2 2 + 3 3 + /* Copyright (c) 2021 Facebook */ 4 4 + #ifndef __LIBBPF_STRSET_H 5 5 + #define __LIBBPF_STRSET_H 6 6 + 7 7 + #include <stdbool.h> 8 8 + #include <stddef.h> 9 9 + 10 10 + struct strset; 11 11 + 12 12 + struct strset *strset__new(size_t max_data_sz, const char *init_data, size_t init_data_sz); 13 13 + void strset__free(struct strset *set); 14 14 + 15 15 + const char *strset__data(const struct strset *set); 16 16 + size_t strset__data_size(const struct strset *set); 17 17 + 18 18 + int strset__find_str(struct strset *set, const char *s); 19 19 + int strset__add_str(struct strset *set, const char *s); 20 20 + 21 21 + #endif /* __LIBBPF_STRSET_H */

+68 -2

tools/lib/bpf/xsk.h

reviewed

··· 3 3 /* 4 4 * AF_XDP user-space access library. 5 5 * 6 6 - * Copyright(c) 2018 - 2019 Intel Corporation. 6 6 + * Copyright (c) 2018 - 2019 Intel Corporation. 7 7 + * Copyright (c) 2019 Facebook 7 8 * 8 9 * Author(s): Magnus Karlsson <magnus.karlsson@intel.com> 9 10 */ ··· 14 13 15 14 #include <stdio.h> 16 15 #include <stdint.h> 16 16 + #include <stdbool.h> 17 17 #include <linux/if_xdp.h> 18 18 19 19 #include "libbpf.h" 20 20 - #include "libbpf_util.h" 21 20 22 21 #ifdef __cplusplus 23 22 extern "C" { 24 23 #endif 24 24 + 25 25 + /* Load-Acquire Store-Release barriers used by the XDP socket 26 26 + * library. The following macros should *NOT* be considered part of 27 27 + * the xsk.h API, and is subject to change anytime. 28 28 + * 29 29 + * LIBRARY INTERNAL 30 30 + */ 31 31 + 32 32 + #define __XSK_READ_ONCE(x) (*(volatile typeof(x) *)&x) 33 33 + #define __XSK_WRITE_ONCE(x, v) (*(volatile typeof(x) *)&x) = (v) 34 34 + 35 35 + #if defined(__i386__) || defined(__x86_64__) 36 36 + # define libbpf_smp_store_release(p, v) \ 37 37 + do { \ 38 38 + asm volatile("" : : : "memory"); \ 39 39 + __XSK_WRITE_ONCE(*p, v); \ 40 40 + } while (0) 41 41 + # define libbpf_smp_load_acquire(p) \ 42 42 + ({ \ 43 43 + typeof(*p) ___p1 = __XSK_READ_ONCE(*p); \ 44 44 + asm volatile("" : : : "memory"); \ 45 45 + ___p1; \ 46 46 + }) 47 47 + #elif defined(__aarch64__) 48 48 + # define libbpf_smp_store_release(p, v) \ 49 49 + asm volatile ("stlr %w1, %0" : "=Q" (*p) : "r" (v) : "memory") 50 50 + # define libbpf_smp_load_acquire(p) \ 51 51 + ({ \ 52 52 + typeof(*p) ___p1; \ 53 53 + asm volatile ("ldar %w0, %1" \ 54 54 + : "=r" (___p1) : "Q" (*p) : "memory"); \ 55 55 + ___p1; \ 56 56 + }) 57 57 + #elif defined(__riscv) 58 58 + # define libbpf_smp_store_release(p, v) \ 59 59 + do { \ 60 60 + asm volatile ("fence rw,w" : : : "memory"); \ 61 61 + __XSK_WRITE_ONCE(*p, v); \ 62 62 + } while (0) 63 63 + # define libbpf_smp_load_acquire(p) \ 64 64 + ({ \ 65 65 + typeof(*p) ___p1 = __XSK_READ_ONCE(*p); \ 66 66 + asm volatile ("fence r,rw" : : : "memory"); \ 67 67 + ___p1; \ 68 68 + }) 69 69 + #endif 70 70 + 71 71 + #ifndef libbpf_smp_store_release 72 72 + #define libbpf_smp_store_release(p, v) \ 73 73 + do { \ 74 74 + __sync_synchronize(); \ 75 75 + __XSK_WRITE_ONCE(*p, v); \ 76 76 + } while (0) 77 77 + #endif 78 78 + 79 79 + #ifndef libbpf_smp_load_acquire 80 80 + #define libbpf_smp_load_acquire(p) \ 81 81 + ({ \ 82 82 + typeof(*p) ___p1 = __XSK_READ_ONCE(*p); \ 83 83 + __sync_synchronize(); \ 84 84 + ___p1; \ 85 85 + }) 86 86 + #endif 87 87 + 88 88 + /* LIBRARY INTERNAL -- END */ 25 89 26 90 /* Do not access these members directly. Use the functions below. */ 27 91 #define DEFINE_XSK_RING(name) \

+27 -7

tools/testing/selftests/bpf/Makefile

reviewed

··· 21 21 22 22 BPF_GCC ?= $(shell command -v bpf-gcc;) 23 23 SAN_CFLAGS ?= 24 24 - CFLAGS += -g -rdynamic -Wall -O2 $(GENFLAGS) $(SAN_CFLAGS) \ 24 24 + CFLAGS += -g -Og -rdynamic -Wall $(GENFLAGS) $(SAN_CFLAGS) \ 25 25 -I$(CURDIR) -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ 26 26 -I$(TOOLSINCDIR) -I$(APIDIR) -I$(OUTPUT) \ 27 27 -Dbpf_prog_load=bpf_prog_test_load \ ··· 201 201 $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/bpftool 202 202 $(Q)$(MAKE) $(submake_extras) -C $(BPFTOOLDIR) \ 203 203 CC=$(HOSTCC) LD=$(HOSTLD) \ 204 204 + EXTRA_CFLAGS='-g -Og' \ 204 205 OUTPUT=$(HOST_BUILD_DIR)/bpftool/ \ 205 206 prefix= DESTDIR=$(HOST_SCRATCH_DIR)/ install 206 207 ··· 219 218 ../../../include/uapi/linux/bpf.h \ 220 219 | $(INCLUDE_DIR) $(BUILD_DIR)/libbpf 221 220 $(Q)$(MAKE) $(submake_extras) -C $(BPFDIR) OUTPUT=$(BUILD_DIR)/libbpf/ \ 221 221 + EXTRA_CFLAGS='-g -Og' \ 222 222 DESTDIR=$(SCRATCH_DIR) prefix= all install_headers 223 223 224 224 ifneq ($(BPFOBJ),$(HOST_BPFOBJ)) ··· 227 225 ../../../include/uapi/linux/bpf.h \ 228 226 | $(INCLUDE_DIR) $(HOST_BUILD_DIR)/libbpf 229 227 $(Q)$(MAKE) $(submake_extras) -C $(BPFDIR) \ 230 230 - OUTPUT=$(HOST_BUILD_DIR)/libbpf/ CC=$(HOSTCC) LD=$(HOSTLD) \ 228 228 + EXTRA_CFLAGS='-g -Og' \ 229 229 + OUTPUT=$(HOST_BUILD_DIR)/libbpf/ CC=$(HOSTCC) LD=$(HOSTLD) \ 231 230 DESTDIR=$(HOST_SCRATCH_DIR)/ prefix= all install_headers 232 231 endif 233 232 234 234 - $(INCLUDE_DIR)/vmlinux.h: $(VMLINUX_BTF) | $(BPFTOOL) $(INCLUDE_DIR) 233 233 + $(INCLUDE_DIR)/vmlinux.h: $(VMLINUX_BTF) $(BPFTOOL) | $(INCLUDE_DIR) 235 234 ifeq ($(VMLINUX_H),) 236 235 $(call msg,GEN,,$@) 237 236 $(Q)$(BPFTOOL) btf dump file $(VMLINUX_BTF) format c > $@ ··· 303 300 304 301 SKEL_BLACKLIST := btf__% test_pinning_invalid.c test_sk_assign.c 305 302 303 303 + LINKED_SKELS := test_static_linked.skel.h 304 304 + 305 305 + test_static_linked.skel.h-deps := test_static_linked1.o test_static_linked2.o 306 306 + 306 307 # Set up extra TRUNNER_XXX "temporary" variables in the environment (relies on 307 308 # $eval()) and pass control to DEFINE_TEST_RUNNER_RULES. 308 309 # Parameters: ··· 327 320 TRUNNER_BPF_SKELS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.skel.h, \ 328 321 $$(filter-out $(SKEL_BLACKLIST), \ 329 322 $$(TRUNNER_BPF_SRCS))) 323 323 + TRUNNER_BPF_SKELS_LINKED := $$(addprefix $$(TRUNNER_OUTPUT)/,$(LINKED_SKELS)) 330 324 TEST_GEN_FILES += $$(TRUNNER_BPF_OBJS) 331 325 332 326 # Evaluate rules now with extra TRUNNER_XXX variables above already defined ··· 360 352 $$(call $(TRUNNER_BPF_BUILD_RULE),$$<,$$@, \ 361 353 $(TRUNNER_BPF_CFLAGS)) 362 354 363 363 - $(TRUNNER_BPF_SKELS): $(TRUNNER_OUTPUT)/%.skel.h: \ 364 364 - $(TRUNNER_OUTPUT)/%.o \ 365 365 - | $(BPFTOOL) $(TRUNNER_OUTPUT) 355 355 + $(TRUNNER_BPF_SKELS): %.skel.h: %.o $(BPFTOOL) | $(TRUNNER_OUTPUT) 366 356 $$(call msg,GEN-SKEL,$(TRUNNER_BINARY),$$@) 367 367 - $(Q)$$(BPFTOOL) gen skeleton $$< > $$@ 357 357 + $(Q)$$(BPFTOOL) gen object $$(<:.o=.linked1.o) $$< 358 358 + $(Q)$$(BPFTOOL) gen object $$(<:.o=.linked2.o) $$(<:.o=.linked1.o) 359 359 + $(Q)$$(BPFTOOL) gen object $$(<:.o=.linked3.o) $$(<:.o=.linked2.o) 360 360 + $(Q)diff $$(<:.o=.linked2.o) $$(<:.o=.linked3.o) 361 361 + $(Q)$$(BPFTOOL) gen skeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.o=)) > $$@ 362 362 + 363 363 + $(TRUNNER_BPF_SKELS_LINKED): $(TRUNNER_BPF_OBJS) $(BPFTOOL) | $(TRUNNER_OUTPUT) 364 364 + $$(call msg,LINK-BPF,$(TRUNNER_BINARY),$$(@:.skel.h=.o)) 365 365 + $(Q)$$(BPFTOOL) gen object $$(@:.skel.h=.linked1.o) $$(addprefix $(TRUNNER_OUTPUT)/,$$($$(@F)-deps)) 366 366 + $(Q)$$(BPFTOOL) gen object $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked1.o) 367 367 + $(Q)$$(BPFTOOL) gen object $$(@:.skel.h=.linked3.o) $$(@:.skel.h=.linked2.o) 368 368 + $(Q)diff $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked3.o) 369 369 + $$(call msg,GEN-SKEL,$(TRUNNER_BINARY),$$@) 370 370 + $(Q)$$(BPFTOOL) gen skeleton $$(@:.skel.h=.linked3.o) name $$(notdir $$(@:.skel.h=)) > $$@ 368 371 endif 369 372 370 373 # ensure we set up tests.h header generation rule just once ··· 397 378 $(TRUNNER_EXTRA_HDRS) \ 398 379 $(TRUNNER_BPF_OBJS) \ 399 380 $(TRUNNER_BPF_SKELS) \ 381 381 + $(TRUNNER_BPF_SKELS_LINKED) \ 400 382 $$(BPFOBJ) | $(TRUNNER_OUTPUT) 401 383 $$(call msg,TEST-OBJ,$(TRUNNER_BINARY),$$@) 402 384 $(Q)cd $$(@D) && $$(CC) -I. $$(CFLAGS) -c $(CURDIR)/$$< $$(LDLIBS) -o $$(@F)

+5 -1

tools/testing/selftests/bpf/get_cgroup_id_user.c

reviewed

··· 57 57 __u32 key = 0, pid; 58 58 int exit_code = 1; 59 59 char buf[256]; 60 60 + const struct timespec req = { 61 61 + .tv_sec = 1, 62 62 + .tv_nsec = 0, 63 63 + }; 60 64 61 65 cgroup_fd = cgroup_setup_and_join(TEST_CGROUP); 62 66 if (CHECK(cgroup_fd < 0, "cgroup_setup_and_join", "err %d errno %d\n", cgroup_fd, errno)) ··· 119 115 goto close_pmu; 120 116 121 117 /* trigger some syscalls */ 122 122 - sleep(1); 118 118 + syscall(__NR_nanosleep, &req, NULL); 123 119 124 120 err = bpf_map_lookup_elem(cgidmap_fd, &key, &kcgid); 125 121 if (CHECK(err, "bpf_map_lookup_elem", "err %d errno %d\n", err, errno))

-5

tools/testing/selftests/bpf/map_tests/array_map_batch_ops.c

reviewed

··· 55 55 int map_fd, *keys, *values, *visited; 56 56 __u32 count, total, total_success; 57 57 const __u32 max_entries = 10; 58 58 - bool nospace_err; 59 58 __u64 batch = 0; 60 59 int err, step; 61 60 DECLARE_LIBBPF_OPTS(bpf_map_batch_opts, opts, ··· 89 90 * elements each. 90 91 */ 91 92 count = step; 92 92 - nospace_err = false; 93 93 while (true) { 94 94 err = bpf_map_lookup_batch(map_fd, 95 95 total ? &batch : NULL, &batch, ··· 104 106 break; 105 107 106 108 } 107 107 - 108 108 - if (nospace_err == true) 109 109 - continue; 110 109 111 110 CHECK(total != max_entries, "lookup with steps", 112 111 "total = %u, max_entries = %u\n", total, max_entries);

+40

tools/testing/selftests/bpf/prog_tests/static_linked.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* Copyright (c) 2019 Facebook */ 3 3 + 4 4 + #include <test_progs.h> 5 5 + #include "test_static_linked.skel.h" 6 6 + 7 7 + void test_static_linked(void) 8 8 + { 9 9 + int err; 10 10 + struct test_static_linked* skel; 11 11 + 12 12 + skel = test_static_linked__open(); 13 13 + if (!ASSERT_OK_PTR(skel, "skel_open")) 14 14 + return; 15 15 + 16 16 + skel->rodata->rovar1 = 1; 17 17 + skel->bss->static_var1 = 2; 18 18 + skel->bss->static_var11 = 3; 19 19 + 20 20 + skel->rodata->rovar2 = 4; 21 21 + skel->bss->static_var2 = 5; 22 22 + skel->bss->static_var22 = 6; 23 23 + 24 24 + err = test_static_linked__load(skel); 25 25 + if (!ASSERT_OK(err, "skel_load")) 26 26 + goto cleanup; 27 27 + 28 28 + err = test_static_linked__attach(skel); 29 29 + if (!ASSERT_OK(err, "skel_attach")) 30 30 + goto cleanup; 31 31 + 32 32 + /* trigger */ 33 33 + usleep(1); 34 34 + 35 35 + ASSERT_EQ(skel->bss->var1, 1 * 2 + 2 + 3, "var1"); 36 36 + ASSERT_EQ(skel->bss->var2, 4 * 3 + 5 + 6, "var2"); 37 37 + 38 38 + cleanup: 39 39 + test_static_linked__destroy(skel); 40 40 + }

+25

tools/testing/selftests/bpf/progs/bind4_prog.c

reviewed

··· 57 57 return 0; 58 58 } 59 59 60 60 + static __inline int bind_reuseport(struct bpf_sock_addr *ctx) 61 61 + { 62 62 + int val = 1; 63 63 + 64 64 + if (bpf_setsockopt(ctx, SOL_SOCKET, SO_REUSEPORT, 65 65 + &val, sizeof(val))) 66 66 + return 1; 67 67 + if (bpf_getsockopt(ctx, SOL_SOCKET, SO_REUSEPORT, 68 68 + &val, sizeof(val)) || !val) 69 69 + return 1; 70 70 + val = 0; 71 71 + if (bpf_setsockopt(ctx, SOL_SOCKET, SO_REUSEPORT, 72 72 + &val, sizeof(val))) 73 73 + return 1; 74 74 + if (bpf_getsockopt(ctx, SOL_SOCKET, SO_REUSEPORT, 75 75 + &val, sizeof(val)) || val) 76 76 + return 1; 77 77 + 78 78 + return 0; 79 79 + } 80 80 + 60 81 static __inline int misc_opts(struct bpf_sock_addr *ctx, int opt) 61 82 { 62 83 int old, tmp, new = 0xeb9f; ··· 146 125 147 126 /* Test for misc socket options. */ 148 127 if (misc_opts(ctx, SO_MARK) || misc_opts(ctx, SO_PRIORITY)) 128 128 + return 0; 129 129 + 130 130 + /* Set reuseport and unset */ 131 131 + if (bind_reuseport(ctx)) 149 132 return 0; 150 133 151 134 ctx->user_ip4 = bpf_htonl(SERV4_REWRITE_IP);

+25

tools/testing/selftests/bpf/progs/bind6_prog.c

reviewed

··· 63 63 return 0; 64 64 } 65 65 66 66 + static __inline int bind_reuseport(struct bpf_sock_addr *ctx) 67 67 + { 68 68 + int val = 1; 69 69 + 70 70 + if (bpf_setsockopt(ctx, SOL_SOCKET, SO_REUSEPORT, 71 71 + &val, sizeof(val))) 72 72 + return 1; 73 73 + if (bpf_getsockopt(ctx, SOL_SOCKET, SO_REUSEPORT, 74 74 + &val, sizeof(val)) || !val) 75 75 + return 1; 76 76 + val = 0; 77 77 + if (bpf_setsockopt(ctx, SOL_SOCKET, SO_REUSEPORT, 78 78 + &val, sizeof(val))) 79 79 + return 1; 80 80 + if (bpf_getsockopt(ctx, SOL_SOCKET, SO_REUSEPORT, 81 81 + &val, sizeof(val)) || val) 82 82 + return 1; 83 83 + 84 84 + return 0; 85 85 + } 86 86 + 66 87 static __inline int misc_opts(struct bpf_sock_addr *ctx, int opt) 67 88 { 68 89 int old, tmp, new = 0xeb9f; ··· 160 139 161 140 /* Test for misc socket options. */ 162 141 if (misc_opts(ctx, SO_MARK) || misc_opts(ctx, SO_PRIORITY)) 142 142 + return 0; 143 143 + 144 144 + /* Set reuseport and unset */ 145 145 + if (bind_reuseport(ctx)) 163 146 return 0; 164 147 165 148 ctx->user_ip6[0] = bpf_htonl(SERV6_REWRITE_IP_0);

+1 -1

tools/testing/selftests/bpf/progs/fentry_test.c

reviewed

··· 64 64 SEC("fentry/bpf_fentry_test7") 65 65 int BPF_PROG(test7, struct bpf_fentry_test_t *arg) 66 66 { 67 67 - if (arg == 0) 67 67 + if (!arg) 68 68 test7_result = 1; 69 69 return 0; 70 70 }

+2 -2

tools/testing/selftests/bpf/progs/fexit_test.c

reviewed

··· 65 65 SEC("fexit/bpf_fentry_test7") 66 66 int BPF_PROG(test7, struct bpf_fentry_test_t *arg) 67 67 { 68 68 - if (arg == 0) 68 68 + if (!arg) 69 69 test7_result = 1; 70 70 return 0; 71 71 } ··· 74 74 SEC("fexit/bpf_fentry_test8") 75 75 int BPF_PROG(test8, struct bpf_fentry_test_t *arg) 76 76 { 77 77 - if (arg->a == 0) 77 77 + if (!arg->a) 78 78 test8_result = 1; 79 79 return 0; 80 80 }

-1

tools/testing/selftests/bpf/progs/skb_pkt_end.c

reviewed

··· 4 4 #include <bpf/bpf_core_read.h> 5 5 #include <bpf/bpf_helpers.h> 6 6 7 7 - #define NULL 0 8 7 #define INLINE __always_inline 9 8 10 9 #define skb_shorter(skb, len) ((void *)(long)(skb)->data + (len) > (void *)(long)skb->data_end)

+1 -1

tools/testing/selftests/bpf/progs/test_global_func10.c

reviewed

··· 14 14 15 15 __noinline int foo(const struct Big *big) 16 16 { 17 17 - if (big == 0) 17 17 + if (!big) 18 18 return 0; 19 19 20 20 return bpf_get_prandom_u32() < big->y;

+30

tools/testing/selftests/bpf/progs/test_static_linked1.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* Copyright (c) 2021 Facebook */ 3 3 + 4 4 + #include <linux/bpf.h> 5 5 + #include <bpf/bpf_helpers.h> 6 6 + 7 7 + /* 8-byte aligned .bss */ 8 8 + static volatile long static_var1; 9 9 + static volatile int static_var11; 10 10 + int var1 = 0; 11 11 + /* 4-byte aligned .rodata */ 12 12 + const volatile int rovar1; 13 13 + 14 14 + /* same "subprog" name in both files */ 15 15 + static __noinline int subprog(int x) 16 16 + { 17 17 + /* but different formula */ 18 18 + return x * 2; 19 19 + } 20 20 + 21 21 + SEC("raw_tp/sys_enter") 22 22 + int handler1(const void *ctx) 23 23 + { 24 24 + var1 = subprog(rovar1) + static_var1 + static_var11; 25 25 + 26 26 + return 0; 27 27 + } 28 28 + 29 29 + char LICENSE[] SEC("license") = "GPL"; 30 30 + int VERSION SEC("version") = 1;

+31

tools/testing/selftests/bpf/progs/test_static_linked2.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* Copyright (c) 2021 Facebook */ 3 3 + 4 4 + #include <linux/bpf.h> 5 5 + #include <bpf/bpf_helpers.h> 6 6 + 7 7 + /* 4-byte aligned .bss */ 8 8 + static volatile int static_var2; 9 9 + static volatile int static_var22; 10 10 + int var2 = 0; 11 11 + /* 8-byte aligned .rodata */ 12 12 + const volatile long rovar2; 13 13 + 14 14 + /* same "subprog" name in both files */ 15 15 + static __noinline int subprog(int x) 16 16 + { 17 17 + /* but different formula */ 18 18 + return x * 3; 19 19 + } 20 20 + 21 21 + SEC("raw_tp/sys_enter") 22 22 + int handler2(const void *ctx) 23 23 + { 24 24 + var2 = subprog(rovar2) + static_var2 + static_var22; 25 25 + 26 26 + return 0; 27 27 + } 28 28 + 29 29 + /* different name and/or type of the variable doesn't matter */ 30 30 + char _license[] SEC("license") = "GPL"; 31 31 + int _version SEC("version") = 1;

+2 -2

tools/testing/selftests/bpf/xdpxceiver.c

reviewed

··· 297 297 static void xsk_populate_fill_ring(struct xsk_umem_info *umem) 298 298 { 299 299 int ret, i; 300 300 - u32 idx; 300 300 + u32 idx = 0; 301 301 302 302 ret = xsk_ring_prod__reserve(&umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS, &idx); 303 303 if (ret != XSK_RING_PROD__DEFAULT_NUM_DESCS) ··· 584 584 585 585 static void tx_only(struct xsk_socket_info *xsk, u32 *frameptr, int batch_size) 586 586 { 587 587 - u32 idx; 587 587 + u32 idx = 0; 588 588 unsigned int i; 589 589 bool tx_invalid_test = stat_test_type == STAT_TEST_TX_INVALID; 590 590 u32 len = tx_invalid_test ? XSK_UMEM__DEFAULT_FRAME_SIZE + 1 : PKT_SIZE;