Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'eth-fbnic-add-timestamping-support'
Vadim Fedorenko says:
====================
eth: fbnic: add timestamping support
The series is to add timestamping support for Meta's NIC driver.
Changelog:
v3 -> v4:
- use adjust_by_scaled_ppm() instead of open coding it
- adjust cached value of high bits of timestamp to be sure it
is older then incoming timestamps
v2 -> v3:
- rebase on top of net-next
- add doc to describe retur value of fbnic_ts40_to_ns()
v1 -> v2:
- adjust comment about using u64 stats locking primitive
- fix typo in the first patch
- Cc Richard
====================
Link: https://patch.msgid.link/20241008181436.4120604-1-vadfed@meta.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
+722
-8
+2
-1
drivers/net/ethernet/meta/fbnic/Makefile
+2
-1
drivers/net/ethernet/meta/fbnic/Makefile
+11
drivers/net/ethernet/meta/fbnic/fbnic.h
+11
drivers/net/ethernet/meta/fbnic/fbnic.h
···
6
6
7
7
#include <linux/interrupt.h>
8
8
#include <linux/io.h>
9
+
#include <linux/ptp_clock_kernel.h>
9
10
#include <linux/types.h>
10
11
#include <linux/workqueue.h>
11
12
···
49
48
50
49
/* Number of TCQs/RCQs available on hardware */
51
50
u16 max_num_queues;
51
+
52
+
/* Lock protecting writes to @time_high, @time_offset of fbnic_netdev,
53
+
* and the HW time CSR machinery.
54
+
*/
55
+
spinlock_t time_lock;
56
+
/* Externally accessible PTP clock, may be NULL */
57
+
struct ptp_clock *ptp;
58
+
struct ptp_clock_info ptp_info;
59
+
/* Last @time_high refresh time in jiffies (to catch stalls) */
60
+
unsigned long last_read;
52
61
53
62
/* Local copy of hardware statistics */
54
63
struct fbnic_hw_stats hw_stats;
+39
drivers/net/ethernet/meta/fbnic/fbnic_csr.h
+39
drivers/net/ethernet/meta/fbnic/fbnic_csr.h
···
413
413
#define FBNIC_TMI_DROP_CTRL 0x04401 /* 0x11004 */
414
414
#define FBNIC_TMI_DROP_CTRL_EN CSR_BIT(0)
415
415
#define FBNIC_CSR_END_TMI 0x0443f /* CSR section delimiter */
416
+
417
+
/* Precision Time Protocol Registers */
418
+
#define FBNIC_CSR_START_PTP 0x04800 /* CSR section delimiter */
419
+
#define FBNIC_PTP_REG_BASE 0x04800 /* 0x12000 */
420
+
421
+
#define FBNIC_PTP_CTRL 0x04800 /* 0x12000 */
422
+
#define FBNIC_PTP_CTRL_EN CSR_BIT(0)
423
+
#define FBNIC_PTP_CTRL_MONO_EN CSR_BIT(4)
424
+
#define FBNIC_PTP_CTRL_TQS_OUT_EN CSR_BIT(8)
425
+
#define FBNIC_PTP_CTRL_MAC_OUT_IVAL CSR_GENMASK(16, 12)
426
+
#define FBNIC_PTP_CTRL_TICK_IVAL CSR_GENMASK(23, 20)
427
+
428
+
#define FBNIC_PTP_ADJUST 0x04801 /* 0x12004 */
429
+
#define FBNIC_PTP_ADJUST_INIT CSR_BIT(0)
430
+
#define FBNIC_PTP_ADJUST_SUB_NUDGE CSR_BIT(8)
431
+
#define FBNIC_PTP_ADJUST_ADD_NUDGE CSR_BIT(16)
432
+
#define FBNIC_PTP_ADJUST_ADDEND_SET CSR_BIT(24)
433
+
434
+
#define FBNIC_PTP_INIT_HI 0x04802 /* 0x12008 */
435
+
#define FBNIC_PTP_INIT_LO 0x04803 /* 0x1200c */
436
+
437
+
#define FBNIC_PTP_NUDGE_NS 0x04804 /* 0x12010 */
438
+
#define FBNIC_PTP_NUDGE_SUBNS 0x04805 /* 0x12014 */
439
+
440
+
#define FBNIC_PTP_ADD_VAL_NS 0x04806 /* 0x12018 */
441
+
#define FBNIC_PTP_ADD_VAL_NS_MASK CSR_GENMASK(15, 0)
442
+
#define FBNIC_PTP_ADD_VAL_SUBNS 0x04807 /* 0x1201c */
443
+
444
+
#define FBNIC_PTP_CTR_VAL_HI 0x04808 /* 0x12020 */
445
+
#define FBNIC_PTP_CTR_VAL_LO 0x04809 /* 0x12024 */
446
+
447
+
#define FBNIC_PTP_MONO_PTP_CTR_HI 0x0480a /* 0x12028 */
448
+
#define FBNIC_PTP_MONO_PTP_CTR_LO 0x0480b /* 0x1202c */
449
+
450
+
#define FBNIC_PTP_CDC_FIFO_STATUS 0x0480c /* 0x12030 */
451
+
#define FBNIC_PTP_SPARE 0x0480d /* 0x12034 */
452
+
#define FBNIC_CSR_END_PTP 0x0480d /* CSR section delimiter */
453
+
416
454
/* Rx Buffer Registers */
417
455
#define FBNIC_CSR_START_RXB 0x08000 /* CSR section delimiter */
418
456
enum {
···
586
548
};
587
549
588
550
#define FBNIC_RPC_ACT_TBL0_DMA_HINT CSR_GENMASK(24, 16)
551
+
#define FBNIC_RPC_ACT_TBL0_TS_ENA CSR_BIT(28)
589
552
#define FBNIC_RPC_ACT_TBL0_RSS_CTXT_ID CSR_BIT(30)
590
553
591
554
#define FBNIC_RPC_ACT_TBL1_DEFAULT 0x0840b /* 0x2102c */
+54
drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c
+54
drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c
···
6
6
#include "fbnic_netdev.h"
7
7
#include "fbnic_tlv.h"
8
8
9
+
static int
10
+
fbnic_get_ts_info(struct net_device *netdev,
11
+
struct kernel_ethtool_ts_info *tsinfo)
12
+
{
13
+
struct fbnic_net *fbn = netdev_priv(netdev);
14
+
15
+
tsinfo->phc_index = ptp_clock_index(fbn->fbd->ptp);
16
+
17
+
tsinfo->so_timestamping =
18
+
SOF_TIMESTAMPING_TX_SOFTWARE |
19
+
SOF_TIMESTAMPING_TX_HARDWARE |
20
+
SOF_TIMESTAMPING_RX_HARDWARE |
21
+
SOF_TIMESTAMPING_RAW_HARDWARE;
22
+
23
+
tsinfo->tx_types =
24
+
BIT(HWTSTAMP_TX_OFF) |
25
+
BIT(HWTSTAMP_TX_ON);
26
+
27
+
tsinfo->rx_filters =
28
+
BIT(HWTSTAMP_FILTER_NONE) |
29
+
BIT(HWTSTAMP_FILTER_PTP_V1_L4_EVENT) |
30
+
BIT(HWTSTAMP_FILTER_PTP_V2_L4_EVENT) |
31
+
BIT(HWTSTAMP_FILTER_PTP_V2_L2_EVENT) |
32
+
BIT(HWTSTAMP_FILTER_PTP_V2_EVENT) |
33
+
BIT(HWTSTAMP_FILTER_ALL);
34
+
35
+
return 0;
36
+
}
37
+
9
38
static void
10
39
fbnic_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo)
11
40
{
···
93
64
&mac_stats->eth_mac.FrameTooLongErrors);
94
65
}
95
66
67
+
static void fbnic_get_ts_stats(struct net_device *netdev,
68
+
struct ethtool_ts_stats *ts_stats)
69
+
{
70
+
struct fbnic_net *fbn = netdev_priv(netdev);
71
+
u64 ts_packets, ts_lost;
72
+
struct fbnic_ring *ring;
73
+
unsigned int start;
74
+
int i;
75
+
76
+
ts_stats->pkts = fbn->tx_stats.ts_packets;
77
+
ts_stats->lost = fbn->tx_stats.ts_lost;
78
+
for (i = 0; i < fbn->num_tx_queues; i++) {
79
+
ring = fbn->tx[i];
80
+
do {
81
+
start = u64_stats_fetch_begin(&ring->stats.syncp);
82
+
ts_packets = ring->stats.ts_packets;
83
+
ts_lost = ring->stats.ts_lost;
84
+
} while (u64_stats_fetch_retry(&ring->stats.syncp, start));
85
+
ts_stats->pkts += ts_packets;
86
+
ts_stats->lost += ts_lost;
87
+
}
88
+
}
89
+
96
90
static const struct ethtool_ops fbnic_ethtool_ops = {
97
91
.get_drvinfo = fbnic_get_drvinfo,
92
+
.get_ts_info = fbnic_get_ts_info,
93
+
.get_ts_stats = fbnic_get_ts_stats,
98
94
.get_eth_mac_stats = fbnic_get_eth_mac_stats,
99
95
};
100
96
+89
-2
drivers/net/ethernet/meta/fbnic/fbnic_netdev.c
+89
-2
drivers/net/ethernet/meta/fbnic/fbnic_netdev.c
···
42
42
goto free_resources;
43
43
}
44
44
45
-
err = fbnic_fw_init_heartbeat(fbd, false);
45
+
err = fbnic_time_start(fbn);
46
46
if (err)
47
47
goto release_ownership;
48
48
49
+
err = fbnic_fw_init_heartbeat(fbd, false);
50
+
if (err)
51
+
goto time_stop;
52
+
49
53
err = fbnic_pcs_irq_enable(fbd);
50
54
if (err)
51
-
goto release_ownership;
55
+
goto time_stop;
52
56
/* Pull the BMC config and initialize the RPC */
53
57
fbnic_bmc_rpc_init(fbd);
54
58
fbnic_rss_reinit(fbd, fbn);
55
59
56
60
return 0;
61
+
time_stop:
62
+
fbnic_time_stop(fbn);
57
63
release_ownership:
58
64
fbnic_fw_xmit_ownership_msg(fbn->fbd, false);
59
65
free_resources:
···
88
82
fbnic_down(fbn);
89
83
fbnic_pcs_irq_disable(fbn->fbd);
90
84
85
+
fbnic_time_stop(fbn);
91
86
fbnic_fw_xmit_ownership_msg(fbn->fbd, false);
92
87
93
88
fbnic_free_resources(fbn);
···
324
317
__dev_mc_unsync(netdev, NULL);
325
318
}
326
319
320
+
static int fbnic_hwtstamp_get(struct net_device *netdev,
321
+
struct kernel_hwtstamp_config *config)
322
+
{
323
+
struct fbnic_net *fbn = netdev_priv(netdev);
324
+
325
+
*config = fbn->hwtstamp_config;
326
+
327
+
return 0;
328
+
}
329
+
330
+
static int fbnic_hwtstamp_set(struct net_device *netdev,
331
+
struct kernel_hwtstamp_config *config,
332
+
struct netlink_ext_ack *extack)
333
+
{
334
+
struct fbnic_net *fbn = netdev_priv(netdev);
335
+
int old_rx_filter;
336
+
337
+
if (config->source != HWTSTAMP_SOURCE_NETDEV)
338
+
return -EOPNOTSUPP;
339
+
340
+
if (!kernel_hwtstamp_config_changed(config, &fbn->hwtstamp_config))
341
+
return 0;
342
+
343
+
/* Upscale the filters */
344
+
switch (config->rx_filter) {
345
+
case HWTSTAMP_FILTER_NONE:
346
+
case HWTSTAMP_FILTER_ALL:
347
+
case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
348
+
case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
349
+
case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
350
+
case HWTSTAMP_FILTER_PTP_V2_EVENT:
351
+
break;
352
+
case HWTSTAMP_FILTER_NTP_ALL:
353
+
config->rx_filter = HWTSTAMP_FILTER_ALL;
354
+
break;
355
+
case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
356
+
case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
357
+
config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
358
+
break;
359
+
case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
360
+
case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
361
+
config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
362
+
break;
363
+
case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
364
+
case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
365
+
config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT;
366
+
break;
367
+
case HWTSTAMP_FILTER_PTP_V2_SYNC:
368
+
case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
369
+
config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
370
+
break;
371
+
default:
372
+
return -ERANGE;
373
+
}
374
+
375
+
/* Configure */
376
+
old_rx_filter = fbn->hwtstamp_config.rx_filter;
377
+
memcpy(&fbn->hwtstamp_config, config, sizeof(*config));
378
+
379
+
if (old_rx_filter != config->rx_filter && netif_running(fbn->netdev)) {
380
+
fbnic_rss_reinit(fbn->fbd, fbn);
381
+
fbnic_write_rules(fbn->fbd);
382
+
}
383
+
384
+
/* Save / report back filter configuration
385
+
* Note that our filter configuration is inexact. Instead of
386
+
* filtering for a specific UDP port or L2 Ethertype we are
387
+
* filtering in all UDP or all non-IP packets for timestamping. So
388
+
* if anything other than FILTER_ALL is requested we report
389
+
* FILTER_SOME indicating that we will be timestamping a few
390
+
* additional packets.
391
+
*/
392
+
if (config->rx_filter > HWTSTAMP_FILTER_ALL)
393
+
config->rx_filter = HWTSTAMP_FILTER_SOME;
394
+
395
+
return 0;
396
+
}
397
+
327
398
static void fbnic_get_stats64(struct net_device *dev,
328
399
struct rtnl_link_stats64 *stats64)
329
400
{
···
479
394
.ndo_set_mac_address = fbnic_set_mac,
480
395
.ndo_set_rx_mode = fbnic_set_rx_mode,
481
396
.ndo_get_stats64 = fbnic_get_stats64,
397
+
.ndo_hwtstamp_get = fbnic_hwtstamp_get,
398
+
.ndo_hwtstamp_set = fbnic_hwtstamp_set,
482
399
};
483
400
484
401
static void fbnic_get_queue_stats_rx(struct net_device *dev, int idx,
+18
drivers/net/ethernet/meta/fbnic/fbnic_netdev.h
+18
drivers/net/ethernet/meta/fbnic/fbnic_netdev.h
···
33
33
u8 fec;
34
34
u8 link_mode;
35
35
36
+
/* Cached top bits of the HW time counter for 40b -> 64b conversion */
37
+
u32 time_high;
38
+
/* Protect readers of @time_offset, writers take @time_lock. */
39
+
struct u64_stats_sync time_seq;
40
+
/* Offset in ns between free running NIC PHC and time set via PTP
41
+
* clock callbacks
42
+
*/
43
+
s64 time_offset;
44
+
36
45
u16 num_tx_queues;
37
46
u16 num_rx_queues;
38
47
···
53
44
struct fbnic_queue_stats tx_stats;
54
45
struct fbnic_queue_stats rx_stats;
55
46
u64 link_down_events;
47
+
48
+
/* Time stampinn filter config */
49
+
struct kernel_hwtstamp_config hwtstamp_config;
56
50
57
51
struct list_head napis;
58
52
};
···
71
59
void fbnic_reset_queues(struct fbnic_net *fbn,
72
60
unsigned int tx, unsigned int rx);
73
61
void fbnic_set_ethtool_ops(struct net_device *dev);
62
+
63
+
int fbnic_ptp_setup(struct fbnic_dev *fbd);
64
+
void fbnic_ptp_destroy(struct fbnic_dev *fbd);
65
+
void fbnic_time_init(struct fbnic_net *fbn);
66
+
int fbnic_time_start(struct fbnic_net *fbn);
67
+
void fbnic_time_stop(struct fbnic_net *fbn);
74
68
75
69
void __fbnic_set_rx_mode(struct net_device *netdev);
76
70
void fbnic_clear_rx_mode(struct net_device *netdev);
+8
-1
drivers/net/ethernet/meta/fbnic/fbnic_pci.c
+8
-1
drivers/net/ethernet/meta/fbnic/fbnic_pci.c
···
300
300
goto init_failure_mode;
301
301
}
302
302
303
+
err = fbnic_ptp_setup(fbd);
304
+
if (err)
305
+
goto ifm_free_netdev;
306
+
303
307
err = fbnic_netdev_register(netdev);
304
308
if (err) {
305
309
dev_err(&pdev->dev, "Netdev registration failed: %d\n", err);
306
-
goto ifm_free_netdev;
310
+
goto ifm_destroy_ptp;
307
311
}
308
312
309
313
return 0;
310
314
315
+
ifm_destroy_ptp:
316
+
fbnic_ptp_destroy(fbd);
311
317
ifm_free_netdev:
312
318
fbnic_netdev_free(fbd);
313
319
init_failure_mode:
···
348
342
349
343
fbnic_netdev_unregister(netdev);
350
344
cancel_delayed_work_sync(&fbd->service_task);
345
+
fbnic_ptp_destroy(fbd);
351
346
fbnic_netdev_free(fbd);
352
347
}
353
348
+31
drivers/net/ethernet/meta/fbnic/fbnic_rpc.c
+31
drivers/net/ethernet/meta/fbnic/fbnic_rpc.c
···
244
244
((_ip) ? FBNIC_RPC_TCAM_ACT1_IP_VALID : 0) | \
245
245
((_v6) ? FBNIC_RPC_TCAM_ACT1_IP_IS_V6 : 0))
246
246
247
+
#define FBNIC_TSTAMP_MASK(_all, _udp, _ether) \
248
+
(((_all) ? ((1u << FBNIC_NUM_HASH_OPT) - 1) : 0) | \
249
+
((_udp) ? (1u << FBNIC_UDP6_HASH_OPT) | \
250
+
(1u << FBNIC_UDP4_HASH_OPT) : 0) | \
251
+
((_ether) ? (1u << FBNIC_ETHER_HASH_OPT) : 0))
252
+
247
253
void fbnic_rss_reinit(struct fbnic_dev *fbd, struct fbnic_net *fbn)
248
254
{
249
255
static const u32 act1_value[FBNIC_NUM_HASH_OPT] = {
···
261
255
FBNIC_ACT1_INIT(0, 0, 1, 0), /* IP4 */
262
256
0 /* Ether */
263
257
};
258
+
u32 tstamp_mask = 0;
264
259
unsigned int i;
265
260
266
261
/* To support scenarios where a BMC is present we must write the
···
270
263
*/
271
264
BUILD_BUG_ON(FBNIC_RSS_EN_NUM_UNICAST * 2 != FBNIC_RSS_EN_NUM_ENTRIES);
272
265
BUILD_BUG_ON(ARRAY_SIZE(act1_value) != FBNIC_NUM_HASH_OPT);
266
+
267
+
/* Set timestamp mask with 1b per flow type */
268
+
if (fbn->hwtstamp_config.rx_filter != HWTSTAMP_FILTER_NONE) {
269
+
switch (fbn->hwtstamp_config.rx_filter) {
270
+
case HWTSTAMP_FILTER_ALL:
271
+
tstamp_mask = FBNIC_TSTAMP_MASK(1, 1, 1);
272
+
break;
273
+
case HWTSTAMP_FILTER_PTP_V2_EVENT:
274
+
tstamp_mask = FBNIC_TSTAMP_MASK(0, 1, 1);
275
+
break;
276
+
case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
277
+
case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
278
+
tstamp_mask = FBNIC_TSTAMP_MASK(0, 1, 0);
279
+
break;
280
+
case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
281
+
tstamp_mask = FBNIC_TSTAMP_MASK(0, 0, 1);
282
+
break;
283
+
default:
284
+
netdev_warn(fbn->netdev, "Unsupported hwtstamp_rx_filter\n");
285
+
break;
286
+
}
287
+
}
273
288
274
289
/* Program RSS hash enable mask for host in action TCAM/table. */
275
290
for (i = fbnic_bmc_present(fbd) ? 0 : FBNIC_RSS_EN_NUM_UNICAST;
···
316
287
317
288
if (!dest)
318
289
dest = FBNIC_RPC_ACT_TBL0_DROP;
290
+
else if (tstamp_mask & (1u << flow_type))
291
+
dest |= FBNIC_RPC_ACT_TBL0_TS_ENA;
319
292
320
293
if (act1_value[flow_type] & FBNIC_RPC_TCAM_ACT1_L4_VALID)
321
294
dest |= FIELD_PREP(FBNIC_RPC_ACT_TBL0_DMA_HINT,
+303
drivers/net/ethernet/meta/fbnic/fbnic_time.c
+303
drivers/net/ethernet/meta/fbnic/fbnic_time.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
/* Copyright (c) Meta Platforms, Inc. and affiliates. */
3
+
4
+
#include <linux/bitfield.h>
5
+
#include <linux/jiffies.h>
6
+
#include <linux/limits.h>
7
+
#include <linux/ptp_clock_kernel.h>
8
+
#include <linux/timer.h>
9
+
10
+
#include "fbnic.h"
11
+
#include "fbnic_csr.h"
12
+
#include "fbnic_netdev.h"
13
+
14
+
/* FBNIC timing & PTP implementation
15
+
* Datapath uses truncated 40b timestamps for scheduling and event reporting.
16
+
* We need to promote those to full 64b, hence we periodically cache the top
17
+
* 32bit of the HW time counter. Since this makes our time reporting non-atomic
18
+
* we leave the HW clock free running and adjust time offsets in SW as needed.
19
+
* Time offset is 64bit - we need a seq counter for 32bit machines.
20
+
* Time offset and the cache of top bits are independent so we don't need
21
+
* a coherent snapshot of both - READ_ONCE()/WRITE_ONCE() + writer side lock
22
+
* are enough.
23
+
*/
24
+
25
+
/* Period of refresh of top bits of timestamp, give ourselves a 8x margin.
26
+
* This should translate to once a minute.
27
+
* The use of nsecs_to_jiffies() should be safe for a <=40b nsec value.
28
+
*/
29
+
#define FBNIC_TS_HIGH_REFRESH_JIF nsecs_to_jiffies((1ULL << 40) / 16)
30
+
31
+
static struct fbnic_dev *fbnic_from_ptp_info(struct ptp_clock_info *ptp)
32
+
{
33
+
return container_of(ptp, struct fbnic_dev, ptp_info);
34
+
}
35
+
36
+
/* This function is "slow" because we could try guessing which high part
37
+
* is correct based on low instead of re-reading, and skip reading @hi
38
+
* twice altogether if @lo is far enough from 0.
39
+
*/
40
+
static u64 __fbnic_time_get_slow(struct fbnic_dev *fbd)
41
+
{
42
+
u32 hi, lo;
43
+
44
+
lockdep_assert_held(&fbd->time_lock);
45
+
46
+
do {
47
+
hi = fbnic_rd32(fbd, FBNIC_PTP_CTR_VAL_HI);
48
+
lo = fbnic_rd32(fbd, FBNIC_PTP_CTR_VAL_LO);
49
+
} while (hi != fbnic_rd32(fbd, FBNIC_PTP_CTR_VAL_HI));
50
+
51
+
return (u64)hi << 32 | lo;
52
+
}
53
+
54
+
static void __fbnic_time_set_addend(struct fbnic_dev *fbd, u64 addend)
55
+
{
56
+
lockdep_assert_held(&fbd->time_lock);
57
+
58
+
fbnic_wr32(fbd, FBNIC_PTP_ADD_VAL_NS,
59
+
FIELD_PREP(FBNIC_PTP_ADD_VAL_NS_MASK, addend >> 32));
60
+
fbnic_wr32(fbd, FBNIC_PTP_ADD_VAL_SUBNS, (u32)addend);
61
+
}
62
+
63
+
static void fbnic_ptp_fresh_check(struct fbnic_dev *fbd)
64
+
{
65
+
if (time_is_after_jiffies(fbd->last_read +
66
+
FBNIC_TS_HIGH_REFRESH_JIF * 3 / 2))
67
+
return;
68
+
69
+
dev_warn(fbd->dev, "NIC timestamp refresh stall, delayed by %lu sec\n",
70
+
(jiffies - fbd->last_read - FBNIC_TS_HIGH_REFRESH_JIF) / HZ);
71
+
}
72
+
73
+
static void fbnic_ptp_refresh_time(struct fbnic_dev *fbd, struct fbnic_net *fbn)
74
+
{
75
+
unsigned long flags;
76
+
u32 hi;
77
+
78
+
spin_lock_irqsave(&fbd->time_lock, flags);
79
+
hi = fbnic_rd32(fbn->fbd, FBNIC_PTP_CTR_VAL_HI);
80
+
if (!fbnic_present(fbd))
81
+
goto out; /* Don't bother handling, reset is pending */
82
+
/* Let's keep high cached value a bit lower to avoid race with
83
+
* incoming timestamps. The logic in fbnic_ts40_to_ns() will
84
+
* take care of overflow in this case. It will make cached time
85
+
* ~1 minute lower and incoming timestamp will always be later
86
+
* then cached time.
87
+
*/
88
+
WRITE_ONCE(fbn->time_high, hi - 16);
89
+
fbd->last_read = jiffies;
90
+
out:
91
+
spin_unlock_irqrestore(&fbd->time_lock, flags);
92
+
}
93
+
94
+
static long fbnic_ptp_do_aux_work(struct ptp_clock_info *ptp)
95
+
{
96
+
struct fbnic_dev *fbd = fbnic_from_ptp_info(ptp);
97
+
struct fbnic_net *fbn;
98
+
99
+
fbn = netdev_priv(fbd->netdev);
100
+
101
+
fbnic_ptp_fresh_check(fbd);
102
+
fbnic_ptp_refresh_time(fbd, fbn);
103
+
104
+
return FBNIC_TS_HIGH_REFRESH_JIF;
105
+
}
106
+
107
+
static int fbnic_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm)
108
+
{
109
+
struct fbnic_dev *fbd = fbnic_from_ptp_info(ptp);
110
+
u64 addend, dclk_period;
111
+
unsigned long flags;
112
+
113
+
/* d_clock is 600 MHz; which in Q16.32 fixed point ns is: */
114
+
dclk_period = (((u64)1000000000) << 32) / FBNIC_CLOCK_FREQ;
115
+
addend = adjust_by_scaled_ppm(dclk_period, scaled_ppm);
116
+
117
+
spin_lock_irqsave(&fbd->time_lock, flags);
118
+
__fbnic_time_set_addend(fbd, addend);
119
+
fbnic_wr32(fbd, FBNIC_PTP_ADJUST, FBNIC_PTP_ADJUST_ADDEND_SET);
120
+
121
+
/* Flush, make sure FBNIC_PTP_ADD_VAL_* is stable for at least 4 clks */
122
+
fbnic_rd32(fbd, FBNIC_PTP_SPARE);
123
+
spin_unlock_irqrestore(&fbd->time_lock, flags);
124
+
125
+
return fbnic_present(fbd) ? 0 : -EIO;
126
+
}
127
+
128
+
static int fbnic_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
129
+
{
130
+
struct fbnic_dev *fbd = fbnic_from_ptp_info(ptp);
131
+
struct fbnic_net *fbn;
132
+
unsigned long flags;
133
+
134
+
fbn = netdev_priv(fbd->netdev);
135
+
136
+
spin_lock_irqsave(&fbd->time_lock, flags);
137
+
u64_stats_update_begin(&fbn->time_seq);
138
+
WRITE_ONCE(fbn->time_offset, READ_ONCE(fbn->time_offset) + delta);
139
+
u64_stats_update_end(&fbn->time_seq);
140
+
spin_unlock_irqrestore(&fbd->time_lock, flags);
141
+
142
+
return 0;
143
+
}
144
+
145
+
static int
146
+
fbnic_ptp_gettimex64(struct ptp_clock_info *ptp, struct timespec64 *ts,
147
+
struct ptp_system_timestamp *sts)
148
+
{
149
+
struct fbnic_dev *fbd = fbnic_from_ptp_info(ptp);
150
+
struct fbnic_net *fbn;
151
+
unsigned long flags;
152
+
u64 time_ns;
153
+
u32 hi, lo;
154
+
155
+
fbn = netdev_priv(fbd->netdev);
156
+
157
+
spin_lock_irqsave(&fbd->time_lock, flags);
158
+
159
+
do {
160
+
hi = fbnic_rd32(fbd, FBNIC_PTP_CTR_VAL_HI);
161
+
ptp_read_system_prets(sts);
162
+
lo = fbnic_rd32(fbd, FBNIC_PTP_CTR_VAL_LO);
163
+
ptp_read_system_postts(sts);
164
+
/* Similarly to comment above __fbnic_time_get_slow()
165
+
* - this can be optimized if needed.
166
+
*/
167
+
} while (hi != fbnic_rd32(fbd, FBNIC_PTP_CTR_VAL_HI));
168
+
169
+
time_ns = ((u64)hi << 32 | lo) + fbn->time_offset;
170
+
spin_unlock_irqrestore(&fbd->time_lock, flags);
171
+
172
+
if (!fbnic_present(fbd))
173
+
return -EIO;
174
+
175
+
*ts = ns_to_timespec64(time_ns);
176
+
177
+
return 0;
178
+
}
179
+
180
+
static int
181
+
fbnic_ptp_settime64(struct ptp_clock_info *ptp, const struct timespec64 *ts)
182
+
{
183
+
struct fbnic_dev *fbd = fbnic_from_ptp_info(ptp);
184
+
struct fbnic_net *fbn;
185
+
unsigned long flags;
186
+
u64 dev_ns, host_ns;
187
+
int ret;
188
+
189
+
fbn = netdev_priv(fbd->netdev);
190
+
191
+
host_ns = timespec64_to_ns(ts);
192
+
193
+
spin_lock_irqsave(&fbd->time_lock, flags);
194
+
195
+
dev_ns = __fbnic_time_get_slow(fbd);
196
+
197
+
if (fbnic_present(fbd)) {
198
+
u64_stats_update_begin(&fbn->time_seq);
199
+
WRITE_ONCE(fbn->time_offset, host_ns - dev_ns);
200
+
u64_stats_update_end(&fbn->time_seq);
201
+
ret = 0;
202
+
} else {
203
+
ret = -EIO;
204
+
}
205
+
spin_unlock_irqrestore(&fbd->time_lock, flags);
206
+
207
+
return ret;
208
+
}
209
+
210
+
static const struct ptp_clock_info fbnic_ptp_info = {
211
+
.owner = THIS_MODULE,
212
+
/* 1,000,000,000 - 1 PPB to ensure increment is positive
213
+
* after max negative adjustment.
214
+
*/
215
+
.max_adj = 999999999,
216
+
.do_aux_work = fbnic_ptp_do_aux_work,
217
+
.adjfine = fbnic_ptp_adjfine,
218
+
.adjtime = fbnic_ptp_adjtime,
219
+
.gettimex64 = fbnic_ptp_gettimex64,
220
+
.settime64 = fbnic_ptp_settime64,
221
+
};
222
+
223
+
static void fbnic_ptp_reset(struct fbnic_dev *fbd)
224
+
{
225
+
struct fbnic_net *fbn = netdev_priv(fbd->netdev);
226
+
u64 dclk_period;
227
+
228
+
fbnic_wr32(fbd, FBNIC_PTP_CTRL,
229
+
FBNIC_PTP_CTRL_EN |
230
+
FIELD_PREP(FBNIC_PTP_CTRL_TICK_IVAL, 1));
231
+
232
+
/* d_clock is 600 MHz; which in Q16.32 fixed point ns is: */
233
+
dclk_period = (((u64)1000000000) << 32) / FBNIC_CLOCK_FREQ;
234
+
235
+
__fbnic_time_set_addend(fbd, dclk_period);
236
+
237
+
fbnic_wr32(fbd, FBNIC_PTP_INIT_HI, 0);
238
+
fbnic_wr32(fbd, FBNIC_PTP_INIT_LO, 0);
239
+
240
+
fbnic_wr32(fbd, FBNIC_PTP_ADJUST, FBNIC_PTP_ADJUST_INIT);
241
+
242
+
fbnic_wr32(fbd, FBNIC_PTP_CTRL,
243
+
FBNIC_PTP_CTRL_EN |
244
+
FBNIC_PTP_CTRL_TQS_OUT_EN |
245
+
FIELD_PREP(FBNIC_PTP_CTRL_MAC_OUT_IVAL, 3) |
246
+
FIELD_PREP(FBNIC_PTP_CTRL_TICK_IVAL, 1));
247
+
248
+
fbnic_rd32(fbd, FBNIC_PTP_SPARE);
249
+
250
+
fbn->time_offset = 0;
251
+
fbn->time_high = 0;
252
+
}
253
+
254
+
void fbnic_time_init(struct fbnic_net *fbn)
255
+
{
256
+
/* This is not really a statistic, but the lockng primitive fits
257
+
* our usecase perfectly, we need an atomic 8 bytes READ_ONCE() /
258
+
* WRITE_ONCE() behavior.
259
+
*/
260
+
u64_stats_init(&fbn->time_seq);
261
+
}
262
+
263
+
int fbnic_time_start(struct fbnic_net *fbn)
264
+
{
265
+
fbnic_ptp_refresh_time(fbn->fbd, fbn);
266
+
/* Assume that fbnic_ptp_do_aux_work() will never be called if not
267
+
* scheduled here
268
+
*/
269
+
return ptp_schedule_worker(fbn->fbd->ptp, FBNIC_TS_HIGH_REFRESH_JIF);
270
+
}
271
+
272
+
void fbnic_time_stop(struct fbnic_net *fbn)
273
+
{
274
+
ptp_cancel_worker_sync(fbn->fbd->ptp);
275
+
fbnic_ptp_fresh_check(fbn->fbd);
276
+
}
277
+
278
+
int fbnic_ptp_setup(struct fbnic_dev *fbd)
279
+
{
280
+
struct device *dev = fbd->dev;
281
+
unsigned long flags;
282
+
283
+
spin_lock_init(&fbd->time_lock);
284
+
285
+
spin_lock_irqsave(&fbd->time_lock, flags); /* Appease lockdep */
286
+
fbnic_ptp_reset(fbd);
287
+
spin_unlock_irqrestore(&fbd->time_lock, flags);
288
+
289
+
memcpy(&fbd->ptp_info, &fbnic_ptp_info, sizeof(fbnic_ptp_info));
290
+
291
+
fbd->ptp = ptp_clock_register(&fbd->ptp_info, dev);
292
+
if (IS_ERR(fbd->ptp))
293
+
dev_err(dev, "Failed to register PTP: %pe\n", fbd->ptp);
294
+
295
+
return PTR_ERR_OR_ZERO(fbd->ptp);
296
+
}
297
+
298
+
void fbnic_ptp_destroy(struct fbnic_dev *fbd)
299
+
{
300
+
if (!fbd->ptp)
301
+
return;
302
+
ptp_clock_unregister(fbd->ptp);
303
+
}
+164
-4
drivers/net/ethernet/meta/fbnic/fbnic_txrx.c
+164
-4
drivers/net/ethernet/meta/fbnic/fbnic_txrx.c
···
12
12
#include "fbnic_netdev.h"
13
13
#include "fbnic_txrx.h"
14
14
15
+
enum {
16
+
FBNIC_XMIT_CB_TS = 0x01,
17
+
};
18
+
15
19
struct fbnic_xmit_cb {
16
20
u32 bytecount;
17
21
u8 desc_count;
22
+
u8 flags;
18
23
int hw_head;
19
24
};
20
25
···
46
41
u32 __iomem *csr_base = fbnic_ring_csr_base(ring);
47
42
48
43
writel(val, csr_base + csr);
44
+
}
45
+
46
+
/**
47
+
* fbnic_ts40_to_ns() - convert descriptor timestamp to PHC time
48
+
* @fbn: netdev priv of the FB NIC
49
+
* @ts40: timestamp read from a descriptor
50
+
*
51
+
* Return: u64 value of PHC time in nanoseconds
52
+
*
53
+
* Convert truncated 40 bit device timestamp as read from a descriptor
54
+
* to the full PHC time in nanoseconds.
55
+
*/
56
+
static __maybe_unused u64 fbnic_ts40_to_ns(struct fbnic_net *fbn, u64 ts40)
57
+
{
58
+
unsigned int s;
59
+
u64 time_ns;
60
+
s64 offset;
61
+
u8 ts_top;
62
+
u32 high;
63
+
64
+
do {
65
+
s = u64_stats_fetch_begin(&fbn->time_seq);
66
+
offset = READ_ONCE(fbn->time_offset);
67
+
} while (u64_stats_fetch_retry(&fbn->time_seq, s));
68
+
69
+
high = READ_ONCE(fbn->time_high);
70
+
71
+
/* Bits 63..40 from periodic clock reads, 39..0 from ts40 */
72
+
time_ns = (u64)(high >> 8) << 40 | ts40;
73
+
74
+
/* Compare bits 32-39 between periodic reads and ts40,
75
+
* see if HW clock may have wrapped since last read. We are sure
76
+
* that periodic reads are always at least ~1 minute behind, so
77
+
* this logic works perfectly fine.
78
+
*/
79
+
ts_top = ts40 >> 32;
80
+
if (ts_top < (u8)high && (u8)high - ts_top > U8_MAX / 2)
81
+
time_ns += 1ULL << 40;
82
+
83
+
return time_ns + offset;
49
84
}
50
85
51
86
static unsigned int fbnic_desc_unused(struct fbnic_ring *ring)
···
155
110
#define FBNIC_TWD_TYPE(_type) \
156
111
cpu_to_le64(FIELD_PREP(FBNIC_TWD_TYPE_MASK, FBNIC_TWD_TYPE_##_type))
157
112
113
+
static bool fbnic_tx_tstamp(struct sk_buff *skb)
114
+
{
115
+
struct fbnic_net *fbn;
116
+
117
+
if (!unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))
118
+
return false;
119
+
120
+
fbn = netdev_priv(skb->dev);
121
+
if (fbn->hwtstamp_config.tx_type == HWTSTAMP_TX_OFF)
122
+
return false;
123
+
124
+
skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
125
+
FBNIC_XMIT_CB(skb)->flags |= FBNIC_XMIT_CB_TS;
126
+
FBNIC_XMIT_CB(skb)->hw_head = -1;
127
+
128
+
return true;
129
+
}
130
+
158
131
static bool
159
132
fbnic_tx_offloads(struct fbnic_ring *ring, struct sk_buff *skb, __le64 *meta)
160
133
{
161
134
unsigned int l2len, i3len;
135
+
136
+
if (fbnic_tx_tstamp(skb))
137
+
*meta |= cpu_to_le64(FBNIC_TWD_FLAG_REQ_TS);
162
138
163
139
if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL))
164
140
return false;
···
270
204
FBNIC_XMIT_CB(skb)->desc_count = ((twd - meta) + 1) & ring->size_mask;
271
205
272
206
ring->tail = tail;
207
+
208
+
/* Record SW timestamp */
209
+
skb_tx_timestamp(skb);
273
210
274
211
/* Verify there is room for another packet */
275
212
fbnic_maybe_stop_tx(skb->dev, ring, FBNIC_MAX_SKB_DESC);
···
385
316
struct fbnic_ring *ring, bool discard,
386
317
unsigned int hw_head)
387
318
{
388
-
u64 total_bytes = 0, total_packets = 0;
319
+
u64 total_bytes = 0, total_packets = 0, ts_lost = 0;
389
320
unsigned int head = ring->head;
390
321
struct netdev_queue *txq;
391
322
unsigned int clean_desc;
···
399
330
desc_cnt = FBNIC_XMIT_CB(skb)->desc_count;
400
331
if (desc_cnt > clean_desc)
401
332
break;
333
+
334
+
if (unlikely(FBNIC_XMIT_CB(skb)->flags & FBNIC_XMIT_CB_TS)) {
335
+
FBNIC_XMIT_CB(skb)->hw_head = hw_head;
336
+
if (likely(!discard))
337
+
break;
338
+
ts_lost++;
339
+
}
402
340
403
341
ring->tx_buf[head] = NULL;
404
342
···
444
368
if (unlikely(discard)) {
445
369
u64_stats_update_begin(&ring->stats.syncp);
446
370
ring->stats.dropped += total_packets;
371
+
ring->stats.ts_lost += ts_lost;
447
372
u64_stats_update_end(&ring->stats.syncp);
448
373
449
374
netdev_tx_completed_queue(txq, total_packets, total_bytes);
···
459
382
netif_txq_completed_wake(txq, total_packets, total_bytes,
460
383
fbnic_desc_unused(ring),
461
384
FBNIC_TX_DESC_WAKEUP);
385
+
}
386
+
387
+
static void fbnic_clean_tsq(struct fbnic_napi_vector *nv,
388
+
struct fbnic_ring *ring,
389
+
u64 tcd, int *ts_head, int *head0)
390
+
{
391
+
struct skb_shared_hwtstamps hwtstamp;
392
+
struct fbnic_net *fbn;
393
+
struct sk_buff *skb;
394
+
int head;
395
+
u64 ns;
396
+
397
+
head = (*ts_head < 0) ? ring->head : *ts_head;
398
+
399
+
do {
400
+
unsigned int desc_cnt;
401
+
402
+
if (head == ring->tail) {
403
+
if (unlikely(net_ratelimit()))
404
+
netdev_err(nv->napi.dev,
405
+
"Tx timestamp without matching packet\n");
406
+
return;
407
+
}
408
+
409
+
skb = ring->tx_buf[head];
410
+
desc_cnt = FBNIC_XMIT_CB(skb)->desc_count;
411
+
412
+
head += desc_cnt;
413
+
head &= ring->size_mask;
414
+
} while (!(FBNIC_XMIT_CB(skb)->flags & FBNIC_XMIT_CB_TS));
415
+
416
+
fbn = netdev_priv(nv->napi.dev);
417
+
ns = fbnic_ts40_to_ns(fbn, FIELD_GET(FBNIC_TCD_TYPE1_TS_MASK, tcd));
418
+
419
+
memset(&hwtstamp, 0, sizeof(hwtstamp));
420
+
hwtstamp.hwtstamp = ns_to_ktime(ns);
421
+
422
+
*ts_head = head;
423
+
424
+
FBNIC_XMIT_CB(skb)->flags &= ~FBNIC_XMIT_CB_TS;
425
+
if (*head0 < 0) {
426
+
head = FBNIC_XMIT_CB(skb)->hw_head;
427
+
if (head >= 0)
428
+
*head0 = head;
429
+
}
430
+
431
+
skb_tstamp_tx(skb, &hwtstamp);
432
+
u64_stats_update_begin(&ring->stats.syncp);
433
+
ring->stats.ts_packets++;
434
+
u64_stats_update_end(&ring->stats.syncp);
462
435
}
463
436
464
437
static void fbnic_page_pool_init(struct fbnic_ring *ring, unsigned int idx,
···
544
417
}
545
418
546
419
static void fbnic_clean_twq(struct fbnic_napi_vector *nv, int napi_budget,
547
-
struct fbnic_q_triad *qt, s32 head0)
420
+
struct fbnic_q_triad *qt, s32 ts_head, s32 head0)
548
421
{
549
422
if (head0 >= 0)
550
423
fbnic_clean_twq0(nv, napi_budget, &qt->sub0, false, head0);
424
+
else if (ts_head >= 0)
425
+
fbnic_clean_twq0(nv, napi_budget, &qt->sub0, false, ts_head);
551
426
}
552
427
553
428
static void
···
557
428
int napi_budget)
558
429
{
559
430
struct fbnic_ring *cmpl = &qt->cmpl;
431
+
s32 head0 = -1, ts_head = -1;
560
432
__le64 *raw_tcd, done;
561
433
u32 head = cmpl->head;
562
-
s32 head0 = -1;
563
434
564
435
done = (head & (cmpl->size_mask + 1)) ? 0 : cpu_to_le64(FBNIC_TCD_DONE);
565
436
raw_tcd = &cmpl->desc[head & cmpl->size_mask];
···
582
453
* they are skipped for now.
583
454
*/
584
455
break;
456
+
case FBNIC_TCD_TYPE_1:
457
+
if (WARN_ON_ONCE(tcd & FBNIC_TCD_TWQ1))
458
+
break;
459
+
460
+
fbnic_clean_tsq(nv, &qt->sub0, tcd, &ts_head, &head0);
461
+
break;
585
462
default:
586
463
break;
587
464
}
···
607
472
}
608
473
609
474
/* Unmap and free processed buffers */
610
-
fbnic_clean_twq(nv, napi_budget, qt, head0);
475
+
fbnic_clean_twq(nv, napi_budget, qt, ts_head, head0);
611
476
}
612
477
613
478
static void fbnic_clean_bdq(struct fbnic_napi_vector *nv, int napi_budget,
···
842
707
/* Set MAC header specific fields */
843
708
skb->protocol = eth_type_trans(skb, nv->napi.dev);
844
709
710
+
/* Add timestamp if present */
711
+
if (pkt->hwtstamp)
712
+
skb_hwtstamps(skb)->hwtstamp = pkt->hwtstamp;
713
+
845
714
return skb;
846
715
}
847
716
···
854
715
return (FBNIC_RCD_META_L4_TYPE_MASK & rcd) ? PKT_HASH_TYPE_L4 :
855
716
(FBNIC_RCD_META_L3_TYPE_MASK & rcd) ? PKT_HASH_TYPE_L3 :
856
717
PKT_HASH_TYPE_L2;
718
+
}
719
+
720
+
static void fbnic_rx_tstamp(struct fbnic_napi_vector *nv, u64 rcd,
721
+
struct fbnic_pkt_buff *pkt)
722
+
{
723
+
struct fbnic_net *fbn;
724
+
u64 ns, ts;
725
+
726
+
if (!FIELD_GET(FBNIC_RCD_OPT_META_TS, rcd))
727
+
return;
728
+
729
+
fbn = netdev_priv(nv->napi.dev);
730
+
ts = FIELD_GET(FBNIC_RCD_OPT_META_TS_MASK, rcd);
731
+
ns = fbnic_ts40_to_ns(fbn, ts);
732
+
733
+
/* Add timestamp to shared info */
734
+
pkt->hwtstamp = ns_to_ktime(ns);
857
735
}
858
736
859
737
static void fbnic_populate_skb_fields(struct fbnic_napi_vector *nv,
···
936
780
/* Only type 0 is currently supported */
937
781
if (FIELD_GET(FBNIC_RCD_OPT_META_TYPE_MASK, rcd))
938
782
break;
783
+
784
+
fbnic_rx_tstamp(nv, rcd, pkt);
939
785
940
786
/* We currently ignore the action table index */
941
787
break;
···
1065
907
fbn->tx_stats.bytes += stats->bytes;
1066
908
fbn->tx_stats.packets += stats->packets;
1067
909
fbn->tx_stats.dropped += stats->dropped;
910
+
fbn->tx_stats.ts_lost += stats->ts_lost;
911
+
fbn->tx_stats.ts_packets += stats->ts_packets;
1068
912
}
1069
913
1070
914
static void fbnic_remove_tx_ring(struct fbnic_net *fbn,
+3
drivers/net/ethernet/meta/fbnic/fbnic_txrx.h
+3
drivers/net/ethernet/meta/fbnic/fbnic_txrx.h
···
47
47
48
48
struct fbnic_pkt_buff {
49
49
struct xdp_buff buff;
50
+
ktime_t hwtstamp;
50
51
u32 data_truesize;
51
52
u16 data_len;
52
53
u16 nr_frags;
···
57
56
u64 packets;
58
57
u64 bytes;
59
58
u64 dropped;
59
+
u64 ts_packets;
60
+
u64 ts_lost;
60
61
struct u64_stats_sync syncp;
61
62
};
62
63