Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'add-more-functionality-to-bnge'
Bhargava Marreddy says:
====================
Add more functionality to BNGE
This patch series adds the infrastructure to make the netdevice
functional. It allocates data structures for core resources,
followed by their initialisation and registration with the firmware.
The core resources include the RX, TX, AGG, CMPL, and NQ rings,
as well as the VNIC. RX/TX functionality will be introduced in the
next patch series to keep this one at a reviewable size.
====================
Link: https://patch.msgid.link/20250919174742.24969-1-bhargava.marreddy@broadcom.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
+3140
-7
+1
drivers/net/ethernet/broadcom/Kconfig
+1
drivers/net/ethernet/broadcom/Kconfig
···
257
257
tristate "Broadcom Ethernet device support"
258
258
depends on PCI
259
259
select NET_DEVLINK
260
+
select PAGE_POOL
260
261
help
261
262
This driver supports Broadcom 50/100/200/400/800 gigabit Ethernet cards.
262
263
The module will be called bng_en. To compile this driver as a module,
+27
drivers/net/ethernet/broadcom/bnge/bnge.h
+27
drivers/net/ethernet/broadcom/bnge/bnge.h
···
102
102
u16 chip_num;
103
103
u8 chip_rev;
104
104
105
+
#if BITS_PER_LONG == 32
106
+
/* ensure atomic 64-bit doorbell writes on 32-bit systems. */
107
+
spinlock_t db_lock;
108
+
#endif
105
109
int db_offset; /* db_offset within db_size */
106
110
int db_size;
107
111
···
133
129
134
130
unsigned long state;
135
131
#define BNGE_STATE_DRV_REGISTERED 0
132
+
#define BNGE_STATE_OPEN 1
136
133
137
134
u64 fw_cap;
138
135
···
160
155
u16 rss_indir_tbl_entries;
161
156
162
157
u32 rss_cap;
158
+
u32 rss_hash_cfg;
163
159
164
160
u16 rx_nr_rings;
165
161
u16 tx_nr_rings;
···
219
213
return true;
220
214
}
221
215
216
+
static inline void bnge_writeq(struct bnge_dev *bd, u64 val,
217
+
void __iomem *addr)
218
+
{
219
+
#if BITS_PER_LONG == 32
220
+
spin_lock(&bd->db_lock);
221
+
lo_hi_writeq(val, addr);
222
+
spin_unlock(&bd->db_lock);
223
+
#else
224
+
writeq(val, addr);
225
+
#endif
226
+
}
227
+
228
+
/* For TX and RX ring doorbells */
229
+
static inline void bnge_db_write(struct bnge_dev *bd, struct bnge_db_info *db,
230
+
u32 idx)
231
+
{
232
+
bnge_writeq(bd, db->db_key64 | DB_RING_IDX(db, idx),
233
+
db->doorbell);
234
+
}
235
+
222
236
bool bnge_aux_registered(struct bnge_dev *bd);
237
+
u16 bnge_aux_get_msix(struct bnge_dev *bd);
223
238
224
239
#endif /* _BNGE_H_ */
+16
drivers/net/ethernet/broadcom/bnge/bnge_core.c
+16
drivers/net/ethernet/broadcom/bnge/bnge_core.c
···
96
96
bnge_free_ctx_mem(bd);
97
97
}
98
98
99
+
static void bnge_set_dflt_rss_hash_type(struct bnge_dev *bd)
100
+
{
101
+
bd->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
102
+
VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
103
+
VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
104
+
VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6 |
105
+
VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
106
+
VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
107
+
}
108
+
99
109
static int bnge_fw_register_dev(struct bnge_dev *bd)
100
110
{
101
111
int rc;
···
146
136
rc = -ENODEV;
147
137
goto err_func_unrgtr;
148
138
}
139
+
140
+
bnge_set_dflt_rss_hash_type(bd);
149
141
150
142
return 0;
151
143
···
307
295
rc);
308
296
goto err_config_uninit;
309
297
}
298
+
299
+
#if BITS_PER_LONG == 32
300
+
spin_lock_init(&bd->db_lock);
301
+
#endif
310
302
311
303
rc = bnge_alloc_irqs(bd);
312
304
if (rc) {
+34
drivers/net/ethernet/broadcom/bnge/bnge_db.h
+34
drivers/net/ethernet/broadcom/bnge/bnge_db.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
/* Copyright (c) 2025 Broadcom */
3
+
4
+
#ifndef _BNGE_DB_H_
5
+
#define _BNGE_DB_H_
6
+
7
+
/* 64-bit doorbell */
8
+
#define DBR_EPOCH_SFT 24
9
+
#define DBR_TOGGLE_SFT 25
10
+
#define DBR_XID_SFT 32
11
+
#define DBR_PATH_L2 (0x1ULL << 56)
12
+
#define DBR_VALID (0x1ULL << 58)
13
+
#define DBR_TYPE_SQ (0x0ULL << 60)
14
+
#define DBR_TYPE_SRQ (0x2ULL << 60)
15
+
#define DBR_TYPE_CQ (0x4ULL << 60)
16
+
#define DBR_TYPE_CQ_ARMALL (0x6ULL << 60)
17
+
#define DBR_TYPE_NQ (0xaULL << 60)
18
+
#define DBR_TYPE_NQ_ARM (0xbULL << 60)
19
+
#define DBR_TYPE_NQ_MASK (0xeULL << 60)
20
+
21
+
struct bnge_db_info {
22
+
void __iomem *doorbell;
23
+
u64 db_key64;
24
+
u32 db_ring_mask;
25
+
u32 db_epoch_mask;
26
+
u8 db_epoch_shift;
27
+
};
28
+
29
+
#define DB_EPOCH(db, idx) (((idx) & (db)->db_epoch_mask) << \
30
+
((db)->db_epoch_shift))
31
+
#define DB_RING_IDX(db, idx) (((idx) & (db)->db_ring_mask) | \
32
+
DB_EPOCH(db, idx))
33
+
34
+
#endif /* _BNGE_DB_H_ */
+482
drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c
+482
drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c
···
6
6
#include <linux/mm.h>
7
7
#include <linux/pci.h>
8
8
#include <linux/bnxt/hsi.h>
9
+
#include <linux/if_vlan.h>
10
+
#include <net/netdev_queues.h>
9
11
10
12
#include "bnge.h"
11
13
#include "bnge_hwrm.h"
···
702
700
qportcfg_exit:
703
701
bnge_hwrm_req_drop(bd, req);
704
702
return rc;
703
+
}
704
+
705
+
int bnge_hwrm_vnic_set_hds(struct bnge_net *bn, struct bnge_vnic_info *vnic)
706
+
{
707
+
u16 hds_thresh = (u16)bn->netdev->cfg_pending->hds_thresh;
708
+
struct hwrm_vnic_plcmodes_cfg_input *req;
709
+
struct bnge_dev *bd = bn->bd;
710
+
int rc;
711
+
712
+
rc = bnge_hwrm_req_init(bd, req, HWRM_VNIC_PLCMODES_CFG);
713
+
if (rc)
714
+
return rc;
715
+
716
+
req->flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT);
717
+
req->enables = cpu_to_le32(BNGE_PLC_EN_JUMBO_THRES_VALID);
718
+
req->jumbo_thresh = cpu_to_le16(bn->rx_buf_use_size);
719
+
720
+
if (bnge_is_agg_reqd(bd)) {
721
+
req->flags |= cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
722
+
VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
723
+
req->enables |=
724
+
cpu_to_le32(BNGE_PLC_EN_HDS_THRES_VALID);
725
+
req->hds_threshold = cpu_to_le16(hds_thresh);
726
+
}
727
+
req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
728
+
return bnge_hwrm_req_send(bd, req);
729
+
}
730
+
731
+
int bnge_hwrm_vnic_ctx_alloc(struct bnge_dev *bd,
732
+
struct bnge_vnic_info *vnic, u16 ctx_idx)
733
+
{
734
+
struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp;
735
+
struct hwrm_vnic_rss_cos_lb_ctx_alloc_input *req;
736
+
int rc;
737
+
738
+
rc = bnge_hwrm_req_init(bd, req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC);
739
+
if (rc)
740
+
return rc;
741
+
742
+
resp = bnge_hwrm_req_hold(bd, req);
743
+
rc = bnge_hwrm_req_send(bd, req);
744
+
if (!rc)
745
+
vnic->fw_rss_cos_lb_ctx[ctx_idx] =
746
+
le16_to_cpu(resp->rss_cos_lb_ctx_id);
747
+
bnge_hwrm_req_drop(bd, req);
748
+
749
+
return rc;
750
+
}
751
+
752
+
static void
753
+
__bnge_hwrm_vnic_set_rss(struct bnge_net *bn,
754
+
struct hwrm_vnic_rss_cfg_input *req,
755
+
struct bnge_vnic_info *vnic)
756
+
{
757
+
struct bnge_dev *bd = bn->bd;
758
+
759
+
bnge_fill_hw_rss_tbl(bn, vnic);
760
+
req->flags |= VNIC_RSS_CFG_REQ_FLAGS_IPSEC_HASH_TYPE_CFG_SUPPORT;
761
+
762
+
req->hash_type = cpu_to_le32(bd->rss_hash_cfg);
763
+
req->hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
764
+
req->ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
765
+
req->hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
766
+
}
767
+
768
+
int bnge_hwrm_vnic_set_rss(struct bnge_net *bn,
769
+
struct bnge_vnic_info *vnic, bool set_rss)
770
+
{
771
+
struct hwrm_vnic_rss_cfg_input *req;
772
+
struct bnge_dev *bd = bn->bd;
773
+
dma_addr_t ring_tbl_map;
774
+
u32 i, nr_ctxs;
775
+
int rc;
776
+
777
+
rc = bnge_hwrm_req_init(bd, req, HWRM_VNIC_RSS_CFG);
778
+
if (rc)
779
+
return rc;
780
+
781
+
req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
782
+
if (!set_rss)
783
+
return bnge_hwrm_req_send(bd, req);
784
+
785
+
__bnge_hwrm_vnic_set_rss(bn, req, vnic);
786
+
ring_tbl_map = vnic->rss_table_dma_addr;
787
+
nr_ctxs = bnge_cal_nr_rss_ctxs(bd->rx_nr_rings);
788
+
789
+
bnge_hwrm_req_hold(bd, req);
790
+
for (i = 0; i < nr_ctxs; ring_tbl_map += BNGE_RSS_TABLE_SIZE, i++) {
791
+
req->ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map);
792
+
req->ring_table_pair_index = i;
793
+
req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
794
+
rc = bnge_hwrm_req_send(bd, req);
795
+
if (rc)
796
+
goto exit;
797
+
}
798
+
799
+
exit:
800
+
bnge_hwrm_req_drop(bd, req);
801
+
return rc;
802
+
}
803
+
804
+
int bnge_hwrm_vnic_cfg(struct bnge_net *bn, struct bnge_vnic_info *vnic)
805
+
{
806
+
struct bnge_rx_ring_info *rxr = &bn->rx_ring[0];
807
+
struct hwrm_vnic_cfg_input *req;
808
+
struct bnge_dev *bd = bn->bd;
809
+
int rc;
810
+
811
+
rc = bnge_hwrm_req_init(bd, req, HWRM_VNIC_CFG);
812
+
if (rc)
813
+
return rc;
814
+
815
+
req->default_rx_ring_id =
816
+
cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
817
+
req->default_cmpl_ring_id =
818
+
cpu_to_le16(bnge_cp_ring_for_rx(rxr));
819
+
req->enables =
820
+
cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
821
+
VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
822
+
vnic->mru = bd->netdev->mtu + ETH_HLEN + VLAN_HLEN;
823
+
req->mru = cpu_to_le16(vnic->mru);
824
+
825
+
req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
826
+
827
+
if (bd->flags & BNGE_EN_STRIP_VLAN)
828
+
req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
829
+
if (vnic->vnic_id == BNGE_VNIC_DEFAULT && bnge_aux_registered(bd))
830
+
req->flags |= cpu_to_le32(BNGE_VNIC_CFG_ROCE_DUAL_MODE);
831
+
832
+
return bnge_hwrm_req_send(bd, req);
833
+
}
834
+
835
+
void bnge_hwrm_update_rss_hash_cfg(struct bnge_net *bn)
836
+
{
837
+
struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT];
838
+
struct hwrm_vnic_rss_qcfg_output *resp;
839
+
struct hwrm_vnic_rss_qcfg_input *req;
840
+
struct bnge_dev *bd = bn->bd;
841
+
842
+
if (bnge_hwrm_req_init(bd, req, HWRM_VNIC_RSS_QCFG))
843
+
return;
844
+
845
+
req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
846
+
/* all contexts configured to same hash_type, zero always exists */
847
+
req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
848
+
resp = bnge_hwrm_req_hold(bd, req);
849
+
if (!bnge_hwrm_req_send(bd, req))
850
+
bd->rss_hash_cfg =
851
+
le32_to_cpu(resp->hash_type) ?: bd->rss_hash_cfg;
852
+
bnge_hwrm_req_drop(bd, req);
853
+
}
854
+
855
+
int bnge_hwrm_l2_filter_free(struct bnge_dev *bd, struct bnge_l2_filter *fltr)
856
+
{
857
+
struct hwrm_cfa_l2_filter_free_input *req;
858
+
int rc;
859
+
860
+
rc = bnge_hwrm_req_init(bd, req, HWRM_CFA_L2_FILTER_FREE);
861
+
if (rc)
862
+
return rc;
863
+
864
+
req->l2_filter_id = fltr->base.filter_id;
865
+
return bnge_hwrm_req_send(bd, req);
866
+
}
867
+
868
+
int bnge_hwrm_l2_filter_alloc(struct bnge_dev *bd, struct bnge_l2_filter *fltr)
869
+
{
870
+
struct hwrm_cfa_l2_filter_alloc_output *resp;
871
+
struct hwrm_cfa_l2_filter_alloc_input *req;
872
+
int rc;
873
+
874
+
rc = bnge_hwrm_req_init(bd, req, HWRM_CFA_L2_FILTER_ALLOC);
875
+
if (rc)
876
+
return rc;
877
+
878
+
req->flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
879
+
880
+
req->flags |= cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
881
+
req->dst_id = cpu_to_le16(fltr->base.fw_vnic_id);
882
+
req->enables =
883
+
cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
884
+
CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
885
+
CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
886
+
ether_addr_copy(req->l2_addr, fltr->l2_key.dst_mac_addr);
887
+
eth_broadcast_addr(req->l2_addr_mask);
888
+
889
+
if (fltr->l2_key.vlan) {
890
+
req->enables |=
891
+
cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN |
892
+
CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN_MASK |
893
+
CFA_L2_FILTER_ALLOC_REQ_ENABLES_NUM_VLANS);
894
+
req->num_vlans = 1;
895
+
req->l2_ivlan = cpu_to_le16(fltr->l2_key.vlan);
896
+
req->l2_ivlan_mask = cpu_to_le16(0xfff);
897
+
}
898
+
899
+
resp = bnge_hwrm_req_hold(bd, req);
900
+
rc = bnge_hwrm_req_send(bd, req);
901
+
if (!rc)
902
+
fltr->base.filter_id = resp->l2_filter_id;
903
+
904
+
bnge_hwrm_req_drop(bd, req);
905
+
return rc;
906
+
}
907
+
908
+
int bnge_hwrm_cfa_l2_set_rx_mask(struct bnge_dev *bd,
909
+
struct bnge_vnic_info *vnic)
910
+
{
911
+
struct hwrm_cfa_l2_set_rx_mask_input *req;
912
+
int rc;
913
+
914
+
rc = bnge_hwrm_req_init(bd, req, HWRM_CFA_L2_SET_RX_MASK);
915
+
if (rc)
916
+
return rc;
917
+
918
+
req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
919
+
if (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST) {
920
+
req->num_mc_entries = cpu_to_le32(vnic->mc_list_count);
921
+
req->mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
922
+
}
923
+
req->mask = cpu_to_le32(vnic->rx_mask);
924
+
return bnge_hwrm_req_send_silent(bd, req);
925
+
}
926
+
927
+
int bnge_hwrm_vnic_alloc(struct bnge_dev *bd, struct bnge_vnic_info *vnic,
928
+
unsigned int nr_rings)
929
+
{
930
+
struct hwrm_vnic_alloc_output *resp;
931
+
struct hwrm_vnic_alloc_input *req;
932
+
unsigned int i;
933
+
int rc;
934
+
935
+
rc = bnge_hwrm_req_init(bd, req, HWRM_VNIC_ALLOC);
936
+
if (rc)
937
+
return rc;
938
+
939
+
for (i = 0; i < BNGE_MAX_CTX_PER_VNIC; i++)
940
+
vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
941
+
if (vnic->vnic_id == BNGE_VNIC_DEFAULT)
942
+
req->flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
943
+
944
+
resp = bnge_hwrm_req_hold(bd, req);
945
+
rc = bnge_hwrm_req_send(bd, req);
946
+
if (!rc)
947
+
vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
948
+
bnge_hwrm_req_drop(bd, req);
949
+
return rc;
950
+
}
951
+
952
+
void bnge_hwrm_vnic_free_one(struct bnge_dev *bd, struct bnge_vnic_info *vnic)
953
+
{
954
+
if (vnic->fw_vnic_id != INVALID_HW_RING_ID) {
955
+
struct hwrm_vnic_free_input *req;
956
+
957
+
if (bnge_hwrm_req_init(bd, req, HWRM_VNIC_FREE))
958
+
return;
959
+
960
+
req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
961
+
962
+
bnge_hwrm_req_send(bd, req);
963
+
vnic->fw_vnic_id = INVALID_HW_RING_ID;
964
+
}
965
+
}
966
+
967
+
void bnge_hwrm_vnic_ctx_free_one(struct bnge_dev *bd,
968
+
struct bnge_vnic_info *vnic, u16 ctx_idx)
969
+
{
970
+
struct hwrm_vnic_rss_cos_lb_ctx_free_input *req;
971
+
972
+
if (bnge_hwrm_req_init(bd, req, HWRM_VNIC_RSS_COS_LB_CTX_FREE))
973
+
return;
974
+
975
+
req->rss_cos_lb_ctx_id =
976
+
cpu_to_le16(vnic->fw_rss_cos_lb_ctx[ctx_idx]);
977
+
978
+
bnge_hwrm_req_send(bd, req);
979
+
vnic->fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
980
+
}
981
+
982
+
void bnge_hwrm_stat_ctx_free(struct bnge_net *bn)
983
+
{
984
+
struct hwrm_stat_ctx_free_input *req;
985
+
struct bnge_dev *bd = bn->bd;
986
+
int i;
987
+
988
+
if (bnge_hwrm_req_init(bd, req, HWRM_STAT_CTX_FREE))
989
+
return;
990
+
991
+
bnge_hwrm_req_hold(bd, req);
992
+
for (i = 0; i < bd->nq_nr_rings; i++) {
993
+
struct bnge_napi *bnapi = bn->bnapi[i];
994
+
struct bnge_nq_ring_info *nqr = &bnapi->nq_ring;
995
+
996
+
if (nqr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
997
+
req->stat_ctx_id = cpu_to_le32(nqr->hw_stats_ctx_id);
998
+
bnge_hwrm_req_send(bd, req);
999
+
1000
+
nqr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
1001
+
}
1002
+
}
1003
+
bnge_hwrm_req_drop(bd, req);
1004
+
}
1005
+
1006
+
int bnge_hwrm_stat_ctx_alloc(struct bnge_net *bn)
1007
+
{
1008
+
struct hwrm_stat_ctx_alloc_output *resp;
1009
+
struct hwrm_stat_ctx_alloc_input *req;
1010
+
struct bnge_dev *bd = bn->bd;
1011
+
int rc, i;
1012
+
1013
+
rc = bnge_hwrm_req_init(bd, req, HWRM_STAT_CTX_ALLOC);
1014
+
if (rc)
1015
+
return rc;
1016
+
1017
+
req->stats_dma_length = cpu_to_le16(bd->hw_ring_stats_size);
1018
+
req->update_period_ms = cpu_to_le32(bn->stats_coal_ticks / 1000);
1019
+
1020
+
resp = bnge_hwrm_req_hold(bd, req);
1021
+
for (i = 0; i < bd->nq_nr_rings; i++) {
1022
+
struct bnge_napi *bnapi = bn->bnapi[i];
1023
+
struct bnge_nq_ring_info *nqr = &bnapi->nq_ring;
1024
+
1025
+
req->stats_dma_addr = cpu_to_le64(nqr->stats.hw_stats_map);
1026
+
1027
+
rc = bnge_hwrm_req_send(bd, req);
1028
+
if (rc)
1029
+
break;
1030
+
1031
+
nqr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
1032
+
bn->grp_info[i].fw_stats_ctx = nqr->hw_stats_ctx_id;
1033
+
}
1034
+
bnge_hwrm_req_drop(bd, req);
1035
+
return rc;
1036
+
}
1037
+
1038
+
int hwrm_ring_free_send_msg(struct bnge_net *bn,
1039
+
struct bnge_ring_struct *ring,
1040
+
u32 ring_type, int cmpl_ring_id)
1041
+
{
1042
+
struct hwrm_ring_free_input *req;
1043
+
struct bnge_dev *bd = bn->bd;
1044
+
int rc;
1045
+
1046
+
rc = bnge_hwrm_req_init(bd, req, HWRM_RING_FREE);
1047
+
if (rc)
1048
+
goto exit;
1049
+
1050
+
req->cmpl_ring = cpu_to_le16(cmpl_ring_id);
1051
+
req->ring_type = ring_type;
1052
+
req->ring_id = cpu_to_le16(ring->fw_ring_id);
1053
+
1054
+
bnge_hwrm_req_hold(bd, req);
1055
+
rc = bnge_hwrm_req_send(bd, req);
1056
+
bnge_hwrm_req_drop(bd, req);
1057
+
exit:
1058
+
if (rc) {
1059
+
netdev_err(bd->netdev, "hwrm_ring_free type %d failed. rc:%d\n", ring_type, rc);
1060
+
return -EIO;
1061
+
}
1062
+
return 0;
1063
+
}
1064
+
1065
+
int hwrm_ring_alloc_send_msg(struct bnge_net *bn,
1066
+
struct bnge_ring_struct *ring,
1067
+
u32 ring_type, u32 map_index)
1068
+
{
1069
+
struct bnge_ring_mem_info *rmem = &ring->ring_mem;
1070
+
struct bnge_ring_grp_info *grp_info;
1071
+
struct hwrm_ring_alloc_output *resp;
1072
+
struct hwrm_ring_alloc_input *req;
1073
+
struct bnge_dev *bd = bn->bd;
1074
+
u16 ring_id, flags = 0;
1075
+
int rc;
1076
+
1077
+
rc = bnge_hwrm_req_init(bd, req, HWRM_RING_ALLOC);
1078
+
if (rc)
1079
+
goto exit;
1080
+
1081
+
req->enables = 0;
1082
+
if (rmem->nr_pages > 1) {
1083
+
req->page_tbl_addr = cpu_to_le64(rmem->dma_pg_tbl);
1084
+
/* Page size is in log2 units */
1085
+
req->page_size = BNGE_PAGE_SHIFT;
1086
+
req->page_tbl_depth = 1;
1087
+
} else {
1088
+
req->page_tbl_addr = cpu_to_le64(rmem->dma_arr[0]);
1089
+
}
1090
+
req->fbo = 0;
1091
+
/* Association of ring index with doorbell index and MSIX number */
1092
+
req->logical_id = cpu_to_le16(map_index);
1093
+
1094
+
switch (ring_type) {
1095
+
case HWRM_RING_ALLOC_TX: {
1096
+
struct bnge_tx_ring_info *txr;
1097
+
1098
+
txr = container_of(ring, struct bnge_tx_ring_info,
1099
+
tx_ring_struct);
1100
+
req->ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
1101
+
/* Association of transmit ring with completion ring */
1102
+
grp_info = &bn->grp_info[ring->grp_idx];
1103
+
req->cmpl_ring_id = cpu_to_le16(bnge_cp_ring_for_tx(txr));
1104
+
req->length = cpu_to_le32(bn->tx_ring_mask + 1);
1105
+
req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
1106
+
req->queue_id = cpu_to_le16(ring->queue_id);
1107
+
req->flags = cpu_to_le16(flags);
1108
+
break;
1109
+
}
1110
+
case HWRM_RING_ALLOC_RX:
1111
+
req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
1112
+
req->length = cpu_to_le32(bn->rx_ring_mask + 1);
1113
+
1114
+
/* Association of rx ring with stats context */
1115
+
grp_info = &bn->grp_info[ring->grp_idx];
1116
+
req->rx_buf_size = cpu_to_le16(bn->rx_buf_use_size);
1117
+
req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
1118
+
req->enables |=
1119
+
cpu_to_le32(RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
1120
+
if (NET_IP_ALIGN == 2)
1121
+
flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
1122
+
req->flags = cpu_to_le16(flags);
1123
+
break;
1124
+
case HWRM_RING_ALLOC_AGG:
1125
+
req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
1126
+
/* Association of agg ring with rx ring */
1127
+
grp_info = &bn->grp_info[ring->grp_idx];
1128
+
req->rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
1129
+
req->rx_buf_size = cpu_to_le16(BNGE_RX_PAGE_SIZE);
1130
+
req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
1131
+
req->enables |=
1132
+
cpu_to_le32(RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
1133
+
RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
1134
+
req->length = cpu_to_le32(bn->rx_agg_ring_mask + 1);
1135
+
break;
1136
+
case HWRM_RING_ALLOC_CMPL:
1137
+
req->ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
1138
+
req->length = cpu_to_le32(bn->cp_ring_mask + 1);
1139
+
/* Association of cp ring with nq */
1140
+
grp_info = &bn->grp_info[map_index];
1141
+
req->nq_ring_id = cpu_to_le16(grp_info->nq_fw_ring_id);
1142
+
req->cq_handle = cpu_to_le64(ring->handle);
1143
+
req->enables |=
1144
+
cpu_to_le32(RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
1145
+
break;
1146
+
case HWRM_RING_ALLOC_NQ:
1147
+
req->ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
1148
+
req->length = cpu_to_le32(bn->cp_ring_mask + 1);
1149
+
req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
1150
+
break;
1151
+
default:
1152
+
netdev_err(bn->netdev, "hwrm alloc invalid ring type %d\n", ring_type);
1153
+
return -EINVAL;
1154
+
}
1155
+
1156
+
resp = bnge_hwrm_req_hold(bd, req);
1157
+
rc = bnge_hwrm_req_send(bd, req);
1158
+
ring_id = le16_to_cpu(resp->ring_id);
1159
+
bnge_hwrm_req_drop(bd, req);
1160
+
1161
+
exit:
1162
+
if (rc) {
1163
+
netdev_err(bd->netdev, "hwrm_ring_alloc type %d failed. rc:%d\n", ring_type, rc);
1164
+
return -EIO;
1165
+
}
1166
+
ring->fw_ring_id = ring_id;
1167
+
return rc;
1168
+
}
1169
+
1170
+
int bnge_hwrm_set_async_event_cr(struct bnge_dev *bd, int idx)
1171
+
{
1172
+
struct hwrm_func_cfg_input *req;
1173
+
int rc;
1174
+
1175
+
rc = bnge_hwrm_req_init(bd, req, HWRM_FUNC_CFG);
1176
+
if (rc)
1177
+
return rc;
1178
+
1179
+
req->fid = cpu_to_le16(0xffff);
1180
+
req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
1181
+
req->async_event_cr = cpu_to_le16(idx);
1182
+
return bnge_hwrm_req_send(bd, req);
705
1183
}
+31
drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.h
+31
drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.h
···
4
4
#ifndef _BNGE_HWRM_LIB_H_
5
5
#define _BNGE_HWRM_LIB_H_
6
6
7
+
#define BNGE_PLC_EN_JUMBO_THRES_VALID \
8
+
VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID
9
+
#define BNGE_PLC_EN_HDS_THRES_VALID \
10
+
VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID
11
+
#define BNGE_VNIC_CFG_ROCE_DUAL_MODE \
12
+
VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE
13
+
7
14
int bnge_hwrm_ver_get(struct bnge_dev *bd);
8
15
int bnge_hwrm_func_reset(struct bnge_dev *bd);
9
16
int bnge_hwrm_fw_set_time(struct bnge_dev *bd);
···
31
24
int bnge_hwrm_func_resc_qcaps(struct bnge_dev *bd);
32
25
int bnge_hwrm_queue_qportcfg(struct bnge_dev *bd);
33
26
27
+
int bnge_hwrm_vnic_set_hds(struct bnge_net *bn, struct bnge_vnic_info *vnic);
28
+
int bnge_hwrm_vnic_ctx_alloc(struct bnge_dev *bd,
29
+
struct bnge_vnic_info *vnic, u16 ctx_idx);
30
+
int bnge_hwrm_vnic_set_rss(struct bnge_net *bn,
31
+
struct bnge_vnic_info *vnic, bool set_rss);
32
+
int bnge_hwrm_vnic_cfg(struct bnge_net *bn, struct bnge_vnic_info *vnic);
33
+
void bnge_hwrm_update_rss_hash_cfg(struct bnge_net *bn);
34
+
int bnge_hwrm_vnic_alloc(struct bnge_dev *bd, struct bnge_vnic_info *vnic,
35
+
unsigned int nr_rings);
36
+
void bnge_hwrm_vnic_free_one(struct bnge_dev *bd, struct bnge_vnic_info *vnic);
37
+
void bnge_hwrm_vnic_ctx_free_one(struct bnge_dev *bd,
38
+
struct bnge_vnic_info *vnic, u16 ctx_idx);
39
+
int bnge_hwrm_l2_filter_free(struct bnge_dev *bd, struct bnge_l2_filter *fltr);
40
+
int bnge_hwrm_l2_filter_alloc(struct bnge_dev *bd, struct bnge_l2_filter *fltr);
41
+
int bnge_hwrm_cfa_l2_set_rx_mask(struct bnge_dev *bd,
42
+
struct bnge_vnic_info *vnic);
43
+
void bnge_hwrm_stat_ctx_free(struct bnge_net *bn);
44
+
int bnge_hwrm_stat_ctx_alloc(struct bnge_net *bn);
45
+
int hwrm_ring_free_send_msg(struct bnge_net *bn, struct bnge_ring_struct *ring,
46
+
u32 ring_type, int cmpl_ring_id);
47
+
int hwrm_ring_alloc_send_msg(struct bnge_net *bn,
48
+
struct bnge_ring_struct *ring,
49
+
u32 ring_type, u32 map_index);
50
+
int bnge_hwrm_set_async_event_cr(struct bnge_dev *bd, int idx);
34
51
#endif /* _BNGE_HWRM_LIB_H_ */
+2217
drivers/net/ethernet/broadcom/bnge/bnge_netdev.c
+2217
drivers/net/ethernet/broadcom/bnge/bnge_netdev.c
···
14
14
#include <linux/if.h>
15
15
#include <net/ip.h>
16
16
#include <linux/skbuff.h>
17
+
#include <net/page_pool/helpers.h>
17
18
18
19
#include "bnge.h"
19
20
#include "bnge_hwrm_lib.h"
20
21
#include "bnge_ethtool.h"
22
+
#include "bnge_rmem.h"
23
+
24
+
#define BNGE_RING_TO_TC_OFF(bd, tx) \
25
+
((tx) % (bd)->tx_nr_rings_per_tc)
26
+
27
+
#define BNGE_RING_TO_TC(bd, tx) \
28
+
((tx) / (bd)->tx_nr_rings_per_tc)
29
+
30
+
#define BNGE_TC_TO_RING_BASE(bd, tc) \
31
+
((tc) * (bd)->tx_nr_rings_per_tc)
32
+
33
+
static void bnge_free_stats_mem(struct bnge_net *bn,
34
+
struct bnge_stats_mem *stats)
35
+
{
36
+
struct bnge_dev *bd = bn->bd;
37
+
38
+
if (stats->hw_stats) {
39
+
dma_free_coherent(bd->dev, stats->len, stats->hw_stats,
40
+
stats->hw_stats_map);
41
+
stats->hw_stats = NULL;
42
+
}
43
+
}
44
+
45
+
static int bnge_alloc_stats_mem(struct bnge_net *bn,
46
+
struct bnge_stats_mem *stats)
47
+
{
48
+
struct bnge_dev *bd = bn->bd;
49
+
50
+
stats->hw_stats = dma_alloc_coherent(bd->dev, stats->len,
51
+
&stats->hw_stats_map, GFP_KERNEL);
52
+
if (!stats->hw_stats)
53
+
return -ENOMEM;
54
+
55
+
return 0;
56
+
}
57
+
58
+
static void bnge_free_ring_stats(struct bnge_net *bn)
59
+
{
60
+
struct bnge_dev *bd = bn->bd;
61
+
int i;
62
+
63
+
if (!bn->bnapi)
64
+
return;
65
+
66
+
for (i = 0; i < bd->nq_nr_rings; i++) {
67
+
struct bnge_napi *bnapi = bn->bnapi[i];
68
+
struct bnge_nq_ring_info *nqr = &bnapi->nq_ring;
69
+
70
+
bnge_free_stats_mem(bn, &nqr->stats);
71
+
}
72
+
}
73
+
74
+
static int bnge_alloc_ring_stats(struct bnge_net *bn)
75
+
{
76
+
struct bnge_dev *bd = bn->bd;
77
+
u32 size, i;
78
+
int rc;
79
+
80
+
size = bd->hw_ring_stats_size;
81
+
82
+
for (i = 0; i < bd->nq_nr_rings; i++) {
83
+
struct bnge_napi *bnapi = bn->bnapi[i];
84
+
struct bnge_nq_ring_info *nqr = &bnapi->nq_ring;
85
+
86
+
nqr->stats.len = size;
87
+
rc = bnge_alloc_stats_mem(bn, &nqr->stats);
88
+
if (rc)
89
+
goto err_free_ring_stats;
90
+
91
+
nqr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
92
+
}
93
+
return 0;
94
+
95
+
err_free_ring_stats:
96
+
bnge_free_ring_stats(bn);
97
+
return rc;
98
+
}
99
+
100
+
static void bnge_free_nq_desc_arr(struct bnge_nq_ring_info *nqr)
101
+
{
102
+
struct bnge_ring_struct *ring = &nqr->ring_struct;
103
+
104
+
kfree(nqr->desc_ring);
105
+
nqr->desc_ring = NULL;
106
+
ring->ring_mem.pg_arr = NULL;
107
+
kfree(nqr->desc_mapping);
108
+
nqr->desc_mapping = NULL;
109
+
ring->ring_mem.dma_arr = NULL;
110
+
}
111
+
112
+
static void bnge_free_cp_desc_arr(struct bnge_cp_ring_info *cpr)
113
+
{
114
+
struct bnge_ring_struct *ring = &cpr->ring_struct;
115
+
116
+
kfree(cpr->desc_ring);
117
+
cpr->desc_ring = NULL;
118
+
ring->ring_mem.pg_arr = NULL;
119
+
kfree(cpr->desc_mapping);
120
+
cpr->desc_mapping = NULL;
121
+
ring->ring_mem.dma_arr = NULL;
122
+
}
123
+
124
+
static int bnge_alloc_nq_desc_arr(struct bnge_nq_ring_info *nqr, int n)
125
+
{
126
+
nqr->desc_ring = kcalloc(n, sizeof(*nqr->desc_ring), GFP_KERNEL);
127
+
if (!nqr->desc_ring)
128
+
return -ENOMEM;
129
+
130
+
nqr->desc_mapping = kcalloc(n, sizeof(*nqr->desc_mapping), GFP_KERNEL);
131
+
if (!nqr->desc_mapping)
132
+
goto err_free_desc_ring;
133
+
return 0;
134
+
135
+
err_free_desc_ring:
136
+
kfree(nqr->desc_ring);
137
+
nqr->desc_ring = NULL;
138
+
return -ENOMEM;
139
+
}
140
+
141
+
static int bnge_alloc_cp_desc_arr(struct bnge_cp_ring_info *cpr, int n)
142
+
{
143
+
cpr->desc_ring = kcalloc(n, sizeof(*cpr->desc_ring), GFP_KERNEL);
144
+
if (!cpr->desc_ring)
145
+
return -ENOMEM;
146
+
147
+
cpr->desc_mapping = kcalloc(n, sizeof(*cpr->desc_mapping), GFP_KERNEL);
148
+
if (!cpr->desc_mapping)
149
+
goto err_free_desc_ring;
150
+
return 0;
151
+
152
+
err_free_desc_ring:
153
+
kfree(cpr->desc_ring);
154
+
cpr->desc_ring = NULL;
155
+
return -ENOMEM;
156
+
}
157
+
158
+
static void bnge_free_nq_arrays(struct bnge_net *bn)
159
+
{
160
+
struct bnge_dev *bd = bn->bd;
161
+
int i;
162
+
163
+
for (i = 0; i < bd->nq_nr_rings; i++) {
164
+
struct bnge_napi *bnapi = bn->bnapi[i];
165
+
166
+
bnge_free_nq_desc_arr(&bnapi->nq_ring);
167
+
}
168
+
}
169
+
170
+
static int bnge_alloc_nq_arrays(struct bnge_net *bn)
171
+
{
172
+
struct bnge_dev *bd = bn->bd;
173
+
int i, rc;
174
+
175
+
for (i = 0; i < bd->nq_nr_rings; i++) {
176
+
struct bnge_napi *bnapi = bn->bnapi[i];
177
+
178
+
rc = bnge_alloc_nq_desc_arr(&bnapi->nq_ring, bn->cp_nr_pages);
179
+
if (rc)
180
+
goto err_free_nq_arrays;
181
+
}
182
+
return 0;
183
+
184
+
err_free_nq_arrays:
185
+
bnge_free_nq_arrays(bn);
186
+
return rc;
187
+
}
188
+
189
+
static void bnge_free_nq_tree(struct bnge_net *bn)
190
+
{
191
+
struct bnge_dev *bd = bn->bd;
192
+
int i;
193
+
194
+
for (i = 0; i < bd->nq_nr_rings; i++) {
195
+
struct bnge_napi *bnapi = bn->bnapi[i];
196
+
struct bnge_nq_ring_info *nqr;
197
+
struct bnge_ring_struct *ring;
198
+
int j;
199
+
200
+
nqr = &bnapi->nq_ring;
201
+
ring = &nqr->ring_struct;
202
+
203
+
bnge_free_ring(bd, &ring->ring_mem);
204
+
205
+
if (!nqr->cp_ring_arr)
206
+
continue;
207
+
208
+
for (j = 0; j < nqr->cp_ring_count; j++) {
209
+
struct bnge_cp_ring_info *cpr = &nqr->cp_ring_arr[j];
210
+
211
+
ring = &cpr->ring_struct;
212
+
bnge_free_ring(bd, &ring->ring_mem);
213
+
bnge_free_cp_desc_arr(cpr);
214
+
}
215
+
kfree(nqr->cp_ring_arr);
216
+
nqr->cp_ring_arr = NULL;
217
+
nqr->cp_ring_count = 0;
218
+
}
219
+
}
220
+
221
+
static int alloc_one_cp_ring(struct bnge_net *bn,
222
+
struct bnge_cp_ring_info *cpr)
223
+
{
224
+
struct bnge_ring_mem_info *rmem;
225
+
struct bnge_ring_struct *ring;
226
+
struct bnge_dev *bd = bn->bd;
227
+
int rc;
228
+
229
+
rc = bnge_alloc_cp_desc_arr(cpr, bn->cp_nr_pages);
230
+
if (rc)
231
+
return -ENOMEM;
232
+
ring = &cpr->ring_struct;
233
+
rmem = &ring->ring_mem;
234
+
rmem->nr_pages = bn->cp_nr_pages;
235
+
rmem->page_size = HW_CMPD_RING_SIZE;
236
+
rmem->pg_arr = (void **)cpr->desc_ring;
237
+
rmem->dma_arr = cpr->desc_mapping;
238
+
rmem->flags = BNGE_RMEM_RING_PTE_FLAG;
239
+
rc = bnge_alloc_ring(bd, rmem);
240
+
if (rc)
241
+
goto err_free_cp_desc_arr;
242
+
return rc;
243
+
244
+
err_free_cp_desc_arr:
245
+
bnge_free_cp_desc_arr(cpr);
246
+
return rc;
247
+
}
248
+
249
+
static int bnge_alloc_nq_tree(struct bnge_net *bn)
250
+
{
251
+
struct bnge_dev *bd = bn->bd;
252
+
int i, j, ulp_msix, rc;
253
+
int tcs = 1;
254
+
255
+
ulp_msix = bnge_aux_get_msix(bd);
256
+
for (i = 0, j = 0; i < bd->nq_nr_rings; i++) {
257
+
bool sh = !!(bd->flags & BNGE_EN_SHARED_CHNL);
258
+
struct bnge_napi *bnapi = bn->bnapi[i];
259
+
struct bnge_nq_ring_info *nqr;
260
+
struct bnge_cp_ring_info *cpr;
261
+
struct bnge_ring_struct *ring;
262
+
int cp_count = 0, k;
263
+
int rx = 0, tx = 0;
264
+
265
+
nqr = &bnapi->nq_ring;
266
+
nqr->bnapi = bnapi;
267
+
ring = &nqr->ring_struct;
268
+
269
+
rc = bnge_alloc_ring(bd, &ring->ring_mem);
270
+
if (rc)
271
+
goto err_free_nq_tree;
272
+
273
+
ring->map_idx = ulp_msix + i;
274
+
275
+
if (i < bd->rx_nr_rings) {
276
+
cp_count++;
277
+
rx = 1;
278
+
}
279
+
280
+
if ((sh && i < bd->tx_nr_rings) ||
281
+
(!sh && i >= bd->rx_nr_rings)) {
282
+
cp_count += tcs;
283
+
tx = 1;
284
+
}
285
+
286
+
nqr->cp_ring_arr = kcalloc(cp_count, sizeof(*cpr),
287
+
GFP_KERNEL);
288
+
if (!nqr->cp_ring_arr) {
289
+
rc = -ENOMEM;
290
+
goto err_free_nq_tree;
291
+
}
292
+
293
+
nqr->cp_ring_count = cp_count;
294
+
295
+
for (k = 0; k < cp_count; k++) {
296
+
cpr = &nqr->cp_ring_arr[k];
297
+
rc = alloc_one_cp_ring(bn, cpr);
298
+
if (rc)
299
+
goto err_free_nq_tree;
300
+
301
+
cpr->bnapi = bnapi;
302
+
cpr->cp_idx = k;
303
+
if (!k && rx) {
304
+
bn->rx_ring[i].rx_cpr = cpr;
305
+
cpr->cp_ring_type = BNGE_NQ_HDL_TYPE_RX;
306
+
} else {
307
+
int n, tc = k - rx;
308
+
309
+
n = BNGE_TC_TO_RING_BASE(bd, tc) + j;
310
+
bn->tx_ring[n].tx_cpr = cpr;
311
+
cpr->cp_ring_type = BNGE_NQ_HDL_TYPE_TX;
312
+
}
313
+
}
314
+
if (tx)
315
+
j++;
316
+
}
317
+
return 0;
318
+
319
+
err_free_nq_tree:
320
+
bnge_free_nq_tree(bn);
321
+
return rc;
322
+
}
323
+
324
+
static bool bnge_separate_head_pool(struct bnge_rx_ring_info *rxr)
325
+
{
326
+
return rxr->need_head_pool || PAGE_SIZE > BNGE_RX_PAGE_SIZE;
327
+
}
328
+
329
+
static void bnge_free_one_rx_ring_bufs(struct bnge_net *bn,
330
+
struct bnge_rx_ring_info *rxr)
331
+
{
332
+
int i, max_idx;
333
+
334
+
if (!rxr->rx_buf_ring)
335
+
return;
336
+
337
+
max_idx = bn->rx_nr_pages * RX_DESC_CNT;
338
+
339
+
for (i = 0; i < max_idx; i++) {
340
+
struct bnge_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i];
341
+
void *data = rx_buf->data;
342
+
343
+
if (!data)
344
+
continue;
345
+
346
+
rx_buf->data = NULL;
347
+
page_pool_free_va(rxr->head_pool, data, true);
348
+
}
349
+
}
350
+
351
+
static void bnge_free_one_agg_ring_bufs(struct bnge_net *bn,
352
+
struct bnge_rx_ring_info *rxr)
353
+
{
354
+
int i, max_idx;
355
+
356
+
if (!rxr->rx_agg_buf_ring)
357
+
return;
358
+
359
+
max_idx = bn->rx_agg_nr_pages * RX_DESC_CNT;
360
+
361
+
for (i = 0; i < max_idx; i++) {
362
+
struct bnge_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_buf_ring[i];
363
+
netmem_ref netmem = rx_agg_buf->netmem;
364
+
365
+
if (!netmem)
366
+
continue;
367
+
368
+
rx_agg_buf->netmem = 0;
369
+
__clear_bit(i, rxr->rx_agg_bmap);
370
+
371
+
page_pool_recycle_direct_netmem(rxr->page_pool, netmem);
372
+
}
373
+
}
374
+
375
+
static void bnge_free_one_rx_ring_pair_bufs(struct bnge_net *bn,
376
+
struct bnge_rx_ring_info *rxr)
377
+
{
378
+
bnge_free_one_rx_ring_bufs(bn, rxr);
379
+
bnge_free_one_agg_ring_bufs(bn, rxr);
380
+
}
381
+
382
+
static void bnge_free_rx_ring_pair_bufs(struct bnge_net *bn)
383
+
{
384
+
struct bnge_dev *bd = bn->bd;
385
+
int i;
386
+
387
+
if (!bn->rx_ring)
388
+
return;
389
+
390
+
for (i = 0; i < bd->rx_nr_rings; i++)
391
+
bnge_free_one_rx_ring_pair_bufs(bn, &bn->rx_ring[i]);
392
+
}
393
+
394
+
static void bnge_free_all_rings_bufs(struct bnge_net *bn)
395
+
{
396
+
bnge_free_rx_ring_pair_bufs(bn);
397
+
}
398
+
399
+
static void bnge_free_rx_rings(struct bnge_net *bn)
400
+
{
401
+
struct bnge_dev *bd = bn->bd;
402
+
int i;
403
+
404
+
for (i = 0; i < bd->rx_nr_rings; i++) {
405
+
struct bnge_rx_ring_info *rxr = &bn->rx_ring[i];
406
+
struct bnge_ring_struct *ring;
407
+
408
+
page_pool_destroy(rxr->page_pool);
409
+
page_pool_destroy(rxr->head_pool);
410
+
rxr->page_pool = rxr->head_pool = NULL;
411
+
412
+
kfree(rxr->rx_agg_bmap);
413
+
rxr->rx_agg_bmap = NULL;
414
+
415
+
ring = &rxr->rx_ring_struct;
416
+
bnge_free_ring(bd, &ring->ring_mem);
417
+
418
+
ring = &rxr->rx_agg_ring_struct;
419
+
bnge_free_ring(bd, &ring->ring_mem);
420
+
}
421
+
}
422
+
423
+
static int bnge_alloc_rx_page_pool(struct bnge_net *bn,
424
+
struct bnge_rx_ring_info *rxr,
425
+
int numa_node)
426
+
{
427
+
const unsigned int agg_size_fac = PAGE_SIZE / BNGE_RX_PAGE_SIZE;
428
+
const unsigned int rx_size_fac = PAGE_SIZE / SZ_4K;
429
+
struct page_pool_params pp = { 0 };
430
+
struct bnge_dev *bd = bn->bd;
431
+
struct page_pool *pool;
432
+
433
+
pp.pool_size = bn->rx_agg_ring_size / agg_size_fac;
434
+
pp.nid = numa_node;
435
+
pp.netdev = bn->netdev;
436
+
pp.dev = bd->dev;
437
+
pp.dma_dir = bn->rx_dir;
438
+
pp.max_len = PAGE_SIZE;
439
+
pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV |
440
+
PP_FLAG_ALLOW_UNREADABLE_NETMEM;
441
+
pp.queue_idx = rxr->bnapi->index;
442
+
443
+
pool = page_pool_create(&pp);
444
+
if (IS_ERR(pool))
445
+
return PTR_ERR(pool);
446
+
rxr->page_pool = pool;
447
+
448
+
rxr->need_head_pool = page_pool_is_unreadable(pool);
449
+
if (bnge_separate_head_pool(rxr)) {
450
+
pp.pool_size = min(bn->rx_ring_size / rx_size_fac, 1024);
451
+
pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
452
+
pool = page_pool_create(&pp);
453
+
if (IS_ERR(pool))
454
+
goto err_destroy_pp;
455
+
} else {
456
+
page_pool_get(pool);
457
+
}
458
+
rxr->head_pool = pool;
459
+
return 0;
460
+
461
+
err_destroy_pp:
462
+
page_pool_destroy(rxr->page_pool);
463
+
rxr->page_pool = NULL;
464
+
return PTR_ERR(pool);
465
+
}
466
+
467
+
static void bnge_enable_rx_page_pool(struct bnge_rx_ring_info *rxr)
468
+
{
469
+
page_pool_enable_direct_recycling(rxr->head_pool, &rxr->bnapi->napi);
470
+
page_pool_enable_direct_recycling(rxr->page_pool, &rxr->bnapi->napi);
471
+
}
472
+
473
+
static int bnge_alloc_rx_agg_bmap(struct bnge_net *bn,
474
+
struct bnge_rx_ring_info *rxr)
475
+
{
476
+
u16 mem_size;
477
+
478
+
rxr->rx_agg_bmap_size = bn->rx_agg_ring_mask + 1;
479
+
mem_size = rxr->rx_agg_bmap_size / 8;
480
+
rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
481
+
if (!rxr->rx_agg_bmap)
482
+
return -ENOMEM;
483
+
484
+
return 0;
485
+
}
486
+
487
+
static int bnge_alloc_rx_rings(struct bnge_net *bn)
488
+
{
489
+
int i, rc = 0, agg_rings = 0, cpu;
490
+
struct bnge_dev *bd = bn->bd;
491
+
492
+
if (bnge_is_agg_reqd(bd))
493
+
agg_rings = 1;
494
+
495
+
for (i = 0; i < bd->rx_nr_rings; i++) {
496
+
struct bnge_rx_ring_info *rxr = &bn->rx_ring[i];
497
+
struct bnge_ring_struct *ring;
498
+
int cpu_node;
499
+
500
+
ring = &rxr->rx_ring_struct;
501
+
502
+
cpu = cpumask_local_spread(i, dev_to_node(bd->dev));
503
+
cpu_node = cpu_to_node(cpu);
504
+
netdev_dbg(bn->netdev, "Allocating page pool for rx_ring[%d] on numa_node: %d\n",
505
+
i, cpu_node);
506
+
rc = bnge_alloc_rx_page_pool(bn, rxr, cpu_node);
507
+
if (rc)
508
+
goto err_free_rx_rings;
509
+
bnge_enable_rx_page_pool(rxr);
510
+
511
+
rc = bnge_alloc_ring(bd, &ring->ring_mem);
512
+
if (rc)
513
+
goto err_free_rx_rings;
514
+
515
+
ring->grp_idx = i;
516
+
if (agg_rings) {
517
+
ring = &rxr->rx_agg_ring_struct;
518
+
rc = bnge_alloc_ring(bd, &ring->ring_mem);
519
+
if (rc)
520
+
goto err_free_rx_rings;
521
+
522
+
ring->grp_idx = i;
523
+
rc = bnge_alloc_rx_agg_bmap(bn, rxr);
524
+
if (rc)
525
+
goto err_free_rx_rings;
526
+
}
527
+
}
528
+
return rc;
529
+
530
+
err_free_rx_rings:
531
+
bnge_free_rx_rings(bn);
532
+
return rc;
533
+
}
534
+
535
+
static void bnge_free_tx_rings(struct bnge_net *bn)
536
+
{
537
+
struct bnge_dev *bd = bn->bd;
538
+
int i;
539
+
540
+
for (i = 0; i < bd->tx_nr_rings; i++) {
541
+
struct bnge_tx_ring_info *txr = &bn->tx_ring[i];
542
+
struct bnge_ring_struct *ring;
543
+
544
+
ring = &txr->tx_ring_struct;
545
+
546
+
bnge_free_ring(bd, &ring->ring_mem);
547
+
}
548
+
}
549
+
550
+
static int bnge_alloc_tx_rings(struct bnge_net *bn)
551
+
{
552
+
struct bnge_dev *bd = bn->bd;
553
+
int i, j, rc;
554
+
555
+
for (i = 0, j = 0; i < bd->tx_nr_rings; i++) {
556
+
struct bnge_tx_ring_info *txr = &bn->tx_ring[i];
557
+
struct bnge_ring_struct *ring;
558
+
u8 qidx;
559
+
560
+
ring = &txr->tx_ring_struct;
561
+
562
+
rc = bnge_alloc_ring(bd, &ring->ring_mem);
563
+
if (rc)
564
+
goto err_free_tx_rings;
565
+
566
+
ring->grp_idx = txr->bnapi->index;
567
+
qidx = bd->tc_to_qidx[j];
568
+
ring->queue_id = bd->q_info[qidx].queue_id;
569
+
if (BNGE_RING_TO_TC_OFF(bd, i) == (bd->tx_nr_rings_per_tc - 1))
570
+
j++;
571
+
}
572
+
return 0;
573
+
574
+
err_free_tx_rings:
575
+
bnge_free_tx_rings(bn);
576
+
return rc;
577
+
}
578
+
579
+
static void bnge_free_vnic_attributes(struct bnge_net *bn)
580
+
{
581
+
struct pci_dev *pdev = bn->bd->pdev;
582
+
struct bnge_vnic_info *vnic;
583
+
int i;
584
+
585
+
if (!bn->vnic_info)
586
+
return;
587
+
588
+
for (i = 0; i < bn->nr_vnics; i++) {
589
+
vnic = &bn->vnic_info[i];
590
+
591
+
kfree(vnic->uc_list);
592
+
vnic->uc_list = NULL;
593
+
594
+
if (vnic->mc_list) {
595
+
dma_free_coherent(&pdev->dev, vnic->mc_list_size,
596
+
vnic->mc_list, vnic->mc_list_mapping);
597
+
vnic->mc_list = NULL;
598
+
}
599
+
600
+
if (vnic->rss_table) {
601
+
dma_free_coherent(&pdev->dev, vnic->rss_table_size,
602
+
vnic->rss_table,
603
+
vnic->rss_table_dma_addr);
604
+
vnic->rss_table = NULL;
605
+
}
606
+
607
+
vnic->rss_hash_key = NULL;
608
+
vnic->flags = 0;
609
+
}
610
+
}
611
+
612
+
static int bnge_alloc_vnic_attributes(struct bnge_net *bn)
613
+
{
614
+
struct bnge_dev *bd = bn->bd;
615
+
struct bnge_vnic_info *vnic;
616
+
int i, size;
617
+
618
+
for (i = 0; i < bn->nr_vnics; i++) {
619
+
vnic = &bn->vnic_info[i];
620
+
621
+
if (vnic->flags & BNGE_VNIC_UCAST_FLAG) {
622
+
int mem_size = (BNGE_MAX_UC_ADDRS - 1) * ETH_ALEN;
623
+
624
+
vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
625
+
if (!vnic->uc_list)
626
+
goto err_free_vnic_attributes;
627
+
}
628
+
629
+
if (vnic->flags & BNGE_VNIC_MCAST_FLAG) {
630
+
vnic->mc_list_size = BNGE_MAX_MC_ADDRS * ETH_ALEN;
631
+
vnic->mc_list =
632
+
dma_alloc_coherent(bd->dev,
633
+
vnic->mc_list_size,
634
+
&vnic->mc_list_mapping,
635
+
GFP_KERNEL);
636
+
if (!vnic->mc_list)
637
+
goto err_free_vnic_attributes;
638
+
}
639
+
640
+
/* Allocate rss table and hash key */
641
+
size = L1_CACHE_ALIGN(BNGE_MAX_RSS_TABLE_SIZE);
642
+
643
+
vnic->rss_table_size = size + HW_HASH_KEY_SIZE;
644
+
vnic->rss_table = dma_alloc_coherent(bd->dev,
645
+
vnic->rss_table_size,
646
+
&vnic->rss_table_dma_addr,
647
+
GFP_KERNEL);
648
+
if (!vnic->rss_table)
649
+
goto err_free_vnic_attributes;
650
+
651
+
vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
652
+
vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
653
+
}
654
+
return 0;
655
+
656
+
err_free_vnic_attributes:
657
+
bnge_free_vnic_attributes(bn);
658
+
return -ENOMEM;
659
+
}
660
+
661
+
static int bnge_alloc_vnics(struct bnge_net *bn)
662
+
{
663
+
int num_vnics;
664
+
665
+
/* Allocate only 1 VNIC for now
666
+
* Additional VNICs will be added based on RFS/NTUPLE in future patches
667
+
*/
668
+
num_vnics = 1;
669
+
670
+
bn->vnic_info = kcalloc(num_vnics, sizeof(struct bnge_vnic_info),
671
+
GFP_KERNEL);
672
+
if (!bn->vnic_info)
673
+
return -ENOMEM;
674
+
675
+
bn->nr_vnics = num_vnics;
676
+
677
+
return 0;
678
+
}
679
+
680
+
static void bnge_free_vnics(struct bnge_net *bn)
681
+
{
682
+
kfree(bn->vnic_info);
683
+
bn->vnic_info = NULL;
684
+
bn->nr_vnics = 0;
685
+
}
686
+
687
+
static void bnge_free_ring_grps(struct bnge_net *bn)
688
+
{
689
+
kfree(bn->grp_info);
690
+
bn->grp_info = NULL;
691
+
}
692
+
693
+
static int bnge_init_ring_grps(struct bnge_net *bn)
694
+
{
695
+
struct bnge_dev *bd = bn->bd;
696
+
int i;
697
+
698
+
bn->grp_info = kcalloc(bd->nq_nr_rings,
699
+
sizeof(struct bnge_ring_grp_info),
700
+
GFP_KERNEL);
701
+
if (!bn->grp_info)
702
+
return -ENOMEM;
703
+
for (i = 0; i < bd->nq_nr_rings; i++) {
704
+
bn->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
705
+
bn->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
706
+
bn->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
707
+
bn->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
708
+
bn->grp_info[i].nq_fw_ring_id = INVALID_HW_RING_ID;
709
+
}
710
+
711
+
return 0;
712
+
}
713
+
714
+
static void bnge_free_core(struct bnge_net *bn)
715
+
{
716
+
bnge_free_vnic_attributes(bn);
717
+
bnge_free_tx_rings(bn);
718
+
bnge_free_rx_rings(bn);
719
+
bnge_free_nq_tree(bn);
720
+
bnge_free_nq_arrays(bn);
721
+
bnge_free_ring_stats(bn);
722
+
bnge_free_ring_grps(bn);
723
+
bnge_free_vnics(bn);
724
+
kfree(bn->tx_ring_map);
725
+
bn->tx_ring_map = NULL;
726
+
kfree(bn->tx_ring);
727
+
bn->tx_ring = NULL;
728
+
kfree(bn->rx_ring);
729
+
bn->rx_ring = NULL;
730
+
kfree(bn->bnapi);
731
+
bn->bnapi = NULL;
732
+
}
733
+
734
+
static int bnge_alloc_core(struct bnge_net *bn)
735
+
{
736
+
struct bnge_dev *bd = bn->bd;
737
+
int i, j, size, arr_size;
738
+
int rc = -ENOMEM;
739
+
void *bnapi;
740
+
741
+
arr_size = L1_CACHE_ALIGN(sizeof(struct bnge_napi *) *
742
+
bd->nq_nr_rings);
743
+
size = L1_CACHE_ALIGN(sizeof(struct bnge_napi));
744
+
bnapi = kzalloc(arr_size + size * bd->nq_nr_rings, GFP_KERNEL);
745
+
if (!bnapi)
746
+
return rc;
747
+
748
+
bn->bnapi = bnapi;
749
+
bnapi += arr_size;
750
+
for (i = 0; i < bd->nq_nr_rings; i++, bnapi += size) {
751
+
struct bnge_nq_ring_info *nqr;
752
+
753
+
bn->bnapi[i] = bnapi;
754
+
bn->bnapi[i]->index = i;
755
+
bn->bnapi[i]->bn = bn;
756
+
nqr = &bn->bnapi[i]->nq_ring;
757
+
nqr->ring_struct.ring_mem.flags = BNGE_RMEM_RING_PTE_FLAG;
758
+
}
759
+
760
+
bn->rx_ring = kcalloc(bd->rx_nr_rings,
761
+
sizeof(struct bnge_rx_ring_info),
762
+
GFP_KERNEL);
763
+
if (!bn->rx_ring)
764
+
goto err_free_core;
765
+
766
+
for (i = 0; i < bd->rx_nr_rings; i++) {
767
+
struct bnge_rx_ring_info *rxr = &bn->rx_ring[i];
768
+
769
+
rxr->rx_ring_struct.ring_mem.flags =
770
+
BNGE_RMEM_RING_PTE_FLAG;
771
+
rxr->rx_agg_ring_struct.ring_mem.flags =
772
+
BNGE_RMEM_RING_PTE_FLAG;
773
+
rxr->bnapi = bn->bnapi[i];
774
+
bn->bnapi[i]->rx_ring = &bn->rx_ring[i];
775
+
}
776
+
777
+
bn->tx_ring = kcalloc(bd->tx_nr_rings,
778
+
sizeof(struct bnge_tx_ring_info),
779
+
GFP_KERNEL);
780
+
if (!bn->tx_ring)
781
+
goto err_free_core;
782
+
783
+
bn->tx_ring_map = kcalloc(bd->tx_nr_rings, sizeof(u16),
784
+
GFP_KERNEL);
785
+
if (!bn->tx_ring_map)
786
+
goto err_free_core;
787
+
788
+
if (bd->flags & BNGE_EN_SHARED_CHNL)
789
+
j = 0;
790
+
else
791
+
j = bd->rx_nr_rings;
792
+
793
+
for (i = 0; i < bd->tx_nr_rings; i++) {
794
+
struct bnge_tx_ring_info *txr = &bn->tx_ring[i];
795
+
struct bnge_napi *bnapi2;
796
+
int k;
797
+
798
+
txr->tx_ring_struct.ring_mem.flags = BNGE_RMEM_RING_PTE_FLAG;
799
+
bn->tx_ring_map[i] = i;
800
+
k = j + BNGE_RING_TO_TC_OFF(bd, i);
801
+
802
+
bnapi2 = bn->bnapi[k];
803
+
txr->txq_index = i;
804
+
txr->tx_napi_idx =
805
+
BNGE_RING_TO_TC(bd, txr->txq_index);
806
+
bnapi2->tx_ring[txr->tx_napi_idx] = txr;
807
+
txr->bnapi = bnapi2;
808
+
}
809
+
810
+
rc = bnge_alloc_ring_stats(bn);
811
+
if (rc)
812
+
goto err_free_core;
813
+
814
+
rc = bnge_alloc_vnics(bn);
815
+
if (rc)
816
+
goto err_free_core;
817
+
818
+
rc = bnge_alloc_nq_arrays(bn);
819
+
if (rc)
820
+
goto err_free_core;
821
+
822
+
bnge_init_ring_struct(bn);
823
+
824
+
rc = bnge_alloc_rx_rings(bn);
825
+
if (rc)
826
+
goto err_free_core;
827
+
828
+
rc = bnge_alloc_tx_rings(bn);
829
+
if (rc)
830
+
goto err_free_core;
831
+
832
+
rc = bnge_alloc_nq_tree(bn);
833
+
if (rc)
834
+
goto err_free_core;
835
+
836
+
bn->vnic_info[BNGE_VNIC_DEFAULT].flags |= BNGE_VNIC_RSS_FLAG |
837
+
BNGE_VNIC_MCAST_FLAG |
838
+
BNGE_VNIC_UCAST_FLAG;
839
+
rc = bnge_alloc_vnic_attributes(bn);
840
+
if (rc)
841
+
goto err_free_core;
842
+
return 0;
843
+
844
+
err_free_core:
845
+
bnge_free_core(bn);
846
+
return rc;
847
+
}
848
+
849
+
u16 bnge_cp_ring_for_rx(struct bnge_rx_ring_info *rxr)
850
+
{
851
+
return rxr->rx_cpr->ring_struct.fw_ring_id;
852
+
}
853
+
854
+
u16 bnge_cp_ring_for_tx(struct bnge_tx_ring_info *txr)
855
+
{
856
+
return txr->tx_cpr->ring_struct.fw_ring_id;
857
+
}
858
+
859
+
static void bnge_db_nq(struct bnge_net *bn, struct bnge_db_info *db, u32 idx)
860
+
{
861
+
bnge_writeq(bn->bd, db->db_key64 | DBR_TYPE_NQ_MASK |
862
+
DB_RING_IDX(db, idx), db->doorbell);
863
+
}
864
+
865
+
static void bnge_db_cq(struct bnge_net *bn, struct bnge_db_info *db, u32 idx)
866
+
{
867
+
bnge_writeq(bn->bd, db->db_key64 | DBR_TYPE_CQ_ARMALL |
868
+
DB_RING_IDX(db, idx), db->doorbell);
869
+
}
870
+
871
+
static int bnge_cp_num_to_irq_num(struct bnge_net *bn, int n)
872
+
{
873
+
struct bnge_napi *bnapi = bn->bnapi[n];
874
+
struct bnge_nq_ring_info *nqr;
875
+
876
+
nqr = &bnapi->nq_ring;
877
+
878
+
return nqr->ring_struct.map_idx;
879
+
}
880
+
881
+
static irqreturn_t bnge_msix(int irq, void *dev_instance)
882
+
{
883
+
/* NAPI scheduling to be added in a future patch */
884
+
return IRQ_HANDLED;
885
+
}
886
+
887
+
static void bnge_init_nq_tree(struct bnge_net *bn)
888
+
{
889
+
struct bnge_dev *bd = bn->bd;
890
+
int i, j;
891
+
892
+
for (i = 0; i < bd->nq_nr_rings; i++) {
893
+
struct bnge_nq_ring_info *nqr = &bn->bnapi[i]->nq_ring;
894
+
struct bnge_ring_struct *ring = &nqr->ring_struct;
895
+
896
+
ring->fw_ring_id = INVALID_HW_RING_ID;
897
+
for (j = 0; j < nqr->cp_ring_count; j++) {
898
+
struct bnge_cp_ring_info *cpr = &nqr->cp_ring_arr[j];
899
+
900
+
ring = &cpr->ring_struct;
901
+
ring->fw_ring_id = INVALID_HW_RING_ID;
902
+
}
903
+
}
904
+
}
905
+
906
+
static netmem_ref __bnge_alloc_rx_netmem(struct bnge_net *bn,
907
+
dma_addr_t *mapping,
908
+
struct bnge_rx_ring_info *rxr,
909
+
unsigned int *offset,
910
+
gfp_t gfp)
911
+
{
912
+
netmem_ref netmem;
913
+
914
+
if (PAGE_SIZE > BNGE_RX_PAGE_SIZE) {
915
+
netmem = page_pool_alloc_frag_netmem(rxr->page_pool, offset,
916
+
BNGE_RX_PAGE_SIZE, gfp);
917
+
} else {
918
+
netmem = page_pool_alloc_netmems(rxr->page_pool, gfp);
919
+
*offset = 0;
920
+
}
921
+
if (!netmem)
922
+
return 0;
923
+
924
+
*mapping = page_pool_get_dma_addr_netmem(netmem) + *offset;
925
+
return netmem;
926
+
}
927
+
928
+
static u8 *__bnge_alloc_rx_frag(struct bnge_net *bn, dma_addr_t *mapping,
929
+
struct bnge_rx_ring_info *rxr,
930
+
gfp_t gfp)
931
+
{
932
+
unsigned int offset;
933
+
struct page *page;
934
+
935
+
page = page_pool_alloc_frag(rxr->head_pool, &offset,
936
+
bn->rx_buf_size, gfp);
937
+
if (!page)
938
+
return NULL;
939
+
940
+
*mapping = page_pool_get_dma_addr(page) + bn->rx_dma_offset + offset;
941
+
return page_address(page) + offset;
942
+
}
943
+
944
+
static int bnge_alloc_rx_data(struct bnge_net *bn,
945
+
struct bnge_rx_ring_info *rxr,
946
+
u16 prod, gfp_t gfp)
947
+
{
948
+
struct bnge_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[RING_RX(bn, prod)];
949
+
struct rx_bd *rxbd;
950
+
dma_addr_t mapping;
951
+
u8 *data;
952
+
953
+
rxbd = &rxr->rx_desc_ring[RX_RING(bn, prod)][RX_IDX(prod)];
954
+
data = __bnge_alloc_rx_frag(bn, &mapping, rxr, gfp);
955
+
if (!data)
956
+
return -ENOMEM;
957
+
958
+
rx_buf->data = data;
959
+
rx_buf->data_ptr = data + bn->rx_offset;
960
+
rx_buf->mapping = mapping;
961
+
962
+
rxbd->rx_bd_haddr = cpu_to_le64(mapping);
963
+
964
+
return 0;
965
+
}
966
+
967
+
static int bnge_alloc_one_rx_ring_bufs(struct bnge_net *bn,
968
+
struct bnge_rx_ring_info *rxr,
969
+
int ring_nr)
970
+
{
971
+
u32 prod = rxr->rx_prod;
972
+
int i, rc = 0;
973
+
974
+
for (i = 0; i < bn->rx_ring_size; i++) {
975
+
rc = bnge_alloc_rx_data(bn, rxr, prod, GFP_KERNEL);
976
+
if (rc)
977
+
break;
978
+
prod = NEXT_RX(prod);
979
+
}
980
+
981
+
/* Abort if not a single buffer can be allocated */
982
+
if (rc && !i) {
983
+
netdev_err(bn->netdev,
984
+
"RX ring %d: allocated %d/%d buffers, abort\n",
985
+
ring_nr, i, bn->rx_ring_size);
986
+
return rc;
987
+
}
988
+
989
+
rxr->rx_prod = prod;
990
+
991
+
if (i < bn->rx_ring_size)
992
+
netdev_warn(bn->netdev,
993
+
"RX ring %d: allocated %d/%d buffers, continuing\n",
994
+
ring_nr, i, bn->rx_ring_size);
995
+
return 0;
996
+
}
997
+
998
+
static u16 bnge_find_next_agg_idx(struct bnge_rx_ring_info *rxr, u16 idx)
999
+
{
1000
+
u16 next, max = rxr->rx_agg_bmap_size;
1001
+
1002
+
next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
1003
+
if (next >= max)
1004
+
next = find_first_zero_bit(rxr->rx_agg_bmap, max);
1005
+
return next;
1006
+
}
1007
+
1008
+
static int bnge_alloc_rx_netmem(struct bnge_net *bn,
1009
+
struct bnge_rx_ring_info *rxr,
1010
+
u16 prod, gfp_t gfp)
1011
+
{
1012
+
struct bnge_sw_rx_agg_bd *rx_agg_buf;
1013
+
u16 sw_prod = rxr->rx_sw_agg_prod;
1014
+
unsigned int offset = 0;
1015
+
struct rx_bd *rxbd;
1016
+
dma_addr_t mapping;
1017
+
netmem_ref netmem;
1018
+
1019
+
rxbd = &rxr->rx_agg_desc_ring[RX_AGG_RING(bn, prod)][RX_IDX(prod)];
1020
+
netmem = __bnge_alloc_rx_netmem(bn, &mapping, rxr, &offset, gfp);
1021
+
if (!netmem)
1022
+
return -ENOMEM;
1023
+
1024
+
if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
1025
+
sw_prod = bnge_find_next_agg_idx(rxr, sw_prod);
1026
+
1027
+
__set_bit(sw_prod, rxr->rx_agg_bmap);
1028
+
rx_agg_buf = &rxr->rx_agg_buf_ring[sw_prod];
1029
+
rxr->rx_sw_agg_prod = RING_RX_AGG(bn, NEXT_RX_AGG(sw_prod));
1030
+
1031
+
rx_agg_buf->netmem = netmem;
1032
+
rx_agg_buf->offset = offset;
1033
+
rx_agg_buf->mapping = mapping;
1034
+
rxbd->rx_bd_haddr = cpu_to_le64(mapping);
1035
+
rxbd->rx_bd_opaque = sw_prod;
1036
+
return 0;
1037
+
}
1038
+
1039
+
static int bnge_alloc_one_agg_ring_bufs(struct bnge_net *bn,
1040
+
struct bnge_rx_ring_info *rxr,
1041
+
int ring_nr)
1042
+
{
1043
+
u32 prod = rxr->rx_agg_prod;
1044
+
int i, rc = 0;
1045
+
1046
+
for (i = 0; i < bn->rx_agg_ring_size; i++) {
1047
+
rc = bnge_alloc_rx_netmem(bn, rxr, prod, GFP_KERNEL);
1048
+
if (rc)
1049
+
break;
1050
+
prod = NEXT_RX_AGG(prod);
1051
+
}
1052
+
1053
+
if (rc && i < MAX_SKB_FRAGS) {
1054
+
netdev_err(bn->netdev,
1055
+
"Agg ring %d: allocated %d/%d buffers (min %d), abort\n",
1056
+
ring_nr, i, bn->rx_agg_ring_size, MAX_SKB_FRAGS);
1057
+
goto err_free_one_agg_ring_bufs;
1058
+
}
1059
+
1060
+
rxr->rx_agg_prod = prod;
1061
+
1062
+
if (i < bn->rx_agg_ring_size)
1063
+
netdev_warn(bn->netdev,
1064
+
"Agg ring %d: allocated %d/%d buffers, continuing\n",
1065
+
ring_nr, i, bn->rx_agg_ring_size);
1066
+
return 0;
1067
+
1068
+
err_free_one_agg_ring_bufs:
1069
+
bnge_free_one_agg_ring_bufs(bn, rxr);
1070
+
return -ENOMEM;
1071
+
}
1072
+
1073
+
static int bnge_alloc_one_rx_ring_pair_bufs(struct bnge_net *bn, int ring_nr)
1074
+
{
1075
+
struct bnge_rx_ring_info *rxr = &bn->rx_ring[ring_nr];
1076
+
int rc;
1077
+
1078
+
rc = bnge_alloc_one_rx_ring_bufs(bn, rxr, ring_nr);
1079
+
if (rc)
1080
+
return rc;
1081
+
1082
+
if (bnge_is_agg_reqd(bn->bd)) {
1083
+
rc = bnge_alloc_one_agg_ring_bufs(bn, rxr, ring_nr);
1084
+
if (rc)
1085
+
goto err_free_one_rx_ring_bufs;
1086
+
}
1087
+
return 0;
1088
+
1089
+
err_free_one_rx_ring_bufs:
1090
+
bnge_free_one_rx_ring_bufs(bn, rxr);
1091
+
return rc;
1092
+
}
1093
+
1094
+
static void bnge_init_rxbd_pages(struct bnge_ring_struct *ring, u32 type)
1095
+
{
1096
+
struct rx_bd **rx_desc_ring;
1097
+
u32 prod;
1098
+
int i;
1099
+
1100
+
rx_desc_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
1101
+
for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
1102
+
struct rx_bd *rxbd = rx_desc_ring[i];
1103
+
int j;
1104
+
1105
+
for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
1106
+
rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
1107
+
rxbd->rx_bd_opaque = prod;
1108
+
}
1109
+
}
1110
+
}
1111
+
1112
+
static void bnge_init_one_rx_ring_rxbd(struct bnge_net *bn,
1113
+
struct bnge_rx_ring_info *rxr)
1114
+
{
1115
+
struct bnge_ring_struct *ring;
1116
+
u32 type;
1117
+
1118
+
type = (bn->rx_buf_use_size << RX_BD_LEN_SHIFT) |
1119
+
RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
1120
+
1121
+
if (NET_IP_ALIGN == 2)
1122
+
type |= RX_BD_FLAGS_SOP;
1123
+
1124
+
ring = &rxr->rx_ring_struct;
1125
+
bnge_init_rxbd_pages(ring, type);
1126
+
ring->fw_ring_id = INVALID_HW_RING_ID;
1127
+
}
1128
+
1129
+
static void bnge_init_one_agg_ring_rxbd(struct bnge_net *bn,
1130
+
struct bnge_rx_ring_info *rxr)
1131
+
{
1132
+
struct bnge_ring_struct *ring;
1133
+
u32 type;
1134
+
1135
+
ring = &rxr->rx_agg_ring_struct;
1136
+
ring->fw_ring_id = INVALID_HW_RING_ID;
1137
+
if (bnge_is_agg_reqd(bn->bd)) {
1138
+
type = ((u32)BNGE_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
1139
+
RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
1140
+
1141
+
bnge_init_rxbd_pages(ring, type);
1142
+
}
1143
+
}
1144
+
1145
+
static void bnge_init_one_rx_ring_pair(struct bnge_net *bn, int ring_nr)
1146
+
{
1147
+
struct bnge_rx_ring_info *rxr;
1148
+
1149
+
rxr = &bn->rx_ring[ring_nr];
1150
+
bnge_init_one_rx_ring_rxbd(bn, rxr);
1151
+
1152
+
netif_queue_set_napi(bn->netdev, ring_nr, NETDEV_QUEUE_TYPE_RX,
1153
+
&rxr->bnapi->napi);
1154
+
1155
+
bnge_init_one_agg_ring_rxbd(bn, rxr);
1156
+
}
1157
+
1158
+
static int bnge_alloc_rx_ring_pair_bufs(struct bnge_net *bn)
1159
+
{
1160
+
int i, rc;
1161
+
1162
+
for (i = 0; i < bn->bd->rx_nr_rings; i++) {
1163
+
rc = bnge_alloc_one_rx_ring_pair_bufs(bn, i);
1164
+
if (rc)
1165
+
goto err_free_rx_ring_pair_bufs;
1166
+
}
1167
+
return 0;
1168
+
1169
+
err_free_rx_ring_pair_bufs:
1170
+
bnge_free_rx_ring_pair_bufs(bn);
1171
+
return rc;
1172
+
}
1173
+
1174
+
static void bnge_init_rx_rings(struct bnge_net *bn)
1175
+
{
1176
+
int i;
1177
+
1178
+
#define BNGE_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
1179
+
#define BNGE_RX_DMA_OFFSET NET_SKB_PAD
1180
+
bn->rx_offset = BNGE_RX_OFFSET;
1181
+
bn->rx_dma_offset = BNGE_RX_DMA_OFFSET;
1182
+
1183
+
for (i = 0; i < bn->bd->rx_nr_rings; i++)
1184
+
bnge_init_one_rx_ring_pair(bn, i);
1185
+
}
1186
+
1187
+
static void bnge_init_tx_rings(struct bnge_net *bn)
1188
+
{
1189
+
int i;
1190
+
1191
+
bn->tx_wake_thresh = max(bn->tx_ring_size / 2, BNGE_MIN_TX_DESC_CNT);
1192
+
1193
+
for (i = 0; i < bn->bd->tx_nr_rings; i++) {
1194
+
struct bnge_tx_ring_info *txr = &bn->tx_ring[i];
1195
+
struct bnge_ring_struct *ring = &txr->tx_ring_struct;
1196
+
1197
+
ring->fw_ring_id = INVALID_HW_RING_ID;
1198
+
1199
+
netif_queue_set_napi(bn->netdev, i, NETDEV_QUEUE_TYPE_TX,
1200
+
&txr->bnapi->napi);
1201
+
}
1202
+
}
1203
+
1204
+
static void bnge_init_vnics(struct bnge_net *bn)
1205
+
{
1206
+
struct bnge_vnic_info *vnic0 = &bn->vnic_info[BNGE_VNIC_DEFAULT];
1207
+
int i;
1208
+
1209
+
for (i = 0; i < bn->nr_vnics; i++) {
1210
+
struct bnge_vnic_info *vnic = &bn->vnic_info[i];
1211
+
int j;
1212
+
1213
+
vnic->fw_vnic_id = INVALID_HW_RING_ID;
1214
+
vnic->vnic_id = i;
1215
+
for (j = 0; j < BNGE_MAX_CTX_PER_VNIC; j++)
1216
+
vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
1217
+
1218
+
if (bn->vnic_info[i].rss_hash_key) {
1219
+
if (i == BNGE_VNIC_DEFAULT) {
1220
+
u8 *key = (void *)vnic->rss_hash_key;
1221
+
int k;
1222
+
1223
+
if (!bn->rss_hash_key_valid &&
1224
+
!bn->rss_hash_key_updated) {
1225
+
get_random_bytes(bn->rss_hash_key,
1226
+
HW_HASH_KEY_SIZE);
1227
+
bn->rss_hash_key_updated = true;
1228
+
}
1229
+
1230
+
memcpy(vnic->rss_hash_key, bn->rss_hash_key,
1231
+
HW_HASH_KEY_SIZE);
1232
+
1233
+
if (!bn->rss_hash_key_updated)
1234
+
continue;
1235
+
1236
+
bn->rss_hash_key_updated = false;
1237
+
bn->rss_hash_key_valid = true;
1238
+
1239
+
bn->toeplitz_prefix = 0;
1240
+
for (k = 0; k < 8; k++) {
1241
+
bn->toeplitz_prefix <<= 8;
1242
+
bn->toeplitz_prefix |= key[k];
1243
+
}
1244
+
} else {
1245
+
memcpy(vnic->rss_hash_key, vnic0->rss_hash_key,
1246
+
HW_HASH_KEY_SIZE);
1247
+
}
1248
+
}
1249
+
}
1250
+
}
1251
+
1252
+
static void bnge_set_db_mask(struct bnge_net *bn, struct bnge_db_info *db,
1253
+
u32 ring_type)
1254
+
{
1255
+
switch (ring_type) {
1256
+
case HWRM_RING_ALLOC_TX:
1257
+
db->db_ring_mask = bn->tx_ring_mask;
1258
+
break;
1259
+
case HWRM_RING_ALLOC_RX:
1260
+
db->db_ring_mask = bn->rx_ring_mask;
1261
+
break;
1262
+
case HWRM_RING_ALLOC_AGG:
1263
+
db->db_ring_mask = bn->rx_agg_ring_mask;
1264
+
break;
1265
+
case HWRM_RING_ALLOC_CMPL:
1266
+
case HWRM_RING_ALLOC_NQ:
1267
+
db->db_ring_mask = bn->cp_ring_mask;
1268
+
break;
1269
+
}
1270
+
db->db_epoch_mask = db->db_ring_mask + 1;
1271
+
db->db_epoch_shift = DBR_EPOCH_SFT - ilog2(db->db_epoch_mask);
1272
+
}
1273
+
1274
+
static void bnge_set_db(struct bnge_net *bn, struct bnge_db_info *db,
1275
+
u32 ring_type, u32 map_idx, u32 xid)
1276
+
{
1277
+
struct bnge_dev *bd = bn->bd;
1278
+
1279
+
switch (ring_type) {
1280
+
case HWRM_RING_ALLOC_TX:
1281
+
db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
1282
+
break;
1283
+
case HWRM_RING_ALLOC_RX:
1284
+
case HWRM_RING_ALLOC_AGG:
1285
+
db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
1286
+
break;
1287
+
case HWRM_RING_ALLOC_CMPL:
1288
+
db->db_key64 = DBR_PATH_L2;
1289
+
break;
1290
+
case HWRM_RING_ALLOC_NQ:
1291
+
db->db_key64 = DBR_PATH_L2;
1292
+
break;
1293
+
}
1294
+
db->db_key64 |= ((u64)xid << DBR_XID_SFT) | DBR_VALID;
1295
+
1296
+
db->doorbell = bd->bar1 + bd->db_offset;
1297
+
bnge_set_db_mask(bn, db, ring_type);
1298
+
}
1299
+
1300
+
static int bnge_hwrm_cp_ring_alloc(struct bnge_net *bn,
1301
+
struct bnge_cp_ring_info *cpr)
1302
+
{
1303
+
const u32 type = HWRM_RING_ALLOC_CMPL;
1304
+
struct bnge_napi *bnapi = cpr->bnapi;
1305
+
struct bnge_ring_struct *ring;
1306
+
u32 map_idx = bnapi->index;
1307
+
int rc;
1308
+
1309
+
ring = &cpr->ring_struct;
1310
+
ring->handle = BNGE_SET_NQ_HDL(cpr);
1311
+
rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx);
1312
+
if (rc)
1313
+
return rc;
1314
+
1315
+
bnge_set_db(bn, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
1316
+
bnge_db_cq(bn, &cpr->cp_db, cpr->cp_raw_cons);
1317
+
1318
+
return 0;
1319
+
}
1320
+
1321
+
static int bnge_hwrm_tx_ring_alloc(struct bnge_net *bn,
1322
+
struct bnge_tx_ring_info *txr, u32 tx_idx)
1323
+
{
1324
+
struct bnge_ring_struct *ring = &txr->tx_ring_struct;
1325
+
const u32 type = HWRM_RING_ALLOC_TX;
1326
+
int rc;
1327
+
1328
+
rc = hwrm_ring_alloc_send_msg(bn, ring, type, tx_idx);
1329
+
if (rc)
1330
+
return rc;
1331
+
1332
+
bnge_set_db(bn, &txr->tx_db, type, tx_idx, ring->fw_ring_id);
1333
+
1334
+
return 0;
1335
+
}
1336
+
1337
+
static int bnge_hwrm_rx_agg_ring_alloc(struct bnge_net *bn,
1338
+
struct bnge_rx_ring_info *rxr)
1339
+
{
1340
+
struct bnge_ring_struct *ring = &rxr->rx_agg_ring_struct;
1341
+
u32 type = HWRM_RING_ALLOC_AGG;
1342
+
struct bnge_dev *bd = bn->bd;
1343
+
u32 grp_idx = ring->grp_idx;
1344
+
u32 map_idx;
1345
+
int rc;
1346
+
1347
+
map_idx = grp_idx + bd->rx_nr_rings;
1348
+
rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx);
1349
+
if (rc)
1350
+
return rc;
1351
+
1352
+
bnge_set_db(bn, &rxr->rx_agg_db, type, map_idx,
1353
+
ring->fw_ring_id);
1354
+
bnge_db_write(bn->bd, &rxr->rx_agg_db, rxr->rx_agg_prod);
1355
+
bnge_db_write(bn->bd, &rxr->rx_db, rxr->rx_prod);
1356
+
bn->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
1357
+
1358
+
return 0;
1359
+
}
1360
+
1361
+
static int bnge_hwrm_rx_ring_alloc(struct bnge_net *bn,
1362
+
struct bnge_rx_ring_info *rxr)
1363
+
{
1364
+
struct bnge_ring_struct *ring = &rxr->rx_ring_struct;
1365
+
struct bnge_napi *bnapi = rxr->bnapi;
1366
+
u32 type = HWRM_RING_ALLOC_RX;
1367
+
u32 map_idx = bnapi->index;
1368
+
int rc;
1369
+
1370
+
rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx);
1371
+
if (rc)
1372
+
return rc;
1373
+
1374
+
bnge_set_db(bn, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
1375
+
bn->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
1376
+
1377
+
return 0;
1378
+
}
1379
+
1380
+
static int bnge_hwrm_ring_alloc(struct bnge_net *bn)
1381
+
{
1382
+
struct bnge_dev *bd = bn->bd;
1383
+
bool agg_rings;
1384
+
int i, rc = 0;
1385
+
1386
+
agg_rings = !!(bnge_is_agg_reqd(bd));
1387
+
for (i = 0; i < bd->nq_nr_rings; i++) {
1388
+
struct bnge_napi *bnapi = bn->bnapi[i];
1389
+
struct bnge_nq_ring_info *nqr = &bnapi->nq_ring;
1390
+
struct bnge_ring_struct *ring = &nqr->ring_struct;
1391
+
u32 type = HWRM_RING_ALLOC_NQ;
1392
+
u32 map_idx = ring->map_idx;
1393
+
unsigned int vector;
1394
+
1395
+
vector = bd->irq_tbl[map_idx].vector;
1396
+
disable_irq_nosync(vector);
1397
+
rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx);
1398
+
if (rc) {
1399
+
enable_irq(vector);
1400
+
goto err_out;
1401
+
}
1402
+
bnge_set_db(bn, &nqr->nq_db, type, map_idx, ring->fw_ring_id);
1403
+
bnge_db_nq(bn, &nqr->nq_db, nqr->nq_raw_cons);
1404
+
enable_irq(vector);
1405
+
bn->grp_info[i].nq_fw_ring_id = ring->fw_ring_id;
1406
+
1407
+
if (!i) {
1408
+
rc = bnge_hwrm_set_async_event_cr(bd, ring->fw_ring_id);
1409
+
if (rc)
1410
+
netdev_warn(bn->netdev, "Failed to set async event completion ring.\n");
1411
+
}
1412
+
}
1413
+
1414
+
for (i = 0; i < bd->tx_nr_rings; i++) {
1415
+
struct bnge_tx_ring_info *txr = &bn->tx_ring[i];
1416
+
1417
+
rc = bnge_hwrm_cp_ring_alloc(bn, txr->tx_cpr);
1418
+
if (rc)
1419
+
goto err_out;
1420
+
rc = bnge_hwrm_tx_ring_alloc(bn, txr, i);
1421
+
if (rc)
1422
+
goto err_out;
1423
+
}
1424
+
1425
+
for (i = 0; i < bd->rx_nr_rings; i++) {
1426
+
struct bnge_rx_ring_info *rxr = &bn->rx_ring[i];
1427
+
struct bnge_cp_ring_info *cpr;
1428
+
struct bnge_ring_struct *ring;
1429
+
struct bnge_napi *bnapi;
1430
+
u32 map_idx, type;
1431
+
1432
+
rc = bnge_hwrm_rx_ring_alloc(bn, rxr);
1433
+
if (rc)
1434
+
goto err_out;
1435
+
/* If we have agg rings, post agg buffers first. */
1436
+
if (!agg_rings)
1437
+
bnge_db_write(bn->bd, &rxr->rx_db, rxr->rx_prod);
1438
+
1439
+
cpr = rxr->rx_cpr;
1440
+
bnapi = rxr->bnapi;
1441
+
type = HWRM_RING_ALLOC_CMPL;
1442
+
map_idx = bnapi->index;
1443
+
1444
+
ring = &cpr->ring_struct;
1445
+
ring->handle = BNGE_SET_NQ_HDL(cpr);
1446
+
rc = hwrm_ring_alloc_send_msg(bn, ring, type, map_idx);
1447
+
if (rc)
1448
+
goto err_out;
1449
+
bnge_set_db(bn, &cpr->cp_db, type, map_idx,
1450
+
ring->fw_ring_id);
1451
+
bnge_db_cq(bn, &cpr->cp_db, cpr->cp_raw_cons);
1452
+
}
1453
+
1454
+
if (agg_rings) {
1455
+
for (i = 0; i < bd->rx_nr_rings; i++) {
1456
+
rc = bnge_hwrm_rx_agg_ring_alloc(bn, &bn->rx_ring[i]);
1457
+
if (rc)
1458
+
goto err_out;
1459
+
}
1460
+
}
1461
+
err_out:
1462
+
return rc;
1463
+
}
1464
+
1465
+
void bnge_fill_hw_rss_tbl(struct bnge_net *bn, struct bnge_vnic_info *vnic)
1466
+
{
1467
+
__le16 *ring_tbl = vnic->rss_table;
1468
+
struct bnge_rx_ring_info *rxr;
1469
+
struct bnge_dev *bd = bn->bd;
1470
+
u16 tbl_size, i;
1471
+
1472
+
tbl_size = bnge_get_rxfh_indir_size(bd);
1473
+
1474
+
for (i = 0; i < tbl_size; i++) {
1475
+
u16 ring_id, j;
1476
+
1477
+
j = bd->rss_indir_tbl[i];
1478
+
rxr = &bn->rx_ring[j];
1479
+
1480
+
ring_id = rxr->rx_ring_struct.fw_ring_id;
1481
+
*ring_tbl++ = cpu_to_le16(ring_id);
1482
+
ring_id = bnge_cp_ring_for_rx(rxr);
1483
+
*ring_tbl++ = cpu_to_le16(ring_id);
1484
+
}
1485
+
}
1486
+
1487
+
static int bnge_hwrm_vnic_rss_cfg(struct bnge_net *bn,
1488
+
struct bnge_vnic_info *vnic)
1489
+
{
1490
+
int rc;
1491
+
1492
+
rc = bnge_hwrm_vnic_set_rss(bn, vnic, true);
1493
+
if (rc) {
1494
+
netdev_err(bn->netdev, "hwrm vnic %d set rss failure rc: %d\n",
1495
+
vnic->vnic_id, rc);
1496
+
return rc;
1497
+
}
1498
+
rc = bnge_hwrm_vnic_cfg(bn, vnic);
1499
+
if (rc)
1500
+
netdev_err(bn->netdev, "hwrm vnic %d cfg failure rc: %d\n",
1501
+
vnic->vnic_id, rc);
1502
+
return rc;
1503
+
}
1504
+
1505
+
static int bnge_setup_vnic(struct bnge_net *bn, struct bnge_vnic_info *vnic)
1506
+
{
1507
+
struct bnge_dev *bd = bn->bd;
1508
+
int rc, i, nr_ctxs;
1509
+
1510
+
nr_ctxs = bnge_cal_nr_rss_ctxs(bd->rx_nr_rings);
1511
+
for (i = 0; i < nr_ctxs; i++) {
1512
+
rc = bnge_hwrm_vnic_ctx_alloc(bd, vnic, i);
1513
+
if (rc) {
1514
+
netdev_err(bn->netdev, "hwrm vnic %d ctx %d alloc failure rc: %d\n",
1515
+
vnic->vnic_id, i, rc);
1516
+
return -ENOMEM;
1517
+
}
1518
+
bn->rsscos_nr_ctxs++;
1519
+
}
1520
+
1521
+
rc = bnge_hwrm_vnic_rss_cfg(bn, vnic);
1522
+
if (rc)
1523
+
return rc;
1524
+
1525
+
if (bnge_is_agg_reqd(bd)) {
1526
+
rc = bnge_hwrm_vnic_set_hds(bn, vnic);
1527
+
if (rc)
1528
+
netdev_err(bn->netdev, "hwrm vnic %d set hds failure rc: %d\n",
1529
+
vnic->vnic_id, rc);
1530
+
}
1531
+
return rc;
1532
+
}
1533
+
1534
+
static void bnge_del_l2_filter(struct bnge_net *bn, struct bnge_l2_filter *fltr)
1535
+
{
1536
+
if (!refcount_dec_and_test(&fltr->refcnt))
1537
+
return;
1538
+
hlist_del_rcu(&fltr->base.hash);
1539
+
kfree_rcu(fltr, base.rcu);
1540
+
}
1541
+
1542
+
static void bnge_init_l2_filter(struct bnge_net *bn,
1543
+
struct bnge_l2_filter *fltr,
1544
+
struct bnge_l2_key *key, u32 idx)
1545
+
{
1546
+
struct hlist_head *head;
1547
+
1548
+
ether_addr_copy(fltr->l2_key.dst_mac_addr, key->dst_mac_addr);
1549
+
fltr->l2_key.vlan = key->vlan;
1550
+
fltr->base.type = BNGE_FLTR_TYPE_L2;
1551
+
1552
+
head = &bn->l2_fltr_hash_tbl[idx];
1553
+
hlist_add_head_rcu(&fltr->base.hash, head);
1554
+
refcount_set(&fltr->refcnt, 1);
1555
+
}
1556
+
1557
+
static struct bnge_l2_filter *__bnge_lookup_l2_filter(struct bnge_net *bn,
1558
+
struct bnge_l2_key *key,
1559
+
u32 idx)
1560
+
{
1561
+
struct bnge_l2_filter *fltr;
1562
+
struct hlist_head *head;
1563
+
1564
+
head = &bn->l2_fltr_hash_tbl[idx];
1565
+
hlist_for_each_entry_rcu(fltr, head, base.hash) {
1566
+
struct bnge_l2_key *l2_key = &fltr->l2_key;
1567
+
1568
+
if (ether_addr_equal(l2_key->dst_mac_addr, key->dst_mac_addr) &&
1569
+
l2_key->vlan == key->vlan)
1570
+
return fltr;
1571
+
}
1572
+
return NULL;
1573
+
}
1574
+
1575
+
static struct bnge_l2_filter *bnge_lookup_l2_filter(struct bnge_net *bn,
1576
+
struct bnge_l2_key *key,
1577
+
u32 idx)
1578
+
{
1579
+
struct bnge_l2_filter *fltr;
1580
+
1581
+
rcu_read_lock();
1582
+
fltr = __bnge_lookup_l2_filter(bn, key, idx);
1583
+
if (fltr)
1584
+
refcount_inc(&fltr->refcnt);
1585
+
rcu_read_unlock();
1586
+
return fltr;
1587
+
}
1588
+
1589
+
static struct bnge_l2_filter *bnge_alloc_l2_filter(struct bnge_net *bn,
1590
+
struct bnge_l2_key *key,
1591
+
gfp_t gfp)
1592
+
{
1593
+
struct bnge_l2_filter *fltr;
1594
+
u32 idx;
1595
+
1596
+
idx = jhash2(&key->filter_key, BNGE_L2_KEY_SIZE, bn->hash_seed) &
1597
+
BNGE_L2_FLTR_HASH_MASK;
1598
+
fltr = bnge_lookup_l2_filter(bn, key, idx);
1599
+
if (fltr)
1600
+
return fltr;
1601
+
1602
+
fltr = kzalloc(sizeof(*fltr), gfp);
1603
+
if (!fltr)
1604
+
return ERR_PTR(-ENOMEM);
1605
+
1606
+
bnge_init_l2_filter(bn, fltr, key, idx);
1607
+
return fltr;
1608
+
}
1609
+
1610
+
static int bnge_hwrm_set_vnic_filter(struct bnge_net *bn, u16 vnic_id, u16 idx,
1611
+
const u8 *mac_addr)
1612
+
{
1613
+
struct bnge_l2_filter *fltr;
1614
+
struct bnge_l2_key key;
1615
+
int rc;
1616
+
1617
+
ether_addr_copy(key.dst_mac_addr, mac_addr);
1618
+
key.vlan = 0;
1619
+
fltr = bnge_alloc_l2_filter(bn, &key, GFP_KERNEL);
1620
+
if (IS_ERR(fltr))
1621
+
return PTR_ERR(fltr);
1622
+
1623
+
fltr->base.fw_vnic_id = bn->vnic_info[vnic_id].fw_vnic_id;
1624
+
rc = bnge_hwrm_l2_filter_alloc(bn->bd, fltr);
1625
+
if (rc)
1626
+
goto err_del_l2_filter;
1627
+
bn->vnic_info[vnic_id].l2_filters[idx] = fltr;
1628
+
return rc;
1629
+
1630
+
err_del_l2_filter:
1631
+
bnge_del_l2_filter(bn, fltr);
1632
+
return rc;
1633
+
}
1634
+
1635
+
static bool bnge_mc_list_updated(struct bnge_net *bn, u32 *rx_mask)
1636
+
{
1637
+
struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT];
1638
+
struct net_device *dev = bn->netdev;
1639
+
struct netdev_hw_addr *ha;
1640
+
int mc_count = 0, off = 0;
1641
+
bool update = false;
1642
+
u8 *haddr;
1643
+
1644
+
netdev_for_each_mc_addr(ha, dev) {
1645
+
if (mc_count >= BNGE_MAX_MC_ADDRS) {
1646
+
*rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
1647
+
vnic->mc_list_count = 0;
1648
+
return false;
1649
+
}
1650
+
haddr = ha->addr;
1651
+
if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
1652
+
memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
1653
+
update = true;
1654
+
}
1655
+
off += ETH_ALEN;
1656
+
mc_count++;
1657
+
}
1658
+
if (mc_count)
1659
+
*rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
1660
+
1661
+
if (mc_count != vnic->mc_list_count) {
1662
+
vnic->mc_list_count = mc_count;
1663
+
update = true;
1664
+
}
1665
+
return update;
1666
+
}
1667
+
1668
+
static bool bnge_uc_list_updated(struct bnge_net *bn)
1669
+
{
1670
+
struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT];
1671
+
struct net_device *dev = bn->netdev;
1672
+
struct netdev_hw_addr *ha;
1673
+
int off = 0;
1674
+
1675
+
if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
1676
+
return true;
1677
+
1678
+
netdev_for_each_uc_addr(ha, dev) {
1679
+
if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
1680
+
return true;
1681
+
1682
+
off += ETH_ALEN;
1683
+
}
1684
+
return false;
1685
+
}
1686
+
1687
+
static bool bnge_promisc_ok(struct bnge_net *bn)
1688
+
{
1689
+
return true;
1690
+
}
1691
+
1692
+
static int bnge_cfg_def_vnic(struct bnge_net *bn)
1693
+
{
1694
+
struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT];
1695
+
struct net_device *dev = bn->netdev;
1696
+
struct bnge_dev *bd = bn->bd;
1697
+
struct netdev_hw_addr *ha;
1698
+
int i, off = 0, rc;
1699
+
bool uc_update;
1700
+
1701
+
netif_addr_lock_bh(dev);
1702
+
uc_update = bnge_uc_list_updated(bn);
1703
+
netif_addr_unlock_bh(dev);
1704
+
1705
+
if (!uc_update)
1706
+
goto skip_uc;
1707
+
1708
+
for (i = 1; i < vnic->uc_filter_count; i++) {
1709
+
struct bnge_l2_filter *fltr = vnic->l2_filters[i];
1710
+
1711
+
bnge_hwrm_l2_filter_free(bd, fltr);
1712
+
bnge_del_l2_filter(bn, fltr);
1713
+
}
1714
+
1715
+
vnic->uc_filter_count = 1;
1716
+
1717
+
netif_addr_lock_bh(dev);
1718
+
if (netdev_uc_count(dev) > (BNGE_MAX_UC_ADDRS - 1)) {
1719
+
vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
1720
+
} else {
1721
+
netdev_for_each_uc_addr(ha, dev) {
1722
+
memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
1723
+
off += ETH_ALEN;
1724
+
vnic->uc_filter_count++;
1725
+
}
1726
+
}
1727
+
netif_addr_unlock_bh(dev);
1728
+
1729
+
for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
1730
+
rc = bnge_hwrm_set_vnic_filter(bn, 0, i, vnic->uc_list + off);
1731
+
if (rc) {
1732
+
netdev_err(dev, "HWRM vnic filter failure rc: %d\n", rc);
1733
+
vnic->uc_filter_count = i;
1734
+
return rc;
1735
+
}
1736
+
}
1737
+
1738
+
skip_uc:
1739
+
if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) &&
1740
+
!bnge_promisc_ok(bn))
1741
+
vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
1742
+
rc = bnge_hwrm_cfa_l2_set_rx_mask(bd, vnic);
1743
+
if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) {
1744
+
netdev_info(dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
1745
+
rc);
1746
+
vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
1747
+
vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
1748
+
vnic->mc_list_count = 0;
1749
+
rc = bnge_hwrm_cfa_l2_set_rx_mask(bd, vnic);
1750
+
}
1751
+
if (rc)
1752
+
netdev_err(dev, "HWRM cfa l2 rx mask failure rc: %d\n",
1753
+
rc);
1754
+
1755
+
return rc;
1756
+
}
1757
+
1758
+
static void bnge_hwrm_vnic_free(struct bnge_net *bn)
1759
+
{
1760
+
int i;
1761
+
1762
+
for (i = 0; i < bn->nr_vnics; i++)
1763
+
bnge_hwrm_vnic_free_one(bn->bd, &bn->vnic_info[i]);
1764
+
}
1765
+
1766
+
static void bnge_hwrm_vnic_ctx_free(struct bnge_net *bn)
1767
+
{
1768
+
int i, j;
1769
+
1770
+
for (i = 0; i < bn->nr_vnics; i++) {
1771
+
struct bnge_vnic_info *vnic = &bn->vnic_info[i];
1772
+
1773
+
for (j = 0; j < BNGE_MAX_CTX_PER_VNIC; j++) {
1774
+
if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
1775
+
bnge_hwrm_vnic_ctx_free_one(bn->bd, vnic, j);
1776
+
}
1777
+
}
1778
+
bn->rsscos_nr_ctxs = 0;
1779
+
}
1780
+
1781
+
static void bnge_hwrm_clear_vnic_filter(struct bnge_net *bn)
1782
+
{
1783
+
struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT];
1784
+
int i;
1785
+
1786
+
for (i = 0; i < vnic->uc_filter_count; i++) {
1787
+
struct bnge_l2_filter *fltr = vnic->l2_filters[i];
1788
+
1789
+
bnge_hwrm_l2_filter_free(bn->bd, fltr);
1790
+
bnge_del_l2_filter(bn, fltr);
1791
+
}
1792
+
1793
+
vnic->uc_filter_count = 0;
1794
+
}
1795
+
1796
+
static void bnge_clear_vnic(struct bnge_net *bn)
1797
+
{
1798
+
bnge_hwrm_clear_vnic_filter(bn);
1799
+
bnge_hwrm_vnic_free(bn);
1800
+
bnge_hwrm_vnic_ctx_free(bn);
1801
+
}
1802
+
1803
+
static void bnge_hwrm_rx_ring_free(struct bnge_net *bn,
1804
+
struct bnge_rx_ring_info *rxr,
1805
+
bool close_path)
1806
+
{
1807
+
struct bnge_ring_struct *ring = &rxr->rx_ring_struct;
1808
+
u32 grp_idx = rxr->bnapi->index;
1809
+
u32 cmpl_ring_id;
1810
+
1811
+
if (ring->fw_ring_id == INVALID_HW_RING_ID)
1812
+
return;
1813
+
1814
+
cmpl_ring_id = bnge_cp_ring_for_rx(rxr);
1815
+
hwrm_ring_free_send_msg(bn, ring,
1816
+
RING_FREE_REQ_RING_TYPE_RX,
1817
+
close_path ? cmpl_ring_id :
1818
+
INVALID_HW_RING_ID);
1819
+
ring->fw_ring_id = INVALID_HW_RING_ID;
1820
+
bn->grp_info[grp_idx].rx_fw_ring_id = INVALID_HW_RING_ID;
1821
+
}
1822
+
1823
+
static void bnge_hwrm_rx_agg_ring_free(struct bnge_net *bn,
1824
+
struct bnge_rx_ring_info *rxr,
1825
+
bool close_path)
1826
+
{
1827
+
struct bnge_ring_struct *ring = &rxr->rx_agg_ring_struct;
1828
+
u32 grp_idx = rxr->bnapi->index;
1829
+
u32 cmpl_ring_id;
1830
+
1831
+
if (ring->fw_ring_id == INVALID_HW_RING_ID)
1832
+
return;
1833
+
1834
+
cmpl_ring_id = bnge_cp_ring_for_rx(rxr);
1835
+
hwrm_ring_free_send_msg(bn, ring, RING_FREE_REQ_RING_TYPE_RX_AGG,
1836
+
close_path ? cmpl_ring_id :
1837
+
INVALID_HW_RING_ID);
1838
+
ring->fw_ring_id = INVALID_HW_RING_ID;
1839
+
bn->grp_info[grp_idx].agg_fw_ring_id = INVALID_HW_RING_ID;
1840
+
}
1841
+
1842
+
static void bnge_hwrm_tx_ring_free(struct bnge_net *bn,
1843
+
struct bnge_tx_ring_info *txr,
1844
+
bool close_path)
1845
+
{
1846
+
struct bnge_ring_struct *ring = &txr->tx_ring_struct;
1847
+
u32 cmpl_ring_id;
1848
+
1849
+
if (ring->fw_ring_id == INVALID_HW_RING_ID)
1850
+
return;
1851
+
1852
+
cmpl_ring_id = close_path ? bnge_cp_ring_for_tx(txr) :
1853
+
INVALID_HW_RING_ID;
1854
+
hwrm_ring_free_send_msg(bn, ring, RING_FREE_REQ_RING_TYPE_TX,
1855
+
cmpl_ring_id);
1856
+
ring->fw_ring_id = INVALID_HW_RING_ID;
1857
+
}
1858
+
1859
+
static void bnge_hwrm_cp_ring_free(struct bnge_net *bn,
1860
+
struct bnge_cp_ring_info *cpr)
1861
+
{
1862
+
struct bnge_ring_struct *ring;
1863
+
1864
+
ring = &cpr->ring_struct;
1865
+
if (ring->fw_ring_id == INVALID_HW_RING_ID)
1866
+
return;
1867
+
1868
+
hwrm_ring_free_send_msg(bn, ring, RING_FREE_REQ_RING_TYPE_L2_CMPL,
1869
+
INVALID_HW_RING_ID);
1870
+
ring->fw_ring_id = INVALID_HW_RING_ID;
1871
+
}
1872
+
1873
+
static void bnge_hwrm_ring_free(struct bnge_net *bn, bool close_path)
1874
+
{
1875
+
struct bnge_dev *bd = bn->bd;
1876
+
int i;
1877
+
1878
+
if (!bn->bnapi)
1879
+
return;
1880
+
1881
+
for (i = 0; i < bd->tx_nr_rings; i++)
1882
+
bnge_hwrm_tx_ring_free(bn, &bn->tx_ring[i], close_path);
1883
+
1884
+
for (i = 0; i < bd->rx_nr_rings; i++) {
1885
+
bnge_hwrm_rx_ring_free(bn, &bn->rx_ring[i], close_path);
1886
+
bnge_hwrm_rx_agg_ring_free(bn, &bn->rx_ring[i], close_path);
1887
+
}
1888
+
1889
+
for (i = 0; i < bd->nq_nr_rings; i++) {
1890
+
struct bnge_napi *bnapi = bn->bnapi[i];
1891
+
struct bnge_nq_ring_info *nqr;
1892
+
struct bnge_ring_struct *ring;
1893
+
int j;
1894
+
1895
+
nqr = &bnapi->nq_ring;
1896
+
for (j = 0; j < nqr->cp_ring_count && nqr->cp_ring_arr; j++)
1897
+
bnge_hwrm_cp_ring_free(bn, &nqr->cp_ring_arr[j]);
1898
+
1899
+
ring = &nqr->ring_struct;
1900
+
if (ring->fw_ring_id != INVALID_HW_RING_ID) {
1901
+
hwrm_ring_free_send_msg(bn, ring,
1902
+
RING_FREE_REQ_RING_TYPE_NQ,
1903
+
INVALID_HW_RING_ID);
1904
+
ring->fw_ring_id = INVALID_HW_RING_ID;
1905
+
bn->grp_info[i].nq_fw_ring_id = INVALID_HW_RING_ID;
1906
+
}
1907
+
}
1908
+
}
1909
+
1910
+
static void bnge_setup_msix(struct bnge_net *bn)
1911
+
{
1912
+
struct net_device *dev = bn->netdev;
1913
+
struct bnge_dev *bd = bn->bd;
1914
+
int len, i;
1915
+
1916
+
len = sizeof(bd->irq_tbl[0].name);
1917
+
for (i = 0; i < bd->nq_nr_rings; i++) {
1918
+
int map_idx = bnge_cp_num_to_irq_num(bn, i);
1919
+
char *attr;
1920
+
1921
+
if (bd->flags & BNGE_EN_SHARED_CHNL)
1922
+
attr = "TxRx";
1923
+
else if (i < bd->rx_nr_rings)
1924
+
attr = "rx";
1925
+
else
1926
+
attr = "tx";
1927
+
1928
+
snprintf(bd->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
1929
+
attr, i);
1930
+
bd->irq_tbl[map_idx].handler = bnge_msix;
1931
+
}
1932
+
}
1933
+
1934
+
static int bnge_setup_interrupts(struct bnge_net *bn)
1935
+
{
1936
+
struct net_device *dev = bn->netdev;
1937
+
struct bnge_dev *bd = bn->bd;
1938
+
1939
+
bnge_setup_msix(bn);
1940
+
1941
+
return netif_set_real_num_queues(dev, bd->tx_nr_rings, bd->rx_nr_rings);
1942
+
}
1943
+
1944
+
static void bnge_hwrm_resource_free(struct bnge_net *bn, bool close_path)
1945
+
{
1946
+
bnge_clear_vnic(bn);
1947
+
bnge_hwrm_ring_free(bn, close_path);
1948
+
bnge_hwrm_stat_ctx_free(bn);
1949
+
}
1950
+
1951
+
static void bnge_free_irq(struct bnge_net *bn)
1952
+
{
1953
+
struct bnge_dev *bd = bn->bd;
1954
+
struct bnge_irq *irq;
1955
+
int i;
1956
+
1957
+
for (i = 0; i < bd->nq_nr_rings; i++) {
1958
+
int map_idx = bnge_cp_num_to_irq_num(bn, i);
1959
+
1960
+
irq = &bd->irq_tbl[map_idx];
1961
+
if (irq->requested) {
1962
+
if (irq->have_cpumask) {
1963
+
irq_set_affinity_hint(irq->vector, NULL);
1964
+
free_cpumask_var(irq->cpu_mask);
1965
+
irq->have_cpumask = 0;
1966
+
}
1967
+
free_irq(irq->vector, bn->bnapi[i]);
1968
+
}
1969
+
1970
+
irq->requested = 0;
1971
+
}
1972
+
}
1973
+
1974
+
static int bnge_request_irq(struct bnge_net *bn)
1975
+
{
1976
+
struct bnge_dev *bd = bn->bd;
1977
+
int i, rc;
1978
+
1979
+
rc = bnge_setup_interrupts(bn);
1980
+
if (rc) {
1981
+
netdev_err(bn->netdev, "bnge_setup_interrupts err: %d\n", rc);
1982
+
return rc;
1983
+
}
1984
+
for (i = 0; i < bd->nq_nr_rings; i++) {
1985
+
int map_idx = bnge_cp_num_to_irq_num(bn, i);
1986
+
struct bnge_irq *irq = &bd->irq_tbl[map_idx];
1987
+
1988
+
rc = request_irq(irq->vector, irq->handler, 0, irq->name,
1989
+
bn->bnapi[i]);
1990
+
if (rc)
1991
+
goto err_free_irq;
1992
+
1993
+
netif_napi_set_irq_locked(&bn->bnapi[i]->napi, irq->vector);
1994
+
irq->requested = 1;
1995
+
1996
+
if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
1997
+
int numa_node = dev_to_node(&bd->pdev->dev);
1998
+
1999
+
irq->have_cpumask = 1;
2000
+
cpumask_set_cpu(cpumask_local_spread(i, numa_node),
2001
+
irq->cpu_mask);
2002
+
rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
2003
+
if (rc) {
2004
+
netdev_warn(bn->netdev,
2005
+
"Set affinity failed, IRQ = %d\n",
2006
+
irq->vector);
2007
+
goto err_free_irq;
2008
+
}
2009
+
}
2010
+
}
2011
+
return 0;
2012
+
2013
+
err_free_irq:
2014
+
bnge_free_irq(bn);
2015
+
return rc;
2016
+
}
2017
+
2018
+
static int bnge_init_chip(struct bnge_net *bn)
2019
+
{
2020
+
struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT];
2021
+
struct bnge_dev *bd = bn->bd;
2022
+
int rc;
2023
+
2024
+
#define BNGE_DEF_STATS_COAL_TICKS 1000000
2025
+
bn->stats_coal_ticks = BNGE_DEF_STATS_COAL_TICKS;
2026
+
2027
+
rc = bnge_hwrm_stat_ctx_alloc(bn);
2028
+
if (rc) {
2029
+
netdev_err(bn->netdev, "hwrm stat ctx alloc failure rc: %d\n", rc);
2030
+
goto err_out;
2031
+
}
2032
+
2033
+
rc = bnge_hwrm_ring_alloc(bn);
2034
+
if (rc) {
2035
+
netdev_err(bn->netdev, "hwrm ring alloc failure rc: %d\n", rc);
2036
+
goto err_out;
2037
+
}
2038
+
2039
+
rc = bnge_hwrm_vnic_alloc(bd, vnic, bd->rx_nr_rings);
2040
+
if (rc) {
2041
+
netdev_err(bn->netdev, "hwrm vnic alloc failure rc: %d\n", rc);
2042
+
goto err_out;
2043
+
}
2044
+
2045
+
rc = bnge_setup_vnic(bn, vnic);
2046
+
if (rc)
2047
+
goto err_out;
2048
+
2049
+
if (bd->rss_cap & BNGE_RSS_CAP_RSS_HASH_TYPE_DELTA)
2050
+
bnge_hwrm_update_rss_hash_cfg(bn);
2051
+
2052
+
/* Filter for default vnic 0 */
2053
+
rc = bnge_hwrm_set_vnic_filter(bn, 0, 0, bn->netdev->dev_addr);
2054
+
if (rc) {
2055
+
netdev_err(bn->netdev, "HWRM vnic filter failure rc: %d\n", rc);
2056
+
goto err_out;
2057
+
}
2058
+
vnic->uc_filter_count = 1;
2059
+
2060
+
vnic->rx_mask = 0;
2061
+
2062
+
if (bn->netdev->flags & IFF_BROADCAST)
2063
+
vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
2064
+
2065
+
if (bn->netdev->flags & IFF_PROMISC)
2066
+
vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
2067
+
2068
+
if (bn->netdev->flags & IFF_ALLMULTI) {
2069
+
vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
2070
+
vnic->mc_list_count = 0;
2071
+
} else if (bn->netdev->flags & IFF_MULTICAST) {
2072
+
u32 mask = 0;
2073
+
2074
+
bnge_mc_list_updated(bn, &mask);
2075
+
vnic->rx_mask |= mask;
2076
+
}
2077
+
2078
+
rc = bnge_cfg_def_vnic(bn);
2079
+
if (rc)
2080
+
goto err_out;
2081
+
return 0;
2082
+
2083
+
err_out:
2084
+
bnge_hwrm_resource_free(bn, 0);
2085
+
return rc;
2086
+
}
2087
+
2088
+
static int bnge_napi_poll(struct napi_struct *napi, int budget)
2089
+
{
2090
+
int work_done = 0;
2091
+
2092
+
/* defer NAPI implementation to next patch series */
2093
+
napi_complete_done(napi, work_done);
2094
+
2095
+
return work_done;
2096
+
}
2097
+
2098
+
static void bnge_init_napi(struct bnge_net *bn)
2099
+
{
2100
+
struct bnge_dev *bd = bn->bd;
2101
+
struct bnge_napi *bnapi;
2102
+
int i;
2103
+
2104
+
for (i = 0; i < bd->nq_nr_rings; i++) {
2105
+
bnapi = bn->bnapi[i];
2106
+
netif_napi_add_config_locked(bn->netdev, &bnapi->napi,
2107
+
bnge_napi_poll, bnapi->index);
2108
+
}
2109
+
}
2110
+
2111
+
static void bnge_del_napi(struct bnge_net *bn)
2112
+
{
2113
+
struct bnge_dev *bd = bn->bd;
2114
+
int i;
2115
+
2116
+
for (i = 0; i < bd->rx_nr_rings; i++)
2117
+
netif_queue_set_napi(bn->netdev, i, NETDEV_QUEUE_TYPE_RX, NULL);
2118
+
for (i = 0; i < bd->tx_nr_rings; i++)
2119
+
netif_queue_set_napi(bn->netdev, i, NETDEV_QUEUE_TYPE_TX, NULL);
2120
+
2121
+
for (i = 0; i < bd->nq_nr_rings; i++) {
2122
+
struct bnge_napi *bnapi = bn->bnapi[i];
2123
+
2124
+
__netif_napi_del_locked(&bnapi->napi);
2125
+
}
2126
+
2127
+
/* Wait for RCU grace period after removing NAPI instances */
2128
+
synchronize_net();
2129
+
}
2130
+
2131
+
static int bnge_init_nic(struct bnge_net *bn)
2132
+
{
2133
+
int rc;
2134
+
2135
+
bnge_init_nq_tree(bn);
2136
+
2137
+
bnge_init_rx_rings(bn);
2138
+
rc = bnge_alloc_rx_ring_pair_bufs(bn);
2139
+
if (rc)
2140
+
return rc;
2141
+
2142
+
bnge_init_tx_rings(bn);
2143
+
2144
+
rc = bnge_init_ring_grps(bn);
2145
+
if (rc)
2146
+
goto err_free_rx_ring_pair_bufs;
2147
+
2148
+
bnge_init_vnics(bn);
2149
+
2150
+
rc = bnge_init_chip(bn);
2151
+
if (rc)
2152
+
goto err_free_ring_grps;
2153
+
return rc;
2154
+
2155
+
err_free_ring_grps:
2156
+
bnge_free_ring_grps(bn);
2157
+
return rc;
2158
+
2159
+
err_free_rx_ring_pair_bufs:
2160
+
bnge_free_rx_ring_pair_bufs(bn);
2161
+
return rc;
2162
+
}
2163
+
2164
+
static int bnge_open_core(struct bnge_net *bn)
2165
+
{
2166
+
struct bnge_dev *bd = bn->bd;
2167
+
int rc;
2168
+
2169
+
netif_carrier_off(bn->netdev);
2170
+
2171
+
rc = bnge_reserve_rings(bd);
2172
+
if (rc) {
2173
+
netdev_err(bn->netdev, "bnge_reserve_rings err: %d\n", rc);
2174
+
return rc;
2175
+
}
2176
+
2177
+
rc = bnge_alloc_core(bn);
2178
+
if (rc) {
2179
+
netdev_err(bn->netdev, "bnge_alloc_core err: %d\n", rc);
2180
+
return rc;
2181
+
}
2182
+
2183
+
bnge_init_napi(bn);
2184
+
rc = bnge_request_irq(bn);
2185
+
if (rc) {
2186
+
netdev_err(bn->netdev, "bnge_request_irq err: %d\n", rc);
2187
+
goto err_del_napi;
2188
+
}
2189
+
2190
+
rc = bnge_init_nic(bn);
2191
+
if (rc) {
2192
+
netdev_err(bn->netdev, "bnge_init_nic err: %d\n", rc);
2193
+
goto err_free_irq;
2194
+
}
2195
+
set_bit(BNGE_STATE_OPEN, &bd->state);
2196
+
return 0;
2197
+
2198
+
err_free_irq:
2199
+
bnge_free_irq(bn);
2200
+
err_del_napi:
2201
+
bnge_del_napi(bn);
2202
+
bnge_free_core(bn);
2203
+
return rc;
2204
+
}
21
2205
22
2206
static netdev_tx_t bnge_start_xmit(struct sk_buff *skb, struct net_device *dev)
23
2207
{
···
2212
28
2213
29
static int bnge_open(struct net_device *dev)
2214
30
{
31
+
struct bnge_net *bn = netdev_priv(dev);
32
+
int rc;
33
+
34
+
rc = bnge_open_core(bn);
35
+
if (rc)
36
+
netdev_err(dev, "bnge_open_core err: %d\n", rc);
37
+
38
+
return rc;
39
+
}
40
+
41
+
static int bnge_shutdown_nic(struct bnge_net *bn)
42
+
{
43
+
/* TODO: close_path = 0 until we make NAPI functional */
44
+
bnge_hwrm_resource_free(bn, 0);
2215
45
return 0;
46
+
}
47
+
48
+
static void bnge_close_core(struct bnge_net *bn)
49
+
{
50
+
struct bnge_dev *bd = bn->bd;
51
+
52
+
clear_bit(BNGE_STATE_OPEN, &bd->state);
53
+
bnge_shutdown_nic(bn);
54
+
bnge_free_all_rings_bufs(bn);
55
+
bnge_free_irq(bn);
56
+
bnge_del_napi(bn);
57
+
58
+
bnge_free_core(bn);
2216
59
}
2217
60
2218
61
static int bnge_close(struct net_device *dev)
2219
62
{
63
+
struct bnge_net *bn = netdev_priv(dev);
64
+
65
+
bnge_close_core(bn);
66
+
2220
67
return 0;
2221
68
}
2222
69
···
2453
238
2454
239
bn->rx_ring_size = BNGE_DEFAULT_RX_RING_SIZE;
2455
240
bn->tx_ring_size = BNGE_DEFAULT_TX_RING_SIZE;
241
+
bn->rx_dir = DMA_FROM_DEVICE;
2456
242
2457
243
bnge_set_tpa_flags(bd);
2458
244
bnge_set_ring_params(bd);
···
2461
245
bnge_init_l2_fltr_tbl(bn);
2462
246
bnge_init_mac_addr(bd);
2463
247
248
+
netdev->request_ops_lock = true;
2464
249
rc = register_netdev(netdev);
2465
250
if (rc) {
2466
251
dev_err(bd->dev, "Register netdev failed rc: %d\n", rc);
+249
-1
drivers/net/ethernet/broadcom/bnge/bnge_netdev.h
+249
-1
drivers/net/ethernet/broadcom/bnge/bnge_netdev.h
···
5
5
#define _BNGE_NETDEV_H_
6
6
7
7
#include <linux/bnxt/hsi.h>
8
+
#include <linux/io-64-nonatomic-lo-hi.h>
9
+
#include <linux/refcount.h>
10
+
#include "bnge_db.h"
8
11
9
12
struct tx_bd {
10
13
__le32 tx_bd_len_flags_type;
···
116
113
};
117
114
118
115
struct bnge_sw_rx_agg_bd {
119
-
struct page *page;
116
+
netmem_ref netmem;
120
117
unsigned int offset;
121
118
dma_addr_t mapping;
119
+
};
120
+
121
+
#define HWRM_RING_ALLOC_TX 0x1
122
+
#define HWRM_RING_ALLOC_RX 0x2
123
+
#define HWRM_RING_ALLOC_AGG 0x4
124
+
#define HWRM_RING_ALLOC_CMPL 0x8
125
+
#define HWRM_RING_ALLOC_NQ 0x10
126
+
127
+
struct bnge_ring_grp_info {
128
+
u16 fw_stats_ctx;
129
+
u16 fw_grp_id;
130
+
u16 rx_fw_ring_id;
131
+
u16 agg_fw_ring_id;
132
+
u16 nq_fw_ring_id;
122
133
};
123
134
124
135
#define BNGE_RX_COPY_THRESH 256
···
149
132
};
150
133
151
134
#define BNGE_NET_EN_TPA (BNGE_NET_EN_GRO | BNGE_NET_EN_LRO)
135
+
136
+
/* Minimum TX BDs for a TX packet with MAX_SKB_FRAGS + 1. We need one extra
137
+
* BD because the first TX BD is always a long BD.
138
+
*/
139
+
#define BNGE_MIN_TX_DESC_CNT (MAX_SKB_FRAGS + 2)
140
+
141
+
#define RX_RING(bn, x) (((x) & (bn)->rx_ring_mask) >> (BNGE_PAGE_SHIFT - 4))
142
+
#define RX_AGG_RING(bn, x) (((x) & (bn)->rx_agg_ring_mask) >> \
143
+
(BNGE_PAGE_SHIFT - 4))
144
+
#define RX_IDX(x) ((x) & (RX_DESC_CNT - 1))
145
+
146
+
#define TX_RING(bn, x) (((x) & (bn)->tx_ring_mask) >> (BNGE_PAGE_SHIFT - 4))
147
+
#define TX_IDX(x) ((x) & (TX_DESC_CNT - 1))
148
+
149
+
#define CP_RING(x) (((x) & ~(CP_DESC_CNT - 1)) >> (BNGE_PAGE_SHIFT - 4))
150
+
#define CP_IDX(x) ((x) & (CP_DESC_CNT - 1))
151
+
152
+
#define RING_RX(bn, idx) ((idx) & (bn)->rx_ring_mask)
153
+
#define NEXT_RX(idx) ((idx) + 1)
154
+
155
+
#define RING_RX_AGG(bn, idx) ((idx) & (bn)->rx_agg_ring_mask)
156
+
#define NEXT_RX_AGG(idx) ((idx) + 1)
157
+
158
+
#define BNGE_NQ_HDL_TYPE_SHIFT 24
159
+
#define BNGE_NQ_HDL_TYPE_RX 0x00
160
+
#define BNGE_NQ_HDL_TYPE_TX 0x01
152
161
153
162
struct bnge_net {
154
163
struct bnge_dev *bd;
···
207
164
struct hlist_head l2_fltr_hash_tbl[BNGE_L2_FLTR_HASH_SIZE];
208
165
u32 hash_seed;
209
166
u64 toeplitz_prefix;
167
+
168
+
struct bnge_napi **bnapi;
169
+
170
+
struct bnge_rx_ring_info *rx_ring;
171
+
struct bnge_tx_ring_info *tx_ring;
172
+
173
+
u16 *tx_ring_map;
174
+
enum dma_data_direction rx_dir;
175
+
176
+
/* grp_info indexed by napi/nq index */
177
+
struct bnge_ring_grp_info *grp_info;
178
+
struct bnge_vnic_info *vnic_info;
179
+
int nr_vnics;
180
+
int total_irqs;
181
+
182
+
u32 tx_wake_thresh;
183
+
u16 rx_offset;
184
+
u16 rx_dma_offset;
185
+
186
+
u8 rss_hash_key[HW_HASH_KEY_SIZE];
187
+
u8 rss_hash_key_valid:1;
188
+
u8 rss_hash_key_updated:1;
189
+
int rsscos_nr_ctxs;
190
+
u32 stats_coal_ticks;
210
191
};
211
192
212
193
#define BNGE_DEFAULT_RX_RING_SIZE 511
···
270
203
#define BNGE_MAX_RX_JUM_DESC_CNT (RX_DESC_CNT * MAX_RX_AGG_PAGES - 1)
271
204
#define BNGE_MAX_TX_DESC_CNT (TX_DESC_CNT * MAX_TX_PAGES - 1)
272
205
206
+
#define BNGE_MAX_TXR_PER_NAPI 8
207
+
208
+
#define bnge_for_each_napi_tx(iter, bnapi, txr) \
209
+
for (iter = 0, txr = (bnapi)->tx_ring[0]; txr; \
210
+
txr = (iter < BNGE_MAX_TXR_PER_NAPI - 1) ? \
211
+
(bnapi)->tx_ring[++iter] : NULL)
212
+
213
+
#define BNGE_SET_NQ_HDL(cpr) \
214
+
(((cpr)->cp_ring_type << BNGE_NQ_HDL_TYPE_SHIFT) | (cpr)->cp_idx)
215
+
216
+
struct bnge_stats_mem {
217
+
u64 *sw_stats;
218
+
u64 *hw_masks;
219
+
void *hw_stats;
220
+
dma_addr_t hw_stats_map;
221
+
int len;
222
+
};
223
+
224
+
struct bnge_cp_ring_info {
225
+
struct bnge_napi *bnapi;
226
+
dma_addr_t *desc_mapping;
227
+
struct tx_cmp **desc_ring;
228
+
struct bnge_ring_struct ring_struct;
229
+
u8 cp_ring_type;
230
+
u8 cp_idx;
231
+
u32 cp_raw_cons;
232
+
struct bnge_db_info cp_db;
233
+
};
234
+
235
+
struct bnge_nq_ring_info {
236
+
struct bnge_napi *bnapi;
237
+
dma_addr_t *desc_mapping;
238
+
struct nqe_cn **desc_ring;
239
+
struct bnge_ring_struct ring_struct;
240
+
u32 nq_raw_cons;
241
+
struct bnge_db_info nq_db;
242
+
243
+
struct bnge_stats_mem stats;
244
+
u32 hw_stats_ctx_id;
245
+
246
+
int cp_ring_count;
247
+
struct bnge_cp_ring_info *cp_ring_arr;
248
+
};
249
+
250
+
struct bnge_rx_ring_info {
251
+
struct bnge_napi *bnapi;
252
+
struct bnge_cp_ring_info *rx_cpr;
253
+
u16 rx_prod;
254
+
u16 rx_agg_prod;
255
+
u16 rx_sw_agg_prod;
256
+
u16 rx_next_cons;
257
+
struct bnge_db_info rx_db;
258
+
struct bnge_db_info rx_agg_db;
259
+
260
+
struct rx_bd *rx_desc_ring[MAX_RX_PAGES];
261
+
struct bnge_sw_rx_bd *rx_buf_ring;
262
+
263
+
struct rx_bd *rx_agg_desc_ring[MAX_RX_AGG_PAGES];
264
+
struct bnge_sw_rx_agg_bd *rx_agg_buf_ring;
265
+
266
+
unsigned long *rx_agg_bmap;
267
+
u16 rx_agg_bmap_size;
268
+
269
+
dma_addr_t rx_desc_mapping[MAX_RX_PAGES];
270
+
dma_addr_t rx_agg_desc_mapping[MAX_RX_AGG_PAGES];
271
+
272
+
struct bnge_ring_struct rx_ring_struct;
273
+
struct bnge_ring_struct rx_agg_ring_struct;
274
+
struct page_pool *page_pool;
275
+
struct page_pool *head_pool;
276
+
bool need_head_pool;
277
+
};
278
+
279
+
struct bnge_tx_ring_info {
280
+
struct bnge_napi *bnapi;
281
+
struct bnge_cp_ring_info *tx_cpr;
282
+
u16 tx_prod;
283
+
u16 tx_cons;
284
+
u16 tx_hw_cons;
285
+
u16 txq_index;
286
+
u8 tx_napi_idx;
287
+
u8 kick_pending;
288
+
struct bnge_db_info tx_db;
289
+
290
+
struct tx_bd *tx_desc_ring[MAX_TX_PAGES];
291
+
struct bnge_sw_tx_bd *tx_buf_ring;
292
+
293
+
dma_addr_t tx_desc_mapping[MAX_TX_PAGES];
294
+
295
+
u32 dev_state;
296
+
#define BNGE_DEV_STATE_CLOSING 0x1
297
+
298
+
struct bnge_ring_struct tx_ring_struct;
299
+
};
300
+
301
+
struct bnge_napi {
302
+
struct napi_struct napi;
303
+
struct bnge_net *bn;
304
+
int index;
305
+
306
+
struct bnge_nq_ring_info nq_ring;
307
+
struct bnge_rx_ring_info *rx_ring;
308
+
struct bnge_tx_ring_info *tx_ring[BNGE_MAX_TXR_PER_NAPI];
309
+
};
310
+
311
+
#define INVALID_STATS_CTX_ID -1
312
+
#define BNGE_VNIC_DEFAULT 0
313
+
#define BNGE_MAX_UC_ADDRS 4
314
+
315
+
struct bnge_vnic_info {
316
+
u16 fw_vnic_id;
317
+
#define BNGE_MAX_CTX_PER_VNIC 8
318
+
u16 fw_rss_cos_lb_ctx[BNGE_MAX_CTX_PER_VNIC];
319
+
u16 mru;
320
+
/* index 0 always dev_addr */
321
+
struct bnge_l2_filter *l2_filters[BNGE_MAX_UC_ADDRS];
322
+
u16 uc_filter_count;
323
+
u8 *uc_list;
324
+
dma_addr_t rss_table_dma_addr;
325
+
__le16 *rss_table;
326
+
dma_addr_t rss_hash_key_dma_addr;
327
+
u64 *rss_hash_key;
328
+
int rss_table_size;
329
+
#define BNGE_RSS_TABLE_ENTRIES 64
330
+
#define BNGE_RSS_TABLE_SIZE (BNGE_RSS_TABLE_ENTRIES * 4)
331
+
#define BNGE_RSS_TABLE_MAX_TBL 8
332
+
#define BNGE_MAX_RSS_TABLE_SIZE \
333
+
(BNGE_RSS_TABLE_SIZE * BNGE_RSS_TABLE_MAX_TBL)
334
+
u32 rx_mask;
335
+
336
+
u8 *mc_list;
337
+
int mc_list_size;
338
+
int mc_list_count;
339
+
dma_addr_t mc_list_mapping;
340
+
#define BNGE_MAX_MC_ADDRS 16
341
+
342
+
u32 flags;
343
+
#define BNGE_VNIC_RSS_FLAG 1
344
+
#define BNGE_VNIC_MCAST_FLAG 4
345
+
#define BNGE_VNIC_UCAST_FLAG 8
346
+
u32 vnic_id;
347
+
};
348
+
349
+
struct bnge_filter_base {
350
+
struct hlist_node hash;
351
+
struct list_head list;
352
+
__le64 filter_id;
353
+
u8 type;
354
+
#define BNGE_FLTR_TYPE_L2 2
355
+
u8 flags;
356
+
u16 rxq;
357
+
u16 fw_vnic_id;
358
+
u16 vf_idx;
359
+
unsigned long state;
360
+
#define BNGE_FLTR_VALID 0
361
+
#define BNGE_FLTR_FW_DELETED 2
362
+
363
+
struct rcu_head rcu;
364
+
};
365
+
366
+
struct bnge_l2_key {
367
+
union {
368
+
struct {
369
+
u8 dst_mac_addr[ETH_ALEN];
370
+
u16 vlan;
371
+
};
372
+
u32 filter_key;
373
+
};
374
+
};
375
+
376
+
#define BNGE_L2_KEY_SIZE (sizeof(struct bnge_l2_key) / 4)
377
+
struct bnge_l2_filter {
378
+
/* base filter must be the first member */
379
+
struct bnge_filter_base base;
380
+
struct bnge_l2_key l2_key;
381
+
refcount_t refcnt;
382
+
};
383
+
384
+
u16 bnge_cp_ring_for_rx(struct bnge_rx_ring_info *rxr);
385
+
u16 bnge_cp_ring_for_tx(struct bnge_tx_ring_info *txr);
386
+
void bnge_fill_hw_rss_tbl(struct bnge_net *bn, struct bnge_vnic_info *vnic);
273
387
#endif /* _BNGE_NETDEV_H_ */
+3
-3
drivers/net/ethernet/broadcom/bnge/bnge_resc.c
+3
-3
drivers/net/ethernet/broadcom/bnge/bnge_resc.c
···
46
46
return min_t(int, roce_msix, num_online_cpus() + 1);
47
47
}
48
48
49
-
static u16 bnge_aux_get_msix(struct bnge_dev *bd)
49
+
u16 bnge_aux_get_msix(struct bnge_dev *bd)
50
50
{
51
51
if (bnge_is_roce_en(bd))
52
52
return bd->aux_num_msix;
···
164
164
return bnge_fix_rings_count(rx, tx, max_nq, sh);
165
165
}
166
166
167
-
static int bnge_cal_nr_rss_ctxs(u16 rx_rings)
167
+
int bnge_cal_nr_rss_ctxs(u16 rx_rings)
168
168
{
169
169
if (!rx_rings)
170
170
return 0;
···
184
184
return 1;
185
185
}
186
186
187
-
static u32 bnge_get_rxfh_indir_size(struct bnge_dev *bd)
187
+
u32 bnge_get_rxfh_indir_size(struct bnge_dev *bd)
188
188
{
189
189
return bnge_cal_nr_rss_ctxs(bd->rx_nr_rings) *
190
190
BNGE_RSS_TABLE_ENTRIES;
+2
drivers/net/ethernet/broadcom/bnge/bnge_resc.h
+2
drivers/net/ethernet/broadcom/bnge/bnge_resc.h
···
72
72
int bnge_net_init_dflt_config(struct bnge_dev *bd);
73
73
void bnge_net_uninit_dflt_config(struct bnge_dev *bd);
74
74
void bnge_aux_init_dflt_config(struct bnge_dev *bd);
75
+
u32 bnge_get_rxfh_indir_size(struct bnge_dev *bd);
76
+
int bnge_cal_nr_rss_ctxs(u16 rx_rings);
75
77
76
78
static inline u32
77
79
bnge_adjust_pow_two(u32 total_ent, u16 ent_per_blk)
+64
-3
drivers/net/ethernet/broadcom/bnge/bnge_rmem.c
+64
-3
drivers/net/ethernet/broadcom/bnge/bnge_rmem.c
···
95
95
&rmem->dma_arr[i],
96
96
GFP_KERNEL);
97
97
if (!rmem->pg_arr[i])
98
-
return -ENOMEM;
98
+
goto err_free_ring;
99
99
100
100
if (rmem->ctx_mem)
101
101
bnge_init_ctx_mem(rmem->ctx_mem, rmem->pg_arr[i],
···
116
116
if (rmem->vmem_size) {
117
117
*rmem->vmem = vzalloc(rmem->vmem_size);
118
118
if (!(*rmem->vmem))
119
-
return -ENOMEM;
119
+
goto err_free_ring;
120
120
}
121
-
122
121
return 0;
122
+
123
+
err_free_ring:
124
+
bnge_free_ring(bd, rmem);
125
+
return -ENOMEM;
123
126
}
124
127
125
128
static int bnge_alloc_ctx_one_lvl(struct bnge_dev *bd,
···
438
435
ctx->flags |= BNGE_CTX_FLAG_INITED;
439
436
440
437
return 0;
438
+
}
439
+
440
+
void bnge_init_ring_struct(struct bnge_net *bn)
441
+
{
442
+
struct bnge_dev *bd = bn->bd;
443
+
int i, j;
444
+
445
+
for (i = 0; i < bd->nq_nr_rings; i++) {
446
+
struct bnge_napi *bnapi = bn->bnapi[i];
447
+
struct bnge_ring_mem_info *rmem;
448
+
struct bnge_nq_ring_info *nqr;
449
+
struct bnge_rx_ring_info *rxr;
450
+
struct bnge_tx_ring_info *txr;
451
+
struct bnge_ring_struct *ring;
452
+
453
+
nqr = &bnapi->nq_ring;
454
+
ring = &nqr->ring_struct;
455
+
rmem = &ring->ring_mem;
456
+
rmem->nr_pages = bn->cp_nr_pages;
457
+
rmem->page_size = HW_CMPD_RING_SIZE;
458
+
rmem->pg_arr = (void **)nqr->desc_ring;
459
+
rmem->dma_arr = nqr->desc_mapping;
460
+
rmem->vmem_size = 0;
461
+
462
+
rxr = bnapi->rx_ring;
463
+
if (!rxr)
464
+
goto skip_rx;
465
+
466
+
ring = &rxr->rx_ring_struct;
467
+
rmem = &ring->ring_mem;
468
+
rmem->nr_pages = bn->rx_nr_pages;
469
+
rmem->page_size = HW_RXBD_RING_SIZE;
470
+
rmem->pg_arr = (void **)rxr->rx_desc_ring;
471
+
rmem->dma_arr = rxr->rx_desc_mapping;
472
+
rmem->vmem_size = SW_RXBD_RING_SIZE * bn->rx_nr_pages;
473
+
rmem->vmem = (void **)&rxr->rx_buf_ring;
474
+
475
+
ring = &rxr->rx_agg_ring_struct;
476
+
rmem = &ring->ring_mem;
477
+
rmem->nr_pages = bn->rx_agg_nr_pages;
478
+
rmem->page_size = HW_RXBD_RING_SIZE;
479
+
rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
480
+
rmem->dma_arr = rxr->rx_agg_desc_mapping;
481
+
rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bn->rx_agg_nr_pages;
482
+
rmem->vmem = (void **)&rxr->rx_agg_buf_ring;
483
+
484
+
skip_rx:
485
+
bnge_for_each_napi_tx(j, bnapi, txr) {
486
+
ring = &txr->tx_ring_struct;
487
+
rmem = &ring->ring_mem;
488
+
rmem->nr_pages = bn->tx_nr_pages;
489
+
rmem->page_size = HW_TXBD_RING_SIZE;
490
+
rmem->pg_arr = (void **)txr->tx_desc_ring;
491
+
rmem->dma_arr = txr->tx_desc_mapping;
492
+
rmem->vmem_size = SW_TXBD_RING_SIZE * bn->tx_nr_pages;
493
+
rmem->vmem = (void **)&txr->tx_buf_ring;
494
+
}
495
+
}
441
496
}
+14
drivers/net/ethernet/broadcom/bnge/bnge_rmem.h
+14
drivers/net/ethernet/broadcom/bnge/bnge_rmem.h
···
6
6
7
7
struct bnge_ctx_mem_type;
8
8
struct bnge_dev;
9
+
struct bnge_net;
9
10
10
11
#define PTU_PTE_VALID 0x1UL
11
12
#define PTU_PTE_LAST 0x2UL
···
181
180
struct bnge_ctx_mem_type ctx_arr[BNGE_CTX_V2_MAX];
182
181
};
183
182
183
+
struct bnge_ring_struct {
184
+
struct bnge_ring_mem_info ring_mem;
185
+
186
+
u16 fw_ring_id;
187
+
union {
188
+
u16 grp_idx;
189
+
u16 map_idx; /* Used by NQs */
190
+
};
191
+
u32 handle;
192
+
u8 queue_id;
193
+
};
194
+
184
195
int bnge_alloc_ring(struct bnge_dev *bd, struct bnge_ring_mem_info *rmem);
185
196
void bnge_free_ring(struct bnge_dev *bd, struct bnge_ring_mem_info *rmem);
186
197
int bnge_alloc_ctx_mem(struct bnge_dev *bd);
187
198
void bnge_free_ctx_mem(struct bnge_dev *bd);
199
+
void bnge_init_ring_struct(struct bnge_net *bn);
188
200
189
201
#endif /* _BNGE_RMEM_H_ */