Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
dmaengine: hisilicon: Adapt DMA driver to HiSilicon IP09
The HiSilicon IP08 and HiSilicon IP09 are DMA iEPs, they
have the same pci device id but different pci revision.
Unfortunately, they have different register layouts, so
the origin driver cannot run on HiSilicon IP09 correctly.
This patch enables the driver to adapt to HiSilicon IP09.
HiSilicon IP09 offers 4 channels, each channel has a send
queue, a complete queue and an interrupt to help to do tasks.
This DMA engine can do memory copy between memory blocks.
Signed-off-by: Jie Hai <haijie1@huawei.com>
Acked-by: Zhou Wang <wangzhou1@hisilicon.com>
Link: https://lore.kernel.org/r/20220830062251.52993-6-haijie1@huawei.com
Signed-off-by: Vinod Koul <vkoul@kernel.org>
authored by
Jie Hai
and committed by
Vinod Koul
fd5273fa
4aa69cf7
+295
-78
+295
-78
drivers/dma/hisi_dma.c
+295
-78
drivers/dma/hisi_dma.c
···
1
1
// SPDX-License-Identifier: GPL-2.0-only
2
-
/* Copyright(c) 2019 HiSilicon Limited. */
2
+
/* Copyright(c) 2019-2022 HiSilicon Limited. */
3
+
3
4
#include <linux/bitfield.h>
4
5
#include <linux/dmaengine.h>
5
6
#include <linux/init.h>
···
10
9
#include <linux/spinlock.h>
11
10
#include "virt-dma.h"
12
11
13
-
#define HISI_DMA_SQ_BASE_L 0x0
14
-
#define HISI_DMA_SQ_BASE_H 0x4
15
-
#define HISI_DMA_SQ_DEPTH 0x8
16
-
#define HISI_DMA_SQ_TAIL_PTR 0xc
17
-
#define HISI_DMA_CQ_BASE_L 0x10
18
-
#define HISI_DMA_CQ_BASE_H 0x14
19
-
#define HISI_DMA_CQ_DEPTH 0x18
20
-
#define HISI_DMA_CQ_HEAD_PTR 0x1c
21
-
#define HISI_DMA_CTRL0 0x20
22
-
#define HISI_DMA_CTRL0_QUEUE_EN_S 0
23
-
#define HISI_DMA_CTRL0_QUEUE_PAUSE_S 4
24
-
#define HISI_DMA_CTRL1 0x24
25
-
#define HISI_DMA_CTRL1_QUEUE_RESET_S 0
26
-
#define HISI_DMA_Q_FSM_STS 0x30
27
-
#define HISI_DMA_FSM_STS_MASK GENMASK(3, 0)
28
-
#define HISI_DMA_INT_STS 0x40
29
-
#define HISI_DMA_INT_STS_MASK GENMASK(12, 0)
30
-
#define HISI_DMA_INT_MSK 0x44
31
-
#define HISI_DMA_MODE 0x217c
32
-
#define HISI_DMA_OFFSET 0x100
12
+
/* HiSilicon DMA register common field define */
13
+
#define HISI_DMA_Q_SQ_BASE_L 0x0
14
+
#define HISI_DMA_Q_SQ_BASE_H 0x4
15
+
#define HISI_DMA_Q_SQ_DEPTH 0x8
16
+
#define HISI_DMA_Q_SQ_TAIL_PTR 0xc
17
+
#define HISI_DMA_Q_CQ_BASE_L 0x10
18
+
#define HISI_DMA_Q_CQ_BASE_H 0x14
19
+
#define HISI_DMA_Q_CQ_DEPTH 0x18
20
+
#define HISI_DMA_Q_CQ_HEAD_PTR 0x1c
21
+
#define HISI_DMA_Q_CTRL0 0x20
22
+
#define HISI_DMA_Q_CTRL0_QUEUE_EN BIT(0)
23
+
#define HISI_DMA_Q_CTRL0_QUEUE_PAUSE BIT(4)
24
+
#define HISI_DMA_Q_CTRL1 0x24
25
+
#define HISI_DMA_Q_CTRL1_QUEUE_RESET BIT(0)
26
+
#define HISI_DMA_Q_FSM_STS 0x30
27
+
#define HISI_DMA_Q_FSM_STS_MASK GENMASK(3, 0)
28
+
#define HISI_DMA_Q_ERR_INT_NUM0 0x84
29
+
#define HISI_DMA_Q_ERR_INT_NUM1 0x88
30
+
#define HISI_DMA_Q_ERR_INT_NUM2 0x8c
33
31
34
-
#define HISI_DMA_MSI_NUM 32
35
-
#define HISI_DMA_CHAN_NUM 30
36
-
#define HISI_DMA_Q_DEPTH_VAL 1024
32
+
/* HiSilicon IP08 DMA register and field define */
33
+
#define HISI_DMA_HIP08_MODE 0x217C
34
+
#define HISI_DMA_HIP08_Q_BASE 0x0
35
+
#define HISI_DMA_HIP08_Q_CTRL0_ERR_ABORT_EN BIT(2)
36
+
#define HISI_DMA_HIP08_Q_INT_STS 0x40
37
+
#define HISI_DMA_HIP08_Q_INT_MSK 0x44
38
+
#define HISI_DMA_HIP08_Q_INT_STS_MASK GENMASK(14, 0)
39
+
#define HISI_DMA_HIP08_Q_ERR_INT_NUM3 0x90
40
+
#define HISI_DMA_HIP08_Q_ERR_INT_NUM4 0x94
41
+
#define HISI_DMA_HIP08_Q_ERR_INT_NUM5 0x98
42
+
#define HISI_DMA_HIP08_Q_ERR_INT_NUM6 0x48
43
+
#define HISI_DMA_HIP08_Q_CTRL0_SQCQ_DRCT BIT(24)
37
44
38
-
#define PCI_BAR_2 2
45
+
/* HiSilicon IP09 DMA register and field define */
46
+
#define HISI_DMA_HIP09_DMA_FLR_DISABLE 0xA00
47
+
#define HISI_DMA_HIP09_DMA_FLR_DISABLE_B BIT(0)
48
+
#define HISI_DMA_HIP09_Q_BASE 0x2000
49
+
#define HISI_DMA_HIP09_Q_CTRL0_ERR_ABORT_EN GENMASK(31, 28)
50
+
#define HISI_DMA_HIP09_Q_CTRL0_SQ_DRCT BIT(26)
51
+
#define HISI_DMA_HIP09_Q_CTRL0_CQ_DRCT BIT(27)
52
+
#define HISI_DMA_HIP09_Q_CTRL1_VA_ENABLE BIT(2)
53
+
#define HISI_DMA_HIP09_Q_INT_STS 0x40
54
+
#define HISI_DMA_HIP09_Q_INT_MSK 0x44
55
+
#define HISI_DMA_HIP09_Q_INT_STS_MASK 0x1
56
+
#define HISI_DMA_HIP09_Q_ERR_INT_STS 0x48
57
+
#define HISI_DMA_HIP09_Q_ERR_INT_MSK 0x4C
58
+
#define HISI_DMA_HIP09_Q_ERR_INT_STS_MASK GENMASK(18, 1)
59
+
#define HISI_DMA_HIP09_PORT_CFG_REG(port_id) (0x800 + \
60
+
(port_id) * 0x20)
61
+
#define HISI_DMA_HIP09_PORT_CFG_LINK_DOWN_MASK_B BIT(16)
39
62
40
-
#define HISI_DMA_POLL_Q_STS_DELAY_US 10
41
-
#define HISI_DMA_POLL_Q_STS_TIME_OUT_US 1000
63
+
#define HISI_DMA_HIP09_MAX_PORT_NUM 16
64
+
65
+
#define HISI_DMA_HIP08_MSI_NUM 32
66
+
#define HISI_DMA_HIP08_CHAN_NUM 30
67
+
#define HISI_DMA_HIP09_MSI_NUM 4
68
+
#define HISI_DMA_HIP09_CHAN_NUM 4
69
+
#define HISI_DMA_REVISION_HIP08B 0x21
70
+
#define HISI_DMA_REVISION_HIP09A 0x30
71
+
72
+
#define HISI_DMA_Q_OFFSET 0x100
73
+
#define HISI_DMA_Q_DEPTH_VAL 1024
74
+
75
+
#define PCI_BAR_2 2
76
+
77
+
#define HISI_DMA_POLL_Q_STS_DELAY_US 10
78
+
#define HISI_DMA_POLL_Q_STS_TIME_OUT_US 1000
79
+
80
+
/*
81
+
* The HIP08B(HiSilicon IP08) and HIP09A(HiSilicon IP09) are DMA iEPs, they
82
+
* have the same pci device id but different pci revision.
83
+
* Unfortunately, they have different register layouts, so two layout
84
+
* enumerations are defined.
85
+
*/
86
+
enum hisi_dma_reg_layout {
87
+
HISI_DMA_REG_LAYOUT_INVALID = 0,
88
+
HISI_DMA_REG_LAYOUT_HIP08,
89
+
HISI_DMA_REG_LAYOUT_HIP09
90
+
};
42
91
43
92
enum hisi_dma_mode {
44
93
EP = 0,
···
159
108
struct dma_device dma_dev;
160
109
u32 chan_num;
161
110
u32 chan_depth;
111
+
enum hisi_dma_reg_layout reg_layout;
112
+
void __iomem *queue_base; /* queue region start of register */
162
113
struct hisi_dma_chan chan[];
163
114
};
115
+
116
+
static enum hisi_dma_reg_layout hisi_dma_get_reg_layout(struct pci_dev *pdev)
117
+
{
118
+
if (pdev->revision == HISI_DMA_REVISION_HIP08B)
119
+
return HISI_DMA_REG_LAYOUT_HIP08;
120
+
else if (pdev->revision >= HISI_DMA_REVISION_HIP09A)
121
+
return HISI_DMA_REG_LAYOUT_HIP09;
122
+
123
+
return HISI_DMA_REG_LAYOUT_INVALID;
124
+
}
125
+
126
+
static u32 hisi_dma_get_chan_num(struct pci_dev *pdev)
127
+
{
128
+
if (pdev->revision == HISI_DMA_REVISION_HIP08B)
129
+
return HISI_DMA_HIP08_CHAN_NUM;
130
+
131
+
return HISI_DMA_HIP09_CHAN_NUM;
132
+
}
133
+
134
+
static u32 hisi_dma_get_msi_num(struct pci_dev *pdev)
135
+
{
136
+
if (pdev->revision == HISI_DMA_REVISION_HIP08B)
137
+
return HISI_DMA_HIP08_MSI_NUM;
138
+
139
+
return HISI_DMA_HIP09_MSI_NUM;
140
+
}
141
+
142
+
static u32 hisi_dma_get_queue_base(struct pci_dev *pdev)
143
+
{
144
+
if (pdev->revision == HISI_DMA_REVISION_HIP08B)
145
+
return HISI_DMA_HIP08_Q_BASE;
146
+
147
+
return HISI_DMA_HIP09_Q_BASE;
148
+
}
164
149
165
150
static inline struct hisi_dma_chan *to_hisi_dma_chan(struct dma_chan *c)
166
151
{
···
211
124
static inline void hisi_dma_chan_write(void __iomem *base, u32 reg, u32 index,
212
125
u32 val)
213
126
{
214
-
writel_relaxed(val, base + reg + index * HISI_DMA_OFFSET);
127
+
writel_relaxed(val, base + reg + index * HISI_DMA_Q_OFFSET);
215
128
}
216
129
217
130
static inline void hisi_dma_update_bit(void __iomem *addr, u32 pos, bool val)
···
219
132
u32 tmp;
220
133
221
134
tmp = readl_relaxed(addr);
222
-
tmp = val ? tmp | BIT(pos) : tmp & ~BIT(pos);
135
+
tmp = val ? tmp | pos : tmp & ~pos;
223
136
writel_relaxed(tmp, addr);
224
137
}
225
138
226
139
static void hisi_dma_pause_dma(struct hisi_dma_dev *hdma_dev, u32 index,
227
140
bool pause)
228
141
{
229
-
void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index *
230
-
HISI_DMA_OFFSET;
142
+
void __iomem *addr;
231
143
232
-
hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_PAUSE_S, pause);
144
+
addr = hdma_dev->queue_base + HISI_DMA_Q_CTRL0 +
145
+
index * HISI_DMA_Q_OFFSET;
146
+
hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL0_QUEUE_PAUSE, pause);
233
147
}
234
148
235
149
static void hisi_dma_enable_dma(struct hisi_dma_dev *hdma_dev, u32 index,
236
150
bool enable)
237
151
{
238
-
void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index *
239
-
HISI_DMA_OFFSET;
152
+
void __iomem *addr;
240
153
241
-
hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_EN_S, enable);
154
+
addr = hdma_dev->queue_base + HISI_DMA_Q_CTRL0 +
155
+
index * HISI_DMA_Q_OFFSET;
156
+
hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL0_QUEUE_EN, enable);
242
157
}
243
158
244
159
static void hisi_dma_mask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
245
160
{
246
-
hisi_dma_chan_write(hdma_dev->base, HISI_DMA_INT_MSK, qp_index,
247
-
HISI_DMA_INT_STS_MASK);
161
+
void __iomem *q_base = hdma_dev->queue_base;
162
+
163
+
if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
164
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_MSK,
165
+
qp_index, HISI_DMA_HIP08_Q_INT_STS_MASK);
166
+
else {
167
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_MSK,
168
+
qp_index, HISI_DMA_HIP09_Q_INT_STS_MASK);
169
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_MSK,
170
+
qp_index,
171
+
HISI_DMA_HIP09_Q_ERR_INT_STS_MASK);
172
+
}
248
173
}
249
174
250
175
static void hisi_dma_unmask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
251
176
{
252
-
void __iomem *base = hdma_dev->base;
177
+
void __iomem *q_base = hdma_dev->queue_base;
253
178
254
-
hisi_dma_chan_write(base, HISI_DMA_INT_STS, qp_index,
255
-
HISI_DMA_INT_STS_MASK);
256
-
hisi_dma_chan_write(base, HISI_DMA_INT_MSK, qp_index, 0);
179
+
if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
180
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_STS,
181
+
qp_index, HISI_DMA_HIP08_Q_INT_STS_MASK);
182
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_MSK,
183
+
qp_index, 0);
184
+
} else {
185
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_STS,
186
+
qp_index, HISI_DMA_HIP09_Q_INT_STS_MASK);
187
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_STS,
188
+
qp_index,
189
+
HISI_DMA_HIP09_Q_ERR_INT_STS_MASK);
190
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_MSK,
191
+
qp_index, 0);
192
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_MSK,
193
+
qp_index, 0);
194
+
}
257
195
}
258
196
259
197
static void hisi_dma_do_reset(struct hisi_dma_dev *hdma_dev, u32 index)
260
198
{
261
-
void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL1 + index *
262
-
HISI_DMA_OFFSET;
199
+
void __iomem *addr;
263
200
264
-
hisi_dma_update_bit(addr, HISI_DMA_CTRL1_QUEUE_RESET_S, 1);
201
+
addr = hdma_dev->queue_base +
202
+
HISI_DMA_Q_CTRL1 + index * HISI_DMA_Q_OFFSET;
203
+
hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL1_QUEUE_RESET, 1);
265
204
}
266
205
267
206
static void hisi_dma_reset_qp_point(struct hisi_dma_dev *hdma_dev, u32 index)
268
207
{
269
-
hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, index, 0);
270
-
hisi_dma_chan_write(hdma_dev->base, HISI_DMA_CQ_HEAD_PTR, index, 0);
208
+
void __iomem *q_base = hdma_dev->queue_base;
209
+
210
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_TAIL_PTR, index, 0);
211
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR, index, 0);
271
212
}
272
213
273
214
static void hisi_dma_reset_or_disable_hw_chan(struct hisi_dma_chan *chan,
···
310
195
hisi_dma_enable_dma(hdma_dev, index, false);
311
196
hisi_dma_mask_irq(hdma_dev, index);
312
197
313
-
addr = hdma_dev->base +
314
-
HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET;
198
+
addr = hdma_dev->queue_base +
199
+
HISI_DMA_Q_FSM_STS + index * HISI_DMA_Q_OFFSET;
315
200
316
201
ret = readl_relaxed_poll_timeout(addr, tmp,
317
-
FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) != RUN,
202
+
FIELD_GET(HISI_DMA_Q_FSM_STS_MASK, tmp) != RUN,
318
203
HISI_DMA_POLL_Q_STS_DELAY_US, HISI_DMA_POLL_Q_STS_TIME_OUT_US);
319
204
if (ret) {
320
205
dev_err(&hdma_dev->pdev->dev, "disable channel timeout!\n");
···
331
216
}
332
217
333
218
ret = readl_relaxed_poll_timeout(addr, tmp,
334
-
FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) == IDLE,
219
+
FIELD_GET(HISI_DMA_Q_FSM_STS_MASK, tmp) == IDLE,
335
220
HISI_DMA_POLL_Q_STS_DELAY_US, HISI_DMA_POLL_Q_STS_TIME_OUT_US);
336
221
if (ret) {
337
222
dev_err(&hdma_dev->pdev->dev, "reset channel timeout!\n");
···
413
298
chan->sq_tail = (chan->sq_tail + 1) % hdma_dev->chan_depth;
414
299
415
300
/* update sq_tail to trigger a new task */
416
-
hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, chan->qp_num,
417
-
chan->sq_tail);
301
+
hisi_dma_chan_write(hdma_dev->queue_base, HISI_DMA_Q_SQ_TAIL_PTR,
302
+
chan->qp_num, chan->sq_tail);
418
303
}
419
304
420
305
static void hisi_dma_issue_pending(struct dma_chan *c)
···
488
373
static void hisi_dma_init_hw_qp(struct hisi_dma_dev *hdma_dev, u32 index)
489
374
{
490
375
struct hisi_dma_chan *chan = &hdma_dev->chan[index];
376
+
void __iomem *q_base = hdma_dev->queue_base;
491
377
u32 hw_depth = hdma_dev->chan_depth - 1;
492
-
void __iomem *base = hdma_dev->base;
378
+
void __iomem *addr;
379
+
u32 tmp;
493
380
494
381
/* set sq, cq base */
495
-
hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_L, index,
382
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_BASE_L, index,
496
383
lower_32_bits(chan->sq_dma));
497
-
hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_H, index,
384
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_BASE_H, index,
498
385
upper_32_bits(chan->sq_dma));
499
-
hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_L, index,
386
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_BASE_L, index,
500
387
lower_32_bits(chan->cq_dma));
501
-
hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_H, index,
388
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_BASE_H, index,
502
389
upper_32_bits(chan->cq_dma));
503
390
504
391
/* set sq, cq depth */
505
-
hisi_dma_chan_write(base, HISI_DMA_SQ_DEPTH, index, hw_depth);
506
-
hisi_dma_chan_write(base, HISI_DMA_CQ_DEPTH, index, hw_depth);
392
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_DEPTH, index, hw_depth);
393
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_DEPTH, index, hw_depth);
507
394
508
395
/* init sq tail and cq head */
509
-
hisi_dma_chan_write(base, HISI_DMA_SQ_TAIL_PTR, index, 0);
510
-
hisi_dma_chan_write(base, HISI_DMA_CQ_HEAD_PTR, index, 0);
396
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_TAIL_PTR, index, 0);
397
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR, index, 0);
398
+
399
+
/* init error interrupt stats */
400
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM0, index, 0);
401
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM1, index, 0);
402
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM2, index, 0);
403
+
404
+
if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
405
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM3,
406
+
index, 0);
407
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM4,
408
+
index, 0);
409
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM5,
410
+
index, 0);
411
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM6,
412
+
index, 0);
413
+
/*
414
+
* init SQ/CQ direction selecting register.
415
+
* "0" is to local side and "1" is to remote side.
416
+
*/
417
+
addr = q_base + HISI_DMA_Q_CTRL0 + index * HISI_DMA_Q_OFFSET;
418
+
hisi_dma_update_bit(addr, HISI_DMA_HIP08_Q_CTRL0_SQCQ_DRCT, 0);
419
+
420
+
/*
421
+
* 0 - Continue to next descriptor if error occurs.
422
+
* 1 - Abort the DMA queue if error occurs.
423
+
*/
424
+
hisi_dma_update_bit(addr,
425
+
HISI_DMA_HIP08_Q_CTRL0_ERR_ABORT_EN, 0);
426
+
} else {
427
+
addr = q_base + HISI_DMA_Q_CTRL0 + index * HISI_DMA_Q_OFFSET;
428
+
429
+
/*
430
+
* init SQ/CQ direction selecting register.
431
+
* "0" is to local side and "1" is to remote side.
432
+
*/
433
+
hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL0_SQ_DRCT, 0);
434
+
hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL0_CQ_DRCT, 0);
435
+
436
+
/*
437
+
* 0 - Continue to next descriptor if error occurs.
438
+
* 1 - Abort the DMA queue if error occurs.
439
+
*/
440
+
441
+
tmp = readl_relaxed(addr);
442
+
tmp &= ~HISI_DMA_HIP09_Q_CTRL0_ERR_ABORT_EN;
443
+
writel_relaxed(tmp, addr);
444
+
445
+
/*
446
+
* 0 - dma should process FLR whith CPU.
447
+
* 1 - dma not process FLR, only cpu process FLR.
448
+
*/
449
+
addr = q_base + HISI_DMA_HIP09_DMA_FLR_DISABLE +
450
+
index * HISI_DMA_Q_OFFSET;
451
+
hisi_dma_update_bit(addr, HISI_DMA_HIP09_DMA_FLR_DISABLE_B, 0);
452
+
453
+
addr = q_base + HISI_DMA_Q_CTRL1 + index * HISI_DMA_Q_OFFSET;
454
+
hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL1_VA_ENABLE, 1);
455
+
}
511
456
}
512
457
513
458
static void hisi_dma_enable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
···
611
436
struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
612
437
struct hisi_dma_desc *desc;
613
438
struct hisi_dma_cqe *cqe;
439
+
void __iomem *q_base;
614
440
615
441
spin_lock(&chan->vc.lock);
616
442
617
443
desc = chan->desc;
618
444
cqe = chan->cq + chan->cq_head;
445
+
q_base = hdma_dev->queue_base;
619
446
if (desc) {
620
447
chan->cq_head = (chan->cq_head + 1) % hdma_dev->chan_depth;
621
-
hisi_dma_chan_write(hdma_dev->base, HISI_DMA_CQ_HEAD_PTR,
448
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR,
622
449
chan->qp_num, chan->cq_head);
623
450
if (FIELD_GET(STATUS_MASK, cqe->w0) == STATUS_SUCC) {
624
451
vchan_cookie_complete(&desc->vd);
···
681
504
static void hisi_dma_set_mode(struct hisi_dma_dev *hdma_dev,
682
505
enum hisi_dma_mode mode)
683
506
{
684
-
writel_relaxed(mode == RC ? 1 : 0, hdma_dev->base + HISI_DMA_MODE);
507
+
if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
508
+
writel_relaxed(mode == RC ? 1 : 0,
509
+
hdma_dev->base + HISI_DMA_HIP08_MODE);
510
+
}
511
+
512
+
static void hisi_dma_init_hw(struct hisi_dma_dev *hdma_dev)
513
+
{
514
+
void __iomem *addr;
515
+
int i;
516
+
517
+
if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP09) {
518
+
for (i = 0; i < HISI_DMA_HIP09_MAX_PORT_NUM; i++) {
519
+
addr = hdma_dev->base + HISI_DMA_HIP09_PORT_CFG_REG(i);
520
+
hisi_dma_update_bit(addr,
521
+
HISI_DMA_HIP09_PORT_CFG_LINK_DOWN_MASK_B, 1);
522
+
}
523
+
}
524
+
}
525
+
526
+
static void hisi_dma_init_dma_dev(struct hisi_dma_dev *hdma_dev)
527
+
{
528
+
struct dma_device *dma_dev;
529
+
530
+
dma_dev = &hdma_dev->dma_dev;
531
+
dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
532
+
dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources;
533
+
dma_dev->device_prep_dma_memcpy = hisi_dma_prep_dma_memcpy;
534
+
dma_dev->device_tx_status = hisi_dma_tx_status;
535
+
dma_dev->device_issue_pending = hisi_dma_issue_pending;
536
+
dma_dev->device_terminate_all = hisi_dma_terminate_all;
537
+
dma_dev->device_synchronize = hisi_dma_synchronize;
538
+
dma_dev->directions = BIT(DMA_MEM_TO_MEM);
539
+
dma_dev->dev = &hdma_dev->pdev->dev;
540
+
INIT_LIST_HEAD(&dma_dev->channels);
685
541
}
686
542
687
543
static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id)
688
544
{
545
+
enum hisi_dma_reg_layout reg_layout;
689
546
struct device *dev = &pdev->dev;
690
547
struct hisi_dma_dev *hdma_dev;
691
548
struct dma_device *dma_dev;
549
+
u32 chan_num;
550
+
u32 msi_num;
692
551
int ret;
552
+
553
+
reg_layout = hisi_dma_get_reg_layout(pdev);
554
+
if (reg_layout == HISI_DMA_REG_LAYOUT_INVALID) {
555
+
dev_err(dev, "unsupported device!\n");
556
+
return -EINVAL;
557
+
}
693
558
694
559
ret = pcim_enable_device(pdev);
695
560
if (ret) {
···
749
530
if (ret)
750
531
return ret;
751
532
752
-
hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, HISI_DMA_CHAN_NUM), GFP_KERNEL);
533
+
chan_num = hisi_dma_get_chan_num(pdev);
534
+
hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, chan_num),
535
+
GFP_KERNEL);
753
536
if (!hdma_dev)
754
537
return -EINVAL;
755
538
756
539
hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2];
757
540
hdma_dev->pdev = pdev;
758
-
hdma_dev->chan_num = HISI_DMA_CHAN_NUM;
759
541
hdma_dev->chan_depth = HISI_DMA_Q_DEPTH_VAL;
542
+
hdma_dev->chan_num = chan_num;
543
+
hdma_dev->reg_layout = reg_layout;
544
+
hdma_dev->queue_base = hdma_dev->base + hisi_dma_get_queue_base(pdev);
760
545
761
546
pci_set_drvdata(pdev, hdma_dev);
762
547
pci_set_master(pdev);
763
548
549
+
msi_num = hisi_dma_get_msi_num(pdev);
550
+
764
551
/* This will be freed by 'pcim_release()'. See 'pcim_enable_device()' */
765
-
ret = pci_alloc_irq_vectors(pdev, HISI_DMA_MSI_NUM, HISI_DMA_MSI_NUM,
766
-
PCI_IRQ_MSI);
552
+
ret = pci_alloc_irq_vectors(pdev, msi_num, msi_num, PCI_IRQ_MSI);
767
553
if (ret < 0) {
768
554
dev_err(dev, "Failed to allocate MSI vectors!\n");
769
555
return ret;
770
556
}
771
557
772
-
dma_dev = &hdma_dev->dma_dev;
773
-
dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
774
-
dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources;
775
-
dma_dev->device_prep_dma_memcpy = hisi_dma_prep_dma_memcpy;
776
-
dma_dev->device_tx_status = hisi_dma_tx_status;
777
-
dma_dev->device_issue_pending = hisi_dma_issue_pending;
778
-
dma_dev->device_terminate_all = hisi_dma_terminate_all;
779
-
dma_dev->device_synchronize = hisi_dma_synchronize;
780
-
dma_dev->directions = BIT(DMA_MEM_TO_MEM);
781
-
dma_dev->dev = dev;
782
-
INIT_LIST_HEAD(&dma_dev->channels);
558
+
hisi_dma_init_dma_dev(hdma_dev);
783
559
784
560
hisi_dma_set_mode(hdma_dev, RC);
561
+
562
+
hisi_dma_init_hw(hdma_dev);
785
563
786
564
ret = hisi_dma_enable_hw_channels(hdma_dev);
787
565
if (ret < 0) {
···
791
575
if (ret)
792
576
return ret;
793
577
578
+
dma_dev = &hdma_dev->dma_dev;
794
579
ret = dmaenginem_async_device_register(dma_dev);
795
580
if (ret < 0)
796
581
dev_err(dev, "failed to register device!\n");