Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'dmaengine-6.1-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine
Pull dmaengine updates from Vinod Koul:
"New Support:
- MT6795 SoC dma controller (AngeloGioacchino Del Regno)
- qcom-adm controller yaml binding (Christian Marangi)
- Renesas r8a779g0 dma controller yaml binding (Geert Uytterhoeven)
- Qualcomm SM6350 GPI dma controller (Luca Weiss)
Updates:
- STM32 DMA-MDMA chaining support (Amelie Delaunay)
- make hsu driver use managed resources (Andy Shevchenko)
- the usual round of idxd driver updates (Dave Jiang & Jerry
Snitselaar)
- apple dma driver iommu and pd properties and remove use
of devres for irqs (Janne Grunau & Martin Povišer)
- device_synchronize support for Xilinx zynqmp driver (Swati
Agarwal)"
* tag 'dmaengine-6.1-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine: (60 commits)
dmaengine: ioat: remove unused declarations in dma.h
dmaengine: ti: k3-udma: Respond TX done if DMA_PREP_INTERRUPT is not requested
dmaengine: zynqmp_dma: Add device_synchronize support
dt-bindings: dma: add additional pbus reset to qcom,adm
dt-bindings: dma: rework qcom,adm Documentation to yaml schema
dt-bindings: dma: apple,admac: Add iommus and power-domains properties
dmaengine: dw-edma: Remove runtime PM support
dmaengine: idxd: add configuration for concurrent batch descriptor processing
dmaengine: idxd: add configuration for concurrent work descriptor processing
dmaengine: idxd: add WQ operation cap restriction support
dmanegine: idxd: reformat opcap output to match bitmap_parse() input
dmaengine: idxd: convert ats_dis to a wq flag
dmaengine: ioat: stop mod_timer from resurrecting deleted timer in __cleanup()
dmaengine: qcom-adm: fix wrong calling convention for prep_slave_sg
dmaengine: qcom-adm: fix wrong sizeof config in slave_config
dmaengine: ti: k3-psil: add additional TX threads for j721e
dmaengine: ti: k3-psil: add additional TX threads for j7200
dmaengine: apple-admac: Trigger shared reset
dmaengine: apple-admac: Do not use devres for IRQs
dmaengine: ti: edma: Remove some unused functions
...
+1851
-410
+35
Documentation/ABI/stable/sysfs-driver-dma-idxd
+35
Documentation/ABI/stable/sysfs-driver-dma-idxd
···
227
227
Description: Indicate the number of retires for an enqcmds submission on a sharedwq.
228
228
A max value to set attribute is capped at 64.
229
229
230
+
What: /sys/bus/dsa/devices/wq<m>.<n>/op_config
231
+
Date: Sept 14, 2022
232
+
KernelVersion: 6.0.0
233
+
Contact: dmaengine@vger.kernel.org
234
+
Description: Shows the operation capability bits displayed in bitmap format
235
+
presented by %*pb printk() output format specifier.
236
+
The attribute can be configured when the WQ is disabled in
237
+
order to configure the WQ to accept specific bits that
238
+
correlates to the operations allowed. It's visible only
239
+
on platforms that support the capability.
240
+
230
241
What: /sys/bus/dsa/devices/engine<m>.<n>/group_id
231
242
Date: Oct 25, 2019
232
243
KernelVersion: 5.6.0
···
266
255
Description: Indicates the number of Read Buffers reserved for the use of
267
256
engines in the group. See DSA spec v1.2 9.2.18 GRPCFG Read Buffers
268
257
Reserved.
258
+
259
+
What: /sys/bus/dsa/devices/group<m>.<n>/desc_progress_limit
260
+
Date: Sept 14, 2022
261
+
KernelVersion: 6.0.0
262
+
Contact: dmaengine@vger.kernel.org
263
+
Description: Allows control of the number of work descriptors that can be
264
+
concurrently processed by an engine in the group as a fraction
265
+
of the Maximum Work Descriptors in Progress value specified in
266
+
the ENGCAP register. The acceptable values are 0 (default),
267
+
1 (1/2 of max value), 2 (1/4 of the max value), and 3 (1/8 of
268
+
the max value). It's visible only on platforms that support
269
+
the capability.
270
+
271
+
What: /sys/bus/dsa/devices/group<m>.<n>/batch_progress_limit
272
+
Date: Sept 14, 2022
273
+
KernelVersion: 6.0.0
274
+
Contact: dmaengine@vger.kernel.org
275
+
Description: Allows control of the number of batch descriptors that can be
276
+
concurrently processed by an engine in the group as a fraction
277
+
of the Maximum Batch Descriptors in Progress value specified in
278
+
the ENGCAP register. The acceptable values are 0 (default),
279
+
1 (1/2 of max value), 2 (1/4 of the max value), and 3 (1/8 of
280
+
the max value). It's visible only on platforms that support
281
+
the capability.
+1
Documentation/arm/index.rst
+1
Documentation/arm/index.rst
+415
Documentation/arm/stm32/stm32-dma-mdma-chaining.rst
+415
Documentation/arm/stm32/stm32-dma-mdma-chaining.rst
···
1
+
.. SPDX-License-Identifier: GPL-2.0
2
+
3
+
=======================
4
+
STM32 DMA-MDMA chaining
5
+
=======================
6
+
7
+
8
+
Introduction
9
+
------------
10
+
11
+
This document describes the STM32 DMA-MDMA chaining feature. But before going
12
+
further, let's introduce the peripherals involved.
13
+
14
+
To offload data transfers from the CPU, STM32 microprocessors (MPUs) embed
15
+
direct memory access controllers (DMA).
16
+
17
+
STM32MP1 SoCs embed both STM32 DMA and STM32 MDMA controllers. STM32 DMA
18
+
request routing capabilities are enhanced by a DMA request multiplexer
19
+
(STM32 DMAMUX).
20
+
21
+
**STM32 DMAMUX**
22
+
23
+
STM32 DMAMUX routes any DMA request from a given peripheral to any STM32 DMA
24
+
controller (STM32MP1 counts two STM32 DMA controllers) channels.
25
+
26
+
**STM32 DMA**
27
+
28
+
STM32 DMA is mainly used to implement central data buffer storage (usually in
29
+
the system SRAM) for different peripheral. It can access external RAMs but
30
+
without the ability to generate convenient burst transfer ensuring the best
31
+
load of the AXI.
32
+
33
+
**STM32 MDMA**
34
+
35
+
STM32 MDMA (Master DMA) is mainly used to manage direct data transfers between
36
+
RAM data buffers without CPU intervention. It can also be used in a
37
+
hierarchical structure that uses STM32 DMA as first level data buffer
38
+
interfaces for AHB peripherals, while the STM32 MDMA acts as a second level
39
+
DMA with better performance. As a AXI/AHB master, STM32 MDMA can take control
40
+
of the AXI/AHB bus.
41
+
42
+
43
+
Principles
44
+
----------
45
+
46
+
STM32 DMA-MDMA chaining feature relies on the strengths of STM32 DMA and
47
+
STM32 MDMA controllers.
48
+
49
+
STM32 DMA has a circular Double Buffer Mode (DBM). At each end of transaction
50
+
(when DMA data counter - DMA_SxNDTR - reaches 0), the memory pointers
51
+
(configured with DMA_SxSM0AR and DMA_SxM1AR) are swapped and the DMA data
52
+
counter is automatically reloaded. This allows the SW or the STM32 MDMA to
53
+
process one memory area while the second memory area is being filled/used by
54
+
the STM32 DMA transfer.
55
+
56
+
With STM32 MDMA linked-list mode, a single request initiates the data array
57
+
(collection of nodes) to be transferred until the linked-list pointer for the
58
+
channel is null. The channel transfer complete of the last node is the end of
59
+
transfer, unless first and last nodes are linked to each other, in such a
60
+
case, the linked-list loops on to create a circular MDMA transfer.
61
+
62
+
STM32 MDMA has direct connections with STM32 DMA. This enables autonomous
63
+
communication and synchronization between peripherals, thus saving CPU
64
+
resources and bus congestion. Transfer Complete signal of STM32 DMA channel
65
+
can triggers STM32 MDMA transfer. STM32 MDMA can clear the request generated
66
+
by the STM32 DMA by writing to its Interrupt Clear register (whose address is
67
+
stored in MDMA_CxMAR, and bit mask in MDMA_CxMDR).
68
+
69
+
.. table:: STM32 MDMA interconnect table with STM32 DMA
70
+
71
+
+--------------+----------------+-----------+------------+
72
+
| STM32 DMAMUX | STM32 DMA | STM32 DMA | STM32 MDMA |
73
+
| channels | channels | Transfer | request |
74
+
| | | complete | |
75
+
| | | signal | |
76
+
+==============+================+===========+============+
77
+
| Channel *0* | DMA1 channel 0 | dma1_tcf0 | *0x00* |
78
+
+--------------+----------------+-----------+------------+
79
+
| Channel *1* | DMA1 channel 1 | dma1_tcf1 | *0x01* |
80
+
+--------------+----------------+-----------+------------+
81
+
| Channel *2* | DMA1 channel 2 | dma1_tcf2 | *0x02* |
82
+
+--------------+----------------+-----------+------------+
83
+
| Channel *3* | DMA1 channel 3 | dma1_tcf3 | *0x03* |
84
+
+--------------+----------------+-----------+------------+
85
+
| Channel *4* | DMA1 channel 4 | dma1_tcf4 | *0x04* |
86
+
+--------------+----------------+-----------+------------+
87
+
| Channel *5* | DMA1 channel 5 | dma1_tcf5 | *0x05* |
88
+
+--------------+----------------+-----------+------------+
89
+
| Channel *6* | DMA1 channel 6 | dma1_tcf6 | *0x06* |
90
+
+--------------+----------------+-----------+------------+
91
+
| Channel *7* | DMA1 channel 7 | dma1_tcf7 | *0x07* |
92
+
+--------------+----------------+-----------+------------+
93
+
| Channel *8* | DMA2 channel 0 | dma2_tcf0 | *0x08* |
94
+
+--------------+----------------+-----------+------------+
95
+
| Channel *9* | DMA2 channel 1 | dma2_tcf1 | *0x09* |
96
+
+--------------+----------------+-----------+------------+
97
+
| Channel *10* | DMA2 channel 2 | dma2_tcf2 | *0x0A* |
98
+
+--------------+----------------+-----------+------------+
99
+
| Channel *11* | DMA2 channel 3 | dma2_tcf3 | *0x0B* |
100
+
+--------------+----------------+-----------+------------+
101
+
| Channel *12* | DMA2 channel 4 | dma2_tcf4 | *0x0C* |
102
+
+--------------+----------------+-----------+------------+
103
+
| Channel *13* | DMA2 channel 5 | dma2_tcf5 | *0x0D* |
104
+
+--------------+----------------+-----------+------------+
105
+
| Channel *14* | DMA2 channel 6 | dma2_tcf6 | *0x0E* |
106
+
+--------------+----------------+-----------+------------+
107
+
| Channel *15* | DMA2 channel 7 | dma2_tcf7 | *0x0F* |
108
+
+--------------+----------------+-----------+------------+
109
+
110
+
STM32 DMA-MDMA chaining feature then uses a SRAM buffer. STM32MP1 SoCs embed
111
+
three fast access static internal RAMs of various size, used for data storage.
112
+
Due to STM32 DMA legacy (within microcontrollers), STM32 DMA performances are
113
+
bad with DDR, while they are optimal with SRAM. Hence the SRAM buffer used
114
+
between STM32 DMA and STM32 MDMA. This buffer is split in two equal periods
115
+
and STM32 DMA uses one period while STM32 MDMA uses the other period
116
+
simultaneously.
117
+
::
118
+
119
+
dma[1:2]-tcf[0:7]
120
+
.----------------.
121
+
____________ ' _________ V____________
122
+
| STM32 DMA | / __|>_ \ | STM32 MDMA |
123
+
|------------| | / \ | |------------|
124
+
| DMA_SxM0AR |<=>| | SRAM | |<=>| []-[]...[] |
125
+
| DMA_SxM1AR | | \_____/ | | |
126
+
|____________| \___<|____/ |____________|
127
+
128
+
STM32 DMA-MDMA chaining uses (struct dma_slave_config).peripheral_config to
129
+
exchange the parameters needed to configure MDMA. These parameters are
130
+
gathered into a u32 array with three values:
131
+
132
+
* the STM32 MDMA request (which is actually the DMAMUX channel ID),
133
+
* the address of the STM32 DMA register to clear the Transfer Complete
134
+
interrupt flag,
135
+
* the mask of the Transfer Complete interrupt flag of the STM32 DMA channel.
136
+
137
+
Device Tree updates for STM32 DMA-MDMA chaining support
138
+
-------------------------------------------------------
139
+
140
+
**1. Allocate a SRAM buffer**
141
+
142
+
SRAM device tree node is defined in SoC device tree. You can refer to it in
143
+
your board device tree to define your SRAM pool.
144
+
::
145
+
146
+
&sram {
147
+
my_foo_device_dma_pool: dma-sram@0 {
148
+
reg = <0x0 0x1000>;
149
+
};
150
+
};
151
+
152
+
Be careful of the start index, in case there are other SRAM consumers.
153
+
Define your pool size strategically: to optimise chaining, the idea is that
154
+
STM32 DMA and STM32 MDMA can work simultaneously, on each buffer of the
155
+
SRAM.
156
+
If the SRAM period is greater than the expected DMA transfer, then STM32 DMA
157
+
and STM32 MDMA will work sequentially instead of simultaneously. It is not a
158
+
functional issue but it is not optimal.
159
+
160
+
Don't forget to refer to your SRAM pool in your device node. You need to
161
+
define a new property.
162
+
::
163
+
164
+
&my_foo_device {
165
+
...
166
+
my_dma_pool = &my_foo_device_dma_pool;
167
+
};
168
+
169
+
Then get this SRAM pool in your foo driver and allocate your SRAM buffer.
170
+
171
+
**2. Allocate a STM32 DMA channel and a STM32 MDMA channel**
172
+
173
+
You need to define an extra channel in your device tree node, in addition to
174
+
the one you should already have for "classic" DMA operation.
175
+
176
+
This new channel must be taken from STM32 MDMA channels, so, the phandle of
177
+
the DMA controller to use is the MDMA controller's one.
178
+
::
179
+
180
+
&my_foo_device {
181
+
[...]
182
+
my_dma_pool = &my_foo_device_dma_pool;
183
+
dmas = <&dmamux1 ...>, // STM32 DMA channel
184
+
<&mdma1 0 0x3 0x1200000a 0 0>; // + STM32 MDMA channel
185
+
};
186
+
187
+
Concerning STM32 MDMA bindings:
188
+
189
+
1. The request line number : whatever the value here, it will be overwritten
190
+
by MDMA driver with the STM32 DMAMUX channel ID passed through
191
+
(struct dma_slave_config).peripheral_config
192
+
193
+
2. The priority level : choose Very High (0x3) so that your channel will
194
+
take priority other the other during request arbitration
195
+
196
+
3. A 32bit mask specifying the DMA channel configuration : source and
197
+
destination address increment, block transfer with 128 bytes per single
198
+
transfer
199
+
200
+
4. The 32bit value specifying the register to be used to acknowledge the
201
+
request: it will be overwritten by MDMA driver, with the DMA channel
202
+
interrupt flag clear register address passed through
203
+
(struct dma_slave_config).peripheral_config
204
+
205
+
5. The 32bit mask specifying the value to be written to acknowledge the
206
+
request: it will be overwritten by MDMA driver, with the DMA channel
207
+
Transfer Complete flag passed through
208
+
(struct dma_slave_config).peripheral_config
209
+
210
+
Driver updates for STM32 DMA-MDMA chaining support in foo driver
211
+
----------------------------------------------------------------
212
+
213
+
**0. (optional) Refactor the original sg_table if dmaengine_prep_slave_sg()**
214
+
215
+
In case of dmaengine_prep_slave_sg(), the original sg_table can't be used as
216
+
is. Two new sg_tables must be created from the original one. One for
217
+
STM32 DMA transfer (where memory address targets now the SRAM buffer instead
218
+
of DDR buffer) and one for STM32 MDMA transfer (where memory address targets
219
+
the DDR buffer).
220
+
221
+
The new sg_list items must fit SRAM period length. Here is an example for
222
+
DMA_DEV_TO_MEM:
223
+
::
224
+
225
+
/*
226
+
* Assuming sgl and nents, respectively the initial scatterlist and its
227
+
* length.
228
+
* Assuming sram_dma_buf and sram_period, respectively the memory
229
+
* allocated from the pool for DMA usage, and the length of the period,
230
+
* which is half of the sram_buf size.
231
+
*/
232
+
struct sg_table new_dma_sgt, new_mdma_sgt;
233
+
struct scatterlist *s, *_sgl;
234
+
dma_addr_t ddr_dma_buf;
235
+
u32 new_nents = 0, len;
236
+
int i;
237
+
238
+
/* Count the number of entries needed */
239
+
for_each_sg(sgl, s, nents, i)
240
+
if (sg_dma_len(s) > sram_period)
241
+
new_nents += DIV_ROUND_UP(sg_dma_len(s), sram_period);
242
+
else
243
+
new_nents++;
244
+
245
+
/* Create sg table for STM32 DMA channel */
246
+
ret = sg_alloc_table(&new_dma_sgt, new_nents, GFP_ATOMIC);
247
+
if (ret)
248
+
dev_err(dev, "DMA sg table alloc failed\n");
249
+
250
+
for_each_sg(new_dma_sgt.sgl, s, new_dma_sgt.nents, i) {
251
+
_sgl = sgl;
252
+
sg_dma_len(s) = min(sg_dma_len(_sgl), sram_period);
253
+
/* Targets the beginning = first half of the sram_buf */
254
+
s->dma_address = sram_buf;
255
+
/*
256
+
* Targets the second half of the sram_buf
257
+
* for odd indexes of the item of the sg_list
258
+
*/
259
+
if (i & 1)
260
+
s->dma_address += sram_period;
261
+
}
262
+
263
+
/* Create sg table for STM32 MDMA channel */
264
+
ret = sg_alloc_table(&new_mdma_sgt, new_nents, GFP_ATOMIC);
265
+
if (ret)
266
+
dev_err(dev, "MDMA sg_table alloc failed\n");
267
+
268
+
_sgl = sgl;
269
+
len = sg_dma_len(sgl);
270
+
ddr_dma_buf = sg_dma_address(sgl);
271
+
for_each_sg(mdma_sgt.sgl, s, mdma_sgt.nents, i) {
272
+
size_t bytes = min_t(size_t, len, sram_period);
273
+
274
+
sg_dma_len(s) = bytes;
275
+
sg_dma_address(s) = ddr_dma_buf;
276
+
len -= bytes;
277
+
278
+
if (!len && sg_next(_sgl)) {
279
+
_sgl = sg_next(_sgl);
280
+
len = sg_dma_len(_sgl);
281
+
ddr_dma_buf = sg_dma_address(_sgl);
282
+
} else {
283
+
ddr_dma_buf += bytes;
284
+
}
285
+
}
286
+
287
+
Don't forget to release these new sg_tables after getting the descriptors
288
+
with dmaengine_prep_slave_sg().
289
+
290
+
**1. Set controller specific parameters**
291
+
292
+
First, use dmaengine_slave_config() with a struct dma_slave_config to
293
+
configure STM32 DMA channel. You just have to take care of DMA addresses,
294
+
the memory address (depending on the transfer direction) must point on your
295
+
SRAM buffer, and set (struct dma_slave_config).peripheral_size != 0.
296
+
297
+
STM32 DMA driver will check (struct dma_slave_config).peripheral_size to
298
+
determine if chaining is being used or not. If it is used, then STM32 DMA
299
+
driver fills (struct dma_slave_config).peripheral_config with an array of
300
+
three u32 : the first one containing STM32 DMAMUX channel ID, the second one
301
+
the channel interrupt flag clear register address, and the third one the
302
+
channel Transfer Complete flag mask.
303
+
304
+
Then, use dmaengine_slave_config with another struct dma_slave_config to
305
+
configure STM32 MDMA channel. Take care of DMA addresses, the device address
306
+
(depending on the transfer direction) must point on your SRAM buffer, and
307
+
the memory address must point to the buffer originally used for "classic"
308
+
DMA operation. Use the previous (struct dma_slave_config).peripheral_size
309
+
and .peripheral_config that have been updated by STM32 DMA driver, to set
310
+
(struct dma_slave_config).peripheral_size and .peripheral_config of the
311
+
struct dma_slave_config to configure STM32 MDMA channel.
312
+
::
313
+
314
+
struct dma_slave_config dma_conf;
315
+
struct dma_slave_config mdma_conf;
316
+
317
+
memset(&dma_conf, 0, sizeof(dma_conf));
318
+
[...]
319
+
config.direction = DMA_DEV_TO_MEM;
320
+
config.dst_addr = sram_dma_buf; // SRAM buffer
321
+
config.peripheral_size = 1; // peripheral_size != 0 => chaining
322
+
323
+
dmaengine_slave_config(dma_chan, &dma_config);
324
+
325
+
memset(&mdma_conf, 0, sizeof(mdma_conf));
326
+
config.direction = DMA_DEV_TO_MEM;
327
+
mdma_conf.src_addr = sram_dma_buf; // SRAM buffer
328
+
mdma_conf.dst_addr = rx_dma_buf; // original memory buffer
329
+
mdma_conf.peripheral_size = dma_conf.peripheral_size; // <- dma_conf
330
+
mdma_conf.peripheral_config = dma_config.peripheral_config; // <- dma_conf
331
+
332
+
dmaengine_slave_config(mdma_chan, &mdma_conf);
333
+
334
+
**2. Get a descriptor for STM32 DMA channel transaction**
335
+
336
+
In the same way you get your descriptor for your "classic" DMA operation,
337
+
you just have to replace the original sg_list (in case of
338
+
dmaengine_prep_slave_sg()) with the new sg_list using SRAM buffer, or to
339
+
replace the original buffer address, length and period (in case of
340
+
dmaengine_prep_dma_cyclic()) with the new SRAM buffer.
341
+
342
+
**3. Get a descriptor for STM32 MDMA channel transaction**
343
+
344
+
If you previously get descriptor (for STM32 DMA) with
345
+
346
+
* dmaengine_prep_slave_sg(), then use dmaengine_prep_slave_sg() for
347
+
STM32 MDMA;
348
+
* dmaengine_prep_dma_cyclic(), then use dmaengine_prep_dma_cyclic() for
349
+
STM32 MDMA.
350
+
351
+
Use the new sg_list using SRAM buffer (in case of dmaengine_prep_slave_sg())
352
+
or, depending on the transfer direction, either the original DDR buffer (in
353
+
case of DMA_DEV_TO_MEM) or the SRAM buffer (in case of DMA_MEM_TO_DEV), the
354
+
source address being previously set with dmaengine_slave_config().
355
+
356
+
**4. Submit both transactions**
357
+
358
+
Before submitting your transactions, you may need to define on which
359
+
descriptor you want a callback to be called at the end of the transfer
360
+
(dmaengine_prep_slave_sg()) or the period (dmaengine_prep_dma_cyclic()).
361
+
Depending on the direction, set the callback on the descriptor that finishes
362
+
the overal transfer:
363
+
364
+
* DMA_DEV_TO_MEM: set the callback on the "MDMA" descriptor
365
+
* DMA_MEM_TO_DEV: set the callback on the "DMA" descriptor
366
+
367
+
Then, submit the descriptors whatever the order, with dmaengine_tx_submit().
368
+
369
+
**5. Issue pending requests (and wait for callback notification)**
370
+
371
+
As STM32 MDMA channel transfer is triggered by STM32 DMA, you must issue
372
+
STM32 MDMA channel before STM32 DMA channel.
373
+
374
+
If any, your callback will be called to warn you about the end of the overal
375
+
transfer or the period completion.
376
+
377
+
Don't forget to terminate both channels. STM32 DMA channel is configured in
378
+
cyclic Double-Buffer mode so it won't be disabled by HW, you need to terminate
379
+
it. STM32 MDMA channel will be stopped by HW in case of sg transfer, but not
380
+
in case of cyclic transfer. You can terminate it whatever the kind of transfer.
381
+
382
+
**STM32 DMA-MDMA chaining DMA_MEM_TO_DEV special case**
383
+
384
+
STM32 DMA-MDMA chaining in DMA_MEM_TO_DEV is a special case. Indeed, the
385
+
STM32 MDMA feeds the SRAM buffer with the DDR data, and the STM32 DMA reads
386
+
data from SRAM buffer. So some data (the first period) have to be copied in
387
+
SRAM buffer when the STM32 DMA starts to read.
388
+
389
+
A trick could be pausing the STM32 DMA channel (that will raise a Transfer
390
+
Complete signal, triggering the STM32 MDMA channel), but the first data read
391
+
by the STM32 DMA could be "wrong". The proper way is to prepare the first SRAM
392
+
period with dmaengine_prep_dma_memcpy(). Then this first period should be
393
+
"removed" from the sg or the cyclic transfer.
394
+
395
+
Due to this complexity, rather use the STM32 DMA-MDMA chaining for
396
+
DMA_DEV_TO_MEM and keep the "classic" DMA usage for DMA_MEM_TO_DEV, unless
397
+
you're not afraid.
398
+
399
+
Resources
400
+
---------
401
+
402
+
Application note, datasheet and reference manual are available on ST website
403
+
(STM32MP1_).
404
+
405
+
Dedicated focus on three application notes (AN5224_, AN4031_ & AN5001_)
406
+
dealing with STM32 DMAMUX, STM32 DMA and STM32 MDMA.
407
+
408
+
.. _STM32MP1: https://www.st.com/en/microcontrollers-microprocessors/stm32mp1-series.html
409
+
.. _AN5224: https://www.st.com/resource/en/application_note/an5224-stm32-dmamux-the-dma-request-router-stmicroelectronics.pdf
410
+
.. _AN4031: https://www.st.com/resource/en/application_note/dm00046011-using-the-stm32f2-stm32f4-and-stm32f7-series-dma-controller-stmicroelectronics.pdf
411
+
.. _AN5001: https://www.st.com/resource/en/application_note/an5001-stm32cube-expansion-package-for-stm32h7-series-mdma-stmicroelectronics.pdf
412
+
413
+
:Authors:
414
+
415
+
- Amelie Delaunay <amelie.delaunay@foss.st.com>
+7
Documentation/devicetree/bindings/dma/apple,admac.yaml
+7
Documentation/devicetree/bindings/dma/apple,admac.yaml
+6
Documentation/devicetree/bindings/dma/arm,pl330.yaml
+6
Documentation/devicetree/bindings/dma/arm,pl330.yaml
+1
Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml
+1
Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml
+99
Documentation/devicetree/bindings/dma/qcom,adm.yaml
+99
Documentation/devicetree/bindings/dma/qcom,adm.yaml
···
1
+
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2
+
%YAML 1.2
3
+
---
4
+
$id: http://devicetree.org/schemas/dma/qcom,adm.yaml#
5
+
$schema: http://devicetree.org/meta-schemas/core.yaml#
6
+
7
+
title: Qualcomm ADM DMA Controller
8
+
9
+
maintainers:
10
+
- Christian Marangi <ansuelsmth@gmail.com>
11
+
- Bjorn Andersson <bjorn.andersson@linaro.org>
12
+
13
+
description: |
14
+
QCOM ADM DMA controller provides DMA capabilities for
15
+
peripheral buses such as NAND and SPI.
16
+
17
+
properties:
18
+
compatible:
19
+
const: qcom,adm
20
+
21
+
reg:
22
+
maxItems: 1
23
+
24
+
interrupts:
25
+
maxItems: 1
26
+
27
+
"#dma-cells":
28
+
const: 1
29
+
30
+
clocks:
31
+
items:
32
+
- description: phandle to the core clock
33
+
- description: phandle to the iface clock
34
+
35
+
clock-names:
36
+
items:
37
+
- const: core
38
+
- const: iface
39
+
40
+
resets:
41
+
items:
42
+
- description: phandle to the clk reset
43
+
- description: phandle to the pbus reset
44
+
- description: phandle to the c0 reset
45
+
- description: phandle to the c1 reset
46
+
- description: phandle to the c2 reset
47
+
48
+
reset-names:
49
+
items:
50
+
- const: clk
51
+
- const: pbus
52
+
- const: c0
53
+
- const: c1
54
+
- const: c2
55
+
56
+
qcom,ee:
57
+
$ref: /schemas/types.yaml#/definitions/uint32
58
+
description: indicates the security domain identifier used in the secure world.
59
+
minimum: 0
60
+
maximum: 255
61
+
62
+
required:
63
+
- compatible
64
+
- reg
65
+
- interrupts
66
+
- "#dma-cells"
67
+
- clocks
68
+
- clock-names
69
+
- resets
70
+
- reset-names
71
+
- qcom,ee
72
+
73
+
additionalProperties: false
74
+
75
+
examples:
76
+
- |
77
+
#include <dt-bindings/clock/qcom,gcc-ipq806x.h>
78
+
#include <dt-bindings/reset/qcom,gcc-ipq806x.h>
79
+
80
+
adm_dma: dma-controller@18300000 {
81
+
compatible = "qcom,adm";
82
+
reg = <0x18300000 0x100000>;
83
+
interrupts = <0 170 0>;
84
+
#dma-cells = <1>;
85
+
86
+
clocks = <&gcc ADM0_CLK>,
87
+
<&gcc ADM0_PBUS_CLK>;
88
+
clock-names = "core", "iface";
89
+
90
+
resets = <&gcc ADM0_RESET>,
91
+
<&gcc ADM0_PBUS_RESET>,
92
+
<&gcc ADM0_C0_RESET>,
93
+
<&gcc ADM0_C1_RESET>,
94
+
<&gcc ADM0_C2_RESET>;
95
+
reset-names = "clk", "pbus", "c0", "c1", "c2";
96
+
qcom,ee = <0>;
97
+
};
98
+
99
+
...
+4
-4
Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml
+4
-4
Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml
···
8
8
9
9
maintainers:
10
10
- Andy Gross <agross@kernel.org>
11
-
- Bjorn Andersson <bjorn.andersson@linaro.org>
11
+
- Bjorn Andersson <andersson@kernel.org>
12
12
13
13
allOf:
14
14
- $ref: "dma-controller.yaml#"
···
20
20
- qcom,bam-v1.3.0
21
21
# MSM8974, APQ8074 and APQ8084
22
22
- qcom,bam-v1.4.0
23
-
# MSM8916
23
+
# MSM8916 and SDM845
24
24
- qcom,bam-v1.7.0
25
25
26
26
clocks:
···
90
90
91
91
dma-controller@f9944000 {
92
92
compatible = "qcom,bam-v1.4.0";
93
-
reg = <0xf9944000 0x15000>;
94
-
interrupts = <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>;
93
+
reg = <0xf9944000 0x19000>;
94
+
interrupts = <GIC_SPI 239 IRQ_TYPE_LEVEL_HIGH>;
95
95
clocks = <&gcc GCC_BLSP2_AHB_CLK>;
96
96
clock-names = "bam_clk";
97
97
#dma-cells = <1>;
+1
Documentation/devicetree/bindings/dma/qcom,gpi.yaml
+1
Documentation/devicetree/bindings/dma/qcom,gpi.yaml
-61
Documentation/devicetree/bindings/dma/qcom_adm.txt
-61
Documentation/devicetree/bindings/dma/qcom_adm.txt
···
1
-
QCOM ADM DMA Controller
2
-
3
-
Required properties:
4
-
- compatible: must contain "qcom,adm" for IPQ/APQ8064 and MSM8960
5
-
- reg: Address range for DMA registers
6
-
- interrupts: Should contain one interrupt shared by all channels
7
-
- #dma-cells: must be <2>. First cell denotes the channel number. Second cell
8
-
denotes CRCI (client rate control interface) flow control assignment.
9
-
- clocks: Should contain the core clock and interface clock.
10
-
- clock-names: Must contain "core" for the core clock and "iface" for the
11
-
interface clock.
12
-
- resets: Must contain an entry for each entry in reset names.
13
-
- reset-names: Must include the following entries:
14
-
- clk
15
-
- c0
16
-
- c1
17
-
- c2
18
-
- qcom,ee: indicates the security domain identifier used in the secure world.
19
-
20
-
Example:
21
-
adm_dma: dma@18300000 {
22
-
compatible = "qcom,adm";
23
-
reg = <0x18300000 0x100000>;
24
-
interrupts = <0 170 0>;
25
-
#dma-cells = <2>;
26
-
27
-
clocks = <&gcc ADM0_CLK>, <&gcc ADM0_PBUS_CLK>;
28
-
clock-names = "core", "iface";
29
-
30
-
resets = <&gcc ADM0_RESET>,
31
-
<&gcc ADM0_C0_RESET>,
32
-
<&gcc ADM0_C1_RESET>,
33
-
<&gcc ADM0_C2_RESET>;
34
-
reset-names = "clk", "c0", "c1", "c2";
35
-
qcom,ee = <0>;
36
-
};
37
-
38
-
DMA clients must use the format descripted in the dma.txt file, using a three
39
-
cell specifier for each channel.
40
-
41
-
Each dmas request consists of 3 cells:
42
-
1. phandle pointing to the DMA controller
43
-
2. channel number
44
-
3. CRCI assignment, if applicable. If no CRCI flow control is required, use 0.
45
-
The CRCI is used for flow control. It identifies the peripheral device that
46
-
is the source/destination for the transferred data.
47
-
48
-
Example:
49
-
50
-
spi4: spi@1a280000 {
51
-
spi-max-frequency = <50000000>;
52
-
53
-
pinctrl-0 = <&spi_pins>;
54
-
pinctrl-names = "default";
55
-
56
-
cs-gpios = <&qcom_pinmux 20 0>;
57
-
58
-
dmas = <&adm_dma 6 9>,
59
-
<&adm_dma 5 10>;
60
-
dma-names = "rx", "tx";
61
-
};
+1
Documentation/devicetree/bindings/dma/renesas,rcar-dmac.yaml
+1
Documentation/devicetree/bindings/dma/renesas,rcar-dmac.yaml
+1
-1
Documentation/devicetree/bindings/dma/ti-dma-crossbar.txt
+1
-1
Documentation/devicetree/bindings/dma/ti-dma-crossbar.txt
···
4
4
- compatible: "ti,dra7-dma-crossbar" for DRA7xx DMA crossbar
5
5
"ti,am335x-edma-crossbar" for AM335x and AM437x
6
6
- reg: Memory map for accessing module
7
-
- #dma-cells: Should be set to to match with the DMA controller's dma-cells
7
+
- #dma-cells: Should be set to match with the DMA controller's dma-cells
8
8
for ti,dra7-dma-crossbar and <3> for ti,am335x-edma-crossbar.
9
9
- dma-requests: Number of DMA requests the crossbar can receive
10
10
- dma-masters: phandle pointing to the DMA controller
+1
MAINTAINERS
+1
MAINTAINERS
+1
-1
drivers/dma/Kconfig
+1
-1
drivers/dma/Kconfig
+1
-1
drivers/dma/amba-pl08x.c
+1
-1
drivers/dma/amba-pl08x.c
···
2367
2367
INIT_LIST_HEAD(&dmadev->channels);
2368
2368
2369
2369
/*
2370
-
* Register as many many memcpy as we have physical channels,
2370
+
* Register as many memcpy as we have physical channels,
2371
2371
* we won't always be able to use all but the code will have
2372
2372
* to cope with that situation.
2373
2373
*/
+35
-10
drivers/dma/apple-admac.c
+35
-10
drivers/dma/apple-admac.c
···
12
12
#include <linux/module.h>
13
13
#include <linux/of_device.h>
14
14
#include <linux/of_dma.h>
15
-
#include <linux/interrupt.h>
15
+
#include <linux/reset.h>
16
16
#include <linux/spinlock.h>
17
+
#include <linux/interrupt.h>
17
18
18
19
#include "dmaengine.h"
19
20
···
96
95
struct dma_device dma;
97
96
struct device *dev;
98
97
__iomem void *base;
98
+
struct reset_control *rstc;
99
99
100
+
int irq;
100
101
int irq_index;
101
102
int nchannels;
102
103
struct admac_chan channels[];
···
727
724
728
725
if (irq < 0)
729
726
return dev_err_probe(&pdev->dev, irq, "no usable interrupt\n");
730
-
731
-
err = devm_request_irq(&pdev->dev, irq, admac_interrupt,
732
-
0, dev_name(&pdev->dev), ad);
733
-
if (err)
734
-
return dev_err_probe(&pdev->dev, err,
735
-
"unable to register interrupt\n");
727
+
ad->irq = irq;
736
728
737
729
ad->base = devm_platform_ioremap_resource(pdev, 0);
738
730
if (IS_ERR(ad->base))
739
731
return dev_err_probe(&pdev->dev, PTR_ERR(ad->base),
740
732
"unable to obtain MMIO resource\n");
733
+
734
+
ad->rstc = devm_reset_control_get_optional_shared(&pdev->dev, NULL);
735
+
if (IS_ERR(ad->rstc))
736
+
return PTR_ERR(ad->rstc);
741
737
742
738
dma = &ad->dma;
743
739
···
776
774
tasklet_setup(&adchan->tasklet, admac_chan_tasklet);
777
775
}
778
776
779
-
err = dma_async_device_register(&ad->dma);
777
+
err = reset_control_reset(ad->rstc);
780
778
if (err)
781
-
return dev_err_probe(&pdev->dev, err, "failed to register DMA device\n");
779
+
return dev_err_probe(&pdev->dev, err,
780
+
"unable to trigger reset\n");
781
+
782
+
err = request_irq(irq, admac_interrupt, 0, dev_name(&pdev->dev), ad);
783
+
if (err) {
784
+
dev_err_probe(&pdev->dev, err,
785
+
"unable to register interrupt\n");
786
+
goto free_reset;
787
+
}
788
+
789
+
err = dma_async_device_register(&ad->dma);
790
+
if (err) {
791
+
dev_err_probe(&pdev->dev, err, "failed to register DMA device\n");
792
+
goto free_irq;
793
+
}
782
794
783
795
err = of_dma_controller_register(pdev->dev.of_node, admac_dma_of_xlate, ad);
784
796
if (err) {
785
797
dma_async_device_unregister(&ad->dma);
786
-
return dev_err_probe(&pdev->dev, err, "failed to register with OF\n");
798
+
dev_err_probe(&pdev->dev, err, "failed to register with OF\n");
799
+
goto free_irq;
787
800
}
788
801
789
802
return 0;
803
+
804
+
free_irq:
805
+
free_irq(ad->irq, ad);
806
+
free_reset:
807
+
reset_control_rearm(ad->rstc);
808
+
return err;
790
809
}
791
810
792
811
static int admac_remove(struct platform_device *pdev)
···
816
793
817
794
of_dma_controller_free(pdev->dev.of_node);
818
795
dma_async_device_unregister(&ad->dma);
796
+
free_irq(ad->irq, ad);
797
+
reset_control_rearm(ad->rstc);
819
798
820
799
return 0;
821
800
}
+1
-4
drivers/dma/at_xdmac.c
+1
-4
drivers/dma/at_xdmac.c
-12
drivers/dma/dw-edma/dw-edma-core.c
-12
drivers/dma/dw-edma/dw-edma-core.c
···
9
9
#include <linux/module.h>
10
10
#include <linux/device.h>
11
11
#include <linux/kernel.h>
12
-
#include <linux/pm_runtime.h>
13
12
#include <linux/dmaengine.h>
14
13
#include <linux/err.h>
15
14
#include <linux/interrupt.h>
···
681
682
if (chan->status != EDMA_ST_IDLE)
682
683
return -EBUSY;
683
684
684
-
pm_runtime_get(chan->dw->chip->dev);
685
-
686
685
return 0;
687
686
}
688
687
689
688
static void dw_edma_free_chan_resources(struct dma_chan *dchan)
690
689
{
691
690
unsigned long timeout = jiffies + msecs_to_jiffies(5000);
692
-
struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
693
691
int ret;
694
692
695
693
while (time_before(jiffies, timeout)) {
···
699
703
700
704
cpu_relax();
701
705
}
702
-
703
-
pm_runtime_put(chan->dw->chip->dev);
704
706
}
705
707
706
708
static int dw_edma_channel_setup(struct dw_edma *dw, bool write,
···
971
977
if (err)
972
978
goto err_irq_free;
973
979
974
-
/* Power management */
975
-
pm_runtime_enable(dev);
976
-
977
980
/* Turn debugfs on */
978
981
dw_edma_v0_core_debugfs_on(dw);
979
982
···
999
1008
/* Free irqs */
1000
1009
for (i = (dw->nr_irqs - 1); i >= 0; i--)
1001
1010
free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]);
1002
-
1003
-
/* Power management */
1004
-
pm_runtime_disable(dev);
1005
1011
1006
1012
/* Deregister eDMA device */
1007
1013
dma_async_device_unregister(&dw->wr_edma);
+551
-93
drivers/dma/hisi_dma.c
+551
-93
drivers/dma/hisi_dma.c
···
1
1
// SPDX-License-Identifier: GPL-2.0-only
2
-
/* Copyright(c) 2019 HiSilicon Limited. */
2
+
/* Copyright(c) 2019-2022 HiSilicon Limited. */
3
+
3
4
#include <linux/bitfield.h>
4
5
#include <linux/dmaengine.h>
5
6
#include <linux/init.h>
···
10
9
#include <linux/spinlock.h>
11
10
#include "virt-dma.h"
12
11
13
-
#define HISI_DMA_SQ_BASE_L 0x0
14
-
#define HISI_DMA_SQ_BASE_H 0x4
15
-
#define HISI_DMA_SQ_DEPTH 0x8
16
-
#define HISI_DMA_SQ_TAIL_PTR 0xc
17
-
#define HISI_DMA_CQ_BASE_L 0x10
18
-
#define HISI_DMA_CQ_BASE_H 0x14
19
-
#define HISI_DMA_CQ_DEPTH 0x18
20
-
#define HISI_DMA_CQ_HEAD_PTR 0x1c
21
-
#define HISI_DMA_CTRL0 0x20
22
-
#define HISI_DMA_CTRL0_QUEUE_EN_S 0
23
-
#define HISI_DMA_CTRL0_QUEUE_PAUSE_S 4
24
-
#define HISI_DMA_CTRL1 0x24
25
-
#define HISI_DMA_CTRL1_QUEUE_RESET_S 0
26
-
#define HISI_DMA_Q_FSM_STS 0x30
27
-
#define HISI_DMA_FSM_STS_MASK GENMASK(3, 0)
28
-
#define HISI_DMA_INT_STS 0x40
29
-
#define HISI_DMA_INT_STS_MASK GENMASK(12, 0)
30
-
#define HISI_DMA_INT_MSK 0x44
31
-
#define HISI_DMA_MODE 0x217c
32
-
#define HISI_DMA_OFFSET 0x100
12
+
/* HiSilicon DMA register common field define */
13
+
#define HISI_DMA_Q_SQ_BASE_L 0x0
14
+
#define HISI_DMA_Q_SQ_BASE_H 0x4
15
+
#define HISI_DMA_Q_SQ_DEPTH 0x8
16
+
#define HISI_DMA_Q_SQ_TAIL_PTR 0xc
17
+
#define HISI_DMA_Q_CQ_BASE_L 0x10
18
+
#define HISI_DMA_Q_CQ_BASE_H 0x14
19
+
#define HISI_DMA_Q_CQ_DEPTH 0x18
20
+
#define HISI_DMA_Q_CQ_HEAD_PTR 0x1c
21
+
#define HISI_DMA_Q_CTRL0 0x20
22
+
#define HISI_DMA_Q_CTRL0_QUEUE_EN BIT(0)
23
+
#define HISI_DMA_Q_CTRL0_QUEUE_PAUSE BIT(4)
24
+
#define HISI_DMA_Q_CTRL1 0x24
25
+
#define HISI_DMA_Q_CTRL1_QUEUE_RESET BIT(0)
26
+
#define HISI_DMA_Q_FSM_STS 0x30
27
+
#define HISI_DMA_Q_FSM_STS_MASK GENMASK(3, 0)
28
+
#define HISI_DMA_Q_ERR_INT_NUM0 0x84
29
+
#define HISI_DMA_Q_ERR_INT_NUM1 0x88
30
+
#define HISI_DMA_Q_ERR_INT_NUM2 0x8c
33
31
34
-
#define HISI_DMA_MSI_NUM 32
35
-
#define HISI_DMA_CHAN_NUM 30
36
-
#define HISI_DMA_Q_DEPTH_VAL 1024
32
+
/* HiSilicon IP08 DMA register and field define */
33
+
#define HISI_DMA_HIP08_MODE 0x217C
34
+
#define HISI_DMA_HIP08_Q_BASE 0x0
35
+
#define HISI_DMA_HIP08_Q_CTRL0_ERR_ABORT_EN BIT(2)
36
+
#define HISI_DMA_HIP08_Q_INT_STS 0x40
37
+
#define HISI_DMA_HIP08_Q_INT_MSK 0x44
38
+
#define HISI_DMA_HIP08_Q_INT_STS_MASK GENMASK(14, 0)
39
+
#define HISI_DMA_HIP08_Q_ERR_INT_NUM3 0x90
40
+
#define HISI_DMA_HIP08_Q_ERR_INT_NUM4 0x94
41
+
#define HISI_DMA_HIP08_Q_ERR_INT_NUM5 0x98
42
+
#define HISI_DMA_HIP08_Q_ERR_INT_NUM6 0x48
43
+
#define HISI_DMA_HIP08_Q_CTRL0_SQCQ_DRCT BIT(24)
37
44
38
-
#define PCI_BAR_2 2
45
+
/* HiSilicon IP09 DMA register and field define */
46
+
#define HISI_DMA_HIP09_DMA_FLR_DISABLE 0xA00
47
+
#define HISI_DMA_HIP09_DMA_FLR_DISABLE_B BIT(0)
48
+
#define HISI_DMA_HIP09_Q_BASE 0x2000
49
+
#define HISI_DMA_HIP09_Q_CTRL0_ERR_ABORT_EN GENMASK(31, 28)
50
+
#define HISI_DMA_HIP09_Q_CTRL0_SQ_DRCT BIT(26)
51
+
#define HISI_DMA_HIP09_Q_CTRL0_CQ_DRCT BIT(27)
52
+
#define HISI_DMA_HIP09_Q_CTRL1_VA_ENABLE BIT(2)
53
+
#define HISI_DMA_HIP09_Q_INT_STS 0x40
54
+
#define HISI_DMA_HIP09_Q_INT_MSK 0x44
55
+
#define HISI_DMA_HIP09_Q_INT_STS_MASK 0x1
56
+
#define HISI_DMA_HIP09_Q_ERR_INT_STS 0x48
57
+
#define HISI_DMA_HIP09_Q_ERR_INT_MSK 0x4C
58
+
#define HISI_DMA_HIP09_Q_ERR_INT_STS_MASK GENMASK(18, 1)
59
+
#define HISI_DMA_HIP09_PORT_CFG_REG(port_id) (0x800 + \
60
+
(port_id) * 0x20)
61
+
#define HISI_DMA_HIP09_PORT_CFG_LINK_DOWN_MASK_B BIT(16)
62
+
63
+
#define HISI_DMA_HIP09_MAX_PORT_NUM 16
64
+
65
+
#define HISI_DMA_HIP08_MSI_NUM 32
66
+
#define HISI_DMA_HIP08_CHAN_NUM 30
67
+
#define HISI_DMA_HIP09_MSI_NUM 4
68
+
#define HISI_DMA_HIP09_CHAN_NUM 4
69
+
#define HISI_DMA_REVISION_HIP08B 0x21
70
+
#define HISI_DMA_REVISION_HIP09A 0x30
71
+
72
+
#define HISI_DMA_Q_OFFSET 0x100
73
+
#define HISI_DMA_Q_DEPTH_VAL 1024
74
+
75
+
#define PCI_BAR_2 2
76
+
77
+
#define HISI_DMA_POLL_Q_STS_DELAY_US 10
78
+
#define HISI_DMA_POLL_Q_STS_TIME_OUT_US 1000
79
+
80
+
#define HISI_DMA_MAX_DIR_NAME_LEN 128
81
+
82
+
/*
83
+
* The HIP08B(HiSilicon IP08) and HIP09A(HiSilicon IP09) are DMA iEPs, they
84
+
* have the same pci device id but different pci revision.
85
+
* Unfortunately, they have different register layouts, so two layout
86
+
* enumerations are defined.
87
+
*/
88
+
enum hisi_dma_reg_layout {
89
+
HISI_DMA_REG_LAYOUT_INVALID = 0,
90
+
HISI_DMA_REG_LAYOUT_HIP08,
91
+
HISI_DMA_REG_LAYOUT_HIP09
92
+
};
39
93
40
94
enum hisi_dma_mode {
41
95
EP = 0,
···
161
105
struct dma_device dma_dev;
162
106
u32 chan_num;
163
107
u32 chan_depth;
108
+
enum hisi_dma_reg_layout reg_layout;
109
+
void __iomem *queue_base; /* queue region start of register */
164
110
struct hisi_dma_chan chan[];
165
111
};
112
+
113
+
#ifdef CONFIG_DEBUG_FS
114
+
115
+
static const struct debugfs_reg32 hisi_dma_comm_chan_regs[] = {
116
+
{"DMA_QUEUE_SQ_DEPTH ", 0x0008ull},
117
+
{"DMA_QUEUE_SQ_TAIL_PTR ", 0x000Cull},
118
+
{"DMA_QUEUE_CQ_DEPTH ", 0x0018ull},
119
+
{"DMA_QUEUE_CQ_HEAD_PTR ", 0x001Cull},
120
+
{"DMA_QUEUE_CTRL0 ", 0x0020ull},
121
+
{"DMA_QUEUE_CTRL1 ", 0x0024ull},
122
+
{"DMA_QUEUE_FSM_STS ", 0x0030ull},
123
+
{"DMA_QUEUE_SQ_STS ", 0x0034ull},
124
+
{"DMA_QUEUE_CQ_TAIL_PTR ", 0x003Cull},
125
+
{"DMA_QUEUE_INT_STS ", 0x0040ull},
126
+
{"DMA_QUEUE_INT_MSK ", 0x0044ull},
127
+
{"DMA_QUEUE_INT_RO ", 0x006Cull},
128
+
};
129
+
130
+
static const struct debugfs_reg32 hisi_dma_hip08_chan_regs[] = {
131
+
{"DMA_QUEUE_BYTE_CNT ", 0x0038ull},
132
+
{"DMA_ERR_INT_NUM6 ", 0x0048ull},
133
+
{"DMA_QUEUE_DESP0 ", 0x0050ull},
134
+
{"DMA_QUEUE_DESP1 ", 0x0054ull},
135
+
{"DMA_QUEUE_DESP2 ", 0x0058ull},
136
+
{"DMA_QUEUE_DESP3 ", 0x005Cull},
137
+
{"DMA_QUEUE_DESP4 ", 0x0074ull},
138
+
{"DMA_QUEUE_DESP5 ", 0x0078ull},
139
+
{"DMA_QUEUE_DESP6 ", 0x007Cull},
140
+
{"DMA_QUEUE_DESP7 ", 0x0080ull},
141
+
{"DMA_ERR_INT_NUM0 ", 0x0084ull},
142
+
{"DMA_ERR_INT_NUM1 ", 0x0088ull},
143
+
{"DMA_ERR_INT_NUM2 ", 0x008Cull},
144
+
{"DMA_ERR_INT_NUM3 ", 0x0090ull},
145
+
{"DMA_ERR_INT_NUM4 ", 0x0094ull},
146
+
{"DMA_ERR_INT_NUM5 ", 0x0098ull},
147
+
{"DMA_QUEUE_SQ_STS2 ", 0x00A4ull},
148
+
};
149
+
150
+
static const struct debugfs_reg32 hisi_dma_hip09_chan_regs[] = {
151
+
{"DMA_QUEUE_ERR_INT_STS ", 0x0048ull},
152
+
{"DMA_QUEUE_ERR_INT_MSK ", 0x004Cull},
153
+
{"DFX_SQ_READ_ERR_PTR ", 0x0068ull},
154
+
{"DFX_DMA_ERR_INT_NUM0 ", 0x0084ull},
155
+
{"DFX_DMA_ERR_INT_NUM1 ", 0x0088ull},
156
+
{"DFX_DMA_ERR_INT_NUM2 ", 0x008Cull},
157
+
{"DFX_DMA_QUEUE_SQ_STS2 ", 0x00A4ull},
158
+
};
159
+
160
+
static const struct debugfs_reg32 hisi_dma_hip08_comm_regs[] = {
161
+
{"DMA_ECC_ERR_ADDR ", 0x2004ull},
162
+
{"DMA_ECC_ECC_CNT ", 0x2014ull},
163
+
{"COMMON_AND_CH_ERR_STS ", 0x2030ull},
164
+
{"LOCAL_CPL_ID_STS_0 ", 0x20E0ull},
165
+
{"LOCAL_CPL_ID_STS_1 ", 0x20E4ull},
166
+
{"LOCAL_CPL_ID_STS_2 ", 0x20E8ull},
167
+
{"LOCAL_CPL_ID_STS_3 ", 0x20ECull},
168
+
{"LOCAL_TLP_NUM ", 0x2158ull},
169
+
{"SQCQ_TLP_NUM ", 0x2164ull},
170
+
{"CPL_NUM ", 0x2168ull},
171
+
{"INF_BACK_PRESS_STS ", 0x2170ull},
172
+
{"DMA_CH_RAS_LEVEL ", 0x2184ull},
173
+
{"DMA_CM_RAS_LEVEL ", 0x2188ull},
174
+
{"DMA_CH_ERR_STS ", 0x2190ull},
175
+
{"DMA_CH_DONE_STS ", 0x2194ull},
176
+
{"DMA_SQ_TAG_STS_0 ", 0x21A0ull},
177
+
{"DMA_SQ_TAG_STS_1 ", 0x21A4ull},
178
+
{"DMA_SQ_TAG_STS_2 ", 0x21A8ull},
179
+
{"DMA_SQ_TAG_STS_3 ", 0x21ACull},
180
+
{"LOCAL_P_ID_STS_0 ", 0x21B0ull},
181
+
{"LOCAL_P_ID_STS_1 ", 0x21B4ull},
182
+
{"LOCAL_P_ID_STS_2 ", 0x21B8ull},
183
+
{"LOCAL_P_ID_STS_3 ", 0x21BCull},
184
+
{"DMA_PREBUFF_INFO_0 ", 0x2200ull},
185
+
{"DMA_CM_TABLE_INFO_0 ", 0x2220ull},
186
+
{"DMA_CM_CE_RO ", 0x2244ull},
187
+
{"DMA_CM_NFE_RO ", 0x2248ull},
188
+
{"DMA_CM_FE_RO ", 0x224Cull},
189
+
};
190
+
191
+
static const struct debugfs_reg32 hisi_dma_hip09_comm_regs[] = {
192
+
{"COMMON_AND_CH_ERR_STS ", 0x0030ull},
193
+
{"DMA_PORT_IDLE_STS ", 0x0150ull},
194
+
{"DMA_CH_RAS_LEVEL ", 0x0184ull},
195
+
{"DMA_CM_RAS_LEVEL ", 0x0188ull},
196
+
{"DMA_CM_CE_RO ", 0x0244ull},
197
+
{"DMA_CM_NFE_RO ", 0x0248ull},
198
+
{"DMA_CM_FE_RO ", 0x024Cull},
199
+
{"DFX_INF_BACK_PRESS_STS0 ", 0x1A40ull},
200
+
{"DFX_INF_BACK_PRESS_STS1 ", 0x1A44ull},
201
+
{"DFX_INF_BACK_PRESS_STS2 ", 0x1A48ull},
202
+
{"DFX_DMA_WRR_DISABLE ", 0x1A4Cull},
203
+
{"DFX_PA_REQ_TLP_NUM ", 0x1C00ull},
204
+
{"DFX_PA_BACK_TLP_NUM ", 0x1C04ull},
205
+
{"DFX_PA_RETRY_TLP_NUM ", 0x1C08ull},
206
+
{"DFX_LOCAL_NP_TLP_NUM ", 0x1C0Cull},
207
+
{"DFX_LOCAL_CPL_HEAD_TLP_NUM ", 0x1C10ull},
208
+
{"DFX_LOCAL_CPL_DATA_TLP_NUM ", 0x1C14ull},
209
+
{"DFX_LOCAL_CPL_EXT_DATA_TLP_NUM ", 0x1C18ull},
210
+
{"DFX_LOCAL_P_HEAD_TLP_NUM ", 0x1C1Cull},
211
+
{"DFX_LOCAL_P_ACK_TLP_NUM ", 0x1C20ull},
212
+
{"DFX_BUF_ALOC_PORT_REQ_NUM ", 0x1C24ull},
213
+
{"DFX_BUF_ALOC_PORT_RESULT_NUM ", 0x1C28ull},
214
+
{"DFX_BUF_FAIL_SIZE_NUM ", 0x1C2Cull},
215
+
{"DFX_BUF_ALOC_SIZE_NUM ", 0x1C30ull},
216
+
{"DFX_BUF_NP_RELEASE_SIZE_NUM ", 0x1C34ull},
217
+
{"DFX_BUF_P_RELEASE_SIZE_NUM ", 0x1C38ull},
218
+
{"DFX_BUF_PORT_RELEASE_SIZE_NUM ", 0x1C3Cull},
219
+
{"DFX_DMA_PREBUF_MEM0_ECC_ERR_ADDR ", 0x1CA8ull},
220
+
{"DFX_DMA_PREBUF_MEM0_ECC_CNT ", 0x1CACull},
221
+
{"DFX_DMA_LOC_NP_OSTB_ECC_ERR_ADDR ", 0x1CB0ull},
222
+
{"DFX_DMA_LOC_NP_OSTB_ECC_CNT ", 0x1CB4ull},
223
+
{"DFX_DMA_PREBUF_MEM1_ECC_ERR_ADDR ", 0x1CC0ull},
224
+
{"DFX_DMA_PREBUF_MEM1_ECC_CNT ", 0x1CC4ull},
225
+
{"DMA_CH_DONE_STS ", 0x02E0ull},
226
+
{"DMA_CH_ERR_STS ", 0x0320ull},
227
+
};
228
+
#endif /* CONFIG_DEBUG_FS*/
229
+
230
+
static enum hisi_dma_reg_layout hisi_dma_get_reg_layout(struct pci_dev *pdev)
231
+
{
232
+
if (pdev->revision == HISI_DMA_REVISION_HIP08B)
233
+
return HISI_DMA_REG_LAYOUT_HIP08;
234
+
else if (pdev->revision >= HISI_DMA_REVISION_HIP09A)
235
+
return HISI_DMA_REG_LAYOUT_HIP09;
236
+
237
+
return HISI_DMA_REG_LAYOUT_INVALID;
238
+
}
239
+
240
+
static u32 hisi_dma_get_chan_num(struct pci_dev *pdev)
241
+
{
242
+
if (pdev->revision == HISI_DMA_REVISION_HIP08B)
243
+
return HISI_DMA_HIP08_CHAN_NUM;
244
+
245
+
return HISI_DMA_HIP09_CHAN_NUM;
246
+
}
247
+
248
+
static u32 hisi_dma_get_msi_num(struct pci_dev *pdev)
249
+
{
250
+
if (pdev->revision == HISI_DMA_REVISION_HIP08B)
251
+
return HISI_DMA_HIP08_MSI_NUM;
252
+
253
+
return HISI_DMA_HIP09_MSI_NUM;
254
+
}
255
+
256
+
static u32 hisi_dma_get_queue_base(struct pci_dev *pdev)
257
+
{
258
+
if (pdev->revision == HISI_DMA_REVISION_HIP08B)
259
+
return HISI_DMA_HIP08_Q_BASE;
260
+
261
+
return HISI_DMA_HIP09_Q_BASE;
262
+
}
166
263
167
264
static inline struct hisi_dma_chan *to_hisi_dma_chan(struct dma_chan *c)
168
265
{
···
330
121
static inline void hisi_dma_chan_write(void __iomem *base, u32 reg, u32 index,
331
122
u32 val)
332
123
{
333
-
writel_relaxed(val, base + reg + index * HISI_DMA_OFFSET);
124
+
writel_relaxed(val, base + reg + index * HISI_DMA_Q_OFFSET);
334
125
}
335
126
336
127
static inline void hisi_dma_update_bit(void __iomem *addr, u32 pos, bool val)
···
338
129
u32 tmp;
339
130
340
131
tmp = readl_relaxed(addr);
341
-
tmp = val ? tmp | BIT(pos) : tmp & ~BIT(pos);
132
+
tmp = val ? tmp | pos : tmp & ~pos;
342
133
writel_relaxed(tmp, addr);
343
134
}
344
135
345
136
static void hisi_dma_pause_dma(struct hisi_dma_dev *hdma_dev, u32 index,
346
137
bool pause)
347
138
{
348
-
void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index *
349
-
HISI_DMA_OFFSET;
139
+
void __iomem *addr;
350
140
351
-
hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_PAUSE_S, pause);
141
+
addr = hdma_dev->queue_base + HISI_DMA_Q_CTRL0 +
142
+
index * HISI_DMA_Q_OFFSET;
143
+
hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL0_QUEUE_PAUSE, pause);
352
144
}
353
145
354
146
static void hisi_dma_enable_dma(struct hisi_dma_dev *hdma_dev, u32 index,
355
147
bool enable)
356
148
{
357
-
void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index *
358
-
HISI_DMA_OFFSET;
149
+
void __iomem *addr;
359
150
360
-
hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_EN_S, enable);
151
+
addr = hdma_dev->queue_base + HISI_DMA_Q_CTRL0 +
152
+
index * HISI_DMA_Q_OFFSET;
153
+
hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL0_QUEUE_EN, enable);
361
154
}
362
155
363
156
static void hisi_dma_mask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
364
157
{
365
-
hisi_dma_chan_write(hdma_dev->base, HISI_DMA_INT_MSK, qp_index,
366
-
HISI_DMA_INT_STS_MASK);
158
+
void __iomem *q_base = hdma_dev->queue_base;
159
+
160
+
if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
161
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_MSK,
162
+
qp_index, HISI_DMA_HIP08_Q_INT_STS_MASK);
163
+
else {
164
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_MSK,
165
+
qp_index, HISI_DMA_HIP09_Q_INT_STS_MASK);
166
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_MSK,
167
+
qp_index,
168
+
HISI_DMA_HIP09_Q_ERR_INT_STS_MASK);
169
+
}
367
170
}
368
171
369
172
static void hisi_dma_unmask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
370
173
{
371
-
void __iomem *base = hdma_dev->base;
174
+
void __iomem *q_base = hdma_dev->queue_base;
372
175
373
-
hisi_dma_chan_write(base, HISI_DMA_INT_STS, qp_index,
374
-
HISI_DMA_INT_STS_MASK);
375
-
hisi_dma_chan_write(base, HISI_DMA_INT_MSK, qp_index, 0);
176
+
if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
177
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_STS,
178
+
qp_index, HISI_DMA_HIP08_Q_INT_STS_MASK);
179
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_MSK,
180
+
qp_index, 0);
181
+
} else {
182
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_STS,
183
+
qp_index, HISI_DMA_HIP09_Q_INT_STS_MASK);
184
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_STS,
185
+
qp_index,
186
+
HISI_DMA_HIP09_Q_ERR_INT_STS_MASK);
187
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_MSK,
188
+
qp_index, 0);
189
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_MSK,
190
+
qp_index, 0);
191
+
}
376
192
}
377
193
378
194
static void hisi_dma_do_reset(struct hisi_dma_dev *hdma_dev, u32 index)
379
195
{
380
-
void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL1 + index *
381
-
HISI_DMA_OFFSET;
196
+
void __iomem *addr;
382
197
383
-
hisi_dma_update_bit(addr, HISI_DMA_CTRL1_QUEUE_RESET_S, 1);
198
+
addr = hdma_dev->queue_base +
199
+
HISI_DMA_Q_CTRL1 + index * HISI_DMA_Q_OFFSET;
200
+
hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL1_QUEUE_RESET, 1);
384
201
}
385
202
386
203
static void hisi_dma_reset_qp_point(struct hisi_dma_dev *hdma_dev, u32 index)
387
204
{
388
-
hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, index, 0);
389
-
hisi_dma_chan_write(hdma_dev->base, HISI_DMA_CQ_HEAD_PTR, index, 0);
205
+
void __iomem *q_base = hdma_dev->queue_base;
206
+
207
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_TAIL_PTR, index, 0);
208
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR, index, 0);
390
209
}
391
210
392
-
static void hisi_dma_reset_hw_chan(struct hisi_dma_chan *chan)
211
+
static void hisi_dma_reset_or_disable_hw_chan(struct hisi_dma_chan *chan,
212
+
bool disable)
393
213
{
394
214
struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
395
215
u32 index = chan->qp_num, tmp;
216
+
void __iomem *addr;
396
217
int ret;
397
218
398
219
hisi_dma_pause_dma(hdma_dev, index, true);
399
220
hisi_dma_enable_dma(hdma_dev, index, false);
400
221
hisi_dma_mask_irq(hdma_dev, index);
401
222
402
-
ret = readl_relaxed_poll_timeout(hdma_dev->base +
403
-
HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp,
404
-
FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) != RUN, 10, 1000);
223
+
addr = hdma_dev->queue_base +
224
+
HISI_DMA_Q_FSM_STS + index * HISI_DMA_Q_OFFSET;
225
+
226
+
ret = readl_relaxed_poll_timeout(addr, tmp,
227
+
FIELD_GET(HISI_DMA_Q_FSM_STS_MASK, tmp) != RUN,
228
+
HISI_DMA_POLL_Q_STS_DELAY_US, HISI_DMA_POLL_Q_STS_TIME_OUT_US);
405
229
if (ret) {
406
230
dev_err(&hdma_dev->pdev->dev, "disable channel timeout!\n");
407
231
WARN_ON(1);
···
443
201
hisi_dma_do_reset(hdma_dev, index);
444
202
hisi_dma_reset_qp_point(hdma_dev, index);
445
203
hisi_dma_pause_dma(hdma_dev, index, false);
446
-
hisi_dma_enable_dma(hdma_dev, index, true);
447
-
hisi_dma_unmask_irq(hdma_dev, index);
448
204
449
-
ret = readl_relaxed_poll_timeout(hdma_dev->base +
450
-
HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp,
451
-
FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) == IDLE, 10, 1000);
205
+
if (!disable) {
206
+
hisi_dma_enable_dma(hdma_dev, index, true);
207
+
hisi_dma_unmask_irq(hdma_dev, index);
208
+
}
209
+
210
+
ret = readl_relaxed_poll_timeout(addr, tmp,
211
+
FIELD_GET(HISI_DMA_Q_FSM_STS_MASK, tmp) == IDLE,
212
+
HISI_DMA_POLL_Q_STS_DELAY_US, HISI_DMA_POLL_Q_STS_TIME_OUT_US);
452
213
if (ret) {
453
214
dev_err(&hdma_dev->pdev->dev, "reset channel timeout!\n");
454
215
WARN_ON(1);
···
463
218
struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
464
219
struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
465
220
466
-
hisi_dma_reset_hw_chan(chan);
221
+
hisi_dma_reset_or_disable_hw_chan(chan, false);
467
222
vchan_free_chan_resources(&chan->vc);
468
223
469
224
memset(chan->sq, 0, sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth);
···
512
267
513
268
vd = vchan_next_desc(&chan->vc);
514
269
if (!vd) {
515
-
dev_err(&hdma_dev->pdev->dev, "no issued task!\n");
516
270
chan->desc = NULL;
517
271
return;
518
272
}
···
532
288
chan->sq_tail = (chan->sq_tail + 1) % hdma_dev->chan_depth;
533
289
534
290
/* update sq_tail to trigger a new task */
535
-
hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, chan->qp_num,
536
-
chan->sq_tail);
291
+
hisi_dma_chan_write(hdma_dev->queue_base, HISI_DMA_Q_SQ_TAIL_PTR,
292
+
chan->qp_num, chan->sq_tail);
537
293
}
538
294
539
295
static void hisi_dma_issue_pending(struct dma_chan *c)
···
543
299
544
300
spin_lock_irqsave(&chan->vc.lock, flags);
545
301
546
-
if (vchan_issue_pending(&chan->vc))
302
+
if (vchan_issue_pending(&chan->vc) && !chan->desc)
547
303
hisi_dma_start_transfer(chan);
548
304
549
305
spin_unlock_irqrestore(&chan->vc.lock, flags);
···
607
363
static void hisi_dma_init_hw_qp(struct hisi_dma_dev *hdma_dev, u32 index)
608
364
{
609
365
struct hisi_dma_chan *chan = &hdma_dev->chan[index];
366
+
void __iomem *q_base = hdma_dev->queue_base;
610
367
u32 hw_depth = hdma_dev->chan_depth - 1;
611
-
void __iomem *base = hdma_dev->base;
368
+
void __iomem *addr;
369
+
u32 tmp;
612
370
613
371
/* set sq, cq base */
614
-
hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_L, index,
372
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_BASE_L, index,
615
373
lower_32_bits(chan->sq_dma));
616
-
hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_H, index,
374
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_BASE_H, index,
617
375
upper_32_bits(chan->sq_dma));
618
-
hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_L, index,
376
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_BASE_L, index,
619
377
lower_32_bits(chan->cq_dma));
620
-
hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_H, index,
378
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_BASE_H, index,
621
379
upper_32_bits(chan->cq_dma));
622
380
623
381
/* set sq, cq depth */
624
-
hisi_dma_chan_write(base, HISI_DMA_SQ_DEPTH, index, hw_depth);
625
-
hisi_dma_chan_write(base, HISI_DMA_CQ_DEPTH, index, hw_depth);
382
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_DEPTH, index, hw_depth);
383
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_DEPTH, index, hw_depth);
626
384
627
385
/* init sq tail and cq head */
628
-
hisi_dma_chan_write(base, HISI_DMA_SQ_TAIL_PTR, index, 0);
629
-
hisi_dma_chan_write(base, HISI_DMA_CQ_HEAD_PTR, index, 0);
386
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_TAIL_PTR, index, 0);
387
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR, index, 0);
388
+
389
+
/* init error interrupt stats */
390
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM0, index, 0);
391
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM1, index, 0);
392
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM2, index, 0);
393
+
394
+
if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
395
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM3,
396
+
index, 0);
397
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM4,
398
+
index, 0);
399
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM5,
400
+
index, 0);
401
+
hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM6,
402
+
index, 0);
403
+
/*
404
+
* init SQ/CQ direction selecting register.
405
+
* "0" is to local side and "1" is to remote side.
406
+
*/
407
+
addr = q_base + HISI_DMA_Q_CTRL0 + index * HISI_DMA_Q_OFFSET;
408
+
hisi_dma_update_bit(addr, HISI_DMA_HIP08_Q_CTRL0_SQCQ_DRCT, 0);
409
+
410
+
/*
411
+
* 0 - Continue to next descriptor if error occurs.
412
+
* 1 - Abort the DMA queue if error occurs.
413
+
*/
414
+
hisi_dma_update_bit(addr,
415
+
HISI_DMA_HIP08_Q_CTRL0_ERR_ABORT_EN, 0);
416
+
} else {
417
+
addr = q_base + HISI_DMA_Q_CTRL0 + index * HISI_DMA_Q_OFFSET;
418
+
419
+
/*
420
+
* init SQ/CQ direction selecting register.
421
+
* "0" is to local side and "1" is to remote side.
422
+
*/
423
+
hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL0_SQ_DRCT, 0);
424
+
hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL0_CQ_DRCT, 0);
425
+
426
+
/*
427
+
* 0 - Continue to next descriptor if error occurs.
428
+
* 1 - Abort the DMA queue if error occurs.
429
+
*/
430
+
431
+
tmp = readl_relaxed(addr);
432
+
tmp &= ~HISI_DMA_HIP09_Q_CTRL0_ERR_ABORT_EN;
433
+
writel_relaxed(tmp, addr);
434
+
435
+
/*
436
+
* 0 - dma should process FLR whith CPU.
437
+
* 1 - dma not process FLR, only cpu process FLR.
438
+
*/
439
+
addr = q_base + HISI_DMA_HIP09_DMA_FLR_DISABLE +
440
+
index * HISI_DMA_Q_OFFSET;
441
+
hisi_dma_update_bit(addr, HISI_DMA_HIP09_DMA_FLR_DISABLE_B, 0);
442
+
443
+
addr = q_base + HISI_DMA_Q_CTRL1 + index * HISI_DMA_Q_OFFSET;
444
+
hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL1_VA_ENABLE, 1);
445
+
}
630
446
}
631
447
632
448
static void hisi_dma_enable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
···
698
394
699
395
static void hisi_dma_disable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
700
396
{
701
-
hisi_dma_reset_hw_chan(&hdma_dev->chan[qp_index]);
397
+
hisi_dma_reset_or_disable_hw_chan(&hdma_dev->chan[qp_index], true);
702
398
}
703
399
704
400
static void hisi_dma_enable_qps(struct hisi_dma_dev *hdma_dev)
···
730
426
struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
731
427
struct hisi_dma_desc *desc;
732
428
struct hisi_dma_cqe *cqe;
429
+
void __iomem *q_base;
733
430
734
431
spin_lock(&chan->vc.lock);
735
432
736
433
desc = chan->desc;
737
434
cqe = chan->cq + chan->cq_head;
435
+
q_base = hdma_dev->queue_base;
738
436
if (desc) {
437
+
chan->cq_head = (chan->cq_head + 1) % hdma_dev->chan_depth;
438
+
hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR,
439
+
chan->qp_num, chan->cq_head);
739
440
if (FIELD_GET(STATUS_MASK, cqe->w0) == STATUS_SUCC) {
740
-
chan->cq_head = (chan->cq_head + 1) %
741
-
hdma_dev->chan_depth;
742
-
hisi_dma_chan_write(hdma_dev->base,
743
-
HISI_DMA_CQ_HEAD_PTR, chan->qp_num,
744
-
chan->cq_head);
745
441
vchan_cookie_complete(&desc->vd);
442
+
hisi_dma_start_transfer(chan);
746
443
} else {
747
444
dev_err(&hdma_dev->pdev->dev, "task error!\n");
748
445
}
749
-
750
-
chan->desc = NULL;
751
446
}
752
447
753
448
spin_unlock(&chan->vc.lock);
···
800
497
static void hisi_dma_set_mode(struct hisi_dma_dev *hdma_dev,
801
498
enum hisi_dma_mode mode)
802
499
{
803
-
writel_relaxed(mode == RC ? 1 : 0, hdma_dev->base + HISI_DMA_MODE);
500
+
if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
501
+
writel_relaxed(mode == RC ? 1 : 0,
502
+
hdma_dev->base + HISI_DMA_HIP08_MODE);
804
503
}
504
+
505
+
static void hisi_dma_init_hw(struct hisi_dma_dev *hdma_dev)
506
+
{
507
+
void __iomem *addr;
508
+
int i;
509
+
510
+
if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP09) {
511
+
for (i = 0; i < HISI_DMA_HIP09_MAX_PORT_NUM; i++) {
512
+
addr = hdma_dev->base + HISI_DMA_HIP09_PORT_CFG_REG(i);
513
+
hisi_dma_update_bit(addr,
514
+
HISI_DMA_HIP09_PORT_CFG_LINK_DOWN_MASK_B, 1);
515
+
}
516
+
}
517
+
}
518
+
519
+
static void hisi_dma_init_dma_dev(struct hisi_dma_dev *hdma_dev)
520
+
{
521
+
struct dma_device *dma_dev;
522
+
523
+
dma_dev = &hdma_dev->dma_dev;
524
+
dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
525
+
dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources;
526
+
dma_dev->device_prep_dma_memcpy = hisi_dma_prep_dma_memcpy;
527
+
dma_dev->device_tx_status = hisi_dma_tx_status;
528
+
dma_dev->device_issue_pending = hisi_dma_issue_pending;
529
+
dma_dev->device_terminate_all = hisi_dma_terminate_all;
530
+
dma_dev->device_synchronize = hisi_dma_synchronize;
531
+
dma_dev->directions = BIT(DMA_MEM_TO_MEM);
532
+
dma_dev->dev = &hdma_dev->pdev->dev;
533
+
INIT_LIST_HEAD(&dma_dev->channels);
534
+
}
535
+
536
+
/* --- debugfs implementation --- */
537
+
#ifdef CONFIG_DEBUG_FS
538
+
#include <linux/debugfs.h>
539
+
static struct debugfs_reg32 *hisi_dma_get_ch_regs(struct hisi_dma_dev *hdma_dev,
540
+
u32 *regs_sz)
541
+
{
542
+
struct device *dev = &hdma_dev->pdev->dev;
543
+
struct debugfs_reg32 *regs;
544
+
u32 regs_sz_comm;
545
+
546
+
regs_sz_comm = ARRAY_SIZE(hisi_dma_comm_chan_regs);
547
+
548
+
if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
549
+
*regs_sz = regs_sz_comm + ARRAY_SIZE(hisi_dma_hip08_chan_regs);
550
+
else
551
+
*regs_sz = regs_sz_comm + ARRAY_SIZE(hisi_dma_hip09_chan_regs);
552
+
553
+
regs = devm_kcalloc(dev, *regs_sz, sizeof(struct debugfs_reg32),
554
+
GFP_KERNEL);
555
+
if (!regs)
556
+
return NULL;
557
+
memcpy(regs, hisi_dma_comm_chan_regs, sizeof(hisi_dma_comm_chan_regs));
558
+
559
+
if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
560
+
memcpy(regs + regs_sz_comm, hisi_dma_hip08_chan_regs,
561
+
sizeof(hisi_dma_hip08_chan_regs));
562
+
else
563
+
memcpy(regs + regs_sz_comm, hisi_dma_hip09_chan_regs,
564
+
sizeof(hisi_dma_hip09_chan_regs));
565
+
566
+
return regs;
567
+
}
568
+
569
+
static int hisi_dma_create_chan_dir(struct hisi_dma_dev *hdma_dev)
570
+
{
571
+
char dir_name[HISI_DMA_MAX_DIR_NAME_LEN];
572
+
struct debugfs_regset32 *regsets;
573
+
struct debugfs_reg32 *regs;
574
+
struct dentry *chan_dir;
575
+
struct device *dev;
576
+
u32 regs_sz;
577
+
int ret;
578
+
int i;
579
+
580
+
dev = &hdma_dev->pdev->dev;
581
+
582
+
regsets = devm_kcalloc(dev, hdma_dev->chan_num,
583
+
sizeof(*regsets), GFP_KERNEL);
584
+
if (!regsets)
585
+
return -ENOMEM;
586
+
587
+
regs = hisi_dma_get_ch_regs(hdma_dev, ®s_sz);
588
+
if (!regs)
589
+
return -ENOMEM;
590
+
591
+
for (i = 0; i < hdma_dev->chan_num; i++) {
592
+
regsets[i].regs = regs;
593
+
regsets[i].nregs = regs_sz;
594
+
regsets[i].base = hdma_dev->queue_base + i * HISI_DMA_Q_OFFSET;
595
+
regsets[i].dev = dev;
596
+
597
+
memset(dir_name, 0, HISI_DMA_MAX_DIR_NAME_LEN);
598
+
ret = sprintf(dir_name, "channel%d", i);
599
+
if (ret < 0)
600
+
return ret;
601
+
602
+
chan_dir = debugfs_create_dir(dir_name,
603
+
hdma_dev->dma_dev.dbg_dev_root);
604
+
debugfs_create_regset32("regs", 0444, chan_dir, ®sets[i]);
605
+
}
606
+
607
+
return 0;
608
+
}
609
+
610
+
static void hisi_dma_create_debugfs(struct hisi_dma_dev *hdma_dev)
611
+
{
612
+
struct debugfs_regset32 *regset;
613
+
struct device *dev;
614
+
int ret;
615
+
616
+
dev = &hdma_dev->pdev->dev;
617
+
618
+
if (hdma_dev->dma_dev.dbg_dev_root == NULL)
619
+
return;
620
+
621
+
regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL);
622
+
if (!regset)
623
+
return;
624
+
625
+
if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
626
+
regset->regs = hisi_dma_hip08_comm_regs;
627
+
regset->nregs = ARRAY_SIZE(hisi_dma_hip08_comm_regs);
628
+
} else {
629
+
regset->regs = hisi_dma_hip09_comm_regs;
630
+
regset->nregs = ARRAY_SIZE(hisi_dma_hip09_comm_regs);
631
+
}
632
+
regset->base = hdma_dev->base;
633
+
regset->dev = dev;
634
+
635
+
debugfs_create_regset32("regs", 0444,
636
+
hdma_dev->dma_dev.dbg_dev_root, regset);
637
+
638
+
ret = hisi_dma_create_chan_dir(hdma_dev);
639
+
if (ret < 0)
640
+
dev_info(&hdma_dev->pdev->dev, "fail to create debugfs for channels!\n");
641
+
}
642
+
#else
643
+
static void hisi_dma_create_debugfs(struct hisi_dma_dev *hdma_dev) { }
644
+
#endif /* CONFIG_DEBUG_FS*/
645
+
/* --- debugfs implementation --- */
805
646
806
647
static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id)
807
648
{
649
+
enum hisi_dma_reg_layout reg_layout;
808
650
struct device *dev = &pdev->dev;
809
651
struct hisi_dma_dev *hdma_dev;
810
652
struct dma_device *dma_dev;
653
+
u32 chan_num;
654
+
u32 msi_num;
811
655
int ret;
656
+
657
+
reg_layout = hisi_dma_get_reg_layout(pdev);
658
+
if (reg_layout == HISI_DMA_REG_LAYOUT_INVALID) {
659
+
dev_err(dev, "unsupported device!\n");
660
+
return -EINVAL;
661
+
}
812
662
813
663
ret = pcim_enable_device(pdev);
814
664
if (ret) {
···
979
523
if (ret)
980
524
return ret;
981
525
982
-
hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, HISI_DMA_CHAN_NUM), GFP_KERNEL);
526
+
chan_num = hisi_dma_get_chan_num(pdev);
527
+
hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, chan_num),
528
+
GFP_KERNEL);
983
529
if (!hdma_dev)
984
530
return -EINVAL;
985
531
986
532
hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2];
987
533
hdma_dev->pdev = pdev;
988
-
hdma_dev->chan_num = HISI_DMA_CHAN_NUM;
989
534
hdma_dev->chan_depth = HISI_DMA_Q_DEPTH_VAL;
535
+
hdma_dev->chan_num = chan_num;
536
+
hdma_dev->reg_layout = reg_layout;
537
+
hdma_dev->queue_base = hdma_dev->base + hisi_dma_get_queue_base(pdev);
990
538
991
539
pci_set_drvdata(pdev, hdma_dev);
992
540
pci_set_master(pdev);
993
541
542
+
msi_num = hisi_dma_get_msi_num(pdev);
543
+
994
544
/* This will be freed by 'pcim_release()'. See 'pcim_enable_device()' */
995
-
ret = pci_alloc_irq_vectors(pdev, HISI_DMA_MSI_NUM, HISI_DMA_MSI_NUM,
996
-
PCI_IRQ_MSI);
545
+
ret = pci_alloc_irq_vectors(pdev, msi_num, msi_num, PCI_IRQ_MSI);
997
546
if (ret < 0) {
998
547
dev_err(dev, "Failed to allocate MSI vectors!\n");
999
548
return ret;
1000
549
}
1001
550
1002
-
dma_dev = &hdma_dev->dma_dev;
1003
-
dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
1004
-
dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources;
1005
-
dma_dev->device_prep_dma_memcpy = hisi_dma_prep_dma_memcpy;
1006
-
dma_dev->device_tx_status = hisi_dma_tx_status;
1007
-
dma_dev->device_issue_pending = hisi_dma_issue_pending;
1008
-
dma_dev->device_terminate_all = hisi_dma_terminate_all;
1009
-
dma_dev->device_synchronize = hisi_dma_synchronize;
1010
-
dma_dev->directions = BIT(DMA_MEM_TO_MEM);
1011
-
dma_dev->dev = dev;
1012
-
INIT_LIST_HEAD(&dma_dev->channels);
551
+
hisi_dma_init_dma_dev(hdma_dev);
1013
552
1014
553
hisi_dma_set_mode(hdma_dev, RC);
554
+
555
+
hisi_dma_init_hw(hdma_dev);
1015
556
1016
557
ret = hisi_dma_enable_hw_channels(hdma_dev);
1017
558
if (ret < 0) {
···
1021
568
if (ret)
1022
569
return ret;
1023
570
571
+
dma_dev = &hdma_dev->dma_dev;
1024
572
ret = dmaenginem_async_device_register(dma_dev);
1025
-
if (ret < 0)
573
+
if (ret < 0) {
1026
574
dev_err(dev, "failed to register device!\n");
575
+
return ret;
576
+
}
1027
577
1028
-
return ret;
578
+
hisi_dma_create_debugfs(hdma_dev);
579
+
580
+
return 0;
1029
581
}
1030
582
1031
583
static const struct pci_device_id hisi_dma_pci_tbl[] = {
+8
drivers/dma/hsu/hsu.c
+8
drivers/dma/hsu/hsu.c
···
16
16
* port 3, and so on.
17
17
*/
18
18
19
+
#include <linux/bits.h>
19
20
#include <linux/delay.h>
21
+
#include <linux/device.h>
20
22
#include <linux/dmaengine.h>
21
23
#include <linux/dma-mapping.h>
22
24
#include <linux/init.h>
25
+
#include <linux/interrupt.h>
26
+
#include <linux/list.h>
23
27
#include <linux/module.h>
28
+
#include <linux/percpu-defs.h>
29
+
#include <linux/scatterlist.h>
24
30
#include <linux/slab.h>
31
+
#include <linux/string.h>
32
+
#include <linux/spinlock.h>
25
33
26
34
#include "hsu.h"
27
35
+8
-4
drivers/dma/hsu/hsu.h
+8
-4
drivers/dma/hsu/hsu.h
···
10
10
#ifndef __DMA_HSU_H__
11
11
#define __DMA_HSU_H__
12
12
13
-
#include <linux/spinlock.h>
13
+
#include <linux/bits.h>
14
+
#include <linux/container_of.h>
15
+
#include <linux/io.h>
16
+
#include <linux/types.h>
17
+
14
18
#include <linux/dma/hsu.h>
15
19
16
20
#include "../virt-dma.h"
···
40
36
41
37
/* Bits in HSU_CH_SR */
42
38
#define HSU_CH_SR_DESCTO(x) BIT(8 + (x))
43
-
#define HSU_CH_SR_DESCTO_ANY (BIT(11) | BIT(10) | BIT(9) | BIT(8))
39
+
#define HSU_CH_SR_DESCTO_ANY GENMASK(11, 8)
44
40
#define HSU_CH_SR_CHE BIT(15)
45
41
#define HSU_CH_SR_DESCE(x) BIT(16 + (x))
46
-
#define HSU_CH_SR_DESCE_ANY (BIT(19) | BIT(18) | BIT(17) | BIT(16))
47
-
#define HSU_CH_SR_CDESC_ANY (BIT(31) | BIT(30))
42
+
#define HSU_CH_SR_DESCE_ANY GENMASK(19, 16)
43
+
#define HSU_CH_SR_CDESC_ANY GENMASK(31, 30)
48
44
49
45
/* Bits in HSU_CH_CR */
50
46
#define HSU_CH_CR_CHA BIT(0)
+21
-26
drivers/dma/hsu/pci.c
+21
-26
drivers/dma/hsu/pci.c
···
10
10
11
11
#include <linux/bitops.h>
12
12
#include <linux/device.h>
13
+
#include <linux/interrupt.h>
13
14
#include <linux/module.h>
14
15
#include <linux/pci.h>
15
16
···
27
26
static irqreturn_t hsu_pci_irq(int irq, void *dev)
28
27
{
29
28
struct hsu_dma_chip *chip = dev;
30
-
u32 dmaisr;
31
-
u32 status;
29
+
unsigned long dmaisr;
32
30
unsigned short i;
31
+
u32 status;
33
32
int ret = 0;
34
33
int err;
35
34
36
35
dmaisr = readl(chip->regs + HSU_PCI_DMAISR);
37
-
for (i = 0; i < chip->hsu->nr_channels; i++) {
38
-
if (dmaisr & 0x1) {
39
-
err = hsu_dma_get_status(chip, i, &status);
40
-
if (err > 0)
41
-
ret |= 1;
42
-
else if (err == 0)
43
-
ret |= hsu_dma_do_irq(chip, i, status);
44
-
}
45
-
dmaisr >>= 1;
36
+
for_each_set_bit(i, &dmaisr, chip->hsu->nr_channels) {
37
+
err = hsu_dma_get_status(chip, i, &status);
38
+
if (err > 0)
39
+
ret |= 1;
40
+
else if (err == 0)
41
+
ret |= hsu_dma_do_irq(chip, i, status);
46
42
}
47
43
48
44
return IRQ_RETVAL(ret);
49
45
}
50
46
47
+
static void hsu_pci_dma_remove(void *chip)
48
+
{
49
+
hsu_dma_remove(chip);
50
+
}
51
+
51
52
static int hsu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
52
53
{
54
+
struct device *dev = &pdev->dev;
53
55
struct hsu_dma_chip *chip;
54
56
int ret;
55
57
···
91
87
if (ret)
92
88
return ret;
93
89
94
-
ret = request_irq(chip->irq, hsu_pci_irq, 0, "hsu_dma_pci", chip);
90
+
ret = devm_add_action_or_reset(dev, hsu_pci_dma_remove, chip);
95
91
if (ret)
96
-
goto err_register_irq;
92
+
return ret;
93
+
94
+
ret = devm_request_irq(dev, chip->irq, hsu_pci_irq, 0, "hsu_dma_pci", chip);
95
+
if (ret)
96
+
return ret;
97
97
98
98
/*
99
99
* On Intel Tangier B0 and Anniedale the interrupt line, disregarding
···
113
105
pci_set_drvdata(pdev, chip);
114
106
115
107
return 0;
116
-
117
-
err_register_irq:
118
-
hsu_dma_remove(chip);
119
-
return ret;
120
-
}
121
-
122
-
static void hsu_pci_remove(struct pci_dev *pdev)
123
-
{
124
-
struct hsu_dma_chip *chip = pci_get_drvdata(pdev);
125
-
126
-
free_irq(chip->irq, chip);
127
-
hsu_dma_remove(chip);
128
108
}
129
109
130
110
static const struct pci_device_id hsu_pci_id_table[] = {
···
126
130
.name = "hsu_dma_pci",
127
131
.id_table = hsu_pci_id_table,
128
132
.probe = hsu_pci_probe,
129
-
.remove = hsu_pci_remove,
130
133
};
131
134
132
135
module_pci_driver(hsu_pci_driver);
+29
-9
drivers/dma/idxd/device.c
+29
-9
drivers/dma/idxd/device.c
···
196
196
}
197
197
198
198
wq->state = IDXD_WQ_ENABLED;
199
+
set_bit(wq->id, idxd->wq_enable_map);
199
200
dev_dbg(dev, "WQ %d enabled\n", wq->id);
200
201
return 0;
201
202
}
···
224
223
225
224
if (reset_config)
226
225
idxd_wq_disable_cleanup(wq);
226
+
clear_bit(wq->id, idxd->wq_enable_map);
227
227
wq->state = IDXD_WQ_DISABLED;
228
228
dev_dbg(dev, "WQ %d disabled\n", wq->id);
229
229
return 0;
···
260
258
operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
261
259
idxd_cmd_exec(idxd, IDXD_CMD_RESET_WQ, operand, NULL);
262
260
idxd_wq_disable_cleanup(wq);
263
-
wq->state = IDXD_WQ_DISABLED;
264
261
}
265
262
266
263
int idxd_wq_map_portal(struct idxd_wq *wq)
···
379
378
struct idxd_device *idxd = wq->idxd;
380
379
381
380
lockdep_assert_held(&wq->wq_lock);
381
+
wq->state = IDXD_WQ_DISABLED;
382
382
memset(wq->wqcfg, 0, idxd->wqcfg_size);
383
383
wq->type = IDXD_WQT_NONE;
384
384
wq->threshold = 0;
385
385
wq->priority = 0;
386
-
wq->ats_dis = 0;
387
386
wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES;
388
387
clear_bit(WQ_FLAG_DEDICATED, &wq->flags);
389
388
clear_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags);
389
+
clear_bit(WQ_FLAG_ATS_DISABLE, &wq->flags);
390
390
memset(wq->name, 0, WQ_NAME_SIZE);
391
391
wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER;
392
392
wq->max_batch_size = WQ_DEFAULT_MAX_BATCH;
393
+
if (wq->opcap_bmap)
394
+
bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS);
393
395
}
394
396
395
397
static void idxd_wq_device_reset_cleanup(struct idxd_wq *wq)
···
709
705
group->tc_a = -1;
710
706
group->tc_b = -1;
711
707
}
708
+
group->desc_progress_limit = 0;
709
+
group->batch_progress_limit = 0;
712
710
}
713
711
}
714
712
···
767
761
768
762
/* setup GRPFLAGS */
769
763
grpcfg_offset = GRPFLGCFG_OFFSET(idxd, group->id);
770
-
iowrite32(group->grpcfg.flags.bits, idxd->reg_base + grpcfg_offset);
771
-
dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n",
764
+
iowrite64(group->grpcfg.flags.bits, idxd->reg_base + grpcfg_offset);
765
+
dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#llx\n",
772
766
group->id, grpcfg_offset,
773
-
ioread32(idxd->reg_base + grpcfg_offset));
767
+
ioread64(idxd->reg_base + grpcfg_offset));
774
768
}
775
769
776
770
static int idxd_groups_config_write(struct idxd_device *idxd)
···
813
807
struct idxd_device *idxd = wq->idxd;
814
808
struct device *dev = &idxd->pdev->dev;
815
809
u32 wq_offset;
816
-
int i;
810
+
int i, n;
817
811
818
812
if (!wq->group)
819
813
return 0;
···
865
859
wq->wqcfg->bof = 1;
866
860
867
861
if (idxd->hw.wq_cap.wq_ats_support)
868
-
wq->wqcfg->wq_ats_disable = wq->ats_dis;
862
+
wq->wqcfg->wq_ats_disable = test_bit(WQ_FLAG_ATS_DISABLE, &wq->flags);
869
863
870
864
/* bytes 12-15 */
871
865
wq->wqcfg->max_xfer_shift = ilog2(wq->max_xfer_bytes);
872
866
wq->wqcfg->max_batch_shift = ilog2(wq->max_batch_size);
867
+
868
+
/* bytes 32-63 */
869
+
if (idxd->hw.wq_cap.op_config && wq->opcap_bmap) {
870
+
memset(wq->wqcfg->op_config, 0, IDXD_MAX_OPCAP_BITS / 8);
871
+
for_each_set_bit(n, wq->opcap_bmap, IDXD_MAX_OPCAP_BITS) {
872
+
int pos = n % BITS_PER_LONG_LONG;
873
+
int idx = n / BITS_PER_LONG_LONG;
874
+
875
+
wq->wqcfg->op_config[idx] |= BIT(pos);
876
+
}
877
+
}
873
878
874
879
dev_dbg(dev, "WQ %d CFGs\n", wq->id);
875
880
for (i = 0; i < WQCFG_STRIDES(idxd); i++) {
···
931
914
group->grpcfg.flags.rdbufs_allowed = group->rdbufs_allowed;
932
915
else
933
916
group->grpcfg.flags.rdbufs_allowed = idxd->max_rdbufs;
917
+
918
+
group->grpcfg.flags.desc_progress_limit = group->desc_progress_limit;
919
+
group->grpcfg.flags.batch_progress_limit = group->batch_progress_limit;
934
920
}
935
921
}
936
922
···
1116
1096
}
1117
1097
1118
1098
grpcfg_offset = GRPFLGCFG_OFFSET(idxd, group->id);
1119
-
group->grpcfg.flags.bits = ioread32(idxd->reg_base + grpcfg_offset);
1120
-
dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n",
1099
+
group->grpcfg.flags.bits = ioread64(idxd->reg_base + grpcfg_offset);
1100
+
dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#llx\n",
1121
1101
group->id, grpcfg_offset, group->grpcfg.flags.bits);
1122
1102
}
1123
1103
+9
-1
drivers/dma/idxd/idxd.h
+9
-1
drivers/dma/idxd/idxd.h
···
11
11
#include <linux/idr.h>
12
12
#include <linux/pci.h>
13
13
#include <linux/ioasid.h>
14
+
#include <linux/bitmap.h>
14
15
#include <linux/perf_event.h>
15
16
#include <uapi/linux/idxd.h>
16
17
#include "registers.h"
···
96
95
u8 rdbufs_reserved;
97
96
int tc_a;
98
97
int tc_b;
98
+
int desc_progress_limit;
99
+
int batch_progress_limit;
99
100
};
100
101
101
102
struct idxd_pmu {
···
135
132
enum idxd_wq_flag {
136
133
WQ_FLAG_DEDICATED = 0,
137
134
WQ_FLAG_BLOCK_ON_FAULT,
135
+
WQ_FLAG_ATS_DISABLE,
138
136
};
139
137
140
138
enum idxd_wq_type {
···
198
194
enum idxd_wq_state state;
199
195
unsigned long flags;
200
196
union wqcfg *wqcfg;
197
+
unsigned long *opcap_bmap;
198
+
201
199
struct dsa_hw_desc **hw_descs;
202
200
int num_descs;
203
201
union {
···
214
208
char name[WQ_NAME_SIZE + 1];
215
209
u64 max_xfer_bytes;
216
210
u32 max_batch_size;
217
-
bool ats_dis;
218
211
};
219
212
220
213
struct idxd_engine {
···
304
299
int rdbuf_limit;
305
300
int nr_rdbufs; /* non-reserved read buffers */
306
301
unsigned int wqcfg_size;
302
+
unsigned long *wq_enable_map;
307
303
308
304
union sw_err_reg sw_err;
309
305
wait_queue_head_t cmd_waitq;
···
314
308
struct work_struct work;
315
309
316
310
struct idxd_pmu *idxd_pmu;
311
+
312
+
unsigned long *opcap_bmap;
317
313
};
318
314
319
315
/* IDXD software descriptor */
+36
drivers/dma/idxd/init.c
+36
drivers/dma/idxd/init.c
···
151
151
if (!idxd->wqs)
152
152
return -ENOMEM;
153
153
154
+
idxd->wq_enable_map = bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL, dev_to_node(dev));
155
+
if (!idxd->wq_enable_map) {
156
+
kfree(idxd->wqs);
157
+
return -ENOMEM;
158
+
}
159
+
154
160
for (i = 0; i < idxd->max_wqs; i++) {
155
161
wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev));
156
162
if (!wq) {
···
190
184
put_device(conf_dev);
191
185
rc = -ENOMEM;
192
186
goto err;
187
+
}
188
+
189
+
if (idxd->hw.wq_cap.op_config) {
190
+
wq->opcap_bmap = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL);
191
+
if (!wq->opcap_bmap) {
192
+
put_device(conf_dev);
193
+
rc = -ENOMEM;
194
+
goto err;
195
+
}
196
+
bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS);
193
197
}
194
198
idxd->wqs[i] = wq;
195
199
}
···
385
369
dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset);
386
370
}
387
371
372
+
static void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count)
373
+
{
374
+
int i, j, nr;
375
+
376
+
for (i = 0, nr = 0; i < count; i++) {
377
+
for (j = 0; j < BITS_PER_LONG_LONG; j++) {
378
+
if (val[i] & BIT(j))
379
+
set_bit(nr, bmap);
380
+
nr++;
381
+
}
382
+
}
383
+
}
384
+
388
385
static void idxd_read_caps(struct idxd_device *idxd)
389
386
{
390
387
struct device *dev = &idxd->pdev->dev;
···
456
427
IDXD_OPCAP_OFFSET + i * sizeof(u64));
457
428
dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]);
458
429
}
430
+
multi_u64_to_bmap(idxd->opcap_bmap, &idxd->hw.opcap.bits[0], 4);
459
431
}
460
432
461
433
static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data)
···
477
447
idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL);
478
448
if (idxd->id < 0)
479
449
return NULL;
450
+
451
+
idxd->opcap_bmap = bitmap_zalloc_node(IDXD_MAX_OPCAP_BITS, GFP_KERNEL, dev_to_node(dev));
452
+
if (!idxd->opcap_bmap) {
453
+
ida_free(&idxd_ida, idxd->id);
454
+
return NULL;
455
+
}
480
456
481
457
device_initialize(conf_dev);
482
458
conf_dev->parent = dev;
+3
-10
drivers/dma/idxd/irq.c
+3
-10
drivers/dma/idxd/irq.c
···
17
17
IRQ_WORK_PROCESS_FAULT,
18
18
};
19
19
20
-
struct idxd_fault {
21
-
struct work_struct work;
22
-
u64 addr;
23
-
struct idxd_device *idxd;
24
-
};
25
-
26
20
struct idxd_resubmit {
27
21
struct work_struct work;
28
22
struct idxd_desc *desc;
···
43
49
goto out;
44
50
45
51
for (i = 0; i < idxd->max_wqs; i++) {
46
-
struct idxd_wq *wq = idxd->wqs[i];
52
+
if (test_bit(i, idxd->wq_enable_map)) {
53
+
struct idxd_wq *wq = idxd->wqs[i];
47
54
48
-
if (wq->state == IDXD_WQ_ENABLED) {
49
55
rc = idxd_wq_enable(wq);
50
56
if (rc < 0) {
57
+
clear_bit(i, idxd->wq_enable_map);
51
58
dev_warn(dev, "Unable to re-enable wq %s\n",
52
59
dev_name(wq_confdev(wq)));
53
60
}
···
319
324
idxd->state = IDXD_DEV_HALTED;
320
325
idxd_wqs_quiesce(idxd);
321
326
idxd_wqs_unmap_portal(idxd);
322
-
spin_lock(&idxd->dev_lock);
323
327
idxd_device_clear_state(idxd);
324
328
dev_err(&idxd->pdev->dev,
325
329
"idxd halted, need %s.\n",
326
330
gensts.reset_type == IDXD_DEVICE_RESET_FLR ?
327
331
"FLR" : "system reset");
328
-
spin_unlock(&idxd->dev_lock);
329
332
return -ENXIO;
330
333
}
331
334
}
+23
-12
drivers/dma/idxd/registers.h
+23
-12
drivers/dma/idxd/registers.h
···
54
54
u64 priority:1;
55
55
u64 occupancy:1;
56
56
u64 occupancy_int:1;
57
-
u64 rsvd3:10;
57
+
u64 op_config:1;
58
+
u64 rsvd3:9;
58
59
};
59
60
u64 bits;
60
61
} __packed;
···
68
67
u64 total_rdbufs:8; /* formerly total_tokens */
69
68
u64 rdbuf_ctrl:1; /* formerly token_en */
70
69
u64 rdbuf_limit:1; /* formerly token_limit */
71
-
u64 rsvd:46;
70
+
u64 progress_limit:1; /* descriptor and batch descriptor */
71
+
u64 rsvd:45;
72
72
};
73
73
u64 bits;
74
74
} __packed;
···
91
89
struct opcap {
92
90
u64 bits[4];
93
91
};
92
+
93
+
#define IDXD_MAX_OPCAP_BITS 256U
94
94
95
95
#define IDXD_OPCAP_OFFSET 0x40
96
96
···
289
285
290
286
union group_flags {
291
287
struct {
292
-
u32 tc_a:3;
293
-
u32 tc_b:3;
294
-
u32 rsvd:1;
295
-
u32 use_rdbuf_limit:1;
296
-
u32 rdbufs_reserved:8;
297
-
u32 rsvd2:4;
298
-
u32 rdbufs_allowed:8;
299
-
u32 rsvd3:4;
288
+
u64 tc_a:3;
289
+
u64 tc_b:3;
290
+
u64 rsvd:1;
291
+
u64 use_rdbuf_limit:1;
292
+
u64 rdbufs_reserved:8;
293
+
u64 rsvd2:4;
294
+
u64 rdbufs_allowed:8;
295
+
u64 rsvd3:4;
296
+
u64 desc_progress_limit:2;
297
+
u64 rsvd4:2;
298
+
u64 batch_progress_limit:2;
299
+
u64 rsvd5:26;
300
300
};
301
-
u32 bits;
301
+
u64 bits;
302
302
} __packed;
303
303
304
304
struct grpcfg {
···
356
348
357
349
/* bytes 28-31 */
358
350
u32 rsvd8;
351
+
352
+
/* bytes 32-63 */
353
+
u64 op_config[4];
359
354
};
360
-
u32 bits[8];
355
+
u32 bits[16];
361
356
} __packed;
362
357
363
358
#define WQCFG_PASID_IDX 2
+178
-9
drivers/dma/idxd/sysfs.c
+178
-9
drivers/dma/idxd/sysfs.c
···
443
443
__ATTR(traffic_class_b, 0644, group_traffic_class_b_show,
444
444
group_traffic_class_b_store);
445
445
446
+
static ssize_t group_desc_progress_limit_show(struct device *dev,
447
+
struct device_attribute *attr,
448
+
char *buf)
449
+
{
450
+
struct idxd_group *group = confdev_to_group(dev);
451
+
452
+
return sysfs_emit(buf, "%d\n", group->desc_progress_limit);
453
+
}
454
+
455
+
static ssize_t group_desc_progress_limit_store(struct device *dev,
456
+
struct device_attribute *attr,
457
+
const char *buf, size_t count)
458
+
{
459
+
struct idxd_group *group = confdev_to_group(dev);
460
+
int val, rc;
461
+
462
+
rc = kstrtoint(buf, 10, &val);
463
+
if (rc < 0)
464
+
return -EINVAL;
465
+
466
+
if (val & ~GENMASK(1, 0))
467
+
return -EINVAL;
468
+
469
+
group->desc_progress_limit = val;
470
+
return count;
471
+
}
472
+
473
+
static struct device_attribute dev_attr_group_desc_progress_limit =
474
+
__ATTR(desc_progress_limit, 0644, group_desc_progress_limit_show,
475
+
group_desc_progress_limit_store);
476
+
477
+
static ssize_t group_batch_progress_limit_show(struct device *dev,
478
+
struct device_attribute *attr,
479
+
char *buf)
480
+
{
481
+
struct idxd_group *group = confdev_to_group(dev);
482
+
483
+
return sysfs_emit(buf, "%d\n", group->batch_progress_limit);
484
+
}
485
+
486
+
static ssize_t group_batch_progress_limit_store(struct device *dev,
487
+
struct device_attribute *attr,
488
+
const char *buf, size_t count)
489
+
{
490
+
struct idxd_group *group = confdev_to_group(dev);
491
+
int val, rc;
492
+
493
+
rc = kstrtoint(buf, 10, &val);
494
+
if (rc < 0)
495
+
return -EINVAL;
496
+
497
+
if (val & ~GENMASK(1, 0))
498
+
return -EINVAL;
499
+
500
+
group->batch_progress_limit = val;
501
+
return count;
502
+
}
503
+
504
+
static struct device_attribute dev_attr_group_batch_progress_limit =
505
+
__ATTR(batch_progress_limit, 0644, group_batch_progress_limit_show,
506
+
group_batch_progress_limit_store);
446
507
static struct attribute *idxd_group_attributes[] = {
447
508
&dev_attr_group_work_queues.attr,
448
509
&dev_attr_group_engines.attr,
···
515
454
&dev_attr_group_read_buffers_reserved.attr,
516
455
&dev_attr_group_traffic_class_a.attr,
517
456
&dev_attr_group_traffic_class_b.attr,
457
+
&dev_attr_group_desc_progress_limit.attr,
458
+
&dev_attr_group_batch_progress_limit.attr,
518
459
NULL,
519
460
};
520
461
462
+
static bool idxd_group_attr_progress_limit_invisible(struct attribute *attr,
463
+
struct idxd_device *idxd)
464
+
{
465
+
return (attr == &dev_attr_group_desc_progress_limit.attr ||
466
+
attr == &dev_attr_group_batch_progress_limit.attr) &&
467
+
!idxd->hw.group_cap.progress_limit;
468
+
}
469
+
470
+
static umode_t idxd_group_attr_visible(struct kobject *kobj,
471
+
struct attribute *attr, int n)
472
+
{
473
+
struct device *dev = container_of(kobj, struct device, kobj);
474
+
struct idxd_group *group = confdev_to_group(dev);
475
+
struct idxd_device *idxd = group->idxd;
476
+
477
+
if (idxd_group_attr_progress_limit_invisible(attr, idxd))
478
+
return 0;
479
+
480
+
return attr->mode;
481
+
}
482
+
521
483
static const struct attribute_group idxd_group_attribute_group = {
522
484
.attrs = idxd_group_attributes,
485
+
.is_visible = idxd_group_attr_visible,
523
486
};
524
487
525
488
static const struct attribute_group *idxd_group_attribute_groups[] = {
···
1058
973
{
1059
974
struct idxd_wq *wq = confdev_to_wq(dev);
1060
975
1061
-
return sysfs_emit(buf, "%u\n", wq->ats_dis);
976
+
return sysfs_emit(buf, "%u\n", test_bit(WQ_FLAG_ATS_DISABLE, &wq->flags));
1062
977
}
1063
978
1064
979
static ssize_t wq_ats_disable_store(struct device *dev, struct device_attribute *attr,
···
1079
994
if (rc < 0)
1080
995
return rc;
1081
996
1082
-
wq->ats_dis = ats_dis;
997
+
if (ats_dis)
998
+
set_bit(WQ_FLAG_ATS_DISABLE, &wq->flags);
999
+
else
1000
+
clear_bit(WQ_FLAG_ATS_DISABLE, &wq->flags);
1083
1001
1084
1002
return count;
1085
1003
}
···
1143
1055
static struct device_attribute dev_attr_wq_enqcmds_retries =
1144
1056
__ATTR(enqcmds_retries, 0644, wq_enqcmds_retries_show, wq_enqcmds_retries_store);
1145
1057
1058
+
static ssize_t wq_op_config_show(struct device *dev,
1059
+
struct device_attribute *attr, char *buf)
1060
+
{
1061
+
struct idxd_wq *wq = confdev_to_wq(dev);
1062
+
1063
+
return sysfs_emit(buf, "%*pb\n", IDXD_MAX_OPCAP_BITS, wq->opcap_bmap);
1064
+
}
1065
+
1066
+
static int idxd_verify_supported_opcap(struct idxd_device *idxd, unsigned long *opmask)
1067
+
{
1068
+
int bit;
1069
+
1070
+
/*
1071
+
* The OPCAP is defined as 256 bits that represents each operation the device
1072
+
* supports per bit. Iterate through all the bits and check if the input mask
1073
+
* is set for bits that are not set in the OPCAP for the device. If no OPCAP
1074
+
* bit is set and input mask has the bit set, then return error.
1075
+
*/
1076
+
for_each_set_bit(bit, opmask, IDXD_MAX_OPCAP_BITS) {
1077
+
if (!test_bit(bit, idxd->opcap_bmap))
1078
+
return -EINVAL;
1079
+
}
1080
+
1081
+
return 0;
1082
+
}
1083
+
1084
+
static ssize_t wq_op_config_store(struct device *dev, struct device_attribute *attr,
1085
+
const char *buf, size_t count)
1086
+
{
1087
+
struct idxd_wq *wq = confdev_to_wq(dev);
1088
+
struct idxd_device *idxd = wq->idxd;
1089
+
unsigned long *opmask;
1090
+
int rc;
1091
+
1092
+
if (wq->state != IDXD_WQ_DISABLED)
1093
+
return -EPERM;
1094
+
1095
+
opmask = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL);
1096
+
if (!opmask)
1097
+
return -ENOMEM;
1098
+
1099
+
rc = bitmap_parse(buf, count, opmask, IDXD_MAX_OPCAP_BITS);
1100
+
if (rc < 0)
1101
+
goto err;
1102
+
1103
+
rc = idxd_verify_supported_opcap(idxd, opmask);
1104
+
if (rc < 0)
1105
+
goto err;
1106
+
1107
+
bitmap_copy(wq->opcap_bmap, opmask, IDXD_MAX_OPCAP_BITS);
1108
+
1109
+
bitmap_free(opmask);
1110
+
return count;
1111
+
1112
+
err:
1113
+
bitmap_free(opmask);
1114
+
return rc;
1115
+
}
1116
+
1117
+
static struct device_attribute dev_attr_wq_op_config =
1118
+
__ATTR(op_config, 0644, wq_op_config_show, wq_op_config_store);
1119
+
1146
1120
static struct attribute *idxd_wq_attributes[] = {
1147
1121
&dev_attr_wq_clients.attr,
1148
1122
&dev_attr_wq_state.attr,
···
1222
1072
&dev_attr_wq_ats_disable.attr,
1223
1073
&dev_attr_wq_occupancy.attr,
1224
1074
&dev_attr_wq_enqcmds_retries.attr,
1075
+
&dev_attr_wq_op_config.attr,
1225
1076
NULL,
1226
1077
};
1227
1078
1079
+
static bool idxd_wq_attr_op_config_invisible(struct attribute *attr,
1080
+
struct idxd_device *idxd)
1081
+
{
1082
+
return attr == &dev_attr_wq_op_config.attr &&
1083
+
!idxd->hw.wq_cap.op_config;
1084
+
}
1085
+
1086
+
static umode_t idxd_wq_attr_visible(struct kobject *kobj,
1087
+
struct attribute *attr, int n)
1088
+
{
1089
+
struct device *dev = container_of(kobj, struct device, kobj);
1090
+
struct idxd_wq *wq = confdev_to_wq(dev);
1091
+
struct idxd_device *idxd = wq->idxd;
1092
+
1093
+
if (idxd_wq_attr_op_config_invisible(attr, idxd))
1094
+
return 0;
1095
+
1096
+
return attr->mode;
1097
+
}
1098
+
1228
1099
static const struct attribute_group idxd_wq_attribute_group = {
1229
1100
.attrs = idxd_wq_attributes,
1101
+
.is_visible = idxd_wq_attr_visible,
1230
1102
};
1231
1103
1232
1104
static const struct attribute_group *idxd_wq_attribute_groups[] = {
···
1260
1088
{
1261
1089
struct idxd_wq *wq = confdev_to_wq(dev);
1262
1090
1091
+
bitmap_free(wq->opcap_bmap);
1263
1092
kfree(wq->wqcfg);
1264
1093
kfree(wq);
1265
1094
}
···
1350
1177
struct device_attribute *attr, char *buf)
1351
1178
{
1352
1179
struct idxd_device *idxd = confdev_to_idxd(dev);
1353
-
int i, rc = 0;
1354
1180
1355
-
for (i = 0; i < 4; i++)
1356
-
rc += sysfs_emit_at(buf, rc, "%#llx ", idxd->hw.opcap.bits[i]);
1357
-
1358
-
rc--;
1359
-
rc += sysfs_emit_at(buf, rc, "\n");
1360
-
return rc;
1181
+
return sysfs_emit(buf, "%*pb\n", IDXD_MAX_OPCAP_BITS, idxd->opcap_bmap);
1361
1182
}
1362
1183
static DEVICE_ATTR_RO(op_cap);
1363
1184
···
1572
1405
struct idxd_device *idxd = confdev_to_idxd(dev);
1573
1406
1574
1407
kfree(idxd->groups);
1408
+
bitmap_free(idxd->wq_enable_map);
1575
1409
kfree(idxd->wqs);
1576
1410
kfree(idxd->engines);
1577
1411
ida_free(&idxd_ida, idxd->id);
1412
+
bitmap_free(idxd->opcap_bmap);
1578
1413
kfree(idxd);
1579
1414
}
1580
1415
+3
-3
drivers/dma/ioat/dma.c
+3
-3
drivers/dma/ioat/dma.c
···
656
656
if (active - i == 0) {
657
657
dev_dbg(to_dev(ioat_chan), "%s: cancel completion timeout\n",
658
658
__func__);
659
-
mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
659
+
mod_timer_pending(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
660
660
}
661
661
662
662
/* microsecond delay by sysfs variable per pending descriptor */
···
682
682
683
683
if (chanerr &
684
684
(IOAT_CHANERR_HANDLE_MASK | IOAT_CHANERR_RECOVER_MASK)) {
685
-
mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
685
+
mod_timer_pending(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
686
686
ioat_eh(ioat_chan);
687
687
}
688
688
}
···
879
879
}
880
880
881
881
if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state))
882
-
mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
882
+
mod_timer_pending(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
883
883
}
884
884
885
885
static void ioat_reboot_chan(struct ioatdma_chan *ioat_chan)
-2
drivers/dma/ioat/dma.h
-2
drivers/dma/ioat/dma.h
···
196
196
extern struct ioat_sysfs_entry ioat_version_attr;
197
197
extern struct ioat_sysfs_entry ioat_cap_attr;
198
198
extern int ioat_pending_level;
199
-
extern int ioat_ring_alloc_order;
200
199
extern struct kobj_type ioat_ktype;
201
200
extern struct kmem_cache *ioat_cache;
202
-
extern int ioat_ring_max_alloc_order;
203
201
extern struct kmem_cache *ioat_sed_cache;
204
202
205
203
static inline struct ioatdma_chan *to_ioat_chan(struct dma_chan *c)
+4
-7
drivers/dma/mxs-dma.c
+4
-7
drivers/dma/mxs-dma.c
···
670
670
return mxs_chan->status;
671
671
}
672
672
673
-
static int __init mxs_dma_init(struct mxs_dma_engine *mxs_dma)
673
+
static int mxs_dma_init(struct mxs_dma_engine *mxs_dma)
674
674
{
675
675
int ret;
676
676
···
741
741
ofdma->of_node);
742
742
}
743
743
744
-
static int __init mxs_dma_probe(struct platform_device *pdev)
744
+
static int mxs_dma_probe(struct platform_device *pdev)
745
745
{
746
746
struct device_node *np = pdev->dev.of_node;
747
747
const struct mxs_dma_type *dma_type;
···
839
839
.name = "mxs-dma",
840
840
.of_match_table = mxs_dma_dt_ids,
841
841
},
842
+
.probe = mxs_dma_probe,
842
843
};
843
844
844
-
static int __init mxs_dma_module_init(void)
845
-
{
846
-
return platform_driver_probe(&mxs_dma_driver, mxs_dma_probe);
847
-
}
848
-
subsys_initcall(mxs_dma_module_init);
845
+
builtin_platform_driver(mxs_dma_driver);
-4
drivers/dma/pl330.c
-4
drivers/dma/pl330.c
···
2752
2752
return NULL;
2753
2753
2754
2754
pch->cyclic = true;
2755
-
desc->txd.flags = flags;
2756
2755
2757
2756
return &desc->txd;
2758
2757
}
···
2802
2803
desc->rqcfg.brst_len = 1;
2803
2804
2804
2805
desc->bytes_requested = len;
2805
-
2806
-
desc->txd.flags = flags;
2807
2806
2808
2807
return &desc->txd;
2809
2808
}
···
2886
2889
}
2887
2890
2888
2891
/* Return the last desc in the chain */
2889
-
desc->txd.flags = flg;
2890
2892
return &desc->txd;
2891
2893
}
2892
2894
+4
-3
drivers/dma/qcom/gpi.c
+4
-3
drivers/dma/qcom/gpi.c
···
1150
1150
{
1151
1151
struct gpii *gpii = (struct gpii *)data;
1152
1152
1153
-
read_lock_bh(&gpii->pm_lock);
1153
+
read_lock(&gpii->pm_lock);
1154
1154
if (!REG_ACCESS_VALID(gpii->pm_state)) {
1155
-
read_unlock_bh(&gpii->pm_lock);
1155
+
read_unlock(&gpii->pm_lock);
1156
1156
dev_err(gpii->gpi_dev->dev, "not processing any events, pm_state:%s\n",
1157
1157
TO_GPI_PM_STR(gpii->pm_state));
1158
1158
return;
···
1163
1163
1164
1164
/* enable IEOB, switching back to interrupts */
1165
1165
gpi_config_interrupts(gpii, MASK_IEOB_SETTINGS, 1);
1166
-
read_unlock_bh(&gpii->pm_lock);
1166
+
read_unlock(&gpii->pm_lock);
1167
1167
}
1168
1168
1169
1169
/* marks all pending events for the channel as stale */
···
2288
2288
static const struct of_device_id gpi_of_match[] = {
2289
2289
{ .compatible = "qcom,sc7280-gpi-dma", .data = (void *)0x10000 },
2290
2290
{ .compatible = "qcom,sdm845-gpi-dma", .data = (void *)0x0 },
2291
+
{ .compatible = "qcom,sm6350-gpi-dma", .data = (void *)0x10000 },
2291
2292
{ .compatible = "qcom,sm8150-gpi-dma", .data = (void *)0x0 },
2292
2293
{ .compatible = "qcom,sm8250-gpi-dma", .data = (void *)0x0 },
2293
2294
{ .compatible = "qcom,sm8350-gpi-dma", .data = (void *)0x10000 },
+14
-8
drivers/dma/qcom/qcom_adm.c
+14
-8
drivers/dma/qcom/qcom_adm.c
···
379
379
if (blk_size < 0) {
380
380
dev_err(adev->dev, "invalid burst value: %d\n",
381
381
burst);
382
-
return ERR_PTR(-EINVAL);
382
+
return NULL;
383
383
}
384
384
385
385
crci = achan->crci & 0xf;
386
386
if (!crci || achan->crci > 0x1f) {
387
387
dev_err(adev->dev, "invalid crci value\n");
388
-
return ERR_PTR(-EINVAL);
388
+
return NULL;
389
389
}
390
390
}
391
391
···
403
403
}
404
404
405
405
async_desc = kzalloc(sizeof(*async_desc), GFP_NOWAIT);
406
-
if (!async_desc)
407
-
return ERR_PTR(-ENOMEM);
406
+
if (!async_desc) {
407
+
dev_err(adev->dev, "not enough memory for async_desc struct\n");
408
+
return NULL;
409
+
}
408
410
409
411
async_desc->mux = achan->mux ? ADM_CRCI_CTL_MUX_SEL : 0;
410
412
async_desc->crci = crci;
···
416
414
sizeof(*cple) + 2 * ADM_DESC_ALIGN;
417
415
418
416
async_desc->cpl = kzalloc(async_desc->dma_len, GFP_NOWAIT);
419
-
if (!async_desc->cpl)
417
+
if (!async_desc->cpl) {
418
+
dev_err(adev->dev, "not enough memory for cpl struct\n");
420
419
goto free;
420
+
}
421
421
422
422
async_desc->adev = adev;
423
423
···
441
437
async_desc->dma_addr = dma_map_single(adev->dev, async_desc->cpl,
442
438
async_desc->dma_len,
443
439
DMA_TO_DEVICE);
444
-
if (dma_mapping_error(adev->dev, async_desc->dma_addr))
440
+
if (dma_mapping_error(adev->dev, async_desc->dma_addr)) {
441
+
dev_err(adev->dev, "dma mapping error for cpl\n");
445
442
goto free;
443
+
}
446
444
447
445
cple_addr = async_desc->dma_addr + ((void *)cple - async_desc->cpl);
448
446
···
460
454
461
455
free:
462
456
kfree(async_desc);
463
-
return ERR_PTR(-ENOMEM);
457
+
return NULL;
464
458
}
465
459
466
460
/**
···
500
494
501
495
spin_lock_irqsave(&achan->vc.lock, flag);
502
496
memcpy(&achan->slave, cfg, sizeof(struct dma_slave_config));
503
-
if (cfg->peripheral_size == sizeof(config))
497
+
if (cfg->peripheral_size == sizeof(*config))
504
498
achan->crci = config->crci;
505
499
spin_unlock_irqrestore(&achan->vc.lock, flag);
506
500
+1
-1
drivers/dma/s3c24xx-dma.c
+1
-1
drivers/dma/s3c24xx-dma.c
···
1094
1094
INIT_LIST_HEAD(&dmadev->channels);
1095
1095
1096
1096
/*
1097
-
* Register as many many memcpy as we have physical channels,
1097
+
* Register as many memcpy as we have physical channels,
1098
1098
* we won't always be able to use all but the code will have
1099
1099
* to cope with that situation.
1100
1100
*/
+2
-6
drivers/dma/sf-pdma/sf-pdma.c
+2
-6
drivers/dma/sf-pdma/sf-pdma.c
···
405
405
chan = &pdma->chans[i];
406
406
407
407
irq = platform_get_irq(pdev, i * 2);
408
-
if (irq < 0) {
409
-
dev_err(&pdev->dev, "ch(%d) Can't get done irq.\n", i);
408
+
if (irq < 0)
410
409
return -EINVAL;
411
-
}
412
410
413
411
r = devm_request_irq(&pdev->dev, irq, sf_pdma_done_isr, 0,
414
412
dev_name(&pdev->dev), (void *)chan);
···
418
420
chan->txirq = irq;
419
421
420
422
irq = platform_get_irq(pdev, (i * 2) + 1);
421
-
if (irq < 0) {
422
-
dev_err(&pdev->dev, "ch(%d) Can't get err irq.\n", i);
423
+
if (irq < 0)
423
424
return -EINVAL;
424
-
}
425
425
426
426
r = devm_request_irq(&pdev->dev, irq, sf_pdma_err_isr, 0,
427
427
dev_name(&pdev->dev), (void *)chan);
+2
-2
drivers/dma/sh/rcar-dmac.c
+2
-2
drivers/dma/sh/rcar-dmac.c
+83
-53
drivers/dma/stm32-dma.c
+83
-53
drivers/dma/stm32-dma.c
···
9
9
* Pierre-Yves Mordret <pierre-yves.mordret@st.com>
10
10
*/
11
11
12
+
#include <linux/bitfield.h>
12
13
#include <linux/clk.h>
13
14
#include <linux/delay.h>
14
15
#include <linux/dmaengine.h>
···
33
32
34
33
#define STM32_DMA_LISR 0x0000 /* DMA Low Int Status Reg */
35
34
#define STM32_DMA_HISR 0x0004 /* DMA High Int Status Reg */
35
+
#define STM32_DMA_ISR(n) (((n) & 4) ? STM32_DMA_HISR : STM32_DMA_LISR)
36
36
#define STM32_DMA_LIFCR 0x0008 /* DMA Low Int Flag Clear Reg */
37
37
#define STM32_DMA_HIFCR 0x000c /* DMA High Int Flag Clear Reg */
38
+
#define STM32_DMA_IFCR(n) (((n) & 4) ? STM32_DMA_HIFCR : STM32_DMA_LIFCR)
38
39
#define STM32_DMA_TCI BIT(5) /* Transfer Complete Interrupt */
39
40
#define STM32_DMA_HTI BIT(4) /* Half Transfer Interrupt */
40
41
#define STM32_DMA_TEI BIT(3) /* Transfer Error Interrupt */
···
46
43
| STM32_DMA_TEI \
47
44
| STM32_DMA_DMEI \
48
45
| STM32_DMA_FEI)
46
+
/*
47
+
* If (chan->id % 4) is 2 or 3, left shift the mask by 16 bits;
48
+
* if (ch % 4) is 1 or 3, additionally left shift the mask by 6 bits.
49
+
*/
50
+
#define STM32_DMA_FLAGS_SHIFT(n) ({ typeof(n) (_n) = (n); \
51
+
(((_n) & 2) << 3) | (((_n) & 1) * 6); })
49
52
50
53
/* DMA Stream x Configuration Register */
51
54
#define STM32_DMA_SCR(x) (0x0010 + 0x18 * (x)) /* x = 0..7 */
52
-
#define STM32_DMA_SCR_REQ(n) ((n & 0x7) << 25)
55
+
#define STM32_DMA_SCR_REQ_MASK GENMASK(27, 25)
53
56
#define STM32_DMA_SCR_MBURST_MASK GENMASK(24, 23)
54
-
#define STM32_DMA_SCR_MBURST(n) ((n & 0x3) << 23)
55
57
#define STM32_DMA_SCR_PBURST_MASK GENMASK(22, 21)
56
-
#define STM32_DMA_SCR_PBURST(n) ((n & 0x3) << 21)
57
58
#define STM32_DMA_SCR_PL_MASK GENMASK(17, 16)
58
-
#define STM32_DMA_SCR_PL(n) ((n & 0x3) << 16)
59
59
#define STM32_DMA_SCR_MSIZE_MASK GENMASK(14, 13)
60
-
#define STM32_DMA_SCR_MSIZE(n) ((n & 0x3) << 13)
61
60
#define STM32_DMA_SCR_PSIZE_MASK GENMASK(12, 11)
62
-
#define STM32_DMA_SCR_PSIZE(n) ((n & 0x3) << 11)
63
-
#define STM32_DMA_SCR_PSIZE_GET(n) ((n & STM32_DMA_SCR_PSIZE_MASK) >> 11)
64
61
#define STM32_DMA_SCR_DIR_MASK GENMASK(7, 6)
65
-
#define STM32_DMA_SCR_DIR(n) ((n & 0x3) << 6)
66
62
#define STM32_DMA_SCR_TRBUFF BIT(20) /* Bufferable transfer for USART/UART */
67
63
#define STM32_DMA_SCR_CT BIT(19) /* Target in double buffer */
68
64
#define STM32_DMA_SCR_DBM BIT(18) /* Double Buffer Mode */
···
98
96
/* DMA stream x FIFO control register */
99
97
#define STM32_DMA_SFCR(x) (0x0024 + 0x18 * (x))
100
98
#define STM32_DMA_SFCR_FTH_MASK GENMASK(1, 0)
101
-
#define STM32_DMA_SFCR_FTH(n) (n & STM32_DMA_SFCR_FTH_MASK)
102
99
#define STM32_DMA_SFCR_FEIE BIT(7) /* FIFO error interrupt enable */
103
100
#define STM32_DMA_SFCR_DMDIS BIT(2) /* Direct mode disable */
104
101
#define STM32_DMA_SFCR_MASK (STM32_DMA_SFCR_FEIE \
···
138
137
139
138
/* DMA Features */
140
139
#define STM32_DMA_THRESHOLD_FTR_MASK GENMASK(1, 0)
141
-
#define STM32_DMA_THRESHOLD_FTR_GET(n) ((n) & STM32_DMA_THRESHOLD_FTR_MASK)
142
140
#define STM32_DMA_DIRECT_MODE_MASK BIT(2)
143
-
#define STM32_DMA_DIRECT_MODE_GET(n) (((n) & STM32_DMA_DIRECT_MODE_MASK) >> 2)
144
141
#define STM32_DMA_ALT_ACK_MODE_MASK BIT(4)
145
-
#define STM32_DMA_ALT_ACK_MODE_GET(n) (((n) & STM32_DMA_ALT_ACK_MODE_MASK) >> 4)
142
+
#define STM32_DMA_MDMA_STREAM_ID_MASK GENMASK(19, 16)
146
143
147
144
enum stm32_dma_width {
148
145
STM32_DMA_BYTE,
···
194
195
struct stm32_dma_sg_req sg_req[];
195
196
};
196
197
198
+
/**
199
+
* struct stm32_dma_mdma_config - STM32 DMA MDMA configuration
200
+
* @stream_id: DMA request to trigger STM32 MDMA transfer
201
+
* @ifcr: DMA interrupt flag clear register address,
202
+
* used by STM32 MDMA to clear DMA Transfer Complete flag
203
+
* @tcf: DMA Transfer Complete flag
204
+
*/
205
+
struct stm32_dma_mdma_config {
206
+
u32 stream_id;
207
+
u32 ifcr;
208
+
u32 tcf;
209
+
};
210
+
197
211
struct stm32_dma_chan {
198
212
struct virt_dma_chan vchan;
199
213
bool config_init;
···
221
209
u32 mem_burst;
222
210
u32 mem_width;
223
211
enum dma_status status;
212
+
bool trig_mdma;
213
+
struct stm32_dma_mdma_config mdma_config;
224
214
};
225
215
226
216
struct stm32_dma_device {
···
402
388
403
389
memcpy(&chan->dma_sconfig, config, sizeof(*config));
404
390
391
+
/* Check if user is requesting DMA to trigger STM32 MDMA */
392
+
if (config->peripheral_size) {
393
+
config->peripheral_config = &chan->mdma_config;
394
+
config->peripheral_size = sizeof(chan->mdma_config);
395
+
chan->trig_mdma = true;
396
+
}
397
+
405
398
chan->config_init = true;
406
399
407
400
return 0;
···
422
401
/*
423
402
* Read "flags" from DMA_xISR register corresponding to the selected
424
403
* DMA channel at the correct bit offset inside that register.
425
-
*
426
-
* If (ch % 4) is 2 or 3, left shift the mask by 16 bits.
427
-
* If (ch % 4) is 1 or 3, additionally left shift the mask by 6 bits.
428
404
*/
429
405
430
-
if (chan->id & 4)
431
-
dma_isr = stm32_dma_read(dmadev, STM32_DMA_HISR);
432
-
else
433
-
dma_isr = stm32_dma_read(dmadev, STM32_DMA_LISR);
434
-
435
-
flags = dma_isr >> (((chan->id & 2) << 3) | ((chan->id & 1) * 6));
406
+
dma_isr = stm32_dma_read(dmadev, STM32_DMA_ISR(chan->id));
407
+
flags = dma_isr >> STM32_DMA_FLAGS_SHIFT(chan->id);
436
408
437
409
return flags & STM32_DMA_MASKI;
438
410
}
···
438
424
/*
439
425
* Write "flags" to the DMA_xIFCR register corresponding to the selected
440
426
* DMA channel at the correct bit offset inside that register.
441
-
*
442
-
* If (ch % 4) is 2 or 3, left shift the mask by 16 bits.
443
-
* If (ch % 4) is 1 or 3, additionally left shift the mask by 6 bits.
444
427
*/
445
428
flags &= STM32_DMA_MASKI;
446
-
dma_ifcr = flags << (((chan->id & 2) << 3) | ((chan->id & 1) * 6));
429
+
dma_ifcr = flags << STM32_DMA_FLAGS_SHIFT(chan->id);
447
430
448
-
if (chan->id & 4)
449
-
stm32_dma_write(dmadev, STM32_DMA_HIFCR, dma_ifcr);
450
-
else
451
-
stm32_dma_write(dmadev, STM32_DMA_LIFCR, dma_ifcr);
431
+
stm32_dma_write(dmadev, STM32_DMA_IFCR(chan->id), dma_ifcr);
452
432
}
453
433
454
434
static int stm32_dma_disable_chan(struct stm32_dma_chan *chan)
···
583
575
584
576
sg_req = &chan->desc->sg_req[chan->next_sg];
585
577
reg = &sg_req->chan_reg;
578
+
579
+
/* When DMA triggers STM32 MDMA, DMA Transfer Complete is managed by STM32 MDMA */
580
+
if (chan->trig_mdma && chan->dma_sconfig.direction != DMA_MEM_TO_DEV)
581
+
reg->dma_scr &= ~STM32_DMA_SCR_TCIE;
586
582
587
583
reg->dma_scr &= ~STM32_DMA_SCR_EN;
588
584
stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), reg->dma_scr);
···
737
725
738
726
if (chan->desc->cyclic) {
739
727
vchan_cyclic_callback(&chan->desc->vdesc);
728
+
if (chan->trig_mdma)
729
+
return;
740
730
stm32_dma_sg_inc(chan);
741
731
/* cyclic while CIRC/DBM disable => post resume reconfiguration needed */
742
732
if (!(scr & (STM32_DMA_SCR_CIRC | STM32_DMA_SCR_DBM)))
···
875
861
sg_req = &chan->desc->sg_req[chan->next_sg - 1];
876
862
877
863
ndtr = sg_req->chan_reg.dma_sndtr;
878
-
offset = (ndtr - chan_reg.dma_sndtr) << STM32_DMA_SCR_PSIZE_GET(chan_reg.dma_scr);
864
+
offset = (ndtr - chan_reg.dma_sndtr);
865
+
offset <<= FIELD_GET(STM32_DMA_SCR_PSIZE_MASK, chan_reg.dma_scr);
879
866
spar = sg_req->chan_reg.dma_spar;
880
867
sm0ar = sg_req->chan_reg.dma_sm0ar;
881
868
sm1ar = sg_req->chan_reg.dma_sm1ar;
···
988
973
if (src_burst_size < 0)
989
974
return src_burst_size;
990
975
991
-
dma_scr = STM32_DMA_SCR_DIR(STM32_DMA_MEM_TO_DEV) |
992
-
STM32_DMA_SCR_PSIZE(dst_bus_width) |
993
-
STM32_DMA_SCR_MSIZE(src_bus_width) |
994
-
STM32_DMA_SCR_PBURST(dst_burst_size) |
995
-
STM32_DMA_SCR_MBURST(src_burst_size);
976
+
dma_scr = FIELD_PREP(STM32_DMA_SCR_DIR_MASK, STM32_DMA_MEM_TO_DEV) |
977
+
FIELD_PREP(STM32_DMA_SCR_PSIZE_MASK, dst_bus_width) |
978
+
FIELD_PREP(STM32_DMA_SCR_MSIZE_MASK, src_bus_width) |
979
+
FIELD_PREP(STM32_DMA_SCR_PBURST_MASK, dst_burst_size) |
980
+
FIELD_PREP(STM32_DMA_SCR_MBURST_MASK, src_burst_size);
996
981
997
982
/* Set FIFO threshold */
998
983
chan->chan_reg.dma_sfcr &= ~STM32_DMA_SFCR_FTH_MASK;
999
984
if (fifoth != STM32_DMA_FIFO_THRESHOLD_NONE)
1000
-
chan->chan_reg.dma_sfcr |= STM32_DMA_SFCR_FTH(fifoth);
985
+
chan->chan_reg.dma_sfcr |= FIELD_PREP(STM32_DMA_SFCR_FTH_MASK, fifoth);
1001
986
1002
987
/* Set peripheral address */
1003
988
chan->chan_reg.dma_spar = chan->dma_sconfig.dst_addr;
···
1045
1030
if (dst_burst_size < 0)
1046
1031
return dst_burst_size;
1047
1032
1048
-
dma_scr = STM32_DMA_SCR_DIR(STM32_DMA_DEV_TO_MEM) |
1049
-
STM32_DMA_SCR_PSIZE(src_bus_width) |
1050
-
STM32_DMA_SCR_MSIZE(dst_bus_width) |
1051
-
STM32_DMA_SCR_PBURST(src_burst_size) |
1052
-
STM32_DMA_SCR_MBURST(dst_burst_size);
1033
+
dma_scr = FIELD_PREP(STM32_DMA_SCR_DIR_MASK, STM32_DMA_DEV_TO_MEM) |
1034
+
FIELD_PREP(STM32_DMA_SCR_PSIZE_MASK, src_bus_width) |
1035
+
FIELD_PREP(STM32_DMA_SCR_MSIZE_MASK, dst_bus_width) |
1036
+
FIELD_PREP(STM32_DMA_SCR_PBURST_MASK, src_burst_size) |
1037
+
FIELD_PREP(STM32_DMA_SCR_MBURST_MASK, dst_burst_size);
1053
1038
1054
1039
/* Set FIFO threshold */
1055
1040
chan->chan_reg.dma_sfcr &= ~STM32_DMA_SFCR_FTH_MASK;
1056
1041
if (fifoth != STM32_DMA_FIFO_THRESHOLD_NONE)
1057
-
chan->chan_reg.dma_sfcr |= STM32_DMA_SFCR_FTH(fifoth);
1042
+
chan->chan_reg.dma_sfcr |= FIELD_PREP(STM32_DMA_SFCR_FTH_MASK, fifoth);
1058
1043
1059
1044
/* Set peripheral address */
1060
1045
chan->chan_reg.dma_spar = chan->dma_sconfig.src_addr;
···
1114
1099
else
1115
1100
chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_PFCTRL;
1116
1101
1102
+
/* Activate Double Buffer Mode if DMA triggers STM32 MDMA and more than 1 sg */
1103
+
if (chan->trig_mdma && sg_len > 1)
1104
+
chan->chan_reg.dma_scr |= STM32_DMA_SCR_DBM;
1105
+
1117
1106
for_each_sg(sgl, sg, sg_len, i) {
1118
1107
ret = stm32_dma_set_xfer_param(chan, direction, &buswidth,
1119
1108
sg_dma_len(sg),
···
1139
1120
desc->sg_req[i].chan_reg.dma_spar = chan->chan_reg.dma_spar;
1140
1121
desc->sg_req[i].chan_reg.dma_sm0ar = sg_dma_address(sg);
1141
1122
desc->sg_req[i].chan_reg.dma_sm1ar = sg_dma_address(sg);
1123
+
if (chan->trig_mdma)
1124
+
desc->sg_req[i].chan_reg.dma_sm1ar += sg_dma_len(sg);
1142
1125
desc->sg_req[i].chan_reg.dma_sndtr = nb_data_items;
1143
1126
}
1144
1127
···
1228
1207
desc->sg_req[i].chan_reg.dma_spar = chan->chan_reg.dma_spar;
1229
1208
desc->sg_req[i].chan_reg.dma_sm0ar = buf_addr;
1230
1209
desc->sg_req[i].chan_reg.dma_sm1ar = buf_addr;
1210
+
if (chan->trig_mdma)
1211
+
desc->sg_req[i].chan_reg.dma_sm1ar += period_len;
1231
1212
desc->sg_req[i].chan_reg.dma_sndtr = nb_data_items;
1232
-
buf_addr += period_len;
1213
+
if (!chan->trig_mdma)
1214
+
buf_addr += period_len;
1233
1215
}
1234
1216
1235
1217
desc->num_sgs = num_periods;
···
1271
1247
1272
1248
stm32_dma_clear_reg(&desc->sg_req[i].chan_reg);
1273
1249
desc->sg_req[i].chan_reg.dma_scr =
1274
-
STM32_DMA_SCR_DIR(STM32_DMA_MEM_TO_MEM) |
1275
-
STM32_DMA_SCR_PBURST(dma_burst) |
1276
-
STM32_DMA_SCR_MBURST(dma_burst) |
1250
+
FIELD_PREP(STM32_DMA_SCR_DIR_MASK, STM32_DMA_MEM_TO_MEM) |
1251
+
FIELD_PREP(STM32_DMA_SCR_PBURST_MASK, dma_burst) |
1252
+
FIELD_PREP(STM32_DMA_SCR_MBURST_MASK, dma_burst) |
1277
1253
STM32_DMA_SCR_MINC |
1278
1254
STM32_DMA_SCR_PINC |
1279
1255
STM32_DMA_SCR_TCIE |
1280
1256
STM32_DMA_SCR_TEIE;
1281
1257
desc->sg_req[i].chan_reg.dma_sfcr |= STM32_DMA_SFCR_MASK;
1282
-
desc->sg_req[i].chan_reg.dma_sfcr |=
1283
-
STM32_DMA_SFCR_FTH(threshold);
1258
+
desc->sg_req[i].chan_reg.dma_sfcr |= FIELD_PREP(STM32_DMA_SFCR_FTH_MASK, threshold);
1284
1259
desc->sg_req[i].chan_reg.dma_spar = src + offset;
1285
1260
desc->sg_req[i].chan_reg.dma_sm0ar = dest + offset;
1286
1261
desc->sg_req[i].chan_reg.dma_sndtr = xfer_count;
···
1298
1275
struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);
1299
1276
1300
1277
dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id));
1301
-
width = STM32_DMA_SCR_PSIZE_GET(dma_scr);
1278
+
width = FIELD_GET(STM32_DMA_SCR_PSIZE_MASK, dma_scr);
1302
1279
ndtr = stm32_dma_read(dmadev, STM32_DMA_SNDTR(chan->id));
1303
1280
1304
1281
return ndtr << width;
···
1504
1481
stm32_dma_clear_reg(&chan->chan_reg);
1505
1482
1506
1483
chan->chan_reg.dma_scr = cfg->stream_config & STM32_DMA_SCR_CFG_MASK;
1507
-
chan->chan_reg.dma_scr |= STM32_DMA_SCR_REQ(cfg->request_line);
1484
+
chan->chan_reg.dma_scr |= FIELD_PREP(STM32_DMA_SCR_REQ_MASK, cfg->request_line);
1508
1485
1509
1486
/* Enable Interrupts */
1510
1487
chan->chan_reg.dma_scr |= STM32_DMA_SCR_TEIE | STM32_DMA_SCR_TCIE;
1511
1488
1512
-
chan->threshold = STM32_DMA_THRESHOLD_FTR_GET(cfg->features);
1513
-
if (STM32_DMA_DIRECT_MODE_GET(cfg->features))
1489
+
chan->threshold = FIELD_GET(STM32_DMA_THRESHOLD_FTR_MASK, cfg->features);
1490
+
if (FIELD_GET(STM32_DMA_DIRECT_MODE_MASK, cfg->features))
1514
1491
chan->threshold = STM32_DMA_FIFO_THRESHOLD_NONE;
1515
-
if (STM32_DMA_ALT_ACK_MODE_GET(cfg->features))
1492
+
if (FIELD_GET(STM32_DMA_ALT_ACK_MODE_MASK, cfg->features))
1516
1493
chan->chan_reg.dma_scr |= STM32_DMA_SCR_TRBUFF;
1494
+
chan->mdma_config.stream_id = FIELD_GET(STM32_DMA_MDMA_STREAM_ID_MASK, cfg->features);
1517
1495
}
1518
1496
1519
1497
static struct dma_chan *stm32_dma_of_xlate(struct of_phandle_args *dma_spec,
···
1654
1630
chan->id = i;
1655
1631
chan->vchan.desc_free = stm32_dma_desc_free;
1656
1632
vchan_init(&chan->vchan, dd);
1633
+
1634
+
chan->mdma_config.ifcr = res->start;
1635
+
chan->mdma_config.ifcr += STM32_DMA_IFCR(chan->id);
1636
+
1637
+
chan->mdma_config.tcf = STM32_DMA_TCI;
1638
+
chan->mdma_config.tcf <<= STM32_DMA_FLAGS_SHIFT(chan->id);
1657
1639
}
1658
1640
1659
1641
ret = dma_async_device_register(dd);
+3
-9
drivers/dma/stm32-dmamux.c
+3
-9
drivers/dma/stm32-dmamux.c
···
39
39
u32 dma_requests; /* Number of DMA requests connected to DMAMUX */
40
40
u32 dmamux_requests; /* Number of DMA requests routed toward DMAs */
41
41
spinlock_t lock; /* Protects register access */
42
-
unsigned long *dma_inuse; /* Used DMA channel */
42
+
DECLARE_BITMAP(dma_inuse, STM32_DMAMUX_MAX_DMA_REQUESTS); /* Used DMA channel */
43
43
u32 ccr[STM32_DMAMUX_MAX_DMA_REQUESTS]; /* Used to backup CCR register
44
44
* in suspend
45
45
*/
46
46
u32 dma_reqs[]; /* Number of DMA Request per DMA masters.
47
47
* [0] holds number of DMA Masters.
48
-
* To be kept at very end end of this structure
48
+
* To be kept at very end of this structure
49
49
*/
50
50
};
51
51
···
147
147
mux->request = dma_spec->args[0];
148
148
149
149
/* craft DMA spec */
150
-
dma_spec->args[3] = dma_spec->args[2];
150
+
dma_spec->args[3] = dma_spec->args[2] | mux->chan_id << 16;
151
151
dma_spec->args[2] = dma_spec->args[1];
152
152
dma_spec->args[1] = 0;
153
153
dma_spec->args[0] = mux->chan_id - min;
···
229
229
230
230
stm32_dmamux->dma_requests = dma_req;
231
231
stm32_dmamux->dma_reqs[0] = count;
232
-
stm32_dmamux->dma_inuse = devm_kcalloc(&pdev->dev,
233
-
BITS_TO_LONGS(dma_req),
234
-
sizeof(unsigned long),
235
-
GFP_KERNEL);
236
-
if (!stm32_dmamux->dma_inuse)
237
-
return -ENOMEM;
238
232
239
233
if (device_property_read_u32(&pdev->dev, "dma-requests",
240
234
&stm32_dmamux->dmamux_requests)) {
+69
-1
drivers/dma/stm32-mdma.c
+69
-1
drivers/dma/stm32-mdma.c
···
199
199
u32 transfer_config;
200
200
u32 mask_addr;
201
201
u32 mask_data;
202
+
bool m2m_hw; /* True when MDMA is triggered by STM32 DMA */
202
203
};
203
204
204
205
struct stm32_mdma_hwdesc {
···
226
225
bool cyclic;
227
226
u32 count;
228
227
struct stm32_mdma_desc_node node[];
228
+
};
229
+
230
+
struct stm32_mdma_dma_config {
231
+
u32 request; /* STM32 DMA channel stream id, triggering MDMA */
232
+
u32 cmar; /* STM32 DMA interrupt flag clear register address */
233
+
u32 cmdr; /* STM32 DMA Transfer Complete flag */
229
234
};
230
235
231
236
struct stm32_mdma_chan {
···
546
539
dst_addr = chan->dma_config.dst_addr;
547
540
548
541
/* Set device data size */
542
+
if (chan_config->m2m_hw)
543
+
dst_addr_width = stm32_mdma_get_max_width(dst_addr, buf_len,
544
+
STM32_MDMA_MAX_BUF_LEN);
549
545
dst_bus_width = stm32_mdma_get_width(chan, dst_addr_width);
550
546
if (dst_bus_width < 0)
551
547
return dst_bus_width;
552
548
ctcr &= ~STM32_MDMA_CTCR_DSIZE_MASK;
553
549
ctcr |= STM32_MDMA_CTCR_DSIZE(dst_bus_width);
550
+
if (chan_config->m2m_hw) {
551
+
ctcr &= ~STM32_MDMA_CTCR_DINCOS_MASK;
552
+
ctcr |= STM32_MDMA_CTCR_DINCOS(dst_bus_width);
553
+
}
554
554
555
555
/* Set device burst value */
556
+
if (chan_config->m2m_hw)
557
+
dst_maxburst = STM32_MDMA_MAX_BUF_LEN / dst_addr_width;
558
+
556
559
dst_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,
557
560
dst_maxburst,
558
561
dst_addr_width);
···
605
588
src_addr = chan->dma_config.src_addr;
606
589
607
590
/* Set device data size */
591
+
if (chan_config->m2m_hw)
592
+
src_addr_width = stm32_mdma_get_max_width(src_addr, buf_len,
593
+
STM32_MDMA_MAX_BUF_LEN);
594
+
608
595
src_bus_width = stm32_mdma_get_width(chan, src_addr_width);
609
596
if (src_bus_width < 0)
610
597
return src_bus_width;
611
598
ctcr &= ~STM32_MDMA_CTCR_SSIZE_MASK;
612
599
ctcr |= STM32_MDMA_CTCR_SSIZE(src_bus_width);
600
+
if (chan_config->m2m_hw) {
601
+
ctcr &= ~STM32_MDMA_CTCR_SINCOS_MASK;
602
+
ctcr |= STM32_MDMA_CTCR_SINCOS(src_bus_width);
603
+
}
613
604
614
605
/* Set device burst value */
606
+
if (chan_config->m2m_hw)
607
+
src_maxburst = STM32_MDMA_MAX_BUF_LEN / src_addr_width;
608
+
615
609
src_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,
616
610
src_maxburst,
617
611
src_addr_width);
···
730
702
{
731
703
struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
732
704
struct dma_slave_config *dma_config = &chan->dma_config;
705
+
struct stm32_mdma_chan_config *chan_config = &chan->chan_config;
733
706
struct scatterlist *sg;
734
707
dma_addr_t src_addr, dst_addr;
735
-
u32 ccr, ctcr, ctbr;
708
+
u32 m2m_hw_period, ccr, ctcr, ctbr;
736
709
int i, ret = 0;
710
+
711
+
if (chan_config->m2m_hw)
712
+
m2m_hw_period = sg_dma_len(sgl);
737
713
738
714
for_each_sg(sgl, sg, sg_len, i) {
739
715
if (sg_dma_len(sg) > STM32_MDMA_MAX_BLOCK_LEN) {
···
748
716
if (direction == DMA_MEM_TO_DEV) {
749
717
src_addr = sg_dma_address(sg);
750
718
dst_addr = dma_config->dst_addr;
719
+
if (chan_config->m2m_hw && (i & 1))
720
+
dst_addr += m2m_hw_period;
751
721
ret = stm32_mdma_set_xfer_param(chan, direction, &ccr,
752
722
&ctcr, &ctbr, src_addr,
753
723
sg_dma_len(sg));
···
757
723
src_addr);
758
724
} else {
759
725
src_addr = dma_config->src_addr;
726
+
if (chan_config->m2m_hw && (i & 1))
727
+
src_addr += m2m_hw_period;
760
728
dst_addr = sg_dma_address(sg);
761
729
ret = stm32_mdma_set_xfer_param(chan, direction, &ccr,
762
730
&ctcr, &ctbr, dst_addr,
···
791
755
unsigned long flags, void *context)
792
756
{
793
757
struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
758
+
struct stm32_mdma_chan_config *chan_config = &chan->chan_config;
794
759
struct stm32_mdma_desc *desc;
795
760
int i, ret;
796
761
···
814
777
if (ret < 0)
815
778
goto xfer_setup_err;
816
779
780
+
/*
781
+
* In case of M2M HW transfer triggered by STM32 DMA, we do not have to clear the
782
+
* transfer complete flag by hardware in order to let the CPU rearm the STM32 DMA
783
+
* with the next sg element and update some data in dmaengine framework.
784
+
*/
785
+
if (chan_config->m2m_hw && direction == DMA_MEM_TO_DEV) {
786
+
struct stm32_mdma_hwdesc *hwdesc;
787
+
788
+
for (i = 0; i < sg_len; i++) {
789
+
hwdesc = desc->node[i].hwdesc;
790
+
hwdesc->cmar = 0;
791
+
hwdesc->cmdr = 0;
792
+
}
793
+
}
794
+
817
795
desc->cyclic = false;
818
796
819
797
return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
···
850
798
struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
851
799
struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
852
800
struct dma_slave_config *dma_config = &chan->dma_config;
801
+
struct stm32_mdma_chan_config *chan_config = &chan->chan_config;
853
802
struct stm32_mdma_desc *desc;
854
803
dma_addr_t src_addr, dst_addr;
855
804
u32 ccr, ctcr, ctbr, count;
···
911
858
if (direction == DMA_MEM_TO_DEV) {
912
859
src_addr = buf_addr + i * period_len;
913
860
dst_addr = dma_config->dst_addr;
861
+
if (chan_config->m2m_hw && (i & 1))
862
+
dst_addr += period_len;
914
863
} else {
915
864
src_addr = dma_config->src_addr;
865
+
if (chan_config->m2m_hw && (i & 1))
866
+
src_addr += period_len;
916
867
dst_addr = buf_addr + i * period_len;
917
868
}
918
869
···
1300
1243
struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1301
1244
1302
1245
memcpy(&chan->dma_config, config, sizeof(*config));
1246
+
1247
+
/* Check if user is requesting STM32 DMA to trigger MDMA */
1248
+
if (config->peripheral_size) {
1249
+
struct stm32_mdma_dma_config *mdma_config;
1250
+
1251
+
mdma_config = (struct stm32_mdma_dma_config *)chan->dma_config.peripheral_config;
1252
+
chan->chan_config.request = mdma_config->request;
1253
+
chan->chan_config.mask_addr = mdma_config->cmar;
1254
+
chan->chan_config.mask_data = mdma_config->cmdr;
1255
+
chan->chan_config.m2m_hw = true;
1256
+
}
1303
1257
1304
1258
return 0;
1305
1259
}
-40
drivers/dma/ti/edma.c
-40
drivers/dma/ti/edma.c
···
352
352
edma_modify(ecc, offset + (i << 2), and, or);
353
353
}
354
354
355
-
static inline void edma_or_array(struct edma_cc *ecc, int offset, int i,
356
-
unsigned or)
357
-
{
358
-
edma_or(ecc, offset + (i << 2), or);
359
-
}
360
-
361
355
static inline void edma_or_array2(struct edma_cc *ecc, int offset, int i, int j,
362
356
unsigned or)
363
357
{
···
362
368
int j, unsigned val)
363
369
{
364
370
edma_write(ecc, offset + ((i * 2 + j) << 2), val);
365
-
}
366
-
367
-
static inline unsigned int edma_shadow0_read(struct edma_cc *ecc, int offset)
368
-
{
369
-
return edma_read(ecc, EDMA_SHADOW0 + offset);
370
371
}
371
372
372
373
static inline unsigned int edma_shadow0_read_array(struct edma_cc *ecc,
···
382
393
edma_write(ecc, EDMA_SHADOW0 + offset + (i << 2), val);
383
394
}
384
395
385
-
static inline unsigned int edma_param_read(struct edma_cc *ecc, int offset,
386
-
int param_no)
387
-
{
388
-
return edma_read(ecc, EDMA_PARM + offset + (param_no << 5));
389
-
}
390
-
391
-
static inline void edma_param_write(struct edma_cc *ecc, int offset,
392
-
int param_no, unsigned val)
393
-
{
394
-
edma_write(ecc, EDMA_PARM + offset + (param_no << 5), val);
395
-
}
396
-
397
396
static inline void edma_param_modify(struct edma_cc *ecc, int offset,
398
397
int param_no, unsigned and, unsigned or)
399
398
{
400
399
edma_modify(ecc, EDMA_PARM + offset + (param_no << 5), and, or);
401
-
}
402
-
403
-
static inline void edma_param_and(struct edma_cc *ecc, int offset, int param_no,
404
-
unsigned and)
405
-
{
406
-
edma_and(ecc, EDMA_PARM + offset + (param_no << 5), and);
407
-
}
408
-
409
-
static inline void edma_param_or(struct edma_cc *ecc, int offset, int param_no,
410
-
unsigned or)
411
-
{
412
-
edma_or(ecc, EDMA_PARM + offset + (param_no << 5), or);
413
400
}
414
401
415
402
static void edma_assign_priority_to_queue(struct edma_cc *ecc, int queue_no,
···
706
741
edma_stop(echan);
707
742
/* REVISIT should probably take out of shadow region 0 */
708
743
edma_setup_interrupt(echan, false);
709
-
}
710
-
711
-
static inline struct edma_cc *to_edma_cc(struct dma_device *d)
712
-
{
713
-
return container_of(d, struct edma_cc, dma_slave);
714
744
}
715
745
716
746
static inline struct edma_chan *to_edma_chan(struct dma_chan *c)
+67
drivers/dma/ti/k3-psil-j7200.c
+67
drivers/dma/ti/k3-psil-j7200.c
···
143
143
144
144
/* PSI-L destination thread IDs, used for TX (DMA_MEM_TO_DEV) */
145
145
static struct psil_ep j7200_dst_ep_map[] = {
146
+
/* PDMA_MCASP - McASP0-2 */
147
+
PSIL_PDMA_MCASP(0xc400),
148
+
PSIL_PDMA_MCASP(0xc401),
149
+
PSIL_PDMA_MCASP(0xc402),
150
+
/* PDMA_SPI_G0 - SPI0-3 */
151
+
PSIL_PDMA_XY_PKT(0xc600),
152
+
PSIL_PDMA_XY_PKT(0xc601),
153
+
PSIL_PDMA_XY_PKT(0xc602),
154
+
PSIL_PDMA_XY_PKT(0xc603),
155
+
PSIL_PDMA_XY_PKT(0xc604),
156
+
PSIL_PDMA_XY_PKT(0xc605),
157
+
PSIL_PDMA_XY_PKT(0xc606),
158
+
PSIL_PDMA_XY_PKT(0xc607),
159
+
PSIL_PDMA_XY_PKT(0xc608),
160
+
PSIL_PDMA_XY_PKT(0xc609),
161
+
PSIL_PDMA_XY_PKT(0xc60a),
162
+
PSIL_PDMA_XY_PKT(0xc60b),
163
+
PSIL_PDMA_XY_PKT(0xc60c),
164
+
PSIL_PDMA_XY_PKT(0xc60d),
165
+
PSIL_PDMA_XY_PKT(0xc60e),
166
+
PSIL_PDMA_XY_PKT(0xc60f),
167
+
/* PDMA_SPI_G1 - SPI4-7 */
168
+
PSIL_PDMA_XY_PKT(0xc610),
169
+
PSIL_PDMA_XY_PKT(0xc611),
170
+
PSIL_PDMA_XY_PKT(0xc612),
171
+
PSIL_PDMA_XY_PKT(0xc613),
172
+
PSIL_PDMA_XY_PKT(0xc614),
173
+
PSIL_PDMA_XY_PKT(0xc615),
174
+
PSIL_PDMA_XY_PKT(0xc616),
175
+
PSIL_PDMA_XY_PKT(0xc617),
176
+
PSIL_PDMA_XY_PKT(0xc618),
177
+
PSIL_PDMA_XY_PKT(0xc619),
178
+
PSIL_PDMA_XY_PKT(0xc61a),
179
+
PSIL_PDMA_XY_PKT(0xc61b),
180
+
PSIL_PDMA_XY_PKT(0xc61c),
181
+
PSIL_PDMA_XY_PKT(0xc61d),
182
+
PSIL_PDMA_XY_PKT(0xc61e),
183
+
PSIL_PDMA_XY_PKT(0xc61f),
184
+
/* PDMA_USART_G0 - UART0-1 */
185
+
PSIL_PDMA_XY_PKT(0xc700),
186
+
PSIL_PDMA_XY_PKT(0xc701),
187
+
/* PDMA_USART_G1 - UART2-3 */
188
+
PSIL_PDMA_XY_PKT(0xc702),
189
+
PSIL_PDMA_XY_PKT(0xc703),
190
+
/* PDMA_USART_G2 - UART4-9 */
191
+
PSIL_PDMA_XY_PKT(0xc704),
192
+
PSIL_PDMA_XY_PKT(0xc705),
193
+
PSIL_PDMA_XY_PKT(0xc706),
194
+
PSIL_PDMA_XY_PKT(0xc707),
195
+
PSIL_PDMA_XY_PKT(0xc708),
196
+
PSIL_PDMA_XY_PKT(0xc709),
146
197
/* CPSW5 */
147
198
PSIL_ETHERNET(0xca00),
148
199
PSIL_ETHERNET(0xca01),
···
212
161
PSIL_ETHERNET(0xf005),
213
162
PSIL_ETHERNET(0xf006),
214
163
PSIL_ETHERNET(0xf007),
164
+
/* MCU_PDMA_MISC_G0 - SPI0 */
165
+
PSIL_PDMA_XY_PKT(0xf100),
166
+
PSIL_PDMA_XY_PKT(0xf101),
167
+
PSIL_PDMA_XY_PKT(0xf102),
168
+
PSIL_PDMA_XY_PKT(0xf103),
169
+
/* MCU_PDMA_MISC_G1 - SPI1-2 */
170
+
PSIL_PDMA_XY_PKT(0xf200),
171
+
PSIL_PDMA_XY_PKT(0xf201),
172
+
PSIL_PDMA_XY_PKT(0xf202),
173
+
PSIL_PDMA_XY_PKT(0xf203),
174
+
PSIL_PDMA_XY_PKT(0xf204),
175
+
PSIL_PDMA_XY_PKT(0xf205),
176
+
PSIL_PDMA_XY_PKT(0xf206),
177
+
PSIL_PDMA_XY_PKT(0xf207),
178
+
/* MCU_PDMA_MISC_G2 - UART0 */
179
+
PSIL_PDMA_XY_PKT(0xf300),
215
180
/* SA2UL */
216
181
PSIL_SA2UL(0xf500, 1),
217
182
PSIL_SA2UL(0xf501, 1),
+79
drivers/dma/ti/k3-psil-j721e.c
+79
drivers/dma/ti/k3-psil-j721e.c
···
266
266
PSIL_ETHERNET(0xc205),
267
267
PSIL_ETHERNET(0xc206),
268
268
PSIL_ETHERNET(0xc207),
269
+
/* PDMA6 (PSIL_PDMA_MCASP_G0) - McASP0-2 */
270
+
PSIL_PDMA_MCASP(0xc400),
271
+
PSIL_PDMA_MCASP(0xc401),
272
+
PSIL_PDMA_MCASP(0xc402),
273
+
/* PDMA7 (PSIL_PDMA_MCASP_G1) - McASP3-11 */
274
+
PSIL_PDMA_MCASP(0xc500),
275
+
PSIL_PDMA_MCASP(0xc501),
276
+
PSIL_PDMA_MCASP(0xc502),
277
+
PSIL_PDMA_MCASP(0xc503),
278
+
PSIL_PDMA_MCASP(0xc504),
279
+
PSIL_PDMA_MCASP(0xc505),
280
+
PSIL_PDMA_MCASP(0xc506),
281
+
PSIL_PDMA_MCASP(0xc507),
282
+
PSIL_PDMA_MCASP(0xc508),
283
+
/* PDMA8 (PDMA_MISC_G0) - SPI0-1 */
284
+
PSIL_PDMA_XY_PKT(0xc600),
285
+
PSIL_PDMA_XY_PKT(0xc601),
286
+
PSIL_PDMA_XY_PKT(0xc602),
287
+
PSIL_PDMA_XY_PKT(0xc603),
288
+
PSIL_PDMA_XY_PKT(0xc604),
289
+
PSIL_PDMA_XY_PKT(0xc605),
290
+
PSIL_PDMA_XY_PKT(0xc606),
291
+
PSIL_PDMA_XY_PKT(0xc607),
292
+
/* PDMA9 (PDMA_MISC_G1) - SPI2-3 */
293
+
PSIL_PDMA_XY_PKT(0xc60c),
294
+
PSIL_PDMA_XY_PKT(0xc60d),
295
+
PSIL_PDMA_XY_PKT(0xc60e),
296
+
PSIL_PDMA_XY_PKT(0xc60f),
297
+
PSIL_PDMA_XY_PKT(0xc610),
298
+
PSIL_PDMA_XY_PKT(0xc611),
299
+
PSIL_PDMA_XY_PKT(0xc612),
300
+
PSIL_PDMA_XY_PKT(0xc613),
301
+
/* PDMA10 (PDMA_MISC_G2) - SPI4-5 */
302
+
PSIL_PDMA_XY_PKT(0xc618),
303
+
PSIL_PDMA_XY_PKT(0xc619),
304
+
PSIL_PDMA_XY_PKT(0xc61a),
305
+
PSIL_PDMA_XY_PKT(0xc61b),
306
+
PSIL_PDMA_XY_PKT(0xc61c),
307
+
PSIL_PDMA_XY_PKT(0xc61d),
308
+
PSIL_PDMA_XY_PKT(0xc61e),
309
+
PSIL_PDMA_XY_PKT(0xc61f),
310
+
/* PDMA11 (PDMA_MISC_G3) */
311
+
PSIL_PDMA_XY_PKT(0xc624),
312
+
PSIL_PDMA_XY_PKT(0xc625),
313
+
PSIL_PDMA_XY_PKT(0xc626),
314
+
PSIL_PDMA_XY_PKT(0xc627),
315
+
PSIL_PDMA_XY_PKT(0xc628),
316
+
PSIL_PDMA_XY_PKT(0xc629),
317
+
PSIL_PDMA_XY_PKT(0xc630),
318
+
PSIL_PDMA_XY_PKT(0xc63a),
319
+
/* PDMA13 (PDMA_USART_G0) - UART0-1 */
320
+
PSIL_PDMA_XY_PKT(0xc700),
321
+
PSIL_PDMA_XY_PKT(0xc701),
322
+
/* PDMA14 (PDMA_USART_G1) - UART2-3 */
323
+
PSIL_PDMA_XY_PKT(0xc702),
324
+
PSIL_PDMA_XY_PKT(0xc703),
325
+
/* PDMA15 (PDMA_USART_G2) - UART4-9 */
326
+
PSIL_PDMA_XY_PKT(0xc704),
327
+
PSIL_PDMA_XY_PKT(0xc705),
328
+
PSIL_PDMA_XY_PKT(0xc706),
329
+
PSIL_PDMA_XY_PKT(0xc707),
330
+
PSIL_PDMA_XY_PKT(0xc708),
331
+
PSIL_PDMA_XY_PKT(0xc709),
269
332
/* CPSW9 */
270
333
PSIL_ETHERNET(0xca00),
271
334
PSIL_ETHERNET(0xca01),
···
347
284
PSIL_ETHERNET(0xf005),
348
285
PSIL_ETHERNET(0xf006),
349
286
PSIL_ETHERNET(0xf007),
287
+
/* MCU_PDMA0 (MCU_PDMA_MISC_G0) - SPI0 */
288
+
PSIL_PDMA_XY_PKT(0xf100),
289
+
PSIL_PDMA_XY_PKT(0xf101),
290
+
PSIL_PDMA_XY_PKT(0xf102),
291
+
PSIL_PDMA_XY_PKT(0xf103),
292
+
/* MCU_PDMA1 (MCU_PDMA_MISC_G1) - SPI1-2 */
293
+
PSIL_PDMA_XY_PKT(0xf200),
294
+
PSIL_PDMA_XY_PKT(0xf201),
295
+
PSIL_PDMA_XY_PKT(0xf202),
296
+
PSIL_PDMA_XY_PKT(0xf203),
297
+
PSIL_PDMA_XY_PKT(0xf204),
298
+
PSIL_PDMA_XY_PKT(0xf205),
299
+
PSIL_PDMA_XY_PKT(0xf206),
300
+
PSIL_PDMA_XY_PKT(0xf207),
301
+
/* MCU_PDMA2 (MCU_PDMA_MISC_G2) - UART0 */
302
+
PSIL_PDMA_XY_PKT(0xf300),
350
303
/* SA2UL */
351
304
PSIL_SA2UL(0xf500, 1),
352
305
PSIL_SA2UL(0xf501, 1),
+27
-10
drivers/dma/ti/k3-udma.c
+27
-10
drivers/dma/ti/k3-udma.c
···
263
263
enum udma_tp_level channel_tpl; /* Channel Throughput Level */
264
264
265
265
u32 tr_trigger_type;
266
+
unsigned long tx_flags;
266
267
267
268
/* PKDMA mapped channel */
268
269
int mapped_channel_id;
···
300
299
struct completion teardown_completed;
301
300
302
301
struct udma_tx_drain tx_drain;
303
-
304
-
u32 bcnt; /* number of bytes completed since the start of the channel */
305
302
306
303
/* Channel configuration parameters */
307
304
struct udma_chan_config config;
···
756
757
}
757
758
}
758
759
760
+
static void udma_decrement_byte_counters(struct udma_chan *uc, u32 val)
761
+
{
762
+
if (uc->desc->dir == DMA_DEV_TO_MEM) {
763
+
udma_rchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val);
764
+
udma_rchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val);
765
+
udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val);
766
+
} else {
767
+
udma_tchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val);
768
+
udma_tchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val);
769
+
if (!uc->bchan)
770
+
udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val);
771
+
}
772
+
}
773
+
759
774
static void udma_reset_counters(struct udma_chan *uc)
760
775
{
761
776
u32 val;
···
803
790
val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);
804
791
udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val);
805
792
}
806
-
807
-
uc->bcnt = 0;
808
793
}
809
794
810
795
static int udma_reset_chan(struct udma_chan *uc, bool hard)
···
1056
1045
{
1057
1046
u32 peer_bcnt, bcnt;
1058
1047
1059
-
/* Only TX towards PDMA is affected */
1048
+
/*
1049
+
* Only TX towards PDMA is affected.
1050
+
* If DMA_PREP_INTERRUPT is not set by consumer then skip the transfer
1051
+
* completion calculation, consumer must ensure that there is no stale
1052
+
* data in DMA fabric in this case.
1053
+
*/
1060
1054
if (uc->config.ep_type == PSIL_EP_NATIVE ||
1061
-
uc->config.dir != DMA_MEM_TO_DEV)
1055
+
uc->config.dir != DMA_MEM_TO_DEV || !(uc->config.tx_flags & DMA_PREP_INTERRUPT))
1062
1056
return true;
1063
1057
1064
1058
peer_bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);
···
1131
1115
if (uc->desc) {
1132
1116
struct udma_desc *d = uc->desc;
1133
1117
1134
-
uc->bcnt += d->residue;
1118
+
udma_decrement_byte_counters(uc, d->residue);
1135
1119
udma_start(uc);
1136
1120
vchan_cookie_complete(&d->vd);
1137
1121
break;
···
1184
1168
vchan_cyclic_callback(&d->vd);
1185
1169
} else {
1186
1170
if (udma_is_desc_really_done(uc, d)) {
1187
-
uc->bcnt += d->residue;
1171
+
udma_decrement_byte_counters(uc, d->residue);
1188
1172
udma_start(uc);
1189
1173
vchan_cookie_complete(&d->vd);
1190
1174
} else {
···
1220
1204
vchan_cyclic_callback(&d->vd);
1221
1205
} else {
1222
1206
/* TODO: figure out the real amount of data */
1223
-
uc->bcnt += d->residue;
1207
+
udma_decrement_byte_counters(uc, d->residue);
1224
1208
udma_start(uc);
1225
1209
vchan_cookie_complete(&d->vd);
1226
1210
}
···
3424
3408
if (!burst)
3425
3409
burst = 1;
3426
3410
3411
+
uc->config.tx_flags = tx_flags;
3412
+
3427
3413
if (uc->config.pkt_mode)
3428
3414
d = udma_prep_slave_sg_pkt(uc, sgl, sglen, dir, tx_flags,
3429
3415
context);
···
3827
3809
bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
3828
3810
}
3829
3811
3830
-
bcnt -= uc->bcnt;
3831
3812
if (bcnt && !(bcnt % uc->desc->residue))
3832
3813
residue = 0;
3833
3814
else
+12
drivers/dma/xilinx/zynqmp_dma.c
+12
drivers/dma/xilinx/zynqmp_dma.c
···
796
796
}
797
797
798
798
/**
799
+
* zynqmp_dma_synchronize - Synchronizes the termination of a transfers to the current context.
800
+
* @dchan: DMA channel pointer
801
+
*/
802
+
static void zynqmp_dma_synchronize(struct dma_chan *dchan)
803
+
{
804
+
struct zynqmp_dma_chan *chan = to_chan(dchan);
805
+
806
+
tasklet_kill(&chan->tasklet);
807
+
}
808
+
809
+
/**
799
810
* zynqmp_dma_prep_memcpy - prepare descriptors for memcpy transaction
800
811
* @dchan: DMA channel
801
812
* @dma_dst: Destination buffer address
···
1068
1057
p = &zdev->common;
1069
1058
p->device_prep_dma_memcpy = zynqmp_dma_prep_memcpy;
1070
1059
p->device_terminate_all = zynqmp_dma_device_terminate_all;
1060
+
p->device_synchronize = zynqmp_dma_synchronize;
1071
1061
p->device_issue_pending = zynqmp_dma_issue_pending;
1072
1062
p->device_alloc_chan_resources = zynqmp_dma_alloc_chan_resources;
1073
1063
p->device_free_chan_resources = zynqmp_dma_free_chan_resources;
+4
-2
include/linux/dma/hsu.h
+4
-2
include/linux/dma/hsu.h
···
8
8
#ifndef _DMA_HSU_H
9
9
#define _DMA_HSU_H
10
10
11
-
#include <linux/device.h>
12
-
#include <linux/interrupt.h>
11
+
#include <linux/errno.h>
12
+
#include <linux/kconfig.h>
13
+
#include <linux/types.h>
13
14
14
15
#include <linux/platform_data/dma-hsu.h>
15
16
17
+
struct device;
16
18
struct hsu_dma;
17
19
18
20
/**