Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'dmaengine-5.16-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine
Pull dmaengine updates from Vinod Koul:
"A bunch of driver updates, no new driver or controller support this
time though:
- Another pile of idxd updates
- pm routines cleanup for at_xdmac driver
- Correct handling of callback_result for few drivers
- zynqmp_dma driver updates and descriptor management refinement
- Hardware handshaking support for dw-axi-dmac
- Support for remotely powered controllers in Qcom bam dma
- tegra driver updates"
* tag 'dmaengine-5.16-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine: (69 commits)
dmaengine: ti: k3-udma: Set r/tchan or rflow to NULL if request fail
dmaengine: ti: k3-udma: Set bchan to NULL if a channel request fail
dmaengine: stm32-dma: avoid 64-bit division in stm32_dma_get_max_width
dmaengine: fsl-edma: support edma memcpy
dmaengine: idxd: fix resource leak on dmaengine driver disable
dmaengine: idxd: cleanup completion record allocation
dmaengine: zynqmp_dma: Correctly handle descriptor callbacks
dmaengine: xilinx_dma: Correctly handle cyclic descriptor callbacks
dmaengine: altera-msgdma: Correctly handle descriptor callbacks
dmaengine: at_xdmac: fix compilation warning
dmaengine: dw-axi-dmac: Simplify assignment in dma_chan_pause()
dmaengine: qcom: bam_dma: Add "powered remotely" mode
dt-bindings: dmaengine: bam_dma: Add "powered remotely" mode
dmaengine: sa11x0: Mark PM functions as __maybe_unused
dmaengine: switch from 'pci_' to 'dma_' API
dmaengine: ioat: switch from 'pci_' to 'dma_' API
dmaengine: hsu: switch from 'pci_' to 'dma_' API
dmaengine: hisi_dma: switch from 'pci_' to 'dma_' API
dmaengine: dw: switch from 'pci_' to 'dma_' API
dmaengine: dw-edma-pcie: switch from 'pci_' to 'dma_' API
...
+490
-330
+2
Documentation/devicetree/bindings/dma/qcom_bam_dma.txt
+2
Documentation/devicetree/bindings/dma/qcom_bam_dma.txt
···
15
15
the secure world.
16
16
- qcom,controlled-remotely : optional, indicates that the bam is controlled by
17
17
remote proccessor i.e. execution environment.
18
+
- qcom,powered-remotely : optional, indicates that the bam is powered up by
19
+
a remote processor but must be initialized by the local processor.
18
20
- num-channels : optional, indicates supported number of DMA channels in a
19
21
remotely controlled bam.
20
22
- qcom,num-ees : optional, indicates supported number of Execution Environments
+1
-1
drivers/dma/Kconfig
+1
-1
drivers/dma/Kconfig
···
717
717
718
718
config XILINX_ZYNQMP_DMA
719
719
tristate "Xilinx ZynqMP DMA Engine"
720
-
depends on (ARCH_ZYNQ || MICROBLAZE || ARM64)
720
+
depends on ARCH_ZYNQ || MICROBLAZE || ARM64 || COMPILE_TEST
721
721
select DMA_ENGINE
722
722
help
723
723
Enable support for Xilinx ZynqMP DMA controller.
+4
-6
drivers/dma/altera-msgdma.c
+4
-6
drivers/dma/altera-msgdma.c
···
585
585
struct msgdma_sw_desc *desc, *next;
586
586
587
587
list_for_each_entry_safe(desc, next, &mdev->done_list, node) {
588
-
dma_async_tx_callback callback;
589
-
void *callback_param;
588
+
struct dmaengine_desc_callback cb;
590
589
591
590
list_del(&desc->node);
592
591
593
-
callback = desc->async_tx.callback;
594
-
callback_param = desc->async_tx.callback_param;
595
-
if (callback) {
592
+
dmaengine_desc_get_callback(&desc->async_tx, &cb);
593
+
if (dmaengine_desc_callback_valid(&cb)) {
596
594
spin_unlock(&mdev->lock);
597
-
callback(callback_param);
595
+
dmaengine_desc_callback_invoke(&cb, NULL);
598
596
spin_lock(&mdev->lock);
599
597
}
600
598
+33
-36
drivers/dma/at_xdmac.c
+33
-36
drivers/dma/at_xdmac.c
···
155
155
#define AT_XDMAC_CC_WRIP (0x1 << 23) /* Write in Progress (read only) */
156
156
#define AT_XDMAC_CC_WRIP_DONE (0x0 << 23)
157
157
#define AT_XDMAC_CC_WRIP_IN_PROGRESS (0x1 << 23)
158
-
#define AT_XDMAC_CC_PERID(i) (0x7f & (i) << 24) /* Channel Peripheral Identifier */
158
+
#define AT_XDMAC_CC_PERID(i) ((0x7f & (i)) << 24) /* Channel Peripheral Identifier */
159
159
#define AT_XDMAC_CDS_MSP 0x2C /* Channel Data Stride Memory Set Pattern */
160
160
#define AT_XDMAC_CSUS 0x30 /* Channel Source Microblock Stride */
161
161
#define AT_XDMAC_CDUS 0x34 /* Channel Destination Microblock Stride */
···
1926
1926
return;
1927
1927
}
1928
1928
1929
-
#ifdef CONFIG_PM
1930
-
static int atmel_xdmac_prepare(struct device *dev)
1929
+
static void at_xdmac_axi_config(struct platform_device *pdev)
1930
+
{
1931
+
struct at_xdmac *atxdmac = (struct at_xdmac *)platform_get_drvdata(pdev);
1932
+
bool dev_m2m = false;
1933
+
u32 dma_requests;
1934
+
1935
+
if (!atxdmac->layout->axi_config)
1936
+
return; /* Not supported */
1937
+
1938
+
if (!of_property_read_u32(pdev->dev.of_node, "dma-requests",
1939
+
&dma_requests)) {
1940
+
dev_info(&pdev->dev, "controller in mem2mem mode.\n");
1941
+
dev_m2m = true;
1942
+
}
1943
+
1944
+
if (dev_m2m) {
1945
+
at_xdmac_write(atxdmac, AT_XDMAC_GCFG, AT_XDMAC_GCFG_M2M);
1946
+
at_xdmac_write(atxdmac, AT_XDMAC_GWAC, AT_XDMAC_GWAC_M2M);
1947
+
} else {
1948
+
at_xdmac_write(atxdmac, AT_XDMAC_GCFG, AT_XDMAC_GCFG_P2M);
1949
+
at_xdmac_write(atxdmac, AT_XDMAC_GWAC, AT_XDMAC_GWAC_P2M);
1950
+
}
1951
+
}
1952
+
1953
+
static int __maybe_unused atmel_xdmac_prepare(struct device *dev)
1931
1954
{
1932
1955
struct at_xdmac *atxdmac = dev_get_drvdata(dev);
1933
1956
struct dma_chan *chan, *_chan;
···
1964
1941
}
1965
1942
return 0;
1966
1943
}
1967
-
#else
1968
-
# define atmel_xdmac_prepare NULL
1969
-
#endif
1970
1944
1971
-
#ifdef CONFIG_PM_SLEEP
1972
-
static int atmel_xdmac_suspend(struct device *dev)
1945
+
static int __maybe_unused atmel_xdmac_suspend(struct device *dev)
1973
1946
{
1974
1947
struct at_xdmac *atxdmac = dev_get_drvdata(dev);
1975
1948
struct dma_chan *chan, *_chan;
···
1989
1970
return 0;
1990
1971
}
1991
1972
1992
-
static int atmel_xdmac_resume(struct device *dev)
1973
+
static int __maybe_unused atmel_xdmac_resume(struct device *dev)
1993
1974
{
1994
1975
struct at_xdmac *atxdmac = dev_get_drvdata(dev);
1995
1976
struct at_xdmac_chan *atchan;
1996
1977
struct dma_chan *chan, *_chan;
1978
+
struct platform_device *pdev = container_of(dev, struct platform_device, dev);
1997
1979
int i;
1998
1980
int ret;
1999
1981
2000
1982
ret = clk_prepare_enable(atxdmac->clk);
2001
1983
if (ret)
2002
1984
return ret;
1985
+
1986
+
at_xdmac_axi_config(pdev);
2003
1987
2004
1988
/* Clear pending interrupts. */
2005
1989
for (i = 0; i < atxdmac->dma.chancnt; i++) {
···
2026
2004
}
2027
2005
}
2028
2006
return 0;
2029
-
}
2030
-
#endif /* CONFIG_PM_SLEEP */
2031
-
2032
-
static void at_xdmac_axi_config(struct platform_device *pdev)
2033
-
{
2034
-
struct at_xdmac *atxdmac = (struct at_xdmac *)platform_get_drvdata(pdev);
2035
-
bool dev_m2m = false;
2036
-
u32 dma_requests;
2037
-
2038
-
if (!atxdmac->layout->axi_config)
2039
-
return; /* Not supported */
2040
-
2041
-
if (!of_property_read_u32(pdev->dev.of_node, "dma-requests",
2042
-
&dma_requests)) {
2043
-
dev_info(&pdev->dev, "controller in mem2mem mode.\n");
2044
-
dev_m2m = true;
2045
-
}
2046
-
2047
-
if (dev_m2m) {
2048
-
at_xdmac_write(atxdmac, AT_XDMAC_GCFG, AT_XDMAC_GCFG_M2M);
2049
-
at_xdmac_write(atxdmac, AT_XDMAC_GWAC, AT_XDMAC_GWAC_M2M);
2050
-
} else {
2051
-
at_xdmac_write(atxdmac, AT_XDMAC_GCFG, AT_XDMAC_GCFG_P2M);
2052
-
at_xdmac_write(atxdmac, AT_XDMAC_GWAC, AT_XDMAC_GWAC_P2M);
2053
-
}
2054
2007
}
2055
2008
2056
2009
static int at_xdmac_probe(struct platform_device *pdev)
···
2207
2210
return 0;
2208
2211
}
2209
2212
2210
-
static const struct dev_pm_ops atmel_xdmac_dev_pm_ops = {
2213
+
static const struct dev_pm_ops __maybe_unused atmel_xdmac_dev_pm_ops = {
2211
2214
.prepare = atmel_xdmac_prepare,
2212
2215
SET_LATE_SYSTEM_SLEEP_PM_OPS(atmel_xdmac_suspend, atmel_xdmac_resume)
2213
2216
};
···
2231
2234
.driver = {
2232
2235
.name = "at_xdmac",
2233
2236
.of_match_table = of_match_ptr(atmel_xdmac_dt_ids),
2234
-
.pm = &atmel_xdmac_dev_pm_ops,
2237
+
.pm = pm_ptr(&atmel_xdmac_dev_pm_ops),
2235
2238
}
2236
2239
};
2237
2240
+1
-1
drivers/dma/bestcomm/ata.c
+1
-1
drivers/dma/bestcomm/ata.c
+11
-11
drivers/dma/bestcomm/bestcomm.c
+11
-11
drivers/dma/bestcomm/bestcomm.c
···
95
95
tsk->bd = bcom_sram_alloc(bd_count * bd_size, 4, &tsk->bd_pa);
96
96
if (!tsk->bd)
97
97
goto error;
98
-
memset(tsk->bd, 0x00, bd_count * bd_size);
98
+
memset_io(tsk->bd, 0x00, bd_count * bd_size);
99
99
100
100
tsk->num_bd = bd_count;
101
101
tsk->bd_size = bd_size;
···
186
186
inc = bcom_task_inc(task);
187
187
188
188
/* Clear & copy */
189
-
memset(var, 0x00, BCOM_VAR_SIZE);
190
-
memset(inc, 0x00, BCOM_INC_SIZE);
189
+
memset_io(var, 0x00, BCOM_VAR_SIZE);
190
+
memset_io(inc, 0x00, BCOM_INC_SIZE);
191
191
192
192
desc_src = (u32 *)(hdr + 1);
193
193
var_src = desc_src + hdr->desc_size;
194
194
inc_src = var_src + hdr->var_size;
195
195
196
-
memcpy(desc, desc_src, hdr->desc_size * sizeof(u32));
197
-
memcpy(var + hdr->first_var, var_src, hdr->var_size * sizeof(u32));
198
-
memcpy(inc, inc_src, hdr->inc_size * sizeof(u32));
196
+
memcpy_toio(desc, desc_src, hdr->desc_size * sizeof(u32));
197
+
memcpy_toio(var + hdr->first_var, var_src, hdr->var_size * sizeof(u32));
198
+
memcpy_toio(inc, inc_src, hdr->inc_size * sizeof(u32));
199
199
200
200
return 0;
201
201
}
···
302
302
return -ENOMEM;
303
303
}
304
304
305
-
memset(bcom_eng->tdt, 0x00, tdt_size);
306
-
memset(bcom_eng->ctx, 0x00, ctx_size);
307
-
memset(bcom_eng->var, 0x00, var_size);
308
-
memset(bcom_eng->fdt, 0x00, fdt_size);
305
+
memset_io(bcom_eng->tdt, 0x00, tdt_size);
306
+
memset_io(bcom_eng->ctx, 0x00, ctx_size);
307
+
memset_io(bcom_eng->var, 0x00, var_size);
308
+
memset_io(bcom_eng->fdt, 0x00, fdt_size);
309
309
310
310
/* Copy the FDT for the EU#3 */
311
-
memcpy(&bcom_eng->fdt[48], fdt_ops, sizeof(fdt_ops));
311
+
memcpy_toio(&bcom_eng->fdt[48], fdt_ops, sizeof(fdt_ops));
312
312
313
313
/* Initialize Task base structure */
314
314
for (task=0; task<BCOM_MAX_TASKS; task++)
+2
-2
drivers/dma/bestcomm/fec.c
+2
-2
drivers/dma/bestcomm/fec.c
···
140
140
tsk->index = 0;
141
141
tsk->outdex = 0;
142
142
143
-
memset(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size);
143
+
memset_io(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size);
144
144
145
145
/* Configure some stuff */
146
146
bcom_set_task_pragma(tsk->tasknum, BCOM_FEC_RX_BD_PRAGMA);
···
241
241
tsk->index = 0;
242
242
tsk->outdex = 0;
243
243
244
-
memset(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size);
244
+
memset_io(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size);
245
245
246
246
/* Configure some stuff */
247
247
bcom_set_task_pragma(tsk->tasknum, BCOM_FEC_TX_BD_PRAGMA);
+2
-2
drivers/dma/bestcomm/gen_bd.c
+2
-2
drivers/dma/bestcomm/gen_bd.c
···
142
142
tsk->index = 0;
143
143
tsk->outdex = 0;
144
144
145
-
memset(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size);
145
+
memset_io(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size);
146
146
147
147
/* Configure some stuff */
148
148
bcom_set_task_pragma(tsk->tasknum, BCOM_GEN_RX_BD_PRAGMA);
···
226
226
tsk->index = 0;
227
227
tsk->outdex = 0;
228
228
229
-
memset(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size);
229
+
memset_io(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size);
230
230
231
231
/* Configure some stuff */
232
232
bcom_set_task_pragma(tsk->tasknum, BCOM_GEN_TX_BD_PRAGMA);
+1
drivers/dma/dma-jz4780.c
+1
drivers/dma/dma-jz4780.c
···
915
915
dd->dst_addr_widths = JZ_DMA_BUSWIDTHS;
916
916
dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
917
917
dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
918
+
dd->max_sg_burst = JZ_DMA_MAX_DESC;
918
919
919
920
/*
920
921
* Enable DMA controller, mark all channels as not programmable.
+1
-2
drivers/dma/dmaengine.c
+1
-2
drivers/dma/dmaengine.c
···
695
695
*/
696
696
struct dma_chan *dma_get_slave_channel(struct dma_chan *chan)
697
697
{
698
-
int err = -EBUSY;
699
-
700
698
/* lock against __dma_request_channel */
701
699
mutex_lock(&dma_list_mutex);
702
700
703
701
if (chan->client_count == 0) {
704
702
struct dma_device *device = chan->device;
703
+
int err;
705
704
706
705
dma_cap_set(DMA_PRIVATE, device->cap_mask);
707
706
device->privatecnt++;
+1
-1
drivers/dma/dmaengine.h
+1
-1
drivers/dma/dmaengine.h
+82
-30
drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
+82
-30
drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
···
79
79
iowrite32(upper_32_bits(val), chan->chan_regs + reg + 4);
80
80
}
81
81
82
+
static inline void axi_chan_config_write(struct axi_dma_chan *chan,
83
+
struct axi_dma_chan_config *config)
84
+
{
85
+
u32 cfg_lo, cfg_hi;
86
+
87
+
cfg_lo = (config->dst_multblk_type << CH_CFG_L_DST_MULTBLK_TYPE_POS |
88
+
config->src_multblk_type << CH_CFG_L_SRC_MULTBLK_TYPE_POS);
89
+
if (chan->chip->dw->hdata->reg_map_8_channels) {
90
+
cfg_hi = config->tt_fc << CH_CFG_H_TT_FC_POS |
91
+
config->hs_sel_src << CH_CFG_H_HS_SEL_SRC_POS |
92
+
config->hs_sel_dst << CH_CFG_H_HS_SEL_DST_POS |
93
+
config->src_per << CH_CFG_H_SRC_PER_POS |
94
+
config->dst_per << CH_CFG_H_DST_PER_POS |
95
+
config->prior << CH_CFG_H_PRIORITY_POS;
96
+
} else {
97
+
cfg_lo |= config->src_per << CH_CFG2_L_SRC_PER_POS |
98
+
config->dst_per << CH_CFG2_L_DST_PER_POS;
99
+
cfg_hi = config->tt_fc << CH_CFG2_H_TT_FC_POS |
100
+
config->hs_sel_src << CH_CFG2_H_HS_SEL_SRC_POS |
101
+
config->hs_sel_dst << CH_CFG2_H_HS_SEL_DST_POS |
102
+
config->prior << CH_CFG2_H_PRIORITY_POS;
103
+
}
104
+
axi_chan_iowrite32(chan, CH_CFG_L, cfg_lo);
105
+
axi_chan_iowrite32(chan, CH_CFG_H, cfg_hi);
106
+
}
107
+
82
108
static inline void axi_dma_disable(struct axi_dma_chip *chip)
83
109
{
84
110
u32 val;
···
180
154
181
155
val = axi_dma_ioread32(chan->chip, DMAC_CHEN);
182
156
val &= ~(BIT(chan->id) << DMAC_CHAN_EN_SHIFT);
183
-
val |= BIT(chan->id) << DMAC_CHAN_EN_WE_SHIFT;
157
+
if (chan->chip->dw->hdata->reg_map_8_channels)
158
+
val |= BIT(chan->id) << DMAC_CHAN_EN_WE_SHIFT;
159
+
else
160
+
val |= BIT(chan->id) << DMAC_CHAN_EN2_WE_SHIFT;
184
161
axi_dma_iowrite32(chan->chip, DMAC_CHEN, val);
185
162
}
186
163
···
192
163
u32 val;
193
164
194
165
val = axi_dma_ioread32(chan->chip, DMAC_CHEN);
195
-
val |= BIT(chan->id) << DMAC_CHAN_EN_SHIFT |
196
-
BIT(chan->id) << DMAC_CHAN_EN_WE_SHIFT;
166
+
if (chan->chip->dw->hdata->reg_map_8_channels)
167
+
val |= BIT(chan->id) << DMAC_CHAN_EN_SHIFT |
168
+
BIT(chan->id) << DMAC_CHAN_EN_WE_SHIFT;
169
+
else
170
+
val |= BIT(chan->id) << DMAC_CHAN_EN_SHIFT |
171
+
BIT(chan->id) << DMAC_CHAN_EN2_WE_SHIFT;
197
172
axi_dma_iowrite32(chan->chip, DMAC_CHEN, val);
198
173
}
199
174
···
212
179
213
180
static void axi_dma_hw_init(struct axi_dma_chip *chip)
214
181
{
182
+
int ret;
215
183
u32 i;
216
184
217
185
for (i = 0; i < chip->dw->hdata->nr_channels; i++) {
218
186
axi_chan_irq_disable(&chip->dw->chan[i], DWAXIDMAC_IRQ_ALL);
219
187
axi_chan_disable(&chip->dw->chan[i]);
220
188
}
189
+
ret = dma_set_mask_and_coherent(chip->dev, DMA_BIT_MASK(64));
190
+
if (ret)
191
+
dev_warn(chip->dev, "Unable to set coherent mask\n");
221
192
}
222
193
223
194
static u32 axi_chan_get_xfer_width(struct axi_dma_chan *chan, dma_addr_t src,
···
373
336
struct axi_dma_desc *first)
374
337
{
375
338
u32 priority = chan->chip->dw->hdata->priority[chan->id];
376
-
u32 reg, irq_mask;
339
+
struct axi_dma_chan_config config;
340
+
u32 irq_mask;
377
341
u8 lms = 0; /* Select AXI0 master for LLI fetching */
378
342
379
343
if (unlikely(axi_chan_is_hw_enable(chan))) {
···
386
348
387
349
axi_dma_enable(chan->chip);
388
350
389
-
reg = (DWAXIDMAC_MBLK_TYPE_LL << CH_CFG_L_DST_MULTBLK_TYPE_POS |
390
-
DWAXIDMAC_MBLK_TYPE_LL << CH_CFG_L_SRC_MULTBLK_TYPE_POS);
391
-
axi_chan_iowrite32(chan, CH_CFG_L, reg);
392
-
393
-
reg = (DWAXIDMAC_TT_FC_MEM_TO_MEM_DMAC << CH_CFG_H_TT_FC_POS |
394
-
priority << CH_CFG_H_PRIORITY_POS |
395
-
DWAXIDMAC_HS_SEL_HW << CH_CFG_H_HS_SEL_DST_POS |
396
-
DWAXIDMAC_HS_SEL_HW << CH_CFG_H_HS_SEL_SRC_POS);
351
+
config.dst_multblk_type = DWAXIDMAC_MBLK_TYPE_LL;
352
+
config.src_multblk_type = DWAXIDMAC_MBLK_TYPE_LL;
353
+
config.tt_fc = DWAXIDMAC_TT_FC_MEM_TO_MEM_DMAC;
354
+
config.prior = priority;
355
+
config.hs_sel_dst = DWAXIDMAC_HS_SEL_HW;
356
+
config.hs_sel_dst = DWAXIDMAC_HS_SEL_HW;
397
357
switch (chan->direction) {
398
358
case DMA_MEM_TO_DEV:
399
359
dw_axi_dma_set_byte_halfword(chan, true);
400
-
reg |= (chan->config.device_fc ?
401
-
DWAXIDMAC_TT_FC_MEM_TO_PER_DST :
402
-
DWAXIDMAC_TT_FC_MEM_TO_PER_DMAC)
403
-
<< CH_CFG_H_TT_FC_POS;
360
+
config.tt_fc = chan->config.device_fc ?
361
+
DWAXIDMAC_TT_FC_MEM_TO_PER_DST :
362
+
DWAXIDMAC_TT_FC_MEM_TO_PER_DMAC;
404
363
if (chan->chip->apb_regs)
405
-
reg |= (chan->id << CH_CFG_H_DST_PER_POS);
364
+
config.dst_per = chan->id;
365
+
else
366
+
config.dst_per = chan->hw_handshake_num;
406
367
break;
407
368
case DMA_DEV_TO_MEM:
408
-
reg |= (chan->config.device_fc ?
409
-
DWAXIDMAC_TT_FC_PER_TO_MEM_SRC :
410
-
DWAXIDMAC_TT_FC_PER_TO_MEM_DMAC)
411
-
<< CH_CFG_H_TT_FC_POS;
369
+
config.tt_fc = chan->config.device_fc ?
370
+
DWAXIDMAC_TT_FC_PER_TO_MEM_SRC :
371
+
DWAXIDMAC_TT_FC_PER_TO_MEM_DMAC;
412
372
if (chan->chip->apb_regs)
413
-
reg |= (chan->id << CH_CFG_H_SRC_PER_POS);
373
+
config.src_per = chan->id;
374
+
else
375
+
config.src_per = chan->hw_handshake_num;
414
376
break;
415
377
default:
416
378
break;
417
379
}
418
-
axi_chan_iowrite32(chan, CH_CFG_H, reg);
380
+
axi_chan_config_write(chan, &config);
419
381
420
382
write_chan_llp(chan, first->hw_desc[0].llp | lms);
421
383
···
1158
1120
1159
1121
spin_lock_irqsave(&chan->vc.lock, flags);
1160
1122
1161
-
val = axi_dma_ioread32(chan->chip, DMAC_CHEN);
1162
-
val |= BIT(chan->id) << DMAC_CHAN_SUSP_SHIFT |
1163
-
BIT(chan->id) << DMAC_CHAN_SUSP_WE_SHIFT;
1164
-
axi_dma_iowrite32(chan->chip, DMAC_CHEN, val);
1123
+
if (chan->chip->dw->hdata->reg_map_8_channels) {
1124
+
val = axi_dma_ioread32(chan->chip, DMAC_CHEN);
1125
+
val |= BIT(chan->id) << DMAC_CHAN_SUSP_SHIFT |
1126
+
BIT(chan->id) << DMAC_CHAN_SUSP_WE_SHIFT;
1127
+
axi_dma_iowrite32(chan->chip, DMAC_CHEN, val);
1128
+
} else {
1129
+
val = BIT(chan->id) << DMAC_CHAN_SUSP2_SHIFT |
1130
+
BIT(chan->id) << DMAC_CHAN_SUSP2_WE_SHIFT;
1131
+
axi_dma_iowrite32(chan->chip, DMAC_CHSUSPREG, val);
1132
+
}
1165
1133
1166
1134
do {
1167
1135
if (axi_chan_irq_read(chan) & DWAXIDMAC_IRQ_SUSPENDED)
···
1191
1147
u32 val;
1192
1148
1193
1149
val = axi_dma_ioread32(chan->chip, DMAC_CHEN);
1194
-
val &= ~(BIT(chan->id) << DMAC_CHAN_SUSP_SHIFT);
1195
-
val |= (BIT(chan->id) << DMAC_CHAN_SUSP_WE_SHIFT);
1196
-
axi_dma_iowrite32(chan->chip, DMAC_CHEN, val);
1150
+
if (chan->chip->dw->hdata->reg_map_8_channels) {
1151
+
val &= ~(BIT(chan->id) << DMAC_CHAN_SUSP_SHIFT);
1152
+
val |= (BIT(chan->id) << DMAC_CHAN_SUSP_WE_SHIFT);
1153
+
axi_dma_iowrite32(chan->chip, DMAC_CHEN, val);
1154
+
} else {
1155
+
val &= ~(BIT(chan->id) << DMAC_CHAN_SUSP2_SHIFT);
1156
+
val |= (BIT(chan->id) << DMAC_CHAN_SUSP2_WE_SHIFT);
1157
+
axi_dma_iowrite32(chan->chip, DMAC_CHSUSPREG, val);
1158
+
}
1197
1159
1198
1160
chan->is_paused = false;
1199
1161
}
···
1291
1241
return -EINVAL;
1292
1242
1293
1243
chip->dw->hdata->nr_channels = tmp;
1244
+
if (tmp <= DMA_REG_MAP_CH_REF)
1245
+
chip->dw->hdata->reg_map_8_channels = true;
1294
1246
1295
1247
ret = device_property_read_u32(dev, "snps,dma-masters", &tmp);
1296
1248
if (ret)
+34
-1
drivers/dma/dw-axi-dmac/dw-axi-dmac.h
+34
-1
drivers/dma/dw-axi-dmac/dw-axi-dmac.h
···
18
18
19
19
#include "../virt-dma.h"
20
20
21
-
#define DMAC_MAX_CHANNELS 8
21
+
#define DMAC_MAX_CHANNELS 16
22
22
#define DMAC_MAX_MASTERS 2
23
23
#define DMAC_MAX_BLK_SIZE 0x200000
24
24
···
30
30
u32 priority[DMAC_MAX_CHANNELS];
31
31
/* maximum supported axi burst length */
32
32
u32 axi_rw_burst_len;
33
+
/* Register map for DMAX_NUM_CHANNELS <= 8 */
34
+
bool reg_map_8_channels;
33
35
bool restrict_axi_burst_len;
34
36
};
35
37
···
105
103
u32 period_len;
106
104
};
107
105
106
+
struct axi_dma_chan_config {
107
+
u8 dst_multblk_type;
108
+
u8 src_multblk_type;
109
+
u8 dst_per;
110
+
u8 src_per;
111
+
u8 tt_fc;
112
+
u8 prior;
113
+
u8 hs_sel_dst;
114
+
u8 hs_sel_src;
115
+
};
116
+
108
117
static inline struct device *dchan2dev(struct dma_chan *dchan)
109
118
{
110
119
return &dchan->dev->device;
···
152
139
#define DMAC_CHEN 0x018 /* R/W DMAC Channel Enable */
153
140
#define DMAC_CHEN_L 0x018 /* R/W DMAC Channel Enable 00-31 */
154
141
#define DMAC_CHEN_H 0x01C /* R/W DMAC Channel Enable 32-63 */
142
+
#define DMAC_CHSUSPREG 0x020 /* R/W DMAC Channel Suspend */
143
+
#define DMAC_CHABORTREG 0x028 /* R/W DMAC Channel Abort */
155
144
#define DMAC_INTSTATUS 0x030 /* R DMAC Interrupt Status */
156
145
#define DMAC_COMMON_INTCLEAR 0x038 /* W DMAC Interrupt Clear */
157
146
#define DMAC_COMMON_INTSTATUS_ENA 0x040 /* R DMAC Interrupt Status Enable */
···
202
187
#define DMA_APB_HS_SEL_BIT_SIZE 0x08 /* HW handshake bits per channel */
203
188
#define DMA_APB_HS_SEL_MASK 0xFF /* HW handshake select masks */
204
189
#define MAX_BLOCK_SIZE 0x1000 /* 1024 blocks * 4 bytes data width */
190
+
#define DMA_REG_MAP_CH_REF 0x08 /* Channel count to choose register map */
205
191
206
192
/* DMAC_CFG */
207
193
#define DMAC_EN_POS 0
···
211
195
#define INT_EN_POS 1
212
196
#define INT_EN_MASK BIT(INT_EN_POS)
213
197
198
+
/* DMAC_CHEN */
214
199
#define DMAC_CHAN_EN_SHIFT 0
215
200
#define DMAC_CHAN_EN_WE_SHIFT 8
216
201
217
202
#define DMAC_CHAN_SUSP_SHIFT 16
218
203
#define DMAC_CHAN_SUSP_WE_SHIFT 24
204
+
205
+
/* DMAC_CHEN2 */
206
+
#define DMAC_CHAN_EN2_WE_SHIFT 16
207
+
208
+
/* DMAC_CHSUSP */
209
+
#define DMAC_CHAN_SUSP2_SHIFT 0
210
+
#define DMAC_CHAN_SUSP2_WE_SHIFT 16
219
211
220
212
/* CH_CTL_H */
221
213
#define CH_CTL_H_ARLEN_EN BIT(6)
···
312
288
DWAXIDMAC_MBLK_TYPE_SHADOW_REG,
313
289
DWAXIDMAC_MBLK_TYPE_LL
314
290
};
291
+
292
+
/* CH_CFG2 */
293
+
#define CH_CFG2_L_SRC_PER_POS 4
294
+
#define CH_CFG2_L_DST_PER_POS 11
295
+
296
+
#define CH_CFG2_H_TT_FC_POS 0
297
+
#define CH_CFG2_H_HS_SEL_SRC_POS 3
298
+
#define CH_CFG2_H_HS_SEL_DST_POS 4
299
+
#define CH_CFG2_H_PRIORITY_POS 20
315
300
316
301
/**
317
302
* DW AXI DMA channel interrupts
-1
drivers/dma/dw-edma/dw-edma-core.c
-1
drivers/dma/dw-edma/dw-edma-core.c
+4
-13
drivers/dma/dw-edma/dw-edma-pcie.c
+4
-13
drivers/dma/dw-edma/dw-edma-pcie.c
···
186
186
pci_set_master(pdev);
187
187
188
188
/* DMA configuration */
189
-
err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
189
+
err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
190
190
if (!err) {
191
-
err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
192
-
if (err) {
193
-
pci_err(pdev, "consistent DMA mask 64 set failed\n");
194
-
return err;
195
-
}
191
+
pci_err(pdev, "DMA mask 64 set failed\n");
192
+
return err;
196
193
} else {
197
194
pci_err(pdev, "DMA mask 64 set failed\n");
198
195
199
-
err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
196
+
err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
200
197
if (err) {
201
198
pci_err(pdev, "DMA mask 32 set failed\n");
202
-
return err;
203
-
}
204
-
205
-
err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
206
-
if (err) {
207
-
pci_err(pdev, "consistent DMA mask 32 set failed\n");
208
199
return err;
209
200
}
210
201
}
+1
-5
drivers/dma/dw/pci.c
+1
-5
drivers/dma/dw/pci.c
···
32
32
pci_set_master(pdev);
33
33
pci_try_set_mwi(pdev);
34
34
35
-
ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
36
-
if (ret)
37
-
return ret;
38
-
39
-
ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
35
+
ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
40
36
if (ret)
41
37
return ret;
42
38
+34
-1
drivers/dma/fsl-edma-common.c
+34
-1
drivers/dma/fsl-edma-common.c
···
348
348
struct fsl_edma_engine *edma = fsl_chan->edma;
349
349
struct edma_regs *regs = &fsl_chan->edma->regs;
350
350
u32 ch = fsl_chan->vchan.chan.chan_id;
351
+
u16 csr = 0;
351
352
352
353
/*
353
354
* TCD parameters are stored in struct fsl_edma_hw_tcd in little
···
373
372
374
373
edma_writel(edma, (s32)tcd->dlast_sga,
375
374
®s->tcd[ch].dlast_sga);
375
+
376
+
if (fsl_chan->is_sw) {
377
+
csr = le16_to_cpu(tcd->csr);
378
+
csr |= EDMA_TCD_CSR_START;
379
+
tcd->csr = cpu_to_le16(csr);
380
+
}
376
381
377
382
edma_writew(edma, (s16)tcd->csr, ®s->tcd[ch].csr);
378
383
}
···
594
587
}
595
588
EXPORT_SYMBOL_GPL(fsl_edma_prep_slave_sg);
596
589
590
+
struct dma_async_tx_descriptor *fsl_edma_prep_memcpy(struct dma_chan *chan,
591
+
dma_addr_t dma_dst, dma_addr_t dma_src,
592
+
size_t len, unsigned long flags)
593
+
{
594
+
struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan);
595
+
struct fsl_edma_desc *fsl_desc;
596
+
597
+
fsl_desc = fsl_edma_alloc_desc(fsl_chan, 1);
598
+
if (!fsl_desc)
599
+
return NULL;
600
+
fsl_desc->iscyclic = false;
601
+
602
+
fsl_chan->is_sw = true;
603
+
604
+
/* To match with copy_align and max_seg_size so 1 tcd is enough */
605
+
fsl_edma_fill_tcd(fsl_desc->tcd[0].vtcd, dma_src, dma_dst,
606
+
EDMA_TCD_ATTR_SSIZE_32BYTE | EDMA_TCD_ATTR_DSIZE_32BYTE,
607
+
32, len, 0, 1, 1, 32, 0, true, true, false);
608
+
609
+
return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags);
610
+
}
611
+
EXPORT_SYMBOL_GPL(fsl_edma_prep_memcpy);
612
+
597
613
void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan)
598
614
{
599
615
struct virt_dma_desc *vdesc;
···
668
638
void fsl_edma_free_chan_resources(struct dma_chan *chan)
669
639
{
670
640
struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan);
641
+
struct fsl_edma_engine *edma = fsl_chan->edma;
671
642
unsigned long flags;
672
643
LIST_HEAD(head);
673
644
674
645
spin_lock_irqsave(&fsl_chan->vchan.lock, flags);
675
646
fsl_edma_disable_request(fsl_chan);
676
-
fsl_edma_chan_mux(fsl_chan, 0, false);
647
+
if (edma->drvdata->dmamuxs)
648
+
fsl_edma_chan_mux(fsl_chan, 0, false);
677
649
fsl_chan->edesc = NULL;
678
650
vchan_get_all_descriptors(&fsl_chan->vchan, &head);
679
651
fsl_edma_unprep_slave_dma(fsl_chan);
···
684
652
vchan_dma_desc_free_list(&fsl_chan->vchan, &head);
685
653
dma_pool_destroy(fsl_chan->tcd_pool);
686
654
fsl_chan->tcd_pool = NULL;
655
+
fsl_chan->is_sw = false;
687
656
}
688
657
EXPORT_SYMBOL_GPL(fsl_edma_free_chan_resources);
689
658
+4
drivers/dma/fsl-edma-common.h
+4
drivers/dma/fsl-edma-common.h
···
121
121
struct fsl_edma_desc *edesc;
122
122
struct dma_slave_config cfg;
123
123
u32 attr;
124
+
bool is_sw;
124
125
struct dma_pool *tcd_pool;
125
126
dma_addr_t dma_dev_addr;
126
127
u32 dma_dev_size;
···
241
240
struct dma_chan *chan, struct scatterlist *sgl,
242
241
unsigned int sg_len, enum dma_transfer_direction direction,
243
242
unsigned long flags, void *context);
243
+
struct dma_async_tx_descriptor *fsl_edma_prep_memcpy(
244
+
struct dma_chan *chan, dma_addr_t dma_dst, dma_addr_t dma_src,
245
+
size_t len, unsigned long flags);
244
246
void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan);
245
247
void fsl_edma_issue_pending(struct dma_chan *chan);
246
248
int fsl_edma_alloc_chan_resources(struct dma_chan *chan);
+7
drivers/dma/fsl-edma.c
+7
drivers/dma/fsl-edma.c
···
17
17
#include <linux/of_address.h>
18
18
#include <linux/of_irq.h>
19
19
#include <linux/of_dma.h>
20
+
#include <linux/dma-mapping.h>
20
21
21
22
#include "fsl-edma-common.h"
22
23
···
373
372
dma_cap_set(DMA_PRIVATE, fsl_edma->dma_dev.cap_mask);
374
373
dma_cap_set(DMA_SLAVE, fsl_edma->dma_dev.cap_mask);
375
374
dma_cap_set(DMA_CYCLIC, fsl_edma->dma_dev.cap_mask);
375
+
dma_cap_set(DMA_MEMCPY, fsl_edma->dma_dev.cap_mask);
376
376
377
377
fsl_edma->dma_dev.dev = &pdev->dev;
378
378
fsl_edma->dma_dev.device_alloc_chan_resources
···
383
381
fsl_edma->dma_dev.device_tx_status = fsl_edma_tx_status;
384
382
fsl_edma->dma_dev.device_prep_slave_sg = fsl_edma_prep_slave_sg;
385
383
fsl_edma->dma_dev.device_prep_dma_cyclic = fsl_edma_prep_dma_cyclic;
384
+
fsl_edma->dma_dev.device_prep_dma_memcpy = fsl_edma_prep_memcpy;
386
385
fsl_edma->dma_dev.device_config = fsl_edma_slave_config;
387
386
fsl_edma->dma_dev.device_pause = fsl_edma_pause;
388
387
fsl_edma->dma_dev.device_resume = fsl_edma_resume;
···
394
391
fsl_edma->dma_dev.src_addr_widths = FSL_EDMA_BUSWIDTHS;
395
392
fsl_edma->dma_dev.dst_addr_widths = FSL_EDMA_BUSWIDTHS;
396
393
fsl_edma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
394
+
395
+
fsl_edma->dma_dev.copy_align = DMAENGINE_ALIGN_32_BYTES;
396
+
/* Per worst case 'nbytes = 1' take CITER as the max_seg_size */
397
+
dma_set_max_seg_size(fsl_edma->dma_dev.dev, 0x3fff);
397
398
398
399
platform_set_drvdata(pdev, fsl_edma);
399
400
+1
-5
drivers/dma/hisi_dma.c
+1
-5
drivers/dma/hisi_dma.c
···
519
519
return ret;
520
520
}
521
521
522
-
ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
523
-
if (ret)
524
-
return ret;
525
-
526
-
ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
522
+
ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
527
523
if (ret)
528
524
return ret;
529
525
+1
-5
drivers/dma/hsu/pci.c
+1
-5
drivers/dma/hsu/pci.c
···
65
65
pci_set_master(pdev);
66
66
pci_try_set_mwi(pdev);
67
67
68
-
ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
69
-
if (ret)
70
-
return ret;
71
-
72
-
ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
68
+
ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
73
69
if (ret)
74
70
return ret;
75
71
+8
-21
drivers/dma/idxd/device.c
+8
-21
drivers/dma/idxd/device.c
···
135
135
struct idxd_device *idxd = wq->idxd;
136
136
struct device *dev = &idxd->pdev->dev;
137
137
int rc, num_descs, i;
138
-
int align;
139
-
u64 tmp;
140
138
141
139
if (wq->type != IDXD_WQT_KERNEL)
142
140
return 0;
···
146
148
if (rc < 0)
147
149
return rc;
148
150
149
-
align = idxd->data->align;
150
-
wq->compls_size = num_descs * idxd->data->compl_size + align;
151
-
wq->compls_raw = dma_alloc_coherent(dev, wq->compls_size,
152
-
&wq->compls_addr_raw, GFP_KERNEL);
153
-
if (!wq->compls_raw) {
151
+
wq->compls_size = num_descs * idxd->data->compl_size;
152
+
wq->compls = dma_alloc_coherent(dev, wq->compls_size, &wq->compls_addr, GFP_KERNEL);
153
+
if (!wq->compls) {
154
154
rc = -ENOMEM;
155
155
goto fail_alloc_compls;
156
156
}
157
-
158
-
/* Adjust alignment */
159
-
wq->compls_addr = (wq->compls_addr_raw + (align - 1)) & ~(align - 1);
160
-
tmp = (u64)wq->compls_raw;
161
-
tmp = (tmp + (align - 1)) & ~(align - 1);
162
-
wq->compls = (struct dsa_completion_record *)tmp;
163
157
164
158
rc = alloc_descs(wq, num_descs);
165
159
if (rc < 0)
···
181
191
fail_sbitmap_init:
182
192
free_descs(wq);
183
193
fail_alloc_descs:
184
-
dma_free_coherent(dev, wq->compls_size, wq->compls_raw,
185
-
wq->compls_addr_raw);
194
+
dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
186
195
fail_alloc_compls:
187
196
free_hw_descs(wq);
188
197
return rc;
···
196
207
197
208
free_hw_descs(wq);
198
209
free_descs(wq);
199
-
dma_free_coherent(dev, wq->compls_size, wq->compls_raw,
200
-
wq->compls_addr_raw);
210
+
dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
201
211
sbitmap_queue_free(&wq->sbq);
202
212
}
203
213
···
415
427
{
416
428
percpu_ref_kill(&wq->wq_active);
417
429
wait_for_completion(&wq->wq_dead);
418
-
percpu_ref_exit(&wq->wq_active);
419
430
}
420
431
421
432
/* Device control bits */
···
571
584
spin_lock(&idxd->dev_lock);
572
585
idxd_device_clear_state(idxd);
573
586
idxd->state = IDXD_DEV_DISABLED;
587
+
idxd_unmask_error_interrupts(idxd);
588
+
idxd_msix_perm_setup(idxd);
574
589
spin_unlock(&idxd->dev_lock);
575
590
}
576
591
···
781
792
struct device *dev = &idxd->pdev->dev;
782
793
783
794
/* Setup bandwidth token limit */
784
-
if (idxd->token_limit) {
795
+
if (idxd->hw.gen_cap.config_en && idxd->token_limit) {
785
796
reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET);
786
797
reg.token_limit = idxd->token_limit;
787
798
iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET);
···
1040
1051
1041
1052
wq->size = wq->wqcfg->wq_size;
1042
1053
wq->threshold = wq->wqcfg->wq_thresh;
1043
-
if (wq->wqcfg->priv)
1044
-
wq->type = IDXD_WQT_KERNEL;
1045
1054
1046
1055
/* The driver does not support shared WQ mode in read-only config yet */
1047
1056
if (wq->wqcfg->mode == 0 || wq->wqcfg->pasid_en)
+3
-2
drivers/dma/idxd/dma.c
+3
-2
drivers/dma/idxd/dma.c
···
311
311
312
312
err_dma:
313
313
idxd_wq_quiesce(wq);
314
+
percpu_ref_exit(&wq->wq_active);
314
315
err_ref:
315
316
idxd_wq_free_resources(wq);
316
317
err_res_alloc:
···
329
328
mutex_lock(&wq->wq_lock);
330
329
idxd_wq_quiesce(wq);
331
330
idxd_unregister_dma_channel(wq);
332
-
__drv_disable_wq(wq);
333
331
idxd_wq_free_resources(wq);
334
-
wq->type = IDXD_WQT_NONE;
332
+
__drv_disable_wq(wq);
333
+
percpu_ref_exit(&wq->wq_active);
335
334
mutex_unlock(&wq->wq_lock);
336
335
}
337
336
-2
drivers/dma/idxd/idxd.h
-2
drivers/dma/idxd/idxd.h
+11
-3
drivers/dma/idxd/init.c
+11
-3
drivers/dma/idxd/init.c
···
797
797
int msixcnt = pci_msix_vec_count(pdev);
798
798
int i;
799
799
800
-
dev_dbg(&pdev->dev, "%s called\n", __func__);
800
+
idxd_unregister_devices(idxd);
801
+
/*
802
+
* When ->release() is called for the idxd->conf_dev, it frees all the memory related
803
+
* to the idxd context. The driver still needs those bits in order to do the rest of
804
+
* the cleanup. However, we do need to unbound the idxd sub-driver. So take a ref
805
+
* on the device here to hold off the freeing while allowing the idxd sub-driver
806
+
* to unbind.
807
+
*/
808
+
get_device(idxd_confdev(idxd));
809
+
device_unregister(idxd_confdev(idxd));
801
810
idxd_shutdown(pdev);
802
811
if (device_pasid_enabled(idxd))
803
812
idxd_disable_system_pasid(idxd);
804
-
idxd_unregister_devices(idxd);
805
813
806
814
for (i = 0; i < msixcnt; i++) {
807
815
irq_entry = &idxd->irq_entries[i];
···
823
815
pci_disable_device(pdev);
824
816
destroy_workqueue(idxd->wq);
825
817
perfmon_pmu_remove(idxd);
826
-
device_unregister(idxd_confdev(idxd));
818
+
put_device(idxd_confdev(idxd));
827
819
}
828
820
829
821
static struct pci_driver idxd_pci_driver = {
+6
-2
drivers/dma/idxd/irq.c
+6
-2
drivers/dma/idxd/irq.c
···
63
63
int i;
64
64
bool err = false;
65
65
66
+
if (cause & IDXD_INTC_HALT_STATE)
67
+
goto halt;
68
+
66
69
if (cause & IDXD_INTC_ERR) {
67
70
spin_lock(&idxd->dev_lock);
68
71
for (i = 0; i < 4; i++)
···
124
121
if (!err)
125
122
return 0;
126
123
124
+
halt:
127
125
gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);
128
126
if (gensts.state == IDXD_DEVICE_STATE_HALT) {
129
127
idxd->state = IDXD_DEV_HALTED;
···
138
134
queue_work(idxd->wq, &idxd->work);
139
135
} else {
140
136
spin_lock(&idxd->dev_lock);
137
+
idxd->state = IDXD_DEV_HALTED;
141
138
idxd_wqs_quiesce(idxd);
142
139
idxd_wqs_unmap_portal(idxd);
143
140
idxd_device_clear_state(idxd);
···
226
221
227
222
list_for_each_entry_safe(desc, n, &irq_entry->work_list, list) {
228
223
if (desc->completion->status) {
229
-
list_del(&desc->list);
230
-
list_add_tail(&desc->list, &flist);
224
+
list_move_tail(&desc->list, &flist);
231
225
}
232
226
}
233
227
+2
-2
drivers/dma/idxd/registers.h
+2
-2
drivers/dma/idxd/registers.h
···
36
36
u64 max_batch_shift:4;
37
37
u64 max_ims_mult:6;
38
38
u64 config_en:1;
39
-
u64 max_descs_per_engine:8;
40
-
u64 rsvd3:24;
39
+
u64 rsvd3:32;
41
40
};
42
41
u64 bits;
43
42
} __packed;
···
157
158
#define IDXD_INTC_CMD 0x02
158
159
#define IDXD_INTC_OCCUPY 0x04
159
160
#define IDXD_INTC_PERFMON_OVFL 0x08
161
+
#define IDXD_INTC_HALT_STATE 0x10
160
162
161
163
#define IDXD_CMD_OFFSET 0xa0
162
164
union idxd_command_reg {
+14
-14
drivers/dma/imx-sdma.c
+14
-14
drivers/dma/imx-sdma.c
···
741
741
unsigned long flags;
742
742
743
743
buf_virt = dma_alloc_coherent(sdma->dev, size, &buf_phys, GFP_KERNEL);
744
-
if (!buf_virt) {
744
+
if (!buf_virt)
745
745
return -ENOMEM;
746
-
}
747
746
748
747
spin_lock_irqsave(&sdma->channel_0_lock, flags);
749
748
···
1226
1227
if (sdmac->peripheral_type == IMX_DMATYPE_ASRC_SP ||
1227
1228
sdmac->peripheral_type == IMX_DMATYPE_ASRC)
1228
1229
sdma_set_watermarklevel_for_p2p(sdmac);
1229
-
} else
1230
+
} else {
1230
1231
__set_bit(sdmac->event_id0, sdmac->event_mask);
1232
+
}
1231
1233
1232
1234
/* Address */
1233
1235
sdmac->shp_addr = sdmac->per_address;
···
1241
1241
}
1242
1242
1243
1243
static int sdma_set_channel_priority(struct sdma_channel *sdmac,
1244
-
unsigned int priority)
1244
+
unsigned int priority)
1245
1245
{
1246
1246
struct sdma_engine *sdma = sdmac->sdma;
1247
1247
int channel = sdmac->channel;
···
1261
1261
int ret = -EBUSY;
1262
1262
1263
1263
sdma->bd0 = dma_alloc_coherent(sdma->dev, PAGE_SIZE, &sdma->bd0_phys,
1264
-
GFP_NOWAIT);
1264
+
GFP_NOWAIT);
1265
1265
if (!sdma->bd0) {
1266
1266
ret = -ENOMEM;
1267
1267
goto out;
···
1284
1284
int ret = 0;
1285
1285
1286
1286
desc->bd = dma_alloc_coherent(desc->sdmac->sdma->dev, bd_size,
1287
-
&desc->bd_phys, GFP_NOWAIT);
1287
+
&desc->bd_phys, GFP_NOWAIT);
1288
1288
if (!desc->bd) {
1289
1289
ret = -ENOMEM;
1290
1290
goto out;
···
1757
1757
#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 46
1758
1758
1759
1759
static void sdma_add_scripts(struct sdma_engine *sdma,
1760
-
const struct sdma_script_start_addrs *addr)
1760
+
const struct sdma_script_start_addrs *addr)
1761
1761
{
1762
1762
s32 *addr_arr = (u32 *)addr;
1763
1763
s32 *saddr_arr = (u32 *)sdma->script_addrs;
···
1840
1840
clk_enable(sdma->clk_ahb);
1841
1841
/* download the RAM image for SDMA */
1842
1842
sdma_load_script(sdma, ram_code,
1843
-
header->ram_code_size,
1844
-
addr->ram_code_start_addr);
1843
+
header->ram_code_size,
1844
+
addr->ram_code_start_addr);
1845
1845
clk_disable(sdma->clk_ipg);
1846
1846
clk_disable(sdma->clk_ahb);
1847
1847
···
1850
1850
sdma->fw_loaded = true;
1851
1851
1852
1852
dev_info(sdma->dev, "loaded firmware %d.%d\n",
1853
-
header->version_major,
1854
-
header->version_minor);
1853
+
header->version_major,
1854
+
header->version_minor);
1855
1855
1856
1856
err_firmware:
1857
1857
release_firmware(fw);
···
1955
1955
writel_relaxed(0, sdma->regs + SDMA_H_C0PTR);
1956
1956
1957
1957
sdma->channel_control = dma_alloc_coherent(sdma->dev,
1958
-
MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control) +
1958
+
MAX_DMA_CHANNELS * sizeof(struct sdma_channel_control) +
1959
1959
sizeof(struct sdma_context_data),
1960
1960
&ccb_phys, GFP_KERNEL);
1961
1961
···
1965
1965
}
1966
1966
1967
1967
sdma->context = (void *)sdma->channel_control +
1968
-
MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control);
1968
+
MAX_DMA_CHANNELS * sizeof(struct sdma_channel_control);
1969
1969
sdma->context_phys = ccb_phys +
1970
-
MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control);
1970
+
MAX_DMA_CHANNELS * sizeof(struct sdma_channel_control);
1971
1971
1972
1972
/* disable all channels */
1973
1973
for (i = 0; i < sdma->drvdata->num_events; i++)
+2
-8
drivers/dma/ioat/init.c
+2
-8
drivers/dma/ioat/init.c
···
1363
1363
if (!iomap)
1364
1364
return -ENOMEM;
1365
1365
1366
-
err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1366
+
err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1367
1367
if (err)
1368
-
err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1369
-
if (err)
1370
-
return err;
1371
-
1372
-
err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1373
-
if (err)
1374
-
err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1368
+
err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1375
1369
if (err)
1376
1370
return err;
1377
1371
+1
-1
drivers/dma/milbeaut-hdmac.c
+1
-1
drivers/dma/milbeaut-hdmac.c
+1
drivers/dma/mmp_pdma.c
+1
drivers/dma/mmp_pdma.c
+2
-8
drivers/dma/plx_dma.c
+2
-8
drivers/dma/plx_dma.c
···
563
563
if (rc)
564
564
return rc;
565
565
566
-
rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
566
+
rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48));
567
567
if (rc)
568
-
rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
569
-
if (rc)
570
-
return rc;
571
-
572
-
rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
573
-
if (rc)
574
-
rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
568
+
rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
575
569
if (rc)
576
570
return rc;
577
571
+57
-33
drivers/dma/qcom/bam_dma.c
+57
-33
drivers/dma/qcom/bam_dma.c
···
388
388
/* execution environment ID, from DT */
389
389
u32 ee;
390
390
bool controlled_remotely;
391
+
bool powered_remotely;
392
+
u32 active_channels;
391
393
392
394
const struct reg_offset_data *layout;
393
395
···
415
413
r.pipe_mult * pipe +
416
414
r.evnt_mult * pipe +
417
415
r.ee_mult * bdev->ee;
416
+
}
417
+
418
+
/**
419
+
* bam_reset() - reset and initialize BAM registers
420
+
* @bdev: bam device
421
+
*/
422
+
static void bam_reset(struct bam_device *bdev)
423
+
{
424
+
u32 val;
425
+
426
+
/* s/w reset bam */
427
+
/* after reset all pipes are disabled and idle */
428
+
val = readl_relaxed(bam_addr(bdev, 0, BAM_CTRL));
429
+
val |= BAM_SW_RST;
430
+
writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
431
+
val &= ~BAM_SW_RST;
432
+
writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
433
+
434
+
/* make sure previous stores are visible before enabling BAM */
435
+
wmb();
436
+
437
+
/* enable bam */
438
+
val |= BAM_EN;
439
+
writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
440
+
441
+
/* set descriptor threshhold, start with 4 bytes */
442
+
writel_relaxed(DEFAULT_CNT_THRSHLD,
443
+
bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD));
444
+
445
+
/* Enable default set of h/w workarounds, ie all except BAM_FULL_PIPE */
446
+
writel_relaxed(BAM_CNFG_BITS_DEFAULT, bam_addr(bdev, 0, BAM_CNFG_BITS));
447
+
448
+
/* enable irqs for errors */
449
+
writel_relaxed(BAM_ERROR_EN | BAM_HRESP_ERR_EN,
450
+
bam_addr(bdev, 0, BAM_IRQ_EN));
451
+
452
+
/* unmask global bam interrupt */
453
+
writel_relaxed(BAM_IRQ_MSK, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
418
454
}
419
455
420
456
/**
···
552
512
return -ENOMEM;
553
513
}
554
514
515
+
if (bdev->active_channels++ == 0 && bdev->powered_remotely)
516
+
bam_reset(bdev);
517
+
555
518
return 0;
556
519
}
557
520
···
607
564
608
565
/* disable irq */
609
566
writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_IRQ_EN));
567
+
568
+
if (--bdev->active_channels == 0 && bdev->powered_remotely) {
569
+
/* s/w reset bam */
570
+
val = readl_relaxed(bam_addr(bdev, 0, BAM_CTRL));
571
+
val |= BAM_SW_RST;
572
+
writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
573
+
}
610
574
611
575
err:
612
576
pm_runtime_mark_last_busy(bdev->dev);
···
1214
1164
bdev->num_channels = val & BAM_NUM_PIPES_MASK;
1215
1165
}
1216
1166
1217
-
if (bdev->controlled_remotely)
1218
-
return 0;
1219
-
1220
-
/* s/w reset bam */
1221
-
/* after reset all pipes are disabled and idle */
1222
-
val = readl_relaxed(bam_addr(bdev, 0, BAM_CTRL));
1223
-
val |= BAM_SW_RST;
1224
-
writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
1225
-
val &= ~BAM_SW_RST;
1226
-
writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
1227
-
1228
-
/* make sure previous stores are visible before enabling BAM */
1229
-
wmb();
1230
-
1231
-
/* enable bam */
1232
-
val |= BAM_EN;
1233
-
writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
1234
-
1235
-
/* set descriptor threshhold, start with 4 bytes */
1236
-
writel_relaxed(DEFAULT_CNT_THRSHLD,
1237
-
bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD));
1238
-
1239
-
/* Enable default set of h/w workarounds, ie all except BAM_FULL_PIPE */
1240
-
writel_relaxed(BAM_CNFG_BITS_DEFAULT, bam_addr(bdev, 0, BAM_CNFG_BITS));
1241
-
1242
-
/* enable irqs for errors */
1243
-
writel_relaxed(BAM_ERROR_EN | BAM_HRESP_ERR_EN,
1244
-
bam_addr(bdev, 0, BAM_IRQ_EN));
1245
-
1246
-
/* unmask global bam interrupt */
1247
-
writel_relaxed(BAM_IRQ_MSK, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
1167
+
/* Reset BAM now if fully controlled locally */
1168
+
if (!bdev->controlled_remotely && !bdev->powered_remotely)
1169
+
bam_reset(bdev);
1248
1170
1249
1171
return 0;
1250
1172
}
···
1279
1257
1280
1258
bdev->controlled_remotely = of_property_read_bool(pdev->dev.of_node,
1281
1259
"qcom,controlled-remotely");
1260
+
bdev->powered_remotely = of_property_read_bool(pdev->dev.of_node,
1261
+
"qcom,powered-remotely");
1282
1262
1283
-
if (bdev->controlled_remotely) {
1263
+
if (bdev->controlled_remotely || bdev->powered_remotely) {
1284
1264
ret = of_property_read_u32(pdev->dev.of_node, "num-channels",
1285
1265
&bdev->num_channels);
1286
1266
if (ret)
···
1294
1270
dev_err(bdev->dev, "num-ees unspecified in dt\n");
1295
1271
}
1296
1272
1297
-
if (bdev->controlled_remotely)
1273
+
if (bdev->controlled_remotely || bdev->powered_remotely)
1298
1274
bdev->bamclk = devm_clk_get_optional(bdev->dev, "bam_clk");
1299
1275
else
1300
1276
bdev->bamclk = devm_clk_get(bdev->dev, "bam_clk");
+3
-8
drivers/dma/sa11x0-dma.c
+3
-8
drivers/dma/sa11x0-dma.c
···
1001
1001
return 0;
1002
1002
}
1003
1003
1004
-
static int sa11x0_dma_suspend(struct device *dev)
1004
+
static __maybe_unused int sa11x0_dma_suspend(struct device *dev)
1005
1005
{
1006
1006
struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
1007
1007
unsigned pch;
···
1039
1039
return 0;
1040
1040
}
1041
1041
1042
-
static int sa11x0_dma_resume(struct device *dev)
1042
+
static __maybe_unused int sa11x0_dma_resume(struct device *dev)
1043
1043
{
1044
1044
struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
1045
1045
unsigned pch;
···
1072
1072
}
1073
1073
1074
1074
static const struct dev_pm_ops sa11x0_dma_pm_ops = {
1075
-
.suspend_noirq = sa11x0_dma_suspend,
1076
-
.resume_noirq = sa11x0_dma_resume,
1077
-
.freeze_noirq = sa11x0_dma_suspend,
1078
-
.thaw_noirq = sa11x0_dma_resume,
1079
-
.poweroff_noirq = sa11x0_dma_suspend,
1080
-
.restore_noirq = sa11x0_dma_resume,
1075
+
SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(sa11x0_dma_suspend, sa11x0_dma_resume)
1081
1076
};
1082
1077
1083
1078
static struct platform_driver sa11x0_dma_driver = {
+7
-6
drivers/dma/sh/rcar-dmac.c
+7
-6
drivers/dma/sh/rcar-dmac.c
···
1916
1916
ret = pm_runtime_resume_and_get(&pdev->dev);
1917
1917
if (ret < 0) {
1918
1918
dev_err(&pdev->dev, "runtime PM get sync failed (%d)\n", ret);
1919
-
return ret;
1919
+
goto err_pm_disable;
1920
1920
}
1921
1921
1922
1922
ret = rcar_dmac_init(dmac);
···
1924
1924
1925
1925
if (ret) {
1926
1926
dev_err(&pdev->dev, "failed to reset device\n");
1927
-
goto error;
1927
+
goto err_pm_disable;
1928
1928
}
1929
1929
1930
1930
/* Initialize engine */
···
1958
1958
for_each_rcar_dmac_chan(i, dmac, chan) {
1959
1959
ret = rcar_dmac_chan_probe(dmac, chan);
1960
1960
if (ret < 0)
1961
-
goto error;
1961
+
goto err_pm_disable;
1962
1962
}
1963
1963
1964
1964
/* Register the DMAC as a DMA provider for DT. */
1965
1965
ret = of_dma_controller_register(pdev->dev.of_node, rcar_dmac_of_xlate,
1966
1966
NULL);
1967
1967
if (ret < 0)
1968
-
goto error;
1968
+
goto err_pm_disable;
1969
1969
1970
1970
/*
1971
1971
* Register the DMA engine device.
···
1974
1974
*/
1975
1975
ret = dma_async_device_register(engine);
1976
1976
if (ret < 0)
1977
-
goto error;
1977
+
goto err_dma_free;
1978
1978
1979
1979
return 0;
1980
1980
1981
-
error:
1981
+
err_dma_free:
1982
1982
of_dma_controller_free(pdev->dev.of_node);
1983
+
err_pm_disable:
1983
1984
pm_runtime_disable(&pdev->dev);
1984
1985
return ret;
1985
1986
}
+15
-1
drivers/dma/sh/rz-dmac.c
+15
-1
drivers/dma/sh/rz-dmac.c
···
18
18
#include <linux/of_dma.h>
19
19
#include <linux/of_platform.h>
20
20
#include <linux/platform_device.h>
21
+
#include <linux/pm_runtime.h>
21
22
#include <linux/slab.h>
22
23
#include <linux/spinlock.h>
23
24
···
574
573
static u8 rz_dmac_ds_to_val_mapping(enum dma_slave_buswidth ds)
575
574
{
576
575
u8 i;
577
-
const enum dma_slave_buswidth ds_lut[] = {
576
+
static const enum dma_slave_buswidth ds_lut[] = {
578
577
DMA_SLAVE_BUSWIDTH_1_BYTE,
579
578
DMA_SLAVE_BUSWIDTH_2_BYTES,
580
579
DMA_SLAVE_BUSWIDTH_4_BYTES,
···
873
872
/* Initialize the channels. */
874
873
INIT_LIST_HEAD(&dmac->engine.channels);
875
874
875
+
pm_runtime_enable(&pdev->dev);
876
+
ret = pm_runtime_resume_and_get(&pdev->dev);
877
+
if (ret < 0) {
878
+
dev_err(&pdev->dev, "pm_runtime_resume_and_get failed\n");
879
+
goto err_pm_disable;
880
+
}
881
+
876
882
for (i = 0; i < dmac->n_channels; i++) {
877
883
ret = rz_dmac_chan_probe(dmac, &dmac->channels[i], i);
878
884
if (ret < 0)
···
933
925
channel->lmdesc.base_dma);
934
926
}
935
927
928
+
pm_runtime_put(&pdev->dev);
929
+
err_pm_disable:
930
+
pm_runtime_disable(&pdev->dev);
931
+
936
932
return ret;
937
933
}
938
934
···
955
943
}
956
944
of_dma_controller_free(pdev->dev.of_node);
957
945
dma_async_device_unregister(&dmac->engine);
946
+
pm_runtime_put(&pdev->dev);
947
+
pm_runtime_disable(&pdev->dev);
958
948
959
949
return 0;
960
950
}
+18
-6
drivers/dma/stm32-dma.c
+18
-6
drivers/dma/stm32-dma.c
···
270
270
u32 threshold)
271
271
{
272
272
enum dma_slave_buswidth max_width;
273
-
u64 addr = buf_addr;
274
273
275
274
if (threshold == STM32_DMA_FIFO_THRESHOLD_FULL)
276
275
max_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
···
280
281
max_width > DMA_SLAVE_BUSWIDTH_1_BYTE)
281
282
max_width = max_width >> 1;
282
283
283
-
if (do_div(addr, max_width))
284
+
if (buf_addr & (max_width - 1))
284
285
max_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
285
286
286
287
return max_width;
···
496
497
spin_lock_irqsave(&chan->vchan.lock, flags);
497
498
498
499
if (chan->desc) {
500
+
dma_cookie_complete(&chan->desc->vdesc.tx);
499
501
vchan_terminate_vdesc(&chan->desc->vdesc);
500
502
if (chan->busy)
501
503
stm32_dma_stop(chan);
···
753
753
if (src_bus_width < 0)
754
754
return src_bus_width;
755
755
756
-
/* Set memory burst size */
757
-
src_maxburst = STM32_DMA_MAX_BURST;
756
+
/*
757
+
* Set memory burst size - burst not possible if address is not aligned on
758
+
* the address boundary equal to the size of the transfer
759
+
*/
760
+
if (buf_addr & (buf_len - 1))
761
+
src_maxburst = 1;
762
+
else
763
+
src_maxburst = STM32_DMA_MAX_BURST;
758
764
src_best_burst = stm32_dma_get_best_burst(buf_len,
759
765
src_maxburst,
760
766
fifoth,
···
809
803
if (dst_bus_width < 0)
810
804
return dst_bus_width;
811
805
812
-
/* Set memory burst size */
813
-
dst_maxburst = STM32_DMA_MAX_BURST;
806
+
/*
807
+
* Set memory burst size - burst not possible if address is not aligned on
808
+
* the address boundary equal to the size of the transfer
809
+
*/
810
+
if (buf_addr & (buf_len - 1))
811
+
dst_maxburst = 1;
812
+
else
813
+
dst_maxburst = STM32_DMA_MAX_BURST;
814
814
dst_best_burst = stm32_dma_get_best_burst(buf_len,
815
815
dst_maxburst,
816
816
fifoth,
+2
-1
drivers/dma/stm32-mdma.c
+2
-1
drivers/dma/stm32-mdma.c
···
1566
1566
if (count < 0)
1567
1567
count = 0;
1568
1568
1569
-
dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev) + sizeof(u32) * count,
1569
+
dmadev = devm_kzalloc(&pdev->dev,
1570
+
struct_size(dmadev, ahb_addr_masks, count),
1570
1571
GFP_KERNEL);
1571
1572
if (!dmadev)
1572
1573
return -ENOMEM;
+39
-19
drivers/dma/tegra210-adma.c
+39
-19
drivers/dma/tegra210-adma.c
···
43
43
#define TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(reqs) (reqs << 4)
44
44
45
45
#define ADMA_CH_FIFO_CTRL 0x2c
46
-
#define TEGRA210_ADMA_CH_FIFO_CTRL_TXSIZE(val) (((val) & 0xf) << 8)
47
-
#define TEGRA210_ADMA_CH_FIFO_CTRL_RXSIZE(val) ((val) & 0xf)
48
-
#define TEGRA186_ADMA_CH_FIFO_CTRL_TXSIZE(val) (((val) & 0x1f) << 8)
49
-
#define TEGRA186_ADMA_CH_FIFO_CTRL_RXSIZE(val) ((val) & 0x1f)
46
+
#define ADMA_CH_TX_FIFO_SIZE_SHIFT 8
47
+
#define ADMA_CH_RX_FIFO_SIZE_SHIFT 0
50
48
51
49
#define ADMA_CH_LOWER_SRC_ADDR 0x34
52
50
#define ADMA_CH_LOWER_TRG_ADDR 0x3c
···
59
61
60
62
#define TEGRA_ADMA_BURST_COMPLETE_TIME 20
61
63
62
-
#define TEGRA210_FIFO_CTRL_DEFAULT (TEGRA210_ADMA_CH_FIFO_CTRL_TXSIZE(3) | \
63
-
TEGRA210_ADMA_CH_FIFO_CTRL_RXSIZE(3))
64
-
65
-
#define TEGRA186_FIFO_CTRL_DEFAULT (TEGRA186_ADMA_CH_FIFO_CTRL_TXSIZE(3) | \
66
-
TEGRA186_ADMA_CH_FIFO_CTRL_RXSIZE(3))
67
-
68
64
#define ADMA_CH_REG_FIELD_VAL(val, mask, shift) (((val) & mask) << shift)
69
65
70
66
struct tegra_adma;
71
67
72
68
/*
73
69
* struct tegra_adma_chip_data - Tegra chip specific data
70
+
* @adma_get_burst_config: Function callback used to set DMA burst size.
74
71
* @global_reg_offset: Register offset of DMA global register.
75
72
* @global_int_clear: Register offset of DMA global interrupt clear.
76
73
* @ch_req_tx_shift: Register offset for AHUB transmit channel select.
77
74
* @ch_req_rx_shift: Register offset for AHUB receive channel select.
78
75
* @ch_base_offset: Register offset of DMA channel registers.
79
-
* @has_outstanding_reqs: If DMA channel can have outstanding requests.
80
76
* @ch_fifo_ctrl: Default value for channel FIFO CTRL register.
81
77
* @ch_req_mask: Mask for Tx or Rx channel select.
82
78
* @ch_req_max: Maximum number of Tx or Rx channels available.
83
79
* @ch_reg_size: Size of DMA channel register space.
84
80
* @nr_channels: Number of DMA channels available.
81
+
* @ch_fifo_size_mask: Mask for FIFO size field.
82
+
* @sreq_index_offset: Slave channel index offset.
83
+
* @has_outstanding_reqs: If DMA channel can have outstanding requests.
85
84
*/
86
85
struct tegra_adma_chip_data {
87
86
unsigned int (*adma_get_burst_config)(unsigned int burst_size);
···
92
97
unsigned int ch_req_max;
93
98
unsigned int ch_reg_size;
94
99
unsigned int nr_channels;
100
+
unsigned int ch_fifo_size_mask;
101
+
unsigned int sreq_index_offset;
95
102
bool has_outstanding_reqs;
96
103
};
97
104
···
557
560
{
558
561
struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs;
559
562
const struct tegra_adma_chip_data *cdata = tdc->tdma->cdata;
560
-
unsigned int burst_size, adma_dir;
563
+
unsigned int burst_size, adma_dir, fifo_size_shift;
561
564
562
565
if (desc->num_periods > ADMA_CH_CONFIG_MAX_BUFS)
563
566
return -EINVAL;
564
567
565
568
switch (direction) {
566
569
case DMA_MEM_TO_DEV:
570
+
fifo_size_shift = ADMA_CH_TX_FIFO_SIZE_SHIFT;
567
571
adma_dir = ADMA_CH_CTRL_DIR_MEM2AHUB;
568
572
burst_size = tdc->sconfig.dst_maxburst;
569
573
ch_regs->config = ADMA_CH_CONFIG_SRC_BUF(desc->num_periods - 1);
···
575
577
break;
576
578
577
579
case DMA_DEV_TO_MEM:
580
+
fifo_size_shift = ADMA_CH_RX_FIFO_SIZE_SHIFT;
578
581
adma_dir = ADMA_CH_CTRL_DIR_AHUB2MEM;
579
582
burst_size = tdc->sconfig.src_maxburst;
580
583
ch_regs->config = ADMA_CH_CONFIG_TRG_BUF(desc->num_periods - 1);
···
597
598
ch_regs->config |= ADMA_CH_CONFIG_WEIGHT_FOR_WRR(1);
598
599
if (cdata->has_outstanding_reqs)
599
600
ch_regs->config |= TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(8);
600
-
ch_regs->fifo_ctrl = cdata->ch_fifo_ctrl;
601
+
602
+
/*
603
+
* 'sreq_index' represents the current ADMAIF channel number and as per
604
+
* HW recommendation its FIFO size should match with the corresponding
605
+
* ADMA channel.
606
+
*
607
+
* ADMA FIFO size is set as per below (based on default ADMAIF channel
608
+
* FIFO sizes):
609
+
* fifo_size = 0x2 (sreq_index > sreq_index_offset)
610
+
* fifo_size = 0x3 (sreq_index <= sreq_index_offset)
611
+
*
612
+
*/
613
+
if (tdc->sreq_index > cdata->sreq_index_offset)
614
+
ch_regs->fifo_ctrl =
615
+
ADMA_CH_REG_FIELD_VAL(2, cdata->ch_fifo_size_mask,
616
+
fifo_size_shift);
617
+
else
618
+
ch_regs->fifo_ctrl =
619
+
ADMA_CH_REG_FIELD_VAL(3, cdata->ch_fifo_size_mask,
620
+
fifo_size_shift);
621
+
601
622
ch_regs->tc = desc->period_len & ADMA_CH_TC_COUNT_MASK;
602
623
603
624
return tegra_adma_request_alloc(tdc, direction);
···
801
782
.ch_req_tx_shift = 28,
802
783
.ch_req_rx_shift = 24,
803
784
.ch_base_offset = 0,
804
-
.has_outstanding_reqs = false,
805
-
.ch_fifo_ctrl = TEGRA210_FIFO_CTRL_DEFAULT,
806
785
.ch_req_mask = 0xf,
807
786
.ch_req_max = 10,
808
787
.ch_reg_size = 0x80,
809
788
.nr_channels = 22,
789
+
.ch_fifo_size_mask = 0xf,
790
+
.sreq_index_offset = 2,
791
+
.has_outstanding_reqs = false,
810
792
};
811
793
812
794
static const struct tegra_adma_chip_data tegra186_chip_data = {
···
817
797
.ch_req_tx_shift = 27,
818
798
.ch_req_rx_shift = 22,
819
799
.ch_base_offset = 0x10000,
820
-
.has_outstanding_reqs = true,
821
-
.ch_fifo_ctrl = TEGRA186_FIFO_CTRL_DEFAULT,
822
800
.ch_req_mask = 0x1f,
823
801
.ch_req_max = 20,
824
802
.ch_reg_size = 0x100,
825
803
.nr_channels = 32,
804
+
.ch_fifo_size_mask = 0x1f,
805
+
.sreq_index_offset = 4,
806
+
.has_outstanding_reqs = true,
826
807
};
827
808
828
809
static const struct of_device_id tegra_adma_of_match[] = {
···
888
867
889
868
pm_runtime_enable(&pdev->dev);
890
869
891
-
ret = pm_runtime_get_sync(&pdev->dev);
870
+
ret = pm_runtime_resume_and_get(&pdev->dev);
892
871
if (ret < 0)
893
872
goto rpm_disable;
894
873
···
961
940
for (i = 0; i < tdma->nr_channels; ++i)
962
941
irq_dispose_mapping(tdma->channels[i].irq);
963
942
964
-
pm_runtime_put_sync(&pdev->dev);
965
943
pm_runtime_disable(&pdev->dev);
966
944
967
945
return 0;
+26
-6
drivers/dma/ti/k3-udma.c
+26
-6
drivers/dma/ti/k3-udma.c
···
1348
1348
{
1349
1349
struct udma_dev *ud = uc->ud;
1350
1350
enum udma_tp_level tpl;
1351
+
int ret;
1351
1352
1352
1353
if (uc->bchan) {
1353
1354
dev_dbg(ud->dev, "chan%d: already have bchan%d allocated\n",
···
1366
1365
tpl = ud->bchan_tpl.levels - 1;
1367
1366
1368
1367
uc->bchan = __udma_reserve_bchan(ud, tpl, -1);
1369
-
if (IS_ERR(uc->bchan))
1370
-
return PTR_ERR(uc->bchan);
1368
+
if (IS_ERR(uc->bchan)) {
1369
+
ret = PTR_ERR(uc->bchan);
1370
+
uc->bchan = NULL;
1371
+
return ret;
1372
+
}
1371
1373
1372
1374
uc->tchan = uc->bchan;
1373
1375
···
1380
1376
static int udma_get_tchan(struct udma_chan *uc)
1381
1377
{
1382
1378
struct udma_dev *ud = uc->ud;
1379
+
int ret;
1383
1380
1384
1381
if (uc->tchan) {
1385
1382
dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n",
···
1395
1390
*/
1396
1391
uc->tchan = __udma_reserve_tchan(ud, uc->config.channel_tpl,
1397
1392
uc->config.mapped_channel_id);
1398
-
if (IS_ERR(uc->tchan))
1399
-
return PTR_ERR(uc->tchan);
1393
+
if (IS_ERR(uc->tchan)) {
1394
+
ret = PTR_ERR(uc->tchan);
1395
+
uc->tchan = NULL;
1396
+
return ret;
1397
+
}
1400
1398
1401
1399
if (ud->tflow_cnt) {
1402
1400
int tflow_id;
···
1429
1421
static int udma_get_rchan(struct udma_chan *uc)
1430
1422
{
1431
1423
struct udma_dev *ud = uc->ud;
1424
+
int ret;
1432
1425
1433
1426
if (uc->rchan) {
1434
1427
dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n",
···
1444
1435
*/
1445
1436
uc->rchan = __udma_reserve_rchan(ud, uc->config.channel_tpl,
1446
1437
uc->config.mapped_channel_id);
1438
+
if (IS_ERR(uc->rchan)) {
1439
+
ret = PTR_ERR(uc->rchan);
1440
+
uc->rchan = NULL;
1441
+
return ret;
1442
+
}
1447
1443
1448
-
return PTR_ERR_OR_ZERO(uc->rchan);
1444
+
return 0;
1449
1445
}
1450
1446
1451
1447
static int udma_get_chan_pair(struct udma_chan *uc)
···
1504
1490
static int udma_get_rflow(struct udma_chan *uc, int flow_id)
1505
1491
{
1506
1492
struct udma_dev *ud = uc->ud;
1493
+
int ret;
1507
1494
1508
1495
if (!uc->rchan) {
1509
1496
dev_err(ud->dev, "chan%d: does not have rchan??\n", uc->id);
···
1518
1503
}
1519
1504
1520
1505
uc->rflow = __udma_get_rflow(ud, flow_id);
1506
+
if (IS_ERR(uc->rflow)) {
1507
+
ret = PTR_ERR(uc->rflow);
1508
+
uc->rflow = NULL;
1509
+
return ret;
1510
+
}
1521
1511
1522
-
return PTR_ERR_OR_ZERO(uc->rflow);
1512
+
return 0;
1523
1513
}
1524
1514
1525
1515
static void bcdma_put_bchan(struct udma_chan *uc)
+6
-8
drivers/dma/xilinx/xilinx_dma.c
+6
-8
drivers/dma/xilinx/xilinx_dma.c
···
792
792
}
793
793
794
794
/**
795
-
* xilinx_dma_tx_descriptor - Allocate transaction descriptor
795
+
* xilinx_dma_alloc_tx_descriptor - Allocate transaction descriptor
796
796
* @chan: Driver specific DMA channel
797
797
*
798
798
* Return: The allocated descriptor on success and NULL on failure.
···
998
998
struct xilinx_dma_tx_descriptor *desc,
999
999
unsigned long *flags)
1000
1000
{
1001
-
dma_async_tx_callback callback;
1002
-
void *callback_param;
1001
+
struct dmaengine_desc_callback cb;
1003
1002
1004
-
callback = desc->async_tx.callback;
1005
-
callback_param = desc->async_tx.callback_param;
1006
-
if (callback) {
1003
+
dmaengine_desc_get_callback(&desc->async_tx, &cb);
1004
+
if (dmaengine_desc_callback_valid(&cb)) {
1007
1005
spin_unlock_irqrestore(&chan->lock, *flags);
1008
-
callback(callback_param);
1006
+
dmaengine_desc_callback_invoke(&cb, NULL);
1009
1007
spin_lock_irqsave(&chan->lock, *flags);
1010
1008
}
1011
1009
}
···
2481
2483
}
2482
2484
2483
2485
/**
2484
-
* xilinx_dma_channel_set_config - Configure VDMA channel
2486
+
* xilinx_vdma_channel_set_config - Configure VDMA channel
2485
2487
* Run-time configuration for Axi VDMA, supports:
2486
2488
* . halt the channel
2487
2489
* . configure interrupt coalescing and inter-packet delay threshold
+1
-14
drivers/dma/xilinx/xilinx_dpdma.c
+1
-14
drivers/dma/xilinx/xilinx_dpdma.c
···
271
271
/* -----------------------------------------------------------------------------
272
272
* DebugFS
273
273
*/
274
-
275
-
#ifdef CONFIG_DEBUG_FS
276
-
277
274
#define XILINX_DPDMA_DEBUGFS_READ_MAX_SIZE 32
278
275
#define XILINX_DPDMA_DEBUGFS_UINT16_MAX_STR "65535"
279
276
···
296
299
297
300
static void xilinx_dpdma_debugfs_desc_done_irq(struct xilinx_dpdma_chan *chan)
298
301
{
299
-
if (chan->id == dpdma_debugfs.chan_id)
302
+
if (IS_ENABLED(CONFIG_DEBUG_FS) && chan->id == dpdma_debugfs.chan_id)
300
303
dpdma_debugfs.xilinx_dpdma_irq_done_count++;
301
304
}
302
305
···
458
461
if (IS_ERR(dent))
459
462
dev_err(xdev->dev, "Failed to create debugfs testcase file\n");
460
463
}
461
-
462
-
#else
463
-
static void xilinx_dpdma_debugfs_init(struct xilinx_dpdma_device *xdev)
464
-
{
465
-
}
466
-
467
-
static void xilinx_dpdma_debugfs_desc_done_irq(struct xilinx_dpdma_chan *chan)
468
-
{
469
-
}
470
-
#endif /* CONFIG_DEBUG_FS */
471
464
472
465
/* -----------------------------------------------------------------------------
473
466
* I/O Accessors
+39
-40
drivers/dma/xilinx/zynqmp_dma.c
+39
-40
drivers/dma/xilinx/zynqmp_dma.c
···
6
6
*/
7
7
8
8
#include <linux/bitops.h>
9
-
#include <linux/dmapool.h>
10
-
#include <linux/dma/xilinx_dma.h>
9
+
#include <linux/dma-mapping.h>
11
10
#include <linux/init.h>
12
11
#include <linux/interrupt.h>
13
12
#include <linux/io.h>
14
13
#include <linux/module.h>
15
-
#include <linux/of_address.h>
16
14
#include <linux/of_dma.h>
17
-
#include <linux/of_irq.h>
18
15
#include <linux/of_platform.h>
19
16
#include <linux/slab.h>
20
17
#include <linux/clk.h>
···
600
603
static void zynqmp_dma_chan_desc_cleanup(struct zynqmp_dma_chan *chan)
601
604
{
602
605
struct zynqmp_dma_desc_sw *desc, *next;
606
+
unsigned long irqflags;
607
+
608
+
spin_lock_irqsave(&chan->lock, irqflags);
603
609
604
610
list_for_each_entry_safe(desc, next, &chan->done_list, node) {
605
-
dma_async_tx_callback callback;
606
-
void *callback_param;
611
+
struct dmaengine_desc_callback cb;
607
612
608
-
callback = desc->async_tx.callback;
609
-
callback_param = desc->async_tx.callback_param;
610
-
if (callback) {
611
-
spin_unlock(&chan->lock);
612
-
callback(callback_param);
613
-
spin_lock(&chan->lock);
613
+
dmaengine_desc_get_callback(&desc->async_tx, &cb);
614
+
if (dmaengine_desc_callback_valid(&cb)) {
615
+
spin_unlock_irqrestore(&chan->lock, irqflags);
616
+
dmaengine_desc_callback_invoke(&cb, NULL);
617
+
spin_lock_irqsave(&chan->lock, irqflags);
614
618
}
615
619
616
620
/* Run any dependencies, then free the descriptor */
617
621
zynqmp_dma_free_descriptor(chan, desc);
618
622
}
623
+
624
+
spin_unlock_irqrestore(&chan->lock, irqflags);
619
625
}
620
626
621
627
/**
···
658
658
*/
659
659
static void zynqmp_dma_free_descriptors(struct zynqmp_dma_chan *chan)
660
660
{
661
+
unsigned long irqflags;
662
+
663
+
spin_lock_irqsave(&chan->lock, irqflags);
661
664
zynqmp_dma_free_desc_list(chan, &chan->active_list);
662
665
zynqmp_dma_free_desc_list(chan, &chan->pending_list);
663
666
zynqmp_dma_free_desc_list(chan, &chan->done_list);
667
+
spin_unlock_irqrestore(&chan->lock, irqflags);
664
668
}
665
669
666
670
/**
···
674
670
static void zynqmp_dma_free_chan_resources(struct dma_chan *dchan)
675
671
{
676
672
struct zynqmp_dma_chan *chan = to_chan(dchan);
677
-
unsigned long irqflags;
678
673
679
-
spin_lock_irqsave(&chan->lock, irqflags);
680
674
zynqmp_dma_free_descriptors(chan);
681
-
spin_unlock_irqrestore(&chan->lock, irqflags);
682
675
dma_free_coherent(chan->dev,
683
676
(2 * ZYNQMP_DMA_DESC_SIZE(chan) * ZYNQMP_DMA_NUM_DESCS),
684
677
chan->desc_pool_v, chan->desc_pool_p);
···
690
689
*/
691
690
static void zynqmp_dma_reset(struct zynqmp_dma_chan *chan)
692
691
{
692
+
unsigned long irqflags;
693
+
693
694
writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS);
694
695
696
+
spin_lock_irqsave(&chan->lock, irqflags);
695
697
zynqmp_dma_complete_descriptor(chan);
698
+
spin_unlock_irqrestore(&chan->lock, irqflags);
696
699
zynqmp_dma_chan_desc_cleanup(chan);
697
700
zynqmp_dma_free_descriptors(chan);
701
+
698
702
zynqmp_dma_init(chan);
699
703
}
700
704
···
755
749
u32 count;
756
750
unsigned long irqflags;
757
751
758
-
spin_lock_irqsave(&chan->lock, irqflags);
759
-
760
752
if (chan->err) {
761
753
zynqmp_dma_reset(chan);
762
754
chan->err = false;
763
-
goto unlock;
755
+
return;
764
756
}
765
757
758
+
spin_lock_irqsave(&chan->lock, irqflags);
766
759
count = readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT);
767
-
768
760
while (count) {
769
761
zynqmp_dma_complete_descriptor(chan);
770
-
zynqmp_dma_chan_desc_cleanup(chan);
771
762
count--;
772
763
}
773
-
774
-
if (chan->idle)
775
-
zynqmp_dma_start_transfer(chan);
776
-
777
-
unlock:
778
764
spin_unlock_irqrestore(&chan->lock, irqflags);
765
+
766
+
zynqmp_dma_chan_desc_cleanup(chan);
767
+
768
+
if (chan->idle) {
769
+
spin_lock_irqsave(&chan->lock, irqflags);
770
+
zynqmp_dma_start_transfer(chan);
771
+
spin_unlock_irqrestore(&chan->lock, irqflags);
772
+
}
779
773
}
780
774
781
775
/**
···
787
781
static int zynqmp_dma_device_terminate_all(struct dma_chan *dchan)
788
782
{
789
783
struct zynqmp_dma_chan *chan = to_chan(dchan);
790
-
unsigned long irqflags;
791
784
792
-
spin_lock_irqsave(&chan->lock, irqflags);
793
785
writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS);
794
786
zynqmp_dma_free_descriptors(chan);
795
-
spin_unlock_irqrestore(&chan->lock, irqflags);
796
787
797
788
return 0;
798
789
}
···
1064
1061
p->dev = &pdev->dev;
1065
1062
1066
1063
zdev->clk_main = devm_clk_get(&pdev->dev, "clk_main");
1067
-
if (IS_ERR(zdev->clk_main)) {
1068
-
dev_err(&pdev->dev, "main clock not found.\n");
1069
-
return PTR_ERR(zdev->clk_main);
1070
-
}
1064
+
if (IS_ERR(zdev->clk_main))
1065
+
return dev_err_probe(&pdev->dev, PTR_ERR(zdev->clk_main),
1066
+
"main clock not found.\n");
1071
1067
1072
1068
zdev->clk_apb = devm_clk_get(&pdev->dev, "clk_apb");
1073
-
if (IS_ERR(zdev->clk_apb)) {
1074
-
dev_err(&pdev->dev, "apb clock not found.\n");
1075
-
return PTR_ERR(zdev->clk_apb);
1076
-
}
1069
+
if (IS_ERR(zdev->clk_apb))
1070
+
return dev_err_probe(&pdev->dev, PTR_ERR(zdev->clk_apb),
1071
+
"apb clock not found.\n");
1077
1072
1078
1073
platform_set_drvdata(pdev, zdev);
1079
1074
pm_runtime_set_autosuspend_delay(zdev->dev, ZDMA_PM_TIMEOUT);
···
1086
1085
1087
1086
ret = zynqmp_dma_chan_probe(zdev, pdev);
1088
1087
if (ret) {
1089
-
dev_err(&pdev->dev, "Probing channel failed\n");
1088
+
dev_err_probe(&pdev->dev, ret, "Probing channel failed\n");
1090
1089
goto err_disable_pm;
1091
1090
}
1092
1091
···
1098
1097
ret = of_dma_controller_register(pdev->dev.of_node,
1099
1098
of_zynqmp_dma_xlate, zdev);
1100
1099
if (ret) {
1101
-
dev_err(&pdev->dev, "Unable to register DMA to DT\n");
1100
+
dev_err_probe(&pdev->dev, ret, "Unable to register DMA to DT\n");
1102
1101
dma_async_device_unregister(&zdev->common);
1103
1102
goto free_chan_resources;
1104
1103
}
1105
1104
1106
1105
pm_runtime_mark_last_busy(zdev->dev);
1107
1106
pm_runtime_put_sync_autosuspend(zdev->dev);
1108
-
1109
-
dev_info(&pdev->dev, "ZynqMP DMA driver Probe success\n");
1110
1107
1111
1108
return 0;
1112
1109
-2
include/linux/dmaengine.h
-2
include/linux/dmaengine.h
···
944
944
void (*device_issue_pending)(struct dma_chan *chan);
945
945
void (*device_release)(struct dma_device *dev);
946
946
/* debugfs support */
947
-
#ifdef CONFIG_DEBUG_FS
948
947
void (*dbg_summary_show)(struct seq_file *s, struct dma_device *dev);
949
948
struct dentry *dbg_dev_root;
950
-
#endif
951
949
};
952
950
953
951
static inline int dmaengine_slave_config(struct dma_chan *chan,