Merge tag 'dmaengine-fix-6.8-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine

-15

Documentation/devicetree/bindings/dma/dma-controller.yaml

··· 19 19 20 20 additionalProperties: true 21 21 22 - examples: 23 - - | 24 - dma: dma-controller@48000000 { 25 - compatible = "ti,omap-sdma"; 26 - reg = <0x48000000 0x1000>; 27 - interrupts = <0 12 0x4>, 28 - <0 13 0x4>, 29 - <0 14 0x4>, 30 - <0 15 0x4>; 31 - #dma-cells = <1>; 32 - dma-channels = <32>; 33 - dma-requests = <127>; 34 - dma-channel-mask = <0xfffe>; 35 - }; 36 - 37 22 ...

-11

Documentation/devicetree/bindings/dma/dma-router.yaml

··· 40 40 41 41 additionalProperties: true 42 42 43 - examples: 44 - - | 45 - sdma_xbar: dma-router@4a002b78 { 46 - compatible = "ti,dra7-dma-crossbar"; 47 - reg = <0x4a002b78 0xfc>; 48 - #dma-cells = <1>; 49 - dma-requests = <205>; 50 - ti,dma-safe-map = <0>; 51 - dma-masters = <&sdma>; 52 - }; 53 - 54 43 ...

+62

Documentation/devicetree/bindings/dma/loongson,ls2x-apbdma.yaml

··· 1 + # SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 + %YAML 1.2 3 + --- 4 + $id: http://devicetree.org/schemas/dma/loongson,ls2x-apbdma.yaml# 5 + $schema: http://devicetree.org/meta-schemas/core.yaml# 6 + 7 + title: Loongson LS2X APB DMA controller 8 + 9 + description: 10 + The Loongson LS2X APB DMA controller is used for transferring data 11 + between system memory and the peripherals on the APB bus. 12 + 13 + maintainers: 14 + - Binbin Zhou <zhoubinbin@loongson.cn> 15 + 16 + allOf: 17 + - $ref: dma-controller.yaml# 18 + 19 + properties: 20 + compatible: 21 + oneOf: 22 + - const: loongson,ls2k1000-apbdma 23 + - items: 24 + - const: loongson,ls2k0500-apbdma 25 + - const: loongson,ls2k1000-apbdma 26 + 27 + reg: 28 + maxItems: 1 29 + 30 + interrupts: 31 + maxItems: 1 32 + 33 + clocks: 34 + maxItems: 1 35 + 36 + '#dma-cells': 37 + const: 1 38 + 39 + required: 40 + - compatible 41 + - reg 42 + - interrupts 43 + - clocks 44 + - '#dma-cells' 45 + 46 + additionalProperties: false 47 + 48 + examples: 49 + - | 50 + #include <dt-bindings/interrupt-controller/irq.h> 51 + #include <dt-bindings/clock/loongson,ls2k-clk.h> 52 + 53 + dma-controller@1fe00c00 { 54 + compatible = "loongson,ls2k1000-apbdma"; 55 + reg = <0x1fe00c00 0x8>; 56 + interrupt-parent = <&liointc1>; 57 + interrupts = <12 IRQ_TYPE_LEVEL_HIGH>; 58 + clocks = <&clk LOONGSON2_APB_CLK>; 59 + #dma-cells = <1>; 60 + }; 61 + 62 + ...

+3

Documentation/devicetree/bindings/dma/nvidia,tegra210-adma.yaml

··· 53 53 ADMA_CHn_CTRL register. 54 54 const: 1 55 55 56 + dma-channel-mask: 57 + maxItems: 1 58 + 56 59 required: 57 60 - compatible 58 61 - reg

+2

Documentation/devicetree/bindings/dma/qcom,gpi.yaml

··· 32 32 - qcom,sm8350-gpi-dma 33 33 - qcom,sm8450-gpi-dma 34 34 - qcom,sm8550-gpi-dma 35 + - qcom,sm8650-gpi-dma 36 + - qcom,x1e80100-gpi-dma 35 37 - const: qcom,sm6350-gpi-dma 36 38 - items: 37 39 - enum:

+1 -1

Documentation/devicetree/bindings/dma/renesas,rz-dmac.yaml

··· 16 16 compatible: 17 17 items: 18 18 - enum: 19 - - renesas,r9a07g043-dmac # RZ/G2UL 19 + - renesas,r9a07g043-dmac # RZ/G2UL and RZ/Five 20 20 - renesas,r9a07g044-dmac # RZ/G2{L,LC} 21 21 - renesas,r9a07g054-dmac # RZ/V2L 22 22 - const: renesas,rz-dmac

+1

Documentation/devicetree/bindings/dma/sifive,fu540-c000-pdma.yaml

··· 29 29 compatible: 30 30 items: 31 31 - enum: 32 + - microchip,mpfs-pdma 32 33 - sifive,fu540-c000-pdma 33 34 - const: sifive,pdma0 34 35 description:

+33 -6

Documentation/devicetree/bindings/dma/ti/k3-bcdma.yaml

··· 37 37 38 38 reg: 39 39 minItems: 3 40 - maxItems: 5 40 + maxItems: 9 41 41 42 42 reg-names: 43 43 minItems: 3 44 - maxItems: 5 44 + maxItems: 9 45 45 46 46 "#dma-cells": 47 47 const: 3 ··· 141 141 ti,sci-rm-range-tchan: false 142 142 143 143 reg: 144 - maxItems: 3 144 + items: 145 + - description: BCDMA Control /Status Registers region 146 + - description: RX Channel Realtime Registers region 147 + - description: Ring Realtime Registers region 145 148 146 149 reg-names: 147 150 items: ··· 164 161 properties: 165 162 reg: 166 163 minItems: 5 164 + items: 165 + - description: BCDMA Control /Status Registers region 166 + - description: Block Copy Channel Realtime Registers region 167 + - description: RX Channel Realtime Registers region 168 + - description: TX Channel Realtime Registers region 169 + - description: Ring Realtime Registers region 170 + - description: Ring Configuration Registers region 171 + - description: TX Channel Configuration Registers region 172 + - description: RX Channel Configuration Registers region 173 + - description: Block Copy Channel Configuration Registers region 167 174 168 175 reg-names: 176 + minItems: 5 169 177 items: 170 178 - const: gcfg 171 179 - const: bchanrt 172 180 - const: rchanrt 173 181 - const: tchanrt 174 182 - const: ringrt 183 + - const: ring 184 + - const: tchan 185 + - const: rchan 186 + - const: bchan 175 187 176 188 required: 177 189 - ti,sci-rm-range-bchan ··· 202 184 ti,sci-rm-range-bchan: false 203 185 204 186 reg: 205 - maxItems: 4 187 + items: 188 + - description: BCDMA Control /Status Registers region 189 + - description: RX Channel Realtime Registers region 190 + - description: TX Channel Realtime Registers region 191 + - description: Ring Realtime Registers region 206 192 207 193 reg-names: 208 194 items: ··· 242 220 <0x0 0x4c000000 0x0 0x20000>, 243 221 <0x0 0x4a820000 0x0 0x20000>, 244 222 <0x0 0x4aa40000 0x0 0x20000>, 245 - <0x0 0x4bc00000 0x0 0x100000>; 246 - reg-names = "gcfg", "bchanrt", "rchanrt", "tchanrt", "ringrt"; 223 + <0x0 0x4bc00000 0x0 0x100000>, 224 + <0x0 0x48600000 0x0 0x8000>, 225 + <0x0 0x484a4000 0x0 0x2000>, 226 + <0x0 0x484c2000 0x0 0x2000>, 227 + <0x0 0x48420000 0x0 0x2000>; 228 + reg-names = "gcfg", "bchanrt", "rchanrt", "tchanrt", "ringrt", 229 + "ring", "tchan", "rchan", "bchan"; 247 230 msi-parent = <&inta_main_dmss>; 248 231 #dma-cells = <3>; 249 232

+23 -3

Documentation/devicetree/bindings/dma/ti/k3-pktdma.yaml

··· 45 45 The second cell is the ASEL value for the channel 46 46 47 47 reg: 48 - maxItems: 4 48 + minItems: 4 49 + items: 50 + - description: Packet DMA Control /Status Registers region 51 + - description: RX Channel Realtime Registers region 52 + - description: TX Channel Realtime Registers region 53 + - description: Ring Realtime Registers region 54 + - description: Ring Configuration Registers region 55 + - description: TX Configuration Registers region 56 + - description: RX Configuration Registers region 57 + - description: RX Flow Configuration Registers region 49 58 50 59 reg-names: 60 + minItems: 4 51 61 items: 52 62 - const: gcfg 53 63 - const: rchanrt 54 64 - const: tchanrt 55 65 - const: ringrt 66 + - const: ring 67 + - const: tchan 68 + - const: rchan 69 + - const: rflow 56 70 57 71 msi-parent: true 58 72 ··· 150 136 reg = <0x0 0x485c0000 0x0 0x100>, 151 137 <0x0 0x4a800000 0x0 0x20000>, 152 138 <0x0 0x4aa00000 0x0 0x40000>, 153 - <0x0 0x4b800000 0x0 0x400000>; 154 - reg-names = "gcfg", "rchanrt", "tchanrt", "ringrt"; 139 + <0x0 0x4b800000 0x0 0x400000>, 140 + <0x0 0x485e0000 0x0 0x20000>, 141 + <0x0 0x484a0000 0x0 0x4000>, 142 + <0x0 0x484c0000 0x0 0x2000>, 143 + <0x0 0x48430000 0x0 0x4000>; 144 + reg-names = "gcfg", "rchanrt", "tchanrt", "ringrt", 145 + "ring", "tchan", "rchan", "rflow"; 146 + 155 147 msi-parent = <&inta_main_dmss>; 156 148 #dma-cells = <2>; 157 149

+17 -3

Documentation/devicetree/bindings/dma/ti/k3-udma.yaml

··· 69 69 - ti,j721e-navss-mcu-udmap 70 70 71 71 reg: 72 - maxItems: 3 72 + minItems: 3 73 + items: 74 + - description: UDMA-P Control /Status Registers region 75 + - description: RX Channel Realtime Registers region 76 + - description: TX Channel Realtime Registers region 77 + - description: TX Configuration Registers region 78 + - description: RX Configuration Registers region 79 + - description: RX Flow Configuration Registers region 73 80 74 81 reg-names: 82 + minItems: 3 75 83 items: 76 84 - const: gcfg 77 85 - const: rchanrt 78 86 - const: tchanrt 87 + - const: tchan 88 + - const: rchan 89 + - const: rflow 79 90 80 91 msi-parent: true 81 92 ··· 169 158 compatible = "ti,am654-navss-main-udmap"; 170 159 reg = <0x0 0x31150000 0x0 0x100>, 171 160 <0x0 0x34000000 0x0 0x100000>, 172 - <0x0 0x35000000 0x0 0x100000>; 173 - reg-names = "gcfg", "rchanrt", "tchanrt"; 161 + <0x0 0x35000000 0x0 0x100000>, 162 + <0x0 0x30b00000 0x0 0x20000>, 163 + <0x0 0x30c00000 0x0 0x8000>, 164 + <0x0 0x30d00000 0x0 0x4000>; 165 + reg-names = "gcfg", "rchanrt", "tchanrt", "tchan", "rchan", "rflow"; 174 166 #dma-cells = <1>; 175 167 176 168 ti,ringacc = <&ringacc>;

+7

MAINTAINERS

··· 12639 12639 F: Documentation/devicetree/bindings/gpio/loongson,ls-gpio.yaml 12640 12640 F: drivers/gpio/gpio-loongson-64bit.c 12641 12641 12642 + LOONGSON LS2X APB DMA DRIVER 12643 + M: Binbin Zhou <zhoubinbin@loongson.cn> 12644 + L: dmaengine@vger.kernel.org 12645 + S: Maintained 12646 + F: Documentation/devicetree/bindings/dma/loongson,ls2x-apbdma.yaml 12647 + F: drivers/dma/ls2x-apb-dma.c 12648 + 12642 12649 LOONGSON LS2X I2C DRIVER 12643 12650 M: Binbin Zhou <zhoubinbin@loongson.cn> 12644 12651 L: linux-i2c@vger.kernel.org

+14

drivers/dma/Kconfig

··· 378 378 Enable support for DMA on NXP LPC18xx/43xx platforms 379 379 with PL080 and multiplexed DMA request lines. 380 380 381 + config LS2X_APB_DMA 382 + tristate "Loongson LS2X APB DMA support" 383 + depends on LOONGARCH || COMPILE_TEST 384 + select DMA_ENGINE 385 + select DMA_VIRTUAL_CHANNELS 386 + help 387 + Support for the Loongson LS2X APB DMA controller driver. The 388 + DMA controller is having single DMA channel which can be 389 + configured for different peripherals like audio, nand, sdio 390 + etc which is in APB bus. 391 + 392 + This DMA controller transfers data from memory to peripheral fifo. 393 + It does not support memory to memory data transfer. 394 + 381 395 config MCF_EDMA 382 396 tristate "Freescale eDMA engine support, ColdFire mcf5441x SoCs" 383 397 depends on M5441x || COMPILE_TEST

+1

drivers/dma/Makefile

··· 48 48 obj-y += idxd/ 49 49 obj-$(CONFIG_K3_DMA) += k3dma.o 50 50 obj-$(CONFIG_LPC18XX_DMAMUX) += lpc18xx-dmamux.o 51 + obj-$(CONFIG_LS2X_APB_DMA) += ls2x-apb-dma.o 51 52 obj-$(CONFIG_MILBEAUT_HDMAC) += milbeaut-hdmac.o 52 53 obj-$(CONFIG_MILBEAUT_XDMAC) += milbeaut-xdmac.o 53 54 obj-$(CONFIG_MMP_PDMA) += mmp_pdma.o

+4 -1

drivers/dma/apple-admac.c

··· 57 57 58 58 #define REG_BUS_WIDTH(ch) (0x8040 + (ch) * 0x200) 59 59 60 + #define BUS_WIDTH_WORD_SIZE GENMASK(3, 0) 61 + #define BUS_WIDTH_FRAME_SIZE GENMASK(7, 4) 60 62 #define BUS_WIDTH_8BIT 0x00 61 63 #define BUS_WIDTH_16BIT 0x01 62 64 #define BUS_WIDTH_32BIT 0x02 ··· 742 740 struct admac_data *ad = adchan->host; 743 741 bool is_tx = admac_chan_direction(adchan->no) == DMA_MEM_TO_DEV; 744 742 int wordsize = 0; 745 - u32 bus_width = 0; 743 + u32 bus_width = readl_relaxed(ad->base + REG_BUS_WIDTH(adchan->no)) & 744 + ~(BUS_WIDTH_WORD_SIZE | BUS_WIDTH_FRAME_SIZE); 746 745 747 746 switch (is_tx ? config->dst_addr_width : config->src_addr_width) { 748 747 case DMA_SLAVE_BUSWIDTH_1_BYTE:

+194 -92

drivers/dma/dma-axi-dmac.c

··· 81 81 #define AXI_DMAC_REG_CURRENT_DEST_ADDR 0x438 82 82 #define AXI_DMAC_REG_PARTIAL_XFER_LEN 0x44c 83 83 #define AXI_DMAC_REG_PARTIAL_XFER_ID 0x450 84 + #define AXI_DMAC_REG_CURRENT_SG_ID 0x454 85 + #define AXI_DMAC_REG_SG_ADDRESS 0x47c 86 + #define AXI_DMAC_REG_SG_ADDRESS_HIGH 0x4bc 84 87 85 88 #define AXI_DMAC_CTRL_ENABLE BIT(0) 86 89 #define AXI_DMAC_CTRL_PAUSE BIT(1) 90 + #define AXI_DMAC_CTRL_ENABLE_SG BIT(2) 87 91 88 92 #define AXI_DMAC_IRQ_SOT BIT(0) 89 93 #define AXI_DMAC_IRQ_EOT BIT(1) ··· 101 97 /* The maximum ID allocated by the hardware is 31 */ 102 98 #define AXI_DMAC_SG_UNUSED 32U 103 99 100 + /* Flags for axi_dmac_hw_desc.flags */ 101 + #define AXI_DMAC_HW_FLAG_LAST BIT(0) 102 + #define AXI_DMAC_HW_FLAG_IRQ BIT(1) 103 + 104 + struct axi_dmac_hw_desc { 105 + u32 flags; 106 + u32 id; 107 + u64 dest_addr; 108 + u64 src_addr; 109 + u64 next_sg_addr; 110 + u32 y_len; 111 + u32 x_len; 112 + u32 src_stride; 113 + u32 dst_stride; 114 + u64 __pad[2]; 115 + }; 116 + 104 117 struct axi_dmac_sg { 105 - dma_addr_t src_addr; 106 - dma_addr_t dest_addr; 107 - unsigned int x_len; 108 - unsigned int y_len; 109 - unsigned int dest_stride; 110 - unsigned int src_stride; 111 - unsigned int id; 112 118 unsigned int partial_len; 113 119 bool schedule_when_free; 120 + 121 + struct axi_dmac_hw_desc *hw; 122 + dma_addr_t hw_phys; 114 123 }; 115 124 116 125 struct axi_dmac_desc { 117 126 struct virt_dma_desc vdesc; 127 + struct axi_dmac_chan *chan; 128 + 118 129 bool cyclic; 119 130 bool have_partial_xfer; 120 131 ··· 158 139 bool hw_partial_xfer; 159 140 bool hw_cyclic; 160 141 bool hw_2d; 142 + bool hw_sg; 161 143 }; 162 144 163 145 struct axi_dmac { ··· 233 213 unsigned int flags = 0; 234 214 unsigned int val; 235 215 236 - val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); 237 - if (val) /* Queue is full, wait for the next SOT IRQ */ 238 - return; 216 + if (!chan->hw_sg) { 217 + val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); 218 + if (val) /* Queue is full, wait for the next SOT IRQ */ 219 + return; 220 + } 239 221 240 222 desc = chan->next_desc; 241 223 ··· 251 229 sg = &desc->sg[desc->num_submitted]; 252 230 253 231 /* Already queued in cyclic mode. Wait for it to finish */ 254 - if (sg->id != AXI_DMAC_SG_UNUSED) { 232 + if (sg->hw->id != AXI_DMAC_SG_UNUSED) { 255 233 sg->schedule_when_free = true; 256 234 return; 257 235 } 258 236 259 - desc->num_submitted++; 260 - if (desc->num_submitted == desc->num_sgs || 261 - desc->have_partial_xfer) { 237 + if (chan->hw_sg) { 238 + chan->next_desc = NULL; 239 + } else if (++desc->num_submitted == desc->num_sgs || 240 + desc->have_partial_xfer) { 262 241 if (desc->cyclic) 263 242 desc->num_submitted = 0; /* Start again */ 264 243 else ··· 269 246 chan->next_desc = desc; 270 247 } 271 248 272 - sg->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID); 249 + sg->hw->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID); 273 250 274 - if (axi_dmac_dest_is_mem(chan)) { 275 - axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->dest_addr); 276 - axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->dest_stride); 277 - } 251 + if (!chan->hw_sg) { 252 + if (axi_dmac_dest_is_mem(chan)) { 253 + axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->hw->dest_addr); 254 + axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->hw->dst_stride); 255 + } 278 256 279 - if (axi_dmac_src_is_mem(chan)) { 280 - axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->src_addr); 281 - axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->src_stride); 257 + if (axi_dmac_src_is_mem(chan)) { 258 + axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->hw->src_addr); 259 + axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->hw->src_stride); 260 + } 282 261 } 283 262 284 263 /* 285 264 * If the hardware supports cyclic transfers and there is no callback to 286 - * call and only a single segment, enable hw cyclic mode to avoid 287 - * unnecessary interrupts. 265 + * call, enable hw cyclic mode to avoid unnecessary interrupts. 288 266 */ 289 - if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback && 290 - desc->num_sgs == 1) 291 - flags |= AXI_DMAC_FLAG_CYCLIC; 267 + if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback) { 268 + if (chan->hw_sg) 269 + desc->sg[desc->num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_IRQ; 270 + else if (desc->num_sgs == 1) 271 + flags |= AXI_DMAC_FLAG_CYCLIC; 272 + } 292 273 293 274 if (chan->hw_partial_xfer) 294 275 flags |= AXI_DMAC_FLAG_PARTIAL_REPORT; 295 276 296 - axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->x_len - 1); 297 - axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->y_len - 1); 277 + if (chan->hw_sg) { 278 + axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS, (u32)sg->hw_phys); 279 + axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS_HIGH, 280 + (u64)sg->hw_phys >> 32); 281 + } else { 282 + axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->hw->x_len); 283 + axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->hw->y_len); 284 + } 298 285 axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, flags); 299 286 axi_dmac_write(dmac, AXI_DMAC_REG_START_TRANSFER, 1); 300 287 } ··· 319 286 struct axi_dmac_sg *sg) 320 287 { 321 288 if (chan->hw_2d) 322 - return sg->x_len * sg->y_len; 289 + return (sg->hw->x_len + 1) * (sg->hw->y_len + 1); 323 290 else 324 - return sg->x_len; 291 + return (sg->hw->x_len + 1); 325 292 } 326 293 327 294 static void axi_dmac_dequeue_partial_xfers(struct axi_dmac_chan *chan) ··· 340 307 list_for_each_entry(desc, &chan->active_descs, vdesc.node) { 341 308 for (i = 0; i < desc->num_sgs; i++) { 342 309 sg = &desc->sg[i]; 343 - if (sg->id == AXI_DMAC_SG_UNUSED) 310 + if (sg->hw->id == AXI_DMAC_SG_UNUSED) 344 311 continue; 345 - if (sg->id == id) { 312 + if (sg->hw->id == id) { 346 313 desc->have_partial_xfer = true; 347 314 sg->partial_len = len; 348 315 found_sg = true; ··· 381 348 rslt->result = DMA_TRANS_NOERROR; 382 349 rslt->residue = 0; 383 350 351 + if (chan->hw_sg) 352 + return; 353 + 384 354 /* 385 355 * We get here if the last completed segment is partial, which 386 356 * means we can compute the residue from that segment onwards ··· 410 374 (completed_transfers & AXI_DMAC_FLAG_PARTIAL_XFER_DONE)) 411 375 axi_dmac_dequeue_partial_xfers(chan); 412 376 413 - do { 414 - sg = &active->sg[active->num_completed]; 415 - if (sg->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */ 416 - break; 417 - if (!(BIT(sg->id) & completed_transfers)) 418 - break; 419 - active->num_completed++; 420 - sg->id = AXI_DMAC_SG_UNUSED; 421 - if (sg->schedule_when_free) { 422 - sg->schedule_when_free = false; 423 - start_next = true; 424 - } 425 - 426 - if (sg->partial_len) 427 - axi_dmac_compute_residue(chan, active); 428 - 429 - if (active->cyclic) 377 + if (chan->hw_sg) { 378 + if (active->cyclic) { 430 379 vchan_cyclic_callback(&active->vdesc); 431 - 432 - if (active->num_completed == active->num_sgs || 433 - sg->partial_len) { 434 - if (active->cyclic) { 435 - active->num_completed = 0; /* wrap around */ 436 - } else { 437 - list_del(&active->vdesc.node); 438 - vchan_cookie_complete(&active->vdesc); 439 - active = axi_dmac_active_desc(chan); 440 - } 380 + } else { 381 + list_del(&active->vdesc.node); 382 + vchan_cookie_complete(&active->vdesc); 383 + active = axi_dmac_active_desc(chan); 384 + start_next = !!active; 441 385 } 442 - } while (active); 386 + } else { 387 + do { 388 + sg = &active->sg[active->num_completed]; 389 + if (sg->hw->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */ 390 + break; 391 + if (!(BIT(sg->hw->id) & completed_transfers)) 392 + break; 393 + active->num_completed++; 394 + sg->hw->id = AXI_DMAC_SG_UNUSED; 395 + if (sg->schedule_when_free) { 396 + sg->schedule_when_free = false; 397 + start_next = true; 398 + } 399 + 400 + if (sg->partial_len) 401 + axi_dmac_compute_residue(chan, active); 402 + 403 + if (active->cyclic) 404 + vchan_cyclic_callback(&active->vdesc); 405 + 406 + if (active->num_completed == active->num_sgs || 407 + sg->partial_len) { 408 + if (active->cyclic) { 409 + active->num_completed = 0; /* wrap around */ 410 + } else { 411 + list_del(&active->vdesc.node); 412 + vchan_cookie_complete(&active->vdesc); 413 + active = axi_dmac_active_desc(chan); 414 + } 415 + } 416 + } while (active); 417 + } 443 418 444 419 return start_next; 445 420 } ··· 514 467 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 515 468 struct axi_dmac *dmac = chan_to_axi_dmac(chan); 516 469 unsigned long flags; 470 + u32 ctrl = AXI_DMAC_CTRL_ENABLE; 517 471 518 - axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, AXI_DMAC_CTRL_ENABLE); 472 + if (chan->hw_sg) 473 + ctrl |= AXI_DMAC_CTRL_ENABLE_SG; 474 + 475 + axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, ctrl); 519 476 520 477 spin_lock_irqsave(&chan->vchan.lock, flags); 521 478 if (vchan_issue_pending(&chan->vchan)) ··· 527 476 spin_unlock_irqrestore(&chan->vchan.lock, flags); 528 477 } 529 478 530 - static struct axi_dmac_desc *axi_dmac_alloc_desc(unsigned int num_sgs) 479 + static struct axi_dmac_desc * 480 + axi_dmac_alloc_desc(struct axi_dmac_chan *chan, unsigned int num_sgs) 531 481 { 482 + struct axi_dmac *dmac = chan_to_axi_dmac(chan); 483 + struct device *dev = dmac->dma_dev.dev; 484 + struct axi_dmac_hw_desc *hws; 532 485 struct axi_dmac_desc *desc; 486 + dma_addr_t hw_phys; 533 487 unsigned int i; 534 488 535 489 desc = kzalloc(struct_size(desc, sg, num_sgs), GFP_NOWAIT); 536 490 if (!desc) 537 491 return NULL; 538 492 desc->num_sgs = num_sgs; 493 + desc->chan = chan; 539 494 540 - for (i = 0; i < num_sgs; i++) 541 - desc->sg[i].id = AXI_DMAC_SG_UNUSED; 495 + hws = dma_alloc_coherent(dev, PAGE_ALIGN(num_sgs * sizeof(*hws)), 496 + &hw_phys, GFP_ATOMIC); 497 + if (!hws) { 498 + kfree(desc); 499 + return NULL; 500 + } 501 + 502 + for (i = 0; i < num_sgs; i++) { 503 + desc->sg[i].hw = &hws[i]; 504 + desc->sg[i].hw_phys = hw_phys + i * sizeof(*hws); 505 + 506 + hws[i].id = AXI_DMAC_SG_UNUSED; 507 + hws[i].flags = 0; 508 + 509 + /* Link hardware descriptors */ 510 + hws[i].next_sg_addr = hw_phys + (i + 1) * sizeof(*hws); 511 + } 512 + 513 + /* The last hardware descriptor will trigger an interrupt */ 514 + desc->sg[num_sgs - 1].hw->flags = AXI_DMAC_HW_FLAG_LAST | AXI_DMAC_HW_FLAG_IRQ; 542 515 543 516 return desc; 517 + } 518 + 519 + static void axi_dmac_free_desc(struct axi_dmac_desc *desc) 520 + { 521 + struct axi_dmac *dmac = chan_to_axi_dmac(desc->chan); 522 + struct device *dev = dmac->dma_dev.dev; 523 + struct axi_dmac_hw_desc *hw = desc->sg[0].hw; 524 + dma_addr_t hw_phys = desc->sg[0].hw_phys; 525 + 526 + dma_free_coherent(dev, PAGE_ALIGN(desc->num_sgs * sizeof(*hw)), 527 + hw, hw_phys); 528 + kfree(desc); 544 529 } 545 530 546 531 static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan, ··· 595 508 segment_size = ((segment_size - 1) | chan->length_align_mask) + 1; 596 509 597 510 for (i = 0; i < num_periods; i++) { 598 - len = period_len; 599 - 600 - while (len > segment_size) { 511 + for (len = period_len; len > segment_size; sg++) { 601 512 if (direction == DMA_DEV_TO_MEM) 602 - sg->dest_addr = addr; 513 + sg->hw->dest_addr = addr; 603 514 else 604 - sg->src_addr = addr; 605 - sg->x_len = segment_size; 606 - sg->y_len = 1; 607 - sg++; 515 + sg->hw->src_addr = addr; 516 + sg->hw->x_len = segment_size - 1; 517 + sg->hw->y_len = 0; 518 + sg->hw->flags = 0; 608 519 addr += segment_size; 609 520 len -= segment_size; 610 521 } 611 522 612 523 if (direction == DMA_DEV_TO_MEM) 613 - sg->dest_addr = addr; 524 + sg->hw->dest_addr = addr; 614 525 else 615 - sg->src_addr = addr; 616 - sg->x_len = len; 617 - sg->y_len = 1; 526 + sg->hw->src_addr = addr; 527 + sg->hw->x_len = len - 1; 528 + sg->hw->y_len = 0; 618 529 sg++; 619 530 addr += len; 620 531 } ··· 639 554 for_each_sg(sgl, sg, sg_len, i) 640 555 num_sgs += DIV_ROUND_UP(sg_dma_len(sg), chan->max_length); 641 556 642 - desc = axi_dmac_alloc_desc(num_sgs); 557 + desc = axi_dmac_alloc_desc(chan, num_sgs); 643 558 if (!desc) 644 559 return NULL; 645 560 ··· 648 563 for_each_sg(sgl, sg, sg_len, i) { 649 564 if (!axi_dmac_check_addr(chan, sg_dma_address(sg)) || 650 565 !axi_dmac_check_len(chan, sg_dma_len(sg))) { 651 - kfree(desc); 566 + axi_dmac_free_desc(desc); 652 567 return NULL; 653 568 } 654 569 ··· 668 583 { 669 584 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 670 585 struct axi_dmac_desc *desc; 671 - unsigned int num_periods, num_segments; 586 + unsigned int num_periods, num_segments, num_sgs; 672 587 673 588 if (direction != chan->direction) 674 589 return NULL; ··· 682 597 683 598 num_periods = buf_len / period_len; 684 599 num_segments = DIV_ROUND_UP(period_len, chan->max_length); 600 + num_sgs = num_periods * num_segments; 685 601 686 - desc = axi_dmac_alloc_desc(num_periods * num_segments); 602 + desc = axi_dmac_alloc_desc(chan, num_sgs); 687 603 if (!desc) 688 604 return NULL; 605 + 606 + /* Chain the last descriptor to the first, and remove its "last" flag */ 607 + desc->sg[num_sgs - 1].hw->next_sg_addr = desc->sg[0].hw_phys; 608 + desc->sg[num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_LAST; 689 609 690 610 axi_dmac_fill_linear_sg(chan, direction, buf_addr, num_periods, 691 611 period_len, desc->sg); ··· 743 653 return NULL; 744 654 } 745 655 746 - desc = axi_dmac_alloc_desc(1); 656 + desc = axi_dmac_alloc_desc(chan, 1); 747 657 if (!desc) 748 658 return NULL; 749 659 750 660 if (axi_dmac_src_is_mem(chan)) { 751 - desc->sg[0].src_addr = xt->src_start; 752 - desc->sg[0].src_stride = xt->sgl[0].size + src_icg; 661 + desc->sg[0].hw->src_addr = xt->src_start; 662 + desc->sg[0].hw->src_stride = xt->sgl[0].size + src_icg; 753 663 } 754 664 755 665 if (axi_dmac_dest_is_mem(chan)) { 756 - desc->sg[0].dest_addr = xt->dst_start; 757 - desc->sg[0].dest_stride = xt->sgl[0].size + dst_icg; 666 + desc->sg[0].hw->dest_addr = xt->dst_start; 667 + desc->sg[0].hw->dst_stride = xt->sgl[0].size + dst_icg; 758 668 } 759 669 760 670 if (chan->hw_2d) { 761 - desc->sg[0].x_len = xt->sgl[0].size; 762 - desc->sg[0].y_len = xt->numf; 671 + desc->sg[0].hw->x_len = xt->sgl[0].size - 1; 672 + desc->sg[0].hw->y_len = xt->numf - 1; 763 673 } else { 764 - desc->sg[0].x_len = xt->sgl[0].size * xt->numf; 765 - desc->sg[0].y_len = 1; 674 + desc->sg[0].hw->x_len = xt->sgl[0].size * xt->numf - 1; 675 + desc->sg[0].hw->y_len = 0; 766 676 } 767 677 768 678 if (flags & DMA_CYCLIC) ··· 778 688 779 689 static void axi_dmac_desc_free(struct virt_dma_desc *vdesc) 780 690 { 781 - kfree(container_of(vdesc, struct axi_dmac_desc, vdesc)); 691 + axi_dmac_free_desc(to_axi_dmac_desc(vdesc)); 782 692 } 783 693 784 694 static bool axi_dmac_regmap_rdwr(struct device *dev, unsigned int reg) ··· 804 714 case AXI_DMAC_REG_CURRENT_DEST_ADDR: 805 715 case AXI_DMAC_REG_PARTIAL_XFER_LEN: 806 716 case AXI_DMAC_REG_PARTIAL_XFER_ID: 717 + case AXI_DMAC_REG_CURRENT_SG_ID: 718 + case AXI_DMAC_REG_SG_ADDRESS: 719 + case AXI_DMAC_REG_SG_ADDRESS_HIGH: 807 720 return true; 808 721 default: 809 722 return false; ··· 959 866 if (axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS) == AXI_DMAC_FLAG_CYCLIC) 960 867 chan->hw_cyclic = true; 961 868 869 + axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS, 0xffffffff); 870 + if (axi_dmac_read(dmac, AXI_DMAC_REG_SG_ADDRESS)) 871 + chan->hw_sg = true; 872 + 962 873 axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, 1); 963 874 if (axi_dmac_read(dmac, AXI_DMAC_REG_Y_LENGTH) == 1) 964 875 chan->hw_2d = true; ··· 1008 911 struct axi_dmac *dmac; 1009 912 struct regmap *regmap; 1010 913 unsigned int version; 914 + u32 irq_mask = 0; 1011 915 int ret; 1012 916 1013 917 dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); ··· 1064 966 dma_dev->dst_addr_widths = BIT(dmac->chan.dest_width); 1065 967 dma_dev->directions = BIT(dmac->chan.direction); 1066 968 dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 969 + dma_dev->max_sg_burst = 31; /* 31 SGs maximum in one burst */ 1067 970 INIT_LIST_HEAD(&dma_dev->channels); 1068 971 1069 972 dmac->chan.vchan.desc_free = axi_dmac_desc_free; ··· 1076 977 1077 978 dma_dev->copy_align = (dmac->chan.address_align_mask + 1); 1078 979 1079 - axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_MASK, 0x00); 980 + if (dmac->chan.hw_sg) 981 + irq_mask |= AXI_DMAC_IRQ_SOT; 982 + 983 + axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_MASK, irq_mask); 1080 984 1081 985 if (of_dma_is_coherent(pdev->dev.of_node)) { 1082 986 ret = axi_dmac_read(dmac, AXI_DMAC_REG_COHERENCY_DESC);

+3

drivers/dma/dmaengine.c

··· 1103 1103 static void __dma_async_device_channel_unregister(struct dma_device *device, 1104 1104 struct dma_chan *chan) 1105 1105 { 1106 + if (chan->local == NULL) 1107 + return; 1108 + 1106 1109 WARN_ONCE(!device->device_release && chan->client_count, 1107 1110 "%s called while %d clients hold a reference\n", 1108 1111 __func__, chan->client_count);

+16 -1

drivers/dma/dmatest.c

··· 21 21 #include <linux/slab.h> 22 22 #include <linux/wait.h> 23 23 24 + static bool nobounce; 25 + module_param(nobounce, bool, 0644); 26 + MODULE_PARM_DESC(nobounce, "Prevent using swiotlb buffer (default: use swiotlb buffer)"); 27 + 24 28 static unsigned int test_buf_size = 16384; 25 29 module_param(test_buf_size, uint, 0644); 26 30 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer"); ··· 94 90 95 91 /** 96 92 * struct dmatest_params - test parameters. 93 + * @nobounce: prevent using swiotlb buffer 97 94 * @buf_size: size of the memcpy test buffer 98 95 * @channel: bus ID of the channel to test 99 96 * @device: bus ID of the DMA Engine to test ··· 111 106 * @polled: use polling for completion instead of interrupts 112 107 */ 113 108 struct dmatest_params { 109 + bool nobounce; 114 110 unsigned int buf_size; 115 111 char channel[20]; 116 112 char device[32]; ··· 221 215 struct dmatest_data { 222 216 u8 **raw; 223 217 u8 **aligned; 218 + gfp_t gfp_flags; 224 219 unsigned int cnt; 225 220 unsigned int off; 226 221 }; ··· 540 533 goto err; 541 534 542 535 for (i = 0; i < d->cnt; i++) { 543 - d->raw[i] = kmalloc(buf_size + align, GFP_KERNEL); 536 + d->raw[i] = kmalloc(buf_size + align, d->gfp_flags); 544 537 if (!d->raw[i]) 545 538 goto err; 546 539 ··· 660 653 pr_err("%u-byte buffer too small for %d-byte alignment\n", 661 654 buf_size, 1 << align); 662 655 goto err_free_coefs; 656 + } 657 + 658 + src->gfp_flags = GFP_KERNEL; 659 + dst->gfp_flags = GFP_KERNEL; 660 + if (params->nobounce) { 661 + src->gfp_flags = GFP_DMA; 662 + dst->gfp_flags = GFP_DMA; 663 663 } 664 664 665 665 if (dmatest_alloc_test_data(src, buf_size, align) < 0) ··· 1107 1093 struct dmatest_params *params = &info->params; 1108 1094 1109 1095 /* Copy test parameters */ 1096 + params->nobounce = nobounce; 1110 1097 params->buf_size = test_buf_size; 1111 1098 strscpy(params->channel, strim(test_channel), sizeof(params->channel)); 1112 1099 strscpy(params->device, strim(test_device), sizeof(params->device));

+2 -2

drivers/dma/dw-edma/dw-edma-v0-debugfs.c

··· 176 176 }; 177 177 struct dentry *regs_dent, *ch_dent; 178 178 int nr_entries, i; 179 - char name[16]; 179 + char name[32]; 180 180 181 181 regs_dent = debugfs_create_dir(WRITE_STR, dent); 182 182 ··· 239 239 }; 240 240 struct dentry *regs_dent, *ch_dent; 241 241 int nr_entries, i; 242 - char name[16]; 242 + char name[32]; 243 243 244 244 regs_dent = debugfs_create_dir(READ_STR, dent); 245 245

+2 -2

drivers/dma/dw-edma/dw-hdma-v0-debugfs.c

··· 116 116 static void dw_hdma_debugfs_regs_wr(struct dw_edma *dw, struct dentry *dent) 117 117 { 118 118 struct dentry *regs_dent, *ch_dent; 119 - char name[16]; 119 + char name[32]; 120 120 int i; 121 121 122 122 regs_dent = debugfs_create_dir(WRITE_STR, dent); ··· 133 133 static void dw_hdma_debugfs_regs_rd(struct dw_edma *dw, struct dentry *dent) 134 134 { 135 135 struct dentry *regs_dent, *ch_dent; 136 - char name[16]; 136 + char name[32]; 137 137 int i; 138 138 139 139 regs_dent = debugfs_create_dir(READ_STR, dent);

+10 -7

drivers/dma/fsl-edma-main.c

··· 9 9 * Vybrid and Layerscape SoCs. 10 10 */ 11 11 12 + #include <dt-bindings/dma/fsl-edma.h> 12 13 #include <linux/module.h> 13 14 #include <linux/interrupt.h> 14 15 #include <linux/clk.h> ··· 21 20 #include <linux/property.h> 22 21 23 22 #include "fsl-edma-common.h" 24 - 25 - #define ARGS_RX BIT(0) 26 - #define ARGS_REMOTE BIT(1) 27 - #define ARGS_MULTI_FIFO BIT(2) 28 23 29 24 static void fsl_edma_synchronize(struct dma_chan *chan) 30 25 { ··· 150 153 i = fsl_chan - fsl_edma->chans; 151 154 152 155 fsl_chan->priority = dma_spec->args[1]; 153 - fsl_chan->is_rxchan = dma_spec->args[2] & ARGS_RX; 154 - fsl_chan->is_remote = dma_spec->args[2] & ARGS_REMOTE; 155 - fsl_chan->is_multi_fifo = dma_spec->args[2] & ARGS_MULTI_FIFO; 156 + fsl_chan->is_rxchan = dma_spec->args[2] & FSL_EDMA_RX; 157 + fsl_chan->is_remote = dma_spec->args[2] & FSL_EDMA_REMOTE; 158 + fsl_chan->is_multi_fifo = dma_spec->args[2] & FSL_EDMA_MULTI_FIFO; 159 + 160 + if ((dma_spec->args[2] & FSL_EDMA_EVEN_CH) && (i & 0x1)) 161 + continue; 162 + 163 + if ((dma_spec->args[2] & FSL_EDMA_ODD_CH) && !(i & 0x1)) 164 + continue; 156 165 157 166 if (!b_chmux && i == dma_spec->args[0]) { 158 167 chan = dma_get_slave_channel(chan);

+1 -1

drivers/dma/fsl-qdma.c

··· 805 805 int i; 806 806 int cpu; 807 807 int ret; 808 - char irq_name[20]; 808 + char irq_name[32]; 809 809 810 810 fsl_qdma->error_irq = 811 811 platform_get_irq_byname(pdev, "qdma-error");

+2 -2

drivers/dma/idxd/cdev.c

··· 165 165 struct idxd_wq *wq = idxd_cdev->wq; 166 166 167 167 cdev_ctx = &ictx[wq->idxd->data->type]; 168 - ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor); 168 + ida_free(&cdev_ctx->minor_ida, idxd_cdev->minor); 169 169 kfree(idxd_cdev); 170 170 } 171 171 ··· 463 463 cdev = &idxd_cdev->cdev; 464 464 dev = cdev_dev(idxd_cdev); 465 465 cdev_ctx = &ictx[wq->idxd->data->type]; 466 - minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL); 466 + minor = ida_alloc_max(&cdev_ctx->minor_ida, MINORMASK, GFP_KERNEL); 467 467 if (minor < 0) { 468 468 kfree(idxd_cdev); 469 469 return minor;

+8 -1

drivers/dma/idxd/device.c

··· 807 807 808 808 static void idxd_device_evl_free(struct idxd_device *idxd) 809 809 { 810 + void *evl_log; 811 + unsigned int evl_log_size; 812 + dma_addr_t evl_dma; 810 813 union gencfg_reg gencfg; 811 814 union genctrl_reg genctrl; 812 815 struct device *dev = &idxd->pdev->dev; ··· 830 827 iowrite64(0, idxd->reg_base + IDXD_EVLCFG_OFFSET); 831 828 iowrite64(0, idxd->reg_base + IDXD_EVLCFG_OFFSET + 8); 832 829 833 - dma_free_coherent(dev, evl->log_size, evl->log, evl->dma); 834 830 bitmap_free(evl->bmap); 831 + evl_log = evl->log; 832 + evl_log_size = evl->log_size; 833 + evl_dma = evl->dma; 835 834 evl->log = NULL; 836 835 evl->size = IDXD_EVL_SIZE_MIN; 837 836 spin_unlock(&evl->lock); 837 + 838 + dma_free_coherent(dev, evl_log_size, evl_log, evl_dma); 838 839 } 839 840 840 841 static void idxd_group_config_write(struct idxd_group *group)

-4

drivers/dma/imx-sdma.c

··· 421 421 * @shp_addr: value for gReg[6] 422 422 * @per_addr: value for gReg[2] 423 423 * @status: status of dma channel 424 - * @context_loaded: ensure context is only loaded once 425 424 * @data: specific sdma interface structure 426 - * @bd_pool: dma_pool for bd 427 425 * @terminate_worker: used to call back into terminate work function 428 426 * @terminated: terminated list 429 427 * @is_ram_script: flag for script in ram ··· 484 486 * @num_script_addrs: Number of script addresses in this image 485 487 * @ram_code_start: offset of SDMA ram image in this firmware image 486 488 * @ram_code_size: size of SDMA ram image 487 - * @script_addrs: Stores the start address of the SDMA scripts 488 - * (in SDMA memory space) 489 489 */ 490 490 struct sdma_firmware_header { 491 491 u32 magic;

+705

drivers/dma/ls2x-apb-dma.c

··· 1 + // SPDX-License-Identifier: GPL-2.0-or-later 2 + /* 3 + * Driver for the Loongson LS2X APB DMA Controller 4 + * 5 + * Copyright (C) 2017-2023 Loongson Corporation 6 + */ 7 + 8 + #include <linux/clk.h> 9 + #include <linux/dma-mapping.h> 10 + #include <linux/dmapool.h> 11 + #include <linux/interrupt.h> 12 + #include <linux/io.h> 13 + #include <linux/io-64-nonatomic-lo-hi.h> 14 + #include <linux/module.h> 15 + #include <linux/of.h> 16 + #include <linux/of_dma.h> 17 + #include <linux/platform_device.h> 18 + #include <linux/slab.h> 19 + 20 + #include "dmaengine.h" 21 + #include "virt-dma.h" 22 + 23 + /* Global Configuration Register */ 24 + #define LDMA_ORDER_ERG 0x0 25 + 26 + /* Bitfield definitions */ 27 + 28 + /* Bitfields in Global Configuration Register */ 29 + #define LDMA_64BIT_EN BIT(0) /* 1: 64 bit support */ 30 + #define LDMA_UNCOHERENT_EN BIT(1) /* 0: cache, 1: uncache */ 31 + #define LDMA_ASK_VALID BIT(2) 32 + #define LDMA_START BIT(3) /* DMA start operation */ 33 + #define LDMA_STOP BIT(4) /* DMA stop operation */ 34 + #define LDMA_CONFIG_MASK GENMASK(4, 0) /* DMA controller config bits mask */ 35 + 36 + /* Bitfields in ndesc_addr field of HW decriptor */ 37 + #define LDMA_DESC_EN BIT(0) /*1: The next descriptor is valid */ 38 + #define LDMA_DESC_ADDR_LOW GENMASK(31, 1) 39 + 40 + /* Bitfields in cmd field of HW decriptor */ 41 + #define LDMA_INT BIT(1) /* Enable DMA interrupts */ 42 + #define LDMA_DATA_DIRECTION BIT(12) /* 1: write to device, 0: read from device */ 43 + 44 + #define LDMA_SLAVE_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ 45 + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) 46 + 47 + #define LDMA_MAX_TRANS_LEN U32_MAX 48 + 49 + /*-- descriptors -----------------------------------------------------*/ 50 + 51 + /* 52 + * struct ls2x_dma_hw_desc - DMA HW descriptor 53 + * @ndesc_addr: the next descriptor low address. 54 + * @mem_addr: memory low address. 55 + * @apb_addr: device buffer address. 56 + * @len: length of a piece of carried content, in words. 57 + * @step_len: length between two moved memory data blocks. 58 + * @step_times: number of blocks to be carried in a single DMA operation. 59 + * @cmd: descriptor command or state. 60 + * @stats: DMA status. 61 + * @high_ndesc_addr: the next descriptor high address. 62 + * @high_mem_addr: memory high address. 63 + * @reserved: reserved 64 + */ 65 + struct ls2x_dma_hw_desc { 66 + u32 ndesc_addr; 67 + u32 mem_addr; 68 + u32 apb_addr; 69 + u32 len; 70 + u32 step_len; 71 + u32 step_times; 72 + u32 cmd; 73 + u32 stats; 74 + u32 high_ndesc_addr; 75 + u32 high_mem_addr; 76 + u32 reserved[2]; 77 + } __packed; 78 + 79 + /* 80 + * struct ls2x_dma_sg - ls2x dma scatter gather entry 81 + * @hw: the pointer to DMA HW descriptor. 82 + * @llp: physical address of the DMA HW descriptor. 83 + * @phys: destination or source address(mem). 84 + * @len: number of Bytes to read. 85 + */ 86 + struct ls2x_dma_sg { 87 + struct ls2x_dma_hw_desc *hw; 88 + dma_addr_t llp; 89 + dma_addr_t phys; 90 + u32 len; 91 + }; 92 + 93 + /* 94 + * struct ls2x_dma_desc - software descriptor 95 + * @vdesc: pointer to the virtual dma descriptor. 96 + * @cyclic: flag to dma cyclic 97 + * @burst_size: burst size of transaction, in words. 98 + * @desc_num: number of sg entries. 99 + * @direction: transfer direction, to or from device. 100 + * @status: dma controller status. 101 + * @sg: array of sgs. 102 + */ 103 + struct ls2x_dma_desc { 104 + struct virt_dma_desc vdesc; 105 + bool cyclic; 106 + size_t burst_size; 107 + u32 desc_num; 108 + enum dma_transfer_direction direction; 109 + enum dma_status status; 110 + struct ls2x_dma_sg sg[] __counted_by(desc_num); 111 + }; 112 + 113 + /*-- Channels --------------------------------------------------------*/ 114 + 115 + /* 116 + * struct ls2x_dma_chan - internal representation of an LS2X APB DMA channel 117 + * @vchan: virtual dma channel entry. 118 + * @desc: pointer to the ls2x sw dma descriptor. 119 + * @pool: hw desc table 120 + * @irq: irq line 121 + * @sconfig: configuration for slave transfers, passed via .device_config 122 + */ 123 + struct ls2x_dma_chan { 124 + struct virt_dma_chan vchan; 125 + struct ls2x_dma_desc *desc; 126 + void *pool; 127 + int irq; 128 + struct dma_slave_config sconfig; 129 + }; 130 + 131 + /*-- Controller ------------------------------------------------------*/ 132 + 133 + /* 134 + * struct ls2x_dma_priv - LS2X APB DMAC specific information 135 + * @ddev: dmaengine dma_device object members 136 + * @dma_clk: DMAC clock source 137 + * @regs: memory mapped register base 138 + * @lchan: channel to store ls2x_dma_chan structures 139 + */ 140 + struct ls2x_dma_priv { 141 + struct dma_device ddev; 142 + struct clk *dma_clk; 143 + void __iomem *regs; 144 + struct ls2x_dma_chan lchan; 145 + }; 146 + 147 + /*-- Helper functions ------------------------------------------------*/ 148 + 149 + static inline struct ls2x_dma_desc *to_ldma_desc(struct virt_dma_desc *vdesc) 150 + { 151 + return container_of(vdesc, struct ls2x_dma_desc, vdesc); 152 + } 153 + 154 + static inline struct ls2x_dma_chan *to_ldma_chan(struct dma_chan *chan) 155 + { 156 + return container_of(chan, struct ls2x_dma_chan, vchan.chan); 157 + } 158 + 159 + static inline struct ls2x_dma_priv *to_ldma_priv(struct dma_device *ddev) 160 + { 161 + return container_of(ddev, struct ls2x_dma_priv, ddev); 162 + } 163 + 164 + static struct device *chan2dev(struct dma_chan *chan) 165 + { 166 + return &chan->dev->device; 167 + } 168 + 169 + static void ls2x_dma_desc_free(struct virt_dma_desc *vdesc) 170 + { 171 + struct ls2x_dma_chan *lchan = to_ldma_chan(vdesc->tx.chan); 172 + struct ls2x_dma_desc *desc = to_ldma_desc(vdesc); 173 + int i; 174 + 175 + for (i = 0; i < desc->desc_num; i++) { 176 + if (desc->sg[i].hw) 177 + dma_pool_free(lchan->pool, desc->sg[i].hw, 178 + desc->sg[i].llp); 179 + } 180 + 181 + kfree(desc); 182 + } 183 + 184 + static void ls2x_dma_write_cmd(struct ls2x_dma_chan *lchan, bool cmd) 185 + { 186 + struct ls2x_dma_priv *priv = to_ldma_priv(lchan->vchan.chan.device); 187 + u64 val; 188 + 189 + val = lo_hi_readq(priv->regs + LDMA_ORDER_ERG) & ~LDMA_CONFIG_MASK; 190 + val |= LDMA_64BIT_EN | cmd; 191 + lo_hi_writeq(val, priv->regs + LDMA_ORDER_ERG); 192 + } 193 + 194 + static void ls2x_dma_start_transfer(struct ls2x_dma_chan *lchan) 195 + { 196 + struct ls2x_dma_priv *priv = to_ldma_priv(lchan->vchan.chan.device); 197 + struct ls2x_dma_sg *ldma_sg; 198 + struct virt_dma_desc *vdesc; 199 + u64 val; 200 + 201 + /* Get the next descriptor */ 202 + vdesc = vchan_next_desc(&lchan->vchan); 203 + if (!vdesc) { 204 + lchan->desc = NULL; 205 + return; 206 + } 207 + 208 + list_del(&vdesc->node); 209 + lchan->desc = to_ldma_desc(vdesc); 210 + ldma_sg = &lchan->desc->sg[0]; 211 + 212 + /* Start DMA */ 213 + lo_hi_writeq(0, priv->regs + LDMA_ORDER_ERG); 214 + val = (ldma_sg->llp & ~LDMA_CONFIG_MASK) | LDMA_64BIT_EN | LDMA_START; 215 + lo_hi_writeq(val, priv->regs + LDMA_ORDER_ERG); 216 + } 217 + 218 + static size_t ls2x_dmac_detect_burst(struct ls2x_dma_chan *lchan) 219 + { 220 + u32 maxburst, buswidth; 221 + 222 + /* Reject definitely invalid configurations */ 223 + if ((lchan->sconfig.src_addr_width & LDMA_SLAVE_BUSWIDTHS) && 224 + (lchan->sconfig.dst_addr_width & LDMA_SLAVE_BUSWIDTHS)) 225 + return 0; 226 + 227 + if (lchan->sconfig.direction == DMA_MEM_TO_DEV) { 228 + maxburst = lchan->sconfig.dst_maxburst; 229 + buswidth = lchan->sconfig.dst_addr_width; 230 + } else { 231 + maxburst = lchan->sconfig.src_maxburst; 232 + buswidth = lchan->sconfig.src_addr_width; 233 + } 234 + 235 + /* If maxburst is zero, fallback to LDMA_MAX_TRANS_LEN */ 236 + return maxburst ? (maxburst * buswidth) >> 2 : LDMA_MAX_TRANS_LEN; 237 + } 238 + 239 + static void ls2x_dma_fill_desc(struct ls2x_dma_chan *lchan, u32 sg_index, 240 + struct ls2x_dma_desc *desc) 241 + { 242 + struct ls2x_dma_sg *ldma_sg = &desc->sg[sg_index]; 243 + u32 num_segments, segment_size; 244 + 245 + if (desc->direction == DMA_MEM_TO_DEV) { 246 + ldma_sg->hw->cmd = LDMA_INT | LDMA_DATA_DIRECTION; 247 + ldma_sg->hw->apb_addr = lchan->sconfig.dst_addr; 248 + } else { 249 + ldma_sg->hw->cmd = LDMA_INT; 250 + ldma_sg->hw->apb_addr = lchan->sconfig.src_addr; 251 + } 252 + 253 + ldma_sg->hw->mem_addr = lower_32_bits(ldma_sg->phys); 254 + ldma_sg->hw->high_mem_addr = upper_32_bits(ldma_sg->phys); 255 + 256 + /* Split into multiple equally sized segments if necessary */ 257 + num_segments = DIV_ROUND_UP((ldma_sg->len + 3) >> 2, desc->burst_size); 258 + segment_size = DIV_ROUND_UP((ldma_sg->len + 3) >> 2, num_segments); 259 + 260 + /* Word count register takes input in words */ 261 + ldma_sg->hw->len = segment_size; 262 + ldma_sg->hw->step_times = num_segments; 263 + ldma_sg->hw->step_len = 0; 264 + 265 + /* lets make a link list */ 266 + if (sg_index) { 267 + desc->sg[sg_index - 1].hw->ndesc_addr = ldma_sg->llp | LDMA_DESC_EN; 268 + desc->sg[sg_index - 1].hw->high_ndesc_addr = upper_32_bits(ldma_sg->llp); 269 + } 270 + } 271 + 272 + /*-- DMA Engine API --------------------------------------------------*/ 273 + 274 + /* 275 + * ls2x_dma_alloc_chan_resources - allocate resources for DMA channel 276 + * @chan: allocate descriptor resources for this channel 277 + * 278 + * return - the number of allocated descriptors 279 + */ 280 + static int ls2x_dma_alloc_chan_resources(struct dma_chan *chan) 281 + { 282 + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); 283 + 284 + /* Create a pool of consistent memory blocks for hardware descriptors */ 285 + lchan->pool = dma_pool_create(dev_name(chan2dev(chan)), 286 + chan->device->dev, PAGE_SIZE, 287 + __alignof__(struct ls2x_dma_hw_desc), 0); 288 + if (!lchan->pool) { 289 + dev_err(chan2dev(chan), "No memory for descriptors\n"); 290 + return -ENOMEM; 291 + } 292 + 293 + return 1; 294 + } 295 + 296 + /* 297 + * ls2x_dma_free_chan_resources - free all channel resources 298 + * @chan: DMA channel 299 + */ 300 + static void ls2x_dma_free_chan_resources(struct dma_chan *chan) 301 + { 302 + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); 303 + 304 + vchan_free_chan_resources(to_virt_chan(chan)); 305 + dma_pool_destroy(lchan->pool); 306 + lchan->pool = NULL; 307 + } 308 + 309 + /* 310 + * ls2x_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction 311 + * @chan: DMA channel 312 + * @sgl: scatterlist to transfer to/from 313 + * @sg_len: number of entries in @scatterlist 314 + * @direction: DMA direction 315 + * @flags: tx descriptor status flags 316 + * @context: transaction context (ignored) 317 + * 318 + * Return: Async transaction descriptor on success and NULL on failure 319 + */ 320 + static struct dma_async_tx_descriptor * 321 + ls2x_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 322 + u32 sg_len, enum dma_transfer_direction direction, 323 + unsigned long flags, void *context) 324 + { 325 + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); 326 + struct ls2x_dma_desc *desc; 327 + struct scatterlist *sg; 328 + size_t burst_size; 329 + int i; 330 + 331 + if (unlikely(!sg_len || !is_slave_direction(direction))) 332 + return NULL; 333 + 334 + burst_size = ls2x_dmac_detect_burst(lchan); 335 + if (!burst_size) 336 + return NULL; 337 + 338 + desc = kzalloc(struct_size(desc, sg, sg_len), GFP_NOWAIT); 339 + if (!desc) 340 + return NULL; 341 + 342 + desc->desc_num = sg_len; 343 + desc->direction = direction; 344 + desc->burst_size = burst_size; 345 + 346 + for_each_sg(sgl, sg, sg_len, i) { 347 + struct ls2x_dma_sg *ldma_sg = &desc->sg[i]; 348 + 349 + /* Allocate DMA capable memory for hardware descriptor */ 350 + ldma_sg->hw = dma_pool_alloc(lchan->pool, GFP_NOWAIT, &ldma_sg->llp); 351 + if (!ldma_sg->hw) { 352 + desc->desc_num = i; 353 + ls2x_dma_desc_free(&desc->vdesc); 354 + return NULL; 355 + } 356 + 357 + ldma_sg->phys = sg_dma_address(sg); 358 + ldma_sg->len = sg_dma_len(sg); 359 + 360 + ls2x_dma_fill_desc(lchan, i, desc); 361 + } 362 + 363 + /* Setting the last descriptor enable bit */ 364 + desc->sg[sg_len - 1].hw->ndesc_addr &= ~LDMA_DESC_EN; 365 + desc->status = DMA_IN_PROGRESS; 366 + 367 + return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags); 368 + } 369 + 370 + /* 371 + * ls2x_dma_prep_dma_cyclic - prepare the cyclic DMA transfer 372 + * @chan: the DMA channel to prepare 373 + * @buf_addr: physical DMA address where the buffer starts 374 + * @buf_len: total number of bytes for the entire buffer 375 + * @period_len: number of bytes for each period 376 + * @direction: transfer direction, to or from device 377 + * @flags: tx descriptor status flags 378 + * 379 + * Return: Async transaction descriptor on success and NULL on failure 380 + */ 381 + static struct dma_async_tx_descriptor * 382 + ls2x_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 383 + size_t period_len, enum dma_transfer_direction direction, 384 + unsigned long flags) 385 + { 386 + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); 387 + struct ls2x_dma_desc *desc; 388 + size_t burst_size; 389 + u32 num_periods; 390 + int i; 391 + 392 + if (unlikely(!buf_len || !period_len)) 393 + return NULL; 394 + 395 + if (unlikely(!is_slave_direction(direction))) 396 + return NULL; 397 + 398 + burst_size = ls2x_dmac_detect_burst(lchan); 399 + if (!burst_size) 400 + return NULL; 401 + 402 + num_periods = buf_len / period_len; 403 + desc = kzalloc(struct_size(desc, sg, num_periods), GFP_NOWAIT); 404 + if (!desc) 405 + return NULL; 406 + 407 + desc->desc_num = num_periods; 408 + desc->direction = direction; 409 + desc->burst_size = burst_size; 410 + 411 + /* Build cyclic linked list */ 412 + for (i = 0; i < num_periods; i++) { 413 + struct ls2x_dma_sg *ldma_sg = &desc->sg[i]; 414 + 415 + /* Allocate DMA capable memory for hardware descriptor */ 416 + ldma_sg->hw = dma_pool_alloc(lchan->pool, GFP_NOWAIT, &ldma_sg->llp); 417 + if (!ldma_sg->hw) { 418 + desc->desc_num = i; 419 + ls2x_dma_desc_free(&desc->vdesc); 420 + return NULL; 421 + } 422 + 423 + ldma_sg->phys = buf_addr + period_len * i; 424 + ldma_sg->len = period_len; 425 + 426 + ls2x_dma_fill_desc(lchan, i, desc); 427 + } 428 + 429 + /* Lets make a cyclic list */ 430 + desc->sg[num_periods - 1].hw->ndesc_addr = desc->sg[0].llp | LDMA_DESC_EN; 431 + desc->sg[num_periods - 1].hw->high_ndesc_addr = upper_32_bits(desc->sg[0].llp); 432 + desc->cyclic = true; 433 + desc->status = DMA_IN_PROGRESS; 434 + 435 + return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags); 436 + } 437 + 438 + /* 439 + * ls2x_slave_config - set slave configuration for channel 440 + * @chan: dma channel 441 + * @cfg: slave configuration 442 + * 443 + * Sets slave configuration for channel 444 + */ 445 + static int ls2x_dma_slave_config(struct dma_chan *chan, 446 + struct dma_slave_config *config) 447 + { 448 + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); 449 + 450 + memcpy(&lchan->sconfig, config, sizeof(*config)); 451 + return 0; 452 + } 453 + 454 + /* 455 + * ls2x_dma_issue_pending - push pending transactions to the hardware 456 + * @chan: channel 457 + * 458 + * When this function is called, all pending transactions are pushed to the 459 + * hardware and executed. 460 + */ 461 + static void ls2x_dma_issue_pending(struct dma_chan *chan) 462 + { 463 + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); 464 + unsigned long flags; 465 + 466 + spin_lock_irqsave(&lchan->vchan.lock, flags); 467 + if (vchan_issue_pending(&lchan->vchan) && !lchan->desc) 468 + ls2x_dma_start_transfer(lchan); 469 + spin_unlock_irqrestore(&lchan->vchan.lock, flags); 470 + } 471 + 472 + /* 473 + * ls2x_dma_terminate_all - terminate all transactions 474 + * @chan: channel 475 + * 476 + * Stops all DMA transactions. 477 + */ 478 + static int ls2x_dma_terminate_all(struct dma_chan *chan) 479 + { 480 + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); 481 + unsigned long flags; 482 + LIST_HEAD(head); 483 + 484 + spin_lock_irqsave(&lchan->vchan.lock, flags); 485 + /* Setting stop cmd */ 486 + ls2x_dma_write_cmd(lchan, LDMA_STOP); 487 + if (lchan->desc) { 488 + vchan_terminate_vdesc(&lchan->desc->vdesc); 489 + lchan->desc = NULL; 490 + } 491 + 492 + vchan_get_all_descriptors(&lchan->vchan, &head); 493 + spin_unlock_irqrestore(&lchan->vchan.lock, flags); 494 + 495 + vchan_dma_desc_free_list(&lchan->vchan, &head); 496 + return 0; 497 + } 498 + 499 + /* 500 + * ls2x_dma_synchronize - Synchronizes the termination of transfers to the 501 + * current context. 502 + * @chan: channel 503 + */ 504 + static void ls2x_dma_synchronize(struct dma_chan *chan) 505 + { 506 + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); 507 + 508 + vchan_synchronize(&lchan->vchan); 509 + } 510 + 511 + static int ls2x_dma_pause(struct dma_chan *chan) 512 + { 513 + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); 514 + unsigned long flags; 515 + 516 + spin_lock_irqsave(&lchan->vchan.lock, flags); 517 + if (lchan->desc && lchan->desc->status == DMA_IN_PROGRESS) { 518 + ls2x_dma_write_cmd(lchan, LDMA_STOP); 519 + lchan->desc->status = DMA_PAUSED; 520 + } 521 + spin_unlock_irqrestore(&lchan->vchan.lock, flags); 522 + 523 + return 0; 524 + } 525 + 526 + static int ls2x_dma_resume(struct dma_chan *chan) 527 + { 528 + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); 529 + unsigned long flags; 530 + 531 + spin_lock_irqsave(&lchan->vchan.lock, flags); 532 + if (lchan->desc && lchan->desc->status == DMA_PAUSED) { 533 + lchan->desc->status = DMA_IN_PROGRESS; 534 + ls2x_dma_write_cmd(lchan, LDMA_START); 535 + } 536 + spin_unlock_irqrestore(&lchan->vchan.lock, flags); 537 + 538 + return 0; 539 + } 540 + 541 + /* 542 + * ls2x_dma_isr - LS2X DMA Interrupt handler 543 + * @irq: IRQ number 544 + * @dev_id: Pointer to ls2x_dma_chan 545 + * 546 + * Return: IRQ_HANDLED/IRQ_NONE 547 + */ 548 + static irqreturn_t ls2x_dma_isr(int irq, void *dev_id) 549 + { 550 + struct ls2x_dma_chan *lchan = dev_id; 551 + struct ls2x_dma_desc *desc; 552 + 553 + spin_lock(&lchan->vchan.lock); 554 + desc = lchan->desc; 555 + if (desc) { 556 + if (desc->cyclic) { 557 + vchan_cyclic_callback(&desc->vdesc); 558 + } else { 559 + desc->status = DMA_COMPLETE; 560 + vchan_cookie_complete(&desc->vdesc); 561 + ls2x_dma_start_transfer(lchan); 562 + } 563 + 564 + /* ls2x_dma_start_transfer() updates lchan->desc */ 565 + if (!lchan->desc) 566 + ls2x_dma_write_cmd(lchan, LDMA_STOP); 567 + } 568 + spin_unlock(&lchan->vchan.lock); 569 + 570 + return IRQ_HANDLED; 571 + } 572 + 573 + static int ls2x_dma_chan_init(struct platform_device *pdev, 574 + struct ls2x_dma_priv *priv) 575 + { 576 + struct ls2x_dma_chan *lchan = &priv->lchan; 577 + struct device *dev = &pdev->dev; 578 + int ret; 579 + 580 + lchan->irq = platform_get_irq(pdev, 0); 581 + if (lchan->irq < 0) 582 + return lchan->irq; 583 + 584 + ret = devm_request_irq(dev, lchan->irq, ls2x_dma_isr, IRQF_TRIGGER_RISING, 585 + dev_name(&pdev->dev), lchan); 586 + if (ret) 587 + return ret; 588 + 589 + /* Initialize channels related values */ 590 + INIT_LIST_HEAD(&priv->ddev.channels); 591 + lchan->vchan.desc_free = ls2x_dma_desc_free; 592 + vchan_init(&lchan->vchan, &priv->ddev); 593 + 594 + return 0; 595 + } 596 + 597 + /* 598 + * ls2x_dma_probe - Driver probe function 599 + * @pdev: Pointer to the platform_device structure 600 + * 601 + * Return: '0' on success and failure value on error 602 + */ 603 + static int ls2x_dma_probe(struct platform_device *pdev) 604 + { 605 + struct device *dev = &pdev->dev; 606 + struct ls2x_dma_priv *priv; 607 + struct dma_device *ddev; 608 + int ret; 609 + 610 + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); 611 + if (!priv) 612 + return -ENOMEM; 613 + 614 + priv->regs = devm_platform_ioremap_resource(pdev, 0); 615 + if (IS_ERR(priv->regs)) 616 + return dev_err_probe(dev, PTR_ERR(priv->regs), 617 + "devm_platform_ioremap_resource failed.\n"); 618 + 619 + priv->dma_clk = devm_clk_get(&pdev->dev, NULL); 620 + if (IS_ERR(priv->dma_clk)) 621 + return dev_err_probe(dev, PTR_ERR(priv->dma_clk), "devm_clk_get failed.\n"); 622 + 623 + ret = clk_prepare_enable(priv->dma_clk); 624 + if (ret) 625 + return dev_err_probe(dev, ret, "clk_prepare_enable failed.\n"); 626 + 627 + ret = ls2x_dma_chan_init(pdev, priv); 628 + if (ret) 629 + goto disable_clk; 630 + 631 + ddev = &priv->ddev; 632 + ddev->dev = dev; 633 + dma_cap_zero(ddev->cap_mask); 634 + dma_cap_set(DMA_SLAVE, ddev->cap_mask); 635 + dma_cap_set(DMA_CYCLIC, ddev->cap_mask); 636 + 637 + ddev->device_alloc_chan_resources = ls2x_dma_alloc_chan_resources; 638 + ddev->device_free_chan_resources = ls2x_dma_free_chan_resources; 639 + ddev->device_tx_status = dma_cookie_status; 640 + ddev->device_issue_pending = ls2x_dma_issue_pending; 641 + ddev->device_prep_slave_sg = ls2x_dma_prep_slave_sg; 642 + ddev->device_prep_dma_cyclic = ls2x_dma_prep_dma_cyclic; 643 + ddev->device_config = ls2x_dma_slave_config; 644 + ddev->device_terminate_all = ls2x_dma_terminate_all; 645 + ddev->device_synchronize = ls2x_dma_synchronize; 646 + ddev->device_pause = ls2x_dma_pause; 647 + ddev->device_resume = ls2x_dma_resume; 648 + 649 + ddev->src_addr_widths = LDMA_SLAVE_BUSWIDTHS; 650 + ddev->dst_addr_widths = LDMA_SLAVE_BUSWIDTHS; 651 + ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 652 + 653 + ret = dma_async_device_register(&priv->ddev); 654 + if (ret < 0) 655 + goto disable_clk; 656 + 657 + ret = of_dma_controller_register(dev->of_node, of_dma_xlate_by_chan_id, priv); 658 + if (ret < 0) 659 + goto unregister_dmac; 660 + 661 + platform_set_drvdata(pdev, priv); 662 + 663 + dev_info(dev, "Loongson LS2X APB DMA driver registered successfully.\n"); 664 + return 0; 665 + 666 + unregister_dmac: 667 + dma_async_device_unregister(&priv->ddev); 668 + disable_clk: 669 + clk_disable_unprepare(priv->dma_clk); 670 + 671 + return ret; 672 + } 673 + 674 + /* 675 + * ls2x_dma_remove - Driver remove function 676 + * @pdev: Pointer to the platform_device structure 677 + */ 678 + static void ls2x_dma_remove(struct platform_device *pdev) 679 + { 680 + struct ls2x_dma_priv *priv = platform_get_drvdata(pdev); 681 + 682 + of_dma_controller_free(pdev->dev.of_node); 683 + dma_async_device_unregister(&priv->ddev); 684 + clk_disable_unprepare(priv->dma_clk); 685 + } 686 + 687 + static const struct of_device_id ls2x_dma_of_match_table[] = { 688 + { .compatible = "loongson,ls2k1000-apbdma" }, 689 + { /* sentinel */ } 690 + }; 691 + MODULE_DEVICE_TABLE(of, ls2x_dma_of_match_table); 692 + 693 + static struct platform_driver ls2x_dmac_driver = { 694 + .probe = ls2x_dma_probe, 695 + .remove_new = ls2x_dma_remove, 696 + .driver = { 697 + .name = "ls2x-apbdma", 698 + .of_match_table = ls2x_dma_of_match_table, 699 + }, 700 + }; 701 + module_platform_driver(ls2x_dmac_driver); 702 + 703 + MODULE_DESCRIPTION("Loongson LS2X APB DMA Controller driver"); 704 + MODULE_AUTHOR("Loongson Technology Corporation Limited"); 705 + MODULE_LICENSE("GPL");

+11 -6

drivers/dma/milbeaut-hdmac.c

··· 531 531 return ret; 532 532 } 533 533 534 - static int milbeaut_hdmac_remove(struct platform_device *pdev) 534 + static void milbeaut_hdmac_remove(struct platform_device *pdev) 535 535 { 536 536 struct milbeaut_hdmac_device *mdev = platform_get_drvdata(pdev); 537 537 struct dma_chan *chan; ··· 546 546 */ 547 547 list_for_each_entry(chan, &mdev->ddev.channels, device_node) { 548 548 ret = dmaengine_terminate_sync(chan); 549 - if (ret) 550 - return ret; 549 + if (ret) { 550 + /* 551 + * This results in resource leakage and maybe also 552 + * use-after-free errors as e.g. *mdev is kfreed. 553 + */ 554 + dev_alert(&pdev->dev, "Failed to terminate channel %d (%pe)\n", 555 + chan->chan_id, ERR_PTR(ret)); 556 + return; 557 + } 551 558 milbeaut_hdmac_free_chan_resources(chan); 552 559 } 553 560 554 561 of_dma_controller_free(pdev->dev.of_node); 555 562 dma_async_device_unregister(&mdev->ddev); 556 563 clk_disable_unprepare(mdev->clk); 557 - 558 - return 0; 559 564 } 560 565 561 566 static const struct of_device_id milbeaut_hdmac_match[] = { ··· 571 566 572 567 static struct platform_driver milbeaut_hdmac_driver = { 573 568 .probe = milbeaut_hdmac_probe, 574 - .remove = milbeaut_hdmac_remove, 569 + .remove_new = milbeaut_hdmac_remove, 575 570 .driver = { 576 571 .name = "milbeaut-m10v-hdmac", 577 572 .of_match_table = milbeaut_hdmac_match,

+11 -6

drivers/dma/milbeaut-xdmac.c

··· 368 368 return ret; 369 369 } 370 370 371 - static int milbeaut_xdmac_remove(struct platform_device *pdev) 371 + static void milbeaut_xdmac_remove(struct platform_device *pdev) 372 372 { 373 373 struct milbeaut_xdmac_device *mdev = platform_get_drvdata(pdev); 374 374 struct dma_chan *chan; ··· 383 383 */ 384 384 list_for_each_entry(chan, &mdev->ddev.channels, device_node) { 385 385 ret = dmaengine_terminate_sync(chan); 386 - if (ret) 387 - return ret; 386 + if (ret) { 387 + /* 388 + * This results in resource leakage and maybe also 389 + * use-after-free errors as e.g. *mdev is kfreed. 390 + */ 391 + dev_alert(&pdev->dev, "Failed to terminate channel %d (%pe)\n", 392 + chan->chan_id, ERR_PTR(ret)); 393 + return; 394 + } 388 395 milbeaut_xdmac_free_chan_resources(chan); 389 396 } 390 397 ··· 399 392 dma_async_device_unregister(&mdev->ddev); 400 393 401 394 disable_xdmac(mdev); 402 - 403 - return 0; 404 395 } 405 396 406 397 static const struct of_device_id milbeaut_xdmac_match[] = { ··· 409 404 410 405 static struct platform_driver milbeaut_xdmac_driver = { 411 406 .probe = milbeaut_xdmac_probe, 412 - .remove = milbeaut_xdmac_remove, 407 + .remove_new = milbeaut_xdmac_remove, 413 408 .driver = { 414 409 .name = "milbeaut-m10v-xdmac", 415 410 .of_match_table = milbeaut_xdmac_match,

+3

drivers/dma/pl330.c

··· 1053 1053 1054 1054 thrd->req_running = idx; 1055 1055 1056 + if (desc->rqtype == DMA_MEM_TO_DEV || desc->rqtype == DMA_DEV_TO_MEM) 1057 + UNTIL(thrd, PL330_STATE_WFP); 1058 + 1056 1059 return true; 1057 1060 } 1058 1061

+41 -3

drivers/dma/sf-pdma/sf-pdma.c

··· 20 20 #include <linux/mod_devicetable.h> 21 21 #include <linux/dma-mapping.h> 22 22 #include <linux/of.h> 23 + #include <linux/of_dma.h> 23 24 #include <linux/slab.h> 24 25 25 26 #include "sf-pdma.h" 27 + 28 + #define PDMA_QUIRK_NO_STRICT_ORDERING BIT(0) 26 29 27 30 #ifndef readq 28 31 static inline unsigned long long readq(void __iomem *addr) ··· 68 65 static void sf_pdma_fill_desc(struct sf_pdma_desc *desc, 69 66 u64 dst, u64 src, u64 size) 70 67 { 71 - desc->xfer_type = PDMA_FULL_SPEED; 68 + desc->xfer_type = desc->chan->pdma->transfer_type; 72 69 desc->xfer_size = size; 73 70 desc->dst_addr = dst; 74 71 desc->src_addr = src; ··· 495 492 496 493 static int sf_pdma_probe(struct platform_device *pdev) 497 494 { 495 + const struct sf_pdma_driver_platdata *ddata; 498 496 struct sf_pdma *pdma; 499 497 int ret, n_chans; 500 498 const enum dma_slave_buswidth widths = ··· 520 516 return -ENOMEM; 521 517 522 518 pdma->n_chans = n_chans; 519 + 520 + pdma->transfer_type = PDMA_FULL_SPEED | PDMA_STRICT_ORDERING; 521 + 522 + ddata = device_get_match_data(&pdev->dev); 523 + if (ddata) { 524 + if (ddata->quirks & PDMA_QUIRK_NO_STRICT_ORDERING) 525 + pdma->transfer_type &= ~PDMA_STRICT_ORDERING; 526 + } 523 527 524 528 pdma->membase = devm_platform_ioremap_resource(pdev, 0); 525 529 if (IS_ERR(pdma->membase)) ··· 575 563 return ret; 576 564 } 577 565 566 + ret = of_dma_controller_register(pdev->dev.of_node, 567 + of_dma_xlate_by_chan_id, pdma); 568 + if (ret < 0) { 569 + dev_err(&pdev->dev, 570 + "Can't register SiFive Platform OF_DMA. (%d)\n", ret); 571 + goto err_unregister; 572 + } 573 + 578 574 return 0; 575 + 576 + err_unregister: 577 + dma_async_device_unregister(&pdma->dma_dev); 578 + 579 + return ret; 579 580 } 580 581 581 582 static void sf_pdma_remove(struct platform_device *pdev) ··· 608 583 tasklet_kill(&ch->err_tasklet); 609 584 } 610 585 586 + if (pdev->dev.of_node) 587 + of_dma_controller_free(pdev->dev.of_node); 588 + 611 589 dma_async_device_unregister(&pdma->dma_dev); 612 590 } 613 591 592 + static const struct sf_pdma_driver_platdata mpfs_pdma = { 593 + .quirks = PDMA_QUIRK_NO_STRICT_ORDERING, 594 + }; 595 + 614 596 static const struct of_device_id sf_pdma_dt_ids[] = { 615 - { .compatible = "sifive,fu540-c000-pdma" }, 616 - { .compatible = "sifive,pdma0" }, 597 + { 598 + .compatible = "sifive,fu540-c000-pdma", 599 + }, { 600 + .compatible = "sifive,pdma0", 601 + }, { 602 + .compatible = "microchip,mpfs-pdma", 603 + .data = &mpfs_pdma, 604 + }, 617 605 {}, 618 606 }; 619 607 MODULE_DEVICE_TABLE(of, sf_pdma_dt_ids);

+7 -1

drivers/dma/sf-pdma/sf-pdma.h

··· 48 48 #define PDMA_ERR_STATUS_MASK GENMASK(31, 31) 49 49 50 50 /* Transfer Type */ 51 - #define PDMA_FULL_SPEED 0xFF000008 51 + #define PDMA_FULL_SPEED 0xFF000000 52 + #define PDMA_STRICT_ORDERING BIT(3) 52 53 53 54 /* Error Recovery */ 54 55 #define MAX_RETRY 1 ··· 113 112 struct dma_device dma_dev; 114 113 void __iomem *membase; 115 114 void __iomem *mappedbase; 115 + u32 transfer_type; 116 116 u32 n_chans; 117 117 struct sf_pdma_chan chans[] __counted_by(n_chans); 118 + }; 119 + 120 + struct sf_pdma_driver_platdata { 121 + u32 quirks; 118 122 }; 119 123 120 124 #endif /* _SF_PDMA_H */

+4 -4

drivers/dma/sh/rz-dmac.c

··· 755 755 756 756 static int rz_dmac_chan_probe(struct rz_dmac *dmac, 757 757 struct rz_dmac_chan *channel, 758 - unsigned int index) 758 + u8 index) 759 759 { 760 760 struct platform_device *pdev = to_platform_device(dmac->dev); 761 761 struct rz_lmdesc *lmdesc; 762 - char pdev_irqname[5]; 762 + char pdev_irqname[6]; 763 763 char *irqname; 764 764 int ret; 765 765 ··· 767 767 channel->mid_rid = -EINVAL; 768 768 769 769 /* Request the channel interrupt. */ 770 - sprintf(pdev_irqname, "ch%u", index); 770 + scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index); 771 771 channel->irq = platform_get_irq_byname(pdev, pdev_irqname); 772 772 if (channel->irq < 0) 773 773 return channel->irq; ··· 845 845 struct dma_device *engine; 846 846 struct rz_dmac *dmac; 847 847 int channel_num; 848 - unsigned int i; 849 848 int ret; 850 849 int irq; 850 + u8 i; 851 851 852 852 dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); 853 853 if (!dmac)

+1 -1

drivers/dma/sh/shdma.h

··· 25 25 const struct sh_dmae_slave_config *config; /* Slave DMA configuration */ 26 26 int xmit_shift; /* log_2(bytes_per_xfer) */ 27 27 void __iomem *base; 28 - char dev_id[16]; /* unique name per DMAC of channel */ 28 + char dev_id[32]; /* unique name per DMAC of channel */ 29 29 int pm_error; 30 30 dma_addr_t slave_addr; 31 31 };

+5 -5

drivers/dma/sh/usb-dmac.c

··· 706 706 707 707 static int usb_dmac_chan_probe(struct usb_dmac *dmac, 708 708 struct usb_dmac_chan *uchan, 709 - unsigned int index) 709 + u8 index) 710 710 { 711 711 struct platform_device *pdev = to_platform_device(dmac->dev); 712 - char pdev_irqname[5]; 712 + char pdev_irqname[6]; 713 713 char *irqname; 714 714 int ret; 715 715 ··· 717 717 uchan->iomem = dmac->iomem + USB_DMAC_CHAN_OFFSET(index); 718 718 719 719 /* Request the channel interrupt. */ 720 - sprintf(pdev_irqname, "ch%u", index); 720 + scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index); 721 721 uchan->irq = platform_get_irq_byname(pdev, pdev_irqname); 722 722 if (uchan->irq < 0) 723 723 return -ENODEV; ··· 768 768 const enum dma_slave_buswidth widths = USB_DMAC_SLAVE_BUSWIDTH; 769 769 struct dma_device *engine; 770 770 struct usb_dmac *dmac; 771 - unsigned int i; 772 771 int ret; 772 + u8 i; 773 773 774 774 dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); 775 775 if (!dmac) ··· 869 869 static void usb_dmac_remove(struct platform_device *pdev) 870 870 { 871 871 struct usb_dmac *dmac = platform_get_drvdata(pdev); 872 - int i; 872 + u8 i; 873 873 874 874 for (i = 0; i < dmac->n_channels; ++i) 875 875 usb_dmac_chan_remove(dmac, &dmac->channels[i]);

+5 -7

drivers/dma/ste_dma40.c

··· 31 31 /** 32 32 * struct stedma40_platform_data - Configuration struct for the dma device. 33 33 * 34 - * @dev_tx: mapping between destination event line and io address 35 - * @dev_rx: mapping between source event line and io address 36 34 * @disabled_channels: A vector, ending with -1, that marks physical channels 37 35 * that are for different reasons not available for the driver. 38 36 * @soft_lli_chans: A vector, that marks physical channels will use LLI by SW 39 37 * which avoids HW bug that exists in some versions of the controller. 40 - * SoftLLI introduces relink overhead that could impact performace for 38 + * SoftLLI introduces relink overhead that could impact performance for 41 39 * certain use cases. 42 40 * @num_of_soft_lli_chans: The number of channels that needs to be configured 43 41 * to use SoftLLI. ··· 182 184 183 185 /* 184 186 * since 9540 and 8540 has the same HW revision 185 - * use v4a for 9540 or ealier 187 + * use v4a for 9540 or earlier 186 188 * use v4b for 8540 or later 187 189 * HW revision: 188 190 * DB8500ed has revision 0 ··· 409 411 * 410 412 * @base: The virtual address of LCLA. 18 bit aligned. 411 413 * @dma_addr: DMA address, if mapped 412 - * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used. 414 + * @base_unaligned: The original kmalloc pointer, if kmalloc is used. 413 415 * This pointer is only there for clean-up on error. 414 416 * @pages: The number of pages needed for all physical channels. 415 417 * Only used later for clean-up on error ··· 1653 1655 1654 1656 return; 1655 1657 check_pending_tx: 1656 - /* Rescue manouver if receiving double interrupts */ 1658 + /* Rescue maneuver if receiving double interrupts */ 1657 1659 if (d40c->pending_tx > 0) 1658 1660 d40c->pending_tx--; 1659 1661 spin_unlock_irqrestore(&d40c->lock, flags); ··· 3410 3412 base->lcla_pool.base = (void *)page_list[i]; 3411 3413 } else { 3412 3414 /* 3413 - * After many attempts and no succees with finding the correct 3415 + * After many attempts and no success with finding the correct 3414 3416 * alignment, try with allocating a big buffer. 3415 3417 */ 3416 3418 dev_warn(base->dev,

+33 -2

drivers/dma/tegra210-adma.c

··· 153 153 void __iomem *base_addr; 154 154 struct clk *ahub_clk; 155 155 unsigned int nr_channels; 156 + unsigned long *dma_chan_mask; 156 157 unsigned long rx_requests_reserved; 157 158 unsigned long tx_requests_reserved; 158 159 ··· 742 741 743 742 for (i = 0; i < tdma->nr_channels; i++) { 744 743 tdc = &tdma->channels[i]; 744 + /* skip for reserved channels */ 745 + if (!tdc->tdma) 746 + continue; 747 + 745 748 ch_reg = &tdc->ch_regs; 746 749 ch_reg->cmd = tdma_ch_read(tdc, ADMA_CH_CMD); 747 750 /* skip if channel is not active */ ··· 784 779 785 780 for (i = 0; i < tdma->nr_channels; i++) { 786 781 tdc = &tdma->channels[i]; 782 + /* skip for reserved channels */ 783 + if (!tdc->tdma) 784 + continue; 787 785 ch_reg = &tdc->ch_regs; 788 786 /* skip if channel was not active earlier */ 789 787 if (!ch_reg->cmd) ··· 875 867 return PTR_ERR(tdma->ahub_clk); 876 868 } 877 869 870 + tdma->dma_chan_mask = devm_kzalloc(&pdev->dev, 871 + BITS_TO_LONGS(tdma->nr_channels) * sizeof(unsigned long), 872 + GFP_KERNEL); 873 + if (!tdma->dma_chan_mask) 874 + return -ENOMEM; 875 + 876 + /* Enable all channels by default */ 877 + bitmap_fill(tdma->dma_chan_mask, tdma->nr_channels); 878 + 879 + ret = of_property_read_u32_array(pdev->dev.of_node, "dma-channel-mask", 880 + (u32 *)tdma->dma_chan_mask, 881 + BITS_TO_U32(tdma->nr_channels)); 882 + if (ret < 0 && (ret != -EINVAL)) { 883 + dev_err(&pdev->dev, "dma-channel-mask is not complete.\n"); 884 + return ret; 885 + } 886 + 878 887 INIT_LIST_HEAD(&tdma->dma_dev.channels); 879 888 for (i = 0; i < tdma->nr_channels; i++) { 880 889 struct tegra_adma_chan *tdc = &tdma->channels[i]; 890 + 891 + /* skip for reserved channels */ 892 + if (!test_bit(i, tdma->dma_chan_mask)) 893 + continue; 881 894 882 895 tdc->chan_addr = tdma->base_addr + cdata->ch_base_offset 883 896 + (cdata->ch_reg_size * i); ··· 986 957 of_dma_controller_free(pdev->dev.of_node); 987 958 dma_async_device_unregister(&tdma->dma_dev); 988 959 989 - for (i = 0; i < tdma->nr_channels; ++i) 990 - irq_dispose_mapping(tdma->channels[i].irq); 960 + for (i = 0; i < tdma->nr_channels; ++i) { 961 + if (tdma->channels[i].irq) 962 + irq_dispose_mapping(tdma->channels[i].irq); 963 + } 991 964 992 965 pm_runtime_disable(&pdev->dev); 993 966 }

+2 -1

drivers/dma/ti/Makefile

··· 12 12 k3-psil-j721s2.o \ 13 13 k3-psil-am62.o \ 14 14 k3-psil-am62a.o \ 15 - k3-psil-j784s4.o 15 + k3-psil-j784s4.o \ 16 + k3-psil-am62p.o 16 17 obj-$(CONFIG_TI_K3_PSIL) += k3-psil-lib.o 17 18 obj-$(CONFIG_TI_DMA_CROSSBAR) += dma-crossbar.o

+325

drivers/dma/ti/k3-psil-am62p.c

··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + /* 3 + * Copyright (C) 2023 Texas Instruments Incorporated - https://www.ti.com 4 + */ 5 + 6 + #include <linux/kernel.h> 7 + 8 + #include "k3-psil-priv.h" 9 + 10 + #define PSIL_PDMA_XY_TR(x) \ 11 + { \ 12 + .thread_id = x, \ 13 + .ep_config = { \ 14 + .ep_type = PSIL_EP_PDMA_XY, \ 15 + .mapped_channel_id = -1, \ 16 + .default_flow_id = -1, \ 17 + }, \ 18 + } 19 + 20 + #define PSIL_PDMA_XY_PKT(x) \ 21 + { \ 22 + .thread_id = x, \ 23 + .ep_config = { \ 24 + .ep_type = PSIL_EP_PDMA_XY, \ 25 + .mapped_channel_id = -1, \ 26 + .default_flow_id = -1, \ 27 + .pkt_mode = 1, \ 28 + }, \ 29 + } 30 + 31 + #define PSIL_ETHERNET(x, ch, flow_base, flow_cnt) \ 32 + { \ 33 + .thread_id = x, \ 34 + .ep_config = { \ 35 + .ep_type = PSIL_EP_NATIVE, \ 36 + .pkt_mode = 1, \ 37 + .needs_epib = 1, \ 38 + .psd_size = 16, \ 39 + .mapped_channel_id = ch, \ 40 + .flow_start = flow_base, \ 41 + .flow_num = flow_cnt, \ 42 + .default_flow_id = flow_base, \ 43 + }, \ 44 + } 45 + 46 + #define PSIL_SAUL(x, ch, flow_base, flow_cnt, default_flow, tx) \ 47 + { \ 48 + .thread_id = x, \ 49 + .ep_config = { \ 50 + .ep_type = PSIL_EP_NATIVE, \ 51 + .pkt_mode = 1, \ 52 + .needs_epib = 1, \ 53 + .psd_size = 64, \ 54 + .mapped_channel_id = ch, \ 55 + .flow_start = flow_base, \ 56 + .flow_num = flow_cnt, \ 57 + .default_flow_id = default_flow, \ 58 + .notdpkt = tx, \ 59 + }, \ 60 + } 61 + 62 + #define PSIL_PDMA_MCASP(x) \ 63 + { \ 64 + .thread_id = x, \ 65 + .ep_config = { \ 66 + .ep_type = PSIL_EP_PDMA_XY, \ 67 + .pdma_acc32 = 1, \ 68 + .pdma_burst = 1, \ 69 + }, \ 70 + } 71 + 72 + #define PSIL_CSI2RX(x) \ 73 + { \ 74 + .thread_id = x, \ 75 + .ep_config = { \ 76 + .ep_type = PSIL_EP_NATIVE, \ 77 + }, \ 78 + } 79 + 80 + /* PSI-L source thread IDs, used for RX (DMA_DEV_TO_MEM) */ 81 + static struct psil_ep am62p_src_ep_map[] = { 82 + /* SAUL */ 83 + PSIL_SAUL(0x7504, 20, 35, 8, 35, 0), 84 + PSIL_SAUL(0x7505, 21, 35, 8, 36, 0), 85 + PSIL_SAUL(0x7506, 22, 43, 8, 43, 0), 86 + PSIL_SAUL(0x7507, 23, 43, 8, 44, 0), 87 + /* PDMA_MAIN0 - SPI0-2 */ 88 + PSIL_PDMA_XY_PKT(0x4300), 89 + PSIL_PDMA_XY_PKT(0x4301), 90 + PSIL_PDMA_XY_PKT(0x4302), 91 + PSIL_PDMA_XY_PKT(0x4303), 92 + PSIL_PDMA_XY_PKT(0x4304), 93 + PSIL_PDMA_XY_PKT(0x4305), 94 + PSIL_PDMA_XY_PKT(0x4306), 95 + PSIL_PDMA_XY_PKT(0x4307), 96 + PSIL_PDMA_XY_PKT(0x4308), 97 + PSIL_PDMA_XY_PKT(0x4309), 98 + PSIL_PDMA_XY_PKT(0x430a), 99 + PSIL_PDMA_XY_PKT(0x430b), 100 + /* PDMA_MAIN1 - UART0-6 */ 101 + PSIL_PDMA_XY_PKT(0x4400), 102 + PSIL_PDMA_XY_PKT(0x4401), 103 + PSIL_PDMA_XY_PKT(0x4402), 104 + PSIL_PDMA_XY_PKT(0x4403), 105 + PSIL_PDMA_XY_PKT(0x4404), 106 + PSIL_PDMA_XY_PKT(0x4405), 107 + PSIL_PDMA_XY_PKT(0x4406), 108 + /* PDMA_MAIN2 - MCASP0-2 */ 109 + PSIL_PDMA_MCASP(0x4500), 110 + PSIL_PDMA_MCASP(0x4501), 111 + PSIL_PDMA_MCASP(0x4502), 112 + /* CPSW3G */ 113 + PSIL_ETHERNET(0x4600, 19, 19, 16), 114 + /* CSI2RX */ 115 + PSIL_CSI2RX(0x5000), 116 + PSIL_CSI2RX(0x5001), 117 + PSIL_CSI2RX(0x5002), 118 + PSIL_CSI2RX(0x5003), 119 + PSIL_CSI2RX(0x5004), 120 + PSIL_CSI2RX(0x5005), 121 + PSIL_CSI2RX(0x5006), 122 + PSIL_CSI2RX(0x5007), 123 + PSIL_CSI2RX(0x5008), 124 + PSIL_CSI2RX(0x5009), 125 + PSIL_CSI2RX(0x500a), 126 + PSIL_CSI2RX(0x500b), 127 + PSIL_CSI2RX(0x500c), 128 + PSIL_CSI2RX(0x500d), 129 + PSIL_CSI2RX(0x500e), 130 + PSIL_CSI2RX(0x500f), 131 + PSIL_CSI2RX(0x5010), 132 + PSIL_CSI2RX(0x5011), 133 + PSIL_CSI2RX(0x5012), 134 + PSIL_CSI2RX(0x5013), 135 + PSIL_CSI2RX(0x5014), 136 + PSIL_CSI2RX(0x5015), 137 + PSIL_CSI2RX(0x5016), 138 + PSIL_CSI2RX(0x5017), 139 + PSIL_CSI2RX(0x5018), 140 + PSIL_CSI2RX(0x5019), 141 + PSIL_CSI2RX(0x501a), 142 + PSIL_CSI2RX(0x501b), 143 + PSIL_CSI2RX(0x501c), 144 + PSIL_CSI2RX(0x501d), 145 + PSIL_CSI2RX(0x501e), 146 + PSIL_CSI2RX(0x501f), 147 + PSIL_CSI2RX(0x5000), 148 + PSIL_CSI2RX(0x5001), 149 + PSIL_CSI2RX(0x5002), 150 + PSIL_CSI2RX(0x5003), 151 + PSIL_CSI2RX(0x5004), 152 + PSIL_CSI2RX(0x5005), 153 + PSIL_CSI2RX(0x5006), 154 + PSIL_CSI2RX(0x5007), 155 + PSIL_CSI2RX(0x5008), 156 + PSIL_CSI2RX(0x5009), 157 + PSIL_CSI2RX(0x500a), 158 + PSIL_CSI2RX(0x500b), 159 + PSIL_CSI2RX(0x500c), 160 + PSIL_CSI2RX(0x500d), 161 + PSIL_CSI2RX(0x500e), 162 + PSIL_CSI2RX(0x500f), 163 + PSIL_CSI2RX(0x5010), 164 + PSIL_CSI2RX(0x5011), 165 + PSIL_CSI2RX(0x5012), 166 + PSIL_CSI2RX(0x5013), 167 + PSIL_CSI2RX(0x5014), 168 + PSIL_CSI2RX(0x5015), 169 + PSIL_CSI2RX(0x5016), 170 + PSIL_CSI2RX(0x5017), 171 + PSIL_CSI2RX(0x5018), 172 + PSIL_CSI2RX(0x5019), 173 + PSIL_CSI2RX(0x501a), 174 + PSIL_CSI2RX(0x501b), 175 + PSIL_CSI2RX(0x501c), 176 + PSIL_CSI2RX(0x501d), 177 + PSIL_CSI2RX(0x501e), 178 + PSIL_CSI2RX(0x501f), 179 + /* CSIRX 1-3 (only for J722S) */ 180 + PSIL_CSI2RX(0x5100), 181 + PSIL_CSI2RX(0x5101), 182 + PSIL_CSI2RX(0x5102), 183 + PSIL_CSI2RX(0x5103), 184 + PSIL_CSI2RX(0x5104), 185 + PSIL_CSI2RX(0x5105), 186 + PSIL_CSI2RX(0x5106), 187 + PSIL_CSI2RX(0x5107), 188 + PSIL_CSI2RX(0x5108), 189 + PSIL_CSI2RX(0x5109), 190 + PSIL_CSI2RX(0x510a), 191 + PSIL_CSI2RX(0x510b), 192 + PSIL_CSI2RX(0x510c), 193 + PSIL_CSI2RX(0x510d), 194 + PSIL_CSI2RX(0x510e), 195 + PSIL_CSI2RX(0x510f), 196 + PSIL_CSI2RX(0x5110), 197 + PSIL_CSI2RX(0x5111), 198 + PSIL_CSI2RX(0x5112), 199 + PSIL_CSI2RX(0x5113), 200 + PSIL_CSI2RX(0x5114), 201 + PSIL_CSI2RX(0x5115), 202 + PSIL_CSI2RX(0x5116), 203 + PSIL_CSI2RX(0x5117), 204 + PSIL_CSI2RX(0x5118), 205 + PSIL_CSI2RX(0x5119), 206 + PSIL_CSI2RX(0x511a), 207 + PSIL_CSI2RX(0x511b), 208 + PSIL_CSI2RX(0x511c), 209 + PSIL_CSI2RX(0x511d), 210 + PSIL_CSI2RX(0x511e), 211 + PSIL_CSI2RX(0x511f), 212 + PSIL_CSI2RX(0x5200), 213 + PSIL_CSI2RX(0x5201), 214 + PSIL_CSI2RX(0x5202), 215 + PSIL_CSI2RX(0x5203), 216 + PSIL_CSI2RX(0x5204), 217 + PSIL_CSI2RX(0x5205), 218 + PSIL_CSI2RX(0x5206), 219 + PSIL_CSI2RX(0x5207), 220 + PSIL_CSI2RX(0x5208), 221 + PSIL_CSI2RX(0x5209), 222 + PSIL_CSI2RX(0x520a), 223 + PSIL_CSI2RX(0x520b), 224 + PSIL_CSI2RX(0x520c), 225 + PSIL_CSI2RX(0x520d), 226 + PSIL_CSI2RX(0x520e), 227 + PSIL_CSI2RX(0x520f), 228 + PSIL_CSI2RX(0x5210), 229 + PSIL_CSI2RX(0x5211), 230 + PSIL_CSI2RX(0x5212), 231 + PSIL_CSI2RX(0x5213), 232 + PSIL_CSI2RX(0x5214), 233 + PSIL_CSI2RX(0x5215), 234 + PSIL_CSI2RX(0x5216), 235 + PSIL_CSI2RX(0x5217), 236 + PSIL_CSI2RX(0x5218), 237 + PSIL_CSI2RX(0x5219), 238 + PSIL_CSI2RX(0x521a), 239 + PSIL_CSI2RX(0x521b), 240 + PSIL_CSI2RX(0x521c), 241 + PSIL_CSI2RX(0x521d), 242 + PSIL_CSI2RX(0x521e), 243 + PSIL_CSI2RX(0x521f), 244 + PSIL_CSI2RX(0x5300), 245 + PSIL_CSI2RX(0x5301), 246 + PSIL_CSI2RX(0x5302), 247 + PSIL_CSI2RX(0x5303), 248 + PSIL_CSI2RX(0x5304), 249 + PSIL_CSI2RX(0x5305), 250 + PSIL_CSI2RX(0x5306), 251 + PSIL_CSI2RX(0x5307), 252 + PSIL_CSI2RX(0x5308), 253 + PSIL_CSI2RX(0x5309), 254 + PSIL_CSI2RX(0x530a), 255 + PSIL_CSI2RX(0x530b), 256 + PSIL_CSI2RX(0x530c), 257 + PSIL_CSI2RX(0x530d), 258 + PSIL_CSI2RX(0x530e), 259 + PSIL_CSI2RX(0x530f), 260 + PSIL_CSI2RX(0x5310), 261 + PSIL_CSI2RX(0x5311), 262 + PSIL_CSI2RX(0x5312), 263 + PSIL_CSI2RX(0x5313), 264 + PSIL_CSI2RX(0x5314), 265 + PSIL_CSI2RX(0x5315), 266 + PSIL_CSI2RX(0x5316), 267 + PSIL_CSI2RX(0x5317), 268 + PSIL_CSI2RX(0x5318), 269 + PSIL_CSI2RX(0x5319), 270 + PSIL_CSI2RX(0x531a), 271 + PSIL_CSI2RX(0x531b), 272 + PSIL_CSI2RX(0x531c), 273 + PSIL_CSI2RX(0x531d), 274 + PSIL_CSI2RX(0x531e), 275 + PSIL_CSI2RX(0x531f), 276 + }; 277 + 278 + /* PSI-L destination thread IDs, used for TX (DMA_MEM_TO_DEV) */ 279 + static struct psil_ep am62p_dst_ep_map[] = { 280 + /* SAUL */ 281 + PSIL_SAUL(0xf500, 27, 83, 8, 83, 1), 282 + PSIL_SAUL(0xf501, 28, 91, 8, 91, 1), 283 + /* PDMA_MAIN0 - SPI0-2 */ 284 + PSIL_PDMA_XY_PKT(0xc300), 285 + PSIL_PDMA_XY_PKT(0xc301), 286 + PSIL_PDMA_XY_PKT(0xc302), 287 + PSIL_PDMA_XY_PKT(0xc303), 288 + PSIL_PDMA_XY_PKT(0xc304), 289 + PSIL_PDMA_XY_PKT(0xc305), 290 + PSIL_PDMA_XY_PKT(0xc306), 291 + PSIL_PDMA_XY_PKT(0xc307), 292 + PSIL_PDMA_XY_PKT(0xc308), 293 + PSIL_PDMA_XY_PKT(0xc309), 294 + PSIL_PDMA_XY_PKT(0xc30a), 295 + PSIL_PDMA_XY_PKT(0xc30b), 296 + /* PDMA_MAIN1 - UART0-6 */ 297 + PSIL_PDMA_XY_PKT(0xc400), 298 + PSIL_PDMA_XY_PKT(0xc401), 299 + PSIL_PDMA_XY_PKT(0xc402), 300 + PSIL_PDMA_XY_PKT(0xc403), 301 + PSIL_PDMA_XY_PKT(0xc404), 302 + PSIL_PDMA_XY_PKT(0xc405), 303 + PSIL_PDMA_XY_PKT(0xc406), 304 + /* PDMA_MAIN2 - MCASP0-2 */ 305 + PSIL_PDMA_MCASP(0xc500), 306 + PSIL_PDMA_MCASP(0xc501), 307 + PSIL_PDMA_MCASP(0xc502), 308 + /* CPSW3G */ 309 + PSIL_ETHERNET(0xc600, 19, 19, 8), 310 + PSIL_ETHERNET(0xc601, 20, 27, 8), 311 + PSIL_ETHERNET(0xc602, 21, 35, 8), 312 + PSIL_ETHERNET(0xc603, 22, 43, 8), 313 + PSIL_ETHERNET(0xc604, 23, 51, 8), 314 + PSIL_ETHERNET(0xc605, 24, 59, 8), 315 + PSIL_ETHERNET(0xc606, 25, 67, 8), 316 + PSIL_ETHERNET(0xc607, 26, 75, 8), 317 + }; 318 + 319 + struct psil_ep_map am62p_ep_map = { 320 + .name = "am62p", 321 + .src = am62p_src_ep_map, 322 + .src_count = ARRAY_SIZE(am62p_src_ep_map), 323 + .dst = am62p_dst_ep_map, 324 + .dst_count = ARRAY_SIZE(am62p_dst_ep_map), 325 + };

+1

drivers/dma/ti/k3-psil-priv.h

··· 45 45 extern struct psil_ep_map am62_ep_map; 46 46 extern struct psil_ep_map am62a_ep_map; 47 47 extern struct psil_ep_map j784s4_ep_map; 48 + extern struct psil_ep_map am62p_ep_map; 48 49 49 50 #endif /* K3_PSIL_PRIV_H_ */

+2

drivers/dma/ti/k3-psil.c

··· 26 26 { .family = "AM62X", .data = &am62_ep_map }, 27 27 { .family = "AM62AX", .data = &am62a_ep_map }, 28 28 { .family = "J784S4", .data = &j784s4_ep_map }, 29 + { .family = "AM62PX", .data = &am62p_ep_map }, 30 + { .family = "J722S", .data = &am62p_ep_map }, 29 31 { /* sentinel */ } 30 32 }; 31 33

+2

drivers/dma/ti/k3-udma.c

··· 4441 4441 { .family = "AM62X", .data = &am64_soc_data }, 4442 4442 { .family = "AM62AX", .data = &am64_soc_data }, 4443 4443 { .family = "J784S4", .data = &j721e_soc_data }, 4444 + { .family = "AM62PX", .data = &am64_soc_data }, 4445 + { .family = "J722S", .data = &am64_soc_data }, 4444 4446 { /* sentinel */ } 4445 4447 }; 4446 4448

+11 -6

drivers/dma/uniphier-mdmac.c

··· 453 453 return ret; 454 454 } 455 455 456 - static int uniphier_mdmac_remove(struct platform_device *pdev) 456 + static void uniphier_mdmac_remove(struct platform_device *pdev) 457 457 { 458 458 struct uniphier_mdmac_device *mdev = platform_get_drvdata(pdev); 459 459 struct dma_chan *chan; ··· 468 468 */ 469 469 list_for_each_entry(chan, &mdev->ddev.channels, device_node) { 470 470 ret = dmaengine_terminate_sync(chan); 471 - if (ret) 472 - return ret; 471 + if (ret) { 472 + /* 473 + * This results in resource leakage and maybe also 474 + * use-after-free errors as e.g. *mdev is kfreed. 475 + */ 476 + dev_alert(&pdev->dev, "Failed to terminate channel %d (%pe)\n", 477 + chan->chan_id, ERR_PTR(ret)); 478 + return; 479 + } 473 480 uniphier_mdmac_free_chan_resources(chan); 474 481 } 475 482 476 483 of_dma_controller_free(pdev->dev.of_node); 477 484 dma_async_device_unregister(&mdev->ddev); 478 485 clk_disable_unprepare(mdev->clk); 479 - 480 - return 0; 481 486 } 482 487 483 488 static const struct of_device_id uniphier_mdmac_match[] = { ··· 493 488 494 489 static struct platform_driver uniphier_mdmac_driver = { 495 490 .probe = uniphier_mdmac_probe, 496 - .remove = uniphier_mdmac_remove, 491 + .remove_new = uniphier_mdmac_remove, 497 492 .driver = { 498 493 .name = "uniphier-mio-dmac", 499 494 .of_match_table = uniphier_mdmac_match,

+11 -6

drivers/dma/uniphier-xdmac.c

··· 563 563 return ret; 564 564 } 565 565 566 - static int uniphier_xdmac_remove(struct platform_device *pdev) 566 + static void uniphier_xdmac_remove(struct platform_device *pdev) 567 567 { 568 568 struct uniphier_xdmac_device *xdev = platform_get_drvdata(pdev); 569 569 struct dma_device *ddev = &xdev->ddev; ··· 579 579 */ 580 580 list_for_each_entry(chan, &ddev->channels, device_node) { 581 581 ret = dmaengine_terminate_sync(chan); 582 - if (ret) 583 - return ret; 582 + if (ret) { 583 + /* 584 + * This results in resource leakage and maybe also 585 + * use-after-free errors as e.g. *xdev is kfreed. 586 + */ 587 + dev_alert(&pdev->dev, "Failed to terminate channel %d (%pe)\n", 588 + chan->chan_id, ERR_PTR(ret)); 589 + return; 590 + } 584 591 uniphier_xdmac_free_chan_resources(chan); 585 592 } 586 593 587 594 of_dma_controller_free(pdev->dev.of_node); 588 595 dma_async_device_unregister(ddev); 589 - 590 - return 0; 591 596 } 592 597 593 598 static const struct of_device_id uniphier_xdmac_match[] = { ··· 603 598 604 599 static struct platform_driver uniphier_xdmac_driver = { 605 600 .probe = uniphier_xdmac_probe, 606 - .remove = uniphier_xdmac_remove, 601 + .remove_new = uniphier_xdmac_remove, 607 602 .driver = { 608 603 .name = "uniphier-xdmac", 609 604 .of_match_table = uniphier_xdmac_match,

+15 -15

drivers/dma/xilinx/xdma-regs.h

··· 64 64 __le64 next_desc; 65 65 }; 66 66 67 - #define XDMA_DESC_SIZE sizeof(struct xdma_hw_desc) 68 - #define XDMA_DESC_BLOCK_SIZE (XDMA_DESC_SIZE * XDMA_DESC_ADJACENT) 69 - #define XDMA_DESC_BLOCK_ALIGN 4096 67 + #define XDMA_DESC_SIZE sizeof(struct xdma_hw_desc) 68 + #define XDMA_DESC_BLOCK_SIZE (XDMA_DESC_SIZE * XDMA_DESC_ADJACENT) 69 + #define XDMA_DESC_BLOCK_ALIGN 32 70 + #define XDMA_DESC_BLOCK_BOUNDARY 4096 70 71 71 72 /* 72 73 * Channel registers ··· 77 76 #define XDMA_CHAN_CONTROL_W1S 0x8 78 77 #define XDMA_CHAN_CONTROL_W1C 0xc 79 78 #define XDMA_CHAN_STATUS 0x40 79 + #define XDMA_CHAN_STATUS_RC 0x44 80 80 #define XDMA_CHAN_COMPLETED_DESC 0x48 81 81 #define XDMA_CHAN_ALIGNMENTS 0x4c 82 82 #define XDMA_CHAN_INTR_ENABLE 0x90 ··· 103 101 #define CHAN_CTRL_IE_MAGIC_STOPPED BIT(4) 104 102 #define CHAN_CTRL_IE_IDLE_STOPPED BIT(6) 105 103 #define CHAN_CTRL_IE_READ_ERROR GENMASK(13, 9) 104 + #define CHAN_CTRL_IE_WRITE_ERROR GENMASK(18, 14) 106 105 #define CHAN_CTRL_IE_DESC_ERROR GENMASK(23, 19) 107 106 #define CHAN_CTRL_NON_INCR_ADDR BIT(25) 108 107 #define CHAN_CTRL_POLL_MODE_WB BIT(26) ··· 114 111 CHAN_CTRL_IE_DESC_ALIGN_MISMATCH | \ 115 112 CHAN_CTRL_IE_MAGIC_STOPPED | \ 116 113 CHAN_CTRL_IE_READ_ERROR | \ 114 + CHAN_CTRL_IE_WRITE_ERROR | \ 117 115 CHAN_CTRL_IE_DESC_ERROR) 116 + 117 + #define XDMA_CHAN_STATUS_MASK CHAN_CTRL_START 118 + 119 + #define XDMA_CHAN_ERROR_MASK (CHAN_CTRL_IE_DESC_ALIGN_MISMATCH | \ 120 + CHAN_CTRL_IE_MAGIC_STOPPED | \ 121 + CHAN_CTRL_IE_READ_ERROR | \ 122 + CHAN_CTRL_IE_WRITE_ERROR | \ 123 + CHAN_CTRL_IE_DESC_ERROR) 118 124 119 125 /* bits of the channel interrupt enable mask */ 120 126 #define CHAN_IM_DESC_ERROR BIT(19) ··· 145 133 #define XDMA_SGDMA_DESC_HI 0x4084 146 134 #define XDMA_SGDMA_DESC_ADJ 0x4088 147 135 #define XDMA_SGDMA_DESC_CREDIT 0x408c 148 - 149 - /* bits of the SG DMA control register */ 150 - #define XDMA_CTRL_RUN_STOP BIT(0) 151 - #define XDMA_CTRL_IE_DESC_STOPPED BIT(1) 152 - #define XDMA_CTRL_IE_DESC_COMPLETED BIT(2) 153 - #define XDMA_CTRL_IE_DESC_ALIGN_MISMATCH BIT(3) 154 - #define XDMA_CTRL_IE_MAGIC_STOPPED BIT(4) 155 - #define XDMA_CTRL_IE_IDLE_STOPPED BIT(6) 156 - #define XDMA_CTRL_IE_READ_ERROR GENMASK(13, 9) 157 - #define XDMA_CTRL_IE_DESC_ERROR GENMASK(23, 19) 158 - #define XDMA_CTRL_NON_INCR_ADDR BIT(25) 159 - #define XDMA_CTRL_POLL_MODE_WB BIT(26) 160 136 161 137 /* 162 138 * interrupt registers

+262 -87

drivers/dma/xilinx/xdma.c

··· 78 78 * @vdesc: Virtual DMA descriptor 79 79 * @chan: DMA channel pointer 80 80 * @dir: Transferring direction of the request 81 - * @dev_addr: Physical address on DMA device side 82 81 * @desc_blocks: Hardware descriptor blocks 83 82 * @dblk_num: Number of hardware descriptor blocks 84 83 * @desc_num: Number of hardware descriptors 85 84 * @completed_desc_num: Completed hardware descriptors 86 85 * @cyclic: Cyclic transfer vs. scatter-gather 86 + * @interleaved_dma: Interleaved DMA transfer 87 87 * @periods: Number of periods in the cyclic transfer 88 88 * @period_size: Size of a period in bytes in cyclic transfers 89 + * @frames_left: Number of frames left in interleaved DMA transfer 90 + * @error: tx error flag 89 91 */ 90 92 struct xdma_desc { 91 93 struct virt_dma_desc vdesc; 92 94 struct xdma_chan *chan; 93 95 enum dma_transfer_direction dir; 94 - u64 dev_addr; 95 96 struct xdma_desc_block *desc_blocks; 96 97 u32 dblk_num; 97 98 u32 desc_num; 98 99 u32 completed_desc_num; 99 100 bool cyclic; 101 + bool interleaved_dma; 100 102 u32 periods; 101 103 u32 period_size; 104 + u32 frames_left; 105 + bool error; 102 106 }; 103 107 104 108 #define XDMA_DEV_STATUS_REG_DMA BIT(0) ··· 280 276 sw_desc->chan = chan; 281 277 sw_desc->desc_num = desc_num; 282 278 sw_desc->cyclic = cyclic; 279 + sw_desc->error = false; 283 280 dblk_num = DIV_ROUND_UP(desc_num, XDMA_DESC_ADJACENT); 284 281 sw_desc->desc_blocks = kcalloc(dblk_num, sizeof(*sw_desc->desc_blocks), 285 282 GFP_NOWAIT); ··· 376 371 return ret; 377 372 378 373 xchan->busy = true; 374 + 375 + return 0; 376 + } 377 + 378 + /** 379 + * xdma_xfer_stop - Stop DMA transfer 380 + * @xchan: DMA channel pointer 381 + */ 382 + static int xdma_xfer_stop(struct xdma_chan *xchan) 383 + { 384 + int ret; 385 + u32 val; 386 + struct xdma_device *xdev = xchan->xdev_hdl; 387 + 388 + /* clear run stop bit to prevent any further auto-triggering */ 389 + ret = regmap_write(xdev->rmap, xchan->base + XDMA_CHAN_CONTROL_W1C, 390 + CHAN_CTRL_RUN_STOP); 391 + if (ret) 392 + return ret; 393 + 394 + /* Clear the channel status register */ 395 + ret = regmap_read(xdev->rmap, xchan->base + XDMA_CHAN_STATUS_RC, &val); 396 + if (ret) 397 + return ret; 398 + 379 399 return 0; 380 400 } 381 401 ··· 506 476 } 507 477 508 478 /** 479 + * xdma_terminate_all - Terminate all transactions 480 + * @chan: DMA channel pointer 481 + */ 482 + static int xdma_terminate_all(struct dma_chan *chan) 483 + { 484 + struct xdma_chan *xdma_chan = to_xdma_chan(chan); 485 + struct virt_dma_desc *vd; 486 + unsigned long flags; 487 + LIST_HEAD(head); 488 + 489 + xdma_xfer_stop(xdma_chan); 490 + 491 + spin_lock_irqsave(&xdma_chan->vchan.lock, flags); 492 + 493 + xdma_chan->busy = false; 494 + vd = vchan_next_desc(&xdma_chan->vchan); 495 + if (vd) { 496 + list_del(&vd->node); 497 + dma_cookie_complete(&vd->tx); 498 + vchan_terminate_vdesc(vd); 499 + } 500 + vchan_get_all_descriptors(&xdma_chan->vchan, &head); 501 + list_splice_tail(&head, &xdma_chan->vchan.desc_terminated); 502 + 503 + spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags); 504 + 505 + return 0; 506 + } 507 + 508 + /** 509 + * xdma_synchronize - Synchronize terminated transactions 510 + * @chan: DMA channel pointer 511 + */ 512 + static void xdma_synchronize(struct dma_chan *chan) 513 + { 514 + struct xdma_chan *xdma_chan = to_xdma_chan(chan); 515 + 516 + vchan_synchronize(&xdma_chan->vchan); 517 + } 518 + 519 + /** 520 + * xdma_fill_descs - Fill hardware descriptors with contiguous memory block addresses 521 + * @sw_desc: tx descriptor state container 522 + * @src_addr: Value for a ->src_addr field of a first descriptor 523 + * @dst_addr: Value for a ->dst_addr field of a first descriptor 524 + * @size: Total size of a contiguous memory block 525 + * @filled_descs_num: Number of filled hardware descriptors for corresponding sw_desc 526 + */ 527 + static inline u32 xdma_fill_descs(struct xdma_desc *sw_desc, u64 src_addr, 528 + u64 dst_addr, u32 size, u32 filled_descs_num) 529 + { 530 + u32 left = size, len, desc_num = filled_descs_num; 531 + struct xdma_desc_block *dblk; 532 + struct xdma_hw_desc *desc; 533 + 534 + dblk = sw_desc->desc_blocks + (desc_num / XDMA_DESC_ADJACENT); 535 + desc = dblk->virt_addr; 536 + desc += desc_num & XDMA_DESC_ADJACENT_MASK; 537 + do { 538 + len = min_t(u32, left, XDMA_DESC_BLEN_MAX); 539 + /* set hardware descriptor */ 540 + desc->bytes = cpu_to_le32(len); 541 + desc->src_addr = cpu_to_le64(src_addr); 542 + desc->dst_addr = cpu_to_le64(dst_addr); 543 + if (!(++desc_num & XDMA_DESC_ADJACENT_MASK)) 544 + desc = (++dblk)->virt_addr; 545 + else 546 + desc++; 547 + 548 + src_addr += len; 549 + dst_addr += len; 550 + left -= len; 551 + } while (left); 552 + 553 + return desc_num - filled_descs_num; 554 + } 555 + 556 + /** 509 557 * xdma_prep_device_sg - prepare a descriptor for a DMA transaction 510 558 * @chan: DMA channel pointer 511 559 * @sgl: Transfer scatter gather list ··· 599 491 { 600 492 struct xdma_chan *xdma_chan = to_xdma_chan(chan); 601 493 struct dma_async_tx_descriptor *tx_desc; 602 - u32 desc_num = 0, i, len, rest; 603 - struct xdma_desc_block *dblk; 604 - struct xdma_hw_desc *desc; 605 494 struct xdma_desc *sw_desc; 606 - u64 dev_addr, *src, *dst; 495 + u32 desc_num = 0, i; 496 + u64 addr, dev_addr, *src, *dst; 607 497 struct scatterlist *sg; 608 - u64 addr; 609 498 610 499 for_each_sg(sgl, sg, sg_len, i) 611 500 desc_num += DIV_ROUND_UP(sg_dma_len(sg), XDMA_DESC_BLEN_MAX); ··· 611 506 if (!sw_desc) 612 507 return NULL; 613 508 sw_desc->dir = dir; 509 + sw_desc->cyclic = false; 510 + sw_desc->interleaved_dma = false; 614 511 615 512 if (dir == DMA_MEM_TO_DEV) { 616 513 dev_addr = xdma_chan->cfg.dst_addr; ··· 624 517 dst = &addr; 625 518 } 626 519 627 - dblk = sw_desc->desc_blocks; 628 - desc = dblk->virt_addr; 629 - desc_num = 1; 520 + desc_num = 0; 630 521 for_each_sg(sgl, sg, sg_len, i) { 631 522 addr = sg_dma_address(sg); 632 - rest = sg_dma_len(sg); 633 - 634 - do { 635 - len = min_t(u32, rest, XDMA_DESC_BLEN_MAX); 636 - /* set hardware descriptor */ 637 - desc->bytes = cpu_to_le32(len); 638 - desc->src_addr = cpu_to_le64(*src); 639 - desc->dst_addr = cpu_to_le64(*dst); 640 - 641 - if (!(desc_num & XDMA_DESC_ADJACENT_MASK)) { 642 - dblk++; 643 - desc = dblk->virt_addr; 644 - } else { 645 - desc++; 646 - } 647 - 648 - desc_num++; 649 - dev_addr += len; 650 - addr += len; 651 - rest -= len; 652 - } while (rest); 523 + desc_num += xdma_fill_descs(sw_desc, *src, *dst, sg_dma_len(sg), desc_num); 524 + dev_addr += sg_dma_len(sg); 653 525 } 654 526 655 527 tx_desc = vchan_tx_prep(&xdma_chan->vchan, &sw_desc->vdesc, flags); ··· 662 576 struct xdma_device *xdev = xdma_chan->xdev_hdl; 663 577 unsigned int periods = size / period_size; 664 578 struct dma_async_tx_descriptor *tx_desc; 665 - struct xdma_desc_block *dblk; 666 - struct xdma_hw_desc *desc; 667 579 struct xdma_desc *sw_desc; 580 + u64 addr, dev_addr, *src, *dst; 581 + u32 desc_num; 668 582 unsigned int i; 669 583 670 584 /* ··· 688 602 sw_desc->periods = periods; 689 603 sw_desc->period_size = period_size; 690 604 sw_desc->dir = dir; 605 + sw_desc->interleaved_dma = false; 691 606 692 - dblk = sw_desc->desc_blocks; 693 - desc = dblk->virt_addr; 607 + addr = address; 608 + if (dir == DMA_MEM_TO_DEV) { 609 + dev_addr = xdma_chan->cfg.dst_addr; 610 + src = &addr; 611 + dst = &dev_addr; 612 + } else { 613 + dev_addr = xdma_chan->cfg.src_addr; 614 + src = &dev_addr; 615 + dst = &addr; 616 + } 694 617 695 - /* fill hardware descriptor */ 618 + desc_num = 0; 696 619 for (i = 0; i < periods; i++) { 697 - desc->bytes = cpu_to_le32(period_size); 698 - if (dir == DMA_MEM_TO_DEV) { 699 - desc->src_addr = cpu_to_le64(address + i * period_size); 700 - desc->dst_addr = cpu_to_le64(xdma_chan->cfg.dst_addr); 701 - } else { 702 - desc->src_addr = cpu_to_le64(xdma_chan->cfg.src_addr); 703 - desc->dst_addr = cpu_to_le64(address + i * period_size); 704 - } 705 - 706 - desc++; 620 + desc_num += xdma_fill_descs(sw_desc, *src, *dst, period_size, desc_num); 621 + addr += i * period_size; 707 622 } 708 623 709 624 tx_desc = vchan_tx_prep(&xdma_chan->vchan, &sw_desc->vdesc, flags); ··· 716 629 failed: 717 630 xdma_free_desc(&sw_desc->vdesc); 718 631 632 + return NULL; 633 + } 634 + 635 + /** 636 + * xdma_prep_interleaved_dma - Prepare virtual descriptor for interleaved DMA transfers 637 + * @chan: DMA channel 638 + * @xt: DMA transfer template 639 + * @flags: tx flags 640 + */ 641 + static struct dma_async_tx_descriptor * 642 + xdma_prep_interleaved_dma(struct dma_chan *chan, 643 + struct dma_interleaved_template *xt, 644 + unsigned long flags) 645 + { 646 + int i; 647 + u32 desc_num = 0, period_size = 0; 648 + struct dma_async_tx_descriptor *tx_desc; 649 + struct xdma_chan *xchan = to_xdma_chan(chan); 650 + struct xdma_desc *sw_desc; 651 + u64 src_addr, dst_addr; 652 + 653 + for (i = 0; i < xt->frame_size; ++i) 654 + desc_num += DIV_ROUND_UP(xt->sgl[i].size, XDMA_DESC_BLEN_MAX); 655 + 656 + sw_desc = xdma_alloc_desc(xchan, desc_num, false); 657 + if (!sw_desc) 658 + return NULL; 659 + sw_desc->dir = xt->dir; 660 + sw_desc->interleaved_dma = true; 661 + sw_desc->cyclic = flags & DMA_PREP_REPEAT; 662 + sw_desc->frames_left = xt->numf; 663 + sw_desc->periods = xt->numf; 664 + 665 + desc_num = 0; 666 + src_addr = xt->src_start; 667 + dst_addr = xt->dst_start; 668 + for (i = 0; i < xt->frame_size; ++i) { 669 + desc_num += xdma_fill_descs(sw_desc, src_addr, dst_addr, xt->sgl[i].size, desc_num); 670 + src_addr += dmaengine_get_src_icg(xt, &xt->sgl[i]) + (xt->src_inc ? 671 + xt->sgl[i].size : 0); 672 + dst_addr += dmaengine_get_dst_icg(xt, &xt->sgl[i]) + (xt->dst_inc ? 673 + xt->sgl[i].size : 0); 674 + period_size += xt->sgl[i].size; 675 + } 676 + sw_desc->period_size = period_size; 677 + 678 + tx_desc = vchan_tx_prep(&xchan->vchan, &sw_desc->vdesc, flags); 679 + if (tx_desc) 680 + return tx_desc; 681 + 682 + xdma_free_desc(&sw_desc->vdesc); 719 683 return NULL; 720 684 } 721 685 ··· 815 677 return -EINVAL; 816 678 } 817 679 818 - xdma_chan->desc_pool = dma_pool_create(dma_chan_name(chan), 819 - dev, XDMA_DESC_BLOCK_SIZE, 820 - XDMA_DESC_BLOCK_ALIGN, 0); 680 + xdma_chan->desc_pool = dma_pool_create(dma_chan_name(chan), dev, XDMA_DESC_BLOCK_SIZE, 681 + XDMA_DESC_BLOCK_ALIGN, XDMA_DESC_BLOCK_BOUNDARY); 821 682 if (!xdma_chan->desc_pool) { 822 683 xdma_err(xdev, "unable to allocate descriptor pool"); 823 684 return -ENOMEM; ··· 843 706 spin_lock_irqsave(&xdma_chan->vchan.lock, flags); 844 707 845 708 vd = vchan_find_desc(&xdma_chan->vchan, cookie); 846 - if (vd) 847 - desc = to_xdma_desc(vd); 848 - if (!desc || !desc->cyclic) { 849 - spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags); 850 - return ret; 709 + if (!vd) 710 + goto out; 711 + 712 + desc = to_xdma_desc(vd); 713 + if (desc->error) { 714 + ret = DMA_ERROR; 715 + } else if (desc->cyclic) { 716 + period_idx = desc->completed_desc_num % desc->periods; 717 + residue = (desc->periods - period_idx) * desc->period_size; 718 + dma_set_residue(state, residue); 851 719 } 852 - 853 - period_idx = desc->completed_desc_num % desc->periods; 854 - residue = (desc->periods - period_idx) * desc->period_size; 855 - 720 + out: 856 721 spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags); 857 - 858 - dma_set_residue(state, residue); 859 722 860 723 return ret; 861 724 } ··· 869 732 { 870 733 struct xdma_chan *xchan = dev_id; 871 734 u32 complete_desc_num = 0; 872 - struct xdma_device *xdev; 873 - struct virt_dma_desc *vd; 735 + struct xdma_device *xdev = xchan->xdev_hdl; 736 + struct virt_dma_desc *vd, *next_vd; 874 737 struct xdma_desc *desc; 875 738 int ret; 876 739 u32 st; 740 + bool repeat_tx; 877 741 878 742 spin_lock(&xchan->vchan.lock); 879 743 ··· 883 745 if (!vd) 884 746 goto out; 885 747 886 - xchan->busy = false; 748 + /* Clear-on-read the status register */ 749 + ret = regmap_read(xdev->rmap, xchan->base + XDMA_CHAN_STATUS_RC, &st); 750 + if (ret) 751 + goto out; 752 + 887 753 desc = to_xdma_desc(vd); 888 - xdev = xchan->xdev_hdl; 754 + 755 + st &= XDMA_CHAN_STATUS_MASK; 756 + if ((st & XDMA_CHAN_ERROR_MASK) || 757 + !(st & (CHAN_CTRL_IE_DESC_COMPLETED | CHAN_CTRL_IE_DESC_STOPPED))) { 758 + desc->error = true; 759 + xdma_err(xdev, "channel error, status register value: 0x%x", st); 760 + goto out; 761 + } 889 762 890 763 ret = regmap_read(xdev->rmap, xchan->base + XDMA_CHAN_COMPLETED_DESC, 891 764 &complete_desc_num); 892 765 if (ret) 893 766 goto out; 894 767 895 - desc->completed_desc_num += complete_desc_num; 768 + if (desc->interleaved_dma) { 769 + xchan->busy = false; 770 + desc->completed_desc_num += complete_desc_num; 771 + if (complete_desc_num == XDMA_DESC_BLOCK_NUM * XDMA_DESC_ADJACENT) { 772 + xdma_xfer_start(xchan); 773 + goto out; 774 + } 896 775 897 - if (desc->cyclic) { 898 - ret = regmap_read(xdev->rmap, xchan->base + XDMA_CHAN_STATUS, 899 - &st); 900 - if (ret) 776 + /* last desc of any frame */ 777 + desc->frames_left--; 778 + if (desc->frames_left) 901 779 goto out; 902 780 903 - regmap_write(xdev->rmap, xchan->base + XDMA_CHAN_STATUS, st); 781 + /* last desc of the last frame */ 782 + repeat_tx = vd->tx.flags & DMA_PREP_REPEAT; 783 + next_vd = list_first_entry_or_null(&vd->node, struct virt_dma_desc, node); 784 + if (next_vd) 785 + repeat_tx = repeat_tx && !(next_vd->tx.flags & DMA_PREP_LOAD_EOT); 786 + if (repeat_tx) { 787 + desc->frames_left = desc->periods; 788 + desc->completed_desc_num = 0; 789 + vchan_cyclic_callback(vd); 790 + } else { 791 + list_del(&vd->node); 792 + vchan_cookie_complete(vd); 793 + } 794 + /* start (or continue) the tx of a first desc on the vc.desc_issued list, if any */ 795 + xdma_xfer_start(xchan); 796 + } else if (!desc->cyclic) { 797 + xchan->busy = false; 798 + desc->completed_desc_num += complete_desc_num; 904 799 800 + /* if all data blocks are transferred, remove and complete the request */ 801 + if (desc->completed_desc_num == desc->desc_num) { 802 + list_del(&vd->node); 803 + vchan_cookie_complete(vd); 804 + goto out; 805 + } 806 + 807 + if (desc->completed_desc_num > desc->desc_num || 808 + complete_desc_num != XDMA_DESC_BLOCK_NUM * XDMA_DESC_ADJACENT) 809 + goto out; 810 + 811 + /* transfer the rest of data */ 812 + xdma_xfer_start(xchan); 813 + } else { 814 + desc->completed_desc_num = complete_desc_num; 905 815 vchan_cyclic_callback(vd); 906 - goto out; 907 816 } 908 - 909 - /* 910 - * if all data blocks are transferred, remove and complete the request 911 - */ 912 - if (desc->completed_desc_num == desc->desc_num) { 913 - list_del(&vd->node); 914 - vchan_cookie_complete(vd); 915 - goto out; 916 - } 917 - 918 - if (desc->completed_desc_num > desc->desc_num || 919 - complete_desc_num != XDMA_DESC_BLOCK_NUM * XDMA_DESC_ADJACENT) 920 - goto out; 921 - 922 - /* transfer the rest of data (SG only) */ 923 - xdma_xfer_start(xchan); 924 817 925 818 out: 926 819 spin_unlock(&xchan->vchan.lock); ··· 1249 1080 dma_cap_set(DMA_SLAVE, xdev->dma_dev.cap_mask); 1250 1081 dma_cap_set(DMA_PRIVATE, xdev->dma_dev.cap_mask); 1251 1082 dma_cap_set(DMA_CYCLIC, xdev->dma_dev.cap_mask); 1083 + dma_cap_set(DMA_INTERLEAVE, xdev->dma_dev.cap_mask); 1084 + dma_cap_set(DMA_REPEAT, xdev->dma_dev.cap_mask); 1085 + dma_cap_set(DMA_LOAD_EOT, xdev->dma_dev.cap_mask); 1252 1086 1253 1087 xdev->dma_dev.dev = &pdev->dev; 1254 1088 xdev->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; ··· 1261 1089 xdev->dma_dev.device_prep_slave_sg = xdma_prep_device_sg; 1262 1090 xdev->dma_dev.device_config = xdma_device_config; 1263 1091 xdev->dma_dev.device_issue_pending = xdma_issue_pending; 1092 + xdev->dma_dev.device_terminate_all = xdma_terminate_all; 1093 + xdev->dma_dev.device_synchronize = xdma_synchronize; 1264 1094 xdev->dma_dev.filter.map = pdata->device_map; 1265 1095 xdev->dma_dev.filter.mapcnt = pdata->device_map_cnt; 1266 1096 xdev->dma_dev.filter.fn = xdma_filter_fn; 1267 1097 xdev->dma_dev.device_prep_dma_cyclic = xdma_prep_dma_cyclic; 1098 + xdev->dma_dev.device_prep_interleaved_dma = xdma_prep_interleaved_dma; 1268 1099 1269 1100 ret = dma_async_device_register(&xdev->dma_dev); 1270 1101 if (ret) {

+1 -1

drivers/dma/xilinx/xilinx_dpdma.c

··· 309 309 310 310 out_str_len = strlen(XILINX_DPDMA_DEBUGFS_UINT16_MAX_STR); 311 311 out_str_len = min_t(size_t, XILINX_DPDMA_DEBUGFS_READ_MAX_SIZE, 312 - out_str_len); 312 + out_str_len + 1); 313 313 snprintf(buf, out_str_len, "%d", 314 314 dpdma_debugfs.xilinx_dpdma_irq_done_count); 315 315

+21

include/dt-bindings/dma/fsl-edma.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ 2 + 3 + #ifndef _FSL_EDMA_DT_BINDING_H_ 4 + #define _FSL_EDMA_DT_BINDING_H_ 5 + 6 + /* Receive Channel */ 7 + #define FSL_EDMA_RX 0x1 8 + 9 + /* iMX8 audio remote DMA */ 10 + #define FSL_EDMA_REMOTE 0x2 11 + 12 + /* FIFO is continue memory region */ 13 + #define FSL_EDMA_MULTI_FIFO 0x4 14 + 15 + /* Channel need stick to even channel */ 16 + #define FSL_EDMA_EVEN_CH 0x8 17 + 18 + /* Channel need stick to odd channel */ 19 + #define FSL_EDMA_ODD_CH 0x10 20 + 21 + #endif

Configure Feed

Configure Feed