+12

Documentation/ABI/stable/sysfs-driver-dma-idxd

reviewed

··· 22 22 KernelVersion: 5.6.0 23 23 Contact: dmaengine@vger.kernel.org 24 24 Description: The largest number of work descriptors in a batch. 25 25 + It's not visible when the device does not support batch. 25 26 26 27 What: /sys/bus/dsa/devices/dsa<m>/max_work_queues_size 27 28 Date: Oct 25, 2019 ··· 50 49 The read buffers represent resources within the DSA 51 50 implementation, and these resources are allocated by engines to 52 51 support operations. See DSA spec v1.2 9.2.4 Total Read Buffers. 52 52 + It's not visible when the device does not support Read Buffer 53 53 + allocation control. 53 54 54 55 What: /sys/bus/dsa/devices/dsa<m>/max_transfer_size 55 56 Date: Oct 25, 2019 ··· 125 122 Description: The maximum number of read buffers that may be in use at 126 123 one time by operations that access low bandwidth memory in the 127 124 device. See DSA spec v1.2 9.2.8 GENCFG on Global Read Buffer Limit. 125 125 + It's not visible when the device does not support Read Buffer 126 126 + allocation control. 128 127 129 128 What: /sys/bus/dsa/devices/dsa<m>/cmd_status 130 129 Date: Aug 28, 2020 ··· 210 205 Contact: dmaengine@vger.kernel.org 211 206 Description: The max batch size for this workqueue. Cannot exceed device 212 207 max batch size. Configurable parameter. 208 208 + It's not visible when the device does not support batch. 213 209 214 210 What: /sys/bus/dsa/devices/wq<m>.<n>/ats_disable 215 211 Date: Nov 13, 2020 ··· 256 250 Contact: dmaengine@vger.kernel.org 257 251 Description: Enable the use of global read buffer limit for the group. See DSA 258 252 spec v1.2 9.2.18 GRPCFG Use Global Read Buffer Limit. 253 253 + It's not visible when the device does not support Read Buffer 254 254 + allocation control. 259 255 260 256 What: /sys/bus/dsa/devices/group<m>.<n>/read_buffers_allowed 261 257 Date: Dec 10, 2021 ··· 266 258 Description: Indicates max number of read buffers that may be in use at one time 267 259 by all engines in the group. See DSA spec v1.2 9.2.18 GRPCFG Read 268 260 Buffers Allowed. 261 261 + It's not visible when the device does not support Read Buffer 262 262 + allocation control. 269 263 270 264 What: /sys/bus/dsa/devices/group<m>.<n>/read_buffers_reserved 271 265 Date: Dec 10, 2021 ··· 276 266 Description: Indicates the number of Read Buffers reserved for the use of 277 267 engines in the group. See DSA spec v1.2 9.2.18 GRPCFG Read Buffers 278 268 Reserved. 269 269 + It's not visible when the device does not support Read Buffer 270 270 + allocation control. 279 271 280 272 What: /sys/bus/dsa/devices/group<m>.<n>/desc_progress_limit 281 273 Date: Sept 14, 2022

+1

Documentation/devicetree/bindings/dma/ingenic,dma.yaml

reviewed

··· 18 18 - enum: 19 19 - ingenic,jz4740-dma 20 20 - ingenic,jz4725b-dma 21 21 + - ingenic,jz4755-dma 21 22 - ingenic,jz4760-dma 22 23 - ingenic,jz4760-bdma 23 24 - ingenic,jz4760-mdma

+6 -1

Documentation/devicetree/bindings/dma/nvidia,tegra186-gpc-dma.yaml

reviewed

··· 39 39 Should contain all of the per-channel DMA interrupts in 40 40 ascending order with respect to the DMA channel index. 41 41 minItems: 1 42 42 - maxItems: 31 42 42 + maxItems: 32 43 43 44 44 resets: 45 45 maxItems: 1 ··· 52 52 53 53 dma-coherent: true 54 54 55 55 + dma-channel-mask: 56 56 + maxItems: 1 57 57 + 55 58 required: 56 59 - compatible 57 60 - reg ··· 63 60 - reset-names 64 61 - "#dma-cells" 65 62 - iommus 63 63 + - dma-channel-mask 66 64 67 65 additionalProperties: false 68 66 ··· 112 108 #dma-cells = <1>; 113 109 iommus = <&smmu TEGRA186_SID_GPCDMA_0>; 114 110 dma-coherent; 111 111 + dma-channel-mask = <0xfffffffe>; 115 112 }; 116 113 ...

+18 -8

Documentation/devicetree/bindings/dma/qcom,gpi.yaml

reviewed

··· 18 18 19 19 properties: 20 20 compatible: 21 21 - enum: 22 22 - - qcom,sc7280-gpi-dma 23 23 - - qcom,sdm845-gpi-dma 24 24 - - qcom,sm6350-gpi-dma 25 25 - - qcom,sm8150-gpi-dma 26 26 - - qcom,sm8250-gpi-dma 27 27 - - qcom,sm8350-gpi-dma 28 28 - - qcom,sm8450-gpi-dma 21 21 + oneOf: 22 22 + - enum: 23 23 + - qcom,sdm845-gpi-dma 24 24 + - qcom,sm6350-gpi-dma 25 25 + - items: 26 26 + - enum: 27 27 + - qcom,sc7280-gpi-dma 28 28 + - qcom,sm6115-gpi-dma 29 29 + - qcom,sm6375-gpi-dma 30 30 + - qcom,sm8350-gpi-dma 31 31 + - qcom,sm8450-gpi-dma 32 32 + - const: qcom,sm6350-gpi-dma 33 33 + - items: 34 34 + - enum: 35 35 + - qcom,sdm670-gpi-dma 36 36 + - qcom,sm8150-gpi-dma 37 37 + - qcom,sm8250-gpi-dma 38 38 + - const: qcom,sdm845-gpi-dma 29 39 30 40 reg: 31 41 maxItems: 1

+1

Documentation/driver-api/driver-model/devres.rst

reviewed

··· 450 450 451 451 SLAVE DMA ENGINE 452 452 devm_acpi_dma_controller_register() 453 453 + devm_acpi_dma_controller_free() 453 454 454 455 SPI 455 456 devm_spi_alloc_master()

-6

MAINTAINERS

reviewed

··· 10460 10460 F: drivers/iommu/intel/ 10461 10461 F: include/linux/intel-svm.h 10462 10462 10463 10463 - INTEL IOP-ADMA DMA DRIVER 10464 10464 - R: Dan Williams <dan.j.williams@intel.com> 10465 10465 - S: Odd fixes 10466 10466 - F: drivers/dma/iop-adma.c 10467 10467 - 10468 10463 INTEL IPU3 CSI-2 CIO2 DRIVER 10469 10464 M: Yong Zhi <yong.zhi@intel.com> 10470 10465 M: Sakari Ailus <sakari.ailus@linux.intel.com> ··· 13624 13629 S: Supported 13625 13630 F: Documentation/devicetree/bindings/dma/atmel-dma.txt 13626 13631 F: drivers/dma/at_hdmac.c 13627 13627 - F: drivers/dma/at_hdmac_regs.h 13628 13632 F: drivers/dma/at_xdmac.c 13629 13633 F: include/dt-bindings/dma/at91.h 13630 13634

+1 -8

drivers/dma/Kconfig

reviewed

··· 97 97 tristate "Atmel AHB DMA support" 98 98 depends on ARCH_AT91 99 99 select DMA_ENGINE 100 100 + select DMA_VIRTUAL_CHANNELS 100 101 help 101 102 Support the Atmel AHB DMA controller. 102 103 ··· 357 356 Say Y here if you have such a chipset. 358 357 359 358 If unsure, say N. 360 360 - 361 361 - config INTEL_IOP_ADMA 362 362 - tristate "Intel IOP32x ADMA support" 363 363 - depends on ARCH_IOP32X || COMPILE_TEST 364 364 - select DMA_ENGINE 365 365 - select ASYNC_TX_ENABLE_CHANNEL_SWITCH 366 366 - help 367 367 - Enable support for the Intel(R) IOP Series RAID engines. 368 359 369 360 config K3_DMA 370 361 tristate "Hisilicon K3 DMA support"

-1

drivers/dma/Makefile

reviewed

··· 44 44 obj-$(CONFIG_INTEL_IDMA64) += idma64.o 45 45 obj-$(CONFIG_INTEL_IOATDMA) += ioat/ 46 46 obj-y += idxd/ 47 47 - obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o 48 47 obj-$(CONFIG_K3_DMA) += k3dma.o 49 48 obj-$(CONFIG_LPC18XX_DMAMUX) += lpc18xx-dmamux.o 50 49 obj-$(CONFIG_MILBEAUT_HDMAC) += milbeaut-hdmac.o

+101 -1

drivers/dma/apple-admac.c

reviewed

··· 21 21 #define NCHANNELS_MAX 64 22 22 #define IRQ_NOUTPUTS 4 23 23 24 24 + /* 25 25 + * For allocation purposes we split the cache 26 26 + * memory into blocks of fixed size (given in bytes). 27 27 + */ 28 28 + #define SRAM_BLOCK 2048 29 29 + 24 30 #define RING_WRITE_SLOT GENMASK(1, 0) 25 31 #define RING_READ_SLOT GENMASK(5, 4) 26 32 #define RING_FULL BIT(9) ··· 42 36 #define REG_TX_STOP 0x0004 43 37 #define REG_RX_START 0x0008 44 38 #define REG_RX_STOP 0x000c 39 39 + #define REG_IMPRINT 0x0090 40 40 + #define REG_TX_SRAM_SIZE 0x0094 41 41 + #define REG_RX_SRAM_SIZE 0x0098 45 42 46 43 #define REG_CHAN_CTL(ch) (0x8000 + (ch) * 0x200) 47 44 #define REG_CHAN_CTL_RST_RINGS BIT(0) ··· 62 53 #define BUS_WIDTH_FRAME_2_WORDS 0x10 63 54 #define BUS_WIDTH_FRAME_4_WORDS 0x20 64 55 65 65 - #define CHAN_BUFSIZE 0x8000 56 56 + #define REG_CHAN_SRAM_CARVEOUT(ch) (0x8050 + (ch) * 0x200) 57 57 + #define CHAN_SRAM_CARVEOUT_SIZE GENMASK(31, 16) 58 58 + #define CHAN_SRAM_CARVEOUT_BASE GENMASK(15, 0) 66 59 67 60 #define REG_CHAN_FIFOCTL(ch) (0x8054 + (ch) * 0x200) 68 61 #define CHAN_FIFOCTL_LIMIT GENMASK(31, 16) ··· 87 76 struct dma_chan chan; 88 77 struct tasklet_struct tasklet; 89 78 79 79 + u32 carveout; 80 80 + 90 81 spinlock_t lock; 91 82 struct admac_tx *current_tx; 92 83 int nperiod_acks; ··· 105 92 struct list_head to_free; 106 93 }; 107 94 95 95 + struct admac_sram { 96 96 + u32 size; 97 97 + /* 98 98 + * SRAM_CARVEOUT has 16-bit fields, so the SRAM cannot be larger than 99 99 + * 64K and a 32-bit bitfield over 2K blocks covers it. 100 100 + */ 101 101 + u32 allocated; 102 102 + }; 103 103 + 108 104 struct admac_data { 109 105 struct dma_device dma; 110 106 struct device *dev; 111 107 __iomem void *base; 112 108 struct reset_control *rstc; 109 109 + 110 110 + struct mutex cache_alloc_lock; 111 111 + struct admac_sram txcache, rxcache; 113 112 114 113 int irq; 115 114 int irq_index; ··· 142 117 143 118 struct list_head node; 144 119 }; 120 120 + 121 121 + static int admac_alloc_sram_carveout(struct admac_data *ad, 122 122 + enum dma_transfer_direction dir, 123 123 + u32 *out) 124 124 + { 125 125 + struct admac_sram *sram; 126 126 + int i, ret = 0, nblocks; 127 127 + 128 128 + if (dir == DMA_MEM_TO_DEV) 129 129 + sram = &ad->txcache; 130 130 + else 131 131 + sram = &ad->rxcache; 132 132 + 133 133 + mutex_lock(&ad->cache_alloc_lock); 134 134 + 135 135 + nblocks = sram->size / SRAM_BLOCK; 136 136 + for (i = 0; i < nblocks; i++) 137 137 + if (!(sram->allocated & BIT(i))) 138 138 + break; 139 139 + 140 140 + if (i < nblocks) { 141 141 + *out = FIELD_PREP(CHAN_SRAM_CARVEOUT_BASE, i * SRAM_BLOCK) | 142 142 + FIELD_PREP(CHAN_SRAM_CARVEOUT_SIZE, SRAM_BLOCK); 143 143 + sram->allocated |= BIT(i); 144 144 + } else { 145 145 + ret = -EBUSY; 146 146 + } 147 147 + 148 148 + mutex_unlock(&ad->cache_alloc_lock); 149 149 + 150 150 + return ret; 151 151 + } 152 152 + 153 153 + static void admac_free_sram_carveout(struct admac_data *ad, 154 154 + enum dma_transfer_direction dir, 155 155 + u32 carveout) 156 156 + { 157 157 + struct admac_sram *sram; 158 158 + u32 base = FIELD_GET(CHAN_SRAM_CARVEOUT_BASE, carveout); 159 159 + int i; 160 160 + 161 161 + if (dir == DMA_MEM_TO_DEV) 162 162 + sram = &ad->txcache; 163 163 + else 164 164 + sram = &ad->rxcache; 165 165 + 166 166 + if (WARN_ON(base >= sram->size)) 167 167 + return; 168 168 + 169 169 + mutex_lock(&ad->cache_alloc_lock); 170 170 + i = base / SRAM_BLOCK; 171 171 + sram->allocated &= ~BIT(i); 172 172 + mutex_unlock(&ad->cache_alloc_lock); 173 173 + } 145 174 146 175 static void admac_modify(struct admac_data *ad, int reg, u32 mask, u32 val) 147 176 { ··· 545 466 static int admac_alloc_chan_resources(struct dma_chan *chan) 546 467 { 547 468 struct admac_chan *adchan = to_admac_chan(chan); 469 469 + struct admac_data *ad = adchan->host; 470 470 + int ret; 548 471 549 472 dma_cookie_init(&adchan->chan); 473 473 + ret = admac_alloc_sram_carveout(ad, admac_chan_direction(adchan->no), 474 474 + &adchan->carveout); 475 475 + if (ret < 0) 476 476 + return ret; 477 477 + 478 478 + writel_relaxed(adchan->carveout, 479 479 + ad->base + REG_CHAN_SRAM_CARVEOUT(adchan->no)); 550 480 return 0; 551 481 } 552 482 553 483 static void admac_free_chan_resources(struct dma_chan *chan) 554 484 { 485 485 + struct admac_chan *adchan = to_admac_chan(chan); 486 486 + 555 487 admac_terminate_all(chan); 556 488 admac_synchronize(chan); 489 489 + admac_free_sram_carveout(adchan->host, admac_chan_direction(adchan->no), 490 490 + adchan->carveout); 557 491 } 558 492 559 493 static struct dma_chan *admac_dma_of_xlate(struct of_phandle_args *dma_spec, ··· 804 712 platform_set_drvdata(pdev, ad); 805 713 ad->dev = &pdev->dev; 806 714 ad->nchannels = nchannels; 715 715 + mutex_init(&ad->cache_alloc_lock); 807 716 808 717 /* 809 718 * The controller has 4 IRQ outputs. Try them all until ··· 893 800 dev_err_probe(&pdev->dev, err, "failed to register with OF\n"); 894 801 goto free_irq; 895 802 } 803 803 + 804 804 + ad->txcache.size = readl_relaxed(ad->base + REG_TX_SRAM_SIZE); 805 805 + ad->rxcache.size = readl_relaxed(ad->base + REG_RX_SRAM_SIZE); 806 806 + 807 807 + dev_info(&pdev->dev, "Audio DMA Controller\n"); 808 808 + dev_info(&pdev->dev, "imprint %x TX cache %u RX cache %u\n", 809 809 + readl_relaxed(ad->base + REG_IMPRINT), ad->txcache.size, ad->rxcache.size); 896 810 897 811 return 0; 898 812

+983 -871

drivers/dma/at_hdmac.c

reviewed

··· 3 3 * Driver for the Atmel AHB DMA Controller (aka HDMA or DMAC on AT91 systems) 4 4 * 5 5 * Copyright (C) 2008 Atmel Corporation 6 6 + * Copyright (C) 2022 Microchip Technology, Inc. and its subsidiaries 6 7 * 7 8 * This supports the Atmel AHB DMA Controller found in several Atmel SoCs. 8 9 * The only Atmel DMA Controller that is not covered by this driver is the one ··· 11 10 */ 12 11 13 12 #include <dt-bindings/dma/at91.h> 13 13 + #include <linux/bitfield.h> 14 14 #include <linux/clk.h> 15 15 #include <linux/dmaengine.h> 16 16 - #include <linux/dma-mapping.h> 17 16 #include <linux/dmapool.h> 17 17 + #include <linux/dma-mapping.h> 18 18 #include <linux/interrupt.h> 19 19 #include <linux/module.h> 20 20 - #include <linux/platform_device.h> 21 21 - #include <linux/slab.h> 22 20 #include <linux/of.h> 21 21 + #include <linux/overflow.h> 23 22 #include <linux/of_device.h> 24 23 #include <linux/of_dma.h> 24 24 + #include <linux/platform_device.h> 25 25 + #include <linux/slab.h> 25 26 26 26 - #include "at_hdmac_regs.h" 27 27 #include "dmaengine.h" 28 28 + #include "virt-dma.h" 28 29 29 30 /* 30 31 * Glossary ··· 37 34 * atc_ / atchan : ATmel DMA Channel entity related 38 35 */ 39 36 40 40 - #define ATC_DEFAULT_CFG (ATC_FIFOCFG_HALFFIFO) 41 41 - #define ATC_DEFAULT_CTRLB (ATC_SIF(AT_DMA_MEM_IF) \ 42 42 - |ATC_DIF(AT_DMA_MEM_IF)) 37 37 + #define AT_DMA_MAX_NR_CHANNELS 8 38 38 + 39 39 + /* Global Configuration Register */ 40 40 + #define AT_DMA_GCFG 0x00 41 41 + #define AT_DMA_IF_BIGEND(i) BIT((i)) /* AHB-Lite Interface i in Big-endian mode */ 42 42 + #define AT_DMA_ARB_CFG BIT(4) /* Arbiter mode. */ 43 43 + 44 44 + /* Controller Enable Register */ 45 45 + #define AT_DMA_EN 0x04 46 46 + #define AT_DMA_ENABLE BIT(0) 47 47 + 48 48 + /* Software Single Request Register */ 49 49 + #define AT_DMA_SREQ 0x08 50 50 + #define AT_DMA_SSREQ(x) BIT((x) << 1) /* Request a source single transfer on channel x */ 51 51 + #define AT_DMA_DSREQ(x) BIT(1 + ((x) << 1)) /* Request a destination single transfer on channel x */ 52 52 + 53 53 + /* Software Chunk Transfer Request Register */ 54 54 + #define AT_DMA_CREQ 0x0c 55 55 + #define AT_DMA_SCREQ(x) BIT((x) << 1) /* Request a source chunk transfer on channel x */ 56 56 + #define AT_DMA_DCREQ(x) BIT(1 + ((x) << 1)) /* Request a destination chunk transfer on channel x */ 57 57 + 58 58 + /* Software Last Transfer Flag Register */ 59 59 + #define AT_DMA_LAST 0x10 60 60 + #define AT_DMA_SLAST(x) BIT((x) << 1) /* This src rq is last tx of buffer on channel x */ 61 61 + #define AT_DMA_DLAST(x) BIT(1 + ((x) << 1)) /* This dst rq is last tx of buffer on channel x */ 62 62 + 63 63 + /* Request Synchronization Register */ 64 64 + #define AT_DMA_SYNC 0x14 65 65 + #define AT_DMA_SYR(h) BIT((h)) /* Synchronize handshake line h */ 66 66 + 67 67 + /* Error, Chained Buffer transfer completed and Buffer transfer completed Interrupt registers */ 68 68 + #define AT_DMA_EBCIER 0x18 /* Enable register */ 69 69 + #define AT_DMA_EBCIDR 0x1c /* Disable register */ 70 70 + #define AT_DMA_EBCIMR 0x20 /* Mask Register */ 71 71 + #define AT_DMA_EBCISR 0x24 /* Status Register */ 72 72 + #define AT_DMA_CBTC_OFFSET 8 73 73 + #define AT_DMA_ERR_OFFSET 16 74 74 + #define AT_DMA_BTC(x) BIT((x)) 75 75 + #define AT_DMA_CBTC(x) BIT(AT_DMA_CBTC_OFFSET + (x)) 76 76 + #define AT_DMA_ERR(x) BIT(AT_DMA_ERR_OFFSET + (x)) 77 77 + 78 78 + /* Channel Handler Enable Register */ 79 79 + #define AT_DMA_CHER 0x28 80 80 + #define AT_DMA_ENA(x) BIT((x)) 81 81 + #define AT_DMA_SUSP(x) BIT(8 + (x)) 82 82 + #define AT_DMA_KEEP(x) BIT(24 + (x)) 83 83 + 84 84 + /* Channel Handler Disable Register */ 85 85 + #define AT_DMA_CHDR 0x2c 86 86 + #define AT_DMA_DIS(x) BIT(x) 87 87 + #define AT_DMA_RES(x) BIT(8 + (x)) 88 88 + 89 89 + /* Channel Handler Status Register */ 90 90 + #define AT_DMA_CHSR 0x30 91 91 + #define AT_DMA_EMPT(x) BIT(16 + (x)) 92 92 + #define AT_DMA_STAL(x) BIT(24 + (x)) 93 93 + 94 94 + /* Channel registers base address */ 95 95 + #define AT_DMA_CH_REGS_BASE 0x3c 96 96 + #define ch_regs(x) (AT_DMA_CH_REGS_BASE + (x) * 0x28) /* Channel x base addr */ 97 97 + 98 98 + /* Hardware register offset for each channel */ 99 99 + #define ATC_SADDR_OFFSET 0x00 /* Source Address Register */ 100 100 + #define ATC_DADDR_OFFSET 0x04 /* Destination Address Register */ 101 101 + #define ATC_DSCR_OFFSET 0x08 /* Descriptor Address Register */ 102 102 + #define ATC_CTRLA_OFFSET 0x0c /* Control A Register */ 103 103 + #define ATC_CTRLB_OFFSET 0x10 /* Control B Register */ 104 104 + #define ATC_CFG_OFFSET 0x14 /* Configuration Register */ 105 105 + #define ATC_SPIP_OFFSET 0x18 /* Src PIP Configuration Register */ 106 106 + #define ATC_DPIP_OFFSET 0x1c /* Dst PIP Configuration Register */ 107 107 + 108 108 + 109 109 + /* Bitfield definitions */ 110 110 + 111 111 + /* Bitfields in DSCR */ 112 112 + #define ATC_DSCR_IF GENMASK(1, 0) /* Dsc feched via AHB-Lite Interface */ 113 113 + 114 114 + /* Bitfields in CTRLA */ 115 115 + #define ATC_BTSIZE_MAX GENMASK(15, 0) /* Maximum Buffer Transfer Size */ 116 116 + #define ATC_BTSIZE GENMASK(15, 0) /* Buffer Transfer Size */ 117 117 + #define ATC_SCSIZE GENMASK(18, 16) /* Source Chunk Transfer Size */ 118 118 + #define ATC_DCSIZE GENMASK(22, 20) /* Destination Chunk Transfer Size */ 119 119 + #define ATC_SRC_WIDTH GENMASK(25, 24) /* Source Single Transfer Size */ 120 120 + #define ATC_DST_WIDTH GENMASK(29, 28) /* Destination Single Transfer Size */ 121 121 + #define ATC_DONE BIT(31) /* Tx Done (only written back in descriptor) */ 122 122 + 123 123 + /* Bitfields in CTRLB */ 124 124 + #define ATC_SIF GENMASK(1, 0) /* Src tx done via AHB-Lite Interface i */ 125 125 + #define ATC_DIF GENMASK(5, 4) /* Dst tx done via AHB-Lite Interface i */ 126 126 + #define AT_DMA_MEM_IF 0x0 /* interface 0 as memory interface */ 127 127 + #define AT_DMA_PER_IF 0x1 /* interface 1 as peripheral interface */ 128 128 + #define ATC_SRC_PIP BIT(8) /* Source Picture-in-Picture enabled */ 129 129 + #define ATC_DST_PIP BIT(12) /* Destination Picture-in-Picture enabled */ 130 130 + #define ATC_SRC_DSCR_DIS BIT(16) /* Src Descriptor fetch disable */ 131 131 + #define ATC_DST_DSCR_DIS BIT(20) /* Dst Descriptor fetch disable */ 132 132 + #define ATC_FC GENMASK(22, 21) /* Choose Flow Controller */ 133 133 + #define ATC_FC_MEM2MEM 0x0 /* Mem-to-Mem (DMA) */ 134 134 + #define ATC_FC_MEM2PER 0x1 /* Mem-to-Periph (DMA) */ 135 135 + #define ATC_FC_PER2MEM 0x2 /* Periph-to-Mem (DMA) */ 136 136 + #define ATC_FC_PER2PER 0x3 /* Periph-to-Periph (DMA) */ 137 137 + #define ATC_FC_PER2MEM_PER 0x4 /* Periph-to-Mem (Peripheral) */ 138 138 + #define ATC_FC_MEM2PER_PER 0x5 /* Mem-to-Periph (Peripheral) */ 139 139 + #define ATC_FC_PER2PER_SRCPER 0x6 /* Periph-to-Periph (Src Peripheral) */ 140 140 + #define ATC_FC_PER2PER_DSTPER 0x7 /* Periph-to-Periph (Dst Peripheral) */ 141 141 + #define ATC_SRC_ADDR_MODE GENMASK(25, 24) 142 142 + #define ATC_SRC_ADDR_MODE_INCR 0x0 /* Incrementing Mode */ 143 143 + #define ATC_SRC_ADDR_MODE_DECR 0x1 /* Decrementing Mode */ 144 144 + #define ATC_SRC_ADDR_MODE_FIXED 0x2 /* Fixed Mode */ 145 145 + #define ATC_DST_ADDR_MODE GENMASK(29, 28) 146 146 + #define ATC_DST_ADDR_MODE_INCR 0x0 /* Incrementing Mode */ 147 147 + #define ATC_DST_ADDR_MODE_DECR 0x1 /* Decrementing Mode */ 148 148 + #define ATC_DST_ADDR_MODE_FIXED 0x2 /* Fixed Mode */ 149 149 + #define ATC_IEN BIT(30) /* BTC interrupt enable (active low) */ 150 150 + #define ATC_AUTO BIT(31) /* Auto multiple buffer tx enable */ 151 151 + 152 152 + /* Bitfields in CFG */ 153 153 + #define ATC_PER_MSB(h) ((0x30U & (h)) >> 4) /* Extract most significant bits of a handshaking identifier */ 154 154 + 155 155 + #define ATC_SRC_PER GENMASK(3, 0) /* Channel src rq associated with periph handshaking ifc h */ 156 156 + #define ATC_DST_PER GENMASK(7, 4) /* Channel dst rq associated with periph handshaking ifc h */ 157 157 + #define ATC_SRC_REP BIT(8) /* Source Replay Mod */ 158 158 + #define ATC_SRC_H2SEL BIT(9) /* Source Handshaking Mod */ 159 159 + #define ATC_SRC_PER_MSB GENMASK(11, 10) /* Channel src rq (most significant bits) */ 160 160 + #define ATC_DST_REP BIT(12) /* Destination Replay Mod */ 161 161 + #define ATC_DST_H2SEL BIT(13) /* Destination Handshaking Mod */ 162 162 + #define ATC_DST_PER_MSB GENMASK(15, 14) /* Channel dst rq (most significant bits) */ 163 163 + #define ATC_SOD BIT(16) /* Stop On Done */ 164 164 + #define ATC_LOCK_IF BIT(20) /* Interface Lock */ 165 165 + #define ATC_LOCK_B BIT(21) /* AHB Bus Lock */ 166 166 + #define ATC_LOCK_IF_L BIT(22) /* Master Interface Arbiter Lock */ 167 167 + #define ATC_AHB_PROT GENMASK(26, 24) /* AHB Protection */ 168 168 + #define ATC_FIFOCFG GENMASK(29, 28) /* FIFO Request Configuration */ 169 169 + #define ATC_FIFOCFG_LARGESTBURST 0x0 170 170 + #define ATC_FIFOCFG_HALFFIFO 0x1 171 171 + #define ATC_FIFOCFG_ENOUGHSPACE 0x2 172 172 + 173 173 + /* Bitfields in SPIP */ 174 174 + #define ATC_SPIP_HOLE GENMASK(15, 0) 175 175 + #define ATC_SPIP_BOUNDARY GENMASK(25, 16) 176 176 + 177 177 + /* Bitfields in DPIP */ 178 178 + #define ATC_DPIP_HOLE GENMASK(15, 0) 179 179 + #define ATC_DPIP_BOUNDARY GENMASK(25, 16) 180 180 + 181 181 + #define ATC_SRC_PER_ID(id) (FIELD_PREP(ATC_SRC_PER_MSB, (id)) | \ 182 182 + FIELD_PREP(ATC_SRC_PER, (id))) 183 183 + #define ATC_DST_PER_ID(id) (FIELD_PREP(ATC_DST_PER_MSB, (id)) | \ 184 184 + FIELD_PREP(ATC_DST_PER, (id))) 185 185 + 186 186 + 187 187 + 188 188 + /*-- descriptors -----------------------------------------------------*/ 189 189 + 190 190 + /* LLI == Linked List Item; aka DMA buffer descriptor */ 191 191 + struct at_lli { 192 192 + /* values that are not changed by hardware */ 193 193 + u32 saddr; 194 194 + u32 daddr; 195 195 + /* value that may get written back: */ 196 196 + u32 ctrla; 197 197 + /* more values that are not changed by hardware */ 198 198 + u32 ctrlb; 199 199 + u32 dscr; /* chain to next lli */ 200 200 + }; 201 201 + 202 202 + /** 203 203 + * struct atdma_sg - atdma scatter gather entry 204 204 + * @len: length of the current Linked List Item. 205 205 + * @lli: linked list item that is passed to the DMA controller 206 206 + * @lli_phys: physical address of the LLI. 207 207 + */ 208 208 + struct atdma_sg { 209 209 + unsigned int len; 210 210 + struct at_lli *lli; 211 211 + dma_addr_t lli_phys; 212 212 + }; 213 213 + 214 214 + /** 215 215 + * struct at_desc - software descriptor 216 216 + * @vd: pointer to the virtual dma descriptor. 217 217 + * @atchan: pointer to the atmel dma channel. 218 218 + * @total_len: total transaction byte count 219 219 + * @sg_len: number of sg entries. 220 220 + * @sg: array of sgs. 221 221 + */ 222 222 + struct at_desc { 223 223 + struct virt_dma_desc vd; 224 224 + struct at_dma_chan *atchan; 225 225 + size_t total_len; 226 226 + unsigned int sglen; 227 227 + /* Interleaved data */ 228 228 + size_t boundary; 229 229 + size_t dst_hole; 230 230 + size_t src_hole; 231 231 + 232 232 + /* Memset temporary buffer */ 233 233 + bool memset_buffer; 234 234 + dma_addr_t memset_paddr; 235 235 + int *memset_vaddr; 236 236 + struct atdma_sg sg[]; 237 237 + }; 238 238 + 239 239 + /*-- Channels --------------------------------------------------------*/ 240 240 + 241 241 + /** 242 242 + * atc_status - information bits stored in channel status flag 243 243 + * 244 244 + * Manipulated with atomic operations. 245 245 + */ 246 246 + enum atc_status { 247 247 + ATC_IS_PAUSED = 1, 248 248 + ATC_IS_CYCLIC = 24, 249 249 + }; 250 250 + 251 251 + /** 252 252 + * struct at_dma_chan - internal representation of an Atmel HDMAC channel 253 253 + * @vc: virtual dma channel entry. 254 254 + * @atdma: pointer to the driver data. 255 255 + * @ch_regs: memory mapped register base 256 256 + * @mask: channel index in a mask 257 257 + * @per_if: peripheral interface 258 258 + * @mem_if: memory interface 259 259 + * @status: transmit status information from irq/prep* functions 260 260 + * to tasklet (use atomic operations) 261 261 + * @save_cfg: configuration register that is saved on suspend/resume cycle 262 262 + * @save_dscr: for cyclic operations, preserve next descriptor address in 263 263 + * the cyclic list on suspend/resume cycle 264 264 + * @dma_sconfig: configuration for slave transfers, passed via 265 265 + * .device_config 266 266 + * @desc: pointer to the atmel dma descriptor. 267 267 + */ 268 268 + struct at_dma_chan { 269 269 + struct virt_dma_chan vc; 270 270 + struct at_dma *atdma; 271 271 + void __iomem *ch_regs; 272 272 + u8 mask; 273 273 + u8 per_if; 274 274 + u8 mem_if; 275 275 + unsigned long status; 276 276 + u32 save_cfg; 277 277 + u32 save_dscr; 278 278 + struct dma_slave_config dma_sconfig; 279 279 + bool cyclic; 280 280 + struct at_desc *desc; 281 281 + }; 282 282 + 283 283 + #define channel_readl(atchan, name) \ 284 284 + __raw_readl((atchan)->ch_regs + ATC_##name##_OFFSET) 285 285 + 286 286 + #define channel_writel(atchan, name, val) \ 287 287 + __raw_writel((val), (atchan)->ch_regs + ATC_##name##_OFFSET) 288 288 + 289 289 + /* 290 290 + * Fix sconfig's burst size according to at_hdmac. We need to convert them as: 291 291 + * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3, 32 -> 4, 64 -> 5, 128 -> 6, 256 -> 7. 292 292 + * 293 293 + * This can be done by finding most significant bit set. 294 294 + */ 295 295 + static inline void convert_burst(u32 *maxburst) 296 296 + { 297 297 + if (*maxburst > 1) 298 298 + *maxburst = fls(*maxburst) - 2; 299 299 + else 300 300 + *maxburst = 0; 301 301 + } 302 302 + 303 303 + /* 304 304 + * Fix sconfig's bus width according to at_hdmac. 305 305 + * 1 byte -> 0, 2 bytes -> 1, 4 bytes -> 2. 306 306 + */ 307 307 + static inline u8 convert_buswidth(enum dma_slave_buswidth addr_width) 308 308 + { 309 309 + switch (addr_width) { 310 310 + case DMA_SLAVE_BUSWIDTH_2_BYTES: 311 311 + return 1; 312 312 + case DMA_SLAVE_BUSWIDTH_4_BYTES: 313 313 + return 2; 314 314 + default: 315 315 + /* For 1 byte width or fallback */ 316 316 + return 0; 317 317 + } 318 318 + } 319 319 + 320 320 + /*-- Controller ------------------------------------------------------*/ 321 321 + 322 322 + /** 323 323 + * struct at_dma - internal representation of an Atmel HDMA Controller 324 324 + * @dma_device: dmaengine dma_device object members 325 325 + * @atdma_devtype: identifier of DMA controller compatibility 326 326 + * @ch_regs: memory mapped register base 327 327 + * @clk: dma controller clock 328 328 + * @save_imr: interrupt mask register that is saved on suspend/resume cycle 329 329 + * @all_chan_mask: all channels availlable in a mask 330 330 + * @lli_pool: hw lli table 331 331 + * @chan: channels table to store at_dma_chan structures 332 332 + */ 333 333 + struct at_dma { 334 334 + struct dma_device dma_device; 335 335 + void __iomem *regs; 336 336 + struct clk *clk; 337 337 + u32 save_imr; 338 338 + 339 339 + u8 all_chan_mask; 340 340 + 341 341 + struct dma_pool *lli_pool; 342 342 + struct dma_pool *memset_pool; 343 343 + /* AT THE END channels table */ 344 344 + struct at_dma_chan chan[]; 345 345 + }; 346 346 + 347 347 + #define dma_readl(atdma, name) \ 348 348 + __raw_readl((atdma)->regs + AT_DMA_##name) 349 349 + #define dma_writel(atdma, name, val) \ 350 350 + __raw_writel((val), (atdma)->regs + AT_DMA_##name) 351 351 + 352 352 + static inline struct at_desc *to_atdma_desc(struct dma_async_tx_descriptor *t) 353 353 + { 354 354 + return container_of(t, struct at_desc, vd.tx); 355 355 + } 356 356 + 357 357 + static inline struct at_dma_chan *to_at_dma_chan(struct dma_chan *chan) 358 358 + { 359 359 + return container_of(chan, struct at_dma_chan, vc.chan); 360 360 + } 361 361 + 362 362 + static inline struct at_dma *to_at_dma(struct dma_device *ddev) 363 363 + { 364 364 + return container_of(ddev, struct at_dma, dma_device); 365 365 + } 366 366 + 367 367 + 368 368 + /*-- Helper functions ------------------------------------------------*/ 369 369 + 370 370 + static struct device *chan2dev(struct dma_chan *chan) 371 371 + { 372 372 + return &chan->dev->device; 373 373 + } 374 374 + 375 375 + #if defined(VERBOSE_DEBUG) 376 376 + static void vdbg_dump_regs(struct at_dma_chan *atchan) 377 377 + { 378 378 + struct at_dma *atdma = to_at_dma(atchan->vc.chan.device); 379 379 + 380 380 + dev_err(chan2dev(&atchan->vc.chan), 381 381 + " channel %d : imr = 0x%x, chsr = 0x%x\n", 382 382 + atchan->vc.chan.chan_id, 383 383 + dma_readl(atdma, EBCIMR), 384 384 + dma_readl(atdma, CHSR)); 385 385 + 386 386 + dev_err(chan2dev(&atchan->vc.chan), 387 387 + " channel: s0x%x d0x%x ctrl0x%x:0x%x cfg0x%x l0x%x\n", 388 388 + channel_readl(atchan, SADDR), 389 389 + channel_readl(atchan, DADDR), 390 390 + channel_readl(atchan, CTRLA), 391 391 + channel_readl(atchan, CTRLB), 392 392 + channel_readl(atchan, CFG), 393 393 + channel_readl(atchan, DSCR)); 394 394 + } 395 395 + #else 396 396 + static void vdbg_dump_regs(struct at_dma_chan *atchan) {} 397 397 + #endif 398 398 + 399 399 + static void atc_dump_lli(struct at_dma_chan *atchan, struct at_lli *lli) 400 400 + { 401 401 + dev_crit(chan2dev(&atchan->vc.chan), 402 402 + "desc: s%pad d%pad ctrl0x%x:0x%x l%pad\n", 403 403 + &lli->saddr, &lli->daddr, 404 404 + lli->ctrla, lli->ctrlb, &lli->dscr); 405 405 + } 406 406 + 407 407 + 408 408 + static void atc_setup_irq(struct at_dma *atdma, int chan_id, int on) 409 409 + { 410 410 + u32 ebci; 411 411 + 412 412 + /* enable interrupts on buffer transfer completion & error */ 413 413 + ebci = AT_DMA_BTC(chan_id) 414 414 + | AT_DMA_ERR(chan_id); 415 415 + if (on) 416 416 + dma_writel(atdma, EBCIER, ebci); 417 417 + else 418 418 + dma_writel(atdma, EBCIDR, ebci); 419 419 + } 420 420 + 421 421 + static void atc_enable_chan_irq(struct at_dma *atdma, int chan_id) 422 422 + { 423 423 + atc_setup_irq(atdma, chan_id, 1); 424 424 + } 425 425 + 426 426 + static void atc_disable_chan_irq(struct at_dma *atdma, int chan_id) 427 427 + { 428 428 + atc_setup_irq(atdma, chan_id, 0); 429 429 + } 430 430 + 431 431 + 432 432 + /** 433 433 + * atc_chan_is_enabled - test if given channel is enabled 434 434 + * @atchan: channel we want to test status 435 435 + */ 436 436 + static inline int atc_chan_is_enabled(struct at_dma_chan *atchan) 437 437 + { 438 438 + struct at_dma *atdma = to_at_dma(atchan->vc.chan.device); 439 439 + 440 440 + return !!(dma_readl(atdma, CHSR) & atchan->mask); 441 441 + } 442 442 + 443 443 + /** 444 444 + * atc_chan_is_paused - test channel pause/resume status 445 445 + * @atchan: channel we want to test status 446 446 + */ 447 447 + static inline int atc_chan_is_paused(struct at_dma_chan *atchan) 448 448 + { 449 449 + return test_bit(ATC_IS_PAUSED, &atchan->status); 450 450 + } 451 451 + 452 452 + /** 453 453 + * atc_chan_is_cyclic - test if given channel has cyclic property set 454 454 + * @atchan: channel we want to test status 455 455 + */ 456 456 + static inline int atc_chan_is_cyclic(struct at_dma_chan *atchan) 457 457 + { 458 458 + return test_bit(ATC_IS_CYCLIC, &atchan->status); 459 459 + } 460 460 + 461 461 + /** 462 462 + * set_lli_eol - set end-of-link to descriptor so it will end transfer 463 463 + * @desc: descriptor, signle or at the end of a chain, to end chain on 464 464 + * @i: index of the atmel scatter gather entry that is at the end of the chain. 465 465 + */ 466 466 + static void set_lli_eol(struct at_desc *desc, unsigned int i) 467 467 + { 468 468 + u32 ctrlb = desc->sg[i].lli->ctrlb; 469 469 + 470 470 + ctrlb &= ~ATC_IEN; 471 471 + ctrlb |= ATC_SRC_DSCR_DIS | ATC_DST_DSCR_DIS; 472 472 + 473 473 + desc->sg[i].lli->ctrlb = ctrlb; 474 474 + desc->sg[i].lli->dscr = 0; 475 475 + } 476 476 + 477 477 + #define ATC_DEFAULT_CFG FIELD_PREP(ATC_FIFOCFG, ATC_FIFOCFG_HALFFIFO) 478 478 + #define ATC_DEFAULT_CTRLB (FIELD_PREP(ATC_SIF, AT_DMA_MEM_IF) | \ 479 479 + FIELD_PREP(ATC_DIF, AT_DMA_MEM_IF)) 43 480 #define ATC_DMA_BUSWIDTHS\ 44 481 (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) |\ 45 482 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |\ ··· 517 74 u32 cfg; 518 75 }; 519 76 520 520 - /* prototypes */ 521 521 - static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx); 522 522 - static void atc_issue_pending(struct dma_chan *chan); 523 523 - 524 524 - 525 525 - /*----------------------------------------------------------------------*/ 526 526 - 527 77 static inline unsigned int atc_get_xfer_width(dma_addr_t src, dma_addr_t dst, 528 78 size_t len) 529 79 { ··· 532 96 return width; 533 97 } 534 98 535 535 - static struct at_desc *atc_first_active(struct at_dma_chan *atchan) 99 99 + static void atdma_lli_chain(struct at_desc *desc, unsigned int i) 536 100 { 537 537 - return list_first_entry(&atchan->active_list, 538 538 - struct at_desc, desc_node); 539 539 - } 101 101 + struct atdma_sg *atdma_sg = &desc->sg[i]; 540 102 541 541 - static struct at_desc *atc_first_queued(struct at_dma_chan *atchan) 542 542 - { 543 543 - return list_first_entry(&atchan->queue, 544 544 - struct at_desc, desc_node); 545 545 - } 546 546 - 547 547 - /** 548 548 - * atc_alloc_descriptor - allocate and return an initialized descriptor 549 549 - * @chan: the channel to allocate descriptors for 550 550 - * @gfp_flags: GFP allocation flags 551 551 - * 552 552 - * Note: The ack-bit is positioned in the descriptor flag at creation time 553 553 - * to make initial allocation more convenient. This bit will be cleared 554 554 - * and control will be given to client at usage time (during 555 555 - * preparation functions). 556 556 - */ 557 557 - static struct at_desc *atc_alloc_descriptor(struct dma_chan *chan, 558 558 - gfp_t gfp_flags) 559 559 - { 560 560 - struct at_desc *desc = NULL; 561 561 - struct at_dma *atdma = to_at_dma(chan->device); 562 562 - dma_addr_t phys; 563 563 - 564 564 - desc = dma_pool_zalloc(atdma->dma_desc_pool, gfp_flags, &phys); 565 565 - if (desc) { 566 566 - INIT_LIST_HEAD(&desc->tx_list); 567 567 - dma_async_tx_descriptor_init(&desc->txd, chan); 568 568 - /* txd.flags will be overwritten in prep functions */ 569 569 - desc->txd.flags = DMA_CTRL_ACK; 570 570 - desc->txd.tx_submit = atc_tx_submit; 571 571 - desc->txd.phys = phys; 572 572 - } 573 573 - 574 574 - return desc; 575 575 - } 576 576 - 577 577 - /** 578 578 - * atc_desc_get - get an unused descriptor from free_list 579 579 - * @atchan: channel we want a new descriptor for 580 580 - */ 581 581 - static struct at_desc *atc_desc_get(struct at_dma_chan *atchan) 582 582 - { 583 583 - struct at_desc *desc, *_desc; 584 584 - struct at_desc *ret = NULL; 585 585 - unsigned long flags; 586 586 - unsigned int i = 0; 587 587 - 588 588 - spin_lock_irqsave(&atchan->lock, flags); 589 589 - list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) { 590 590 - i++; 591 591 - if (async_tx_test_ack(&desc->txd)) { 592 592 - list_del(&desc->desc_node); 593 593 - ret = desc; 594 594 - break; 595 595 - } 596 596 - dev_dbg(chan2dev(&atchan->chan_common), 597 597 - "desc %p not ACKed\n", desc); 598 598 - } 599 599 - spin_unlock_irqrestore(&atchan->lock, flags); 600 600 - dev_vdbg(chan2dev(&atchan->chan_common), 601 601 - "scanned %u descriptors on freelist\n", i); 602 602 - 603 603 - /* no more descriptor available in initial pool: create one more */ 604 604 - if (!ret) 605 605 - ret = atc_alloc_descriptor(&atchan->chan_common, GFP_NOWAIT); 606 606 - 607 607 - return ret; 608 608 - } 609 609 - 610 610 - /** 611 611 - * atc_desc_put - move a descriptor, including any children, to the free list 612 612 - * @atchan: channel we work on 613 613 - * @desc: descriptor, at the head of a chain, to move to free list 614 614 - */ 615 615 - static void atc_desc_put(struct at_dma_chan *atchan, struct at_desc *desc) 616 616 - { 617 617 - if (desc) { 618 618 - struct at_desc *child; 619 619 - unsigned long flags; 620 620 - 621 621 - spin_lock_irqsave(&atchan->lock, flags); 622 622 - list_for_each_entry(child, &desc->tx_list, desc_node) 623 623 - dev_vdbg(chan2dev(&atchan->chan_common), 624 624 - "moving child desc %p to freelist\n", 625 625 - child); 626 626 - list_splice_init(&desc->tx_list, &atchan->free_list); 627 627 - dev_vdbg(chan2dev(&atchan->chan_common), 628 628 - "moving desc %p to freelist\n", desc); 629 629 - list_add(&desc->desc_node, &atchan->free_list); 630 630 - spin_unlock_irqrestore(&atchan->lock, flags); 631 631 - } 632 632 - } 633 633 - 634 634 - /** 635 635 - * atc_desc_chain - build chain adding a descriptor 636 636 - * @first: address of first descriptor of the chain 637 637 - * @prev: address of previous descriptor of the chain 638 638 - * @desc: descriptor to queue 639 639 - * 640 640 - * Called from prep_* functions 641 641 - */ 642 642 - static void atc_desc_chain(struct at_desc **first, struct at_desc **prev, 643 643 - struct at_desc *desc) 644 644 - { 645 645 - if (!(*first)) { 646 646 - *first = desc; 647 647 - } else { 648 648 - /* inform the HW lli about chaining */ 649 649 - (*prev)->lli.dscr = desc->txd.phys; 650 650 - /* insert the link descriptor to the LD ring */ 651 651 - list_add_tail(&desc->desc_node, 652 652 - &(*first)->tx_list); 653 653 - } 654 654 - *prev = desc; 103 103 + if (i) 104 104 + desc->sg[i - 1].lli->dscr = atdma_sg->lli_phys; 655 105 } 656 106 657 107 /** 658 108 * atc_dostart - starts the DMA engine for real 659 109 * @atchan: the channel we want to start 660 660 - * @first: first descriptor in the list we want to begin with 661 661 - * 662 662 - * Called with atchan->lock held and bh disabled 663 110 */ 664 664 - static void atc_dostart(struct at_dma_chan *atchan, struct at_desc *first) 111 111 + static void atc_dostart(struct at_dma_chan *atchan) 665 112 { 666 666 - struct at_dma *atdma = to_at_dma(atchan->chan_common.device); 113 113 + struct virt_dma_desc *vd = vchan_next_desc(&atchan->vc); 114 114 + struct at_desc *desc; 667 115 668 668 - /* ASSERT: channel is idle */ 669 669 - if (atc_chan_is_enabled(atchan)) { 670 670 - dev_err(chan2dev(&atchan->chan_common), 671 671 - "BUG: Attempted to start non-idle channel\n"); 672 672 - dev_err(chan2dev(&atchan->chan_common), 673 673 - " channel: s0x%x d0x%x ctrl0x%x:0x%x l0x%x\n", 674 674 - channel_readl(atchan, SADDR), 675 675 - channel_readl(atchan, DADDR), 676 676 - channel_readl(atchan, CTRLA), 677 677 - channel_readl(atchan, CTRLB), 678 678 - channel_readl(atchan, DSCR)); 679 679 - 680 680 - /* The tasklet will hopefully advance the queue... */ 116 116 + if (!vd) { 117 117 + atchan->desc = NULL; 681 118 return; 682 119 } 683 120 684 121 vdbg_dump_regs(atchan); 685 122 123 123 + list_del(&vd->node); 124 124 + atchan->desc = desc = to_atdma_desc(&vd->tx); 125 125 + 686 126 channel_writel(atchan, SADDR, 0); 687 127 channel_writel(atchan, DADDR, 0); 688 128 channel_writel(atchan, CTRLA, 0); 689 129 channel_writel(atchan, CTRLB, 0); 690 690 - channel_writel(atchan, DSCR, first->txd.phys); 691 691 - channel_writel(atchan, SPIP, ATC_SPIP_HOLE(first->src_hole) | 692 692 - ATC_SPIP_BOUNDARY(first->boundary)); 693 693 - channel_writel(atchan, DPIP, ATC_DPIP_HOLE(first->dst_hole) | 694 694 - ATC_DPIP_BOUNDARY(first->boundary)); 130 130 + channel_writel(atchan, DSCR, desc->sg[0].lli_phys); 131 131 + channel_writel(atchan, SPIP, 132 132 + FIELD_PREP(ATC_SPIP_HOLE, desc->src_hole) | 133 133 + FIELD_PREP(ATC_SPIP_BOUNDARY, desc->boundary)); 134 134 + channel_writel(atchan, DPIP, 135 135 + FIELD_PREP(ATC_DPIP_HOLE, desc->dst_hole) | 136 136 + FIELD_PREP(ATC_DPIP_BOUNDARY, desc->boundary)); 137 137 + 695 138 /* Don't allow CPU to reorder channel enable. */ 696 139 wmb(); 697 697 - dma_writel(atdma, CHER, atchan->mask); 140 140 + dma_writel(atchan->atdma, CHER, atchan->mask); 698 141 699 142 vdbg_dump_regs(atchan); 700 143 } 701 144 702 702 - /* 703 703 - * atc_get_desc_by_cookie - get the descriptor of a cookie 704 704 - * @atchan: the DMA channel 705 705 - * @cookie: the cookie to get the descriptor for 706 706 - */ 707 707 - static struct at_desc *atc_get_desc_by_cookie(struct at_dma_chan *atchan, 708 708 - dma_cookie_t cookie) 145 145 + static void atdma_desc_free(struct virt_dma_desc *vd) 709 146 { 710 710 - struct at_desc *desc, *_desc; 147 147 + struct at_dma *atdma = to_at_dma(vd->tx.chan->device); 148 148 + struct at_desc *desc = to_atdma_desc(&vd->tx); 149 149 + unsigned int i; 711 150 712 712 - list_for_each_entry_safe(desc, _desc, &atchan->queue, desc_node) { 713 713 - if (desc->txd.cookie == cookie) 714 714 - return desc; 151 151 + for (i = 0; i < desc->sglen; i++) { 152 152 + if (desc->sg[i].lli) 153 153 + dma_pool_free(atdma->lli_pool, desc->sg[i].lli, 154 154 + desc->sg[i].lli_phys); 715 155 } 716 156 717 717 - list_for_each_entry_safe(desc, _desc, &atchan->active_list, desc_node) { 718 718 - if (desc->txd.cookie == cookie) 719 719 - return desc; 157 157 + /* If the transfer was a memset, free our temporary buffer */ 158 158 + if (desc->memset_buffer) { 159 159 + dma_pool_free(atdma->memset_pool, desc->memset_vaddr, 160 160 + desc->memset_paddr); 161 161 + desc->memset_buffer = false; 720 162 } 721 163 722 722 - return NULL; 164 164 + kfree(desc); 723 165 } 724 166 725 167 /** ··· 607 293 * @current_len: the number of bytes left before reading CTRLA 608 294 * @ctrla: the value of CTRLA 609 295 */ 610 610 - static inline int atc_calc_bytes_left(int current_len, u32 ctrla) 296 296 + static inline u32 atc_calc_bytes_left(u32 current_len, u32 ctrla) 611 297 { 612 612 - u32 btsize = (ctrla & ATC_BTSIZE_MAX); 613 613 - u32 src_width = ATC_REG_TO_SRC_WIDTH(ctrla); 298 298 + u32 btsize = FIELD_GET(ATC_BTSIZE, ctrla); 299 299 + u32 src_width = FIELD_GET(ATC_SRC_WIDTH, ctrla); 614 300 615 301 /* 616 302 * According to the datasheet, when reading the Control A Register ··· 622 308 } 623 309 624 310 /** 625 625 - * atc_get_bytes_left - get the number of bytes residue for a cookie 626 626 - * @chan: DMA channel 627 627 - * @cookie: transaction identifier to check status of 311 311 + * atc_get_llis_residue - Get residue for a hardware linked list transfer 312 312 + * 313 313 + * Calculate the residue by removing the length of the Linked List Item (LLI) 314 314 + * already transferred from the total length. To get the current LLI we can use 315 315 + * the value of the channel's DSCR register and compare it against the DSCR 316 316 + * value of each LLI. 317 317 + * 318 318 + * The CTRLA register provides us with the amount of data already read from the 319 319 + * source for the LLI. So we can compute a more accurate residue by also 320 320 + * removing the number of bytes corresponding to this amount of data. 321 321 + * 322 322 + * However, the DSCR and CTRLA registers cannot be read both atomically. Hence a 323 323 + * race condition may occur: the first read register may refer to one LLI 324 324 + * whereas the second read may refer to a later LLI in the list because of the 325 325 + * DMA transfer progression inbetween the two reads. 326 326 + * 327 327 + * One solution could have been to pause the DMA transfer, read the DSCR and 328 328 + * CTRLA then resume the DMA transfer. Nonetheless, this approach presents some 329 329 + * drawbacks: 330 330 + * - If the DMA transfer is paused, RX overruns or TX underruns are more likey 331 331 + * to occur depending on the system latency. Taking the USART driver as an 332 332 + * example, it uses a cyclic DMA transfer to read data from the Receive 333 333 + * Holding Register (RHR) to avoid RX overruns since the RHR is not protected 334 334 + * by any FIFO on most Atmel SoCs. So pausing the DMA transfer to compute the 335 335 + * residue would break the USART driver design. 336 336 + * - The atc_pause() function masks interrupts but we'd rather avoid to do so 337 337 + * for system latency purpose. 338 338 + * 339 339 + * Then we'd rather use another solution: the DSCR is read a first time, the 340 340 + * CTRLA is read in turn, next the DSCR is read a second time. If the two 341 341 + * consecutive read values of the DSCR are the same then we assume both refers 342 342 + * to the very same LLI as well as the CTRLA value read inbetween does. For 343 343 + * cyclic tranfers, the assumption is that a full loop is "not so fast". If the 344 344 + * two DSCR values are different, we read again the CTRLA then the DSCR till two 345 345 + * consecutive read values from DSCR are equal or till the maximum trials is 346 346 + * reach. This algorithm is very unlikely not to find a stable value for DSCR. 347 347 + * @atchan: pointer to an atmel hdmac channel. 348 348 + * @desc: pointer to the descriptor for which the residue is calculated. 349 349 + * @residue: residue to be set to dma_tx_state. 350 350 + * Returns 0 on success, -errno otherwise. 628 351 */ 629 629 - static int atc_get_bytes_left(struct dma_chan *chan, dma_cookie_t cookie) 352 352 + static int atc_get_llis_residue(struct at_dma_chan *atchan, 353 353 + struct at_desc *desc, u32 *residue) 630 354 { 631 631 - struct at_dma_chan *atchan = to_at_dma_chan(chan); 632 632 - struct at_desc *desc_first = atc_first_active(atchan); 633 633 - struct at_desc *desc; 634 634 - int ret; 635 635 - u32 ctrla, dscr; 355 355 + u32 len, ctrla, dscr; 636 356 unsigned int i; 637 357 638 638 - /* 639 639 - * If the cookie doesn't match to the currently running transfer then 640 640 - * we can return the total length of the associated DMA transfer, 641 641 - * because it is still queued. 642 642 - */ 643 643 - desc = atc_get_desc_by_cookie(atchan, cookie); 644 644 - if (desc == NULL) 645 645 - return -EINVAL; 646 646 - else if (desc != desc_first) 647 647 - return desc->total_len; 358 358 + len = desc->total_len; 359 359 + dscr = channel_readl(atchan, DSCR); 360 360 + rmb(); /* ensure DSCR is read before CTRLA */ 361 361 + ctrla = channel_readl(atchan, CTRLA); 362 362 + for (i = 0; i < ATC_MAX_DSCR_TRIALS; ++i) { 363 363 + u32 new_dscr; 648 364 649 649 - /* cookie matches to the currently running transfer */ 650 650 - ret = desc_first->total_len; 651 651 - 652 652 - if (desc_first->lli.dscr) { 653 653 - /* hardware linked list transfer */ 365 365 + rmb(); /* ensure DSCR is read after CTRLA */ 366 366 + new_dscr = channel_readl(atchan, DSCR); 654 367 655 368 /* 656 656 - * Calculate the residue by removing the length of the child 657 657 - * descriptors already transferred from the total length. 658 658 - * To get the current child descriptor we can use the value of 659 659 - * the channel's DSCR register and compare it against the value 660 660 - * of the hardware linked list structure of each child 661 661 - * descriptor. 662 662 - * 663 663 - * The CTRLA register provides us with the amount of data 664 664 - * already read from the source for the current child 665 665 - * descriptor. So we can compute a more accurate residue by also 666 666 - * removing the number of bytes corresponding to this amount of 667 667 - * data. 668 668 - * 669 669 - * However, the DSCR and CTRLA registers cannot be read both 670 670 - * atomically. Hence a race condition may occur: the first read 671 671 - * register may refer to one child descriptor whereas the second 672 672 - * read may refer to a later child descriptor in the list 673 673 - * because of the DMA transfer progression inbetween the two 674 674 - * reads. 675 675 - * 676 676 - * One solution could have been to pause the DMA transfer, read 677 677 - * the DSCR and CTRLA then resume the DMA transfer. Nonetheless, 678 678 - * this approach presents some drawbacks: 679 679 - * - If the DMA transfer is paused, RX overruns or TX underruns 680 680 - * are more likey to occur depending on the system latency. 681 681 - * Taking the USART driver as an example, it uses a cyclic DMA 682 682 - * transfer to read data from the Receive Holding Register 683 683 - * (RHR) to avoid RX overruns since the RHR is not protected 684 684 - * by any FIFO on most Atmel SoCs. So pausing the DMA transfer 685 685 - * to compute the residue would break the USART driver design. 686 686 - * - The atc_pause() function masks interrupts but we'd rather 687 687 - * avoid to do so for system latency purpose. 688 688 - * 689 689 - * Then we'd rather use another solution: the DSCR is read a 690 690 - * first time, the CTRLA is read in turn, next the DSCR is read 691 691 - * a second time. If the two consecutive read values of the DSCR 692 692 - * are the same then we assume both refers to the very same 693 693 - * child descriptor as well as the CTRLA value read inbetween 694 694 - * does. For cyclic tranfers, the assumption is that a full loop 695 695 - * is "not so fast". 696 696 - * If the two DSCR values are different, we read again the CTRLA 697 697 - * then the DSCR till two consecutive read values from DSCR are 698 698 - * equal or till the maxium trials is reach. 699 699 - * This algorithm is very unlikely not to find a stable value for 700 700 - * DSCR. 369 369 + * If the DSCR register value has not changed inside the DMA 370 370 + * controller since the previous read, we assume that both the 371 371 + * dscr and ctrla values refers to the very same descriptor. 701 372 */ 373 373 + if (likely(new_dscr == dscr)) 374 374 + break; 702 375 703 703 - dscr = channel_readl(atchan, DSCR); 376 376 + /* 377 377 + * DSCR has changed inside the DMA controller, so the previouly 378 378 + * read value of CTRLA may refer to an already processed 379 379 + * descriptor hence could be outdated. We need to update ctrla 380 380 + * to match the current descriptor. 381 381 + */ 382 382 + dscr = new_dscr; 704 383 rmb(); /* ensure DSCR is read before CTRLA */ 705 384 ctrla = channel_readl(atchan, CTRLA); 706 706 - for (i = 0; i < ATC_MAX_DSCR_TRIALS; ++i) { 707 707 - u32 new_dscr; 385 385 + } 386 386 + if (unlikely(i == ATC_MAX_DSCR_TRIALS)) 387 387 + return -ETIMEDOUT; 708 388 709 709 - rmb(); /* ensure DSCR is read after CTRLA */ 710 710 - new_dscr = channel_readl(atchan, DSCR); 389 389 + /* For the first descriptor we can be more accurate. */ 390 390 + if (desc->sg[0].lli->dscr == dscr) { 391 391 + *residue = atc_calc_bytes_left(len, ctrla); 392 392 + return 0; 393 393 + } 394 394 + len -= desc->sg[0].len; 711 395 712 712 - /* 713 713 - * If the DSCR register value has not changed inside the 714 714 - * DMA controller since the previous read, we assume 715 715 - * that both the dscr and ctrla values refers to the 716 716 - * very same descriptor. 717 717 - */ 718 718 - if (likely(new_dscr == dscr)) 719 719 - break; 720 720 - 721 721 - /* 722 722 - * DSCR has changed inside the DMA controller, so the 723 723 - * previouly read value of CTRLA may refer to an already 724 724 - * processed descriptor hence could be outdated. 725 725 - * We need to update ctrla to match the current 726 726 - * descriptor. 727 727 - */ 728 728 - dscr = new_dscr; 729 729 - rmb(); /* ensure DSCR is read before CTRLA */ 730 730 - ctrla = channel_readl(atchan, CTRLA); 731 731 - } 732 732 - if (unlikely(i == ATC_MAX_DSCR_TRIALS)) 733 733 - return -ETIMEDOUT; 734 734 - 735 735 - /* for the first descriptor we can be more accurate */ 736 736 - if (desc_first->lli.dscr == dscr) 737 737 - return atc_calc_bytes_left(ret, ctrla); 738 738 - 739 739 - ret -= desc_first->len; 740 740 - list_for_each_entry(desc, &desc_first->tx_list, desc_node) { 741 741 - if (desc->lli.dscr == dscr) 742 742 - break; 743 743 - 744 744 - ret -= desc->len; 745 745 - } 746 746 - 747 747 - /* 748 748 - * For the current descriptor in the chain we can calculate 749 749 - * the remaining bytes using the channel's register. 750 750 - */ 751 751 - ret = atc_calc_bytes_left(ret, ctrla); 752 752 - } else { 753 753 - /* single transfer */ 754 754 - ctrla = channel_readl(atchan, CTRLA); 755 755 - ret = atc_calc_bytes_left(ret, ctrla); 396 396 + for (i = 1; i < desc->sglen; i++) { 397 397 + if (desc->sg[i].lli && desc->sg[i].lli->dscr == dscr) 398 398 + break; 399 399 + len -= desc->sg[i].len; 756 400 } 757 401 758 758 - return ret; 402 402 + /* 403 403 + * For the current LLI in the chain we can calculate the remaining bytes 404 404 + * using the channel's CTRLA register. 405 405 + */ 406 406 + *residue = atc_calc_bytes_left(len, ctrla); 407 407 + return 0; 408 408 + 759 409 } 760 410 761 411 /** 762 762 - * atc_chain_complete - finish work for one transaction chain 763 763 - * @atchan: channel we work on 764 764 - * @desc: descriptor at the head of the chain we want do complete 412 412 + * atc_get_residue - get the number of bytes residue for a cookie. 413 413 + * The residue is passed by address and updated on success. 414 414 + * @chan: DMA channel 415 415 + * @cookie: transaction identifier to check status of 416 416 + * @residue: residue to be updated. 417 417 + * Return 0 on success, -errono otherwise. 765 418 */ 766 766 - static void 767 767 - atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) 419 419 + static int atc_get_residue(struct dma_chan *chan, dma_cookie_t cookie, 420 420 + u32 *residue) 768 421 { 769 769 - struct dma_async_tx_descriptor *txd = &desc->txd; 770 770 - struct at_dma *atdma = to_at_dma(atchan->chan_common.device); 771 771 - unsigned long flags; 422 422 + struct at_dma_chan *atchan = to_at_dma_chan(chan); 423 423 + struct virt_dma_desc *vd; 424 424 + struct at_desc *desc = NULL; 425 425 + u32 len, ctrla; 772 426 773 773 - dev_vdbg(chan2dev(&atchan->chan_common), 774 774 - "descriptor %u complete\n", txd->cookie); 427 427 + vd = vchan_find_desc(&atchan->vc, cookie); 428 428 + if (vd) 429 429 + desc = to_atdma_desc(&vd->tx); 430 430 + else if (atchan->desc && atchan->desc->vd.tx.cookie == cookie) 431 431 + desc = atchan->desc; 775 432 776 776 - spin_lock_irqsave(&atchan->lock, flags); 433 433 + if (!desc) 434 434 + return -EINVAL; 777 435 778 778 - /* mark the descriptor as complete for non cyclic cases only */ 779 779 - if (!atc_chan_is_cyclic(atchan)) 780 780 - dma_cookie_complete(txd); 436 436 + if (desc->sg[0].lli->dscr) 437 437 + /* hardware linked list transfer */ 438 438 + return atc_get_llis_residue(atchan, desc, residue); 781 439 782 782 - spin_unlock_irqrestore(&atchan->lock, flags); 783 783 - 784 784 - dma_descriptor_unmap(txd); 785 785 - /* for cyclic transfers, 786 786 - * no need to replay callback function while stopping */ 787 787 - if (!atc_chan_is_cyclic(atchan)) 788 788 - dmaengine_desc_get_callback_invoke(txd, NULL); 789 789 - 790 790 - dma_run_dependencies(txd); 791 791 - 792 792 - spin_lock_irqsave(&atchan->lock, flags); 793 793 - /* move children to free_list */ 794 794 - list_splice_init(&desc->tx_list, &atchan->free_list); 795 795 - /* add myself to free_list */ 796 796 - list_add(&desc->desc_node, &atchan->free_list); 797 797 - spin_unlock_irqrestore(&atchan->lock, flags); 798 798 - 799 799 - /* If the transfer was a memset, free our temporary buffer */ 800 800 - if (desc->memset_buffer) { 801 801 - dma_pool_free(atdma->memset_pool, desc->memset_vaddr, 802 802 - desc->memset_paddr); 803 803 - desc->memset_buffer = false; 804 804 - } 440 440 + /* single transfer */ 441 441 + len = desc->total_len; 442 442 + ctrla = channel_readl(atchan, CTRLA); 443 443 + *residue = atc_calc_bytes_left(len, ctrla); 444 444 + return 0; 805 445 } 806 806 - 807 807 - /** 808 808 - * atc_advance_work - at the end of a transaction, move forward 809 809 - * @atchan: channel where the transaction ended 810 810 - */ 811 811 - static void atc_advance_work(struct at_dma_chan *atchan) 812 812 - { 813 813 - struct at_desc *desc; 814 814 - unsigned long flags; 815 815 - 816 816 - dev_vdbg(chan2dev(&atchan->chan_common), "advance_work\n"); 817 817 - 818 818 - spin_lock_irqsave(&atchan->lock, flags); 819 819 - if (atc_chan_is_enabled(atchan) || list_empty(&atchan->active_list)) 820 820 - return spin_unlock_irqrestore(&atchan->lock, flags); 821 821 - 822 822 - desc = atc_first_active(atchan); 823 823 - /* Remove the transfer node from the active list. */ 824 824 - list_del_init(&desc->desc_node); 825 825 - spin_unlock_irqrestore(&atchan->lock, flags); 826 826 - atc_chain_complete(atchan, desc); 827 827 - 828 828 - /* advance work */ 829 829 - spin_lock_irqsave(&atchan->lock, flags); 830 830 - if (!list_empty(&atchan->active_list)) { 831 831 - desc = atc_first_queued(atchan); 832 832 - list_move_tail(&desc->desc_node, &atchan->active_list); 833 833 - atc_dostart(atchan, desc); 834 834 - } 835 835 - spin_unlock_irqrestore(&atchan->lock, flags); 836 836 - } 837 837 - 838 446 839 447 /** 840 448 * atc_handle_error - handle errors reported by DMA controller 841 841 - * @atchan: channel where error occurs 449 449 + * @atchan: channel where error occurs. 450 450 + * @i: channel index 842 451 */ 843 843 - static void atc_handle_error(struct at_dma_chan *atchan) 452 452 + static void atc_handle_error(struct at_dma_chan *atchan, unsigned int i) 844 453 { 845 845 - struct at_desc *bad_desc; 846 846 - struct at_desc *desc; 847 847 - struct at_desc *child; 848 848 - unsigned long flags; 454 454 + struct at_desc *desc = atchan->desc; 849 455 850 850 - spin_lock_irqsave(&atchan->lock, flags); 851 851 - /* 852 852 - * The descriptor currently at the head of the active list is 853 853 - * broked. Since we don't have any way to report errors, we'll 854 854 - * just have to scream loudly and try to carry on. 855 855 - */ 856 856 - bad_desc = atc_first_active(atchan); 857 857 - list_del_init(&bad_desc->desc_node); 858 858 - 859 859 - /* Try to restart the controller */ 860 860 - if (!list_empty(&atchan->active_list)) { 861 861 - desc = atc_first_queued(atchan); 862 862 - list_move_tail(&desc->desc_node, &atchan->active_list); 863 863 - atc_dostart(atchan, desc); 864 864 - } 456 456 + /* Disable channel on AHB error */ 457 457 + dma_writel(atchan->atdma, CHDR, AT_DMA_RES(i) | atchan->mask); 865 458 866 459 /* 867 460 * KERN_CRITICAL may seem harsh, but since this only happens ··· 777 556 * controller flagged an error instead of scribbling over 778 557 * random memory locations. 779 558 */ 780 780 - dev_crit(chan2dev(&atchan->chan_common), 781 781 - "Bad descriptor submitted for DMA!\n"); 782 782 - dev_crit(chan2dev(&atchan->chan_common), 783 783 - " cookie: %d\n", bad_desc->txd.cookie); 784 784 - atc_dump_lli(atchan, &bad_desc->lli); 785 785 - list_for_each_entry(child, &bad_desc->tx_list, desc_node) 786 786 - atc_dump_lli(atchan, &child->lli); 787 787 - 788 788 - spin_unlock_irqrestore(&atchan->lock, flags); 789 789 - 790 790 - /* Pretend the descriptor completed successfully */ 791 791 - atc_chain_complete(atchan, bad_desc); 559 559 + dev_crit(chan2dev(&atchan->vc.chan), "Bad descriptor submitted for DMA!\n"); 560 560 + dev_crit(chan2dev(&atchan->vc.chan), "cookie: %d\n", 561 561 + desc->vd.tx.cookie); 562 562 + for (i = 0; i < desc->sglen; i++) 563 563 + atc_dump_lli(atchan, desc->sg[i].lli); 792 564 } 793 565 794 794 - /** 795 795 - * atc_handle_cyclic - at the end of a period, run callback function 796 796 - * @atchan: channel used for cyclic operations 797 797 - */ 798 798 - static void atc_handle_cyclic(struct at_dma_chan *atchan) 566 566 + static void atdma_handle_chan_done(struct at_dma_chan *atchan, u32 pending, 567 567 + unsigned int i) 799 568 { 800 800 - struct at_desc *first = atc_first_active(atchan); 801 801 - struct dma_async_tx_descriptor *txd = &first->txd; 569 569 + struct at_desc *desc; 802 570 803 803 - dev_vdbg(chan2dev(&atchan->chan_common), 804 804 - "new cyclic period llp 0x%08x\n", 805 805 - channel_readl(atchan, DSCR)); 571 571 + spin_lock(&atchan->vc.lock); 572 572 + desc = atchan->desc; 806 573 807 807 - dmaengine_desc_get_callback_invoke(txd, NULL); 808 808 - } 574 574 + if (desc) { 575 575 + if (pending & AT_DMA_ERR(i)) { 576 576 + atc_handle_error(atchan, i); 577 577 + /* Pretend the descriptor completed successfully */ 578 578 + } 809 579 810 810 - /*-- IRQ & Tasklet ---------------------------------------------------*/ 811 811 - 812 812 - static void atc_tasklet(struct tasklet_struct *t) 813 813 - { 814 814 - struct at_dma_chan *atchan = from_tasklet(atchan, t, tasklet); 815 815 - 816 816 - if (test_and_clear_bit(ATC_IS_ERROR, &atchan->status)) 817 817 - return atc_handle_error(atchan); 818 818 - 819 819 - if (atc_chan_is_cyclic(atchan)) 820 820 - return atc_handle_cyclic(atchan); 821 821 - 822 822 - atc_advance_work(atchan); 580 580 + if (atc_chan_is_cyclic(atchan)) { 581 581 + vchan_cyclic_callback(&desc->vd); 582 582 + } else { 583 583 + vchan_cookie_complete(&desc->vd); 584 584 + atchan->desc = NULL; 585 585 + if (!(atc_chan_is_enabled(atchan))) 586 586 + atc_dostart(atchan); 587 587 + } 588 588 + } 589 589 + spin_unlock(&atchan->vc.lock); 823 590 } 824 591 825 592 static irqreturn_t at_dma_interrupt(int irq, void *dev_id) 826 593 { 827 827 - struct at_dma *atdma = (struct at_dma *)dev_id; 594 594 + struct at_dma *atdma = dev_id; 828 595 struct at_dma_chan *atchan; 829 596 int i; 830 597 u32 status, pending, imr; ··· 826 617 if (!pending) 827 618 break; 828 619 829 829 - dev_vdbg(atdma->dma_common.dev, 620 620 + dev_vdbg(atdma->dma_device.dev, 830 621 "interrupt: status = 0x%08x, 0x%08x, 0x%08x\n", 831 622 status, imr, pending); 832 623 833 833 - for (i = 0; i < atdma->dma_common.chancnt; i++) { 624 624 + for (i = 0; i < atdma->dma_device.chancnt; i++) { 834 625 atchan = &atdma->chan[i]; 835 835 - if (pending & (AT_DMA_BTC(i) | AT_DMA_ERR(i))) { 836 836 - if (pending & AT_DMA_ERR(i)) { 837 837 - /* Disable channel on AHB error */ 838 838 - dma_writel(atdma, CHDR, 839 839 - AT_DMA_RES(i) | atchan->mask); 840 840 - /* Give information to tasklet */ 841 841 - set_bit(ATC_IS_ERROR, &atchan->status); 842 842 - } 843 843 - tasklet_schedule(&atchan->tasklet); 844 844 - ret = IRQ_HANDLED; 845 845 - } 626 626 + if (!(pending & (AT_DMA_BTC(i) | AT_DMA_ERR(i)))) 627 627 + continue; 628 628 + atdma_handle_chan_done(atchan, pending, i); 629 629 + ret = IRQ_HANDLED; 846 630 } 847 631 848 632 } while (pending); ··· 843 641 return ret; 844 642 } 845 643 846 846 - 847 644 /*-- DMA Engine API --------------------------------------------------*/ 848 848 - 849 849 - /** 850 850 - * atc_tx_submit - set the prepared descriptor(s) to be executed by the engine 851 851 - * @tx: descriptor at the head of the transaction chain 852 852 - * 853 853 - * Queue chain if DMA engine is working already 854 854 - * 855 855 - * Cookie increment and adding to active_list or queue must be atomic 856 856 - */ 857 857 - static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx) 858 858 - { 859 859 - struct at_desc *desc = txd_to_at_desc(tx); 860 860 - struct at_dma_chan *atchan = to_at_dma_chan(tx->chan); 861 861 - dma_cookie_t cookie; 862 862 - unsigned long flags; 863 863 - 864 864 - spin_lock_irqsave(&atchan->lock, flags); 865 865 - cookie = dma_cookie_assign(tx); 866 866 - 867 867 - list_add_tail(&desc->desc_node, &atchan->queue); 868 868 - spin_unlock_irqrestore(&atchan->lock, flags); 869 869 - 870 870 - dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u\n", 871 871 - desc->txd.cookie); 872 872 - return cookie; 873 873 - } 874 874 - 875 645 /** 876 646 * atc_prep_dma_interleaved - prepare memory to memory interleaved operation 877 647 * @chan: the channel to prepare operation on ··· 855 681 struct dma_interleaved_template *xt, 856 682 unsigned long flags) 857 683 { 684 684 + struct at_dma *atdma = to_at_dma(chan->device); 858 685 struct at_dma_chan *atchan = to_at_dma_chan(chan); 859 686 struct data_chunk *first; 860 860 - struct at_desc *desc = NULL; 687 687 + struct atdma_sg *atdma_sg; 688 688 + struct at_desc *desc; 689 689 + struct at_lli *lli; 861 690 size_t xfer_count; 862 691 unsigned int dwidth; 863 692 u32 ctrla; ··· 899 722 len += chunk->size; 900 723 } 901 724 902 902 - dwidth = atc_get_xfer_width(xt->src_start, 903 903 - xt->dst_start, len); 725 725 + dwidth = atc_get_xfer_width(xt->src_start, xt->dst_start, len); 904 726 905 727 xfer_count = len >> dwidth; 906 728 if (xfer_count > ATC_BTSIZE_MAX) { ··· 907 731 return NULL; 908 732 } 909 733 910 910 - ctrla = ATC_SRC_WIDTH(dwidth) | 911 911 - ATC_DST_WIDTH(dwidth); 734 734 + ctrla = FIELD_PREP(ATC_SRC_WIDTH, dwidth) | 735 735 + FIELD_PREP(ATC_DST_WIDTH, dwidth); 912 736 913 913 - ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN 914 914 - | ATC_SRC_ADDR_MODE_INCR 915 915 - | ATC_DST_ADDR_MODE_INCR 916 916 - | ATC_SRC_PIP 917 917 - | ATC_DST_PIP 918 918 - | ATC_FC_MEM2MEM; 737 737 + ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN | 738 738 + FIELD_PREP(ATC_SRC_ADDR_MODE, ATC_SRC_ADDR_MODE_INCR) | 739 739 + FIELD_PREP(ATC_DST_ADDR_MODE, ATC_DST_ADDR_MODE_INCR) | 740 740 + ATC_SRC_PIP | ATC_DST_PIP | 741 741 + FIELD_PREP(ATC_FC, ATC_FC_MEM2MEM); 919 742 920 920 - /* create the transfer */ 921 921 - desc = atc_desc_get(atchan); 922 922 - if (!desc) { 923 923 - dev_err(chan2dev(chan), 924 924 - "%s: couldn't allocate our descriptor\n", __func__); 743 743 + desc = kzalloc(struct_size(desc, sg, 1), GFP_ATOMIC); 744 744 + if (!desc) 745 745 + return NULL; 746 746 + desc->sglen = 1; 747 747 + 748 748 + atdma_sg = desc->sg; 749 749 + atdma_sg->lli = dma_pool_alloc(atdma->lli_pool, GFP_NOWAIT, 750 750 + &atdma_sg->lli_phys); 751 751 + if (!atdma_sg->lli) { 752 752 + kfree(desc); 925 753 return NULL; 926 754 } 755 755 + lli = atdma_sg->lli; 927 756 928 928 - desc->lli.saddr = xt->src_start; 929 929 - desc->lli.daddr = xt->dst_start; 930 930 - desc->lli.ctrla = ctrla | xfer_count; 931 931 - desc->lli.ctrlb = ctrlb; 757 757 + lli->saddr = xt->src_start; 758 758 + lli->daddr = xt->dst_start; 759 759 + lli->ctrla = ctrla | xfer_count; 760 760 + lli->ctrlb = ctrlb; 932 761 933 762 desc->boundary = first->size >> dwidth; 934 763 desc->dst_hole = (dmaengine_get_dst_icg(xt, first) >> dwidth) + 1; 935 764 desc->src_hole = (dmaengine_get_src_icg(xt, first) >> dwidth) + 1; 936 765 937 937 - desc->txd.cookie = -EBUSY; 938 938 - desc->total_len = desc->len = len; 766 766 + atdma_sg->len = len; 767 767 + desc->total_len = len; 939 768 940 940 - /* set end-of-link to the last link descriptor of list*/ 941 941 - set_desc_eol(desc); 942 942 - 943 943 - desc->txd.flags = flags; /* client is in control of this ack */ 944 944 - 945 945 - return &desc->txd; 769 769 + set_lli_eol(desc, 0); 770 770 + return vchan_tx_prep(&atchan->vc, &desc->vd, flags); 946 771 } 947 772 948 773 /** ··· 958 781 atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 959 782 size_t len, unsigned long flags) 960 783 { 784 784 + struct at_dma *atdma = to_at_dma(chan->device); 961 785 struct at_dma_chan *atchan = to_at_dma_chan(chan); 962 786 struct at_desc *desc = NULL; 963 963 - struct at_desc *first = NULL; 964 964 - struct at_desc *prev = NULL; 965 787 size_t xfer_count; 966 788 size_t offset; 789 789 + size_t sg_len; 967 790 unsigned int src_width; 968 791 unsigned int dst_width; 792 792 + unsigned int i; 969 793 u32 ctrla; 970 794 u32 ctrlb; 971 795 972 972 - dev_vdbg(chan2dev(chan), "prep_dma_memcpy: d%pad s%pad l0x%zx f0x%lx\n", 973 973 - &dest, &src, len, flags); 796 796 + dev_dbg(chan2dev(chan), "prep_dma_memcpy: d%pad s%pad l0x%zx f0x%lx\n", 797 797 + &dest, &src, len, flags); 974 798 975 799 if (unlikely(!len)) { 976 976 - dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n"); 800 800 + dev_err(chan2dev(chan), "prep_dma_memcpy: length is zero!\n"); 977 801 return NULL; 978 802 } 979 803 980 980 - ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN 981 981 - | ATC_SRC_ADDR_MODE_INCR 982 982 - | ATC_DST_ADDR_MODE_INCR 983 983 - | ATC_FC_MEM2MEM; 804 804 + sg_len = DIV_ROUND_UP(len, ATC_BTSIZE_MAX); 805 805 + desc = kzalloc(struct_size(desc, sg, sg_len), GFP_ATOMIC); 806 806 + if (!desc) 807 807 + return NULL; 808 808 + desc->sglen = sg_len; 809 809 + 810 810 + ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN | 811 811 + FIELD_PREP(ATC_SRC_ADDR_MODE, ATC_SRC_ADDR_MODE_INCR) | 812 812 + FIELD_PREP(ATC_DST_ADDR_MODE, ATC_DST_ADDR_MODE_INCR) | 813 813 + FIELD_PREP(ATC_FC, ATC_FC_MEM2MEM); 984 814 985 815 /* 986 816 * We can be a lot more clever here, but this should take care ··· 995 811 */ 996 812 src_width = dst_width = atc_get_xfer_width(src, dest, len); 997 813 998 998 - ctrla = ATC_SRC_WIDTH(src_width) | 999 999 - ATC_DST_WIDTH(dst_width); 814 814 + ctrla = FIELD_PREP(ATC_SRC_WIDTH, src_width) | 815 815 + FIELD_PREP(ATC_DST_WIDTH, dst_width); 1000 816 1001 1001 - for (offset = 0; offset < len; offset += xfer_count << src_width) { 1002 1002 - xfer_count = min_t(size_t, (len - offset) >> src_width, 1003 1003 - ATC_BTSIZE_MAX); 817 817 + for (offset = 0, i = 0; offset < len; 818 818 + offset += xfer_count << src_width, i++) { 819 819 + struct atdma_sg *atdma_sg = &desc->sg[i]; 820 820 + struct at_lli *lli; 1004 821 1005 1005 - desc = atc_desc_get(atchan); 1006 1006 - if (!desc) 822 822 + atdma_sg->lli = dma_pool_alloc(atdma->lli_pool, GFP_NOWAIT, 823 823 + &atdma_sg->lli_phys); 824 824 + if (!atdma_sg->lli) 1007 825 goto err_desc_get; 826 826 + lli = atdma_sg->lli; 1008 827 1009 1009 - desc->lli.saddr = src + offset; 1010 1010 - desc->lli.daddr = dest + offset; 1011 1011 - desc->lli.ctrla = ctrla | xfer_count; 1012 1012 - desc->lli.ctrlb = ctrlb; 828 828 + xfer_count = min_t(size_t, (len - offset) >> src_width, 829 829 + ATC_BTSIZE_MAX); 1013 830 1014 1014 - desc->txd.cookie = 0; 1015 1015 - desc->len = xfer_count << src_width; 831 831 + lli->saddr = src + offset; 832 832 + lli->daddr = dest + offset; 833 833 + lli->ctrla = ctrla | xfer_count; 834 834 + lli->ctrlb = ctrlb; 1016 835 1017 1017 - atc_desc_chain(&first, &prev, desc); 836 836 + desc->sg[i].len = xfer_count << src_width; 837 837 + 838 838 + atdma_lli_chain(desc, i); 1018 839 } 1019 840 1020 1020 - /* First descriptor of the chain embedds additional information */ 1021 1021 - first->txd.cookie = -EBUSY; 1022 1022 - first->total_len = len; 841 841 + desc->total_len = len; 1023 842 1024 843 /* set end-of-link to the last link descriptor of list*/ 1025 1025 - set_desc_eol(desc); 844 844 + set_lli_eol(desc, i - 1); 1026 845 1027 1027 - first->txd.flags = flags; /* client is in control of this ack */ 1028 1028 - 1029 1029 - return &first->txd; 846 846 + return vchan_tx_prep(&atchan->vc, &desc->vd, flags); 1030 847 1031 848 err_desc_get: 1032 1032 - atc_desc_put(atchan, first); 849 849 + atdma_desc_free(&desc->vd); 1033 850 return NULL; 1034 851 } 1035 852 1036 1036 - static struct at_desc *atc_create_memset_desc(struct dma_chan *chan, 1037 1037 - dma_addr_t psrc, 1038 1038 - dma_addr_t pdst, 1039 1039 - size_t len) 853 853 + static int atdma_create_memset_lli(struct dma_chan *chan, 854 854 + struct atdma_sg *atdma_sg, 855 855 + dma_addr_t psrc, dma_addr_t pdst, size_t len) 1040 856 { 1041 1041 - struct at_dma_chan *atchan = to_at_dma_chan(chan); 1042 1042 - struct at_desc *desc; 857 857 + struct at_dma *atdma = to_at_dma(chan->device); 858 858 + struct at_lli *lli; 1043 859 size_t xfer_count; 1044 1044 - 1045 1045 - u32 ctrla = ATC_SRC_WIDTH(2) | ATC_DST_WIDTH(2); 860 860 + u32 ctrla = FIELD_PREP(ATC_SRC_WIDTH, 2) | FIELD_PREP(ATC_DST_WIDTH, 2); 1046 861 u32 ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN | 1047 1047 - ATC_SRC_ADDR_MODE_FIXED | 1048 1048 - ATC_DST_ADDR_MODE_INCR | 1049 1049 - ATC_FC_MEM2MEM; 862 862 + FIELD_PREP(ATC_SRC_ADDR_MODE, ATC_SRC_ADDR_MODE_FIXED) | 863 863 + FIELD_PREP(ATC_DST_ADDR_MODE, ATC_DST_ADDR_MODE_INCR) | 864 864 + FIELD_PREP(ATC_FC, ATC_FC_MEM2MEM); 1050 865 1051 866 xfer_count = len >> 2; 1052 867 if (xfer_count > ATC_BTSIZE_MAX) { 1053 1053 - dev_err(chan2dev(chan), "%s: buffer is too big\n", 1054 1054 - __func__); 1055 1055 - return NULL; 868 868 + dev_err(chan2dev(chan), "%s: buffer is too big\n", __func__); 869 869 + return -EINVAL; 1056 870 } 1057 871 1058 1058 - desc = atc_desc_get(atchan); 1059 1059 - if (!desc) { 1060 1060 - dev_err(chan2dev(chan), "%s: can't get a descriptor\n", 1061 1061 - __func__); 1062 1062 - return NULL; 1063 1063 - } 872 872 + atdma_sg->lli = dma_pool_alloc(atdma->lli_pool, GFP_NOWAIT, 873 873 + &atdma_sg->lli_phys); 874 874 + if (!atdma_sg->lli) 875 875 + return -ENOMEM; 876 876 + lli = atdma_sg->lli; 1064 877 1065 1065 - desc->lli.saddr = psrc; 1066 1066 - desc->lli.daddr = pdst; 1067 1067 - desc->lli.ctrla = ctrla | xfer_count; 1068 1068 - desc->lli.ctrlb = ctrlb; 878 878 + lli->saddr = psrc; 879 879 + lli->daddr = pdst; 880 880 + lli->ctrla = ctrla | xfer_count; 881 881 + lli->ctrlb = ctrlb; 1069 882 1070 1070 - desc->txd.cookie = 0; 1071 1071 - desc->len = len; 883 883 + atdma_sg->len = len; 1072 884 1073 1073 - return desc; 885 885 + return 0; 1074 886 } 1075 887 1076 888 /** ··· 1081 901 atc_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value, 1082 902 size_t len, unsigned long flags) 1083 903 { 904 904 + struct at_dma_chan *atchan = to_at_dma_chan(chan); 1084 905 struct at_dma *atdma = to_at_dma(chan->device); 1085 906 struct at_desc *desc; 1086 907 void __iomem *vaddr; 1087 908 dma_addr_t paddr; 1088 909 char fill_pattern; 910 910 + int ret; 1089 911 1090 912 dev_vdbg(chan2dev(chan), "%s: d%pad v0x%x l0x%zx f0x%lx\n", __func__, 1091 913 &dest, value, len, flags); ··· 1118 936 (fill_pattern << 8) | 1119 937 fill_pattern; 1120 938 1121 1121 - desc = atc_create_memset_desc(chan, paddr, dest, len); 1122 1122 - if (!desc) { 1123 1123 - dev_err(chan2dev(chan), "%s: couldn't get a descriptor\n", 1124 1124 - __func__); 939 939 + desc = kzalloc(struct_size(desc, sg, 1), GFP_ATOMIC); 940 940 + if (!desc) 1125 941 goto err_free_buffer; 1126 1126 - } 942 942 + desc->sglen = 1; 943 943 + 944 944 + ret = atdma_create_memset_lli(chan, desc->sg, paddr, dest, len); 945 945 + if (ret) 946 946 + goto err_free_desc; 1127 947 1128 948 desc->memset_paddr = paddr; 1129 949 desc->memset_vaddr = vaddr; 1130 950 desc->memset_buffer = true; 1131 951 1132 1132 - desc->txd.cookie = -EBUSY; 1133 952 desc->total_len = len; 1134 953 1135 954 /* set end-of-link on the descriptor */ 1136 1136 - set_desc_eol(desc); 955 955 + set_lli_eol(desc, 0); 1137 956 1138 1138 - desc->txd.flags = flags; 957 957 + return vchan_tx_prep(&atchan->vc, &desc->vd, flags); 1139 958 1140 1140 - return &desc->txd; 1141 1141 - 959 959 + err_free_desc: 960 960 + kfree(desc); 1142 961 err_free_buffer: 1143 962 dma_pool_free(atdma->memset_pool, vaddr, paddr); 1144 963 return NULL; ··· 1153 970 { 1154 971 struct at_dma_chan *atchan = to_at_dma_chan(chan); 1155 972 struct at_dma *atdma = to_at_dma(chan->device); 1156 1156 - struct at_desc *desc = NULL, *first = NULL, *prev = NULL; 973 973 + struct at_desc *desc; 1157 974 struct scatterlist *sg; 1158 975 void __iomem *vaddr; 1159 976 dma_addr_t paddr; 1160 977 size_t total_len = 0; 1161 978 int i; 979 979 + int ret; 1162 980 1163 981 dev_vdbg(chan2dev(chan), "%s: v0x%x l0x%zx f0x%lx\n", __func__, 1164 982 value, sg_len, flags); ··· 1178 994 } 1179 995 *(u32*)vaddr = value; 1180 996 997 997 + desc = kzalloc(struct_size(desc, sg, sg_len), GFP_ATOMIC); 998 998 + if (!desc) 999 999 + goto err_free_dma_buf; 1000 1000 + desc->sglen = sg_len; 1001 1001 + 1181 1002 for_each_sg(sgl, sg, sg_len, i) { 1182 1003 dma_addr_t dest = sg_dma_address(sg); 1183 1004 size_t len = sg_dma_len(sg); ··· 1193 1004 if (!is_dma_fill_aligned(chan->device, dest, 0, len)) { 1194 1005 dev_err(chan2dev(chan), "%s: buffer is not aligned\n", 1195 1006 __func__); 1196 1196 - goto err_put_desc; 1007 1007 + goto err_free_desc; 1197 1008 } 1198 1009 1199 1199 - desc = atc_create_memset_desc(chan, paddr, dest, len); 1200 1200 - if (!desc) 1201 1201 - goto err_put_desc; 1010 1010 + ret = atdma_create_memset_lli(chan, &desc->sg[i], paddr, dest, 1011 1011 + len); 1012 1012 + if (ret) 1013 1013 + goto err_free_desc; 1202 1014 1203 1203 - atc_desc_chain(&first, &prev, desc); 1204 1204 - 1015 1015 + atdma_lli_chain(desc, i); 1205 1016 total_len += len; 1206 1017 } 1207 1018 1208 1208 - /* 1209 1209 - * Only set the buffer pointers on the last descriptor to 1210 1210 - * avoid free'ing while we have our transfer still going 1211 1211 - */ 1212 1019 desc->memset_paddr = paddr; 1213 1020 desc->memset_vaddr = vaddr; 1214 1021 desc->memset_buffer = true; 1215 1022 1216 1216 - first->txd.cookie = -EBUSY; 1217 1217 - first->total_len = total_len; 1023 1023 + desc->total_len = total_len; 1218 1024 1219 1025 /* set end-of-link on the descriptor */ 1220 1220 - set_desc_eol(desc); 1026 1026 + set_lli_eol(desc, i - 1); 1221 1027 1222 1222 - first->txd.flags = flags; 1028 1028 + return vchan_tx_prep(&atchan->vc, &desc->vd, flags); 1223 1029 1224 1224 - return &first->txd; 1225 1225 - 1226 1226 - err_put_desc: 1227 1227 - atc_desc_put(atchan, first); 1030 1030 + err_free_desc: 1031 1031 + atdma_desc_free(&desc->vd); 1032 1032 + err_free_dma_buf: 1033 1033 + dma_pool_free(atdma->memset_pool, vaddr, paddr); 1228 1034 return NULL; 1229 1035 } 1230 1036 ··· 1237 1053 unsigned int sg_len, enum dma_transfer_direction direction, 1238 1054 unsigned long flags, void *context) 1239 1055 { 1056 1056 + struct at_dma *atdma = to_at_dma(chan->device); 1240 1057 struct at_dma_chan *atchan = to_at_dma_chan(chan); 1241 1058 struct at_dma_slave *atslave = chan->private; 1242 1059 struct dma_slave_config *sconfig = &atchan->dma_sconfig; 1243 1243 - struct at_desc *first = NULL; 1244 1244 - struct at_desc *prev = NULL; 1060 1060 + struct at_desc *desc; 1245 1061 u32 ctrla; 1246 1062 u32 ctrlb; 1247 1063 dma_addr_t reg; ··· 1261 1077 return NULL; 1262 1078 } 1263 1079 1264 1264 - ctrla = ATC_SCSIZE(sconfig->src_maxburst) 1265 1265 - | ATC_DCSIZE(sconfig->dst_maxburst); 1080 1080 + desc = kzalloc(struct_size(desc, sg, sg_len), GFP_ATOMIC); 1081 1081 + if (!desc) 1082 1082 + return NULL; 1083 1083 + desc->sglen = sg_len; 1084 1084 + 1085 1085 + ctrla = FIELD_PREP(ATC_SCSIZE, sconfig->src_maxburst) | 1086 1086 + FIELD_PREP(ATC_DCSIZE, sconfig->dst_maxburst); 1266 1087 ctrlb = ATC_IEN; 1267 1088 1268 1089 switch (direction) { 1269 1090 case DMA_MEM_TO_DEV: 1270 1091 reg_width = convert_buswidth(sconfig->dst_addr_width); 1271 1271 - ctrla |= ATC_DST_WIDTH(reg_width); 1272 1272 - ctrlb |= ATC_DST_ADDR_MODE_FIXED 1273 1273 - | ATC_SRC_ADDR_MODE_INCR 1274 1274 - | ATC_FC_MEM2PER 1275 1275 - | ATC_SIF(atchan->mem_if) | ATC_DIF(atchan->per_if); 1092 1092 + ctrla |= FIELD_PREP(ATC_DST_WIDTH, reg_width); 1093 1093 + ctrlb |= FIELD_PREP(ATC_DST_ADDR_MODE, 1094 1094 + ATC_DST_ADDR_MODE_FIXED) | 1095 1095 + FIELD_PREP(ATC_SRC_ADDR_MODE, ATC_SRC_ADDR_MODE_INCR) | 1096 1096 + FIELD_PREP(ATC_FC, ATC_FC_MEM2PER) | 1097 1097 + FIELD_PREP(ATC_SIF, atchan->mem_if) | 1098 1098 + FIELD_PREP(ATC_DIF, atchan->per_if); 1276 1099 reg = sconfig->dst_addr; 1277 1100 for_each_sg(sgl, sg, sg_len, i) { 1278 1278 - struct at_desc *desc; 1101 1101 + struct atdma_sg *atdma_sg = &desc->sg[i]; 1102 1102 + struct at_lli *lli; 1279 1103 u32 len; 1280 1104 u32 mem; 1281 1105 1282 1282 - desc = atc_desc_get(atchan); 1283 1283 - if (!desc) 1106 1106 + atdma_sg->lli = dma_pool_alloc(atdma->lli_pool, 1107 1107 + GFP_NOWAIT, 1108 1108 + &atdma_sg->lli_phys); 1109 1109 + if (!atdma_sg->lli) 1284 1110 goto err_desc_get; 1111 1111 + lli = atdma_sg->lli; 1285 1112 1286 1113 mem = sg_dma_address(sg); 1287 1114 len = sg_dma_len(sg); ··· 1305 1110 if (unlikely(mem & 3 || len & 3)) 1306 1111 mem_width = 0; 1307 1112 1308 1308 - desc->lli.saddr = mem; 1309 1309 - desc->lli.daddr = reg; 1310 1310 - desc->lli.ctrla = ctrla 1311 1311 - | ATC_SRC_WIDTH(mem_width) 1312 1312 - | len >> mem_width; 1313 1313 - desc->lli.ctrlb = ctrlb; 1314 1314 - desc->len = len; 1113 1113 + lli->saddr = mem; 1114 1114 + lli->daddr = reg; 1115 1115 + lli->ctrla = ctrla | 1116 1116 + FIELD_PREP(ATC_SRC_WIDTH, mem_width) | 1117 1117 + len >> mem_width; 1118 1118 + lli->ctrlb = ctrlb; 1315 1119 1316 1316 - atc_desc_chain(&first, &prev, desc); 1120 1120 + atdma_sg->len = len; 1317 1121 total_len += len; 1122 1122 + 1123 1123 + desc->sg[i].len = len; 1124 1124 + atdma_lli_chain(desc, i); 1318 1125 } 1319 1126 break; 1320 1127 case DMA_DEV_TO_MEM: 1321 1128 reg_width = convert_buswidth(sconfig->src_addr_width); 1322 1322 - ctrla |= ATC_SRC_WIDTH(reg_width); 1323 1323 - ctrlb |= ATC_DST_ADDR_MODE_INCR 1324 1324 - | ATC_SRC_ADDR_MODE_FIXED 1325 1325 - | ATC_FC_PER2MEM 1326 1326 - | ATC_SIF(atchan->per_if) | ATC_DIF(atchan->mem_if); 1129 1129 + ctrla |= FIELD_PREP(ATC_SRC_WIDTH, reg_width); 1130 1130 + ctrlb |= FIELD_PREP(ATC_DST_ADDR_MODE, ATC_DST_ADDR_MODE_INCR) | 1131 1131 + FIELD_PREP(ATC_SRC_ADDR_MODE, 1132 1132 + ATC_SRC_ADDR_MODE_FIXED) | 1133 1133 + FIELD_PREP(ATC_FC, ATC_FC_PER2MEM) | 1134 1134 + FIELD_PREP(ATC_SIF, atchan->per_if) | 1135 1135 + FIELD_PREP(ATC_DIF, atchan->mem_if); 1327 1136 1328 1137 reg = sconfig->src_addr; 1329 1138 for_each_sg(sgl, sg, sg_len, i) { 1330 1330 - struct at_desc *desc; 1139 1139 + struct atdma_sg *atdma_sg = &desc->sg[i]; 1140 1140 + struct at_lli *lli; 1331 1141 u32 len; 1332 1142 u32 mem; 1333 1143 1334 1334 - desc = atc_desc_get(atchan); 1335 1335 - if (!desc) 1144 1144 + atdma_sg->lli = dma_pool_alloc(atdma->lli_pool, 1145 1145 + GFP_NOWAIT, 1146 1146 + &atdma_sg->lli_phys); 1147 1147 + if (!atdma_sg->lli) 1336 1148 goto err_desc_get; 1149 1149 + lli = atdma_sg->lli; 1337 1150 1338 1151 mem = sg_dma_address(sg); 1339 1152 len = sg_dma_len(sg); ··· 1354 1151 if (unlikely(mem & 3 || len & 3)) 1355 1152 mem_width = 0; 1356 1153 1357 1357 - desc->lli.saddr = reg; 1358 1358 - desc->lli.daddr = mem; 1359 1359 - desc->lli.ctrla = ctrla 1360 1360 - | ATC_DST_WIDTH(mem_width) 1361 1361 - | len >> reg_width; 1362 1362 - desc->lli.ctrlb = ctrlb; 1363 1363 - desc->len = len; 1154 1154 + lli->saddr = reg; 1155 1155 + lli->daddr = mem; 1156 1156 + lli->ctrla = ctrla | 1157 1157 + FIELD_PREP(ATC_DST_WIDTH, mem_width) | 1158 1158 + len >> reg_width; 1159 1159 + lli->ctrlb = ctrlb; 1364 1160 1365 1365 - atc_desc_chain(&first, &prev, desc); 1161 1161 + desc->sg[i].len = len; 1366 1162 total_len += len; 1163 1163 + 1164 1164 + atdma_lli_chain(desc, i); 1367 1165 } 1368 1166 break; 1369 1167 default: ··· 1372 1168 } 1373 1169 1374 1170 /* set end-of-link to the last link descriptor of list*/ 1375 1375 - set_desc_eol(prev); 1171 1171 + set_lli_eol(desc, i - 1); 1376 1172 1377 1377 - /* First descriptor of the chain embedds additional information */ 1378 1378 - first->txd.cookie = -EBUSY; 1379 1379 - first->total_len = total_len; 1173 1173 + desc->total_len = total_len; 1380 1174 1381 1381 - /* first link descriptor of list is responsible of flags */ 1382 1382 - first->txd.flags = flags; /* client is in control of this ack */ 1383 1383 - 1384 1384 - return &first->txd; 1175 1175 + return vchan_tx_prep(&atchan->vc, &desc->vd, flags); 1385 1176 1386 1177 err_desc_get: 1387 1178 dev_err(chan2dev(chan), "not enough descriptors available\n"); 1388 1179 err: 1389 1389 - atc_desc_put(atchan, first); 1180 1180 + atdma_desc_free(&desc->vd); 1390 1181 return NULL; 1391 1182 } 1392 1183 ··· 1411 1212 */ 1412 1213 static int 1413 1214 atc_dma_cyclic_fill_desc(struct dma_chan *chan, struct at_desc *desc, 1414 1414 - unsigned int period_index, dma_addr_t buf_addr, 1215 1215 + unsigned int i, dma_addr_t buf_addr, 1415 1216 unsigned int reg_width, size_t period_len, 1416 1217 enum dma_transfer_direction direction) 1417 1218 { 1219 1219 + struct at_dma *atdma = to_at_dma(chan->device); 1418 1220 struct at_dma_chan *atchan = to_at_dma_chan(chan); 1419 1221 struct dma_slave_config *sconfig = &atchan->dma_sconfig; 1420 1420 - u32 ctrla; 1222 1222 + struct atdma_sg *atdma_sg = &desc->sg[i]; 1223 1223 + struct at_lli *lli; 1421 1224 1422 1422 - /* prepare common CRTLA value */ 1423 1423 - ctrla = ATC_SCSIZE(sconfig->src_maxburst) 1424 1424 - | ATC_DCSIZE(sconfig->dst_maxburst) 1425 1425 - | ATC_DST_WIDTH(reg_width) 1426 1426 - | ATC_SRC_WIDTH(reg_width) 1427 1427 - | period_len >> reg_width; 1225 1225 + atdma_sg->lli = dma_pool_alloc(atdma->lli_pool, GFP_ATOMIC, 1226 1226 + &atdma_sg->lli_phys); 1227 1227 + if (!atdma_sg->lli) 1228 1228 + return -ENOMEM; 1229 1229 + lli = atdma_sg->lli; 1428 1230 1429 1231 switch (direction) { 1430 1232 case DMA_MEM_TO_DEV: 1431 1431 - desc->lli.saddr = buf_addr + (period_len * period_index); 1432 1432 - desc->lli.daddr = sconfig->dst_addr; 1433 1433 - desc->lli.ctrla = ctrla; 1434 1434 - desc->lli.ctrlb = ATC_DST_ADDR_MODE_FIXED 1435 1435 - | ATC_SRC_ADDR_MODE_INCR 1436 1436 - | ATC_FC_MEM2PER 1437 1437 - | ATC_SIF(atchan->mem_if) 1438 1438 - | ATC_DIF(atchan->per_if); 1439 1439 - desc->len = period_len; 1233 1233 + lli->saddr = buf_addr + (period_len * i); 1234 1234 + lli->daddr = sconfig->dst_addr; 1235 1235 + lli->ctrlb = FIELD_PREP(ATC_DST_ADDR_MODE, 1236 1236 + ATC_DST_ADDR_MODE_FIXED) | 1237 1237 + FIELD_PREP(ATC_SRC_ADDR_MODE, 1238 1238 + ATC_SRC_ADDR_MODE_INCR) | 1239 1239 + FIELD_PREP(ATC_FC, ATC_FC_MEM2PER) | 1240 1240 + FIELD_PREP(ATC_SIF, atchan->mem_if) | 1241 1241 + FIELD_PREP(ATC_DIF, atchan->per_if); 1242 1242 + 1440 1243 break; 1441 1244 1442 1245 case DMA_DEV_TO_MEM: 1443 1443 - desc->lli.saddr = sconfig->src_addr; 1444 1444 - desc->lli.daddr = buf_addr + (period_len * period_index); 1445 1445 - desc->lli.ctrla = ctrla; 1446 1446 - desc->lli.ctrlb = ATC_DST_ADDR_MODE_INCR 1447 1447 - | ATC_SRC_ADDR_MODE_FIXED 1448 1448 - | ATC_FC_PER2MEM 1449 1449 - | ATC_SIF(atchan->per_if) 1450 1450 - | ATC_DIF(atchan->mem_if); 1451 1451 - desc->len = period_len; 1246 1246 + lli->saddr = sconfig->src_addr; 1247 1247 + lli->daddr = buf_addr + (period_len * i); 1248 1248 + lli->ctrlb = FIELD_PREP(ATC_DST_ADDR_MODE, 1249 1249 + ATC_DST_ADDR_MODE_INCR) | 1250 1250 + FIELD_PREP(ATC_SRC_ADDR_MODE, 1251 1251 + ATC_SRC_ADDR_MODE_FIXED) | 1252 1252 + FIELD_PREP(ATC_FC, ATC_FC_PER2MEM) | 1253 1253 + FIELD_PREP(ATC_SIF, atchan->per_if) | 1254 1254 + FIELD_PREP(ATC_DIF, atchan->mem_if); 1452 1255 break; 1453 1256 1454 1257 default: 1455 1258 return -EINVAL; 1456 1259 } 1260 1260 + 1261 1261 + lli->ctrla = FIELD_PREP(ATC_SCSIZE, sconfig->src_maxburst) | 1262 1262 + FIELD_PREP(ATC_DCSIZE, sconfig->dst_maxburst) | 1263 1263 + FIELD_PREP(ATC_DST_WIDTH, reg_width) | 1264 1264 + FIELD_PREP(ATC_SRC_WIDTH, reg_width) | 1265 1265 + period_len >> reg_width; 1266 1266 + desc->sg[i].len = period_len; 1457 1267 1458 1268 return 0; 1459 1269 } ··· 1484 1276 struct at_dma_chan *atchan = to_at_dma_chan(chan); 1485 1277 struct at_dma_slave *atslave = chan->private; 1486 1278 struct dma_slave_config *sconfig = &atchan->dma_sconfig; 1487 1487 - struct at_desc *first = NULL; 1488 1488 - struct at_desc *prev = NULL; 1279 1279 + struct at_desc *desc; 1489 1280 unsigned long was_cyclic; 1490 1281 unsigned int reg_width; 1491 1282 unsigned int periods = buf_len / period_len; ··· 1518 1311 if (atc_dma_cyclic_check_values(reg_width, buf_addr, period_len)) 1519 1312 goto err_out; 1520 1313 1314 1314 + desc = kzalloc(struct_size(desc, sg, periods), GFP_ATOMIC); 1315 1315 + if (!desc) 1316 1316 + goto err_out; 1317 1317 + desc->sglen = periods; 1318 1318 + 1521 1319 /* build cyclic linked list */ 1522 1320 for (i = 0; i < periods; i++) { 1523 1523 - struct at_desc *desc; 1524 1524 - 1525 1525 - desc = atc_desc_get(atchan); 1526 1526 - if (!desc) 1527 1527 - goto err_desc_get; 1528 1528 - 1529 1321 if (atc_dma_cyclic_fill_desc(chan, desc, i, buf_addr, 1530 1322 reg_width, period_len, direction)) 1531 1531 - goto err_desc_get; 1532 1532 - 1533 1533 - atc_desc_chain(&first, &prev, desc); 1323 1323 + goto err_fill_desc; 1324 1324 + atdma_lli_chain(desc, i); 1534 1325 } 1535 1535 - 1326 1326 + desc->total_len = buf_len; 1536 1327 /* lets make a cyclic list */ 1537 1537 - prev->lli.dscr = first->txd.phys; 1328 1328 + desc->sg[i - 1].lli->dscr = desc->sg[0].lli_phys; 1538 1329 1539 1539 - /* First descriptor of the chain embedds additional information */ 1540 1540 - first->txd.cookie = -EBUSY; 1541 1541 - first->total_len = buf_len; 1330 1330 + return vchan_tx_prep(&atchan->vc, &desc->vd, flags); 1542 1331 1543 1543 - return &first->txd; 1544 1544 - 1545 1545 - err_desc_get: 1546 1546 - dev_err(chan2dev(chan), "not enough descriptors available\n"); 1547 1547 - atc_desc_put(atchan, first); 1332 1332 + err_fill_desc: 1333 1333 + atdma_desc_free(&desc->vd); 1548 1334 err_out: 1549 1335 clear_bit(ATC_IS_CYCLIC, &atchan->status); 1550 1336 return NULL; ··· 1566 1366 { 1567 1367 struct at_dma_chan *atchan = to_at_dma_chan(chan); 1568 1368 struct at_dma *atdma = to_at_dma(chan->device); 1569 1569 - int chan_id = atchan->chan_common.chan_id; 1369 1369 + int chan_id = atchan->vc.chan.chan_id; 1570 1370 unsigned long flags; 1571 1371 1572 1372 dev_vdbg(chan2dev(chan), "%s\n", __func__); 1573 1373 1574 1574 - spin_lock_irqsave(&atchan->lock, flags); 1374 1374 + spin_lock_irqsave(&atchan->vc.lock, flags); 1575 1375 1576 1376 dma_writel(atdma, CHER, AT_DMA_SUSP(chan_id)); 1577 1377 set_bit(ATC_IS_PAUSED, &atchan->status); 1578 1378 1579 1579 - spin_unlock_irqrestore(&atchan->lock, flags); 1379 1379 + spin_unlock_irqrestore(&atchan->vc.lock, flags); 1580 1380 1581 1381 return 0; 1582 1382 } ··· 1585 1385 { 1586 1386 struct at_dma_chan *atchan = to_at_dma_chan(chan); 1587 1387 struct at_dma *atdma = to_at_dma(chan->device); 1588 1588 - int chan_id = atchan->chan_common.chan_id; 1388 1388 + int chan_id = atchan->vc.chan.chan_id; 1589 1389 unsigned long flags; 1590 1390 1591 1391 dev_vdbg(chan2dev(chan), "%s\n", __func__); ··· 1593 1393 if (!atc_chan_is_paused(atchan)) 1594 1394 return 0; 1595 1395 1596 1596 - spin_lock_irqsave(&atchan->lock, flags); 1396 1396 + spin_lock_irqsave(&atchan->vc.lock, flags); 1597 1397 1598 1398 dma_writel(atdma, CHDR, AT_DMA_RES(chan_id)); 1599 1399 clear_bit(ATC_IS_PAUSED, &atchan->status); 1600 1400 1601 1601 - spin_unlock_irqrestore(&atchan->lock, flags); 1401 1401 + spin_unlock_irqrestore(&atchan->vc.lock, flags); 1602 1402 1603 1403 return 0; 1604 1404 } ··· 1607 1407 { 1608 1408 struct at_dma_chan *atchan = to_at_dma_chan(chan); 1609 1409 struct at_dma *atdma = to_at_dma(chan->device); 1610 1610 - int chan_id = atchan->chan_common.chan_id; 1410 1410 + int chan_id = atchan->vc.chan.chan_id; 1611 1411 unsigned long flags; 1412 1412 + 1413 1413 + LIST_HEAD(list); 1612 1414 1613 1415 dev_vdbg(chan2dev(chan), "%s\n", __func__); 1614 1416 ··· 1620 1418 * channel. We still have to poll the channel enable bit due 1621 1419 * to AHB/HSB limitations. 1622 1420 */ 1623 1623 - spin_lock_irqsave(&atchan->lock, flags); 1421 1421 + spin_lock_irqsave(&atchan->vc.lock, flags); 1624 1422 1625 1423 /* disabling channel: must also remove suspend state */ 1626 1424 dma_writel(atdma, CHDR, AT_DMA_RES(chan_id) | atchan->mask); ··· 1629 1427 while (dma_readl(atdma, CHSR) & atchan->mask) 1630 1428 cpu_relax(); 1631 1429 1632 1632 - /* active_list entries will end up before queued entries */ 1633 1633 - list_splice_tail_init(&atchan->queue, &atchan->free_list); 1634 1634 - list_splice_tail_init(&atchan->active_list, &atchan->free_list); 1430 1430 + if (atchan->desc) { 1431 1431 + vchan_terminate_vdesc(&atchan->desc->vd); 1432 1432 + atchan->desc = NULL; 1433 1433 + } 1434 1434 + 1435 1435 + vchan_get_all_descriptors(&atchan->vc, &list); 1635 1436 1636 1437 clear_bit(ATC_IS_PAUSED, &atchan->status); 1637 1438 /* if channel dedicated to cyclic operations, free it */ 1638 1439 clear_bit(ATC_IS_CYCLIC, &atchan->status); 1639 1440 1640 1640 - spin_unlock_irqrestore(&atchan->lock, flags); 1441 1441 + spin_unlock_irqrestore(&atchan->vc.lock, flags); 1442 1442 + 1443 1443 + vchan_dma_desc_free_list(&atchan->vc, &list); 1641 1444 1642 1445 return 0; 1643 1446 } ··· 1664 1457 { 1665 1458 struct at_dma_chan *atchan = to_at_dma_chan(chan); 1666 1459 unsigned long flags; 1667 1667 - enum dma_status ret; 1668 1668 - int bytes = 0; 1460 1460 + enum dma_status dma_status; 1461 1461 + u32 residue; 1462 1462 + int ret; 1669 1463 1670 1670 - ret = dma_cookie_status(chan, cookie, txstate); 1671 1671 - if (ret == DMA_COMPLETE) 1672 1672 - return ret; 1673 1673 - /* 1674 1674 - * There's no point calculating the residue if there's 1675 1675 - * no txstate to store the value. 1676 1676 - */ 1677 1677 - if (!txstate) 1678 1678 - return DMA_ERROR; 1464 1464 + dma_status = dma_cookie_status(chan, cookie, txstate); 1465 1465 + if (dma_status == DMA_COMPLETE || !txstate) 1466 1466 + return dma_status; 1679 1467 1680 1680 - spin_lock_irqsave(&atchan->lock, flags); 1681 1681 - 1468 1468 + spin_lock_irqsave(&atchan->vc.lock, flags); 1682 1469 /* Get number of bytes left in the active transactions */ 1683 1683 - bytes = atc_get_bytes_left(chan, cookie); 1470 1470 + ret = atc_get_residue(chan, cookie, &residue); 1471 1471 + spin_unlock_irqrestore(&atchan->vc.lock, flags); 1684 1472 1685 1685 - spin_unlock_irqrestore(&atchan->lock, flags); 1686 1686 - 1687 1687 - if (unlikely(bytes < 0)) { 1473 1473 + if (unlikely(ret < 0)) { 1688 1474 dev_vdbg(chan2dev(chan), "get residual bytes error\n"); 1689 1475 return DMA_ERROR; 1690 1476 } else { 1691 1691 - dma_set_residue(txstate, bytes); 1477 1477 + dma_set_residue(txstate, residue); 1692 1478 } 1693 1479 1694 1694 - dev_vdbg(chan2dev(chan), "tx_status %d: cookie = %d residue = %d\n", 1695 1695 - ret, cookie, bytes); 1480 1480 + dev_vdbg(chan2dev(chan), "tx_status %d: cookie = %d residue = %u\n", 1481 1481 + dma_status, cookie, residue); 1696 1482 1697 1697 - return ret; 1483 1483 + return dma_status; 1698 1484 } 1699 1485 1700 1700 - /** 1701 1701 - * atc_issue_pending - takes the first transaction descriptor in the pending 1702 1702 - * queue and starts the transfer. 1703 1703 - * @chan: target DMA channel 1704 1704 - */ 1705 1486 static void atc_issue_pending(struct dma_chan *chan) 1706 1487 { 1707 1488 struct at_dma_chan *atchan = to_at_dma_chan(chan); 1708 1708 - struct at_desc *desc; 1709 1489 unsigned long flags; 1710 1490 1711 1711 - dev_vdbg(chan2dev(chan), "issue_pending\n"); 1712 1712 - 1713 1713 - spin_lock_irqsave(&atchan->lock, flags); 1714 1714 - if (atc_chan_is_enabled(atchan) || list_empty(&atchan->queue)) 1715 1715 - return spin_unlock_irqrestore(&atchan->lock, flags); 1716 1716 - 1717 1717 - desc = atc_first_queued(atchan); 1718 1718 - list_move_tail(&desc->desc_node, &atchan->active_list); 1719 1719 - atc_dostart(atchan, desc); 1720 1720 - spin_unlock_irqrestore(&atchan->lock, flags); 1491 1491 + spin_lock_irqsave(&atchan->vc.lock, flags); 1492 1492 + if (vchan_issue_pending(&atchan->vc) && !atchan->desc) { 1493 1493 + if (!(atc_chan_is_enabled(atchan))) 1494 1494 + atc_dostart(atchan); 1495 1495 + } 1496 1496 + spin_unlock_irqrestore(&atchan->vc.lock, flags); 1721 1497 } 1722 1498 1723 1499 /** ··· 1713 1523 { 1714 1524 struct at_dma_chan *atchan = to_at_dma_chan(chan); 1715 1525 struct at_dma *atdma = to_at_dma(chan->device); 1716 1716 - struct at_desc *desc; 1717 1526 struct at_dma_slave *atslave; 1718 1718 - int i; 1719 1527 u32 cfg; 1720 1528 1721 1529 dev_vdbg(chan2dev(chan), "alloc_chan_resources\n"); ··· 1721 1533 /* ASSERT: channel is idle */ 1722 1534 if (atc_chan_is_enabled(atchan)) { 1723 1535 dev_dbg(chan2dev(chan), "DMA channel not idle ?\n"); 1724 1724 - return -EIO; 1725 1725 - } 1726 1726 - 1727 1727 - if (!list_empty(&atchan->free_list)) { 1728 1728 - dev_dbg(chan2dev(chan), "can't allocate channel resources (channel not freed from a previous use)\n"); 1729 1536 return -EIO; 1730 1537 } 1731 1538 ··· 1732 1549 * We need controller-specific data to set up slave 1733 1550 * transfers. 1734 1551 */ 1735 1735 - BUG_ON(!atslave->dma_dev || atslave->dma_dev != atdma->dma_common.dev); 1552 1552 + BUG_ON(!atslave->dma_dev || atslave->dma_dev != atdma->dma_device.dev); 1736 1553 1737 1554 /* if cfg configuration specified take it instead of default */ 1738 1555 if (atslave->cfg) 1739 1556 cfg = atslave->cfg; 1740 1557 } 1741 1558 1742 1742 - /* Allocate initial pool of descriptors */ 1743 1743 - for (i = 0; i < init_nr_desc_per_channel; i++) { 1744 1744 - desc = atc_alloc_descriptor(chan, GFP_KERNEL); 1745 1745 - if (!desc) { 1746 1746 - dev_err(atdma->dma_common.dev, 1747 1747 - "Only %d initial descriptors\n", i); 1748 1748 - break; 1749 1749 - } 1750 1750 - list_add_tail(&desc->desc_node, &atchan->free_list); 1751 1751 - } 1752 1752 - 1753 1753 - dma_cookie_init(chan); 1754 1754 - 1755 1559 /* channel parameters */ 1756 1560 channel_writel(atchan, CFG, cfg); 1757 1561 1758 1758 - dev_dbg(chan2dev(chan), 1759 1759 - "alloc_chan_resources: allocated %d descriptors\n", i); 1760 1760 - 1761 1761 - return i; 1562 1562 + return 0; 1762 1563 } 1763 1564 1764 1565 /** ··· 1752 1585 static void atc_free_chan_resources(struct dma_chan *chan) 1753 1586 { 1754 1587 struct at_dma_chan *atchan = to_at_dma_chan(chan); 1755 1755 - struct at_dma *atdma = to_at_dma(chan->device); 1756 1756 - struct at_desc *desc, *_desc; 1757 1757 - LIST_HEAD(list); 1758 1588 1759 1759 - /* ASSERT: channel is idle */ 1760 1760 - BUG_ON(!list_empty(&atchan->active_list)); 1761 1761 - BUG_ON(!list_empty(&atchan->queue)); 1762 1589 BUG_ON(atc_chan_is_enabled(atchan)); 1763 1590 1764 1764 - list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) { 1765 1765 - dev_vdbg(chan2dev(chan), " freeing descriptor %p\n", desc); 1766 1766 - list_del(&desc->desc_node); 1767 1767 - /* free link descriptor */ 1768 1768 - dma_pool_free(atdma->dma_desc_pool, desc, desc->txd.phys); 1769 1769 - } 1770 1770 - list_splice_init(&atchan->free_list, &list); 1591 1591 + vchan_free_chan_resources(to_virt_chan(chan)); 1771 1592 atchan->status = 0; 1772 1593 1773 1594 /* ··· 1806 1651 return NULL; 1807 1652 } 1808 1653 1809 1809 - atslave->cfg = ATC_DST_H2SEL_HW | ATC_SRC_H2SEL_HW; 1654 1654 + atslave->cfg = ATC_DST_H2SEL | ATC_SRC_H2SEL; 1810 1655 /* 1811 1656 * We can fill both SRC_PER and DST_PER, one of these fields will be 1812 1657 * ignored depending on DMA transfer direction. 1813 1658 */ 1814 1659 per_id = dma_spec->args[1] & AT91_DMA_CFG_PER_ID_MASK; 1815 1815 - atslave->cfg |= ATC_DST_PER_MSB(per_id) | ATC_DST_PER(per_id) 1816 1816 - | ATC_SRC_PER_MSB(per_id) | ATC_SRC_PER(per_id); 1660 1660 + atslave->cfg |= ATC_DST_PER_ID(per_id) | ATC_SRC_PER_ID(per_id); 1817 1661 /* 1818 1662 * We have to translate the value we get from the device tree since 1819 1663 * the half FIFO configuration value had to be 0 to keep backward ··· 1820 1666 */ 1821 1667 switch (dma_spec->args[1] & AT91_DMA_CFG_FIFOCFG_MASK) { 1822 1668 case AT91_DMA_CFG_FIFOCFG_ALAP: 1823 1823 - atslave->cfg |= ATC_FIFOCFG_LARGESTBURST; 1669 1669 + atslave->cfg |= FIELD_PREP(ATC_FIFOCFG, 1670 1670 + ATC_FIFOCFG_LARGESTBURST); 1824 1671 break; 1825 1672 case AT91_DMA_CFG_FIFOCFG_ASAP: 1826 1826 - atslave->cfg |= ATC_FIFOCFG_ENOUGHSPACE; 1673 1673 + atslave->cfg |= FIELD_PREP(ATC_FIFOCFG, 1674 1674 + ATC_FIFOCFG_ENOUGHSPACE); 1827 1675 break; 1828 1676 case AT91_DMA_CFG_FIFOCFG_HALF: 1829 1677 default: 1830 1830 - atslave->cfg |= ATC_FIFOCFG_HALFFIFO; 1678 1678 + atslave->cfg |= FIELD_PREP(ATC_FIFOCFG, ATC_FIFOCFG_HALFFIFO); 1831 1679 } 1832 1680 atslave->dma_dev = &dmac_pdev->dev; 1833 1681 ··· 1924 1768 1925 1769 static int __init at_dma_probe(struct platform_device *pdev) 1926 1770 { 1927 1927 - struct resource *io; 1928 1771 struct at_dma *atdma; 1929 1929 - size_t size; 1930 1772 int irq; 1931 1773 int err; 1932 1774 int i; ··· 1944 1790 if (!plat_dat) 1945 1791 return -ENODEV; 1946 1792 1947 1947 - io = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1948 1948 - if (!io) 1949 1949 - return -EINVAL; 1793 1793 + atdma = devm_kzalloc(&pdev->dev, 1794 1794 + struct_size(atdma, chan, plat_dat->nr_channels), 1795 1795 + GFP_KERNEL); 1796 1796 + if (!atdma) 1797 1797 + return -ENOMEM; 1798 1798 + 1799 1799 + atdma->regs = devm_platform_ioremap_resource(pdev, 0); 1800 1800 + if (IS_ERR(atdma->regs)) 1801 1801 + return PTR_ERR(atdma->regs); 1950 1802 1951 1803 irq = platform_get_irq(pdev, 0); 1952 1804 if (irq < 0) 1953 1805 return irq; 1954 1806 1955 1955 - size = sizeof(struct at_dma); 1956 1956 - size += plat_dat->nr_channels * sizeof(struct at_dma_chan); 1957 1957 - atdma = kzalloc(size, GFP_KERNEL); 1958 1958 - if (!atdma) 1959 1959 - return -ENOMEM; 1960 1960 - 1961 1807 /* discover transaction capabilities */ 1962 1962 - atdma->dma_common.cap_mask = plat_dat->cap_mask; 1808 1808 + atdma->dma_device.cap_mask = plat_dat->cap_mask; 1963 1809 atdma->all_chan_mask = (1 << plat_dat->nr_channels) - 1; 1964 1810 1965 1965 - size = resource_size(io); 1966 1966 - if (!request_mem_region(io->start, size, pdev->dev.driver->name)) { 1967 1967 - err = -EBUSY; 1968 1968 - goto err_kfree; 1969 1969 - } 1811 1811 + atdma->clk = devm_clk_get(&pdev->dev, "dma_clk"); 1812 1812 + if (IS_ERR(atdma->clk)) 1813 1813 + return PTR_ERR(atdma->clk); 1970 1814 1971 1971 - atdma->regs = ioremap(io->start, size); 1972 1972 - if (!atdma->regs) { 1973 1973 - err = -ENOMEM; 1974 1974 - goto err_release_r; 1975 1975 - } 1976 1976 - 1977 1977 - atdma->clk = clk_get(&pdev->dev, "dma_clk"); 1978 1978 - if (IS_ERR(atdma->clk)) { 1979 1979 - err = PTR_ERR(atdma->clk); 1980 1980 - goto err_clk; 1981 1981 - } 1982 1815 err = clk_prepare_enable(atdma->clk); 1983 1816 if (err) 1984 1984 - goto err_clk_prepare; 1817 1817 + return err; 1985 1818 1986 1819 /* force dma off, just in case */ 1987 1820 at_dma_off(atdma); ··· 1980 1839 platform_set_drvdata(pdev, atdma); 1981 1840 1982 1841 /* create a pool of consistent memory blocks for hardware descriptors */ 1983 1983 - atdma->dma_desc_pool = dma_pool_create("at_hdmac_desc_pool", 1984 1984 - &pdev->dev, sizeof(struct at_desc), 1985 1985 - 4 /* word alignment */, 0); 1986 1986 - if (!atdma->dma_desc_pool) { 1987 1987 - dev_err(&pdev->dev, "No memory for descriptors dma pool\n"); 1842 1842 + atdma->lli_pool = dma_pool_create("at_hdmac_lli_pool", 1843 1843 + &pdev->dev, sizeof(struct at_lli), 1844 1844 + 4 /* word alignment */, 0); 1845 1845 + if (!atdma->lli_pool) { 1846 1846 + dev_err(&pdev->dev, "Unable to allocate DMA LLI descriptor pool\n"); 1988 1847 err = -ENOMEM; 1989 1848 goto err_desc_pool_create; 1990 1849 } ··· 2003 1862 cpu_relax(); 2004 1863 2005 1864 /* initialize channels related values */ 2006 2006 - INIT_LIST_HEAD(&atdma->dma_common.channels); 1865 1865 + INIT_LIST_HEAD(&atdma->dma_device.channels); 2007 1866 for (i = 0; i < plat_dat->nr_channels; i++) { 2008 1867 struct at_dma_chan *atchan = &atdma->chan[i]; 2009 1868 2010 1869 atchan->mem_if = AT_DMA_MEM_IF; 2011 1870 atchan->per_if = AT_DMA_PER_IF; 2012 2012 - atchan->chan_common.device = &atdma->dma_common; 2013 2013 - dma_cookie_init(&atchan->chan_common); 2014 2014 - list_add_tail(&atchan->chan_common.device_node, 2015 2015 - &atdma->dma_common.channels); 2016 1871 2017 1872 atchan->ch_regs = atdma->regs + ch_regs(i); 2018 2018 - spin_lock_init(&atchan->lock); 2019 1873 atchan->mask = 1 << i; 2020 1874 2021 2021 - INIT_LIST_HEAD(&atchan->active_list); 2022 2022 - INIT_LIST_HEAD(&atchan->queue); 2023 2023 - INIT_LIST_HEAD(&atchan->free_list); 2024 2024 - 2025 2025 - tasklet_setup(&atchan->tasklet, atc_tasklet); 1875 1875 + atchan->atdma = atdma; 1876 1876 + atchan->vc.desc_free = atdma_desc_free; 1877 1877 + vchan_init(&atchan->vc, &atdma->dma_device); 2026 1878 atc_enable_chan_irq(atdma, i); 2027 1879 } 2028 1880 2029 1881 /* set base routines */ 2030 2030 - atdma->dma_common.device_alloc_chan_resources = atc_alloc_chan_resources; 2031 2031 - atdma->dma_common.device_free_chan_resources = atc_free_chan_resources; 2032 2032 - atdma->dma_common.device_tx_status = atc_tx_status; 2033 2033 - atdma->dma_common.device_issue_pending = atc_issue_pending; 2034 2034 - atdma->dma_common.dev = &pdev->dev; 1882 1882 + atdma->dma_device.device_alloc_chan_resources = atc_alloc_chan_resources; 1883 1883 + atdma->dma_device.device_free_chan_resources = atc_free_chan_resources; 1884 1884 + atdma->dma_device.device_tx_status = atc_tx_status; 1885 1885 + atdma->dma_device.device_issue_pending = atc_issue_pending; 1886 1886 + atdma->dma_device.dev = &pdev->dev; 2035 1887 2036 1888 /* set prep routines based on capability */ 2037 2037 - if (dma_has_cap(DMA_INTERLEAVE, atdma->dma_common.cap_mask)) 2038 2038 - atdma->dma_common.device_prep_interleaved_dma = atc_prep_dma_interleaved; 1889 1889 + if (dma_has_cap(DMA_INTERLEAVE, atdma->dma_device.cap_mask)) 1890 1890 + atdma->dma_device.device_prep_interleaved_dma = atc_prep_dma_interleaved; 2039 1891 2040 2040 - if (dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask)) 2041 2041 - atdma->dma_common.device_prep_dma_memcpy = atc_prep_dma_memcpy; 1892 1892 + if (dma_has_cap(DMA_MEMCPY, atdma->dma_device.cap_mask)) 1893 1893 + atdma->dma_device.device_prep_dma_memcpy = atc_prep_dma_memcpy; 2042 1894 2043 2043 - if (dma_has_cap(DMA_MEMSET, atdma->dma_common.cap_mask)) { 2044 2044 - atdma->dma_common.device_prep_dma_memset = atc_prep_dma_memset; 2045 2045 - atdma->dma_common.device_prep_dma_memset_sg = atc_prep_dma_memset_sg; 2046 2046 - atdma->dma_common.fill_align = DMAENGINE_ALIGN_4_BYTES; 1895 1895 + if (dma_has_cap(DMA_MEMSET, atdma->dma_device.cap_mask)) { 1896 1896 + atdma->dma_device.device_prep_dma_memset = atc_prep_dma_memset; 1897 1897 + atdma->dma_device.device_prep_dma_memset_sg = atc_prep_dma_memset_sg; 1898 1898 + atdma->dma_device.fill_align = DMAENGINE_ALIGN_4_BYTES; 2047 1899 } 2048 1900 2049 2049 - if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)) { 2050 2050 - atdma->dma_common.device_prep_slave_sg = atc_prep_slave_sg; 1901 1901 + if (dma_has_cap(DMA_SLAVE, atdma->dma_device.cap_mask)) { 1902 1902 + atdma->dma_device.device_prep_slave_sg = atc_prep_slave_sg; 2051 1903 /* controller can do slave DMA: can trigger cyclic transfers */ 2052 2052 - dma_cap_set(DMA_CYCLIC, atdma->dma_common.cap_mask); 2053 2053 - atdma->dma_common.device_prep_dma_cyclic = atc_prep_dma_cyclic; 2054 2054 - atdma->dma_common.device_config = atc_config; 2055 2055 - atdma->dma_common.device_pause = atc_pause; 2056 2056 - atdma->dma_common.device_resume = atc_resume; 2057 2057 - atdma->dma_common.device_terminate_all = atc_terminate_all; 2058 2058 - atdma->dma_common.src_addr_widths = ATC_DMA_BUSWIDTHS; 2059 2059 - atdma->dma_common.dst_addr_widths = ATC_DMA_BUSWIDTHS; 2060 2060 - atdma->dma_common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 2061 2061 - atdma->dma_common.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 1904 1904 + dma_cap_set(DMA_CYCLIC, atdma->dma_device.cap_mask); 1905 1905 + atdma->dma_device.device_prep_dma_cyclic = atc_prep_dma_cyclic; 1906 1906 + atdma->dma_device.device_config = atc_config; 1907 1907 + atdma->dma_device.device_pause = atc_pause; 1908 1908 + atdma->dma_device.device_resume = atc_resume; 1909 1909 + atdma->dma_device.device_terminate_all = atc_terminate_all; 1910 1910 + atdma->dma_device.src_addr_widths = ATC_DMA_BUSWIDTHS; 1911 1911 + atdma->dma_device.dst_addr_widths = ATC_DMA_BUSWIDTHS; 1912 1912 + atdma->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 1913 1913 + atdma->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 2062 1914 } 2063 1915 2064 1916 dma_writel(atdma, EN, AT_DMA_ENABLE); 2065 1917 2066 1918 dev_info(&pdev->dev, "Atmel AHB DMA Controller ( %s%s%s), %d channels\n", 2067 2067 - dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask) ? "cpy " : "", 2068 2068 - dma_has_cap(DMA_MEMSET, atdma->dma_common.cap_mask) ? "set " : "", 2069 2069 - dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask) ? "slave " : "", 1919 1919 + dma_has_cap(DMA_MEMCPY, atdma->dma_device.cap_mask) ? "cpy " : "", 1920 1920 + dma_has_cap(DMA_MEMSET, atdma->dma_device.cap_mask) ? "set " : "", 1921 1921 + dma_has_cap(DMA_SLAVE, atdma->dma_device.cap_mask) ? "slave " : "", 2070 1922 plat_dat->nr_channels); 2071 1923 2072 2072 - err = dma_async_device_register(&atdma->dma_common); 1924 1924 + err = dma_async_device_register(&atdma->dma_device); 2073 1925 if (err) { 2074 1926 dev_err(&pdev->dev, "Unable to register: %d.\n", err); 2075 1927 goto err_dma_async_device_register; ··· 2085 1951 return 0; 2086 1952 2087 1953 err_of_dma_controller_register: 2088 2088 - dma_async_device_unregister(&atdma->dma_common); 1954 1954 + dma_async_device_unregister(&atdma->dma_device); 2089 1955 err_dma_async_device_register: 2090 1956 dma_pool_destroy(atdma->memset_pool); 2091 1957 err_memset_pool_create: 2092 2092 - dma_pool_destroy(atdma->dma_desc_pool); 1958 1958 + dma_pool_destroy(atdma->lli_pool); 2093 1959 err_desc_pool_create: 2094 1960 free_irq(platform_get_irq(pdev, 0), atdma); 2095 1961 err_irq: 2096 1962 clk_disable_unprepare(atdma->clk); 2097 2097 - err_clk_prepare: 2098 2098 - clk_put(atdma->clk); 2099 2099 - err_clk: 2100 2100 - iounmap(atdma->regs); 2101 2101 - atdma->regs = NULL; 2102 2102 - err_release_r: 2103 2103 - release_mem_region(io->start, size); 2104 2104 - err_kfree: 2105 2105 - kfree(atdma); 2106 1963 return err; 2107 1964 } 2108 1965 ··· 2101 1976 { 2102 1977 struct at_dma *atdma = platform_get_drvdata(pdev); 2103 1978 struct dma_chan *chan, *_chan; 2104 2104 - struct resource *io; 2105 1979 2106 1980 at_dma_off(atdma); 2107 1981 if (pdev->dev.of_node) 2108 1982 of_dma_controller_free(pdev->dev.of_node); 2109 2109 - dma_async_device_unregister(&atdma->dma_common); 1983 1983 + dma_async_device_unregister(&atdma->dma_device); 2110 1984 2111 1985 dma_pool_destroy(atdma->memset_pool); 2112 2112 - dma_pool_destroy(atdma->dma_desc_pool); 1986 1986 + dma_pool_destroy(atdma->lli_pool); 2113 1987 free_irq(platform_get_irq(pdev, 0), atdma); 2114 1988 2115 2115 - list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels, 1989 1989 + list_for_each_entry_safe(chan, _chan, &atdma->dma_device.channels, 2116 1990 device_node) { 2117 2117 - struct at_dma_chan *atchan = to_at_dma_chan(chan); 2118 2118 - 2119 1991 /* Disable interrupts */ 2120 1992 atc_disable_chan_irq(atdma, chan->chan_id); 2121 2121 - 2122 2122 - tasklet_kill(&atchan->tasklet); 2123 1993 list_del(&chan->device_node); 2124 1994 } 2125 1995 2126 1996 clk_disable_unprepare(atdma->clk); 2127 2127 - clk_put(atdma->clk); 2128 2128 - 2129 2129 - iounmap(atdma->regs); 2130 2130 - atdma->regs = NULL; 2131 2131 - 2132 2132 - io = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2133 2133 - release_mem_region(io->start, resource_size(io)); 2134 2134 - 2135 2135 - kfree(atdma); 2136 1997 2137 1998 return 0; 2138 1999 } ··· 2136 2025 struct at_dma *atdma = dev_get_drvdata(dev); 2137 2026 struct dma_chan *chan, *_chan; 2138 2027 2139 2139 - list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels, 2028 2028 + list_for_each_entry_safe(chan, _chan, &atdma->dma_device.channels, 2140 2029 device_node) { 2141 2030 struct at_dma_chan *atchan = to_at_dma_chan(chan); 2142 2031 /* wait for transaction completion (except in cyclic case) */ ··· 2148 2037 2149 2038 static void atc_suspend_cyclic(struct at_dma_chan *atchan) 2150 2039 { 2151 2151 - struct dma_chan *chan = &atchan->chan_common; 2040 2040 + struct dma_chan *chan = &atchan->vc.chan; 2152 2041 2153 2042 /* Channel should be paused by user 2154 2043 * do it anyway even if it is not done already */ ··· 2171 2060 struct dma_chan *chan, *_chan; 2172 2061 2173 2062 /* preserve data */ 2174 2174 - list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels, 2063 2063 + list_for_each_entry_safe(chan, _chan, &atdma->dma_device.channels, 2175 2064 device_node) { 2176 2065 struct at_dma_chan *atchan = to_at_dma_chan(chan); 2177 2066 ··· 2189 2078 2190 2079 static void atc_resume_cyclic(struct at_dma_chan *atchan) 2191 2080 { 2192 2192 - struct at_dma *atdma = to_at_dma(atchan->chan_common.device); 2081 2081 + struct at_dma *atdma = to_at_dma(atchan->vc.chan.device); 2193 2082 2194 2083 /* restore channel status for cyclic descriptors list: 2195 2084 * next descriptor in the cyclic list at the time of suspend */ ··· 2221 2110 2222 2111 /* restore saved data */ 2223 2112 dma_writel(atdma, EBCIER, atdma->save_imr); 2224 2224 - list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels, 2113 2113 + list_for_each_entry_safe(chan, _chan, &atdma->dma_device.channels, 2225 2114 device_node) { 2226 2115 struct at_dma_chan *atchan = to_at_dma_chan(chan); 2227 2116 ··· 2232 2121 return 0; 2233 2122 } 2234 2123 2235 2235 - static const struct dev_pm_ops at_dma_dev_pm_ops = { 2124 2124 + static const struct dev_pm_ops __maybe_unused at_dma_dev_pm_ops = { 2236 2125 .prepare = at_dma_prepare, 2237 2126 .suspend_noirq = at_dma_suspend_noirq, 2238 2127 .resume_noirq = at_dma_resume_noirq, ··· 2244 2133 .id_table = atdma_devtypes, 2245 2134 .driver = { 2246 2135 .name = "at_hdmac", 2247 2247 - .pm = &at_dma_dev_pm_ops, 2136 2136 + .pm = pm_ptr(&at_dma_dev_pm_ops), 2248 2137 .of_match_table = of_match_ptr(atmel_dma_dt_ids), 2249 2138 }, 2250 2139 }; ··· 2263 2152 2264 2153 MODULE_DESCRIPTION("Atmel AHB DMA Controller driver"); 2265 2154 MODULE_AUTHOR("Nicolas Ferre <nicolas.ferre@atmel.com>"); 2155 2155 + MODULE_AUTHOR("Tudor Ambarus <tudor.ambarus@microchip.com>"); 2266 2156 MODULE_LICENSE("GPL"); 2267 2157 MODULE_ALIAS("platform:at_hdmac");

-478

drivers/dma/at_hdmac_regs.h

reviewed

··· 1 1 - /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 2 - /* 3 3 - * Header file for the Atmel AHB DMA Controller driver 4 4 - * 5 5 - * Copyright (C) 2008 Atmel Corporation 6 6 - */ 7 7 - #ifndef AT_HDMAC_REGS_H 8 8 - #define AT_HDMAC_REGS_H 9 9 - 10 10 - #define AT_DMA_MAX_NR_CHANNELS 8 11 11 - 12 12 - 13 13 - #define AT_DMA_GCFG 0x00 /* Global Configuration Register */ 14 14 - #define AT_DMA_IF_BIGEND(i) (0x1 << (i)) /* AHB-Lite Interface i in Big-endian mode */ 15 15 - #define AT_DMA_ARB_CFG (0x1 << 4) /* Arbiter mode. */ 16 16 - #define AT_DMA_ARB_CFG_FIXED (0x0 << 4) 17 17 - #define AT_DMA_ARB_CFG_ROUND_ROBIN (0x1 << 4) 18 18 - 19 19 - #define AT_DMA_EN 0x04 /* Controller Enable Register */ 20 20 - #define AT_DMA_ENABLE (0x1 << 0) 21 21 - 22 22 - #define AT_DMA_SREQ 0x08 /* Software Single Request Register */ 23 23 - #define AT_DMA_SSREQ(x) (0x1 << ((x) << 1)) /* Request a source single transfer on channel x */ 24 24 - #define AT_DMA_DSREQ(x) (0x1 << (1 + ((x) << 1))) /* Request a destination single transfer on channel x */ 25 25 - 26 26 - #define AT_DMA_CREQ 0x0C /* Software Chunk Transfer Request Register */ 27 27 - #define AT_DMA_SCREQ(x) (0x1 << ((x) << 1)) /* Request a source chunk transfer on channel x */ 28 28 - #define AT_DMA_DCREQ(x) (0x1 << (1 + ((x) << 1))) /* Request a destination chunk transfer on channel x */ 29 29 - 30 30 - #define AT_DMA_LAST 0x10 /* Software Last Transfer Flag Register */ 31 31 - #define AT_DMA_SLAST(x) (0x1 << ((x) << 1)) /* This src rq is last tx of buffer on channel x */ 32 32 - #define AT_DMA_DLAST(x) (0x1 << (1 + ((x) << 1))) /* This dst rq is last tx of buffer on channel x */ 33 33 - 34 34 - #define AT_DMA_SYNC 0x14 /* Request Synchronization Register */ 35 35 - #define AT_DMA_SYR(h) (0x1 << (h)) /* Synchronize handshake line h */ 36 36 - 37 37 - /* Error, Chained Buffer transfer completed and Buffer transfer completed Interrupt registers */ 38 38 - #define AT_DMA_EBCIER 0x18 /* Enable register */ 39 39 - #define AT_DMA_EBCIDR 0x1C /* Disable register */ 40 40 - #define AT_DMA_EBCIMR 0x20 /* Mask Register */ 41 41 - #define AT_DMA_EBCISR 0x24 /* Status Register */ 42 42 - #define AT_DMA_CBTC_OFFSET 8 43 43 - #define AT_DMA_ERR_OFFSET 16 44 44 - #define AT_DMA_BTC(x) (0x1 << (x)) 45 45 - #define AT_DMA_CBTC(x) (0x1 << (AT_DMA_CBTC_OFFSET + (x))) 46 46 - #define AT_DMA_ERR(x) (0x1 << (AT_DMA_ERR_OFFSET + (x))) 47 47 - 48 48 - #define AT_DMA_CHER 0x28 /* Channel Handler Enable Register */ 49 49 - #define AT_DMA_ENA(x) (0x1 << (x)) 50 50 - #define AT_DMA_SUSP(x) (0x1 << ( 8 + (x))) 51 51 - #define AT_DMA_KEEP(x) (0x1 << (24 + (x))) 52 52 - 53 53 - #define AT_DMA_CHDR 0x2C /* Channel Handler Disable Register */ 54 54 - #define AT_DMA_DIS(x) (0x1 << (x)) 55 55 - #define AT_DMA_RES(x) (0x1 << ( 8 + (x))) 56 56 - 57 57 - #define AT_DMA_CHSR 0x30 /* Channel Handler Status Register */ 58 58 - #define AT_DMA_EMPT(x) (0x1 << (16 + (x))) 59 59 - #define AT_DMA_STAL(x) (0x1 << (24 + (x))) 60 60 - 61 61 - 62 62 - #define AT_DMA_CH_REGS_BASE 0x3C /* Channel registers base address */ 63 63 - #define ch_regs(x) (AT_DMA_CH_REGS_BASE + (x) * 0x28) /* Channel x base addr */ 64 64 - 65 65 - /* Hardware register offset for each channel */ 66 66 - #define ATC_SADDR_OFFSET 0x00 /* Source Address Register */ 67 67 - #define ATC_DADDR_OFFSET 0x04 /* Destination Address Register */ 68 68 - #define ATC_DSCR_OFFSET 0x08 /* Descriptor Address Register */ 69 69 - #define ATC_CTRLA_OFFSET 0x0C /* Control A Register */ 70 70 - #define ATC_CTRLB_OFFSET 0x10 /* Control B Register */ 71 71 - #define ATC_CFG_OFFSET 0x14 /* Configuration Register */ 72 72 - #define ATC_SPIP_OFFSET 0x18 /* Src PIP Configuration Register */ 73 73 - #define ATC_DPIP_OFFSET 0x1C /* Dst PIP Configuration Register */ 74 74 - 75 75 - 76 76 - /* Bitfield definitions */ 77 77 - 78 78 - /* Bitfields in DSCR */ 79 79 - #define ATC_DSCR_IF(i) (0x3 & (i)) /* Dsc feched via AHB-Lite Interface i */ 80 80 - 81 81 - /* Bitfields in CTRLA */ 82 82 - #define ATC_BTSIZE_MAX 0xFFFFUL /* Maximum Buffer Transfer Size */ 83 83 - #define ATC_BTSIZE(x) (ATC_BTSIZE_MAX & (x)) /* Buffer Transfer Size */ 84 84 - #define ATC_SCSIZE_MASK (0x7 << 16) /* Source Chunk Transfer Size */ 85 85 - #define ATC_SCSIZE(x) (ATC_SCSIZE_MASK & ((x) << 16)) 86 86 - #define ATC_SCSIZE_1 (0x0 << 16) 87 87 - #define ATC_SCSIZE_4 (0x1 << 16) 88 88 - #define ATC_SCSIZE_8 (0x2 << 16) 89 89 - #define ATC_SCSIZE_16 (0x3 << 16) 90 90 - #define ATC_SCSIZE_32 (0x4 << 16) 91 91 - #define ATC_SCSIZE_64 (0x5 << 16) 92 92 - #define ATC_SCSIZE_128 (0x6 << 16) 93 93 - #define ATC_SCSIZE_256 (0x7 << 16) 94 94 - #define ATC_DCSIZE_MASK (0x7 << 20) /* Destination Chunk Transfer Size */ 95 95 - #define ATC_DCSIZE(x) (ATC_DCSIZE_MASK & ((x) << 20)) 96 96 - #define ATC_DCSIZE_1 (0x0 << 20) 97 97 - #define ATC_DCSIZE_4 (0x1 << 20) 98 98 - #define ATC_DCSIZE_8 (0x2 << 20) 99 99 - #define ATC_DCSIZE_16 (0x3 << 20) 100 100 - #define ATC_DCSIZE_32 (0x4 << 20) 101 101 - #define ATC_DCSIZE_64 (0x5 << 20) 102 102 - #define ATC_DCSIZE_128 (0x6 << 20) 103 103 - #define ATC_DCSIZE_256 (0x7 << 20) 104 104 - #define ATC_SRC_WIDTH_MASK (0x3 << 24) /* Source Single Transfer Size */ 105 105 - #define ATC_SRC_WIDTH(x) ((x) << 24) 106 106 - #define ATC_SRC_WIDTH_BYTE (0x0 << 24) 107 107 - #define ATC_SRC_WIDTH_HALFWORD (0x1 << 24) 108 108 - #define ATC_SRC_WIDTH_WORD (0x2 << 24) 109 109 - #define ATC_REG_TO_SRC_WIDTH(r) (((r) >> 24) & 0x3) 110 110 - #define ATC_DST_WIDTH_MASK (0x3 << 28) /* Destination Single Transfer Size */ 111 111 - #define ATC_DST_WIDTH(x) ((x) << 28) 112 112 - #define ATC_DST_WIDTH_BYTE (0x0 << 28) 113 113 - #define ATC_DST_WIDTH_HALFWORD (0x1 << 28) 114 114 - #define ATC_DST_WIDTH_WORD (0x2 << 28) 115 115 - #define ATC_DONE (0x1 << 31) /* Tx Done (only written back in descriptor) */ 116 116 - 117 117 - /* Bitfields in CTRLB */ 118 118 - #define ATC_SIF(i) (0x3 & (i)) /* Src tx done via AHB-Lite Interface i */ 119 119 - #define ATC_DIF(i) ((0x3 & (i)) << 4) /* Dst tx done via AHB-Lite Interface i */ 120 120 - /* Specify AHB interfaces */ 121 121 - #define AT_DMA_MEM_IF 0 /* interface 0 as memory interface */ 122 122 - #define AT_DMA_PER_IF 1 /* interface 1 as peripheral interface */ 123 123 - 124 124 - #define ATC_SRC_PIP (0x1 << 8) /* Source Picture-in-Picture enabled */ 125 125 - #define ATC_DST_PIP (0x1 << 12) /* Destination Picture-in-Picture enabled */ 126 126 - #define ATC_SRC_DSCR_DIS (0x1 << 16) /* Src Descriptor fetch disable */ 127 127 - #define ATC_DST_DSCR_DIS (0x1 << 20) /* Dst Descriptor fetch disable */ 128 128 - #define ATC_FC_MASK (0x7 << 21) /* Choose Flow Controller */ 129 129 - #define ATC_FC_MEM2MEM (0x0 << 21) /* Mem-to-Mem (DMA) */ 130 130 - #define ATC_FC_MEM2PER (0x1 << 21) /* Mem-to-Periph (DMA) */ 131 131 - #define ATC_FC_PER2MEM (0x2 << 21) /* Periph-to-Mem (DMA) */ 132 132 - #define ATC_FC_PER2PER (0x3 << 21) /* Periph-to-Periph (DMA) */ 133 133 - #define ATC_FC_PER2MEM_PER (0x4 << 21) /* Periph-to-Mem (Peripheral) */ 134 134 - #define ATC_FC_MEM2PER_PER (0x5 << 21) /* Mem-to-Periph (Peripheral) */ 135 135 - #define ATC_FC_PER2PER_SRCPER (0x6 << 21) /* Periph-to-Periph (Src Peripheral) */ 136 136 - #define ATC_FC_PER2PER_DSTPER (0x7 << 21) /* Periph-to-Periph (Dst Peripheral) */ 137 137 - #define ATC_SRC_ADDR_MODE_MASK (0x3 << 24) 138 138 - #define ATC_SRC_ADDR_MODE_INCR (0x0 << 24) /* Incrementing Mode */ 139 139 - #define ATC_SRC_ADDR_MODE_DECR (0x1 << 24) /* Decrementing Mode */ 140 140 - #define ATC_SRC_ADDR_MODE_FIXED (0x2 << 24) /* Fixed Mode */ 141 141 - #define ATC_DST_ADDR_MODE_MASK (0x3 << 28) 142 142 - #define ATC_DST_ADDR_MODE_INCR (0x0 << 28) /* Incrementing Mode */ 143 143 - #define ATC_DST_ADDR_MODE_DECR (0x1 << 28) /* Decrementing Mode */ 144 144 - #define ATC_DST_ADDR_MODE_FIXED (0x2 << 28) /* Fixed Mode */ 145 145 - #define ATC_IEN (0x1 << 30) /* BTC interrupt enable (active low) */ 146 146 - #define ATC_AUTO (0x1 << 31) /* Auto multiple buffer tx enable */ 147 147 - 148 148 - /* Bitfields in CFG */ 149 149 - #define ATC_PER_MSB(h) ((0x30U & (h)) >> 4) /* Extract most significant bits of a handshaking identifier */ 150 150 - 151 151 - #define ATC_SRC_PER(h) (0xFU & (h)) /* Channel src rq associated with periph handshaking ifc h */ 152 152 - #define ATC_DST_PER(h) ((0xFU & (h)) << 4) /* Channel dst rq associated with periph handshaking ifc h */ 153 153 - #define ATC_SRC_REP (0x1 << 8) /* Source Replay Mod */ 154 154 - #define ATC_SRC_H2SEL (0x1 << 9) /* Source Handshaking Mod */ 155 155 - #define ATC_SRC_H2SEL_SW (0x0 << 9) 156 156 - #define ATC_SRC_H2SEL_HW (0x1 << 9) 157 157 - #define ATC_SRC_PER_MSB(h) (ATC_PER_MSB(h) << 10) /* Channel src rq (most significant bits) */ 158 158 - #define ATC_DST_REP (0x1 << 12) /* Destination Replay Mod */ 159 159 - #define ATC_DST_H2SEL (0x1 << 13) /* Destination Handshaking Mod */ 160 160 - #define ATC_DST_H2SEL_SW (0x0 << 13) 161 161 - #define ATC_DST_H2SEL_HW (0x1 << 13) 162 162 - #define ATC_DST_PER_MSB(h) (ATC_PER_MSB(h) << 14) /* Channel dst rq (most significant bits) */ 163 163 - #define ATC_SOD (0x1 << 16) /* Stop On Done */ 164 164 - #define ATC_LOCK_IF (0x1 << 20) /* Interface Lock */ 165 165 - #define ATC_LOCK_B (0x1 << 21) /* AHB Bus Lock */ 166 166 - #define ATC_LOCK_IF_L (0x1 << 22) /* Master Interface Arbiter Lock */ 167 167 - #define ATC_LOCK_IF_L_CHUNK (0x0 << 22) 168 168 - #define ATC_LOCK_IF_L_BUFFER (0x1 << 22) 169 169 - #define ATC_AHB_PROT_MASK (0x7 << 24) /* AHB Protection */ 170 170 - #define ATC_FIFOCFG_MASK (0x3 << 28) /* FIFO Request Configuration */ 171 171 - #define ATC_FIFOCFG_LARGESTBURST (0x0 << 28) 172 172 - #define ATC_FIFOCFG_HALFFIFO (0x1 << 28) 173 173 - #define ATC_FIFOCFG_ENOUGHSPACE (0x2 << 28) 174 174 - 175 175 - /* Bitfields in SPIP */ 176 176 - #define ATC_SPIP_HOLE(x) (0xFFFFU & (x)) 177 177 - #define ATC_SPIP_BOUNDARY(x) ((0x3FF & (x)) << 16) 178 178 - 179 179 - /* Bitfields in DPIP */ 180 180 - #define ATC_DPIP_HOLE(x) (0xFFFFU & (x)) 181 181 - #define ATC_DPIP_BOUNDARY(x) ((0x3FF & (x)) << 16) 182 182 - 183 183 - 184 184 - /*-- descriptors -----------------------------------------------------*/ 185 185 - 186 186 - /* LLI == Linked List Item; aka DMA buffer descriptor */ 187 187 - struct at_lli { 188 188 - /* values that are not changed by hardware */ 189 189 - u32 saddr; 190 190 - u32 daddr; 191 191 - /* value that may get written back: */ 192 192 - u32 ctrla; 193 193 - /* more values that are not changed by hardware */ 194 194 - u32 ctrlb; 195 195 - u32 dscr; /* chain to next lli */ 196 196 - }; 197 197 - 198 198 - /** 199 199 - * struct at_desc - software descriptor 200 200 - * @at_lli: hardware lli structure 201 201 - * @txd: support for the async_tx api 202 202 - * @desc_node: node on the channed descriptors list 203 203 - * @len: descriptor byte count 204 204 - * @total_len: total transaction byte count 205 205 - */ 206 206 - struct at_desc { 207 207 - /* FIRST values the hardware uses */ 208 208 - struct at_lli lli; 209 209 - 210 210 - /* THEN values for driver housekeeping */ 211 211 - struct list_head tx_list; 212 212 - struct dma_async_tx_descriptor txd; 213 213 - struct list_head desc_node; 214 214 - size_t len; 215 215 - size_t total_len; 216 216 - 217 217 - /* Interleaved data */ 218 218 - size_t boundary; 219 219 - size_t dst_hole; 220 220 - size_t src_hole; 221 221 - 222 222 - /* Memset temporary buffer */ 223 223 - bool memset_buffer; 224 224 - dma_addr_t memset_paddr; 225 225 - int *memset_vaddr; 226 226 - }; 227 227 - 228 228 - static inline struct at_desc * 229 229 - txd_to_at_desc(struct dma_async_tx_descriptor *txd) 230 230 - { 231 231 - return container_of(txd, struct at_desc, txd); 232 232 - } 233 233 - 234 234 - 235 235 - /*-- Channels --------------------------------------------------------*/ 236 236 - 237 237 - /** 238 238 - * atc_status - information bits stored in channel status flag 239 239 - * 240 240 - * Manipulated with atomic operations. 241 241 - */ 242 242 - enum atc_status { 243 243 - ATC_IS_ERROR = 0, 244 244 - ATC_IS_PAUSED = 1, 245 245 - ATC_IS_CYCLIC = 24, 246 246 - }; 247 247 - 248 248 - /** 249 249 - * struct at_dma_chan - internal representation of an Atmel HDMAC channel 250 250 - * @chan_common: common dmaengine channel object members 251 251 - * @device: parent device 252 252 - * @ch_regs: memory mapped register base 253 253 - * @mask: channel index in a mask 254 254 - * @per_if: peripheral interface 255 255 - * @mem_if: memory interface 256 256 - * @status: transmit status information from irq/prep* functions 257 257 - * to tasklet (use atomic operations) 258 258 - * @tasklet: bottom half to finish transaction work 259 259 - * @save_cfg: configuration register that is saved on suspend/resume cycle 260 260 - * @save_dscr: for cyclic operations, preserve next descriptor address in 261 261 - * the cyclic list on suspend/resume cycle 262 262 - * @dma_sconfig: configuration for slave transfers, passed via 263 263 - * .device_config 264 264 - * @lock: serializes enqueue/dequeue operations to descriptors lists 265 265 - * @active_list: list of descriptors dmaengine is being running on 266 266 - * @queue: list of descriptors ready to be submitted to engine 267 267 - * @free_list: list of descriptors usable by the channel 268 268 - */ 269 269 - struct at_dma_chan { 270 270 - struct dma_chan chan_common; 271 271 - struct at_dma *device; 272 272 - void __iomem *ch_regs; 273 273 - u8 mask; 274 274 - u8 per_if; 275 275 - u8 mem_if; 276 276 - unsigned long status; 277 277 - struct tasklet_struct tasklet; 278 278 - u32 save_cfg; 279 279 - u32 save_dscr; 280 280 - struct dma_slave_config dma_sconfig; 281 281 - 282 282 - spinlock_t lock; 283 283 - 284 284 - /* these other elements are all protected by lock */ 285 285 - struct list_head active_list; 286 286 - struct list_head queue; 287 287 - struct list_head free_list; 288 288 - }; 289 289 - 290 290 - #define channel_readl(atchan, name) \ 291 291 - __raw_readl((atchan)->ch_regs + ATC_##name##_OFFSET) 292 292 - 293 293 - #define channel_writel(atchan, name, val) \ 294 294 - __raw_writel((val), (atchan)->ch_regs + ATC_##name##_OFFSET) 295 295 - 296 296 - static inline struct at_dma_chan *to_at_dma_chan(struct dma_chan *dchan) 297 297 - { 298 298 - return container_of(dchan, struct at_dma_chan, chan_common); 299 299 - } 300 300 - 301 301 - /* 302 302 - * Fix sconfig's burst size according to at_hdmac. We need to convert them as: 303 303 - * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3, 32 -> 4, 64 -> 5, 128 -> 6, 256 -> 7. 304 304 - * 305 305 - * This can be done by finding most significant bit set. 306 306 - */ 307 307 - static inline void convert_burst(u32 *maxburst) 308 308 - { 309 309 - if (*maxburst > 1) 310 310 - *maxburst = fls(*maxburst) - 2; 311 311 - else 312 312 - *maxburst = 0; 313 313 - } 314 314 - 315 315 - /* 316 316 - * Fix sconfig's bus width according to at_hdmac. 317 317 - * 1 byte -> 0, 2 bytes -> 1, 4 bytes -> 2. 318 318 - */ 319 319 - static inline u8 convert_buswidth(enum dma_slave_buswidth addr_width) 320 320 - { 321 321 - switch (addr_width) { 322 322 - case DMA_SLAVE_BUSWIDTH_2_BYTES: 323 323 - return 1; 324 324 - case DMA_SLAVE_BUSWIDTH_4_BYTES: 325 325 - return 2; 326 326 - default: 327 327 - /* For 1 byte width or fallback */ 328 328 - return 0; 329 329 - } 330 330 - } 331 331 - 332 332 - /*-- Controller ------------------------------------------------------*/ 333 333 - 334 334 - /** 335 335 - * struct at_dma - internal representation of an Atmel HDMA Controller 336 336 - * @chan_common: common dmaengine dma_device object members 337 337 - * @atdma_devtype: identifier of DMA controller compatibility 338 338 - * @ch_regs: memory mapped register base 339 339 - * @clk: dma controller clock 340 340 - * @save_imr: interrupt mask register that is saved on suspend/resume cycle 341 341 - * @all_chan_mask: all channels availlable in a mask 342 342 - * @dma_desc_pool: base of DMA descriptor region (DMA address) 343 343 - * @chan: channels table to store at_dma_chan structures 344 344 - */ 345 345 - struct at_dma { 346 346 - struct dma_device dma_common; 347 347 - void __iomem *regs; 348 348 - struct clk *clk; 349 349 - u32 save_imr; 350 350 - 351 351 - u8 all_chan_mask; 352 352 - 353 353 - struct dma_pool *dma_desc_pool; 354 354 - struct dma_pool *memset_pool; 355 355 - /* AT THE END channels table */ 356 356 - struct at_dma_chan chan[]; 357 357 - }; 358 358 - 359 359 - #define dma_readl(atdma, name) \ 360 360 - __raw_readl((atdma)->regs + AT_DMA_##name) 361 361 - #define dma_writel(atdma, name, val) \ 362 362 - __raw_writel((val), (atdma)->regs + AT_DMA_##name) 363 363 - 364 364 - static inline struct at_dma *to_at_dma(struct dma_device *ddev) 365 365 - { 366 366 - return container_of(ddev, struct at_dma, dma_common); 367 367 - } 368 368 - 369 369 - 370 370 - /*-- Helper functions ------------------------------------------------*/ 371 371 - 372 372 - static struct device *chan2dev(struct dma_chan *chan) 373 373 - { 374 374 - return &chan->dev->device; 375 375 - } 376 376 - 377 377 - #if defined(VERBOSE_DEBUG) 378 378 - static void vdbg_dump_regs(struct at_dma_chan *atchan) 379 379 - { 380 380 - struct at_dma *atdma = to_at_dma(atchan->chan_common.device); 381 381 - 382 382 - dev_err(chan2dev(&atchan->chan_common), 383 383 - " channel %d : imr = 0x%x, chsr = 0x%x\n", 384 384 - atchan->chan_common.chan_id, 385 385 - dma_readl(atdma, EBCIMR), 386 386 - dma_readl(atdma, CHSR)); 387 387 - 388 388 - dev_err(chan2dev(&atchan->chan_common), 389 389 - " channel: s0x%x d0x%x ctrl0x%x:0x%x cfg0x%x l0x%x\n", 390 390 - channel_readl(atchan, SADDR), 391 391 - channel_readl(atchan, DADDR), 392 392 - channel_readl(atchan, CTRLA), 393 393 - channel_readl(atchan, CTRLB), 394 394 - channel_readl(atchan, CFG), 395 395 - channel_readl(atchan, DSCR)); 396 396 - } 397 397 - #else 398 398 - static void vdbg_dump_regs(struct at_dma_chan *atchan) {} 399 399 - #endif 400 400 - 401 401 - static void atc_dump_lli(struct at_dma_chan *atchan, struct at_lli *lli) 402 402 - { 403 403 - dev_crit(chan2dev(&atchan->chan_common), 404 404 - "desc: s%pad d%pad ctrl0x%x:0x%x l%pad\n", 405 405 - &lli->saddr, &lli->daddr, 406 406 - lli->ctrla, lli->ctrlb, &lli->dscr); 407 407 - } 408 408 - 409 409 - 410 410 - static void atc_setup_irq(struct at_dma *atdma, int chan_id, int on) 411 411 - { 412 412 - u32 ebci; 413 413 - 414 414 - /* enable interrupts on buffer transfer completion & error */ 415 415 - ebci = AT_DMA_BTC(chan_id) 416 416 - | AT_DMA_ERR(chan_id); 417 417 - if (on) 418 418 - dma_writel(atdma, EBCIER, ebci); 419 419 - else 420 420 - dma_writel(atdma, EBCIDR, ebci); 421 421 - } 422 422 - 423 423 - static void atc_enable_chan_irq(struct at_dma *atdma, int chan_id) 424 424 - { 425 425 - atc_setup_irq(atdma, chan_id, 1); 426 426 - } 427 427 - 428 428 - static void atc_disable_chan_irq(struct at_dma *atdma, int chan_id) 429 429 - { 430 430 - atc_setup_irq(atdma, chan_id, 0); 431 431 - } 432 432 - 433 433 - 434 434 - /** 435 435 - * atc_chan_is_enabled - test if given channel is enabled 436 436 - * @atchan: channel we want to test status 437 437 - */ 438 438 - static inline int atc_chan_is_enabled(struct at_dma_chan *atchan) 439 439 - { 440 440 - struct at_dma *atdma = to_at_dma(atchan->chan_common.device); 441 441 - 442 442 - return !!(dma_readl(atdma, CHSR) & atchan->mask); 443 443 - } 444 444 - 445 445 - /** 446 446 - * atc_chan_is_paused - test channel pause/resume status 447 447 - * @atchan: channel we want to test status 448 448 - */ 449 449 - static inline int atc_chan_is_paused(struct at_dma_chan *atchan) 450 450 - { 451 451 - return test_bit(ATC_IS_PAUSED, &atchan->status); 452 452 - } 453 453 - 454 454 - /** 455 455 - * atc_chan_is_cyclic - test if given channel has cyclic property set 456 456 - * @atchan: channel we want to test status 457 457 - */ 458 458 - static inline int atc_chan_is_cyclic(struct at_dma_chan *atchan) 459 459 - { 460 460 - return test_bit(ATC_IS_CYCLIC, &atchan->status); 461 461 - } 462 462 - 463 463 - /** 464 464 - * set_desc_eol - set end-of-link to descriptor so it will end transfer 465 465 - * @desc: descriptor, signle or at the end of a chain, to end chain on 466 466 - */ 467 467 - static void set_desc_eol(struct at_desc *desc) 468 468 - { 469 469 - u32 ctrlb = desc->lli.ctrlb; 470 470 - 471 471 - ctrlb &= ~ATC_IEN; 472 472 - ctrlb |= ATC_SRC_DSCR_DIS | ATC_DST_DSCR_DIS; 473 473 - 474 474 - desc->lli.ctrlb = ctrlb; 475 475 - desc->lli.dscr = 0; 476 476 - } 477 477 - 478 478 - #endif /* AT_HDMAC_REGS_H */

+8

drivers/dma/dma-jz4780.c

reviewed

··· 1038 1038 JZ_SOC_DATA_BREAK_LINKS, 1039 1039 }; 1040 1040 1041 1041 + static const struct jz4780_dma_soc_data jz4755_dma_soc_data = { 1042 1042 + .nb_channels = 4, 1043 1043 + .transfer_ord_max = 5, 1044 1044 + .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC | 1045 1045 + JZ_SOC_DATA_BREAK_LINKS, 1046 1046 + }; 1047 1047 + 1041 1048 static const struct jz4780_dma_soc_data jz4760_dma_soc_data = { 1042 1049 .nb_channels = 5, 1043 1050 .transfer_ord_max = 6, ··· 1108 1101 static const struct of_device_id jz4780_dma_dt_match[] = { 1109 1102 { .compatible = "ingenic,jz4740-dma", .data = &jz4740_dma_soc_data }, 1110 1103 { .compatible = "ingenic,jz4725b-dma", .data = &jz4725b_dma_soc_data }, 1104 1104 + { .compatible = "ingenic,jz4755-dma", .data = &jz4755_dma_soc_data }, 1111 1105 { .compatible = "ingenic,jz4760-dma", .data = &jz4760_dma_soc_data }, 1112 1106 { .compatible = "ingenic,jz4760-mdma", .data = &jz4760_mdma_soc_data }, 1113 1107 { .compatible = "ingenic,jz4760-bdma", .data = &jz4760_bdma_soc_data },

+4 -4

drivers/dma/idma64.c

reviewed

··· 600 600 return 0; 601 601 } 602 602 603 603 - static int idma64_remove(struct idma64_chip *chip) 603 603 + static void idma64_remove(struct idma64_chip *chip) 604 604 { 605 605 struct idma64 *idma64 = chip->idma64; 606 606 unsigned short i; ··· 618 618 619 619 tasklet_kill(&idma64c->vchan.task); 620 620 } 621 621 - 622 622 - return 0; 623 621 } 624 622 625 623 /* ---------------------------------------------------------------------- */ ··· 662 664 { 663 665 struct idma64_chip *chip = platform_get_drvdata(pdev); 664 666 665 665 - return idma64_remove(chip); 667 667 + idma64_remove(chip); 668 668 + 669 669 + return 0; 666 670 } 667 671 668 672 static int __maybe_unused idma64_pm_suspend(struct device *dev)

-1

drivers/dma/idxd/device.c

reviewed

··· 7 7 #include <linux/io-64-nonatomic-lo-hi.h> 8 8 #include <linux/dmaengine.h> 9 9 #include <linux/irq.h> 10 10 - #include <linux/msi.h> 11 10 #include <uapi/linux/idxd.h> 12 11 #include "../dmaengine.h" 13 12 #include "idxd.h"

+68

drivers/dma/idxd/sysfs.c

reviewed

··· 528 528 !idxd->hw.group_cap.progress_limit; 529 529 } 530 530 531 531 + static bool idxd_group_attr_read_buffers_invisible(struct attribute *attr, 532 532 + struct idxd_device *idxd) 533 533 + { 534 534 + /* 535 535 + * Intel IAA does not support Read Buffer allocation control, 536 536 + * make these attributes invisible. 537 537 + */ 538 538 + return (attr == &dev_attr_group_use_token_limit.attr || 539 539 + attr == &dev_attr_group_use_read_buffer_limit.attr || 540 540 + attr == &dev_attr_group_tokens_allowed.attr || 541 541 + attr == &dev_attr_group_read_buffers_allowed.attr || 542 542 + attr == &dev_attr_group_tokens_reserved.attr || 543 543 + attr == &dev_attr_group_read_buffers_reserved.attr) && 544 544 + idxd->data->type == IDXD_TYPE_IAX; 545 545 + } 546 546 + 531 547 static umode_t idxd_group_attr_visible(struct kobject *kobj, 532 548 struct attribute *attr, int n) 533 549 { ··· 552 536 struct idxd_device *idxd = group->idxd; 553 537 554 538 if (idxd_group_attr_progress_limit_invisible(attr, idxd)) 539 539 + return 0; 540 540 + 541 541 + if (idxd_group_attr_read_buffers_invisible(attr, idxd)) 555 542 return 0; 556 543 557 544 return attr->mode; ··· 1252 1233 !idxd->hw.wq_cap.op_config; 1253 1234 } 1254 1235 1236 1236 + static bool idxd_wq_attr_max_batch_size_invisible(struct attribute *attr, 1237 1237 + struct idxd_device *idxd) 1238 1238 + { 1239 1239 + /* Intel IAA does not support batch processing, make it invisible */ 1240 1240 + return attr == &dev_attr_wq_max_batch_size.attr && 1241 1241 + idxd->data->type == IDXD_TYPE_IAX; 1242 1242 + } 1243 1243 + 1255 1244 static umode_t idxd_wq_attr_visible(struct kobject *kobj, 1256 1245 struct attribute *attr, int n) 1257 1246 { ··· 1268 1241 struct idxd_device *idxd = wq->idxd; 1269 1242 1270 1243 if (idxd_wq_attr_op_config_invisible(attr, idxd)) 1244 1244 + return 0; 1245 1245 + 1246 1246 + if (idxd_wq_attr_max_batch_size_invisible(attr, idxd)) 1271 1247 return 0; 1272 1248 1273 1249 return attr->mode; ··· 1563 1533 } 1564 1534 static DEVICE_ATTR_RW(cmd_status); 1565 1535 1536 1536 + static bool idxd_device_attr_max_batch_size_invisible(struct attribute *attr, 1537 1537 + struct idxd_device *idxd) 1538 1538 + { 1539 1539 + /* Intel IAA does not support batch processing, make it invisible */ 1540 1540 + return attr == &dev_attr_max_batch_size.attr && 1541 1541 + idxd->data->type == IDXD_TYPE_IAX; 1542 1542 + } 1543 1543 + 1544 1544 + static bool idxd_device_attr_read_buffers_invisible(struct attribute *attr, 1545 1545 + struct idxd_device *idxd) 1546 1546 + { 1547 1547 + /* 1548 1548 + * Intel IAA does not support Read Buffer allocation control, 1549 1549 + * make these attributes invisible. 1550 1550 + */ 1551 1551 + return (attr == &dev_attr_max_tokens.attr || 1552 1552 + attr == &dev_attr_max_read_buffers.attr || 1553 1553 + attr == &dev_attr_token_limit.attr || 1554 1554 + attr == &dev_attr_read_buffer_limit.attr) && 1555 1555 + idxd->data->type == IDXD_TYPE_IAX; 1556 1556 + } 1557 1557 + 1558 1558 + static umode_t idxd_device_attr_visible(struct kobject *kobj, 1559 1559 + struct attribute *attr, int n) 1560 1560 + { 1561 1561 + struct device *dev = container_of(kobj, struct device, kobj); 1562 1562 + struct idxd_device *idxd = confdev_to_idxd(dev); 1563 1563 + 1564 1564 + if (idxd_device_attr_max_batch_size_invisible(attr, idxd)) 1565 1565 + return 0; 1566 1566 + 1567 1567 + if (idxd_device_attr_read_buffers_invisible(attr, idxd)) 1568 1568 + return 0; 1569 1569 + 1570 1570 + return attr->mode; 1571 1571 + } 1572 1572 + 1566 1573 static struct attribute *idxd_device_attributes[] = { 1567 1574 &dev_attr_version.attr, 1568 1575 &dev_attr_max_groups.attr, ··· 1627 1560 1628 1561 static const struct attribute_group idxd_device_attribute_group = { 1629 1562 .attrs = idxd_device_attributes, 1563 1563 + .is_visible = idxd_device_attr_visible, 1630 1564 }; 1631 1565 1632 1566 static const struct attribute_group *idxd_attribute_groups[] = {

+1 -1

drivers/dma/ioat/dma.c

reviewed

··· 33 33 static int idle_timeout = 2000; 34 34 module_param(idle_timeout, int, 0644); 35 35 MODULE_PARM_DESC(idle_timeout, 36 36 - "set ioat idel timeout [msec] (default 2000 [msec])"); 36 36 + "set ioat idle timeout [msec] (default 2000 [msec])"); 37 37 38 38 #define IDLE_TIMEOUT msecs_to_jiffies(idle_timeout) 39 39 #define COMPLETION_TIMEOUT msecs_to_jiffies(completion_timeout)

-1554

drivers/dma/iop-adma.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-only 2 2 - /* 3 3 - * offload engine driver for the Intel Xscale series of i/o processors 4 4 - * Copyright © 2006, Intel Corporation. 5 5 - */ 6 6 - 7 7 - /* 8 8 - * This driver supports the asynchrounous DMA copy and RAID engines available 9 9 - * on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) 10 10 - */ 11 11 - 12 12 - #include <linux/init.h> 13 13 - #include <linux/module.h> 14 14 - #include <linux/delay.h> 15 15 - #include <linux/dma-mapping.h> 16 16 - #include <linux/spinlock.h> 17 17 - #include <linux/interrupt.h> 18 18 - #include <linux/platform_device.h> 19 19 - #include <linux/prefetch.h> 20 20 - #include <linux/memory.h> 21 21 - #include <linux/ioport.h> 22 22 - #include <linux/raid/pq.h> 23 23 - #include <linux/slab.h> 24 24 - 25 25 - #include "iop-adma.h" 26 26 - #include "dmaengine.h" 27 27 - 28 28 - #define to_iop_adma_chan(chan) container_of(chan, struct iop_adma_chan, common) 29 29 - #define to_iop_adma_device(dev) \ 30 30 - container_of(dev, struct iop_adma_device, common) 31 31 - #define tx_to_iop_adma_slot(tx) \ 32 32 - container_of(tx, struct iop_adma_desc_slot, async_tx) 33 33 - 34 34 - /** 35 35 - * iop_adma_free_slots - flags descriptor slots for reuse 36 36 - * @slot: Slot to free 37 37 - * Caller must hold &iop_chan->lock while calling this function 38 38 - */ 39 39 - static void iop_adma_free_slots(struct iop_adma_desc_slot *slot) 40 40 - { 41 41 - int stride = slot->slots_per_op; 42 42 - 43 43 - while (stride--) { 44 44 - slot->slots_per_op = 0; 45 45 - slot = list_entry(slot->slot_node.next, 46 46 - struct iop_adma_desc_slot, 47 47 - slot_node); 48 48 - } 49 49 - } 50 50 - 51 51 - static dma_cookie_t 52 52 - iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc, 53 53 - struct iop_adma_chan *iop_chan, dma_cookie_t cookie) 54 54 - { 55 55 - struct dma_async_tx_descriptor *tx = &desc->async_tx; 56 56 - 57 57 - BUG_ON(tx->cookie < 0); 58 58 - if (tx->cookie > 0) { 59 59 - cookie = tx->cookie; 60 60 - tx->cookie = 0; 61 61 - 62 62 - /* call the callback (must not sleep or submit new 63 63 - * operations to this channel) 64 64 - */ 65 65 - dmaengine_desc_get_callback_invoke(tx, NULL); 66 66 - 67 67 - dma_descriptor_unmap(tx); 68 68 - if (desc->group_head) 69 69 - desc->group_head = NULL; 70 70 - } 71 71 - 72 72 - /* run dependent operations */ 73 73 - dma_run_dependencies(tx); 74 74 - 75 75 - return cookie; 76 76 - } 77 77 - 78 78 - static int 79 79 - iop_adma_clean_slot(struct iop_adma_desc_slot *desc, 80 80 - struct iop_adma_chan *iop_chan) 81 81 - { 82 82 - /* the client is allowed to attach dependent operations 83 83 - * until 'ack' is set 84 84 - */ 85 85 - if (!async_tx_test_ack(&desc->async_tx)) 86 86 - return 0; 87 87 - 88 88 - /* leave the last descriptor in the chain 89 89 - * so we can append to it 90 90 - */ 91 91 - if (desc->chain_node.next == &iop_chan->chain) 92 92 - return 1; 93 93 - 94 94 - dev_dbg(iop_chan->device->common.dev, 95 95 - "\tfree slot: %d slots_per_op: %d\n", 96 96 - desc->idx, desc->slots_per_op); 97 97 - 98 98 - list_del(&desc->chain_node); 99 99 - iop_adma_free_slots(desc); 100 100 - 101 101 - return 0; 102 102 - } 103 103 - 104 104 - static void __iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan) 105 105 - { 106 106 - struct iop_adma_desc_slot *iter, *_iter, *grp_start = NULL; 107 107 - dma_cookie_t cookie = 0; 108 108 - u32 current_desc = iop_chan_get_current_descriptor(iop_chan); 109 109 - int busy = iop_chan_is_busy(iop_chan); 110 110 - int seen_current = 0, slot_cnt = 0, slots_per_op = 0; 111 111 - 112 112 - dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); 113 113 - /* free completed slots from the chain starting with 114 114 - * the oldest descriptor 115 115 - */ 116 116 - list_for_each_entry_safe(iter, _iter, &iop_chan->chain, 117 117 - chain_node) { 118 118 - pr_debug("\tcookie: %d slot: %d busy: %d " 119 119 - "this_desc: %pad next_desc: %#llx ack: %d\n", 120 120 - iter->async_tx.cookie, iter->idx, busy, 121 121 - &iter->async_tx.phys, (u64)iop_desc_get_next_desc(iter), 122 122 - async_tx_test_ack(&iter->async_tx)); 123 123 - prefetch(_iter); 124 124 - prefetch(&_iter->async_tx); 125 125 - 126 126 - /* do not advance past the current descriptor loaded into the 127 127 - * hardware channel, subsequent descriptors are either in 128 128 - * process or have not been submitted 129 129 - */ 130 130 - if (seen_current) 131 131 - break; 132 132 - 133 133 - /* stop the search if we reach the current descriptor and the 134 134 - * channel is busy, or if it appears that the current descriptor 135 135 - * needs to be re-read (i.e. has been appended to) 136 136 - */ 137 137 - if (iter->async_tx.phys == current_desc) { 138 138 - BUG_ON(seen_current++); 139 139 - if (busy || iop_desc_get_next_desc(iter)) 140 140 - break; 141 141 - } 142 142 - 143 143 - /* detect the start of a group transaction */ 144 144 - if (!slot_cnt && !slots_per_op) { 145 145 - slot_cnt = iter->slot_cnt; 146 146 - slots_per_op = iter->slots_per_op; 147 147 - if (slot_cnt <= slots_per_op) { 148 148 - slot_cnt = 0; 149 149 - slots_per_op = 0; 150 150 - } 151 151 - } 152 152 - 153 153 - if (slot_cnt) { 154 154 - pr_debug("\tgroup++\n"); 155 155 - if (!grp_start) 156 156 - grp_start = iter; 157 157 - slot_cnt -= slots_per_op; 158 158 - } 159 159 - 160 160 - /* all the members of a group are complete */ 161 161 - if (slots_per_op != 0 && slot_cnt == 0) { 162 162 - struct iop_adma_desc_slot *grp_iter, *_grp_iter; 163 163 - int end_of_chain = 0; 164 164 - pr_debug("\tgroup end\n"); 165 165 - 166 166 - /* collect the total results */ 167 167 - if (grp_start->xor_check_result) { 168 168 - u32 zero_sum_result = 0; 169 169 - slot_cnt = grp_start->slot_cnt; 170 170 - grp_iter = grp_start; 171 171 - 172 172 - list_for_each_entry_from(grp_iter, 173 173 - &iop_chan->chain, chain_node) { 174 174 - zero_sum_result |= 175 175 - iop_desc_get_zero_result(grp_iter); 176 176 - pr_debug("\titer%d result: %d\n", 177 177 - grp_iter->idx, zero_sum_result); 178 178 - slot_cnt -= slots_per_op; 179 179 - if (slot_cnt == 0) 180 180 - break; 181 181 - } 182 182 - pr_debug("\tgrp_start->xor_check_result: %p\n", 183 183 - grp_start->xor_check_result); 184 184 - *grp_start->xor_check_result = zero_sum_result; 185 185 - } 186 186 - 187 187 - /* clean up the group */ 188 188 - slot_cnt = grp_start->slot_cnt; 189 189 - grp_iter = grp_start; 190 190 - list_for_each_entry_safe_from(grp_iter, _grp_iter, 191 191 - &iop_chan->chain, chain_node) { 192 192 - cookie = iop_adma_run_tx_complete_actions( 193 193 - grp_iter, iop_chan, cookie); 194 194 - 195 195 - slot_cnt -= slots_per_op; 196 196 - end_of_chain = iop_adma_clean_slot(grp_iter, 197 197 - iop_chan); 198 198 - 199 199 - if (slot_cnt == 0 || end_of_chain) 200 200 - break; 201 201 - } 202 202 - 203 203 - /* the group should be complete at this point */ 204 204 - BUG_ON(slot_cnt); 205 205 - 206 206 - slots_per_op = 0; 207 207 - grp_start = NULL; 208 208 - if (end_of_chain) 209 209 - break; 210 210 - else 211 211 - continue; 212 212 - } else if (slots_per_op) /* wait for group completion */ 213 213 - continue; 214 214 - 215 215 - /* write back zero sum results (single descriptor case) */ 216 216 - if (iter->xor_check_result && iter->async_tx.cookie) 217 217 - *iter->xor_check_result = 218 218 - iop_desc_get_zero_result(iter); 219 219 - 220 220 - cookie = iop_adma_run_tx_complete_actions( 221 221 - iter, iop_chan, cookie); 222 222 - 223 223 - if (iop_adma_clean_slot(iter, iop_chan)) 224 224 - break; 225 225 - } 226 226 - 227 227 - if (cookie > 0) { 228 228 - iop_chan->common.completed_cookie = cookie; 229 229 - pr_debug("\tcompleted cookie %d\n", cookie); 230 230 - } 231 231 - } 232 232 - 233 233 - static void 234 234 - iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan) 235 235 - { 236 236 - spin_lock_bh(&iop_chan->lock); 237 237 - __iop_adma_slot_cleanup(iop_chan); 238 238 - spin_unlock_bh(&iop_chan->lock); 239 239 - } 240 240 - 241 241 - static void iop_adma_tasklet(struct tasklet_struct *t) 242 242 - { 243 243 - struct iop_adma_chan *iop_chan = from_tasklet(iop_chan, t, 244 244 - irq_tasklet); 245 245 - 246 246 - /* lockdep will flag depedency submissions as potentially 247 247 - * recursive locking, this is not the case as a dependency 248 248 - * submission will never recurse a channels submit routine. 249 249 - * There are checks in async_tx.c to prevent this. 250 250 - */ 251 251 - spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING); 252 252 - __iop_adma_slot_cleanup(iop_chan); 253 253 - spin_unlock(&iop_chan->lock); 254 254 - } 255 255 - 256 256 - static struct iop_adma_desc_slot * 257 257 - iop_adma_alloc_slots(struct iop_adma_chan *iop_chan, int num_slots, 258 258 - int slots_per_op) 259 259 - { 260 260 - struct iop_adma_desc_slot *iter, *_iter, *alloc_start = NULL; 261 261 - LIST_HEAD(chain); 262 262 - int slots_found, retry = 0; 263 263 - 264 264 - /* start search from the last allocated descrtiptor 265 265 - * if a contiguous allocation can not be found start searching 266 266 - * from the beginning of the list 267 267 - */ 268 268 - retry: 269 269 - slots_found = 0; 270 270 - if (retry == 0) 271 271 - iter = iop_chan->last_used; 272 272 - else 273 273 - iter = list_entry(&iop_chan->all_slots, 274 274 - struct iop_adma_desc_slot, 275 275 - slot_node); 276 276 - 277 277 - list_for_each_entry_safe_continue( 278 278 - iter, _iter, &iop_chan->all_slots, slot_node) { 279 279 - prefetch(_iter); 280 280 - prefetch(&_iter->async_tx); 281 281 - if (iter->slots_per_op) { 282 282 - /* give up after finding the first busy slot 283 283 - * on the second pass through the list 284 284 - */ 285 285 - if (retry) 286 286 - break; 287 287 - 288 288 - slots_found = 0; 289 289 - continue; 290 290 - } 291 291 - 292 292 - /* start the allocation if the slot is correctly aligned */ 293 293 - if (!slots_found++) { 294 294 - if (iop_desc_is_aligned(iter, slots_per_op)) 295 295 - alloc_start = iter; 296 296 - else { 297 297 - slots_found = 0; 298 298 - continue; 299 299 - } 300 300 - } 301 301 - 302 302 - if (slots_found == num_slots) { 303 303 - struct iop_adma_desc_slot *alloc_tail = NULL; 304 304 - struct iop_adma_desc_slot *last_used = NULL; 305 305 - iter = alloc_start; 306 306 - while (num_slots) { 307 307 - int i; 308 308 - dev_dbg(iop_chan->device->common.dev, 309 309 - "allocated slot: %d " 310 310 - "(desc %p phys: %#llx) slots_per_op %d\n", 311 311 - iter->idx, iter->hw_desc, 312 312 - (u64)iter->async_tx.phys, slots_per_op); 313 313 - 314 314 - /* pre-ack all but the last descriptor */ 315 315 - if (num_slots != slots_per_op) 316 316 - async_tx_ack(&iter->async_tx); 317 317 - 318 318 - list_add_tail(&iter->chain_node, &chain); 319 319 - alloc_tail = iter; 320 320 - iter->async_tx.cookie = 0; 321 321 - iter->slot_cnt = num_slots; 322 322 - iter->xor_check_result = NULL; 323 323 - for (i = 0; i < slots_per_op; i++) { 324 324 - iter->slots_per_op = slots_per_op - i; 325 325 - last_used = iter; 326 326 - iter = list_entry(iter->slot_node.next, 327 327 - struct iop_adma_desc_slot, 328 328 - slot_node); 329 329 - } 330 330 - num_slots -= slots_per_op; 331 331 - } 332 332 - alloc_tail->group_head = alloc_start; 333 333 - alloc_tail->async_tx.cookie = -EBUSY; 334 334 - list_splice(&chain, &alloc_tail->tx_list); 335 335 - iop_chan->last_used = last_used; 336 336 - iop_desc_clear_next_desc(alloc_start); 337 337 - iop_desc_clear_next_desc(alloc_tail); 338 338 - return alloc_tail; 339 339 - } 340 340 - } 341 341 - if (!retry++) 342 342 - goto retry; 343 343 - 344 344 - /* perform direct reclaim if the allocation fails */ 345 345 - __iop_adma_slot_cleanup(iop_chan); 346 346 - 347 347 - return NULL; 348 348 - } 349 349 - 350 350 - static void iop_adma_check_threshold(struct iop_adma_chan *iop_chan) 351 351 - { 352 352 - dev_dbg(iop_chan->device->common.dev, "pending: %d\n", 353 353 - iop_chan->pending); 354 354 - 355 355 - if (iop_chan->pending >= IOP_ADMA_THRESHOLD) { 356 356 - iop_chan->pending = 0; 357 357 - iop_chan_append(iop_chan); 358 358 - } 359 359 - } 360 360 - 361 361 - static dma_cookie_t 362 362 - iop_adma_tx_submit(struct dma_async_tx_descriptor *tx) 363 363 - { 364 364 - struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx); 365 365 - struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan); 366 366 - struct iop_adma_desc_slot *grp_start, *old_chain_tail; 367 367 - int slot_cnt; 368 368 - dma_cookie_t cookie; 369 369 - dma_addr_t next_dma; 370 370 - 371 371 - grp_start = sw_desc->group_head; 372 372 - slot_cnt = grp_start->slot_cnt; 373 373 - 374 374 - spin_lock_bh(&iop_chan->lock); 375 375 - cookie = dma_cookie_assign(tx); 376 376 - 377 377 - old_chain_tail = list_entry(iop_chan->chain.prev, 378 378 - struct iop_adma_desc_slot, chain_node); 379 379 - list_splice_init(&sw_desc->tx_list, 380 380 - &old_chain_tail->chain_node); 381 381 - 382 382 - /* fix up the hardware chain */ 383 383 - next_dma = grp_start->async_tx.phys; 384 384 - iop_desc_set_next_desc(old_chain_tail, next_dma); 385 385 - BUG_ON(iop_desc_get_next_desc(old_chain_tail) != next_dma); /* flush */ 386 386 - 387 387 - /* check for pre-chained descriptors */ 388 388 - iop_paranoia(iop_desc_get_next_desc(sw_desc)); 389 389 - 390 390 - /* increment the pending count by the number of slots 391 391 - * memcpy operations have a 1:1 (slot:operation) relation 392 392 - * other operations are heavier and will pop the threshold 393 393 - * more often. 394 394 - */ 395 395 - iop_chan->pending += slot_cnt; 396 396 - iop_adma_check_threshold(iop_chan); 397 397 - spin_unlock_bh(&iop_chan->lock); 398 398 - 399 399 - dev_dbg(iop_chan->device->common.dev, "%s cookie: %d slot: %d\n", 400 400 - __func__, sw_desc->async_tx.cookie, sw_desc->idx); 401 401 - 402 402 - return cookie; 403 403 - } 404 404 - 405 405 - static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan); 406 406 - static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan); 407 407 - 408 408 - /** 409 409 - * iop_adma_alloc_chan_resources - returns the number of allocated descriptors 410 410 - * @chan: allocate descriptor resources for this channel 411 411 - * 412 412 - * Note: We keep the slots for 1 operation on iop_chan->chain at all times. To 413 413 - * avoid deadlock, via async_xor, num_descs_in_pool must at a minimum be 414 414 - * greater than 2x the number slots needed to satisfy a device->max_xor 415 415 - * request. 416 416 - * */ 417 417 - static int iop_adma_alloc_chan_resources(struct dma_chan *chan) 418 418 - { 419 419 - char *hw_desc; 420 420 - dma_addr_t dma_desc; 421 421 - int idx; 422 422 - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 423 423 - struct iop_adma_desc_slot *slot = NULL; 424 424 - int init = iop_chan->slots_allocated ? 0 : 1; 425 425 - struct iop_adma_platform_data *plat_data = 426 426 - dev_get_platdata(&iop_chan->device->pdev->dev); 427 427 - int num_descs_in_pool = plat_data->pool_size/IOP_ADMA_SLOT_SIZE; 428 428 - 429 429 - /* Allocate descriptor slots */ 430 430 - do { 431 431 - idx = iop_chan->slots_allocated; 432 432 - if (idx == num_descs_in_pool) 433 433 - break; 434 434 - 435 435 - slot = kzalloc(sizeof(*slot), GFP_KERNEL); 436 436 - if (!slot) { 437 437 - printk(KERN_INFO "IOP ADMA Channel only initialized" 438 438 - " %d descriptor slots", idx); 439 439 - break; 440 440 - } 441 441 - hw_desc = (char *) iop_chan->device->dma_desc_pool_virt; 442 442 - slot->hw_desc = (void *) &hw_desc[idx * IOP_ADMA_SLOT_SIZE]; 443 443 - 444 444 - dma_async_tx_descriptor_init(&slot->async_tx, chan); 445 445 - slot->async_tx.tx_submit = iop_adma_tx_submit; 446 446 - INIT_LIST_HEAD(&slot->tx_list); 447 447 - INIT_LIST_HEAD(&slot->chain_node); 448 448 - INIT_LIST_HEAD(&slot->slot_node); 449 449 - dma_desc = iop_chan->device->dma_desc_pool; 450 450 - slot->async_tx.phys = dma_desc + idx * IOP_ADMA_SLOT_SIZE; 451 451 - slot->idx = idx; 452 452 - 453 453 - spin_lock_bh(&iop_chan->lock); 454 454 - iop_chan->slots_allocated++; 455 455 - list_add_tail(&slot->slot_node, &iop_chan->all_slots); 456 456 - spin_unlock_bh(&iop_chan->lock); 457 457 - } while (iop_chan->slots_allocated < num_descs_in_pool); 458 458 - 459 459 - if (idx && !iop_chan->last_used) 460 460 - iop_chan->last_used = list_entry(iop_chan->all_slots.next, 461 461 - struct iop_adma_desc_slot, 462 462 - slot_node); 463 463 - 464 464 - dev_dbg(iop_chan->device->common.dev, 465 465 - "allocated %d descriptor slots last_used: %p\n", 466 466 - iop_chan->slots_allocated, iop_chan->last_used); 467 467 - 468 468 - /* initialize the channel and the chain with a null operation */ 469 469 - if (init) { 470 470 - if (dma_has_cap(DMA_MEMCPY, 471 471 - iop_chan->device->common.cap_mask)) 472 472 - iop_chan_start_null_memcpy(iop_chan); 473 473 - else if (dma_has_cap(DMA_XOR, 474 474 - iop_chan->device->common.cap_mask)) 475 475 - iop_chan_start_null_xor(iop_chan); 476 476 - else 477 477 - BUG(); 478 478 - } 479 479 - 480 480 - return (idx > 0) ? idx : -ENOMEM; 481 481 - } 482 482 - 483 483 - static struct dma_async_tx_descriptor * 484 484 - iop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags) 485 485 - { 486 486 - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 487 487 - struct iop_adma_desc_slot *sw_desc, *grp_start; 488 488 - int slot_cnt, slots_per_op; 489 489 - 490 490 - dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); 491 491 - 492 492 - spin_lock_bh(&iop_chan->lock); 493 493 - slot_cnt = iop_chan_interrupt_slot_count(&slots_per_op, iop_chan); 494 494 - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 495 495 - if (sw_desc) { 496 496 - grp_start = sw_desc->group_head; 497 497 - iop_desc_init_interrupt(grp_start, iop_chan); 498 498 - sw_desc->async_tx.flags = flags; 499 499 - } 500 500 - spin_unlock_bh(&iop_chan->lock); 501 501 - 502 502 - return sw_desc ? &sw_desc->async_tx : NULL; 503 503 - } 504 504 - 505 505 - static struct dma_async_tx_descriptor * 506 506 - iop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest, 507 507 - dma_addr_t dma_src, size_t len, unsigned long flags) 508 508 - { 509 509 - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 510 510 - struct iop_adma_desc_slot *sw_desc, *grp_start; 511 511 - int slot_cnt, slots_per_op; 512 512 - 513 513 - if (unlikely(!len)) 514 514 - return NULL; 515 515 - BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT); 516 516 - 517 517 - dev_dbg(iop_chan->device->common.dev, "%s len: %zu\n", 518 518 - __func__, len); 519 519 - 520 520 - spin_lock_bh(&iop_chan->lock); 521 521 - slot_cnt = iop_chan_memcpy_slot_count(len, &slots_per_op); 522 522 - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 523 523 - if (sw_desc) { 524 524 - grp_start = sw_desc->group_head; 525 525 - iop_desc_init_memcpy(grp_start, flags); 526 526 - iop_desc_set_byte_count(grp_start, iop_chan, len); 527 527 - iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); 528 528 - iop_desc_set_memcpy_src_addr(grp_start, dma_src); 529 529 - sw_desc->async_tx.flags = flags; 530 530 - } 531 531 - spin_unlock_bh(&iop_chan->lock); 532 532 - 533 533 - return sw_desc ? &sw_desc->async_tx : NULL; 534 534 - } 535 535 - 536 536 - static struct dma_async_tx_descriptor * 537 537 - iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest, 538 538 - dma_addr_t *dma_src, unsigned int src_cnt, size_t len, 539 539 - unsigned long flags) 540 540 - { 541 541 - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 542 542 - struct iop_adma_desc_slot *sw_desc, *grp_start; 543 543 - int slot_cnt, slots_per_op; 544 544 - 545 545 - if (unlikely(!len)) 546 546 - return NULL; 547 547 - BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); 548 548 - 549 549 - dev_dbg(iop_chan->device->common.dev, 550 550 - "%s src_cnt: %d len: %zu flags: %lx\n", 551 551 - __func__, src_cnt, len, flags); 552 552 - 553 553 - spin_lock_bh(&iop_chan->lock); 554 554 - slot_cnt = iop_chan_xor_slot_count(len, src_cnt, &slots_per_op); 555 555 - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 556 556 - if (sw_desc) { 557 557 - grp_start = sw_desc->group_head; 558 558 - iop_desc_init_xor(grp_start, src_cnt, flags); 559 559 - iop_desc_set_byte_count(grp_start, iop_chan, len); 560 560 - iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); 561 561 - sw_desc->async_tx.flags = flags; 562 562 - while (src_cnt--) 563 563 - iop_desc_set_xor_src_addr(grp_start, src_cnt, 564 564 - dma_src[src_cnt]); 565 565 - } 566 566 - spin_unlock_bh(&iop_chan->lock); 567 567 - 568 568 - return sw_desc ? &sw_desc->async_tx : NULL; 569 569 - } 570 570 - 571 571 - static struct dma_async_tx_descriptor * 572 572 - iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src, 573 573 - unsigned int src_cnt, size_t len, u32 *result, 574 574 - unsigned long flags) 575 575 - { 576 576 - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 577 577 - struct iop_adma_desc_slot *sw_desc, *grp_start; 578 578 - int slot_cnt, slots_per_op; 579 579 - 580 580 - if (unlikely(!len)) 581 581 - return NULL; 582 582 - 583 583 - dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %zu\n", 584 584 - __func__, src_cnt, len); 585 585 - 586 586 - spin_lock_bh(&iop_chan->lock); 587 587 - slot_cnt = iop_chan_zero_sum_slot_count(len, src_cnt, &slots_per_op); 588 588 - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 589 589 - if (sw_desc) { 590 590 - grp_start = sw_desc->group_head; 591 591 - iop_desc_init_zero_sum(grp_start, src_cnt, flags); 592 592 - iop_desc_set_zero_sum_byte_count(grp_start, len); 593 593 - grp_start->xor_check_result = result; 594 594 - pr_debug("\t%s: grp_start->xor_check_result: %p\n", 595 595 - __func__, grp_start->xor_check_result); 596 596 - sw_desc->async_tx.flags = flags; 597 597 - while (src_cnt--) 598 598 - iop_desc_set_zero_sum_src_addr(grp_start, src_cnt, 599 599 - dma_src[src_cnt]); 600 600 - } 601 601 - spin_unlock_bh(&iop_chan->lock); 602 602 - 603 603 - return sw_desc ? &sw_desc->async_tx : NULL; 604 604 - } 605 605 - 606 606 - static struct dma_async_tx_descriptor * 607 607 - iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, 608 608 - unsigned int src_cnt, const unsigned char *scf, size_t len, 609 609 - unsigned long flags) 610 610 - { 611 611 - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 612 612 - struct iop_adma_desc_slot *sw_desc, *g; 613 613 - int slot_cnt, slots_per_op; 614 614 - int continue_srcs; 615 615 - 616 616 - if (unlikely(!len)) 617 617 - return NULL; 618 618 - BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); 619 619 - 620 620 - dev_dbg(iop_chan->device->common.dev, 621 621 - "%s src_cnt: %d len: %zu flags: %lx\n", 622 622 - __func__, src_cnt, len, flags); 623 623 - 624 624 - if (dmaf_p_disabled_continue(flags)) 625 625 - continue_srcs = 1+src_cnt; 626 626 - else if (dmaf_continue(flags)) 627 627 - continue_srcs = 3+src_cnt; 628 628 - else 629 629 - continue_srcs = 0+src_cnt; 630 630 - 631 631 - spin_lock_bh(&iop_chan->lock); 632 632 - slot_cnt = iop_chan_pq_slot_count(len, continue_srcs, &slots_per_op); 633 633 - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 634 634 - if (sw_desc) { 635 635 - int i; 636 636 - 637 637 - g = sw_desc->group_head; 638 638 - iop_desc_set_byte_count(g, iop_chan, len); 639 639 - 640 640 - /* even if P is disabled its destination address (bits 641 641 - * [3:0]) must match Q. It is ok if P points to an 642 642 - * invalid address, it won't be written. 643 643 - */ 644 644 - if (flags & DMA_PREP_PQ_DISABLE_P) 645 645 - dst[0] = dst[1] & 0x7; 646 646 - 647 647 - iop_desc_set_pq_addr(g, dst); 648 648 - sw_desc->async_tx.flags = flags; 649 649 - for (i = 0; i < src_cnt; i++) 650 650 - iop_desc_set_pq_src_addr(g, i, src[i], scf[i]); 651 651 - 652 652 - /* if we are continuing a previous operation factor in 653 653 - * the old p and q values, see the comment for dma_maxpq 654 654 - * in include/linux/dmaengine.h 655 655 - */ 656 656 - if (dmaf_p_disabled_continue(flags)) 657 657 - iop_desc_set_pq_src_addr(g, i++, dst[1], 1); 658 658 - else if (dmaf_continue(flags)) { 659 659 - iop_desc_set_pq_src_addr(g, i++, dst[0], 0); 660 660 - iop_desc_set_pq_src_addr(g, i++, dst[1], 1); 661 661 - iop_desc_set_pq_src_addr(g, i++, dst[1], 0); 662 662 - } 663 663 - iop_desc_init_pq(g, i, flags); 664 664 - } 665 665 - spin_unlock_bh(&iop_chan->lock); 666 666 - 667 667 - return sw_desc ? &sw_desc->async_tx : NULL; 668 668 - } 669 669 - 670 670 - static struct dma_async_tx_descriptor * 671 671 - iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, 672 672 - unsigned int src_cnt, const unsigned char *scf, 673 673 - size_t len, enum sum_check_flags *pqres, 674 674 - unsigned long flags) 675 675 - { 676 676 - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 677 677 - struct iop_adma_desc_slot *sw_desc, *g; 678 678 - int slot_cnt, slots_per_op; 679 679 - 680 680 - if (unlikely(!len)) 681 681 - return NULL; 682 682 - BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); 683 683 - 684 684 - dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %zu\n", 685 685 - __func__, src_cnt, len); 686 686 - 687 687 - spin_lock_bh(&iop_chan->lock); 688 688 - slot_cnt = iop_chan_pq_zero_sum_slot_count(len, src_cnt + 2, &slots_per_op); 689 689 - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 690 690 - if (sw_desc) { 691 691 - /* for validate operations p and q are tagged onto the 692 692 - * end of the source list 693 693 - */ 694 694 - int pq_idx = src_cnt; 695 695 - 696 696 - g = sw_desc->group_head; 697 697 - iop_desc_init_pq_zero_sum(g, src_cnt+2, flags); 698 698 - iop_desc_set_pq_zero_sum_byte_count(g, len); 699 699 - g->pq_check_result = pqres; 700 700 - pr_debug("\t%s: g->pq_check_result: %p\n", 701 701 - __func__, g->pq_check_result); 702 702 - sw_desc->async_tx.flags = flags; 703 703 - while (src_cnt--) 704 704 - iop_desc_set_pq_zero_sum_src_addr(g, src_cnt, 705 705 - src[src_cnt], 706 706 - scf[src_cnt]); 707 707 - iop_desc_set_pq_zero_sum_addr(g, pq_idx, src); 708 708 - } 709 709 - spin_unlock_bh(&iop_chan->lock); 710 710 - 711 711 - return sw_desc ? &sw_desc->async_tx : NULL; 712 712 - } 713 713 - 714 714 - static void iop_adma_free_chan_resources(struct dma_chan *chan) 715 715 - { 716 716 - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 717 717 - struct iop_adma_desc_slot *iter, *_iter; 718 718 - int in_use_descs = 0; 719 719 - 720 720 - iop_adma_slot_cleanup(iop_chan); 721 721 - 722 722 - spin_lock_bh(&iop_chan->lock); 723 723 - list_for_each_entry_safe(iter, _iter, &iop_chan->chain, 724 724 - chain_node) { 725 725 - in_use_descs++; 726 726 - list_del(&iter->chain_node); 727 727 - } 728 728 - list_for_each_entry_safe_reverse( 729 729 - iter, _iter, &iop_chan->all_slots, slot_node) { 730 730 - list_del(&iter->slot_node); 731 731 - kfree(iter); 732 732 - iop_chan->slots_allocated--; 733 733 - } 734 734 - iop_chan->last_used = NULL; 735 735 - 736 736 - dev_dbg(iop_chan->device->common.dev, "%s slots_allocated %d\n", 737 737 - __func__, iop_chan->slots_allocated); 738 738 - spin_unlock_bh(&iop_chan->lock); 739 739 - 740 740 - /* one is ok since we left it on there on purpose */ 741 741 - if (in_use_descs > 1) 742 742 - printk(KERN_ERR "IOP: Freeing %d in use descriptors!\n", 743 743 - in_use_descs - 1); 744 744 - } 745 745 - 746 746 - /** 747 747 - * iop_adma_status - poll the status of an ADMA transaction 748 748 - * @chan: ADMA channel handle 749 749 - * @cookie: ADMA transaction identifier 750 750 - * @txstate: a holder for the current state of the channel or NULL 751 751 - */ 752 752 - static enum dma_status iop_adma_status(struct dma_chan *chan, 753 753 - dma_cookie_t cookie, 754 754 - struct dma_tx_state *txstate) 755 755 - { 756 756 - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 757 757 - int ret; 758 758 - 759 759 - ret = dma_cookie_status(chan, cookie, txstate); 760 760 - if (ret == DMA_COMPLETE) 761 761 - return ret; 762 762 - 763 763 - iop_adma_slot_cleanup(iop_chan); 764 764 - 765 765 - return dma_cookie_status(chan, cookie, txstate); 766 766 - } 767 767 - 768 768 - static irqreturn_t iop_adma_eot_handler(int irq, void *data) 769 769 - { 770 770 - struct iop_adma_chan *chan = data; 771 771 - 772 772 - dev_dbg(chan->device->common.dev, "%s\n", __func__); 773 773 - 774 774 - tasklet_schedule(&chan->irq_tasklet); 775 775 - 776 776 - iop_adma_device_clear_eot_status(chan); 777 777 - 778 778 - return IRQ_HANDLED; 779 779 - } 780 780 - 781 781 - static irqreturn_t iop_adma_eoc_handler(int irq, void *data) 782 782 - { 783 783 - struct iop_adma_chan *chan = data; 784 784 - 785 785 - dev_dbg(chan->device->common.dev, "%s\n", __func__); 786 786 - 787 787 - tasklet_schedule(&chan->irq_tasklet); 788 788 - 789 789 - iop_adma_device_clear_eoc_status(chan); 790 790 - 791 791 - return IRQ_HANDLED; 792 792 - } 793 793 - 794 794 - static irqreturn_t iop_adma_err_handler(int irq, void *data) 795 795 - { 796 796 - struct iop_adma_chan *chan = data; 797 797 - unsigned long status = iop_chan_get_status(chan); 798 798 - 799 799 - dev_err(chan->device->common.dev, 800 800 - "error ( %s%s%s%s%s%s%s)\n", 801 801 - iop_is_err_int_parity(status, chan) ? "int_parity " : "", 802 802 - iop_is_err_mcu_abort(status, chan) ? "mcu_abort " : "", 803 803 - iop_is_err_int_tabort(status, chan) ? "int_tabort " : "", 804 804 - iop_is_err_int_mabort(status, chan) ? "int_mabort " : "", 805 805 - iop_is_err_pci_tabort(status, chan) ? "pci_tabort " : "", 806 806 - iop_is_err_pci_mabort(status, chan) ? "pci_mabort " : "", 807 807 - iop_is_err_split_tx(status, chan) ? "split_tx " : ""); 808 808 - 809 809 - iop_adma_device_clear_err_status(chan); 810 810 - 811 811 - BUG(); 812 812 - 813 813 - return IRQ_HANDLED; 814 814 - } 815 815 - 816 816 - static void iop_adma_issue_pending(struct dma_chan *chan) 817 817 - { 818 818 - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 819 819 - 820 820 - if (iop_chan->pending) { 821 821 - iop_chan->pending = 0; 822 822 - iop_chan_append(iop_chan); 823 823 - } 824 824 - } 825 825 - 826 826 - /* 827 827 - * Perform a transaction to verify the HW works. 828 828 - */ 829 829 - #define IOP_ADMA_TEST_SIZE 2000 830 830 - 831 831 - static int iop_adma_memcpy_self_test(struct iop_adma_device *device) 832 832 - { 833 833 - int i; 834 834 - void *src, *dest; 835 835 - dma_addr_t src_dma, dest_dma; 836 836 - struct dma_chan *dma_chan; 837 837 - dma_cookie_t cookie; 838 838 - struct dma_async_tx_descriptor *tx; 839 839 - int err = 0; 840 840 - struct iop_adma_chan *iop_chan; 841 841 - 842 842 - dev_dbg(device->common.dev, "%s\n", __func__); 843 843 - 844 844 - src = kmalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL); 845 845 - if (!src) 846 846 - return -ENOMEM; 847 847 - dest = kzalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL); 848 848 - if (!dest) { 849 849 - kfree(src); 850 850 - return -ENOMEM; 851 851 - } 852 852 - 853 853 - /* Fill in src buffer */ 854 854 - for (i = 0; i < IOP_ADMA_TEST_SIZE; i++) 855 855 - ((u8 *) src)[i] = (u8)i; 856 856 - 857 857 - /* Start copy, using first DMA channel */ 858 858 - dma_chan = container_of(device->common.channels.next, 859 859 - struct dma_chan, 860 860 - device_node); 861 861 - if (iop_adma_alloc_chan_resources(dma_chan) < 1) { 862 862 - err = -ENODEV; 863 863 - goto out; 864 864 - } 865 865 - 866 866 - dest_dma = dma_map_single(dma_chan->device->dev, dest, 867 867 - IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE); 868 868 - src_dma = dma_map_single(dma_chan->device->dev, src, 869 869 - IOP_ADMA_TEST_SIZE, DMA_TO_DEVICE); 870 870 - tx = iop_adma_prep_dma_memcpy(dma_chan, dest_dma, src_dma, 871 871 - IOP_ADMA_TEST_SIZE, 872 872 - DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 873 873 - 874 874 - cookie = iop_adma_tx_submit(tx); 875 875 - iop_adma_issue_pending(dma_chan); 876 876 - msleep(1); 877 877 - 878 878 - if (iop_adma_status(dma_chan, cookie, NULL) != 879 879 - DMA_COMPLETE) { 880 880 - dev_err(dma_chan->device->dev, 881 881 - "Self-test copy timed out, disabling\n"); 882 882 - err = -ENODEV; 883 883 - goto free_resources; 884 884 - } 885 885 - 886 886 - iop_chan = to_iop_adma_chan(dma_chan); 887 887 - dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma, 888 888 - IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE); 889 889 - if (memcmp(src, dest, IOP_ADMA_TEST_SIZE)) { 890 890 - dev_err(dma_chan->device->dev, 891 891 - "Self-test copy failed compare, disabling\n"); 892 892 - err = -ENODEV; 893 893 - goto free_resources; 894 894 - } 895 895 - 896 896 - free_resources: 897 897 - iop_adma_free_chan_resources(dma_chan); 898 898 - out: 899 899 - kfree(src); 900 900 - kfree(dest); 901 901 - return err; 902 902 - } 903 903 - 904 904 - #define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */ 905 905 - static int 906 906 - iop_adma_xor_val_self_test(struct iop_adma_device *device) 907 907 - { 908 908 - int i, src_idx; 909 909 - struct page *dest; 910 910 - struct page *xor_srcs[IOP_ADMA_NUM_SRC_TEST]; 911 911 - struct page *zero_sum_srcs[IOP_ADMA_NUM_SRC_TEST + 1]; 912 912 - dma_addr_t dma_srcs[IOP_ADMA_NUM_SRC_TEST + 1]; 913 913 - dma_addr_t dest_dma; 914 914 - struct dma_async_tx_descriptor *tx; 915 915 - struct dma_chan *dma_chan; 916 916 - dma_cookie_t cookie; 917 917 - u8 cmp_byte = 0; 918 918 - u32 cmp_word; 919 919 - u32 zero_sum_result; 920 920 - int err = 0; 921 921 - struct iop_adma_chan *iop_chan; 922 922 - 923 923 - dev_dbg(device->common.dev, "%s\n", __func__); 924 924 - 925 925 - for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) { 926 926 - xor_srcs[src_idx] = alloc_page(GFP_KERNEL); 927 927 - if (!xor_srcs[src_idx]) { 928 928 - while (src_idx--) 929 929 - __free_page(xor_srcs[src_idx]); 930 930 - return -ENOMEM; 931 931 - } 932 932 - } 933 933 - 934 934 - dest = alloc_page(GFP_KERNEL); 935 935 - if (!dest) { 936 936 - while (src_idx--) 937 937 - __free_page(xor_srcs[src_idx]); 938 938 - return -ENOMEM; 939 939 - } 940 940 - 941 941 - /* Fill in src buffers */ 942 942 - for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) { 943 943 - u8 *ptr = page_address(xor_srcs[src_idx]); 944 944 - for (i = 0; i < PAGE_SIZE; i++) 945 945 - ptr[i] = (1 << src_idx); 946 946 - } 947 947 - 948 948 - for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) 949 949 - cmp_byte ^= (u8) (1 << src_idx); 950 950 - 951 951 - cmp_word = (cmp_byte << 24) | (cmp_byte << 16) | 952 952 - (cmp_byte << 8) | cmp_byte; 953 953 - 954 954 - memset(page_address(dest), 0, PAGE_SIZE); 955 955 - 956 956 - dma_chan = container_of(device->common.channels.next, 957 957 - struct dma_chan, 958 958 - device_node); 959 959 - if (iop_adma_alloc_chan_resources(dma_chan) < 1) { 960 960 - err = -ENODEV; 961 961 - goto out; 962 962 - } 963 963 - 964 964 - /* test xor */ 965 965 - dest_dma = dma_map_page(dma_chan->device->dev, dest, 0, 966 966 - PAGE_SIZE, DMA_FROM_DEVICE); 967 967 - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) 968 968 - dma_srcs[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i], 969 969 - 0, PAGE_SIZE, DMA_TO_DEVICE); 970 970 - tx = iop_adma_prep_dma_xor(dma_chan, dest_dma, dma_srcs, 971 971 - IOP_ADMA_NUM_SRC_TEST, PAGE_SIZE, 972 972 - DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 973 973 - 974 974 - cookie = iop_adma_tx_submit(tx); 975 975 - iop_adma_issue_pending(dma_chan); 976 976 - msleep(8); 977 977 - 978 978 - if (iop_adma_status(dma_chan, cookie, NULL) != 979 979 - DMA_COMPLETE) { 980 980 - dev_err(dma_chan->device->dev, 981 981 - "Self-test xor timed out, disabling\n"); 982 982 - err = -ENODEV; 983 983 - goto free_resources; 984 984 - } 985 985 - 986 986 - iop_chan = to_iop_adma_chan(dma_chan); 987 987 - dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma, 988 988 - PAGE_SIZE, DMA_FROM_DEVICE); 989 989 - for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) { 990 990 - u32 *ptr = page_address(dest); 991 991 - if (ptr[i] != cmp_word) { 992 992 - dev_err(dma_chan->device->dev, 993 993 - "Self-test xor failed compare, disabling\n"); 994 994 - err = -ENODEV; 995 995 - goto free_resources; 996 996 - } 997 997 - } 998 998 - dma_sync_single_for_device(&iop_chan->device->pdev->dev, dest_dma, 999 999 - PAGE_SIZE, DMA_TO_DEVICE); 1000 1000 - 1001 1001 - /* skip zero sum if the capability is not present */ 1002 1002 - if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask)) 1003 1003 - goto free_resources; 1004 1004 - 1005 1005 - /* zero sum the sources with the destintation page */ 1006 1006 - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) 1007 1007 - zero_sum_srcs[i] = xor_srcs[i]; 1008 1008 - zero_sum_srcs[i] = dest; 1009 1009 - 1010 1010 - zero_sum_result = 1; 1011 1011 - 1012 1012 - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++) 1013 1013 - dma_srcs[i] = dma_map_page(dma_chan->device->dev, 1014 1014 - zero_sum_srcs[i], 0, PAGE_SIZE, 1015 1015 - DMA_TO_DEVICE); 1016 1016 - tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs, 1017 1017 - IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE, 1018 1018 - &zero_sum_result, 1019 1019 - DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1020 1020 - 1021 1021 - cookie = iop_adma_tx_submit(tx); 1022 1022 - iop_adma_issue_pending(dma_chan); 1023 1023 - msleep(8); 1024 1024 - 1025 1025 - if (iop_adma_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { 1026 1026 - dev_err(dma_chan->device->dev, 1027 1027 - "Self-test zero sum timed out, disabling\n"); 1028 1028 - err = -ENODEV; 1029 1029 - goto free_resources; 1030 1030 - } 1031 1031 - 1032 1032 - if (zero_sum_result != 0) { 1033 1033 - dev_err(dma_chan->device->dev, 1034 1034 - "Self-test zero sum failed compare, disabling\n"); 1035 1035 - err = -ENODEV; 1036 1036 - goto free_resources; 1037 1037 - } 1038 1038 - 1039 1039 - /* test for non-zero parity sum */ 1040 1040 - zero_sum_result = 0; 1041 1041 - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++) 1042 1042 - dma_srcs[i] = dma_map_page(dma_chan->device->dev, 1043 1043 - zero_sum_srcs[i], 0, PAGE_SIZE, 1044 1044 - DMA_TO_DEVICE); 1045 1045 - tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs, 1046 1046 - IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE, 1047 1047 - &zero_sum_result, 1048 1048 - DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1049 1049 - 1050 1050 - cookie = iop_adma_tx_submit(tx); 1051 1051 - iop_adma_issue_pending(dma_chan); 1052 1052 - msleep(8); 1053 1053 - 1054 1054 - if (iop_adma_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { 1055 1055 - dev_err(dma_chan->device->dev, 1056 1056 - "Self-test non-zero sum timed out, disabling\n"); 1057 1057 - err = -ENODEV; 1058 1058 - goto free_resources; 1059 1059 - } 1060 1060 - 1061 1061 - if (zero_sum_result != 1) { 1062 1062 - dev_err(dma_chan->device->dev, 1063 1063 - "Self-test non-zero sum failed compare, disabling\n"); 1064 1064 - err = -ENODEV; 1065 1065 - goto free_resources; 1066 1066 - } 1067 1067 - 1068 1068 - free_resources: 1069 1069 - iop_adma_free_chan_resources(dma_chan); 1070 1070 - out: 1071 1071 - src_idx = IOP_ADMA_NUM_SRC_TEST; 1072 1072 - while (src_idx--) 1073 1073 - __free_page(xor_srcs[src_idx]); 1074 1074 - __free_page(dest); 1075 1075 - return err; 1076 1076 - } 1077 1077 - 1078 1078 - #ifdef CONFIG_RAID6_PQ 1079 1079 - static int 1080 1080 - iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device) 1081 1081 - { 1082 1082 - /* combined sources, software pq results, and extra hw pq results */ 1083 1083 - struct page *pq[IOP_ADMA_NUM_SRC_TEST+2+2]; 1084 1084 - /* ptr to the extra hw pq buffers defined above */ 1085 1085 - struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST+2]; 1086 1086 - /* address conversion buffers (dma_map / page_address) */ 1087 1087 - void *pq_sw[IOP_ADMA_NUM_SRC_TEST+2]; 1088 1088 - dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST+2]; 1089 1089 - dma_addr_t *pq_dest = &pq_src[IOP_ADMA_NUM_SRC_TEST]; 1090 1090 - 1091 1091 - int i; 1092 1092 - struct dma_async_tx_descriptor *tx; 1093 1093 - struct dma_chan *dma_chan; 1094 1094 - dma_cookie_t cookie; 1095 1095 - u32 zero_sum_result; 1096 1096 - int err = 0; 1097 1097 - struct device *dev; 1098 1098 - 1099 1099 - dev_dbg(device->common.dev, "%s\n", __func__); 1100 1100 - 1101 1101 - for (i = 0; i < ARRAY_SIZE(pq); i++) { 1102 1102 - pq[i] = alloc_page(GFP_KERNEL); 1103 1103 - if (!pq[i]) { 1104 1104 - while (i--) 1105 1105 - __free_page(pq[i]); 1106 1106 - return -ENOMEM; 1107 1107 - } 1108 1108 - } 1109 1109 - 1110 1110 - /* Fill in src buffers */ 1111 1111 - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) { 1112 1112 - pq_sw[i] = page_address(pq[i]); 1113 1113 - memset(pq_sw[i], 0x11111111 * (1<<i), PAGE_SIZE); 1114 1114 - } 1115 1115 - pq_sw[i] = page_address(pq[i]); 1116 1116 - pq_sw[i+1] = page_address(pq[i+1]); 1117 1117 - 1118 1118 - dma_chan = container_of(device->common.channels.next, 1119 1119 - struct dma_chan, 1120 1120 - device_node); 1121 1121 - if (iop_adma_alloc_chan_resources(dma_chan) < 1) { 1122 1122 - err = -ENODEV; 1123 1123 - goto out; 1124 1124 - } 1125 1125 - 1126 1126 - dev = dma_chan->device->dev; 1127 1127 - 1128 1128 - /* initialize the dests */ 1129 1129 - memset(page_address(pq_hw[0]), 0 , PAGE_SIZE); 1130 1130 - memset(page_address(pq_hw[1]), 0 , PAGE_SIZE); 1131 1131 - 1132 1132 - /* test pq */ 1133 1133 - pq_dest[0] = dma_map_page(dev, pq_hw[0], 0, PAGE_SIZE, DMA_FROM_DEVICE); 1134 1134 - pq_dest[1] = dma_map_page(dev, pq_hw[1], 0, PAGE_SIZE, DMA_FROM_DEVICE); 1135 1135 - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) 1136 1136 - pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, 1137 1137 - DMA_TO_DEVICE); 1138 1138 - 1139 1139 - tx = iop_adma_prep_dma_pq(dma_chan, pq_dest, pq_src, 1140 1140 - IOP_ADMA_NUM_SRC_TEST, (u8 *)raid6_gfexp, 1141 1141 - PAGE_SIZE, 1142 1142 - DMA_PREP_INTERRUPT | 1143 1143 - DMA_CTRL_ACK); 1144 1144 - 1145 1145 - cookie = iop_adma_tx_submit(tx); 1146 1146 - iop_adma_issue_pending(dma_chan); 1147 1147 - msleep(8); 1148 1148 - 1149 1149 - if (iop_adma_status(dma_chan, cookie, NULL) != 1150 1150 - DMA_COMPLETE) { 1151 1151 - dev_err(dev, "Self-test pq timed out, disabling\n"); 1152 1152 - err = -ENODEV; 1153 1153 - goto free_resources; 1154 1154 - } 1155 1155 - 1156 1156 - raid6_call.gen_syndrome(IOP_ADMA_NUM_SRC_TEST+2, PAGE_SIZE, pq_sw); 1157 1157 - 1158 1158 - if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST], 1159 1159 - page_address(pq_hw[0]), PAGE_SIZE) != 0) { 1160 1160 - dev_err(dev, "Self-test p failed compare, disabling\n"); 1161 1161 - err = -ENODEV; 1162 1162 - goto free_resources; 1163 1163 - } 1164 1164 - if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST+1], 1165 1165 - page_address(pq_hw[1]), PAGE_SIZE) != 0) { 1166 1166 - dev_err(dev, "Self-test q failed compare, disabling\n"); 1167 1167 - err = -ENODEV; 1168 1168 - goto free_resources; 1169 1169 - } 1170 1170 - 1171 1171 - /* test correct zero sum using the software generated pq values */ 1172 1172 - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++) 1173 1173 - pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, 1174 1174 - DMA_TO_DEVICE); 1175 1175 - 1176 1176 - zero_sum_result = ~0; 1177 1177 - tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST], 1178 1178 - pq_src, IOP_ADMA_NUM_SRC_TEST, 1179 1179 - raid6_gfexp, PAGE_SIZE, &zero_sum_result, 1180 1180 - DMA_PREP_INTERRUPT|DMA_CTRL_ACK); 1181 1181 - 1182 1182 - cookie = iop_adma_tx_submit(tx); 1183 1183 - iop_adma_issue_pending(dma_chan); 1184 1184 - msleep(8); 1185 1185 - 1186 1186 - if (iop_adma_status(dma_chan, cookie, NULL) != 1187 1187 - DMA_COMPLETE) { 1188 1188 - dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n"); 1189 1189 - err = -ENODEV; 1190 1190 - goto free_resources; 1191 1191 - } 1192 1192 - 1193 1193 - if (zero_sum_result != 0) { 1194 1194 - dev_err(dev, "Self-test pq-zero-sum failed to validate: %x\n", 1195 1195 - zero_sum_result); 1196 1196 - err = -ENODEV; 1197 1197 - goto free_resources; 1198 1198 - } 1199 1199 - 1200 1200 - /* test incorrect zero sum */ 1201 1201 - i = IOP_ADMA_NUM_SRC_TEST; 1202 1202 - memset(pq_sw[i] + 100, 0, 100); 1203 1203 - memset(pq_sw[i+1] + 200, 0, 200); 1204 1204 - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++) 1205 1205 - pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, 1206 1206 - DMA_TO_DEVICE); 1207 1207 - 1208 1208 - zero_sum_result = 0; 1209 1209 - tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST], 1210 1210 - pq_src, IOP_ADMA_NUM_SRC_TEST, 1211 1211 - raid6_gfexp, PAGE_SIZE, &zero_sum_result, 1212 1212 - DMA_PREP_INTERRUPT|DMA_CTRL_ACK); 1213 1213 - 1214 1214 - cookie = iop_adma_tx_submit(tx); 1215 1215 - iop_adma_issue_pending(dma_chan); 1216 1216 - msleep(8); 1217 1217 - 1218 1218 - if (iop_adma_status(dma_chan, cookie, NULL) != 1219 1219 - DMA_COMPLETE) { 1220 1220 - dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n"); 1221 1221 - err = -ENODEV; 1222 1222 - goto free_resources; 1223 1223 - } 1224 1224 - 1225 1225 - if (zero_sum_result != (SUM_CHECK_P_RESULT | SUM_CHECK_Q_RESULT)) { 1226 1226 - dev_err(dev, "Self-test !pq-zero-sum failed to validate: %x\n", 1227 1227 - zero_sum_result); 1228 1228 - err = -ENODEV; 1229 1229 - goto free_resources; 1230 1230 - } 1231 1231 - 1232 1232 - free_resources: 1233 1233 - iop_adma_free_chan_resources(dma_chan); 1234 1234 - out: 1235 1235 - i = ARRAY_SIZE(pq); 1236 1236 - while (i--) 1237 1237 - __free_page(pq[i]); 1238 1238 - return err; 1239 1239 - } 1240 1240 - #endif 1241 1241 - 1242 1242 - static int iop_adma_remove(struct platform_device *dev) 1243 1243 - { 1244 1244 - struct iop_adma_device *device = platform_get_drvdata(dev); 1245 1245 - struct dma_chan *chan, *_chan; 1246 1246 - struct iop_adma_chan *iop_chan; 1247 1247 - struct iop_adma_platform_data *plat_data = dev_get_platdata(&dev->dev); 1248 1248 - 1249 1249 - dma_async_device_unregister(&device->common); 1250 1250 - 1251 1251 - dma_free_coherent(&dev->dev, plat_data->pool_size, 1252 1252 - device->dma_desc_pool_virt, device->dma_desc_pool); 1253 1253 - 1254 1254 - list_for_each_entry_safe(chan, _chan, &device->common.channels, 1255 1255 - device_node) { 1256 1256 - iop_chan = to_iop_adma_chan(chan); 1257 1257 - list_del(&chan->device_node); 1258 1258 - kfree(iop_chan); 1259 1259 - } 1260 1260 - kfree(device); 1261 1261 - 1262 1262 - return 0; 1263 1263 - } 1264 1264 - 1265 1265 - static int iop_adma_probe(struct platform_device *pdev) 1266 1266 - { 1267 1267 - struct resource *res; 1268 1268 - int ret = 0, i; 1269 1269 - struct iop_adma_device *adev; 1270 1270 - struct iop_adma_chan *iop_chan; 1271 1271 - struct dma_device *dma_dev; 1272 1272 - struct iop_adma_platform_data *plat_data = dev_get_platdata(&pdev->dev); 1273 1273 - 1274 1274 - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1275 1275 - if (!res) 1276 1276 - return -ENODEV; 1277 1277 - 1278 1278 - if (!devm_request_mem_region(&pdev->dev, res->start, 1279 1279 - resource_size(res), pdev->name)) 1280 1280 - return -EBUSY; 1281 1281 - 1282 1282 - adev = kzalloc(sizeof(*adev), GFP_KERNEL); 1283 1283 - if (!adev) 1284 1284 - return -ENOMEM; 1285 1285 - dma_dev = &adev->common; 1286 1286 - 1287 1287 - /* allocate coherent memory for hardware descriptors 1288 1288 - * note: writecombine gives slightly better performance, but 1289 1289 - * requires that we explicitly flush the writes 1290 1290 - */ 1291 1291 - adev->dma_desc_pool_virt = dma_alloc_wc(&pdev->dev, 1292 1292 - plat_data->pool_size, 1293 1293 - &adev->dma_desc_pool, 1294 1294 - GFP_KERNEL); 1295 1295 - if (!adev->dma_desc_pool_virt) { 1296 1296 - ret = -ENOMEM; 1297 1297 - goto err_free_adev; 1298 1298 - } 1299 1299 - 1300 1300 - dev_dbg(&pdev->dev, "%s: allocated descriptor pool virt %p phys %pad\n", 1301 1301 - __func__, adev->dma_desc_pool_virt, &adev->dma_desc_pool); 1302 1302 - 1303 1303 - adev->id = plat_data->hw_id; 1304 1304 - 1305 1305 - /* discover transaction capabilites from the platform data */ 1306 1306 - dma_dev->cap_mask = plat_data->cap_mask; 1307 1307 - 1308 1308 - adev->pdev = pdev; 1309 1309 - platform_set_drvdata(pdev, adev); 1310 1310 - 1311 1311 - INIT_LIST_HEAD(&dma_dev->channels); 1312 1312 - 1313 1313 - /* set base routines */ 1314 1314 - dma_dev->device_alloc_chan_resources = iop_adma_alloc_chan_resources; 1315 1315 - dma_dev->device_free_chan_resources = iop_adma_free_chan_resources; 1316 1316 - dma_dev->device_tx_status = iop_adma_status; 1317 1317 - dma_dev->device_issue_pending = iop_adma_issue_pending; 1318 1318 - dma_dev->dev = &pdev->dev; 1319 1319 - 1320 1320 - /* set prep routines based on capability */ 1321 1321 - if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) 1322 1322 - dma_dev->device_prep_dma_memcpy = iop_adma_prep_dma_memcpy; 1323 1323 - if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 1324 1324 - dma_dev->max_xor = iop_adma_get_max_xor(); 1325 1325 - dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor; 1326 1326 - } 1327 1327 - if (dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask)) 1328 1328 - dma_dev->device_prep_dma_xor_val = 1329 1329 - iop_adma_prep_dma_xor_val; 1330 1330 - if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { 1331 1331 - dma_set_maxpq(dma_dev, iop_adma_get_max_pq(), 0); 1332 1332 - dma_dev->device_prep_dma_pq = iop_adma_prep_dma_pq; 1333 1333 - } 1334 1334 - if (dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) 1335 1335 - dma_dev->device_prep_dma_pq_val = 1336 1336 - iop_adma_prep_dma_pq_val; 1337 1337 - if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask)) 1338 1338 - dma_dev->device_prep_dma_interrupt = 1339 1339 - iop_adma_prep_dma_interrupt; 1340 1340 - 1341 1341 - iop_chan = kzalloc(sizeof(*iop_chan), GFP_KERNEL); 1342 1342 - if (!iop_chan) { 1343 1343 - ret = -ENOMEM; 1344 1344 - goto err_free_dma; 1345 1345 - } 1346 1346 - iop_chan->device = adev; 1347 1347 - 1348 1348 - iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start, 1349 1349 - resource_size(res)); 1350 1350 - if (!iop_chan->mmr_base) { 1351 1351 - ret = -ENOMEM; 1352 1352 - goto err_free_iop_chan; 1353 1353 - } 1354 1354 - tasklet_setup(&iop_chan->irq_tasklet, iop_adma_tasklet); 1355 1355 - 1356 1356 - /* clear errors before enabling interrupts */ 1357 1357 - iop_adma_device_clear_err_status(iop_chan); 1358 1358 - 1359 1359 - for (i = 0; i < 3; i++) { 1360 1360 - static const irq_handler_t handler[] = { 1361 1361 - iop_adma_eot_handler, 1362 1362 - iop_adma_eoc_handler, 1363 1363 - iop_adma_err_handler 1364 1364 - }; 1365 1365 - int irq = platform_get_irq(pdev, i); 1366 1366 - if (irq < 0) { 1367 1367 - ret = -ENXIO; 1368 1368 - goto err_free_iop_chan; 1369 1369 - } else { 1370 1370 - ret = devm_request_irq(&pdev->dev, irq, 1371 1371 - handler[i], 0, pdev->name, iop_chan); 1372 1372 - if (ret) 1373 1373 - goto err_free_iop_chan; 1374 1374 - } 1375 1375 - } 1376 1376 - 1377 1377 - spin_lock_init(&iop_chan->lock); 1378 1378 - INIT_LIST_HEAD(&iop_chan->chain); 1379 1379 - INIT_LIST_HEAD(&iop_chan->all_slots); 1380 1380 - iop_chan->common.device = dma_dev; 1381 1381 - dma_cookie_init(&iop_chan->common); 1382 1382 - list_add_tail(&iop_chan->common.device_node, &dma_dev->channels); 1383 1383 - 1384 1384 - if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { 1385 1385 - ret = iop_adma_memcpy_self_test(adev); 1386 1386 - dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret); 1387 1387 - if (ret) 1388 1388 - goto err_free_iop_chan; 1389 1389 - } 1390 1390 - 1391 1391 - if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 1392 1392 - ret = iop_adma_xor_val_self_test(adev); 1393 1393 - dev_dbg(&pdev->dev, "xor self test returned %d\n", ret); 1394 1394 - if (ret) 1395 1395 - goto err_free_iop_chan; 1396 1396 - } 1397 1397 - 1398 1398 - if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) && 1399 1399 - dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) { 1400 1400 - #ifdef CONFIG_RAID6_PQ 1401 1401 - ret = iop_adma_pq_zero_sum_self_test(adev); 1402 1402 - dev_dbg(&pdev->dev, "pq self test returned %d\n", ret); 1403 1403 - #else 1404 1404 - /* can not test raid6, so do not publish capability */ 1405 1405 - dma_cap_clear(DMA_PQ, dma_dev->cap_mask); 1406 1406 - dma_cap_clear(DMA_PQ_VAL, dma_dev->cap_mask); 1407 1407 - ret = 0; 1408 1408 - #endif 1409 1409 - if (ret) 1410 1410 - goto err_free_iop_chan; 1411 1411 - } 1412 1412 - 1413 1413 - dev_info(&pdev->dev, "Intel(R) IOP: ( %s%s%s%s%s%s)\n", 1414 1414 - dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "", 1415 1415 - dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask) ? "pq_val " : "", 1416 1416 - dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "", 1417 1417 - dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask) ? "xor_val " : "", 1418 1418 - dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "", 1419 1419 - dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : ""); 1420 1420 - 1421 1421 - dma_async_device_register(dma_dev); 1422 1422 - goto out; 1423 1423 - 1424 1424 - err_free_iop_chan: 1425 1425 - kfree(iop_chan); 1426 1426 - err_free_dma: 1427 1427 - dma_free_coherent(&adev->pdev->dev, plat_data->pool_size, 1428 1428 - adev->dma_desc_pool_virt, adev->dma_desc_pool); 1429 1429 - err_free_adev: 1430 1430 - kfree(adev); 1431 1431 - out: 1432 1432 - return ret; 1433 1433 - } 1434 1434 - 1435 1435 - static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan) 1436 1436 - { 1437 1437 - struct iop_adma_desc_slot *sw_desc, *grp_start; 1438 1438 - dma_cookie_t cookie; 1439 1439 - int slot_cnt, slots_per_op; 1440 1440 - 1441 1441 - dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); 1442 1442 - 1443 1443 - spin_lock_bh(&iop_chan->lock); 1444 1444 - slot_cnt = iop_chan_memcpy_slot_count(0, &slots_per_op); 1445 1445 - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 1446 1446 - if (sw_desc) { 1447 1447 - grp_start = sw_desc->group_head; 1448 1448 - 1449 1449 - list_splice_init(&sw_desc->tx_list, &iop_chan->chain); 1450 1450 - async_tx_ack(&sw_desc->async_tx); 1451 1451 - iop_desc_init_memcpy(grp_start, 0); 1452 1452 - iop_desc_set_byte_count(grp_start, iop_chan, 0); 1453 1453 - iop_desc_set_dest_addr(grp_start, iop_chan, 0); 1454 1454 - iop_desc_set_memcpy_src_addr(grp_start, 0); 1455 1455 - 1456 1456 - cookie = dma_cookie_assign(&sw_desc->async_tx); 1457 1457 - 1458 1458 - /* initialize the completed cookie to be less than 1459 1459 - * the most recently used cookie 1460 1460 - */ 1461 1461 - iop_chan->common.completed_cookie = cookie - 1; 1462 1462 - 1463 1463 - /* channel should not be busy */ 1464 1464 - BUG_ON(iop_chan_is_busy(iop_chan)); 1465 1465 - 1466 1466 - /* clear any prior error-status bits */ 1467 1467 - iop_adma_device_clear_err_status(iop_chan); 1468 1468 - 1469 1469 - /* disable operation */ 1470 1470 - iop_chan_disable(iop_chan); 1471 1471 - 1472 1472 - /* set the descriptor address */ 1473 1473 - iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys); 1474 1474 - 1475 1475 - /* 1/ don't add pre-chained descriptors 1476 1476 - * 2/ dummy read to flush next_desc write 1477 1477 - */ 1478 1478 - BUG_ON(iop_desc_get_next_desc(sw_desc)); 1479 1479 - 1480 1480 - /* run the descriptor */ 1481 1481 - iop_chan_enable(iop_chan); 1482 1482 - } else 1483 1483 - dev_err(iop_chan->device->common.dev, 1484 1484 - "failed to allocate null descriptor\n"); 1485 1485 - spin_unlock_bh(&iop_chan->lock); 1486 1486 - } 1487 1487 - 1488 1488 - static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan) 1489 1489 - { 1490 1490 - struct iop_adma_desc_slot *sw_desc, *grp_start; 1491 1491 - dma_cookie_t cookie; 1492 1492 - int slot_cnt, slots_per_op; 1493 1493 - 1494 1494 - dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); 1495 1495 - 1496 1496 - spin_lock_bh(&iop_chan->lock); 1497 1497 - slot_cnt = iop_chan_xor_slot_count(0, 2, &slots_per_op); 1498 1498 - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 1499 1499 - if (sw_desc) { 1500 1500 - grp_start = sw_desc->group_head; 1501 1501 - list_splice_init(&sw_desc->tx_list, &iop_chan->chain); 1502 1502 - async_tx_ack(&sw_desc->async_tx); 1503 1503 - iop_desc_init_null_xor(grp_start, 2, 0); 1504 1504 - iop_desc_set_byte_count(grp_start, iop_chan, 0); 1505 1505 - iop_desc_set_dest_addr(grp_start, iop_chan, 0); 1506 1506 - iop_desc_set_xor_src_addr(grp_start, 0, 0); 1507 1507 - iop_desc_set_xor_src_addr(grp_start, 1, 0); 1508 1508 - 1509 1509 - cookie = dma_cookie_assign(&sw_desc->async_tx); 1510 1510 - 1511 1511 - /* initialize the completed cookie to be less than 1512 1512 - * the most recently used cookie 1513 1513 - */ 1514 1514 - iop_chan->common.completed_cookie = cookie - 1; 1515 1515 - 1516 1516 - /* channel should not be busy */ 1517 1517 - BUG_ON(iop_chan_is_busy(iop_chan)); 1518 1518 - 1519 1519 - /* clear any prior error-status bits */ 1520 1520 - iop_adma_device_clear_err_status(iop_chan); 1521 1521 - 1522 1522 - /* disable operation */ 1523 1523 - iop_chan_disable(iop_chan); 1524 1524 - 1525 1525 - /* set the descriptor address */ 1526 1526 - iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys); 1527 1527 - 1528 1528 - /* 1/ don't add pre-chained descriptors 1529 1529 - * 2/ dummy read to flush next_desc write 1530 1530 - */ 1531 1531 - BUG_ON(iop_desc_get_next_desc(sw_desc)); 1532 1532 - 1533 1533 - /* run the descriptor */ 1534 1534 - iop_chan_enable(iop_chan); 1535 1535 - } else 1536 1536 - dev_err(iop_chan->device->common.dev, 1537 1537 - "failed to allocate null descriptor\n"); 1538 1538 - spin_unlock_bh(&iop_chan->lock); 1539 1539 - } 1540 1540 - 1541 1541 - static struct platform_driver iop_adma_driver = { 1542 1542 - .probe = iop_adma_probe, 1543 1543 - .remove = iop_adma_remove, 1544 1544 - .driver = { 1545 1545 - .name = "iop-adma", 1546 1546 - }, 1547 1547 - }; 1548 1548 - 1549 1549 - module_platform_driver(iop_adma_driver); 1550 1550 - 1551 1551 - MODULE_AUTHOR("Intel Corporation"); 1552 1552 - MODULE_DESCRIPTION("IOP ADMA Engine Driver"); 1553 1553 - MODULE_LICENSE("GPL"); 1554 1554 - MODULE_ALIAS("platform:iop-adma");

-914

drivers/dma/iop-adma.h

reviewed

··· 1 1 - /* SPDX-License-Identifier: GPL-2.0-only */ 2 2 - /* 3 3 - * Copyright © 2006, Intel Corporation. 4 4 - */ 5 5 - #ifndef _ADMA_H 6 6 - #define _ADMA_H 7 7 - #include <linux/types.h> 8 8 - #include <linux/io.h> 9 9 - #include <linux/platform_data/dma-iop32x.h> 10 10 - 11 11 - /* Memory copy units */ 12 12 - #define DMA_CCR(chan) (chan->mmr_base + 0x0) 13 13 - #define DMA_CSR(chan) (chan->mmr_base + 0x4) 14 14 - #define DMA_DAR(chan) (chan->mmr_base + 0xc) 15 15 - #define DMA_NDAR(chan) (chan->mmr_base + 0x10) 16 16 - #define DMA_PADR(chan) (chan->mmr_base + 0x14) 17 17 - #define DMA_PUADR(chan) (chan->mmr_base + 0x18) 18 18 - #define DMA_LADR(chan) (chan->mmr_base + 0x1c) 19 19 - #define DMA_BCR(chan) (chan->mmr_base + 0x20) 20 20 - #define DMA_DCR(chan) (chan->mmr_base + 0x24) 21 21 - 22 22 - /* Application accelerator unit */ 23 23 - #define AAU_ACR(chan) (chan->mmr_base + 0x0) 24 24 - #define AAU_ASR(chan) (chan->mmr_base + 0x4) 25 25 - #define AAU_ADAR(chan) (chan->mmr_base + 0x8) 26 26 - #define AAU_ANDAR(chan) (chan->mmr_base + 0xc) 27 27 - #define AAU_SAR(src, chan) (chan->mmr_base + (0x10 + ((src) << 2))) 28 28 - #define AAU_DAR(chan) (chan->mmr_base + 0x20) 29 29 - #define AAU_ABCR(chan) (chan->mmr_base + 0x24) 30 30 - #define AAU_ADCR(chan) (chan->mmr_base + 0x28) 31 31 - #define AAU_SAR_EDCR(src_edc) (chan->mmr_base + (0x02c + ((src_edc-4) << 2))) 32 32 - #define AAU_EDCR0_IDX 8 33 33 - #define AAU_EDCR1_IDX 17 34 34 - #define AAU_EDCR2_IDX 26 35 35 - 36 36 - struct iop3xx_aau_desc_ctrl { 37 37 - unsigned int int_en:1; 38 38 - unsigned int blk1_cmd_ctrl:3; 39 39 - unsigned int blk2_cmd_ctrl:3; 40 40 - unsigned int blk3_cmd_ctrl:3; 41 41 - unsigned int blk4_cmd_ctrl:3; 42 42 - unsigned int blk5_cmd_ctrl:3; 43 43 - unsigned int blk6_cmd_ctrl:3; 44 44 - unsigned int blk7_cmd_ctrl:3; 45 45 - unsigned int blk8_cmd_ctrl:3; 46 46 - unsigned int blk_ctrl:2; 47 47 - unsigned int dual_xor_en:1; 48 48 - unsigned int tx_complete:1; 49 49 - unsigned int zero_result_err:1; 50 50 - unsigned int zero_result_en:1; 51 51 - unsigned int dest_write_en:1; 52 52 - }; 53 53 - 54 54 - struct iop3xx_aau_e_desc_ctrl { 55 55 - unsigned int reserved:1; 56 56 - unsigned int blk1_cmd_ctrl:3; 57 57 - unsigned int blk2_cmd_ctrl:3; 58 58 - unsigned int blk3_cmd_ctrl:3; 59 59 - unsigned int blk4_cmd_ctrl:3; 60 60 - unsigned int blk5_cmd_ctrl:3; 61 61 - unsigned int blk6_cmd_ctrl:3; 62 62 - unsigned int blk7_cmd_ctrl:3; 63 63 - unsigned int blk8_cmd_ctrl:3; 64 64 - unsigned int reserved2:7; 65 65 - }; 66 66 - 67 67 - struct iop3xx_dma_desc_ctrl { 68 68 - unsigned int pci_transaction:4; 69 69 - unsigned int int_en:1; 70 70 - unsigned int dac_cycle_en:1; 71 71 - unsigned int mem_to_mem_en:1; 72 72 - unsigned int crc_data_tx_en:1; 73 73 - unsigned int crc_gen_en:1; 74 74 - unsigned int crc_seed_dis:1; 75 75 - unsigned int reserved:21; 76 76 - unsigned int crc_tx_complete:1; 77 77 - }; 78 78 - 79 79 - struct iop3xx_desc_dma { 80 80 - u32 next_desc; 81 81 - union { 82 82 - u32 pci_src_addr; 83 83 - u32 pci_dest_addr; 84 84 - u32 src_addr; 85 85 - }; 86 86 - union { 87 87 - u32 upper_pci_src_addr; 88 88 - u32 upper_pci_dest_addr; 89 89 - }; 90 90 - union { 91 91 - u32 local_pci_src_addr; 92 92 - u32 local_pci_dest_addr; 93 93 - u32 dest_addr; 94 94 - }; 95 95 - u32 byte_count; 96 96 - union { 97 97 - u32 desc_ctrl; 98 98 - struct iop3xx_dma_desc_ctrl desc_ctrl_field; 99 99 - }; 100 100 - u32 crc_addr; 101 101 - }; 102 102 - 103 103 - struct iop3xx_desc_aau { 104 104 - u32 next_desc; 105 105 - u32 src[4]; 106 106 - u32 dest_addr; 107 107 - u32 byte_count; 108 108 - union { 109 109 - u32 desc_ctrl; 110 110 - struct iop3xx_aau_desc_ctrl desc_ctrl_field; 111 111 - }; 112 112 - union { 113 113 - u32 src_addr; 114 114 - u32 e_desc_ctrl; 115 115 - struct iop3xx_aau_e_desc_ctrl e_desc_ctrl_field; 116 116 - } src_edc[31]; 117 117 - }; 118 118 - 119 119 - struct iop3xx_aau_gfmr { 120 120 - unsigned int gfmr1:8; 121 121 - unsigned int gfmr2:8; 122 122 - unsigned int gfmr3:8; 123 123 - unsigned int gfmr4:8; 124 124 - }; 125 125 - 126 126 - struct iop3xx_desc_pq_xor { 127 127 - u32 next_desc; 128 128 - u32 src[3]; 129 129 - union { 130 130 - u32 data_mult1; 131 131 - struct iop3xx_aau_gfmr data_mult1_field; 132 132 - }; 133 133 - u32 dest_addr; 134 134 - u32 byte_count; 135 135 - union { 136 136 - u32 desc_ctrl; 137 137 - struct iop3xx_aau_desc_ctrl desc_ctrl_field; 138 138 - }; 139 139 - union { 140 140 - u32 src_addr; 141 141 - u32 e_desc_ctrl; 142 142 - struct iop3xx_aau_e_desc_ctrl e_desc_ctrl_field; 143 143 - u32 data_multiplier; 144 144 - struct iop3xx_aau_gfmr data_mult_field; 145 145 - u32 reserved; 146 146 - } src_edc_gfmr[19]; 147 147 - }; 148 148 - 149 149 - struct iop3xx_desc_dual_xor { 150 150 - u32 next_desc; 151 151 - u32 src0_addr; 152 152 - u32 src1_addr; 153 153 - u32 h_src_addr; 154 154 - u32 d_src_addr; 155 155 - u32 h_dest_addr; 156 156 - u32 byte_count; 157 157 - union { 158 158 - u32 desc_ctrl; 159 159 - struct iop3xx_aau_desc_ctrl desc_ctrl_field; 160 160 - }; 161 161 - u32 d_dest_addr; 162 162 - }; 163 163 - 164 164 - union iop3xx_desc { 165 165 - struct iop3xx_desc_aau *aau; 166 166 - struct iop3xx_desc_dma *dma; 167 167 - struct iop3xx_desc_pq_xor *pq_xor; 168 168 - struct iop3xx_desc_dual_xor *dual_xor; 169 169 - void *ptr; 170 170 - }; 171 171 - 172 172 - /* No support for p+q operations */ 173 173 - static inline int 174 174 - iop_chan_pq_slot_count(size_t len, int src_cnt, int *slots_per_op) 175 175 - { 176 176 - BUG(); 177 177 - return 0; 178 178 - } 179 179 - 180 180 - static inline void 181 181 - iop_desc_init_pq(struct iop_adma_desc_slot *desc, int src_cnt, 182 182 - unsigned long flags) 183 183 - { 184 184 - BUG(); 185 185 - } 186 186 - 187 187 - static inline void 188 188 - iop_desc_set_pq_addr(struct iop_adma_desc_slot *desc, dma_addr_t *addr) 189 189 - { 190 190 - BUG(); 191 191 - } 192 192 - 193 193 - static inline void 194 194 - iop_desc_set_pq_src_addr(struct iop_adma_desc_slot *desc, int src_idx, 195 195 - dma_addr_t addr, unsigned char coef) 196 196 - { 197 197 - BUG(); 198 198 - } 199 199 - 200 200 - static inline int 201 201 - iop_chan_pq_zero_sum_slot_count(size_t len, int src_cnt, int *slots_per_op) 202 202 - { 203 203 - BUG(); 204 204 - return 0; 205 205 - } 206 206 - 207 207 - static inline void 208 208 - iop_desc_init_pq_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt, 209 209 - unsigned long flags) 210 210 - { 211 211 - BUG(); 212 212 - } 213 213 - 214 214 - static inline void 215 215 - iop_desc_set_pq_zero_sum_byte_count(struct iop_adma_desc_slot *desc, u32 len) 216 216 - { 217 217 - BUG(); 218 218 - } 219 219 - 220 220 - #define iop_desc_set_pq_zero_sum_src_addr iop_desc_set_pq_src_addr 221 221 - 222 222 - static inline void 223 223 - iop_desc_set_pq_zero_sum_addr(struct iop_adma_desc_slot *desc, int pq_idx, 224 224 - dma_addr_t *src) 225 225 - { 226 226 - BUG(); 227 227 - } 228 228 - 229 229 - static inline int iop_adma_get_max_xor(void) 230 230 - { 231 231 - return 32; 232 232 - } 233 233 - 234 234 - static inline int iop_adma_get_max_pq(void) 235 235 - { 236 236 - BUG(); 237 237 - return 0; 238 238 - } 239 239 - 240 240 - static inline u32 iop_chan_get_current_descriptor(struct iop_adma_chan *chan) 241 241 - { 242 242 - int id = chan->device->id; 243 243 - 244 244 - switch (id) { 245 245 - case DMA0_ID: 246 246 - case DMA1_ID: 247 247 - return __raw_readl(DMA_DAR(chan)); 248 248 - case AAU_ID: 249 249 - return __raw_readl(AAU_ADAR(chan)); 250 250 - default: 251 251 - BUG(); 252 252 - } 253 253 - return 0; 254 254 - } 255 255 - 256 256 - static inline void iop_chan_set_next_descriptor(struct iop_adma_chan *chan, 257 257 - u32 next_desc_addr) 258 258 - { 259 259 - int id = chan->device->id; 260 260 - 261 261 - switch (id) { 262 262 - case DMA0_ID: 263 263 - case DMA1_ID: 264 264 - __raw_writel(next_desc_addr, DMA_NDAR(chan)); 265 265 - break; 266 266 - case AAU_ID: 267 267 - __raw_writel(next_desc_addr, AAU_ANDAR(chan)); 268 268 - break; 269 269 - } 270 270 - 271 271 - } 272 272 - 273 273 - #define IOP_ADMA_STATUS_BUSY (1 << 10) 274 274 - #define IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT (1024) 275 275 - #define IOP_ADMA_XOR_MAX_BYTE_COUNT (16 * 1024 * 1024) 276 276 - #define IOP_ADMA_MAX_BYTE_COUNT (16 * 1024 * 1024) 277 277 - 278 278 - static inline int iop_chan_is_busy(struct iop_adma_chan *chan) 279 279 - { 280 280 - u32 status = __raw_readl(DMA_CSR(chan)); 281 281 - return (status & IOP_ADMA_STATUS_BUSY) ? 1 : 0; 282 282 - } 283 283 - 284 284 - static inline int iop_desc_is_aligned(struct iop_adma_desc_slot *desc, 285 285 - int num_slots) 286 286 - { 287 287 - /* num_slots will only ever be 1, 2, 4, or 8 */ 288 288 - return (desc->idx & (num_slots - 1)) ? 0 : 1; 289 289 - } 290 290 - 291 291 - /* to do: support large (i.e. > hw max) buffer sizes */ 292 292 - static inline int iop_chan_memcpy_slot_count(size_t len, int *slots_per_op) 293 293 - { 294 294 - *slots_per_op = 1; 295 295 - return 1; 296 296 - } 297 297 - 298 298 - /* to do: support large (i.e. > hw max) buffer sizes */ 299 299 - static inline int iop_chan_memset_slot_count(size_t len, int *slots_per_op) 300 300 - { 301 301 - *slots_per_op = 1; 302 302 - return 1; 303 303 - } 304 304 - 305 305 - static inline int iop3xx_aau_xor_slot_count(size_t len, int src_cnt, 306 306 - int *slots_per_op) 307 307 - { 308 308 - static const char slot_count_table[] = { 309 309 - 1, 1, 1, 1, /* 01 - 04 */ 310 310 - 2, 2, 2, 2, /* 05 - 08 */ 311 311 - 4, 4, 4, 4, /* 09 - 12 */ 312 312 - 4, 4, 4, 4, /* 13 - 16 */ 313 313 - 8, 8, 8, 8, /* 17 - 20 */ 314 314 - 8, 8, 8, 8, /* 21 - 24 */ 315 315 - 8, 8, 8, 8, /* 25 - 28 */ 316 316 - 8, 8, 8, 8, /* 29 - 32 */ 317 317 - }; 318 318 - *slots_per_op = slot_count_table[src_cnt - 1]; 319 319 - return *slots_per_op; 320 320 - } 321 321 - 322 322 - static inline int 323 323 - iop_chan_interrupt_slot_count(int *slots_per_op, struct iop_adma_chan *chan) 324 324 - { 325 325 - switch (chan->device->id) { 326 326 - case DMA0_ID: 327 327 - case DMA1_ID: 328 328 - return iop_chan_memcpy_slot_count(0, slots_per_op); 329 329 - case AAU_ID: 330 330 - return iop3xx_aau_xor_slot_count(0, 2, slots_per_op); 331 331 - default: 332 332 - BUG(); 333 333 - } 334 334 - return 0; 335 335 - } 336 336 - 337 337 - static inline int iop_chan_xor_slot_count(size_t len, int src_cnt, 338 338 - int *slots_per_op) 339 339 - { 340 340 - int slot_cnt = iop3xx_aau_xor_slot_count(len, src_cnt, slots_per_op); 341 341 - 342 342 - if (len <= IOP_ADMA_XOR_MAX_BYTE_COUNT) 343 343 - return slot_cnt; 344 344 - 345 345 - len -= IOP_ADMA_XOR_MAX_BYTE_COUNT; 346 346 - while (len > IOP_ADMA_XOR_MAX_BYTE_COUNT) { 347 347 - len -= IOP_ADMA_XOR_MAX_BYTE_COUNT; 348 348 - slot_cnt += *slots_per_op; 349 349 - } 350 350 - 351 351 - slot_cnt += *slots_per_op; 352 352 - 353 353 - return slot_cnt; 354 354 - } 355 355 - 356 356 - /* zero sum on iop3xx is limited to 1k at a time so it requires multiple 357 357 - * descriptors 358 358 - */ 359 359 - static inline int iop_chan_zero_sum_slot_count(size_t len, int src_cnt, 360 360 - int *slots_per_op) 361 361 - { 362 362 - int slot_cnt = iop3xx_aau_xor_slot_count(len, src_cnt, slots_per_op); 363 363 - 364 364 - if (len <= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) 365 365 - return slot_cnt; 366 366 - 367 367 - len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT; 368 368 - while (len > IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) { 369 369 - len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT; 370 370 - slot_cnt += *slots_per_op; 371 371 - } 372 372 - 373 373 - slot_cnt += *slots_per_op; 374 374 - 375 375 - return slot_cnt; 376 376 - } 377 377 - 378 378 - static inline u32 iop_desc_get_byte_count(struct iop_adma_desc_slot *desc, 379 379 - struct iop_adma_chan *chan) 380 380 - { 381 381 - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; 382 382 - 383 383 - switch (chan->device->id) { 384 384 - case DMA0_ID: 385 385 - case DMA1_ID: 386 386 - return hw_desc.dma->byte_count; 387 387 - case AAU_ID: 388 388 - return hw_desc.aau->byte_count; 389 389 - default: 390 390 - BUG(); 391 391 - } 392 392 - return 0; 393 393 - } 394 394 - 395 395 - /* translate the src_idx to a descriptor word index */ 396 396 - static inline int __desc_idx(int src_idx) 397 397 - { 398 398 - static const int desc_idx_table[] = { 0, 0, 0, 0, 399 399 - 0, 1, 2, 3, 400 400 - 5, 6, 7, 8, 401 401 - 9, 10, 11, 12, 402 402 - 14, 15, 16, 17, 403 403 - 18, 19, 20, 21, 404 404 - 23, 24, 25, 26, 405 405 - 27, 28, 29, 30, 406 406 - }; 407 407 - 408 408 - return desc_idx_table[src_idx]; 409 409 - } 410 410 - 411 411 - static inline u32 iop_desc_get_src_addr(struct iop_adma_desc_slot *desc, 412 412 - struct iop_adma_chan *chan, 413 413 - int src_idx) 414 414 - { 415 415 - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; 416 416 - 417 417 - switch (chan->device->id) { 418 418 - case DMA0_ID: 419 419 - case DMA1_ID: 420 420 - return hw_desc.dma->src_addr; 421 421 - case AAU_ID: 422 422 - break; 423 423 - default: 424 424 - BUG(); 425 425 - } 426 426 - 427 427 - if (src_idx < 4) 428 428 - return hw_desc.aau->src[src_idx]; 429 429 - else 430 430 - return hw_desc.aau->src_edc[__desc_idx(src_idx)].src_addr; 431 431 - } 432 432 - 433 433 - static inline void iop3xx_aau_desc_set_src_addr(struct iop3xx_desc_aau *hw_desc, 434 434 - int src_idx, dma_addr_t addr) 435 435 - { 436 436 - if (src_idx < 4) 437 437 - hw_desc->src[src_idx] = addr; 438 438 - else 439 439 - hw_desc->src_edc[__desc_idx(src_idx)].src_addr = addr; 440 440 - } 441 441 - 442 442 - static inline void 443 443 - iop_desc_init_memcpy(struct iop_adma_desc_slot *desc, unsigned long flags) 444 444 - { 445 445 - struct iop3xx_desc_dma *hw_desc = desc->hw_desc; 446 446 - union { 447 447 - u32 value; 448 448 - struct iop3xx_dma_desc_ctrl field; 449 449 - } u_desc_ctrl; 450 450 - 451 451 - u_desc_ctrl.value = 0; 452 452 - u_desc_ctrl.field.mem_to_mem_en = 1; 453 453 - u_desc_ctrl.field.pci_transaction = 0xe; /* memory read block */ 454 454 - u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT; 455 455 - hw_desc->desc_ctrl = u_desc_ctrl.value; 456 456 - hw_desc->upper_pci_src_addr = 0; 457 457 - hw_desc->crc_addr = 0; 458 458 - } 459 459 - 460 460 - static inline void 461 461 - iop_desc_init_memset(struct iop_adma_desc_slot *desc, unsigned long flags) 462 462 - { 463 463 - struct iop3xx_desc_aau *hw_desc = desc->hw_desc; 464 464 - union { 465 465 - u32 value; 466 466 - struct iop3xx_aau_desc_ctrl field; 467 467 - } u_desc_ctrl; 468 468 - 469 469 - u_desc_ctrl.value = 0; 470 470 - u_desc_ctrl.field.blk1_cmd_ctrl = 0x2; /* memory block fill */ 471 471 - u_desc_ctrl.field.dest_write_en = 1; 472 472 - u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT; 473 473 - hw_desc->desc_ctrl = u_desc_ctrl.value; 474 474 - } 475 475 - 476 476 - static inline u32 477 477 - iop3xx_desc_init_xor(struct iop3xx_desc_aau *hw_desc, int src_cnt, 478 478 - unsigned long flags) 479 479 - { 480 480 - int i, shift; 481 481 - u32 edcr; 482 482 - union { 483 483 - u32 value; 484 484 - struct iop3xx_aau_desc_ctrl field; 485 485 - } u_desc_ctrl; 486 486 - 487 487 - u_desc_ctrl.value = 0; 488 488 - switch (src_cnt) { 489 489 - case 25 ... 32: 490 490 - u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */ 491 491 - edcr = 0; 492 492 - shift = 1; 493 493 - for (i = 24; i < src_cnt; i++) { 494 494 - edcr |= (1 << shift); 495 495 - shift += 3; 496 496 - } 497 497 - hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = edcr; 498 498 - src_cnt = 24; 499 499 - fallthrough; 500 500 - case 17 ... 24: 501 501 - if (!u_desc_ctrl.field.blk_ctrl) { 502 502 - hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0; 503 503 - u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */ 504 504 - } 505 505 - edcr = 0; 506 506 - shift = 1; 507 507 - for (i = 16; i < src_cnt; i++) { 508 508 - edcr |= (1 << shift); 509 509 - shift += 3; 510 510 - } 511 511 - hw_desc->src_edc[AAU_EDCR1_IDX].e_desc_ctrl = edcr; 512 512 - src_cnt = 16; 513 513 - fallthrough; 514 514 - case 9 ... 16: 515 515 - if (!u_desc_ctrl.field.blk_ctrl) 516 516 - u_desc_ctrl.field.blk_ctrl = 0x2; /* use EDCR0 */ 517 517 - edcr = 0; 518 518 - shift = 1; 519 519 - for (i = 8; i < src_cnt; i++) { 520 520 - edcr |= (1 << shift); 521 521 - shift += 3; 522 522 - } 523 523 - hw_desc->src_edc[AAU_EDCR0_IDX].e_desc_ctrl = edcr; 524 524 - src_cnt = 8; 525 525 - fallthrough; 526 526 - case 2 ... 8: 527 527 - shift = 1; 528 528 - for (i = 0; i < src_cnt; i++) { 529 529 - u_desc_ctrl.value |= (1 << shift); 530 530 - shift += 3; 531 531 - } 532 532 - 533 533 - if (!u_desc_ctrl.field.blk_ctrl && src_cnt > 4) 534 534 - u_desc_ctrl.field.blk_ctrl = 0x1; /* use mini-desc */ 535 535 - } 536 536 - 537 537 - u_desc_ctrl.field.dest_write_en = 1; 538 538 - u_desc_ctrl.field.blk1_cmd_ctrl = 0x7; /* direct fill */ 539 539 - u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT; 540 540 - hw_desc->desc_ctrl = u_desc_ctrl.value; 541 541 - 542 542 - return u_desc_ctrl.value; 543 543 - } 544 544 - 545 545 - static inline void 546 546 - iop_desc_init_xor(struct iop_adma_desc_slot *desc, int src_cnt, 547 547 - unsigned long flags) 548 548 - { 549 549 - iop3xx_desc_init_xor(desc->hw_desc, src_cnt, flags); 550 550 - } 551 551 - 552 552 - /* return the number of operations */ 553 553 - static inline int 554 554 - iop_desc_init_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt, 555 555 - unsigned long flags) 556 556 - { 557 557 - int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op; 558 558 - struct iop3xx_desc_aau *hw_desc, *prev_hw_desc, *iter; 559 559 - union { 560 560 - u32 value; 561 561 - struct iop3xx_aau_desc_ctrl field; 562 562 - } u_desc_ctrl; 563 563 - int i, j; 564 564 - 565 565 - hw_desc = desc->hw_desc; 566 566 - 567 567 - for (i = 0, j = 0; (slot_cnt -= slots_per_op) >= 0; 568 568 - i += slots_per_op, j++) { 569 569 - iter = iop_hw_desc_slot_idx(hw_desc, i); 570 570 - u_desc_ctrl.value = iop3xx_desc_init_xor(iter, src_cnt, flags); 571 571 - u_desc_ctrl.field.dest_write_en = 0; 572 572 - u_desc_ctrl.field.zero_result_en = 1; 573 573 - u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT; 574 574 - iter->desc_ctrl = u_desc_ctrl.value; 575 575 - 576 576 - /* for the subsequent descriptors preserve the store queue 577 577 - * and chain them together 578 578 - */ 579 579 - if (i) { 580 580 - prev_hw_desc = 581 581 - iop_hw_desc_slot_idx(hw_desc, i - slots_per_op); 582 582 - prev_hw_desc->next_desc = 583 583 - (u32) (desc->async_tx.phys + (i << 5)); 584 584 - } 585 585 - } 586 586 - 587 587 - return j; 588 588 - } 589 589 - 590 590 - static inline void 591 591 - iop_desc_init_null_xor(struct iop_adma_desc_slot *desc, int src_cnt, 592 592 - unsigned long flags) 593 593 - { 594 594 - struct iop3xx_desc_aau *hw_desc = desc->hw_desc; 595 595 - union { 596 596 - u32 value; 597 597 - struct iop3xx_aau_desc_ctrl field; 598 598 - } u_desc_ctrl; 599 599 - 600 600 - u_desc_ctrl.value = 0; 601 601 - switch (src_cnt) { 602 602 - case 25 ... 32: 603 603 - u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */ 604 604 - hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0; 605 605 - fallthrough; 606 606 - case 17 ... 24: 607 607 - if (!u_desc_ctrl.field.blk_ctrl) { 608 608 - hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0; 609 609 - u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */ 610 610 - } 611 611 - hw_desc->src_edc[AAU_EDCR1_IDX].e_desc_ctrl = 0; 612 612 - fallthrough; 613 613 - case 9 ... 16: 614 614 - if (!u_desc_ctrl.field.blk_ctrl) 615 615 - u_desc_ctrl.field.blk_ctrl = 0x2; /* use EDCR0 */ 616 616 - hw_desc->src_edc[AAU_EDCR0_IDX].e_desc_ctrl = 0; 617 617 - fallthrough; 618 618 - case 1 ... 8: 619 619 - if (!u_desc_ctrl.field.blk_ctrl && src_cnt > 4) 620 620 - u_desc_ctrl.field.blk_ctrl = 0x1; /* use mini-desc */ 621 621 - } 622 622 - 623 623 - u_desc_ctrl.field.dest_write_en = 0; 624 624 - u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT; 625 625 - hw_desc->desc_ctrl = u_desc_ctrl.value; 626 626 - } 627 627 - 628 628 - static inline void iop_desc_set_byte_count(struct iop_adma_desc_slot *desc, 629 629 - struct iop_adma_chan *chan, 630 630 - u32 byte_count) 631 631 - { 632 632 - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; 633 633 - 634 634 - switch (chan->device->id) { 635 635 - case DMA0_ID: 636 636 - case DMA1_ID: 637 637 - hw_desc.dma->byte_count = byte_count; 638 638 - break; 639 639 - case AAU_ID: 640 640 - hw_desc.aau->byte_count = byte_count; 641 641 - break; 642 642 - default: 643 643 - BUG(); 644 644 - } 645 645 - } 646 646 - 647 647 - static inline void 648 648 - iop_desc_init_interrupt(struct iop_adma_desc_slot *desc, 649 649 - struct iop_adma_chan *chan) 650 650 - { 651 651 - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; 652 652 - 653 653 - switch (chan->device->id) { 654 654 - case DMA0_ID: 655 655 - case DMA1_ID: 656 656 - iop_desc_init_memcpy(desc, 1); 657 657 - hw_desc.dma->byte_count = 0; 658 658 - hw_desc.dma->dest_addr = 0; 659 659 - hw_desc.dma->src_addr = 0; 660 660 - break; 661 661 - case AAU_ID: 662 662 - iop_desc_init_null_xor(desc, 2, 1); 663 663 - hw_desc.aau->byte_count = 0; 664 664 - hw_desc.aau->dest_addr = 0; 665 665 - hw_desc.aau->src[0] = 0; 666 666 - hw_desc.aau->src[1] = 0; 667 667 - break; 668 668 - default: 669 669 - BUG(); 670 670 - } 671 671 - } 672 672 - 673 673 - static inline void 674 674 - iop_desc_set_zero_sum_byte_count(struct iop_adma_desc_slot *desc, u32 len) 675 675 - { 676 676 - int slots_per_op = desc->slots_per_op; 677 677 - struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter; 678 678 - int i = 0; 679 679 - 680 680 - if (len <= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) { 681 681 - hw_desc->byte_count = len; 682 682 - } else { 683 683 - do { 684 684 - iter = iop_hw_desc_slot_idx(hw_desc, i); 685 685 - iter->byte_count = IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT; 686 686 - len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT; 687 687 - i += slots_per_op; 688 688 - } while (len > IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT); 689 689 - 690 690 - iter = iop_hw_desc_slot_idx(hw_desc, i); 691 691 - iter->byte_count = len; 692 692 - } 693 693 - } 694 694 - 695 695 - static inline void iop_desc_set_dest_addr(struct iop_adma_desc_slot *desc, 696 696 - struct iop_adma_chan *chan, 697 697 - dma_addr_t addr) 698 698 - { 699 699 - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; 700 700 - 701 701 - switch (chan->device->id) { 702 702 - case DMA0_ID: 703 703 - case DMA1_ID: 704 704 - hw_desc.dma->dest_addr = addr; 705 705 - break; 706 706 - case AAU_ID: 707 707 - hw_desc.aau->dest_addr = addr; 708 708 - break; 709 709 - default: 710 710 - BUG(); 711 711 - } 712 712 - } 713 713 - 714 714 - static inline void iop_desc_set_memcpy_src_addr(struct iop_adma_desc_slot *desc, 715 715 - dma_addr_t addr) 716 716 - { 717 717 - struct iop3xx_desc_dma *hw_desc = desc->hw_desc; 718 718 - hw_desc->src_addr = addr; 719 719 - } 720 720 - 721 721 - static inline void 722 722 - iop_desc_set_zero_sum_src_addr(struct iop_adma_desc_slot *desc, int src_idx, 723 723 - dma_addr_t addr) 724 724 - { 725 725 - 726 726 - struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter; 727 727 - int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op; 728 728 - int i; 729 729 - 730 730 - for (i = 0; (slot_cnt -= slots_per_op) >= 0; 731 731 - i += slots_per_op, addr += IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) { 732 732 - iter = iop_hw_desc_slot_idx(hw_desc, i); 733 733 - iop3xx_aau_desc_set_src_addr(iter, src_idx, addr); 734 734 - } 735 735 - } 736 736 - 737 737 - static inline void iop_desc_set_xor_src_addr(struct iop_adma_desc_slot *desc, 738 738 - int src_idx, dma_addr_t addr) 739 739 - { 740 740 - 741 741 - struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter; 742 742 - int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op; 743 743 - int i; 744 744 - 745 745 - for (i = 0; (slot_cnt -= slots_per_op) >= 0; 746 746 - i += slots_per_op, addr += IOP_ADMA_XOR_MAX_BYTE_COUNT) { 747 747 - iter = iop_hw_desc_slot_idx(hw_desc, i); 748 748 - iop3xx_aau_desc_set_src_addr(iter, src_idx, addr); 749 749 - } 750 750 - } 751 751 - 752 752 - static inline void iop_desc_set_next_desc(struct iop_adma_desc_slot *desc, 753 753 - u32 next_desc_addr) 754 754 - { 755 755 - /* hw_desc->next_desc is the same location for all channels */ 756 756 - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; 757 757 - 758 758 - iop_paranoia(hw_desc.dma->next_desc); 759 759 - hw_desc.dma->next_desc = next_desc_addr; 760 760 - } 761 761 - 762 762 - static inline u32 iop_desc_get_next_desc(struct iop_adma_desc_slot *desc) 763 763 - { 764 764 - /* hw_desc->next_desc is the same location for all channels */ 765 765 - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; 766 766 - return hw_desc.dma->next_desc; 767 767 - } 768 768 - 769 769 - static inline void iop_desc_clear_next_desc(struct iop_adma_desc_slot *desc) 770 770 - { 771 771 - /* hw_desc->next_desc is the same location for all channels */ 772 772 - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; 773 773 - hw_desc.dma->next_desc = 0; 774 774 - } 775 775 - 776 776 - static inline void iop_desc_set_block_fill_val(struct iop_adma_desc_slot *desc, 777 777 - u32 val) 778 778 - { 779 779 - struct iop3xx_desc_aau *hw_desc = desc->hw_desc; 780 780 - hw_desc->src[0] = val; 781 781 - } 782 782 - 783 783 - static inline enum sum_check_flags 784 784 - iop_desc_get_zero_result(struct iop_adma_desc_slot *desc) 785 785 - { 786 786 - struct iop3xx_desc_aau *hw_desc = desc->hw_desc; 787 787 - struct iop3xx_aau_desc_ctrl desc_ctrl = hw_desc->desc_ctrl_field; 788 788 - 789 789 - iop_paranoia(!(desc_ctrl.tx_complete && desc_ctrl.zero_result_en)); 790 790 - return desc_ctrl.zero_result_err << SUM_CHECK_P; 791 791 - } 792 792 - 793 793 - static inline void iop_chan_append(struct iop_adma_chan *chan) 794 794 - { 795 795 - u32 dma_chan_ctrl; 796 796 - 797 797 - dma_chan_ctrl = __raw_readl(DMA_CCR(chan)); 798 798 - dma_chan_ctrl |= 0x2; 799 799 - __raw_writel(dma_chan_ctrl, DMA_CCR(chan)); 800 800 - } 801 801 - 802 802 - static inline u32 iop_chan_get_status(struct iop_adma_chan *chan) 803 803 - { 804 804 - return __raw_readl(DMA_CSR(chan)); 805 805 - } 806 806 - 807 807 - static inline void iop_chan_disable(struct iop_adma_chan *chan) 808 808 - { 809 809 - u32 dma_chan_ctrl = __raw_readl(DMA_CCR(chan)); 810 810 - dma_chan_ctrl &= ~1; 811 811 - __raw_writel(dma_chan_ctrl, DMA_CCR(chan)); 812 812 - } 813 813 - 814 814 - static inline void iop_chan_enable(struct iop_adma_chan *chan) 815 815 - { 816 816 - u32 dma_chan_ctrl = __raw_readl(DMA_CCR(chan)); 817 817 - 818 818 - dma_chan_ctrl |= 1; 819 819 - __raw_writel(dma_chan_ctrl, DMA_CCR(chan)); 820 820 - } 821 821 - 822 822 - static inline void iop_adma_device_clear_eot_status(struct iop_adma_chan *chan) 823 823 - { 824 824 - u32 status = __raw_readl(DMA_CSR(chan)); 825 825 - status &= (1 << 9); 826 826 - __raw_writel(status, DMA_CSR(chan)); 827 827 - } 828 828 - 829 829 - static inline void iop_adma_device_clear_eoc_status(struct iop_adma_chan *chan) 830 830 - { 831 831 - u32 status = __raw_readl(DMA_CSR(chan)); 832 832 - status &= (1 << 8); 833 833 - __raw_writel(status, DMA_CSR(chan)); 834 834 - } 835 835 - 836 836 - static inline void iop_adma_device_clear_err_status(struct iop_adma_chan *chan) 837 837 - { 838 838 - u32 status = __raw_readl(DMA_CSR(chan)); 839 839 - 840 840 - switch (chan->device->id) { 841 841 - case DMA0_ID: 842 842 - case DMA1_ID: 843 843 - status &= (1 << 5) | (1 << 3) | (1 << 2) | (1 << 1); 844 844 - break; 845 845 - case AAU_ID: 846 846 - status &= (1 << 5); 847 847 - break; 848 848 - default: 849 849 - BUG(); 850 850 - } 851 851 - 852 852 - __raw_writel(status, DMA_CSR(chan)); 853 853 - } 854 854 - 855 855 - static inline int 856 856 - iop_is_err_int_parity(unsigned long status, struct iop_adma_chan *chan) 857 857 - { 858 858 - return 0; 859 859 - } 860 860 - 861 861 - static inline int 862 862 - iop_is_err_mcu_abort(unsigned long status, struct iop_adma_chan *chan) 863 863 - { 864 864 - return 0; 865 865 - } 866 866 - 867 867 - static inline int 868 868 - iop_is_err_int_tabort(unsigned long status, struct iop_adma_chan *chan) 869 869 - { 870 870 - return 0; 871 871 - } 872 872 - 873 873 - static inline int 874 874 - iop_is_err_int_mabort(unsigned long status, struct iop_adma_chan *chan) 875 875 - { 876 876 - return test_bit(5, &status); 877 877 - } 878 878 - 879 879 - static inline int 880 880 - iop_is_err_pci_tabort(unsigned long status, struct iop_adma_chan *chan) 881 881 - { 882 882 - switch (chan->device->id) { 883 883 - case DMA0_ID: 884 884 - case DMA1_ID: 885 885 - return test_bit(2, &status); 886 886 - default: 887 887 - return 0; 888 888 - } 889 889 - } 890 890 - 891 891 - static inline int 892 892 - iop_is_err_pci_mabort(unsigned long status, struct iop_adma_chan *chan) 893 893 - { 894 894 - switch (chan->device->id) { 895 895 - case DMA0_ID: 896 896 - case DMA1_ID: 897 897 - return test_bit(3, &status); 898 898 - default: 899 899 - return 0; 900 900 - } 901 901 - } 902 902 - 903 903 - static inline int 904 904 - iop_is_err_split_tx(unsigned long status, struct iop_adma_chan *chan) 905 905 - { 906 906 - switch (chan->device->id) { 907 907 - case DMA0_ID: 908 908 - case DMA1_ID: 909 909 - return test_bit(1, &status); 910 910 - default: 911 911 - return 0; 912 912 - } 913 913 - } 914 914 - #endif /* _ADMA_H */

+6 -1

drivers/dma/qcom/gpi.c

reviewed

··· 2286 2286 } 2287 2287 2288 2288 static const struct of_device_id gpi_of_match[] = { 2289 2289 - { .compatible = "qcom,sc7280-gpi-dma", .data = (void *)0x10000 }, 2290 2289 { .compatible = "qcom,sdm845-gpi-dma", .data = (void *)0x0 }, 2291 2290 { .compatible = "qcom,sm6350-gpi-dma", .data = (void *)0x10000 }, 2291 2291 + /* 2292 2292 + * Do not grow the list for compatible devices. Instead use 2293 2293 + * qcom,sdm845-gpi-dma (for ee_offset = 0x0) or qcom,sm6350-gpi-dma 2294 2294 + * (for ee_offset = 0x10000). 2295 2295 + */ 2296 2296 + { .compatible = "qcom,sc7280-gpi-dma", .data = (void *)0x10000 }, 2292 2297 { .compatible = "qcom,sm8150-gpi-dma", .data = (void *)0x0 }, 2293 2298 { .compatible = "qcom,sm8250-gpi-dma", .data = (void *)0x0 }, 2294 2299 { .compatible = "qcom,sm8350-gpi-dma", .data = (void *)0x10000 },

-48

drivers/dma/sh/shdma-arm.h

reviewed

··· 1 1 - /* SPDX-License-Identifier: GPL-2.0 */ 2 2 - /* 3 3 - * Renesas SuperH DMA Engine support 4 4 - * 5 5 - * Copyright (C) 2013 Renesas Electronics, Inc. 6 6 - */ 7 7 - 8 8 - #ifndef SHDMA_ARM_H 9 9 - #define SHDMA_ARM_H 10 10 - 11 11 - #include "shdma.h" 12 12 - 13 13 - /* Transmit sizes and respective CHCR register values */ 14 14 - enum { 15 15 - XMIT_SZ_8BIT = 0, 16 16 - XMIT_SZ_16BIT = 1, 17 17 - XMIT_SZ_32BIT = 2, 18 18 - XMIT_SZ_64BIT = 7, 19 19 - XMIT_SZ_128BIT = 3, 20 20 - XMIT_SZ_256BIT = 4, 21 21 - XMIT_SZ_512BIT = 5, 22 22 - }; 23 23 - 24 24 - /* log2(size / 8) - used to calculate number of transfers */ 25 25 - #define SH_DMAE_TS_SHIFT { \ 26 26 - [XMIT_SZ_8BIT] = 0, \ 27 27 - [XMIT_SZ_16BIT] = 1, \ 28 28 - [XMIT_SZ_32BIT] = 2, \ 29 29 - [XMIT_SZ_64BIT] = 3, \ 30 30 - [XMIT_SZ_128BIT] = 4, \ 31 31 - [XMIT_SZ_256BIT] = 5, \ 32 32 - [XMIT_SZ_512BIT] = 6, \ 33 33 - } 34 34 - 35 35 - #define TS_LOW_BIT 0x3 /* --xx */ 36 36 - #define TS_HI_BIT 0xc /* xx-- */ 37 37 - 38 38 - #define TS_LOW_SHIFT (3) 39 39 - #define TS_HI_SHIFT (20 - 2) /* 2 bits for shifted low TS */ 40 40 - 41 41 - #define TS_INDEX2VAL(i) \ 42 42 - ((((i) & TS_LOW_BIT) << TS_LOW_SHIFT) |\ 43 43 - (((i) & TS_HI_BIT) << TS_HI_SHIFT)) 44 44 - 45 45 - #define CHCR_TX(xmit_sz) (DM_FIX | SM_INC | RS_ERS | TS_INDEX2VAL((xmit_sz))) 46 46 - #define CHCR_RX(xmit_sz) (DM_INC | SM_FIX | RS_ERS | TS_INDEX2VAL((xmit_sz))) 47 47 - 48 48 - #endif

+30 -7

drivers/dma/tegra186-gpc-dma.c

reviewed

··· 161 161 #define TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT 5000 /* 5 msec */ 162 162 163 163 /* Channel base address offset from GPCDMA base address */ 164 164 - #define TEGRA_GPCDMA_CHANNEL_BASE_ADD_OFFSET 0x20000 164 164 + #define TEGRA_GPCDMA_CHANNEL_BASE_ADDR_OFFSET 0x10000 165 165 + 166 166 + /* Default channel mask reserving channel0 */ 167 167 + #define TEGRA_GPCDMA_DEFAULT_CHANNEL_MASK 0xfffffffe 165 168 166 169 struct tegra_dma; 167 170 struct tegra_dma_channel; ··· 249 246 const struct tegra_dma_chip_data *chip_data; 250 247 unsigned long sid_m2d_reserved; 251 248 unsigned long sid_d2m_reserved; 249 249 + u32 chan_mask; 252 250 void __iomem *base_addr; 253 251 struct device *dev; 254 252 struct dma_device dma_dev; ··· 1292 1288 } 1293 1289 1294 1290 static const struct tegra_dma_chip_data tegra186_dma_chip_data = { 1295 1295 - .nr_channels = 31, 1291 1291 + .nr_channels = 32, 1296 1292 .channel_reg_size = SZ_64K, 1297 1293 .max_dma_count = SZ_1G, 1298 1294 .hw_support_pause = false, ··· 1300 1296 }; 1301 1297 1302 1298 static const struct tegra_dma_chip_data tegra194_dma_chip_data = { 1303 1303 - .nr_channels = 31, 1299 1299 + .nr_channels = 32, 1304 1300 .channel_reg_size = SZ_64K, 1305 1301 .max_dma_count = SZ_1G, 1306 1302 .hw_support_pause = true, ··· 1308 1304 }; 1309 1305 1310 1306 static const struct tegra_dma_chip_data tegra234_dma_chip_data = { 1311 1311 - .nr_channels = 31, 1307 1307 + .nr_channels = 32, 1312 1308 .channel_reg_size = SZ_64K, 1313 1309 .max_dma_count = SZ_1G, 1314 1310 .hw_support_pause = true, ··· 1384 1380 } 1385 1381 stream_id = iommu_spec->ids[0] & 0xffff; 1386 1382 1383 1383 + ret = device_property_read_u32(&pdev->dev, "dma-channel-mask", 1384 1384 + &tdma->chan_mask); 1385 1385 + if (ret) { 1386 1386 + dev_warn(&pdev->dev, 1387 1387 + "Missing dma-channel-mask property, using default channel mask %#x\n", 1388 1388 + TEGRA_GPCDMA_DEFAULT_CHANNEL_MASK); 1389 1389 + tdma->chan_mask = TEGRA_GPCDMA_DEFAULT_CHANNEL_MASK; 1390 1390 + } 1391 1391 + 1387 1392 INIT_LIST_HEAD(&tdma->dma_dev.channels); 1388 1393 for (i = 0; i < cdata->nr_channels; i++) { 1389 1394 struct tegra_dma_channel *tdc = &tdma->channels[i]; 1395 1395 + 1396 1396 + /* Check for channel mask */ 1397 1397 + if (!(tdma->chan_mask & BIT(i))) 1398 1398 + continue; 1390 1399 1391 1400 tdc->irq = platform_get_irq(pdev, i); 1392 1401 if (tdc->irq < 0) 1393 1402 return tdc->irq; 1394 1403 1395 1395 - tdc->chan_base_offset = TEGRA_GPCDMA_CHANNEL_BASE_ADD_OFFSET + 1404 1404 + tdc->chan_base_offset = TEGRA_GPCDMA_CHANNEL_BASE_ADDR_OFFSET + 1396 1405 i * cdata->channel_reg_size; 1397 1406 snprintf(tdc->name, sizeof(tdc->name), "gpcdma.%d", i); 1398 1407 tdc->tdma = tdma; ··· 1466 1449 return ret; 1467 1450 } 1468 1451 1469 1469 - dev_info(&pdev->dev, "GPC DMA driver register %d channels\n", 1470 1470 - cdata->nr_channels); 1452 1452 + dev_info(&pdev->dev, "GPC DMA driver register %lu channels\n", 1453 1453 + hweight_long(tdma->chan_mask)); 1471 1454 1472 1455 return 0; 1473 1456 } ··· 1490 1473 for (i = 0; i < tdma->chip_data->nr_channels; i++) { 1491 1474 struct tegra_dma_channel *tdc = &tdma->channels[i]; 1492 1475 1476 1476 + if (!(tdma->chan_mask & BIT(i))) 1477 1477 + continue; 1478 1478 + 1493 1479 if (tdc->dma_desc) { 1494 1480 dev_err(tdma->dev, "channel %u busy\n", i); 1495 1481 return -EBUSY; ··· 1511 1491 1512 1492 for (i = 0; i < tdma->chip_data->nr_channels; i++) { 1513 1493 struct tegra_dma_channel *tdc = &tdma->channels[i]; 1494 1494 + 1495 1495 + if (!(tdma->chan_mask & BIT(i))) 1496 1496 + continue; 1514 1497 1515 1498 tegra_dma_program_sid(tdc, tdc->stream_id); 1516 1499 }

+4 -3

drivers/dma/ti/Kconfig

reviewed

··· 35 35 DMA engine is found on OMAP and DRA7xx parts. 36 36 37 37 config TI_K3_UDMA 38 38 - bool "Texas Instruments UDMA support" 38 38 + tristate "Texas Instruments UDMA support" 39 39 depends on ARCH_K3 40 40 depends on TI_SCI_PROTOCOL 41 41 depends on TI_SCI_INTA_IRQCHIP ··· 48 48 DMA engine is used in AM65x and j721e. 49 49 50 50 config TI_K3_UDMA_GLUE_LAYER 51 51 - bool "Texas Instruments UDMA Glue layer for non DMAengine users" 51 51 + tristate "Texas Instruments UDMA Glue layer for non DMAengine users" 52 52 depends on ARCH_K3 53 53 depends on TI_K3_UDMA 54 54 help ··· 56 56 If unsure, say N. 57 57 58 58 config TI_K3_PSIL 59 59 - bool 59 59 + tristate 60 60 + default TI_K3_UDMA 60 61 61 62 config TI_DMA_CROSSBAR 62 63 bool

+8 -7

drivers/dma/ti/Makefile

reviewed

··· 4 4 obj-$(CONFIG_DMA_OMAP) += omap-dma.o 5 5 obj-$(CONFIG_TI_K3_UDMA) += k3-udma.o 6 6 obj-$(CONFIG_TI_K3_UDMA_GLUE_LAYER) += k3-udma-glue.o 7 7 - obj-$(CONFIG_TI_K3_PSIL) += k3-psil.o \ 8 8 - k3-psil-am654.o \ 9 9 - k3-psil-j721e.o \ 10 10 - k3-psil-j7200.o \ 11 11 - k3-psil-am64.o \ 12 12 - k3-psil-j721s2.o \ 13 13 - k3-psil-am62.o 7 7 + k3-psil-lib-objs := k3-psil.o \ 8 8 + k3-psil-am654.o \ 9 9 + k3-psil-j721e.o \ 10 10 + k3-psil-j7200.o \ 11 11 + k3-psil-am64.o \ 12 12 + k3-psil-j721s2.o \ 13 13 + k3-psil-am62.o 14 14 + obj-$(CONFIG_TI_K3_PSIL) += k3-psil-lib.o 14 15 obj-$(CONFIG_TI_DMA_CROSSBAR) += dma-crossbar.o

+2

drivers/dma/ti/k3-psil.c

reviewed

··· 5 5 */ 6 6 7 7 #include <linux/kernel.h> 8 8 + #include <linux/module.h> 8 9 #include <linux/device.h> 9 10 #include <linux/init.h> 10 11 #include <linux/mutex.h> ··· 102 101 return 0; 103 102 } 104 103 EXPORT_SYMBOL_GPL(psil_set_new_ep_config); 104 104 + MODULE_LICENSE("GPL v2");

+4 -1

drivers/dma/ti/k3-udma-glue.c

reviewed

··· 6 6 * 7 7 */ 8 8 9 9 + #include <linux/module.h> 9 10 #include <linux/atomic.h> 10 11 #include <linux/delay.h> 11 12 #include <linux/dma-mapping.h> ··· 1437 1436 { 1438 1437 return class_register(&k3_udma_glue_devclass); 1439 1438 } 1440 1440 - arch_initcall(k3_udma_glue_class_init); 1439 1439 + 1440 1440 + module_init(k3_udma_glue_class_init); 1441 1441 + MODULE_LICENSE("GPL v2");

+5 -35

drivers/dma/ti/k3-udma.c

reviewed

··· 5 5 */ 6 6 7 7 #include <linux/kernel.h> 8 8 + #include <linux/module.h> 8 9 #include <linux/delay.h> 9 10 #include <linux/dmaengine.h> 10 11 #include <linux/dma-mapping.h> ··· 4336 4335 .compatible = "ti,j721e-navss-mcu-udmap", 4337 4336 .data = &j721e_mcu_data, 4338 4337 }, 4339 4339 - { /* Sentinel */ }, 4340 4340 - }; 4341 4341 - 4342 4342 - static const struct of_device_id bcdma_of_match[] = { 4343 4338 { 4344 4339 .compatible = "ti,am64-dmss-bcdma", 4345 4340 .data = &am64_bcdma_data, 4346 4341 }, 4347 4347 - { /* Sentinel */ }, 4348 4348 - }; 4349 4349 - 4350 4350 - static const struct of_device_id pktdma_of_match[] = { 4351 4342 { 4352 4343 .compatible = "ti,am64-dmss-pktdma", 4353 4344 .data = &am64_pktdma_data, ··· 5264 5271 return -ENOMEM; 5265 5272 5266 5273 match = of_match_node(udma_of_match, dev->of_node); 5267 5267 - if (!match) 5268 5268 - match = of_match_node(bcdma_of_match, dev->of_node); 5269 5274 if (!match) { 5270 5270 - match = of_match_node(pktdma_of_match, dev->of_node); 5271 5271 - if (!match) { 5272 5272 - dev_err(dev, "No compatible match found\n"); 5273 5273 - return -ENODEV; 5274 5274 - } 5275 5275 + dev_err(dev, "No compatible match found\n"); 5276 5276 + return -ENODEV; 5275 5277 } 5276 5278 ud->match_data = match->data; 5277 5279 ··· 5499 5511 }, 5500 5512 .probe = udma_probe, 5501 5513 }; 5502 5502 - builtin_platform_driver(udma_driver); 5503 5514 5504 5504 - static struct platform_driver bcdma_driver = { 5505 5505 - .driver = { 5506 5506 - .name = "ti-bcdma", 5507 5507 - .of_match_table = bcdma_of_match, 5508 5508 - .suppress_bind_attrs = true, 5509 5509 - }, 5510 5510 - .probe = udma_probe, 5511 5511 - }; 5512 5512 - builtin_platform_driver(bcdma_driver); 5513 5513 - 5514 5514 - static struct platform_driver pktdma_driver = { 5515 5515 - .driver = { 5516 5516 - .name = "ti-pktdma", 5517 5517 - .of_match_table = pktdma_of_match, 5518 5518 - .suppress_bind_attrs = true, 5519 5519 - }, 5520 5520 - .probe = udma_probe, 5521 5521 - }; 5522 5522 - builtin_platform_driver(pktdma_driver); 5515 5515 + module_platform_driver(udma_driver); 5516 5516 + MODULE_LICENSE("GPL v2"); 5523 5517 5524 5518 /* Private interfaces to UDMA */ 5525 5519 #include "k3-udma-private.c"

+3 -1

drivers/dma/xilinx/xilinx_dma.c

reviewed

··· 1659 1659 * xilinx_dma_device_config - Configure the DMA channel 1660 1660 * @dchan: DMA channel 1661 1661 * @config: channel configuration 1662 1662 + * 1663 1663 + * Return: 0 always. 1662 1664 */ 1663 1665 static int xilinx_dma_device_config(struct dma_chan *dchan, 1664 1666 struct dma_slave_config *config) ··· 2926 2924 * @xdev: Driver specific device structure 2927 2925 * @node: Device node 2928 2926 * 2929 2929 - * Return: 0 always. 2927 2927 + * Return: '0' on success and failure value on error. 2930 2928 */ 2931 2929 static int xilinx_dma_child_probe(struct xilinx_dma_device *xdev, 2932 2930 struct device_node *node)

+1

drivers/of/irq.c

reviewed

··· 730 730 731 731 return NULL; 732 732 } 733 733 + EXPORT_SYMBOL_GPL(of_msi_get_domain); 733 734 734 735 /** 735 736 * of_msi_configure - Set the msi_domain field of a device

+1 -1

include/uapi/linux/idxd.h

reviewed

··· 295 295 }; 296 296 297 297 uint32_t delta_rec_size; 298 298 - uint32_t crc_val; 298 298 + uint64_t crc_val; 299 299 300 300 /* DIF check & strip */ 301 301 struct {