Merge tag 'dmaengine-fix-6.11' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine

+8 -18

drivers/dma/dw-edma/dw-hdma-v0-core.c

··· 17 17 DW_HDMA_V0_CB = BIT(0), 18 18 DW_HDMA_V0_TCB = BIT(1), 19 19 DW_HDMA_V0_LLP = BIT(2), 20 - DW_HDMA_V0_LIE = BIT(3), 21 - DW_HDMA_V0_RIE = BIT(4), 20 + DW_HDMA_V0_LWIE = BIT(3), 21 + DW_HDMA_V0_RWIE = BIT(4), 22 22 DW_HDMA_V0_CCS = BIT(8), 23 23 DW_HDMA_V0_LLE = BIT(9), 24 24 }; ··· 195 195 static void dw_hdma_v0_core_write_chunk(struct dw_edma_chunk *chunk) 196 196 { 197 197 struct dw_edma_burst *child; 198 - struct dw_edma_chan *chan = chunk->chan; 199 198 u32 control = 0, i = 0; 200 - int j; 201 199 202 200 if (chunk->cb) 203 201 control = DW_HDMA_V0_CB; 204 202 205 - j = chunk->bursts_alloc; 206 - list_for_each_entry(child, &chunk->burst->list, list) { 207 - j--; 208 - if (!j) { 209 - control |= DW_HDMA_V0_LIE; 210 - if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) 211 - control |= DW_HDMA_V0_RIE; 212 - } 213 - 203 + list_for_each_entry(child, &chunk->burst->list, list) 214 204 dw_hdma_v0_write_ll_data(chunk, i++, control, child->sz, 215 205 child->sar, child->dar); 216 - } 217 206 218 207 control = DW_HDMA_V0_LLP | DW_HDMA_V0_TCB; 219 208 if (!chunk->cb) ··· 236 247 if (first) { 237 248 /* Enable engine */ 238 249 SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0)); 239 - /* Interrupt enable&unmask - done, abort */ 240 - tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup) | 241 - HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK | 242 - HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_LOCAL_ABORT_INT_EN; 250 + /* Interrupt unmask - stop, abort */ 251 + tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup); 252 + tmp &= ~(HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); 253 + /* Interrupt enable - stop, abort */ 254 + tmp |= HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_LOCAL_ABORT_INT_EN; 243 255 if (!(dw->chip->flags & DW_EDMA_CHIP_LOCAL)) 244 256 tmp |= HDMA_V0_REMOTE_STOP_INT_EN | HDMA_V0_REMOTE_ABORT_INT_EN; 245 257 SET_CH_32(dw, chan->dir, chan->id, int_setup, tmp);

+109 -22

drivers/dma/dw/core.c

··· 16 16 #include <linux/init.h> 17 17 #include <linux/interrupt.h> 18 18 #include <linux/io.h> 19 + #include <linux/log2.h> 19 20 #include <linux/mm.h> 20 21 #include <linux/module.h> 21 22 #include <linux/slab.h> ··· 622 621 struct dw_desc *prev; 623 622 struct dw_desc *first; 624 623 u32 ctllo, ctlhi; 625 - u8 m_master = dwc->dws.m_master; 626 - u8 lms = DWC_LLP_LMS(m_master); 624 + u8 lms = DWC_LLP_LMS(dwc->dws.m_master); 627 625 dma_addr_t reg; 628 626 unsigned int reg_width; 629 627 unsigned int mem_width; 630 - unsigned int data_width = dw->pdata->data_width[m_master]; 631 628 unsigned int i; 632 629 struct scatterlist *sg; 633 630 size_t total_len = 0; ··· 659 660 mem = sg_dma_address(sg); 660 661 len = sg_dma_len(sg); 661 662 662 - mem_width = __ffs(data_width | mem | len); 663 + mem_width = __ffs(sconfig->src_addr_width | mem | len); 663 664 664 665 slave_sg_todev_fill_desc: 665 666 desc = dwc_desc_get(dwc); ··· 719 720 lli_write(desc, sar, reg); 720 721 lli_write(desc, dar, mem); 721 722 lli_write(desc, ctlhi, ctlhi); 722 - mem_width = __ffs(data_width | mem); 723 + mem_width = __ffs(sconfig->dst_addr_width | mem); 723 724 lli_write(desc, ctllo, ctllo | DWC_CTLL_DST_WIDTH(mem_width)); 724 725 desc->len = dlen; 725 726 ··· 779 780 } 780 781 EXPORT_SYMBOL_GPL(dw_dma_filter); 781 782 782 - static int dwc_config(struct dma_chan *chan, struct dma_slave_config *sconfig) 783 + static int dwc_verify_maxburst(struct dma_chan *chan) 784 + { 785 + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); 786 + 787 + dwc->dma_sconfig.src_maxburst = 788 + clamp(dwc->dma_sconfig.src_maxburst, 1U, dwc->max_burst); 789 + dwc->dma_sconfig.dst_maxburst = 790 + clamp(dwc->dma_sconfig.dst_maxburst, 1U, dwc->max_burst); 791 + 792 + dwc->dma_sconfig.src_maxburst = 793 + rounddown_pow_of_two(dwc->dma_sconfig.src_maxburst); 794 + dwc->dma_sconfig.dst_maxburst = 795 + rounddown_pow_of_two(dwc->dma_sconfig.dst_maxburst); 796 + 797 + return 0; 798 + } 799 + 800 + static int dwc_verify_p_buswidth(struct dma_chan *chan) 783 801 { 784 802 struct dw_dma_chan *dwc = to_dw_dma_chan(chan); 785 803 struct dw_dma *dw = to_dw_dma(chan->device); 804 + u32 reg_width, max_width; 805 + 806 + if (dwc->dma_sconfig.direction == DMA_MEM_TO_DEV) 807 + reg_width = dwc->dma_sconfig.dst_addr_width; 808 + else if (dwc->dma_sconfig.direction == DMA_DEV_TO_MEM) 809 + reg_width = dwc->dma_sconfig.src_addr_width; 810 + else /* DMA_MEM_TO_MEM */ 811 + return 0; 812 + 813 + max_width = dw->pdata->data_width[dwc->dws.p_master]; 814 + 815 + /* Fall-back to 1-byte transfer width if undefined */ 816 + if (reg_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) 817 + reg_width = DMA_SLAVE_BUSWIDTH_1_BYTE; 818 + else if (!is_power_of_2(reg_width) || reg_width > max_width) 819 + return -EINVAL; 820 + else /* bus width is valid */ 821 + return 0; 822 + 823 + /* Update undefined addr width value */ 824 + if (dwc->dma_sconfig.direction == DMA_MEM_TO_DEV) 825 + dwc->dma_sconfig.dst_addr_width = reg_width; 826 + else /* DMA_DEV_TO_MEM */ 827 + dwc->dma_sconfig.src_addr_width = reg_width; 828 + 829 + return 0; 830 + } 831 + 832 + static int dwc_verify_m_buswidth(struct dma_chan *chan) 833 + { 834 + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); 835 + struct dw_dma *dw = to_dw_dma(chan->device); 836 + u32 reg_width, reg_burst, mem_width; 837 + 838 + mem_width = dw->pdata->data_width[dwc->dws.m_master]; 839 + 840 + /* 841 + * It's possible to have a data portion locked in the DMA FIFO in case 842 + * of the channel suspension. Subsequent channel disabling will cause 843 + * that data silent loss. In order to prevent that maintain the src and 844 + * dst transfer widths coherency by means of the relation: 845 + * (CTLx.SRC_TR_WIDTH * CTLx.SRC_MSIZE >= CTLx.DST_TR_WIDTH) 846 + * Look for the details in the commit message that brings this change. 847 + * 848 + * Note the DMA configs utilized in the calculations below must have 849 + * been verified to have correct values by this method call. 850 + */ 851 + if (dwc->dma_sconfig.direction == DMA_MEM_TO_DEV) { 852 + reg_width = dwc->dma_sconfig.dst_addr_width; 853 + if (mem_width < reg_width) 854 + return -EINVAL; 855 + 856 + dwc->dma_sconfig.src_addr_width = mem_width; 857 + } else if (dwc->dma_sconfig.direction == DMA_DEV_TO_MEM) { 858 + reg_width = dwc->dma_sconfig.src_addr_width; 859 + reg_burst = dwc->dma_sconfig.src_maxburst; 860 + 861 + dwc->dma_sconfig.dst_addr_width = min(mem_width, reg_width * reg_burst); 862 + } 863 + 864 + return 0; 865 + } 866 + 867 + static int dwc_config(struct dma_chan *chan, struct dma_slave_config *sconfig) 868 + { 869 + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); 870 + int ret; 786 871 787 872 memcpy(&dwc->dma_sconfig, sconfig, sizeof(*sconfig)); 788 873 789 - dwc->dma_sconfig.src_maxburst = 790 - clamp(dwc->dma_sconfig.src_maxburst, 0U, dwc->max_burst); 791 - dwc->dma_sconfig.dst_maxburst = 792 - clamp(dwc->dma_sconfig.dst_maxburst, 0U, dwc->max_burst); 874 + ret = dwc_verify_maxburst(chan); 875 + if (ret) 876 + return ret; 793 877 794 - dw->encode_maxburst(dwc, &dwc->dma_sconfig.src_maxburst); 795 - dw->encode_maxburst(dwc, &dwc->dma_sconfig.dst_maxburst); 878 + ret = dwc_verify_p_buswidth(chan); 879 + if (ret) 880 + return ret; 881 + 882 + ret = dwc_verify_m_buswidth(chan); 883 + if (ret) 884 + return ret; 796 885 797 886 return 0; 798 887 } ··· 1155 1068 bool autocfg = false; 1156 1069 unsigned int dw_params; 1157 1070 unsigned int i; 1158 - int err; 1071 + int ret; 1159 1072 1160 1073 dw->pdata = devm_kzalloc(chip->dev, sizeof(*dw->pdata), GFP_KERNEL); 1161 1074 if (!dw->pdata) ··· 1171 1084 1172 1085 autocfg = dw_params >> DW_PARAMS_EN & 1; 1173 1086 if (!autocfg) { 1174 - err = -EINVAL; 1087 + ret = -EINVAL; 1175 1088 goto err_pdata; 1176 1089 } 1177 1090 ··· 1191 1104 pdata->chan_allocation_order = CHAN_ALLOCATION_ASCENDING; 1192 1105 pdata->chan_priority = CHAN_PRIORITY_ASCENDING; 1193 1106 } else if (chip->pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS) { 1194 - err = -EINVAL; 1107 + ret = -EINVAL; 1195 1108 goto err_pdata; 1196 1109 } else { 1197 1110 memcpy(dw->pdata, chip->pdata, sizeof(*dw->pdata)); ··· 1203 1116 dw->chan = devm_kcalloc(chip->dev, pdata->nr_channels, sizeof(*dw->chan), 1204 1117 GFP_KERNEL); 1205 1118 if (!dw->chan) { 1206 - err = -ENOMEM; 1119 + ret = -ENOMEM; 1207 1120 goto err_pdata; 1208 1121 } 1209 1122 ··· 1221 1134 sizeof(struct dw_desc), 4, 0); 1222 1135 if (!dw->desc_pool) { 1223 1136 dev_err(chip->dev, "No memory for descriptors dma pool\n"); 1224 - err = -ENOMEM; 1137 + ret = -ENOMEM; 1225 1138 goto err_pdata; 1226 1139 } 1227 1140 1228 1141 tasklet_setup(&dw->tasklet, dw_dma_tasklet); 1229 1142 1230 - err = request_irq(chip->irq, dw_dma_interrupt, IRQF_SHARED, 1143 + ret = request_irq(chip->irq, dw_dma_interrupt, IRQF_SHARED, 1231 1144 dw->name, dw); 1232 - if (err) 1145 + if (ret) 1233 1146 goto err_pdata; 1234 1147 1235 1148 INIT_LIST_HEAD(&dw->dma.channels); ··· 1341 1254 */ 1342 1255 dma_set_max_seg_size(dw->dma.dev, dw->chan[0].block_size); 1343 1256 1344 - err = dma_async_device_register(&dw->dma); 1345 - if (err) 1257 + ret = dma_async_device_register(&dw->dma); 1258 + if (ret) 1346 1259 goto err_dma_register; 1347 1260 1348 1261 dev_info(chip->dev, "DesignWare DMA Controller, %d channels\n", ··· 1356 1269 free_irq(chip->irq, dw); 1357 1270 err_pdata: 1358 1271 pm_runtime_put_sync_suspend(chip->dev); 1359 - return err; 1272 + return ret; 1360 1273 } 1361 1274 1362 1275 int do_dma_remove(struct dw_dma_chip *chip)

+26 -18

drivers/dma/dw/dw.c

··· 64 64 return DWC_CTLH_BLOCK_TS(block) << width; 65 65 } 66 66 67 - static u32 dw_dma_prepare_ctllo(struct dw_dma_chan *dwc) 68 - { 69 - struct dma_slave_config *sconfig = &dwc->dma_sconfig; 70 - u8 smsize = (dwc->direction == DMA_DEV_TO_MEM) ? sconfig->src_maxburst : 0; 71 - u8 dmsize = (dwc->direction == DMA_MEM_TO_DEV) ? sconfig->dst_maxburst : 0; 72 - u8 p_master = dwc->dws.p_master; 73 - u8 m_master = dwc->dws.m_master; 74 - u8 dms = (dwc->direction == DMA_MEM_TO_DEV) ? p_master : m_master; 75 - u8 sms = (dwc->direction == DMA_DEV_TO_MEM) ? p_master : m_master; 76 - 77 - return DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN | 78 - DWC_CTLL_DST_MSIZE(dmsize) | DWC_CTLL_SRC_MSIZE(smsize) | 79 - DWC_CTLL_DMS(dms) | DWC_CTLL_SMS(sms); 80 - } 81 - 82 - static void dw_dma_encode_maxburst(struct dw_dma_chan *dwc, u32 *maxburst) 67 + static inline u8 dw_dma_encode_maxburst(u32 maxburst) 83 68 { 84 69 /* 85 70 * Fix burst size according to dw_dmac. We need to convert them as: 86 71 * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3. 87 72 */ 88 - *maxburst = *maxburst > 1 ? fls(*maxburst) - 2 : 0; 73 + return maxburst > 1 ? fls(maxburst) - 2 : 0; 74 + } 75 + 76 + static u32 dw_dma_prepare_ctllo(struct dw_dma_chan *dwc) 77 + { 78 + struct dma_slave_config *sconfig = &dwc->dma_sconfig; 79 + u8 smsize = 0, dmsize = 0; 80 + u8 sms, dms; 81 + 82 + if (dwc->direction == DMA_MEM_TO_DEV) { 83 + sms = dwc->dws.m_master; 84 + dms = dwc->dws.p_master; 85 + dmsize = dw_dma_encode_maxburst(sconfig->dst_maxburst); 86 + } else if (dwc->direction == DMA_DEV_TO_MEM) { 87 + sms = dwc->dws.p_master; 88 + dms = dwc->dws.m_master; 89 + smsize = dw_dma_encode_maxburst(sconfig->src_maxburst); 90 + } else /* DMA_MEM_TO_MEM */ { 91 + sms = dwc->dws.m_master; 92 + dms = dwc->dws.m_master; 93 + } 94 + 95 + return DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN | 96 + DWC_CTLL_DST_MSIZE(dmsize) | DWC_CTLL_SRC_MSIZE(smsize) | 97 + DWC_CTLL_DMS(dms) | DWC_CTLL_SMS(sms); 89 98 } 90 99 91 100 static void dw_dma_set_device_name(struct dw_dma *dw, int id) ··· 125 116 dw->suspend_chan = dw_dma_suspend_chan; 126 117 dw->resume_chan = dw_dma_resume_chan; 127 118 dw->prepare_ctllo = dw_dma_prepare_ctllo; 128 - dw->encode_maxburst = dw_dma_encode_maxburst; 129 119 dw->bytes2block = dw_dma_bytes2block; 130 120 dw->block2bytes = dw_dma_block2bytes; 131 121

+11 -8

drivers/dma/dw/idma32.c

··· 199 199 return IDMA32C_CTLH_BLOCK_TS(block); 200 200 } 201 201 202 + static inline u8 idma32_encode_maxburst(u32 maxburst) 203 + { 204 + return maxburst > 1 ? fls(maxburst) - 1 : 0; 205 + } 206 + 202 207 static u32 idma32_prepare_ctllo(struct dw_dma_chan *dwc) 203 208 { 204 209 struct dma_slave_config *sconfig = &dwc->dma_sconfig; 205 - u8 smsize = (dwc->direction == DMA_DEV_TO_MEM) ? sconfig->src_maxburst : 0; 206 - u8 dmsize = (dwc->direction == DMA_MEM_TO_DEV) ? sconfig->dst_maxburst : 0; 210 + u8 smsize = 0, dmsize = 0; 211 + 212 + if (dwc->direction == DMA_MEM_TO_DEV) 213 + dmsize = idma32_encode_maxburst(sconfig->dst_maxburst); 214 + else if (dwc->direction == DMA_DEV_TO_MEM) 215 + smsize = idma32_encode_maxburst(sconfig->src_maxburst); 207 216 208 217 return DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN | 209 218 DWC_CTLL_DST_MSIZE(dmsize) | DWC_CTLL_SRC_MSIZE(smsize); 210 - } 211 - 212 - static void idma32_encode_maxburst(struct dw_dma_chan *dwc, u32 *maxburst) 213 - { 214 - *maxburst = *maxburst > 1 ? fls(*maxburst) - 1 : 0; 215 219 } 216 220 217 221 static void idma32_set_device_name(struct dw_dma *dw, int id) ··· 274 270 dw->suspend_chan = idma32_suspend_chan; 275 271 dw->resume_chan = idma32_resume_chan; 276 272 dw->prepare_ctllo = idma32_prepare_ctllo; 277 - dw->encode_maxburst = idma32_encode_maxburst; 278 273 dw->bytes2block = idma32_bytes2block; 279 274 dw->block2bytes = idma32_block2bytes; 280 275

+10 -10

drivers/dma/dw/platform.c

··· 29 29 struct dw_dma_chip_pdata *data; 30 30 struct dw_dma_chip *chip; 31 31 struct device *dev = &pdev->dev; 32 - int err; 32 + int ret; 33 33 34 34 match = device_get_match_data(dev); 35 35 if (!match) ··· 51 51 if (IS_ERR(chip->regs)) 52 52 return PTR_ERR(chip->regs); 53 53 54 - err = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 55 - if (err) 56 - return err; 54 + ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 55 + if (ret) 56 + return ret; 57 57 58 58 if (!data->pdata) 59 59 data->pdata = dev_get_platdata(dev); ··· 69 69 chip->clk = devm_clk_get_optional(chip->dev, "hclk"); 70 70 if (IS_ERR(chip->clk)) 71 71 return PTR_ERR(chip->clk); 72 - err = clk_prepare_enable(chip->clk); 73 - if (err) 74 - return err; 72 + ret = clk_prepare_enable(chip->clk); 73 + if (ret) 74 + return ret; 75 75 76 76 pm_runtime_enable(&pdev->dev); 77 77 78 - err = data->probe(chip); 79 - if (err) 78 + ret = data->probe(chip); 79 + if (ret) 80 80 goto err_dw_dma_probe; 81 81 82 82 platform_set_drvdata(pdev, data); ··· 90 90 err_dw_dma_probe: 91 91 pm_runtime_disable(&pdev->dev); 92 92 clk_disable_unprepare(chip->clk); 93 - return err; 93 + return ret; 94 94 } 95 95 96 96 static void dw_remove(struct platform_device *pdev)

-1

drivers/dma/dw/regs.h

··· 327 327 void (*suspend_chan)(struct dw_dma_chan *dwc, bool drain); 328 328 void (*resume_chan)(struct dw_dma_chan *dwc, bool drain); 329 329 u32 (*prepare_ctllo)(struct dw_dma_chan *dwc); 330 - void (*encode_maxburst)(struct dw_dma_chan *dwc, u32 *maxburst); 331 330 u32 (*bytes2block)(struct dw_dma_chan *dwc, size_t bytes, 332 331 unsigned int width, size_t *len); 333 332 size_t (*block2bytes)(struct dw_dma_chan *dwc, u32 block, u32 width);

+1 -1

drivers/dma/stm32/stm32-dma3.c

··· 403 403 swdesc = kzalloc(struct_size(swdesc, lli, count), GFP_NOWAIT); 404 404 if (!swdesc) 405 405 return NULL; 406 + swdesc->lli_size = count; 406 407 407 408 for (i = 0; i < count; i++) { 408 409 swdesc->lli[i].hwdesc = dma_pool_zalloc(chan->lli_pool, GFP_NOWAIT, ··· 411 410 if (!swdesc->lli[i].hwdesc) 412 411 goto err_pool_free; 413 412 } 414 - swdesc->lli_size = count; 415 413 swdesc->ccr = 0; 416 414 417 415 /* Set LL base address */

+3 -3

drivers/dma/ti/omap-dma.c

··· 1186 1186 d->dev_addr = dev_addr; 1187 1187 d->fi = burst; 1188 1188 d->es = es; 1189 + d->sglen = 1; 1189 1190 d->sg[0].addr = buf_addr; 1190 1191 d->sg[0].en = period_len / es_bytes[es]; 1191 1192 d->sg[0].fn = buf_len / period_len; 1192 - d->sglen = 1; 1193 1193 1194 1194 d->ccr = c->ccr; 1195 1195 if (dir == DMA_DEV_TO_MEM) ··· 1258 1258 d->dev_addr = src; 1259 1259 d->fi = 0; 1260 1260 d->es = data_type; 1261 + d->sglen = 1; 1261 1262 d->sg[0].en = len / BIT(data_type); 1262 1263 d->sg[0].fn = 1; 1263 1264 d->sg[0].addr = dest; 1264 - d->sglen = 1; 1265 1265 d->ccr = c->ccr; 1266 1266 d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_POSTINC; 1267 1267 ··· 1309 1309 if (data_type > CSDP_DATA_TYPE_32) 1310 1310 data_type = CSDP_DATA_TYPE_32; 1311 1311 1312 + d->sglen = 1; 1312 1313 sg = &d->sg[0]; 1313 1314 d->dir = DMA_MEM_TO_MEM; 1314 1315 d->dev_addr = xt->src_start; ··· 1317 1316 sg->en = xt->sgl[0].size / BIT(data_type); 1318 1317 sg->fn = xt->numf; 1319 1318 sg->addr = xt->dst_start; 1320 - d->sglen = 1; 1321 1319 d->ccr = c->ccr; 1322 1320 1323 1321 src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);

Configure Feed

Configure Feed