+1

Documentation/devicetree/bindings/dma/renesas,usb-dmac.txt

reviewed

··· 16 16 - "renesas,r8a7794-usb-dmac" (R-Car E2) 17 17 - "renesas,r8a7795-usb-dmac" (R-Car H3) 18 18 - "renesas,r8a7796-usb-dmac" (R-Car M3-W) 19 19 + - "renesas,r8a77961-usb-dmac" (R-Car M3-W+) 19 20 - "renesas,r8a77965-usb-dmac" (R-Car M3-N) 20 21 - "renesas,r8a77990-usb-dmac" (R-Car E3) 21 22 - "renesas,r8a77995-usb-dmac" (R-Car D3)

+63

Documentation/devicetree/bindings/dma/socionext,uniphier-xdmac.yaml

reviewed

··· 1 1 + # SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 2 + %YAML 1.2 3 3 + --- 4 4 + $id: http://devicetree.org/schemas/dma/socionext,uniphier-xdmac.yaml# 5 5 + $schema: http://devicetree.org/meta-schemas/core.yaml# 6 6 + 7 7 + title: Socionext UniPhier external DMA controller 8 8 + 9 9 + description: | 10 10 + This describes the devicetree bindings for an external DMA engine to perform 11 11 + memory-to-memory or peripheral-to-memory data transfer capable of supporting 12 12 + 16 channels, implemented in Socionext UniPhier SoCs. 13 13 + 14 14 + maintainers: 15 15 + - Kunihiko Hayashi <hayashi.kunihiko@socionext.com> 16 16 + 17 17 + allOf: 18 18 + - $ref: "dma-controller.yaml#" 19 19 + 20 20 + properties: 21 21 + compatible: 22 22 + const: socionext,uniphier-xdmac 23 23 + 24 24 + reg: 25 25 + items: 26 26 + - description: XDMAC base register region (offset and length) 27 27 + - description: XDMAC extension register region (offset and length) 28 28 + 29 29 + interrupts: 30 30 + maxItems: 1 31 31 + 32 32 + "#dma-cells": 33 33 + const: 2 34 34 + description: | 35 35 + DMA request from clients consists of 2 cells: 36 36 + 1. Channel index 37 37 + 2. Transfer request factor number, If no transfer factor, use 0. 38 38 + The number is SoC-specific, and this should be specified with 39 39 + relation to the device to use the DMA controller. 40 40 + 41 41 + dma-channels: 42 42 + minimum: 1 43 43 + maximum: 16 44 44 + 45 45 + additionalProperties: false 46 46 + 47 47 + required: 48 48 + - compatible 49 49 + - reg 50 50 + - interrupts 51 51 + - "#dma-cells" 52 52 + 53 53 + examples: 54 54 + - | 55 55 + xdmac: dma-controller@5fc10000 { 56 56 + compatible = "socionext,uniphier-xdmac"; 57 57 + reg = <0x5fc10000 0x1000>, <0x5fc20000 0x800>; 58 58 + interrupts = <0 188 4>; 59 59 + #dma-cells = <2>; 60 60 + dma-channels = <16>; 61 61 + }; 62 62 + 63 63 + ...

+1 -1

Documentation/devicetree/bindings/dma/ti-edma.txt

reviewed

··· 180 180 }; 181 181 182 182 edma1_tptc1: tptc@27b8000 { 183 183 - compatible = "ti, k2g-edma3-tptc", "ti,edma3-tptc"; 183 183 + compatible = "ti,k2g-edma3-tptc", "ti,edma3-tptc"; 184 184 reg = <0x027b8000 0x400>; 185 185 power-domains = <&k2g_pds 0x4f>; 186 186 };

+18 -1

Documentation/devicetree/bindings/dma/ti/k3-udma.yaml

reviewed

··· 45 45 46 46 properties: 47 47 "#dma-cells": 48 48 - const: 1 48 48 + minimum: 1 49 49 + maximum: 2 49 50 description: | 50 51 The cell is the PSI-L thread ID of the remote (to UDMAP) end. 51 52 Valid ranges for thread ID depends on the data movement direction: ··· 55 54 56 55 Please refer to the device documentation for the PSI-L thread map and also 57 56 the PSI-L peripheral chapter for the correct thread ID. 57 57 + 58 58 + When #dma-cells is 2, the second parameter is the channel ATYPE. 58 59 59 60 compatible: 60 61 enum: ··· 133 130 - ti,sci-rm-range-tchan 134 131 - ti,sci-rm-range-rchan 135 132 - ti,sci-rm-range-rflow 133 133 + 134 134 + if: 135 135 + properties: 136 136 + "#dma-cells": 137 137 + const: 2 138 138 + then: 139 139 + properties: 140 140 + ti,udma-atype: 141 141 + description: ATYPE value which should be used by non slave channels 142 142 + allOf: 143 143 + - $ref: /schemas/types.yaml#/definitions/uint32 144 144 + 145 145 + required: 146 146 + - ti,udma-atype 136 147 137 148 examples: 138 149 - |+

+13 -2

drivers/dma/Kconfig

reviewed

··· 616 616 integrated in chips such as the Toshiba TX4927/38/39. 617 617 618 618 config TEGRA20_APB_DMA 619 619 - bool "NVIDIA Tegra20 APB DMA support" 620 620 - depends on ARCH_TEGRA 619 619 + tristate "NVIDIA Tegra20 APB DMA support" 620 620 + depends on ARCH_TEGRA || COMPILE_TEST 621 621 select DMA_ENGINE 622 622 help 623 623 Support for the NVIDIA Tegra20 APB DMA controller driver. The ··· 657 657 Enable support for the MIO DMAC (Media I/O DMA controller) on the 658 658 UniPhier platform. This DMA controller is used as the external 659 659 DMA engine of the SD/eMMC controllers of the LD4, Pro4, sLD8 SoCs. 660 660 + 661 661 + config UNIPHIER_XDMAC 662 662 + tristate "UniPhier XDMAC support" 663 663 + depends on ARCH_UNIPHIER || COMPILE_TEST 664 664 + depends on OF 665 665 + select DMA_ENGINE 666 666 + select DMA_VIRTUAL_CHANNELS 667 667 + help 668 668 + Enable support for the XDMAC (external DMA controller) on the 669 669 + UniPhier platform. This DMA controller can transfer data from 670 670 + memory to memory, memory to peripheral and peripheral to memory. 660 671 661 672 config XGENE_DMA 662 673 tristate "APM X-Gene DMA support"

+1

drivers/dma/Makefile

reviewed

··· 78 78 obj-$(CONFIG_TEGRA210_ADMA) += tegra210-adma.o 79 79 obj-$(CONFIG_TIMB_DMA) += timb_dma.o 80 80 obj-$(CONFIG_UNIPHIER_MDMAC) += uniphier-mdmac.o 81 81 + obj-$(CONFIG_UNIPHIER_XDMAC) += uniphier-xdmac.o 81 82 obj-$(CONFIG_XGENE_DMA) += xgene-dma.o 82 83 obj-$(CONFIG_ZX_DMA) += zx_dma.o 83 84 obj-$(CONFIG_ST_FDMA) += st_fdma.o

+52 -69

drivers/dma/at_hdmac.c

reviewed

··· 146 146 "scanned %u descriptors on freelist\n", i); 147 147 148 148 /* no more descriptor available in initial pool: create one more */ 149 149 - if (!ret) { 150 150 - ret = atc_alloc_descriptor(&atchan->chan_common, GFP_ATOMIC); 151 151 - if (ret) { 152 152 - spin_lock_irqsave(&atchan->lock, flags); 153 153 - atchan->descs_allocated++; 154 154 - spin_unlock_irqrestore(&atchan->lock, flags); 155 155 - } else { 156 156 - dev_err(chan2dev(&atchan->chan_common), 157 157 - "not enough descriptors available\n"); 158 158 - } 159 159 - } 149 149 + if (!ret) 150 150 + ret = atc_alloc_descriptor(&atchan->chan_common, GFP_NOWAIT); 160 151 161 152 return ret; 162 153 } ··· 426 435 * atc_chain_complete - finish work for one transaction chain 427 436 * @atchan: channel we work on 428 437 * @desc: descriptor at the head of the chain we want do complete 429 429 - * 430 430 - * Called with atchan->lock held and bh disabled */ 438 438 + */ 431 439 static void 432 440 atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) 433 441 { 434 442 struct dma_async_tx_descriptor *txd = &desc->txd; 435 443 struct at_dma *atdma = to_at_dma(atchan->chan_common.device); 444 444 + unsigned long flags; 436 445 437 446 dev_vdbg(chan2dev(&atchan->chan_common), 438 447 "descriptor %u complete\n", txd->cookie); 448 448 + 449 449 + spin_lock_irqsave(&atchan->lock, flags); 439 450 440 451 /* mark the descriptor as complete for non cyclic cases only */ 441 452 if (!atc_chan_is_cyclic(atchan)) ··· 455 462 /* move myself to free_list */ 456 463 list_move(&desc->desc_node, &atchan->free_list); 457 464 465 465 + spin_unlock_irqrestore(&atchan->lock, flags); 466 466 + 458 467 dma_descriptor_unmap(txd); 459 468 /* for cyclic transfers, 460 469 * no need to replay callback function while stopping */ 461 461 - if (!atc_chan_is_cyclic(atchan)) { 462 462 - /* 463 463 - * The API requires that no submissions are done from a 464 464 - * callback, so we don't need to drop the lock here 465 465 - */ 470 470 + if (!atc_chan_is_cyclic(atchan)) 466 471 dmaengine_desc_get_callback_invoke(txd, NULL); 467 467 - } 468 472 469 473 dma_run_dependencies(txd); 470 474 } ··· 479 489 { 480 490 struct at_desc *desc, *_desc; 481 491 LIST_HEAD(list); 492 492 + unsigned long flags; 482 493 483 494 dev_vdbg(chan2dev(&atchan->chan_common), "complete all\n"); 495 495 + 496 496 + spin_lock_irqsave(&atchan->lock, flags); 484 497 485 498 /* 486 499 * Submit queued descriptors ASAP, i.e. before we go through ··· 496 503 /* empty queue list by moving descriptors (if any) to active_list */ 497 504 list_splice_init(&atchan->queue, &atchan->active_list); 498 505 506 506 + spin_unlock_irqrestore(&atchan->lock, flags); 507 507 + 499 508 list_for_each_entry_safe(desc, _desc, &list, desc_node) 500 509 atc_chain_complete(atchan, desc); 501 510 } ··· 505 510 /** 506 511 * atc_advance_work - at the end of a transaction, move forward 507 512 * @atchan: channel where the transaction ended 508 508 - * 509 509 - * Called with atchan->lock held and bh disabled 510 513 */ 511 514 static void atc_advance_work(struct at_dma_chan *atchan) 512 515 { 516 516 + unsigned long flags; 517 517 + int ret; 518 518 + 513 519 dev_vdbg(chan2dev(&atchan->chan_common), "advance_work\n"); 514 520 515 515 - if (atc_chan_is_enabled(atchan)) 521 521 + spin_lock_irqsave(&atchan->lock, flags); 522 522 + ret = atc_chan_is_enabled(atchan); 523 523 + spin_unlock_irqrestore(&atchan->lock, flags); 524 524 + if (ret) 516 525 return; 517 526 518 527 if (list_empty(&atchan->active_list) || 519 519 - list_is_singular(&atchan->active_list)) { 520 520 - atc_complete_all(atchan); 521 521 - } else { 522 522 - atc_chain_complete(atchan, atc_first_active(atchan)); 523 523 - /* advance work */ 524 524 - atc_dostart(atchan, atc_first_active(atchan)); 525 525 - } 528 528 + list_is_singular(&atchan->active_list)) 529 529 + return atc_complete_all(atchan); 530 530 + 531 531 + atc_chain_complete(atchan, atc_first_active(atchan)); 532 532 + 533 533 + /* advance work */ 534 534 + spin_lock_irqsave(&atchan->lock, flags); 535 535 + atc_dostart(atchan, atc_first_active(atchan)); 536 536 + spin_unlock_irqrestore(&atchan->lock, flags); 526 537 } 527 538 528 539 529 540 /** 530 541 * atc_handle_error - handle errors reported by DMA controller 531 542 * @atchan: channel where error occurs 532 532 - * 533 533 - * Called with atchan->lock held and bh disabled 534 543 */ 535 544 static void atc_handle_error(struct at_dma_chan *atchan) 536 545 { 537 546 struct at_desc *bad_desc; 538 547 struct at_desc *child; 548 548 + unsigned long flags; 539 549 550 550 + spin_lock_irqsave(&atchan->lock, flags); 540 551 /* 541 552 * The descriptor currently at the head of the active list is 542 553 * broked. Since we don't have any way to report errors, we'll ··· 574 573 list_for_each_entry(child, &bad_desc->tx_list, desc_node) 575 574 atc_dump_lli(atchan, &child->lli); 576 575 576 576 + spin_unlock_irqrestore(&atchan->lock, flags); 577 577 + 577 578 /* Pretend the descriptor completed successfully */ 578 579 atc_chain_complete(atchan, bad_desc); 579 580 } ··· 583 580 /** 584 581 * atc_handle_cyclic - at the end of a period, run callback function 585 582 * @atchan: channel used for cyclic operations 586 586 - * 587 587 - * Called with atchan->lock held and bh disabled 588 583 */ 589 584 static void atc_handle_cyclic(struct at_dma_chan *atchan) 590 585 { ··· 601 600 static void atc_tasklet(unsigned long data) 602 601 { 603 602 struct at_dma_chan *atchan = (struct at_dma_chan *)data; 604 604 - unsigned long flags; 605 603 606 606 - spin_lock_irqsave(&atchan->lock, flags); 607 604 if (test_and_clear_bit(ATC_IS_ERROR, &atchan->status)) 608 608 - atc_handle_error(atchan); 609 609 - else if (atc_chan_is_cyclic(atchan)) 610 610 - atc_handle_cyclic(atchan); 611 611 - else 612 612 - atc_advance_work(atchan); 605 605 + return atc_handle_error(atchan); 613 606 614 614 - spin_unlock_irqrestore(&atchan->lock, flags); 607 607 + if (atc_chan_is_cyclic(atchan)) 608 608 + return atc_handle_cyclic(atchan); 609 609 + 610 610 + atc_advance_work(atchan); 615 611 } 616 612 617 613 static irqreturn_t at_dma_interrupt(int irq, void *dev_id) ··· 938 940 return NULL; 939 941 } 940 942 941 941 - vaddr = dma_pool_alloc(atdma->memset_pool, GFP_ATOMIC, &paddr); 943 943 + vaddr = dma_pool_alloc(atdma->memset_pool, GFP_NOWAIT, &paddr); 942 944 if (!vaddr) { 943 945 dev_err(chan2dev(chan), "%s: couldn't allocate buffer\n", 944 946 __func__); ··· 996 998 return NULL; 997 999 } 998 1000 999 999 - vaddr = dma_pool_alloc(atdma->memset_pool, GFP_ATOMIC, &paddr); 1001 1001 + vaddr = dma_pool_alloc(atdma->memset_pool, GFP_NOWAIT, &paddr); 1000 1002 if (!vaddr) { 1001 1003 dev_err(chan2dev(chan), "%s: couldn't allocate buffer\n", 1002 1004 __func__); ··· 1444 1446 list_splice_init(&atchan->queue, &list); 1445 1447 list_splice_init(&atchan->active_list, &list); 1446 1448 1449 1449 + spin_unlock_irqrestore(&atchan->lock, flags); 1450 1450 + 1447 1451 /* Flush all pending and queued descriptors */ 1448 1452 list_for_each_entry_safe(desc, _desc, &list, desc_node) 1449 1453 atc_chain_complete(atchan, desc); ··· 1453 1453 clear_bit(ATC_IS_PAUSED, &atchan->status); 1454 1454 /* if channel dedicated to cyclic operations, free it */ 1455 1455 clear_bit(ATC_IS_CYCLIC, &atchan->status); 1456 1456 - 1457 1457 - spin_unlock_irqrestore(&atchan->lock, flags); 1458 1456 1459 1457 return 0; 1460 1458 } ··· 1514 1516 static void atc_issue_pending(struct dma_chan *chan) 1515 1517 { 1516 1518 struct at_dma_chan *atchan = to_at_dma_chan(chan); 1517 1517 - unsigned long flags; 1518 1519 1519 1520 dev_vdbg(chan2dev(chan), "issue_pending\n"); 1520 1521 ··· 1521 1524 if (atc_chan_is_cyclic(atchan)) 1522 1525 return; 1523 1526 1524 1524 - spin_lock_irqsave(&atchan->lock, flags); 1525 1527 atc_advance_work(atchan); 1526 1526 - spin_unlock_irqrestore(&atchan->lock, flags); 1527 1528 } 1528 1529 1529 1530 /** 1530 1531 * atc_alloc_chan_resources - allocate resources for DMA channel 1531 1532 * @chan: allocate descriptor resources for this channel 1532 1532 - * @client: current client requesting the channel be ready for requests 1533 1533 * 1534 1534 * return - the number of allocated descriptors 1535 1535 */ ··· 1536 1542 struct at_dma *atdma = to_at_dma(chan->device); 1537 1543 struct at_desc *desc; 1538 1544 struct at_dma_slave *atslave; 1539 1539 - unsigned long flags; 1540 1545 int i; 1541 1546 u32 cfg; 1542 1542 - LIST_HEAD(tmp_list); 1543 1547 1544 1548 dev_vdbg(chan2dev(chan), "alloc_chan_resources\n"); 1545 1549 1546 1550 /* ASSERT: channel is idle */ 1547 1551 if (atc_chan_is_enabled(atchan)) { 1548 1552 dev_dbg(chan2dev(chan), "DMA channel not idle ?\n"); 1553 1553 + return -EIO; 1554 1554 + } 1555 1555 + 1556 1556 + if (!list_empty(&atchan->free_list)) { 1557 1557 + dev_dbg(chan2dev(chan), "can't allocate channel resources (channel not freed from a previous use)\n"); 1549 1558 return -EIO; 1550 1559 } 1551 1560 ··· 1567 1570 cfg = atslave->cfg; 1568 1571 } 1569 1572 1570 1570 - /* have we already been set up? 1571 1571 - * reconfigure channel but no need to reallocate descriptors */ 1572 1572 - if (!list_empty(&atchan->free_list)) 1573 1573 - return atchan->descs_allocated; 1574 1574 - 1575 1573 /* Allocate initial pool of descriptors */ 1576 1574 for (i = 0; i < init_nr_desc_per_channel; i++) { 1577 1575 desc = atc_alloc_descriptor(chan, GFP_KERNEL); ··· 1575 1583 "Only %d initial descriptors\n", i); 1576 1584 break; 1577 1585 } 1578 1578 - list_add_tail(&desc->desc_node, &tmp_list); 1586 1586 + list_add_tail(&desc->desc_node, &atchan->free_list); 1579 1587 } 1580 1588 1581 1581 - spin_lock_irqsave(&atchan->lock, flags); 1582 1582 - atchan->descs_allocated = i; 1583 1583 - list_splice(&tmp_list, &atchan->free_list); 1584 1589 dma_cookie_init(chan); 1585 1585 - spin_unlock_irqrestore(&atchan->lock, flags); 1586 1590 1587 1591 /* channel parameters */ 1588 1592 channel_writel(atchan, CFG, cfg); 1589 1593 1590 1594 dev_dbg(chan2dev(chan), 1591 1591 - "alloc_chan_resources: allocated %d descriptors\n", 1592 1592 - atchan->descs_allocated); 1595 1595 + "alloc_chan_resources: allocated %d descriptors\n", i); 1593 1596 1594 1594 - return atchan->descs_allocated; 1597 1597 + return i; 1595 1598 } 1596 1599 1597 1600 /** ··· 1600 1613 struct at_desc *desc, *_desc; 1601 1614 LIST_HEAD(list); 1602 1615 1603 1603 - dev_dbg(chan2dev(chan), "free_chan_resources: (descs allocated=%u)\n", 1604 1604 - atchan->descs_allocated); 1605 1605 - 1606 1616 /* ASSERT: channel is idle */ 1607 1617 BUG_ON(!list_empty(&atchan->active_list)); 1608 1618 BUG_ON(!list_empty(&atchan->queue)); ··· 1612 1628 dma_pool_free(atdma->dma_desc_pool, desc, desc->txd.phys); 1613 1629 } 1614 1630 list_splice_init(&atchan->free_list, &list); 1615 1615 - atchan->descs_allocated = 0; 1616 1631 atchan->status = 0; 1617 1632 1618 1633 /* ··· 1654 1671 dma_cap_zero(mask); 1655 1672 dma_cap_set(DMA_SLAVE, mask); 1656 1673 1657 1657 - atslave = kzalloc(sizeof(*atslave), GFP_KERNEL); 1674 1674 + atslave = kmalloc(sizeof(*atslave), GFP_KERNEL); 1658 1675 if (!atslave) 1659 1676 return NULL; 1660 1677

-2

drivers/dma/at_hdmac_regs.h

reviewed

··· 243 243 * @active_list: list of descriptors dmaengine is being running on 244 244 * @queue: list of descriptors ready to be submitted to engine 245 245 * @free_list: list of descriptors usable by the channel 246 246 - * @descs_allocated: records the actual size of the descriptor pool 247 246 */ 248 247 struct at_dma_chan { 249 248 struct dma_chan chan_common; ··· 263 264 struct list_head active_list; 264 265 struct list_head queue; 265 266 struct list_head free_list; 266 266 - unsigned int descs_allocated; 267 267 }; 268 268 269 269 #define channel_readl(atchan, name) \

+18 -26

drivers/dma/at_xdmac.c

reviewed

··· 1543 1543 static void at_xdmac_advance_work(struct at_xdmac_chan *atchan) 1544 1544 { 1545 1545 struct at_xdmac_desc *desc; 1546 1546 - unsigned long flags; 1547 1547 - 1548 1548 - spin_lock_irqsave(&atchan->lock, flags); 1549 1546 1550 1547 /* 1551 1548 * If channel is enabled, do nothing, advance_work will be triggered ··· 1556 1559 if (!desc->active_xfer) 1557 1560 at_xdmac_start_xfer(atchan, desc); 1558 1561 } 1559 1559 - 1560 1560 - spin_unlock_irqrestore(&atchan->lock, flags); 1561 1562 } 1562 1563 1563 1564 static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan) ··· 1591 1596 if (atchan->irq_status & AT_XDMAC_CIS_ROIS) 1592 1597 dev_err(chan2dev(&atchan->chan), "request overflow error!!!"); 1593 1598 1594 1594 - spin_lock_bh(&atchan->lock); 1599 1599 + spin_lock_irq(&atchan->lock); 1595 1600 1596 1601 /* Channel must be disabled first as it's not done automatically */ 1597 1602 at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask); ··· 1602 1607 struct at_xdmac_desc, 1603 1608 xfer_node); 1604 1609 1605 1605 - spin_unlock_bh(&atchan->lock); 1610 1610 + spin_unlock_irq(&atchan->lock); 1606 1611 1607 1612 /* Print bad descriptor's details if needed */ 1608 1613 dev_dbg(chan2dev(&atchan->chan), ··· 1635 1640 if (atchan->irq_status & error_mask) 1636 1641 at_xdmac_handle_error(atchan); 1637 1642 1638 1638 - spin_lock(&atchan->lock); 1643 1643 + spin_lock_irq(&atchan->lock); 1639 1644 desc = list_first_entry(&atchan->xfers_list, 1640 1645 struct at_xdmac_desc, 1641 1646 xfer_node); 1642 1647 dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc); 1643 1648 if (!desc->active_xfer) { 1644 1649 dev_err(chan2dev(&atchan->chan), "Xfer not active: exiting"); 1645 1645 - spin_unlock(&atchan->lock); 1650 1650 + spin_unlock_irq(&atchan->lock); 1646 1651 return; 1647 1652 } 1648 1653 1649 1654 txd = &desc->tx_dma_desc; 1650 1655 1651 1656 at_xdmac_remove_xfer(atchan, desc); 1652 1652 - spin_unlock(&atchan->lock); 1657 1657 + spin_unlock_irq(&atchan->lock); 1653 1658 1654 1654 - if (!at_xdmac_chan_is_cyclic(atchan)) { 1655 1655 - dma_cookie_complete(txd); 1656 1656 - if (txd->flags & DMA_PREP_INTERRUPT) 1657 1657 - dmaengine_desc_get_callback_invoke(txd, NULL); 1658 1658 - } 1659 1659 + dma_cookie_complete(txd); 1660 1660 + if (txd->flags & DMA_PREP_INTERRUPT) 1661 1661 + dmaengine_desc_get_callback_invoke(txd, NULL); 1659 1662 1660 1663 dma_run_dependencies(txd); 1661 1664 1665 1665 + spin_lock_irq(&atchan->lock); 1662 1666 at_xdmac_advance_work(atchan); 1667 1667 + spin_unlock_irq(&atchan->lock); 1663 1668 } 1664 1669 } 1665 1670 ··· 1720 1725 static void at_xdmac_issue_pending(struct dma_chan *chan) 1721 1726 { 1722 1727 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); 1728 1728 + unsigned long flags; 1723 1729 1724 1730 dev_dbg(chan2dev(&atchan->chan), "%s\n", __func__); 1725 1731 1726 1726 - if (!at_xdmac_chan_is_cyclic(atchan)) 1732 1732 + if (!at_xdmac_chan_is_cyclic(atchan)) { 1733 1733 + spin_lock_irqsave(&atchan->lock, flags); 1727 1734 at_xdmac_advance_work(atchan); 1735 1735 + spin_unlock_irqrestore(&atchan->lock, flags); 1736 1736 + } 1728 1737 1729 1738 return; 1730 1739 } ··· 1821 1822 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); 1822 1823 struct at_xdmac_desc *desc; 1823 1824 int i; 1824 1824 - unsigned long flags; 1825 1825 - 1826 1826 - spin_lock_irqsave(&atchan->lock, flags); 1827 1825 1828 1826 if (at_xdmac_chan_is_enabled(atchan)) { 1829 1827 dev_err(chan2dev(chan), 1830 1828 "can't allocate channel resources (channel enabled)\n"); 1831 1831 - i = -EIO; 1832 1832 - goto spin_unlock; 1829 1829 + return -EIO; 1833 1830 } 1834 1831 1835 1832 if (!list_empty(&atchan->free_descs_list)) { 1836 1833 dev_err(chan2dev(chan), 1837 1834 "can't allocate channel resources (channel not free from a previous use)\n"); 1838 1838 - i = -EIO; 1839 1839 - goto spin_unlock; 1835 1835 + return -EIO; 1840 1836 } 1841 1837 1842 1838 for (i = 0; i < init_nr_desc_per_channel; i++) { 1843 1843 - desc = at_xdmac_alloc_desc(chan, GFP_ATOMIC); 1839 1839 + desc = at_xdmac_alloc_desc(chan, GFP_KERNEL); 1844 1840 if (!desc) { 1845 1841 dev_warn(chan2dev(chan), 1846 1842 "only %d descriptors have been allocated\n", i); ··· 1848 1854 1849 1855 dev_dbg(chan2dev(chan), "%s: allocated %d descriptors\n", __func__, i); 1850 1856 1851 1851 - spin_unlock: 1852 1852 - spin_unlock_irqrestore(&atchan->lock, flags); 1853 1857 return i; 1854 1858 } 1855 1859

+1 -1

drivers/dma/bcm-sba-raid.c

reviewed

··· 120 120 struct brcm_message msg; 121 121 struct dma_async_tx_descriptor tx; 122 122 /* SBA commands */ 123 123 - struct brcm_sba_command cmds[0]; 123 123 + struct brcm_sba_command cmds[]; 124 124 }; 125 125 126 126 enum sba_version {

+101 -1

drivers/dma/dmaengine.c

reviewed

··· 58 58 static LIST_HEAD(dma_device_list); 59 59 static long dmaengine_ref_count; 60 60 61 61 + /* --- debugfs implementation --- */ 62 62 + #ifdef CONFIG_DEBUG_FS 63 63 + #include <linux/debugfs.h> 64 64 + 65 65 + static struct dentry *rootdir; 66 66 + 67 67 + static void dmaengine_debug_register(struct dma_device *dma_dev) 68 68 + { 69 69 + dma_dev->dbg_dev_root = debugfs_create_dir(dev_name(dma_dev->dev), 70 70 + rootdir); 71 71 + if (IS_ERR(dma_dev->dbg_dev_root)) 72 72 + dma_dev->dbg_dev_root = NULL; 73 73 + } 74 74 + 75 75 + static void dmaengine_debug_unregister(struct dma_device *dma_dev) 76 76 + { 77 77 + debugfs_remove_recursive(dma_dev->dbg_dev_root); 78 78 + dma_dev->dbg_dev_root = NULL; 79 79 + } 80 80 + 81 81 + static void dmaengine_dbg_summary_show(struct seq_file *s, 82 82 + struct dma_device *dma_dev) 83 83 + { 84 84 + struct dma_chan *chan; 85 85 + 86 86 + list_for_each_entry(chan, &dma_dev->channels, device_node) { 87 87 + if (chan->client_count) { 88 88 + seq_printf(s, " %-13s| %s", dma_chan_name(chan), 89 89 + chan->dbg_client_name ?: "in-use"); 90 90 + 91 91 + if (chan->router) 92 92 + seq_printf(s, " (via router: %s)\n", 93 93 + dev_name(chan->router->dev)); 94 94 + else 95 95 + seq_puts(s, "\n"); 96 96 + } 97 97 + } 98 98 + } 99 99 + 100 100 + static int dmaengine_summary_show(struct seq_file *s, void *data) 101 101 + { 102 102 + struct dma_device *dma_dev = NULL; 103 103 + 104 104 + mutex_lock(&dma_list_mutex); 105 105 + list_for_each_entry(dma_dev, &dma_device_list, global_node) { 106 106 + seq_printf(s, "dma%d (%s): number of channels: %u\n", 107 107 + dma_dev->dev_id, dev_name(dma_dev->dev), 108 108 + dma_dev->chancnt); 109 109 + 110 110 + if (dma_dev->dbg_summary_show) 111 111 + dma_dev->dbg_summary_show(s, dma_dev); 112 112 + else 113 113 + dmaengine_dbg_summary_show(s, dma_dev); 114 114 + 115 115 + if (!list_is_last(&dma_dev->global_node, &dma_device_list)) 116 116 + seq_puts(s, "\n"); 117 117 + } 118 118 + mutex_unlock(&dma_list_mutex); 119 119 + 120 120 + return 0; 121 121 + } 122 122 + DEFINE_SHOW_ATTRIBUTE(dmaengine_summary); 123 123 + 124 124 + static void __init dmaengine_debugfs_init(void) 125 125 + { 126 126 + rootdir = debugfs_create_dir("dmaengine", NULL); 127 127 + 128 128 + /* /sys/kernel/debug/dmaengine/summary */ 129 129 + debugfs_create_file("summary", 0444, rootdir, NULL, 130 130 + &dmaengine_summary_fops); 131 131 + } 132 132 + #else 133 133 + static inline void dmaengine_debugfs_init(void) { } 134 134 + static inline int dmaengine_debug_register(struct dma_device *dma_dev) 135 135 + { 136 136 + return 0; 137 137 + } 138 138 + 139 139 + static inline void dmaengine_debug_unregister(struct dma_device *dma_dev) { } 140 140 + #endif /* DEBUG_FS */ 141 141 + 61 142 /* --- sysfs implementation --- */ 62 143 63 144 #define DMA_SLAVE_NAME "slave" ··· 841 760 return chan ? chan : ERR_PTR(-EPROBE_DEFER); 842 761 843 762 found: 763 763 + #ifdef CONFIG_DEBUG_FS 764 764 + chan->dbg_client_name = kasprintf(GFP_KERNEL, "%s:%s", dev_name(dev), 765 765 + name); 766 766 + #endif 767 767 + 844 768 chan->name = kasprintf(GFP_KERNEL, "dma:%s", name); 845 769 if (!chan->name) 846 770 return chan; ··· 923 837 chan->name = NULL; 924 838 chan->slave = NULL; 925 839 } 840 840 + 841 841 + #ifdef CONFIG_DEBUG_FS 842 842 + kfree(chan->dbg_client_name); 843 843 + chan->dbg_client_name = NULL; 844 844 + #endif 926 845 mutex_unlock(&dma_list_mutex); 927 846 } 928 847 EXPORT_SYMBOL_GPL(dma_release_channel); ··· 1287 1196 dma_channel_rebalance(); 1288 1197 mutex_unlock(&dma_list_mutex); 1289 1198 1199 1199 + dmaengine_debug_register(device); 1200 1200 + 1290 1201 return 0; 1291 1202 1292 1203 err_out: ··· 1321 1228 void dma_async_device_unregister(struct dma_device *device) 1322 1229 { 1323 1230 struct dma_chan *chan, *n; 1231 1231 + 1232 1232 + dmaengine_debug_unregister(device); 1324 1233 1325 1234 list_for_each_entry_safe(chan, n, &device->channels, device_node) 1326 1235 __dma_async_device_channel_unregister(device, chan); ··· 1654 1559 1655 1560 if (err) 1656 1561 return err; 1657 1657 - return class_register(&dma_devclass); 1562 1562 + 1563 1563 + err = class_register(&dma_devclass); 1564 1564 + if (!err) 1565 1565 + dmaengine_debugfs_init(); 1566 1566 + 1567 1567 + return err; 1658 1568 } 1659 1569 arch_initcall(dma_bus_init);

+16

drivers/dma/dmaengine.h

reviewed

··· 182 182 struct dma_chan *dma_get_slave_channel(struct dma_chan *chan); 183 183 struct dma_chan *dma_get_any_slave_channel(struct dma_device *device); 184 184 185 185 + #ifdef CONFIG_DEBUG_FS 186 186 + #include <linux/debugfs.h> 187 187 + 188 188 + static inline struct dentry * 189 189 + dmaengine_get_debugfs_root(struct dma_device *dma_dev) { 190 190 + return dma_dev->dbg_dev_root; 191 191 + } 192 192 + #else 193 193 + struct dentry; 194 194 + static inline struct dentry * 195 195 + dmaengine_get_debugfs_root(struct dma_device *dma_dev) 196 196 + { 197 197 + return NULL; 198 198 + } 199 199 + #endif /* CONFIG_DEBUG_FS */ 200 200 + 185 201 #endif

+15

drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c

reviewed

··· 790 790 return 0; 791 791 } 792 792 793 793 + static void dpaa2_qdma_shutdown(struct fsl_mc_device *ls_dev) 794 794 + { 795 795 + struct dpaa2_qdma_priv *priv; 796 796 + struct device *dev; 797 797 + 798 798 + dev = &ls_dev->dev; 799 799 + priv = dev_get_drvdata(dev); 800 800 + 801 801 + dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle); 802 802 + dpaa2_dpdmai_dpio_unbind(priv); 803 803 + dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle); 804 804 + dpdmai_destroy(priv->mc_io, 0, ls_dev->mc_handle); 805 805 + } 806 806 + 793 807 static const struct fsl_mc_device_id dpaa2_qdma_id_table[] = { 794 808 { 795 809 .vendor = FSL_MC_VENDOR_FREESCALE, ··· 819 805 }, 820 806 .probe = dpaa2_qdma_probe, 821 807 .remove = dpaa2_qdma_remove, 808 808 + .shutdown = dpaa2_qdma_shutdown, 822 809 .match_id_table = dpaa2_qdma_id_table 823 810 }; 824 811

+21

drivers/dma/fsl-dpaa2-qdma/dpdmai.c

reviewed

··· 160 160 } 161 161 162 162 /** 163 163 + * dpdmai_destroy() - Destroy the DPDMAI object and release all its resources. 164 164 + * @mc_io: Pointer to MC portal's I/O object 165 165 + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' 166 166 + * @token: Token of DPDMAI object 167 167 + * 168 168 + * Return: '0' on Success; error code otherwise. 169 169 + */ 170 170 + int dpdmai_destroy(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) 171 171 + { 172 172 + struct fsl_mc_command cmd = { 0 }; 173 173 + 174 174 + /* prepare command */ 175 175 + cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_DESTROY, 176 176 + cmd_flags, token); 177 177 + 178 178 + /* send command to mc*/ 179 179 + return mc_send_command(mc_io, &cmd); 180 180 + } 181 181 + EXPORT_SYMBOL_GPL(dpdmai_destroy); 182 182 + 183 183 + /** 163 184 * dpdmai_enable() - Enable the DPDMAI, allow sending and receiving frames. 164 185 * @mc_io: Pointer to MC portal's I/O object 165 186 * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_'

+2

drivers/dma/fsl-dpaa2-qdma/dpdmai.h

reviewed

··· 18 18 #define DPDMAI_CMDID_CLOSE DPDMAI_CMDID_FORMAT(0x800) 19 19 #define DPDMAI_CMDID_OPEN DPDMAI_CMDID_FORMAT(0x80E) 20 20 #define DPDMAI_CMDID_CREATE DPDMAI_CMDID_FORMAT(0x90E) 21 21 + #define DPDMAI_CMDID_DESTROY DPDMAI_CMDID_FORMAT(0x900) 21 22 22 23 #define DPDMAI_CMDID_ENABLE DPDMAI_CMDID_FORMAT(0x002) 23 24 #define DPDMAI_CMDID_DISABLE DPDMAI_CMDID_FORMAT(0x003) ··· 161 160 int dpdmai_open(struct fsl_mc_io *mc_io, u32 cmd_flags, 162 161 int dpdmai_id, u16 *token); 163 162 int dpdmai_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); 163 163 + int dpdmai_destroy(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); 164 164 int dpdmai_create(struct fsl_mc_io *mc_io, u32 cmd_flags, 165 165 const struct dpdmai_cfg *cfg, u16 *token); 166 166 int dpdmai_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token);

+2 -2

drivers/dma/idxd/cdev.c

reviewed

··· 74 74 struct idxd_device *idxd; 75 75 struct idxd_wq *wq; 76 76 struct device *dev; 77 77 - struct idxd_cdev *idxd_cdev; 78 77 79 78 wq = inode_wq(inode); 80 79 idxd = wq->idxd; 81 80 dev = &idxd->pdev->dev; 82 82 - idxd_cdev = &wq->idxd_cdev; 83 81 84 82 dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq)); 85 83 ··· 137 139 138 140 dev_dbg(&pdev->dev, "%s called\n", __func__); 139 141 rc = check_vma(wq, vma, __func__); 142 142 + if (rc < 0) 143 143 + return rc; 140 144 141 145 vma->vm_flags |= VM_DONTCOPY; 142 146 pfn = (base + idxd_get_wq_portal_full_offset(wq->id,

+2 -2

drivers/dma/idxd/device.c

reviewed

··· 584 584 struct idxd_group *group = &idxd->groups[i]; 585 585 586 586 if (group->tc_a == -1) 587 587 - group->grpcfg.flags.tc_a = 0; 587 587 + group->tc_a = group->grpcfg.flags.tc_a = 0; 588 588 else 589 589 group->grpcfg.flags.tc_a = group->tc_a; 590 590 if (group->tc_b == -1) 591 591 - group->grpcfg.flags.tc_b = 1; 591 591 + group->tc_b = group->grpcfg.flags.tc_b = 1; 592 592 else 593 593 group->grpcfg.flags.tc_b = group->tc_b; 594 594 group->grpcfg.flags.use_token_limit = group->use_token_limit;

+12 -7

drivers/dma/idxd/sysfs.c

reviewed

··· 419 419 struct idxd_device *idxd = engine->idxd; 420 420 long id; 421 421 int rc; 422 422 - struct idxd_group *prevg, *group; 422 422 + struct idxd_group *prevg; 423 423 424 424 rc = kstrtol(buf, 10, &id); 425 425 if (rc < 0) ··· 439 439 return count; 440 440 } 441 441 442 442 - group = &idxd->groups[id]; 443 442 prevg = engine->group; 444 443 445 444 if (prevg) ··· 512 513 if (idxd->state == IDXD_DEV_ENABLED) 513 514 return -EPERM; 514 515 515 515 - if (idxd->token_limit == 0) 516 516 - return -EPERM; 517 517 - 518 516 if (val > idxd->max_tokens) 519 517 return -EINVAL; 520 518 ··· 557 561 if (idxd->state == IDXD_DEV_ENABLED) 558 562 return -EPERM; 559 563 560 560 - if (idxd->token_limit == 0) 561 561 - return -EPERM; 562 564 if (val < 4 * group->num_engines || 563 565 val > group->tokens_reserved + idxd->nr_tokens) 564 566 return -EINVAL; ··· 1174 1180 } 1175 1181 static DEVICE_ATTR_RO(op_cap); 1176 1182 1183 1183 + static ssize_t gen_cap_show(struct device *dev, 1184 1184 + struct device_attribute *attr, char *buf) 1185 1185 + { 1186 1186 + struct idxd_device *idxd = 1187 1187 + container_of(dev, struct idxd_device, conf_dev); 1188 1188 + 1189 1189 + return sprintf(buf, "%#llx\n", idxd->hw.gen_cap.bits); 1190 1190 + } 1191 1191 + static DEVICE_ATTR_RO(gen_cap); 1192 1192 + 1177 1193 static ssize_t configurable_show(struct device *dev, 1178 1194 struct device_attribute *attr, char *buf) 1179 1195 { ··· 1321 1317 &dev_attr_max_batch_size.attr, 1322 1318 &dev_attr_max_transfer_size.attr, 1323 1319 &dev_attr_op_cap.attr, 1320 1320 + &dev_attr_gen_cap.attr, 1324 1321 &dev_attr_configurable.attr, 1325 1322 &dev_attr_clients.attr, 1326 1323 &dev_attr_state.attr,

+1 -1

drivers/dma/ioat/dca.c

reviewed

··· 102 102 int max_requesters; 103 103 int requester_count; 104 104 u8 tag_map[IOAT_TAG_MAP_LEN]; 105 105 - struct ioat_dca_slot req_slots[0]; 105 105 + struct ioat_dca_slot req_slots[]; 106 106 }; 107 107 108 108 static int ioat_dca_dev_managed(struct dca_provider *dca,

+1 -1

drivers/dma/ppc4xx/adma.c

reviewed

··· 4303 4303 for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) { 4304 4304 if (ppc440spe_adma_devices[i] == -1) 4305 4305 continue; 4306 4306 - size += snprintf(buf + size, PAGE_SIZE - size, 4306 4306 + size += scnprintf(buf + size, PAGE_SIZE - size, 4307 4307 "PPC440SP(E)-ADMA.%d: %s\n", i, 4308 4308 ppc_adma_errors[ppc440spe_adma_devices[i]]); 4309 4309 }

+1 -1

drivers/dma/sa11x0-dma.c

reviewed

··· 78 78 bool cyclic; 79 79 80 80 unsigned sglen; 81 81 - struct sa11x0_dma_sg sg[0]; 81 81 + struct sa11x0_dma_sg sg[]; 82 82 }; 83 83 84 84 struct sa11x0_dma_phy;

+1 -1

drivers/dma/sh/rcar-dmac.c

reviewed

··· 1219 1219 sg_len = buf_len / period_len; 1220 1220 if (sg_len > RCAR_DMAC_MAX_SG_LEN) { 1221 1221 dev_err(chan->device->dev, 1222 1222 - "chan%u: sg length %d exceds limit %d", 1222 1222 + "chan%u: sg length %d exceeds limit %d", 1223 1223 rchan->index, sg_len, RCAR_DMAC_MAX_SG_LEN); 1224 1224 return NULL; 1225 1225 }

+1 -1

drivers/dma/sh/shdma-base.c

reviewed

··· 709 709 BUG_ON(!schan->desc_num); 710 710 711 711 if (sg_len > SHDMA_MAX_SG_LEN) { 712 712 - dev_err(schan->dev, "sg length %d exceds limit %d", 712 712 + dev_err(schan->dev, "sg length %d exceeds limit %d", 713 713 sg_len, SHDMA_MAX_SG_LEN); 714 714 return NULL; 715 715 }

+25 -1

drivers/dma/sprd-dma.c

reviewed

··· 212 212 struct clk *ashb_clk; 213 213 int irq; 214 214 u32 total_chns; 215 215 - struct sprd_dma_chn channels[0]; 215 215 + struct sprd_dma_chn channels[]; 216 216 }; 217 217 218 218 static void sprd_dma_free_desc(struct virt_dma_desc *vd); ··· 486 486 return 0; 487 487 } 488 488 489 489 + static void sprd_dma_set_pending(struct sprd_dma_chn *schan, bool enable) 490 490 + { 491 491 + struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan); 492 492 + u32 reg, val, req_id; 493 493 + 494 494 + if (schan->dev_id == SPRD_DMA_SOFTWARE_UID) 495 495 + return; 496 496 + 497 497 + /* The DMA request id always starts from 0. */ 498 498 + req_id = schan->dev_id - 1; 499 499 + 500 500 + if (req_id < 32) { 501 501 + reg = SPRD_DMA_GLB_REQ_PEND0_EN; 502 502 + val = BIT(req_id); 503 503 + } else { 504 504 + reg = SPRD_DMA_GLB_REQ_PEND1_EN; 505 505 + val = BIT(req_id - 32); 506 506 + } 507 507 + 508 508 + sprd_dma_glb_update(sdev, reg, val, enable ? val : 0); 509 509 + } 510 510 + 489 511 static void sprd_dma_set_chn_config(struct sprd_dma_chn *schan, 490 512 struct sprd_dma_desc *sdesc) 491 513 { ··· 554 532 */ 555 533 sprd_dma_set_chn_config(schan, schan->cur_desc); 556 534 sprd_dma_set_uid(schan); 535 535 + sprd_dma_set_pending(schan, true); 557 536 sprd_dma_enable_chn(schan); 558 537 559 538 if (schan->dev_id == SPRD_DMA_SOFTWARE_UID && ··· 566 543 static void sprd_dma_stop(struct sprd_dma_chn *schan) 567 544 { 568 545 sprd_dma_stop_and_disable(schan); 546 546 + sprd_dma_set_pending(schan, false); 569 547 sprd_dma_unset_uid(schan); 570 548 sprd_dma_clear_int(schan); 571 549 schan->cur_desc = NULL;

+66 -28

drivers/dma/stm32-dma.c

reviewed

··· 15 15 #include <linux/dma-mapping.h> 16 16 #include <linux/err.h> 17 17 #include <linux/init.h> 18 18 + #include <linux/iopoll.h> 18 19 #include <linux/jiffies.h> 19 20 #include <linux/list.h> 20 21 #include <linux/module.h> ··· 208 207 struct dma_device ddev; 209 208 void __iomem *base; 210 209 struct clk *clk; 211 211 - struct reset_control *rst; 212 210 bool mem2mem; 213 211 struct stm32_dma_chan chan[STM32_DMA_MAX_CHANNELS]; 214 212 }; ··· 422 422 static int stm32_dma_disable_chan(struct stm32_dma_chan *chan) 423 423 { 424 424 struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); 425 425 - unsigned long timeout = jiffies + msecs_to_jiffies(5000); 426 426 - u32 dma_scr, id; 425 425 + u32 dma_scr, id, reg; 427 426 428 427 id = chan->id; 429 429 - dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id)); 428 428 + reg = STM32_DMA_SCR(id); 429 429 + dma_scr = stm32_dma_read(dmadev, reg); 430 430 431 431 if (dma_scr & STM32_DMA_SCR_EN) { 432 432 dma_scr &= ~STM32_DMA_SCR_EN; 433 433 - stm32_dma_write(dmadev, STM32_DMA_SCR(id), dma_scr); 433 433 + stm32_dma_write(dmadev, reg, dma_scr); 434 434 435 435 - do { 436 436 - dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id)); 437 437 - dma_scr &= STM32_DMA_SCR_EN; 438 438 - if (!dma_scr) 439 439 - break; 440 440 - 441 441 - if (time_after_eq(jiffies, timeout)) { 442 442 - dev_err(chan2dev(chan), "%s: timeout!\n", 443 443 - __func__); 444 444 - return -EBUSY; 445 445 - } 446 446 - cond_resched(); 447 447 - } while (1); 435 435 + return readl_relaxed_poll_timeout_atomic(dmadev->base + reg, 436 436 + dma_scr, !(dma_scr & STM32_DMA_SCR_EN), 437 437 + 10, 1000000); 448 438 } 449 439 450 440 return 0; ··· 478 488 479 489 spin_lock_irqsave(&chan->vchan.lock, flags); 480 490 481 481 - if (chan->busy) { 482 482 - stm32_dma_stop(chan); 491 491 + if (chan->desc) { 492 492 + vchan_terminate_vdesc(&chan->desc->vdesc); 493 493 + if (chan->busy) 494 494 + stm32_dma_stop(chan); 483 495 chan->desc = NULL; 484 496 } 485 497 ··· 537 545 if (!vdesc) 538 546 return; 539 547 548 548 + list_del(&vdesc->node); 549 549 + 540 550 chan->desc = to_stm32_dma_desc(vdesc); 541 551 chan->next_sg = 0; 542 552 } ··· 549 555 sg_req = &chan->desc->sg_req[chan->next_sg]; 550 556 reg = &sg_req->chan_reg; 551 557 558 558 + reg->dma_scr &= ~STM32_DMA_SCR_EN; 552 559 stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), reg->dma_scr); 553 560 stm32_dma_write(dmadev, STM32_DMA_SPAR(chan->id), reg->dma_spar); 554 561 stm32_dma_write(dmadev, STM32_DMA_SM0AR(chan->id), reg->dma_sm0ar); ··· 617 622 } else { 618 623 chan->busy = false; 619 624 if (chan->next_sg == chan->desc->num_sgs) { 620 620 - list_del(&chan->desc->vdesc.node); 621 625 vchan_cookie_complete(&chan->desc->vdesc); 622 626 chan->desc = NULL; 623 627 } ··· 1269 1275 struct dma_device *dd; 1270 1276 const struct of_device_id *match; 1271 1277 struct resource *res; 1278 1278 + struct reset_control *rst; 1272 1279 int i, ret; 1273 1280 1274 1281 match = of_match_device(stm32_dma_of_match, &pdev->dev); ··· 1291 1296 1292 1297 dmadev->clk = devm_clk_get(&pdev->dev, NULL); 1293 1298 if (IS_ERR(dmadev->clk)) { 1294 1294 - dev_err(&pdev->dev, "Error: Missing controller clock\n"); 1295 1295 - return PTR_ERR(dmadev->clk); 1299 1299 + ret = PTR_ERR(dmadev->clk); 1300 1300 + if (ret != -EPROBE_DEFER) 1301 1301 + dev_err(&pdev->dev, "Can't get clock\n"); 1302 1302 + return ret; 1296 1303 } 1297 1304 1298 1305 ret = clk_prepare_enable(dmadev->clk); ··· 1306 1309 dmadev->mem2mem = of_property_read_bool(pdev->dev.of_node, 1307 1310 "st,mem2mem"); 1308 1311 1309 1309 - dmadev->rst = devm_reset_control_get(&pdev->dev, NULL); 1310 1310 - if (!IS_ERR(dmadev->rst)) { 1311 1311 - reset_control_assert(dmadev->rst); 1312 1312 + rst = devm_reset_control_get(&pdev->dev, NULL); 1313 1313 + if (IS_ERR(rst)) { 1314 1314 + ret = PTR_ERR(rst); 1315 1315 + if (ret == -EPROBE_DEFER) 1316 1316 + goto clk_free; 1317 1317 + } else { 1318 1318 + reset_control_assert(rst); 1312 1319 udelay(2); 1313 1313 - reset_control_deassert(dmadev->rst); 1320 1320 + reset_control_deassert(rst); 1314 1321 } 1322 1322 + 1323 1323 + dma_set_max_seg_size(&pdev->dev, STM32_DMA_ALIGNED_MAX_DATA_ITEMS); 1315 1324 1316 1325 dma_cap_set(DMA_SLAVE, dd->cap_mask); 1317 1326 dma_cap_set(DMA_PRIVATE, dd->cap_mask); ··· 1339 1336 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 1340 1337 dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 1341 1338 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 1339 1339 + dd->copy_align = DMAENGINE_ALIGN_32_BYTES; 1342 1340 dd->max_burst = STM32_DMA_MAX_BURST; 1341 1341 + dd->descriptor_reuse = true; 1343 1342 dd->dev = &pdev->dev; 1344 1343 INIT_LIST_HEAD(&dd->channels); 1345 1344 ··· 1432 1427 } 1433 1428 #endif 1434 1429 1430 1430 + #ifdef CONFIG_PM_SLEEP 1431 1431 + static int stm32_dma_suspend(struct device *dev) 1432 1432 + { 1433 1433 + struct stm32_dma_device *dmadev = dev_get_drvdata(dev); 1434 1434 + int id, ret, scr; 1435 1435 + 1436 1436 + ret = pm_runtime_get_sync(dev); 1437 1437 + if (ret < 0) 1438 1438 + return ret; 1439 1439 + 1440 1440 + for (id = 0; id < STM32_DMA_MAX_CHANNELS; id++) { 1441 1441 + scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id)); 1442 1442 + if (scr & STM32_DMA_SCR_EN) { 1443 1443 + dev_warn(dev, "Suspend is prevented by Chan %i\n", id); 1444 1444 + return -EBUSY; 1445 1445 + } 1446 1446 + } 1447 1447 + 1448 1448 + pm_runtime_put_sync(dev); 1449 1449 + 1450 1450 + pm_runtime_force_suspend(dev); 1451 1451 + 1452 1452 + return 0; 1453 1453 + } 1454 1454 + 1455 1455 + static int stm32_dma_resume(struct device *dev) 1456 1456 + { 1457 1457 + return pm_runtime_force_resume(dev); 1458 1458 + } 1459 1459 + #endif 1460 1460 + 1435 1461 static const struct dev_pm_ops stm32_dma_pm_ops = { 1462 1462 + SET_SYSTEM_SLEEP_PM_OPS(stm32_dma_suspend, stm32_dma_resume) 1436 1463 SET_RUNTIME_PM_OPS(stm32_dma_runtime_suspend, 1437 1464 stm32_dma_runtime_resume, NULL) 1438 1465 }; ··· 1475 1438 .of_match_table = stm32_dma_of_match, 1476 1439 .pm = &stm32_dma_pm_ops, 1477 1440 }, 1441 1441 + .probe = stm32_dma_probe, 1478 1442 }; 1479 1443 1480 1444 static int __init stm32_dma_init(void) 1481 1445 { 1482 1482 - return platform_driver_probe(&stm32_dma_driver, stm32_dma_probe); 1446 1446 + return platform_driver_register(&stm32_dma_driver); 1483 1447 } 1484 1448 subsys_initcall(stm32_dma_init);

+77 -16

drivers/dma/stm32-dmamux.c

reviewed

··· 35 35 struct stm32_dmamux_data { 36 36 struct dma_router dmarouter; 37 37 struct clk *clk; 38 38 - struct reset_control *rst; 39 38 void __iomem *iomem; 40 39 u32 dma_requests; /* Number of DMA requests connected to DMAMUX */ 41 40 u32 dmamux_requests; /* Number of DMA requests routed toward DMAs */ 42 41 spinlock_t lock; /* Protects register access */ 43 42 unsigned long *dma_inuse; /* Used DMA channel */ 43 43 + u32 ccr[STM32_DMAMUX_MAX_DMA_REQUESTS]; /* Used to backup CCR register 44 44 + * in suspend 45 45 + */ 44 46 u32 dma_reqs[]; /* Number of DMA Request per DMA masters. 45 47 * [0] holds number of DMA Masters. 46 48 * To be kept at very end end of this structure ··· 181 179 struct stm32_dmamux_data *stm32_dmamux; 182 180 struct resource *res; 183 181 void __iomem *iomem; 182 182 + struct reset_control *rst; 184 183 int i, count, ret; 185 184 u32 dma_req; 186 185 ··· 254 251 stm32_dmamux->clk = devm_clk_get(&pdev->dev, NULL); 255 252 if (IS_ERR(stm32_dmamux->clk)) { 256 253 ret = PTR_ERR(stm32_dmamux->clk); 257 257 - if (ret == -EPROBE_DEFER) 258 258 - dev_info(&pdev->dev, "Missing controller clock\n"); 254 254 + if (ret != -EPROBE_DEFER) 255 255 + dev_err(&pdev->dev, "Missing clock controller\n"); 259 256 return ret; 260 257 } 261 258 262 262 - stm32_dmamux->rst = devm_reset_control_get(&pdev->dev, NULL); 263 263 - if (!IS_ERR(stm32_dmamux->rst)) { 264 264 - reset_control_assert(stm32_dmamux->rst); 259 259 + ret = clk_prepare_enable(stm32_dmamux->clk); 260 260 + if (ret < 0) { 261 261 + dev_err(&pdev->dev, "clk_prep_enable error: %d\n", ret); 262 262 + return ret; 263 263 + } 264 264 + 265 265 + rst = devm_reset_control_get(&pdev->dev, NULL); 266 266 + if (IS_ERR(rst)) { 267 267 + ret = PTR_ERR(rst); 268 268 + if (ret == -EPROBE_DEFER) 269 269 + goto err_clk; 270 270 + } else { 271 271 + reset_control_assert(rst); 265 272 udelay(2); 266 266 - reset_control_deassert(stm32_dmamux->rst); 273 273 + reset_control_deassert(rst); 267 274 } 268 275 269 276 stm32_dmamux->iomem = iomem; ··· 284 271 pm_runtime_set_active(&pdev->dev); 285 272 pm_runtime_enable(&pdev->dev); 286 273 287 287 - if (!IS_ERR(stm32_dmamux->clk)) { 288 288 - ret = clk_prepare_enable(stm32_dmamux->clk); 289 289 - if (ret < 0) { 290 290 - dev_err(&pdev->dev, "clk_prep_enable error: %d\n", ret); 291 291 - return ret; 292 292 - } 293 293 - } 294 294 - 295 274 pm_runtime_get_noresume(&pdev->dev); 296 275 297 276 /* Reset the dmamux */ ··· 292 287 293 288 pm_runtime_put(&pdev->dev); 294 289 295 295 - return of_dma_router_register(node, stm32_dmamux_route_allocate, 290 290 + ret = of_dma_router_register(node, stm32_dmamux_route_allocate, 296 291 &stm32_dmamux->dmarouter); 292 292 + if (ret) 293 293 + goto err_clk; 294 294 + 295 295 + return 0; 296 296 + 297 297 + err_clk: 298 298 + clk_disable_unprepare(stm32_dmamux->clk); 299 299 + 300 300 + return ret; 297 301 } 298 302 299 303 #ifdef CONFIG_PM ··· 332 318 } 333 319 #endif 334 320 321 321 + #ifdef CONFIG_PM_SLEEP 322 322 + static int stm32_dmamux_suspend(struct device *dev) 323 323 + { 324 324 + struct platform_device *pdev = to_platform_device(dev); 325 325 + struct stm32_dmamux_data *stm32_dmamux = platform_get_drvdata(pdev); 326 326 + int i, ret; 327 327 + 328 328 + ret = pm_runtime_get_sync(dev); 329 329 + if (ret < 0) 330 330 + return ret; 331 331 + 332 332 + for (i = 0; i < stm32_dmamux->dma_requests; i++) 333 333 + stm32_dmamux->ccr[i] = stm32_dmamux_read(stm32_dmamux->iomem, 334 334 + STM32_DMAMUX_CCR(i)); 335 335 + 336 336 + pm_runtime_put_sync(dev); 337 337 + 338 338 + pm_runtime_force_suspend(dev); 339 339 + 340 340 + return 0; 341 341 + } 342 342 + 343 343 + static int stm32_dmamux_resume(struct device *dev) 344 344 + { 345 345 + struct platform_device *pdev = to_platform_device(dev); 346 346 + struct stm32_dmamux_data *stm32_dmamux = platform_get_drvdata(pdev); 347 347 + int i, ret; 348 348 + 349 349 + ret = pm_runtime_force_resume(dev); 350 350 + if (ret < 0) 351 351 + return ret; 352 352 + 353 353 + ret = pm_runtime_get_sync(dev); 354 354 + if (ret < 0) 355 355 + return ret; 356 356 + 357 357 + for (i = 0; i < stm32_dmamux->dma_requests; i++) 358 358 + stm32_dmamux_write(stm32_dmamux->iomem, STM32_DMAMUX_CCR(i), 359 359 + stm32_dmamux->ccr[i]); 360 360 + 361 361 + pm_runtime_put_sync(dev); 362 362 + 363 363 + return 0; 364 364 + } 365 365 + #endif 366 366 + 335 367 static const struct dev_pm_ops stm32_dmamux_pm_ops = { 368 368 + SET_SYSTEM_SLEEP_PM_OPS(stm32_dmamux_suspend, stm32_dmamux_resume) 336 369 SET_RUNTIME_PM_OPS(stm32_dmamux_runtime_suspend, 337 370 stm32_dmamux_runtime_resume, NULL) 338 371 };

+62 -16

drivers/dma/stm32-mdma.c

reviewed

··· 273 273 void __iomem *base; 274 274 struct clk *clk; 275 275 int irq; 276 276 - struct reset_control *rst; 277 276 u32 nr_channels; 278 277 u32 nr_requests; 279 278 u32 nr_ahb_addr_masks; ··· 1126 1127 return; 1127 1128 } 1128 1129 1130 1130 + list_del(&vdesc->node); 1131 1131 + 1129 1132 chan->desc = to_stm32_mdma_desc(vdesc); 1130 1133 hwdesc = chan->desc->node[0].hwdesc; 1131 1134 chan->curr_hwdesc = 0; ··· 1243 1242 LIST_HEAD(head); 1244 1243 1245 1244 spin_lock_irqsave(&chan->vchan.lock, flags); 1246 1246 - if (chan->busy) { 1247 1247 - stm32_mdma_stop(chan); 1245 1245 + if (chan->desc) { 1246 1246 + vchan_terminate_vdesc(&chan->desc->vdesc); 1247 1247 + if (chan->busy) 1248 1248 + stm32_mdma_stop(chan); 1248 1249 chan->desc = NULL; 1249 1250 } 1250 1251 vchan_get_all_descriptors(&chan->vchan, &head); ··· 1334 1331 1335 1332 static void stm32_mdma_xfer_end(struct stm32_mdma_chan *chan) 1336 1333 { 1337 1337 - list_del(&chan->desc->vdesc.node); 1338 1334 vchan_cookie_complete(&chan->desc->vdesc); 1339 1335 chan->desc = NULL; 1340 1336 chan->busy = false; ··· 1534 1532 struct dma_device *dd; 1535 1533 struct device_node *of_node; 1536 1534 struct resource *res; 1535 1535 + struct reset_control *rst; 1537 1536 u32 nr_channels, nr_requests; 1538 1537 int i, count, ret; 1539 1538 ··· 1582 1579 dmadev->clk = devm_clk_get(&pdev->dev, NULL); 1583 1580 if (IS_ERR(dmadev->clk)) { 1584 1581 ret = PTR_ERR(dmadev->clk); 1585 1585 - if (ret == -EPROBE_DEFER) 1586 1586 - dev_info(&pdev->dev, "Missing controller clock\n"); 1582 1582 + if (ret != -EPROBE_DEFER) 1583 1583 + dev_err(&pdev->dev, "Missing clock controller\n"); 1587 1584 return ret; 1588 1585 } 1589 1586 ··· 1593 1590 return ret; 1594 1591 } 1595 1592 1596 1596 - dmadev->rst = devm_reset_control_get(&pdev->dev, NULL); 1597 1597 - if (!IS_ERR(dmadev->rst)) { 1598 1598 - reset_control_assert(dmadev->rst); 1593 1593 + rst = devm_reset_control_get(&pdev->dev, NULL); 1594 1594 + if (IS_ERR(rst)) { 1595 1595 + ret = PTR_ERR(rst); 1596 1596 + if (ret == -EPROBE_DEFER) 1597 1597 + goto err_clk; 1598 1598 + } else { 1599 1599 + reset_control_assert(rst); 1599 1600 udelay(2); 1600 1600 - reset_control_deassert(dmadev->rst); 1601 1601 + reset_control_deassert(rst); 1601 1602 } 1602 1603 1603 1604 dd = &dmadev->ddev; ··· 1621 1614 dd->device_resume = stm32_mdma_resume; 1622 1615 dd->device_terminate_all = stm32_mdma_terminate_all; 1623 1616 dd->device_synchronize = stm32_mdma_synchronize; 1617 1617 + dd->descriptor_reuse = true; 1618 1618 + 1624 1619 dd->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | 1625 1620 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | 1626 1621 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | ··· 1646 1637 } 1647 1638 1648 1639 dmadev->irq = platform_get_irq(pdev, 0); 1649 1649 - if (dmadev->irq < 0) 1650 1650 - return dmadev->irq; 1640 1640 + if (dmadev->irq < 0) { 1641 1641 + ret = dmadev->irq; 1642 1642 + goto err_clk; 1643 1643 + } 1651 1644 1652 1645 ret = devm_request_irq(&pdev->dev, dmadev->irq, stm32_mdma_irq_handler, 1653 1646 0, dev_name(&pdev->dev), dmadev); 1654 1647 if (ret) { 1655 1648 dev_err(&pdev->dev, "failed to request IRQ\n"); 1656 1656 - return ret; 1649 1649 + goto err_clk; 1657 1650 } 1658 1651 1659 1652 ret = dmaenginem_async_device_register(dd); 1660 1653 if (ret) 1661 1661 - return ret; 1654 1654 + goto err_clk; 1662 1655 1663 1656 ret = of_dma_controller_register(of_node, stm32_mdma_of_xlate, dmadev); 1664 1657 if (ret < 0) { 1665 1658 dev_err(&pdev->dev, 1666 1659 "STM32 MDMA DMA OF registration failed %d\n", ret); 1667 1667 - goto err_unregister; 1660 1660 + goto err_clk; 1668 1661 } 1669 1662 1670 1663 platform_set_drvdata(pdev, dmadev); ··· 1679 1668 1680 1669 return 0; 1681 1670 1682 1682 - err_unregister: 1671 1671 + err_clk: 1672 1672 + clk_disable_unprepare(dmadev->clk); 1673 1673 + 1683 1674 return ret; 1684 1675 } 1685 1676 ··· 1710 1697 } 1711 1698 #endif 1712 1699 1700 1700 + #ifdef CONFIG_PM_SLEEP 1701 1701 + static int stm32_mdma_pm_suspend(struct device *dev) 1702 1702 + { 1703 1703 + struct stm32_mdma_device *dmadev = dev_get_drvdata(dev); 1704 1704 + u32 ccr, id; 1705 1705 + int ret; 1706 1706 + 1707 1707 + ret = pm_runtime_get_sync(dev); 1708 1708 + if (ret < 0) 1709 1709 + return ret; 1710 1710 + 1711 1711 + for (id = 0; id < dmadev->nr_channels; id++) { 1712 1712 + ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(id)); 1713 1713 + if (ccr & STM32_MDMA_CCR_EN) { 1714 1714 + dev_warn(dev, "Suspend is prevented by Chan %i\n", id); 1715 1715 + return -EBUSY; 1716 1716 + } 1717 1717 + } 1718 1718 + 1719 1719 + pm_runtime_put_sync(dev); 1720 1720 + 1721 1721 + pm_runtime_force_suspend(dev); 1722 1722 + 1723 1723 + return 0; 1724 1724 + } 1725 1725 + 1726 1726 + static int stm32_mdma_pm_resume(struct device *dev) 1727 1727 + { 1728 1728 + return pm_runtime_force_resume(dev); 1729 1729 + } 1730 1730 + #endif 1731 1731 + 1713 1732 static const struct dev_pm_ops stm32_mdma_pm_ops = { 1733 1733 + SET_SYSTEM_SLEEP_PM_OPS(stm32_mdma_pm_suspend, stm32_mdma_pm_resume) 1714 1734 SET_RUNTIME_PM_OPS(stm32_mdma_runtime_suspend, 1715 1735 stm32_mdma_runtime_resume, NULL) 1716 1736 };

+3 -1

drivers/dma/sun4i-dma.c

reviewed

··· 697 697 dest = sconfig->dst_addr; 698 698 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) | 699 699 SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) | 700 700 - SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type); 700 700 + SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) | 701 701 + SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode); 701 702 } else { 702 703 src = sconfig->src_addr; 703 704 dest = buf; 704 705 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) | 706 706 + SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) | 705 707 SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) | 706 708 SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode); 707 709 }

+284 -266

drivers/dma/tegra20-apb-dma.c

reviewed

··· 24 24 #include <linux/pm_runtime.h> 25 25 #include <linux/reset.h> 26 26 #include <linux/slab.h> 27 27 + #include <linux/wait.h> 27 28 28 29 #include "dmaengine.h" 29 30 ··· 60 59 #define TEGRA_APBDMA_STATUS_COUNT_MASK 0xFFFC 61 60 62 61 #define TEGRA_APBDMA_CHAN_CSRE 0x00C 63 63 - #define TEGRA_APBDMA_CHAN_CSRE_PAUSE (1 << 31) 62 62 + #define TEGRA_APBDMA_CHAN_CSRE_PAUSE BIT(31) 64 63 65 64 /* AHB memory address */ 66 65 #define TEGRA_APBDMA_CHAN_AHBPTR 0x010 ··· 121 120 * @support_separate_wcount_reg: Support separate word count register. 122 121 */ 123 122 struct tegra_dma_chip_data { 124 124 - int nr_channels; 125 125 - int channel_reg_size; 126 126 - int max_dma_count; 123 123 + unsigned int nr_channels; 124 124 + unsigned int channel_reg_size; 125 125 + unsigned int max_dma_count; 127 126 bool support_channel_pause; 128 127 bool support_separate_wcount_reg; 129 128 }; 130 129 131 130 /* DMA channel registers */ 132 131 struct tegra_dma_channel_regs { 133 133 - unsigned long csr; 134 134 - unsigned long ahb_ptr; 135 135 - unsigned long apb_ptr; 136 136 - unsigned long ahb_seq; 137 137 - unsigned long apb_seq; 138 138 - unsigned long wcount; 132 132 + u32 csr; 133 133 + u32 ahb_ptr; 134 134 + u32 apb_ptr; 135 135 + u32 ahb_seq; 136 136 + u32 apb_seq; 137 137 + u32 wcount; 139 138 }; 140 139 141 140 /* ··· 169 168 struct list_head node; 170 169 struct list_head tx_list; 171 170 struct list_head cb_node; 172 172 - int cb_count; 171 171 + unsigned int cb_count; 173 172 }; 174 173 175 174 struct tegra_dma_channel; ··· 182 181 struct dma_chan dma_chan; 183 182 char name[12]; 184 183 bool config_init; 185 185 - int id; 186 186 - int irq; 184 184 + unsigned int id; 187 185 void __iomem *chan_addr; 188 186 spinlock_t lock; 189 187 bool busy; ··· 202 202 /* Channel-slave specific configuration */ 203 203 unsigned int slave_id; 204 204 struct dma_slave_config dma_sconfig; 205 205 - struct tegra_dma_channel_regs channel_reg; 205 205 + struct tegra_dma_channel_regs channel_reg; 206 206 + 207 207 + struct wait_queue_head wq; 206 208 }; 207 209 208 210 /* tegra_dma: Tegra DMA specific information */ ··· 224 222 */ 225 223 u32 global_pause_count; 226 224 227 227 - /* Some register need to be cache before suspend */ 228 228 - u32 reg_gen; 229 229 - 230 225 /* Last member of the structure */ 231 226 struct tegra_dma_channel channels[0]; 232 227 }; ··· 239 240 } 240 241 241 242 static inline void tdc_write(struct tegra_dma_channel *tdc, 242 242 - u32 reg, u32 val) 243 243 + u32 reg, u32 val) 243 244 { 244 245 writel(val, tdc->chan_addr + reg); 245 246 } ··· 254 255 return container_of(dc, struct tegra_dma_channel, dma_chan); 255 256 } 256 257 257 257 - static inline struct tegra_dma_desc *txd_to_tegra_dma_desc( 258 258 - struct dma_async_tx_descriptor *td) 258 258 + static inline struct tegra_dma_desc * 259 259 + txd_to_tegra_dma_desc(struct dma_async_tx_descriptor *td) 259 260 { 260 261 return container_of(td, struct tegra_dma_desc, txd); 261 262 } ··· 266 267 } 267 268 268 269 static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *tx); 269 269 - static int tegra_dma_runtime_suspend(struct device *dev); 270 270 - static int tegra_dma_runtime_resume(struct device *dev); 271 270 272 271 /* Get DMA desc from free list, if not there then allocate it. */ 273 273 - static struct tegra_dma_desc *tegra_dma_desc_get( 274 274 - struct tegra_dma_channel *tdc) 272 272 + static struct tegra_dma_desc *tegra_dma_desc_get(struct tegra_dma_channel *tdc) 275 273 { 276 274 struct tegra_dma_desc *dma_desc; 277 275 unsigned long flags; ··· 295 299 dma_async_tx_descriptor_init(&dma_desc->txd, &tdc->dma_chan); 296 300 dma_desc->txd.tx_submit = tegra_dma_tx_submit; 297 301 dma_desc->txd.flags = 0; 302 302 + 298 303 return dma_desc; 299 304 } 300 305 301 306 static void tegra_dma_desc_put(struct tegra_dma_channel *tdc, 302 302 - struct tegra_dma_desc *dma_desc) 307 307 + struct tegra_dma_desc *dma_desc) 303 308 { 304 309 unsigned long flags; 305 310 ··· 311 314 spin_unlock_irqrestore(&tdc->lock, flags); 312 315 } 313 316 314 314 - static struct tegra_dma_sg_req *tegra_dma_sg_req_get( 315 315 - struct tegra_dma_channel *tdc) 317 317 + static struct tegra_dma_sg_req * 318 318 + tegra_dma_sg_req_get(struct tegra_dma_channel *tdc) 316 319 { 317 317 - struct tegra_dma_sg_req *sg_req = NULL; 320 320 + struct tegra_dma_sg_req *sg_req; 318 321 unsigned long flags; 319 322 320 323 spin_lock_irqsave(&tdc->lock, flags); 321 324 if (!list_empty(&tdc->free_sg_req)) { 322 322 - sg_req = list_first_entry(&tdc->free_sg_req, 323 323 - typeof(*sg_req), node); 325 325 + sg_req = list_first_entry(&tdc->free_sg_req, typeof(*sg_req), 326 326 + node); 324 327 list_del(&sg_req->node); 325 328 spin_unlock_irqrestore(&tdc->lock, flags); 326 329 return sg_req; 327 330 } 328 331 spin_unlock_irqrestore(&tdc->lock, flags); 329 332 330 330 - sg_req = kzalloc(sizeof(struct tegra_dma_sg_req), GFP_NOWAIT); 333 333 + sg_req = kzalloc(sizeof(*sg_req), GFP_NOWAIT); 331 334 332 335 return sg_req; 333 336 } 334 337 335 338 static int tegra_dma_slave_config(struct dma_chan *dc, 336 336 - struct dma_slave_config *sconfig) 339 339 + struct dma_slave_config *sconfig) 337 340 { 338 341 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 339 342 ··· 350 353 tdc->slave_id = sconfig->slave_id; 351 354 } 352 355 tdc->config_init = true; 356 356 + 353 357 return 0; 354 358 } 355 359 356 360 static void tegra_dma_global_pause(struct tegra_dma_channel *tdc, 357 357 - bool wait_for_burst_complete) 361 361 + bool wait_for_burst_complete) 358 362 { 359 363 struct tegra_dma *tdma = tdc->tdma; 360 364 ··· 390 392 } 391 393 392 394 static void tegra_dma_pause(struct tegra_dma_channel *tdc, 393 393 - bool wait_for_burst_complete) 395 395 + bool wait_for_burst_complete) 394 396 { 395 397 struct tegra_dma *tdma = tdc->tdma; 396 398 397 399 if (tdma->chip_data->support_channel_pause) { 398 400 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE, 399 399 - TEGRA_APBDMA_CHAN_CSRE_PAUSE); 401 401 + TEGRA_APBDMA_CHAN_CSRE_PAUSE); 400 402 if (wait_for_burst_complete) 401 403 udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME); 402 404 } else { ··· 408 410 { 409 411 struct tegra_dma *tdma = tdc->tdma; 410 412 411 411 - if (tdma->chip_data->support_channel_pause) { 413 413 + if (tdma->chip_data->support_channel_pause) 412 414 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE, 0); 413 413 - } else { 415 415 + else 414 416 tegra_dma_global_resume(tdc); 415 415 - } 416 417 } 417 418 418 419 static void tegra_dma_stop(struct tegra_dma_channel *tdc) 419 420 { 420 420 - u32 csr; 421 421 - u32 status; 421 421 + u32 csr, status; 422 422 423 423 /* Disable interrupts */ 424 424 csr = tdc_read(tdc, TEGRA_APBDMA_CHAN_CSR); ··· 437 441 } 438 442 439 443 static void tegra_dma_start(struct tegra_dma_channel *tdc, 440 440 - struct tegra_dma_sg_req *sg_req) 444 444 + struct tegra_dma_sg_req *sg_req) 441 445 { 442 446 struct tegra_dma_channel_regs *ch_regs = &sg_req->ch_regs; 443 447 ··· 451 455 452 456 /* Start DMA */ 453 457 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, 454 454 - ch_regs->csr | TEGRA_APBDMA_CSR_ENB); 458 458 + ch_regs->csr | TEGRA_APBDMA_CSR_ENB); 455 459 } 456 460 457 461 static void tegra_dma_configure_for_next(struct tegra_dma_channel *tdc, 458 458 - struct tegra_dma_sg_req *nsg_req) 462 462 + struct tegra_dma_sg_req *nsg_req) 459 463 { 460 464 unsigned long status; 461 465 ··· 489 493 tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, nsg_req->ch_regs.ahb_ptr); 490 494 if (tdc->tdma->chip_data->support_separate_wcount_reg) 491 495 tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT, 492 492 - nsg_req->ch_regs.wcount); 496 496 + nsg_req->ch_regs.wcount); 493 497 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, 494 494 - nsg_req->ch_regs.csr | TEGRA_APBDMA_CSR_ENB); 498 498 + nsg_req->ch_regs.csr | TEGRA_APBDMA_CSR_ENB); 495 499 nsg_req->configured = true; 496 500 nsg_req->words_xferred = 0; 497 501 ··· 502 506 { 503 507 struct tegra_dma_sg_req *sg_req; 504 508 505 505 - if (list_empty(&tdc->pending_sg_req)) 506 506 - return; 507 507 - 508 508 - sg_req = list_first_entry(&tdc->pending_sg_req, 509 509 - typeof(*sg_req), node); 509 509 + sg_req = list_first_entry(&tdc->pending_sg_req, typeof(*sg_req), node); 510 510 tegra_dma_start(tdc, sg_req); 511 511 sg_req->configured = true; 512 512 sg_req->words_xferred = 0; ··· 511 519 512 520 static void tdc_configure_next_head_desc(struct tegra_dma_channel *tdc) 513 521 { 514 514 - struct tegra_dma_sg_req *hsgreq; 515 515 - struct tegra_dma_sg_req *hnsgreq; 516 516 - 517 517 - if (list_empty(&tdc->pending_sg_req)) 518 518 - return; 522 522 + struct tegra_dma_sg_req *hsgreq, *hnsgreq; 519 523 520 524 hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node); 521 525 if (!list_is_last(&hsgreq->node, &tdc->pending_sg_req)) { 522 522 - hnsgreq = list_first_entry(&hsgreq->node, 523 523 - typeof(*hnsgreq), node); 526 526 + hnsgreq = list_first_entry(&hsgreq->node, typeof(*hnsgreq), 527 527 + node); 524 528 tegra_dma_configure_for_next(tdc, hnsgreq); 525 529 } 526 530 } 527 531 528 528 - static inline int get_current_xferred_count(struct tegra_dma_channel *tdc, 529 529 - struct tegra_dma_sg_req *sg_req, unsigned long status) 532 532 + static inline unsigned int 533 533 + get_current_xferred_count(struct tegra_dma_channel *tdc, 534 534 + struct tegra_dma_sg_req *sg_req, 535 535 + unsigned long status) 530 536 { 531 537 return sg_req->req_len - (status & TEGRA_APBDMA_STATUS_COUNT_MASK) - 4; 532 538 } 533 539 534 540 static void tegra_dma_abort_all(struct tegra_dma_channel *tdc) 535 541 { 536 536 - struct tegra_dma_sg_req *sgreq; 537 542 struct tegra_dma_desc *dma_desc; 543 543 + struct tegra_dma_sg_req *sgreq; 538 544 539 545 while (!list_empty(&tdc->pending_sg_req)) { 540 540 - sgreq = list_first_entry(&tdc->pending_sg_req, 541 541 - typeof(*sgreq), node); 546 546 + sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), 547 547 + node); 542 548 list_move_tail(&sgreq->node, &tdc->free_sg_req); 543 549 if (sgreq->last_sg) { 544 550 dma_desc = sgreq->dma_desc; ··· 546 556 /* Add in cb list if it is not there. */ 547 557 if (!dma_desc->cb_count) 548 558 list_add_tail(&dma_desc->cb_node, 549 549 - &tdc->cb_desc); 559 559 + &tdc->cb_desc); 550 560 dma_desc->cb_count++; 551 561 } 552 562 } ··· 554 564 } 555 565 556 566 static bool handle_continuous_head_request(struct tegra_dma_channel *tdc, 557 557 - struct tegra_dma_sg_req *last_sg_req, bool to_terminate) 567 567 + bool to_terminate) 558 568 { 559 559 - struct tegra_dma_sg_req *hsgreq = NULL; 560 560 - 561 561 - if (list_empty(&tdc->pending_sg_req)) { 562 562 - dev_err(tdc2dev(tdc), "DMA is running without req\n"); 563 563 - tegra_dma_stop(tdc); 564 564 - return false; 565 565 - } 569 569 + struct tegra_dma_sg_req *hsgreq; 566 570 567 571 /* 568 572 * Check that head req on list should be in flight. ··· 566 582 hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node); 567 583 if (!hsgreq->configured) { 568 584 tegra_dma_stop(tdc); 569 569 - dev_err(tdc2dev(tdc), "Error in DMA transfer, aborting DMA\n"); 585 585 + pm_runtime_put(tdc->tdma->dev); 586 586 + dev_err(tdc2dev(tdc), "DMA transfer underflow, aborting DMA\n"); 570 587 tegra_dma_abort_all(tdc); 571 588 return false; 572 589 } ··· 575 590 /* Configure next request */ 576 591 if (!to_terminate) 577 592 tdc_configure_next_head_desc(tdc); 593 593 + 578 594 return true; 579 595 } 580 596 581 597 static void handle_once_dma_done(struct tegra_dma_channel *tdc, 582 582 - bool to_terminate) 598 598 + bool to_terminate) 583 599 { 584 584 - struct tegra_dma_sg_req *sgreq; 585 600 struct tegra_dma_desc *dma_desc; 601 601 + struct tegra_dma_sg_req *sgreq; 586 602 587 603 tdc->busy = false; 588 604 sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node); ··· 602 616 list_add_tail(&sgreq->node, &tdc->free_sg_req); 603 617 604 618 /* Do not start DMA if it is going to be terminate */ 605 605 - if (to_terminate || list_empty(&tdc->pending_sg_req)) 619 619 + if (to_terminate) 606 620 return; 621 621 + 622 622 + if (list_empty(&tdc->pending_sg_req)) { 623 623 + pm_runtime_put(tdc->tdma->dev); 624 624 + return; 625 625 + } 607 626 608 627 tdc_start_head_req(tdc); 609 628 } 610 629 611 630 static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc, 612 612 - bool to_terminate) 631 631 + bool to_terminate) 613 632 { 614 614 - struct tegra_dma_sg_req *sgreq; 615 633 struct tegra_dma_desc *dma_desc; 634 634 + struct tegra_dma_sg_req *sgreq; 616 635 bool st; 617 636 618 637 sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node); ··· 638 647 if (!list_is_last(&sgreq->node, &tdc->pending_sg_req)) { 639 648 list_move_tail(&sgreq->node, &tdc->pending_sg_req); 640 649 sgreq->configured = false; 641 641 - st = handle_continuous_head_request(tdc, sgreq, to_terminate); 650 650 + st = handle_continuous_head_request(tdc, to_terminate); 642 651 if (!st) 643 652 dma_desc->dma_status = DMA_ERROR; 644 653 } ··· 649 658 struct tegra_dma_channel *tdc = (struct tegra_dma_channel *)data; 650 659 struct dmaengine_desc_callback cb; 651 660 struct tegra_dma_desc *dma_desc; 661 661 + unsigned int cb_count; 652 662 unsigned long flags; 653 653 - int cb_count; 654 663 655 664 spin_lock_irqsave(&tdc->lock, flags); 656 665 while (!list_empty(&tdc->cb_desc)) { 657 657 - dma_desc = list_first_entry(&tdc->cb_desc, 658 658 - typeof(*dma_desc), cb_node); 666 666 + dma_desc = list_first_entry(&tdc->cb_desc, typeof(*dma_desc), 667 667 + cb_node); 659 668 list_del(&dma_desc->cb_node); 660 669 dmaengine_desc_get_callback(&dma_desc->txd, &cb); 661 670 cb_count = dma_desc->cb_count; ··· 673 682 static irqreturn_t tegra_dma_isr(int irq, void *dev_id) 674 683 { 675 684 struct tegra_dma_channel *tdc = dev_id; 676 676 - unsigned long status; 677 677 - unsigned long flags; 685 685 + u32 status; 678 686 679 679 - spin_lock_irqsave(&tdc->lock, flags); 687 687 + spin_lock(&tdc->lock); 680 688 681 689 trace_tegra_dma_isr(&tdc->dma_chan, irq); 682 690 status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); ··· 683 693 tdc_write(tdc, TEGRA_APBDMA_CHAN_STATUS, status); 684 694 tdc->isr_handler(tdc, false); 685 695 tasklet_schedule(&tdc->tasklet); 686 686 - spin_unlock_irqrestore(&tdc->lock, flags); 696 696 + wake_up_all(&tdc->wq); 697 697 + spin_unlock(&tdc->lock); 687 698 return IRQ_HANDLED; 688 699 } 689 700 690 690 - spin_unlock_irqrestore(&tdc->lock, flags); 691 691 - dev_info(tdc2dev(tdc), 692 692 - "Interrupt already served status 0x%08lx\n", status); 701 701 + spin_unlock(&tdc->lock); 702 702 + dev_info(tdc2dev(tdc), "Interrupt already served status 0x%08x\n", 703 703 + status); 704 704 + 693 705 return IRQ_NONE; 694 706 } 695 707 ··· 707 715 cookie = dma_cookie_assign(&dma_desc->txd); 708 716 list_splice_tail_init(&dma_desc->tx_list, &tdc->pending_sg_req); 709 717 spin_unlock_irqrestore(&tdc->lock, flags); 718 718 + 710 719 return cookie; 711 720 } 712 721 ··· 715 722 { 716 723 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 717 724 unsigned long flags; 725 725 + int err; 718 726 719 727 spin_lock_irqsave(&tdc->lock, flags); 720 728 if (list_empty(&tdc->pending_sg_req)) { ··· 723 729 goto end; 724 730 } 725 731 if (!tdc->busy) { 732 732 + err = pm_runtime_get_sync(tdc->tdma->dev); 733 733 + if (err < 0) { 734 734 + dev_err(tdc2dev(tdc), "Failed to enable DMA\n"); 735 735 + goto end; 736 736 + } 737 737 + 726 738 tdc_start_head_req(tdc); 727 739 728 740 /* Continuous single mode: Configure next req */ ··· 748 748 static int tegra_dma_terminate_all(struct dma_chan *dc) 749 749 { 750 750 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 751 751 - struct tegra_dma_sg_req *sgreq; 752 751 struct tegra_dma_desc *dma_desc; 752 752 + struct tegra_dma_sg_req *sgreq; 753 753 unsigned long flags; 754 754 - unsigned long status; 755 755 - unsigned long wcount; 754 754 + u32 status, wcount; 756 755 bool was_busy; 757 756 758 757 spin_lock_irqsave(&tdc->lock, flags); ··· 777 778 tegra_dma_stop(tdc); 778 779 779 780 if (!list_empty(&tdc->pending_sg_req) && was_busy) { 780 780 - sgreq = list_first_entry(&tdc->pending_sg_req, 781 781 - typeof(*sgreq), node); 781 781 + sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), 782 782 + node); 782 783 sgreq->dma_desc->bytes_transferred += 783 784 get_current_xferred_count(tdc, sgreq, wcount); 784 785 } 785 786 tegra_dma_resume(tdc); 786 787 788 788 + pm_runtime_put(tdc->tdma->dev); 789 789 + wake_up_all(&tdc->wq); 790 790 + 787 791 skip_dma_stop: 788 792 tegra_dma_abort_all(tdc); 789 793 790 794 while (!list_empty(&tdc->cb_desc)) { 791 791 - dma_desc = list_first_entry(&tdc->cb_desc, 792 792 - typeof(*dma_desc), cb_node); 795 795 + dma_desc = list_first_entry(&tdc->cb_desc, typeof(*dma_desc), 796 796 + cb_node); 793 797 list_del(&dma_desc->cb_node); 794 798 dma_desc->cb_count = 0; 795 799 } 796 800 spin_unlock_irqrestore(&tdc->lock, flags); 801 801 + 797 802 return 0; 803 803 + } 804 804 + 805 805 + static bool tegra_dma_eoc_interrupt_deasserted(struct tegra_dma_channel *tdc) 806 806 + { 807 807 + unsigned long flags; 808 808 + u32 status; 809 809 + 810 810 + spin_lock_irqsave(&tdc->lock, flags); 811 811 + status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); 812 812 + spin_unlock_irqrestore(&tdc->lock, flags); 813 813 + 814 814 + return !(status & TEGRA_APBDMA_STATUS_ISE_EOC); 815 815 + } 816 816 + 817 817 + static void tegra_dma_synchronize(struct dma_chan *dc) 818 818 + { 819 819 + struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 820 820 + 821 821 + /* 822 822 + * CPU, which handles interrupt, could be busy in 823 823 + * uninterruptible state, in this case sibling CPU 824 824 + * should wait until interrupt is handled. 825 825 + */ 826 826 + wait_event(tdc->wq, tegra_dma_eoc_interrupt_deasserted(tdc)); 827 827 + 828 828 + tasklet_kill(&tdc->tasklet); 798 829 } 799 830 800 831 static unsigned int tegra_dma_sg_bytes_xferred(struct tegra_dma_channel *tdc, 801 832 struct tegra_dma_sg_req *sg_req) 802 833 { 803 803 - unsigned long status, wcount = 0; 834 834 + u32 status, wcount = 0; 804 835 805 836 if (!list_is_first(&sg_req->node, &tdc->pending_sg_req)) 806 837 return 0; ··· 887 858 } 888 859 889 860 static enum dma_status tegra_dma_tx_status(struct dma_chan *dc, 890 890 - dma_cookie_t cookie, struct dma_tx_state *txstate) 861 861 + dma_cookie_t cookie, 862 862 + struct dma_tx_state *txstate) 891 863 { 892 864 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 893 865 struct tegra_dma_desc *dma_desc; ··· 935 905 936 906 trace_tegra_dma_tx_status(&tdc->dma_chan, cookie, txstate); 937 907 spin_unlock_irqrestore(&tdc->lock, flags); 908 908 + 938 909 return ret; 939 910 } 940 911 941 941 - static inline int get_bus_width(struct tegra_dma_channel *tdc, 942 942 - enum dma_slave_buswidth slave_bw) 912 912 + static inline unsigned int get_bus_width(struct tegra_dma_channel *tdc, 913 913 + enum dma_slave_buswidth slave_bw) 943 914 { 944 915 switch (slave_bw) { 945 916 case DMA_SLAVE_BUSWIDTH_1_BYTE: ··· 953 922 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_64; 954 923 default: 955 924 dev_warn(tdc2dev(tdc), 956 956 - "slave bw is not supported, using 32bits\n"); 925 925 + "slave bw is not supported, using 32bits\n"); 957 926 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32; 958 927 } 959 928 } 960 929 961 961 - static inline int get_burst_size(struct tegra_dma_channel *tdc, 962 962 - u32 burst_size, enum dma_slave_buswidth slave_bw, int len) 930 930 + static inline unsigned int get_burst_size(struct tegra_dma_channel *tdc, 931 931 + u32 burst_size, 932 932 + enum dma_slave_buswidth slave_bw, 933 933 + u32 len) 963 934 { 964 964 - int burst_byte; 965 965 - int burst_ahb_width; 935 935 + unsigned int burst_byte, burst_ahb_width; 966 936 967 937 /* 968 938 * burst_size from client is in terms of the bus_width. ··· 990 958 } 991 959 992 960 static int get_transfer_param(struct tegra_dma_channel *tdc, 993 993 - enum dma_transfer_direction direction, unsigned long *apb_addr, 994 994 - unsigned long *apb_seq, unsigned long *csr, unsigned int *burst_size, 995 995 - enum dma_slave_buswidth *slave_bw) 961 961 + enum dma_transfer_direction direction, 962 962 + u32 *apb_addr, 963 963 + u32 *apb_seq, 964 964 + u32 *csr, 965 965 + unsigned int *burst_size, 966 966 + enum dma_slave_buswidth *slave_bw) 996 967 { 997 968 switch (direction) { 998 969 case DMA_MEM_TO_DEV: ··· 1016 981 1017 982 default: 1018 983 dev_err(tdc2dev(tdc), "DMA direction is not supported\n"); 1019 1019 - return -EINVAL; 984 984 + break; 1020 985 } 986 986 + 1021 987 return -EINVAL; 1022 988 } 1023 989 1024 990 static void tegra_dma_prep_wcount(struct tegra_dma_channel *tdc, 1025 1025 - struct tegra_dma_channel_regs *ch_regs, u32 len) 991 991 + struct tegra_dma_channel_regs *ch_regs, 992 992 + u32 len) 1026 993 { 1027 994 u32 len_field = (len - 4) & 0xFFFC; 1028 995 ··· 1034 997 ch_regs->csr |= len_field; 1035 998 } 1036 999 1037 1037 - static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( 1038 1038 - struct dma_chan *dc, struct scatterlist *sgl, unsigned int sg_len, 1039 1039 - enum dma_transfer_direction direction, unsigned long flags, 1040 1040 - void *context) 1000 1000 + static struct dma_async_tx_descriptor * 1001 1001 + tegra_dma_prep_slave_sg(struct dma_chan *dc, 1002 1002 + struct scatterlist *sgl, 1003 1003 + unsigned int sg_len, 1004 1004 + enum dma_transfer_direction direction, 1005 1005 + unsigned long flags, 1006 1006 + void *context) 1041 1007 { 1042 1008 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 1043 1043 - struct tegra_dma_desc *dma_desc; 1044 1044 - unsigned int i; 1045 1045 - struct scatterlist *sg; 1046 1046 - unsigned long csr, ahb_seq, apb_ptr, apb_seq; 1047 1047 - struct list_head req_list; 1048 1048 - struct tegra_dma_sg_req *sg_req = NULL; 1049 1049 - u32 burst_size; 1009 1009 + struct tegra_dma_sg_req *sg_req = NULL; 1010 1010 + u32 csr, ahb_seq, apb_ptr, apb_seq; 1050 1011 enum dma_slave_buswidth slave_bw; 1012 1012 + struct tegra_dma_desc *dma_desc; 1013 1013 + struct list_head req_list; 1014 1014 + struct scatterlist *sg; 1015 1015 + unsigned int burst_size; 1016 1016 + unsigned int i; 1051 1017 1052 1018 if (!tdc->config_init) { 1053 1019 dev_err(tdc2dev(tdc), "DMA channel is not configured\n"); ··· 1062 1022 } 1063 1023 1064 1024 if (get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr, 1065 1065 - &burst_size, &slave_bw) < 0) 1025 1025 + &burst_size, &slave_bw) < 0) 1066 1026 return NULL; 1067 1027 1068 1028 INIT_LIST_HEAD(&req_list); ··· 1108 1068 len = sg_dma_len(sg); 1109 1069 1110 1070 if ((len & 3) || (mem & 3) || 1111 1111 - (len > tdc->tdma->chip_data->max_dma_count)) { 1071 1071 + len > tdc->tdma->chip_data->max_dma_count) { 1112 1072 dev_err(tdc2dev(tdc), 1113 1073 "DMA length/memory address is not supported\n"); 1114 1074 tegra_dma_desc_put(tdc, dma_desc); ··· 1160 1120 return &dma_desc->txd; 1161 1121 } 1162 1122 1163 1163 - static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic( 1164 1164 - struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len, 1165 1165 - size_t period_len, enum dma_transfer_direction direction, 1166 1166 - unsigned long flags) 1123 1123 + static struct dma_async_tx_descriptor * 1124 1124 + tegra_dma_prep_dma_cyclic(struct dma_chan *dc, dma_addr_t buf_addr, 1125 1125 + size_t buf_len, 1126 1126 + size_t period_len, 1127 1127 + enum dma_transfer_direction direction, 1128 1128 + unsigned long flags) 1167 1129 { 1168 1130 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 1169 1169 - struct tegra_dma_desc *dma_desc = NULL; 1170 1131 struct tegra_dma_sg_req *sg_req = NULL; 1171 1171 - unsigned long csr, ahb_seq, apb_ptr, apb_seq; 1172 1172 - int len; 1173 1173 - size_t remain_len; 1174 1174 - dma_addr_t mem = buf_addr; 1175 1175 - u32 burst_size; 1132 1132 + u32 csr, ahb_seq, apb_ptr, apb_seq; 1176 1133 enum dma_slave_buswidth slave_bw; 1134 1134 + struct tegra_dma_desc *dma_desc; 1135 1135 + dma_addr_t mem = buf_addr; 1136 1136 + unsigned int burst_size; 1137 1137 + size_t len, remain_len; 1177 1138 1178 1139 if (!buf_len || !period_len) { 1179 1140 dev_err(tdc2dev(tdc), "Invalid buffer/period len\n"); ··· 1208 1167 1209 1168 len = period_len; 1210 1169 if ((len & 3) || (buf_addr & 3) || 1211 1211 - (len > tdc->tdma->chip_data->max_dma_count)) { 1170 1170 + len > tdc->tdma->chip_data->max_dma_count) { 1212 1171 dev_err(tdc2dev(tdc), "Req len/mem address is not correct\n"); 1213 1172 return NULL; 1214 1173 } 1215 1174 1216 1175 if (get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr, 1217 1217 - &burst_size, &slave_bw) < 0) 1176 1176 + &burst_size, &slave_bw) < 0) 1218 1177 return NULL; 1219 1178 1220 1179 ahb_seq = TEGRA_APBDMA_AHBSEQ_INTR_ENB; ··· 1300 1259 static int tegra_dma_alloc_chan_resources(struct dma_chan *dc) 1301 1260 { 1302 1261 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 1303 1303 - struct tegra_dma *tdma = tdc->tdma; 1304 1304 - int ret; 1305 1262 1306 1263 dma_cookie_init(&tdc->dma_chan); 1307 1307 - tdc->config_init = false; 1308 1308 - 1309 1309 - ret = pm_runtime_get_sync(tdma->dev); 1310 1310 - if (ret < 0) 1311 1311 - return ret; 1312 1264 1313 1265 return 0; 1314 1266 } ··· 1309 1275 static void tegra_dma_free_chan_resources(struct dma_chan *dc) 1310 1276 { 1311 1277 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 1312 1312 - struct tegra_dma *tdma = tdc->tdma; 1313 1278 struct tegra_dma_desc *dma_desc; 1314 1279 struct tegra_dma_sg_req *sg_req; 1315 1280 struct list_head dma_desc_list; 1316 1281 struct list_head sg_req_list; 1317 1317 - unsigned long flags; 1318 1282 1319 1283 INIT_LIST_HEAD(&dma_desc_list); 1320 1284 INIT_LIST_HEAD(&sg_req_list); 1321 1285 1322 1286 dev_dbg(tdc2dev(tdc), "Freeing channel %d\n", tdc->id); 1323 1287 1324 1324 - if (tdc->busy) 1325 1325 - tegra_dma_terminate_all(dc); 1288 1288 + tegra_dma_terminate_all(dc); 1289 1289 + tasklet_kill(&tdc->tasklet); 1326 1290 1327 1327 - spin_lock_irqsave(&tdc->lock, flags); 1328 1291 list_splice_init(&tdc->pending_sg_req, &sg_req_list); 1329 1292 list_splice_init(&tdc->free_sg_req, &sg_req_list); 1330 1293 list_splice_init(&tdc->free_dma_desc, &dma_desc_list); 1331 1294 INIT_LIST_HEAD(&tdc->cb_desc); 1332 1295 tdc->config_init = false; 1333 1296 tdc->isr_handler = NULL; 1334 1334 - spin_unlock_irqrestore(&tdc->lock, flags); 1335 1297 1336 1298 while (!list_empty(&dma_desc_list)) { 1337 1337 - dma_desc = list_first_entry(&dma_desc_list, 1338 1338 - typeof(*dma_desc), node); 1299 1299 + dma_desc = list_first_entry(&dma_desc_list, typeof(*dma_desc), 1300 1300 + node); 1339 1301 list_del(&dma_desc->node); 1340 1302 kfree(dma_desc); 1341 1303 } ··· 1341 1311 list_del(&sg_req->node); 1342 1312 kfree(sg_req); 1343 1313 } 1344 1344 - pm_runtime_put(tdma->dev); 1345 1314 1346 1315 tdc->slave_id = TEGRA_APBDMA_SLAVE_ID_INVALID; 1347 1316 } ··· 1349 1320 struct of_dma *ofdma) 1350 1321 { 1351 1322 struct tegra_dma *tdma = ofdma->of_dma_data; 1352 1352 - struct dma_chan *chan; 1353 1323 struct tegra_dma_channel *tdc; 1324 1324 + struct dma_chan *chan; 1354 1325 1355 1326 if (dma_spec->args[0] > TEGRA_APBDMA_CSR_REQ_SEL_MASK) { 1356 1327 dev_err(tdma->dev, "Invalid slave id: %d\n", dma_spec->args[0]); ··· 1403 1374 .support_separate_wcount_reg = true, 1404 1375 }; 1405 1376 1406 1406 - static int tegra_dma_probe(struct platform_device *pdev) 1377 1377 + static int tegra_dma_init_hw(struct tegra_dma *tdma) 1407 1378 { 1408 1408 - struct resource *res; 1409 1409 - struct tegra_dma *tdma; 1410 1410 - int ret; 1411 1411 - int i; 1412 1412 - const struct tegra_dma_chip_data *cdata; 1379 1379 + int err; 1413 1380 1414 1414 - cdata = of_device_get_match_data(&pdev->dev); 1415 1415 - if (!cdata) { 1416 1416 - dev_err(&pdev->dev, "Error: No device match data found\n"); 1417 1417 - return -ENODEV; 1381 1381 + err = reset_control_assert(tdma->rst); 1382 1382 + if (err) { 1383 1383 + dev_err(tdma->dev, "failed to assert reset: %d\n", err); 1384 1384 + return err; 1418 1385 } 1419 1386 1420 1420 - tdma = devm_kzalloc(&pdev->dev, 1421 1421 - struct_size(tdma, channels, cdata->nr_channels), 1422 1422 - GFP_KERNEL); 1387 1387 + err = clk_enable(tdma->dma_clk); 1388 1388 + if (err) { 1389 1389 + dev_err(tdma->dev, "failed to enable clk: %d\n", err); 1390 1390 + return err; 1391 1391 + } 1392 1392 + 1393 1393 + /* reset DMA controller */ 1394 1394 + udelay(2); 1395 1395 + reset_control_deassert(tdma->rst); 1396 1396 + 1397 1397 + /* enable global DMA registers */ 1398 1398 + tdma_write(tdma, TEGRA_APBDMA_GENERAL, TEGRA_APBDMA_GENERAL_ENABLE); 1399 1399 + tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0); 1400 1400 + tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFF); 1401 1401 + 1402 1402 + clk_disable(tdma->dma_clk); 1403 1403 + 1404 1404 + return 0; 1405 1405 + } 1406 1406 + 1407 1407 + static int tegra_dma_probe(struct platform_device *pdev) 1408 1408 + { 1409 1409 + const struct tegra_dma_chip_data *cdata; 1410 1410 + struct tegra_dma *tdma; 1411 1411 + unsigned int i; 1412 1412 + size_t size; 1413 1413 + int ret; 1414 1414 + 1415 1415 + cdata = of_device_get_match_data(&pdev->dev); 1416 1416 + size = struct_size(tdma, channels, cdata->nr_channels); 1417 1417 + 1418 1418 + tdma = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); 1423 1419 if (!tdma) 1424 1420 return -ENOMEM; 1425 1421 ··· 1452 1398 tdma->chip_data = cdata; 1453 1399 platform_set_drvdata(pdev, tdma); 1454 1400 1455 1455 - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1456 1456 - tdma->base_addr = devm_ioremap_resource(&pdev->dev, res); 1401 1401 + tdma->base_addr = devm_platform_ioremap_resource(pdev, 0); 1457 1402 if (IS_ERR(tdma->base_addr)) 1458 1403 return PTR_ERR(tdma->base_addr); 1459 1404 ··· 1470 1417 1471 1418 spin_lock_init(&tdma->global_lock); 1472 1419 1473 1473 - pm_runtime_enable(&pdev->dev); 1474 1474 - if (!pm_runtime_enabled(&pdev->dev)) 1475 1475 - ret = tegra_dma_runtime_resume(&pdev->dev); 1476 1476 - else 1477 1477 - ret = pm_runtime_get_sync(&pdev->dev); 1478 1478 - 1479 1479 - if (ret < 0) { 1480 1480 - pm_runtime_disable(&pdev->dev); 1420 1420 + ret = clk_prepare(tdma->dma_clk); 1421 1421 + if (ret) 1481 1422 return ret; 1482 1482 - } 1483 1423 1484 1484 - /* Reset DMA controller */ 1485 1485 - reset_control_assert(tdma->rst); 1486 1486 - udelay(2); 1487 1487 - reset_control_deassert(tdma->rst); 1424 1424 + ret = tegra_dma_init_hw(tdma); 1425 1425 + if (ret) 1426 1426 + goto err_clk_unprepare; 1488 1427 1489 1489 - /* Enable global DMA registers */ 1490 1490 - tdma_write(tdma, TEGRA_APBDMA_GENERAL, TEGRA_APBDMA_GENERAL_ENABLE); 1491 1491 - tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0); 1492 1492 - tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFFul); 1493 1493 - 1494 1494 - pm_runtime_put(&pdev->dev); 1428 1428 + pm_runtime_irq_safe(&pdev->dev); 1429 1429 + pm_runtime_enable(&pdev->dev); 1495 1430 1496 1431 INIT_LIST_HEAD(&tdma->dma_dev.channels); 1497 1432 for (i = 0; i < cdata->nr_channels; i++) { 1498 1433 struct tegra_dma_channel *tdc = &tdma->channels[i]; 1434 1434 + int irq; 1499 1435 1500 1436 tdc->chan_addr = tdma->base_addr + 1501 1437 TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET + 1502 1438 (i * cdata->channel_reg_size); 1503 1439 1504 1504 - res = platform_get_resource(pdev, IORESOURCE_IRQ, i); 1505 1505 - if (!res) { 1506 1506 - ret = -EINVAL; 1440 1440 + irq = platform_get_irq(pdev, i); 1441 1441 + if (irq < 0) { 1442 1442 + ret = irq; 1507 1443 dev_err(&pdev->dev, "No irq resource for chan %d\n", i); 1508 1508 - goto err_irq; 1444 1444 + goto err_pm_disable; 1509 1445 } 1510 1510 - tdc->irq = res->start; 1446 1446 + 1511 1447 snprintf(tdc->name, sizeof(tdc->name), "apbdma.%d", i); 1512 1512 - ret = request_irq(tdc->irq, tegra_dma_isr, 0, tdc->name, tdc); 1448 1448 + ret = devm_request_irq(&pdev->dev, irq, tegra_dma_isr, 0, 1449 1449 + tdc->name, tdc); 1513 1450 if (ret) { 1514 1451 dev_err(&pdev->dev, 1515 1452 "request_irq failed with err %d channel %d\n", 1516 1453 ret, i); 1517 1517 - goto err_irq; 1454 1454 + goto err_pm_disable; 1518 1455 } 1519 1456 1520 1457 tdc->dma_chan.device = &tdma->dma_dev; 1521 1458 dma_cookie_init(&tdc->dma_chan); 1522 1459 list_add_tail(&tdc->dma_chan.device_node, 1523 1523 - &tdma->dma_dev.channels); 1460 1460 + &tdma->dma_dev.channels); 1524 1461 tdc->tdma = tdma; 1525 1462 tdc->id = i; 1526 1463 tdc->slave_id = TEGRA_APBDMA_SLAVE_ID_INVALID; 1527 1464 1528 1465 tasklet_init(&tdc->tasklet, tegra_dma_tasklet, 1529 1529 - (unsigned long)tdc); 1466 1466 + (unsigned long)tdc); 1530 1467 spin_lock_init(&tdc->lock); 1468 1468 + init_waitqueue_head(&tdc->wq); 1531 1469 1532 1470 INIT_LIST_HEAD(&tdc->pending_sg_req); 1533 1471 INIT_LIST_HEAD(&tdc->free_sg_req); ··· 1550 1506 tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 1551 1507 tdma->dma_dev.device_config = tegra_dma_slave_config; 1552 1508 tdma->dma_dev.device_terminate_all = tegra_dma_terminate_all; 1509 1509 + tdma->dma_dev.device_synchronize = tegra_dma_synchronize; 1553 1510 tdma->dma_dev.device_tx_status = tegra_dma_tx_status; 1554 1511 tdma->dma_dev.device_issue_pending = tegra_dma_issue_pending; 1555 1512 ··· 1558 1513 if (ret < 0) { 1559 1514 dev_err(&pdev->dev, 1560 1515 "Tegra20 APB DMA driver registration failed %d\n", ret); 1561 1561 - goto err_irq; 1516 1516 + goto err_pm_disable; 1562 1517 } 1563 1518 1564 1519 ret = of_dma_controller_register(pdev->dev.of_node, ··· 1569 1524 goto err_unregister_dma_dev; 1570 1525 } 1571 1526 1572 1572 - dev_info(&pdev->dev, "Tegra20 APB DMA driver register %d channels\n", 1573 1573 - cdata->nr_channels); 1527 1527 + dev_info(&pdev->dev, "Tegra20 APB DMA driver registered %u channels\n", 1528 1528 + cdata->nr_channels); 1529 1529 + 1574 1530 return 0; 1575 1531 1576 1532 err_unregister_dma_dev: 1577 1533 dma_async_device_unregister(&tdma->dma_dev); 1578 1578 - err_irq: 1579 1579 - while (--i >= 0) { 1580 1580 - struct tegra_dma_channel *tdc = &tdma->channels[i]; 1581 1534 1582 1582 - free_irq(tdc->irq, tdc); 1583 1583 - tasklet_kill(&tdc->tasklet); 1584 1584 - } 1585 1585 - 1535 1535 + err_pm_disable: 1586 1536 pm_runtime_disable(&pdev->dev); 1587 1587 - if (!pm_runtime_status_suspended(&pdev->dev)) 1588 1588 - tegra_dma_runtime_suspend(&pdev->dev); 1537 1537 + 1538 1538 + err_clk_unprepare: 1539 1539 + clk_unprepare(tdma->dma_clk); 1540 1540 + 1589 1541 return ret; 1590 1542 } 1591 1543 1592 1544 static int tegra_dma_remove(struct platform_device *pdev) 1593 1545 { 1594 1546 struct tegra_dma *tdma = platform_get_drvdata(pdev); 1595 1595 - int i; 1596 1596 - struct tegra_dma_channel *tdc; 1597 1547 1548 1548 + of_dma_controller_free(pdev->dev.of_node); 1598 1549 dma_async_device_unregister(&tdma->dma_dev); 1599 1599 - 1600 1600 - for (i = 0; i < tdma->chip_data->nr_channels; ++i) { 1601 1601 - tdc = &tdma->channels[i]; 1602 1602 - free_irq(tdc->irq, tdc); 1603 1603 - tasklet_kill(&tdc->tasklet); 1604 1604 - } 1605 1605 - 1606 1550 pm_runtime_disable(&pdev->dev); 1607 1607 - if (!pm_runtime_status_suspended(&pdev->dev)) 1608 1608 - tegra_dma_runtime_suspend(&pdev->dev); 1551 1551 + clk_unprepare(tdma->dma_clk); 1609 1552 1610 1553 return 0; 1611 1554 } 1612 1555 1613 1613 - static int tegra_dma_runtime_suspend(struct device *dev) 1556 1556 + static int __maybe_unused tegra_dma_runtime_suspend(struct device *dev) 1614 1557 { 1615 1558 struct tegra_dma *tdma = dev_get_drvdata(dev); 1616 1616 - int i; 1617 1559 1618 1618 - tdma->reg_gen = tdma_read(tdma, TEGRA_APBDMA_GENERAL); 1619 1619 - for (i = 0; i < tdma->chip_data->nr_channels; i++) { 1620 1620 - struct tegra_dma_channel *tdc = &tdma->channels[i]; 1621 1621 - struct tegra_dma_channel_regs *ch_reg = &tdc->channel_reg; 1622 1622 - 1623 1623 - /* Only save the state of DMA channels that are in use */ 1624 1624 - if (!tdc->config_init) 1625 1625 - continue; 1626 1626 - 1627 1627 - ch_reg->csr = tdc_read(tdc, TEGRA_APBDMA_CHAN_CSR); 1628 1628 - ch_reg->ahb_ptr = tdc_read(tdc, TEGRA_APBDMA_CHAN_AHBPTR); 1629 1629 - ch_reg->apb_ptr = tdc_read(tdc, TEGRA_APBDMA_CHAN_APBPTR); 1630 1630 - ch_reg->ahb_seq = tdc_read(tdc, TEGRA_APBDMA_CHAN_AHBSEQ); 1631 1631 - ch_reg->apb_seq = tdc_read(tdc, TEGRA_APBDMA_CHAN_APBSEQ); 1632 1632 - if (tdma->chip_data->support_separate_wcount_reg) 1633 1633 - ch_reg->wcount = tdc_read(tdc, 1634 1634 - TEGRA_APBDMA_CHAN_WCOUNT); 1635 1635 - } 1636 1636 - 1637 1637 - clk_disable_unprepare(tdma->dma_clk); 1560 1560 + clk_disable(tdma->dma_clk); 1638 1561 1639 1562 return 0; 1640 1563 } 1641 1564 1642 1642 - static int tegra_dma_runtime_resume(struct device *dev) 1565 1565 + static int __maybe_unused tegra_dma_runtime_resume(struct device *dev) 1643 1566 { 1644 1567 struct tegra_dma *tdma = dev_get_drvdata(dev); 1645 1645 - int i, ret; 1646 1568 1647 1647 - ret = clk_prepare_enable(tdma->dma_clk); 1648 1648 - if (ret < 0) { 1649 1649 - dev_err(dev, "clk_enable failed: %d\n", ret); 1650 1650 - return ret; 1651 1651 - } 1569 1569 + return clk_enable(tdma->dma_clk); 1570 1570 + } 1652 1571 1653 1653 - tdma_write(tdma, TEGRA_APBDMA_GENERAL, tdma->reg_gen); 1654 1654 - tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0); 1655 1655 - tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFFul); 1572 1572 + static int __maybe_unused tegra_dma_dev_suspend(struct device *dev) 1573 1573 + { 1574 1574 + struct tegra_dma *tdma = dev_get_drvdata(dev); 1575 1575 + unsigned long flags; 1576 1576 + unsigned int i; 1577 1577 + bool busy; 1656 1578 1657 1579 for (i = 0; i < tdma->chip_data->nr_channels; i++) { 1658 1580 struct tegra_dma_channel *tdc = &tdma->channels[i]; 1659 1659 - struct tegra_dma_channel_regs *ch_reg = &tdc->channel_reg; 1660 1581 1661 1661 - /* Only restore the state of DMA channels that are in use */ 1662 1662 - if (!tdc->config_init) 1663 1663 - continue; 1582 1582 + tasklet_kill(&tdc->tasklet); 1664 1583 1665 1665 - if (tdma->chip_data->support_separate_wcount_reg) 1666 1666 - tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT, 1667 1667 - ch_reg->wcount); 1668 1668 - tdc_write(tdc, TEGRA_APBDMA_CHAN_APBSEQ, ch_reg->apb_seq); 1669 1669 - tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, ch_reg->apb_ptr); 1670 1670 - tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBSEQ, ch_reg->ahb_seq); 1671 1671 - tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, ch_reg->ahb_ptr); 1672 1672 - tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, 1673 1673 - (ch_reg->csr & ~TEGRA_APBDMA_CSR_ENB)); 1584 1584 + spin_lock_irqsave(&tdc->lock, flags); 1585 1585 + busy = tdc->busy; 1586 1586 + spin_unlock_irqrestore(&tdc->lock, flags); 1587 1587 + 1588 1588 + if (busy) { 1589 1589 + dev_err(tdma->dev, "channel %u busy\n", i); 1590 1590 + return -EBUSY; 1591 1591 + } 1674 1592 } 1675 1593 1676 1676 - return 0; 1594 1594 + return pm_runtime_force_suspend(dev); 1595 1595 + } 1596 1596 + 1597 1597 + static int __maybe_unused tegra_dma_dev_resume(struct device *dev) 1598 1598 + { 1599 1599 + struct tegra_dma *tdma = dev_get_drvdata(dev); 1600 1600 + int err; 1601 1601 + 1602 1602 + err = tegra_dma_init_hw(tdma); 1603 1603 + if (err) 1604 1604 + return err; 1605 1605 + 1606 1606 + return pm_runtime_force_resume(dev); 1677 1607 } 1678 1608 1679 1609 static const struct dev_pm_ops tegra_dma_dev_pm_ops = { 1680 1610 SET_RUNTIME_PM_OPS(tegra_dma_runtime_suspend, tegra_dma_runtime_resume, 1681 1611 NULL) 1682 1682 - SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, 1683 1683 - pm_runtime_force_resume) 1612 1612 + SET_SYSTEM_SLEEP_PM_OPS(tegra_dma_dev_suspend, tegra_dma_dev_resume) 1684 1613 }; 1685 1614 1686 1615 static const struct of_device_id tegra_dma_of_match[] = { ··· 1687 1668 1688 1669 module_platform_driver(tegra_dmac_driver); 1689 1670 1690 1690 - MODULE_ALIAS("platform:tegra20-apbdma"); 1691 1671 MODULE_DESCRIPTION("NVIDIA Tegra APB DMA Controller driver"); 1692 1672 MODULE_AUTHOR("Laxman Dewangan <ldewangan@nvidia.com>"); 1693 1673 MODULE_LICENSE("GPL v2");

+1 -1

drivers/dma/tegra210-adma.c

reviewed

··· 164 164 const struct tegra_adma_chip_data *cdata; 165 165 166 166 /* Last member of the structure */ 167 167 - struct tegra_adma_chan channels[0]; 167 167 + struct tegra_adma_chan channels[]; 168 168 }; 169 169 170 170 static inline void tdma_write(struct tegra_adma *tdma, u32 reg, u32 val)

+2 -6

drivers/dma/ti/dma-crossbar.c

reviewed

··· 133 133 const struct of_device_id *match; 134 134 struct device_node *dma_node; 135 135 struct ti_am335x_xbar_data *xbar; 136 136 - struct resource *res; 137 136 void __iomem *iomem; 138 137 int i, ret; 139 138 ··· 172 173 xbar->xbar_events = TI_AM335X_XBAR_LINES; 173 174 } 174 175 175 175 - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 176 176 - iomem = devm_ioremap_resource(&pdev->dev, res); 176 176 + iomem = devm_platform_ioremap_resource(pdev, 0); 177 177 if (IS_ERR(iomem)) 178 178 return PTR_ERR(iomem); 179 179 ··· 321 323 struct device_node *dma_node; 322 324 struct ti_dra7_xbar_data *xbar; 323 325 struct property *prop; 324 324 - struct resource *res; 325 326 u32 safe_val; 326 327 int sz; 327 328 void __iomem *iomem; ··· 400 403 kfree(rsv_events); 401 404 } 402 405 403 403 - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 404 404 - iomem = devm_ioremap_resource(&pdev->dev, res); 406 406 + iomem = devm_platform_ioremap_resource(pdev, 0); 405 407 if (IS_ERR(iomem)) 406 408 return PTR_ERR(iomem); 407 409

+79

drivers/dma/ti/edma.c

reviewed

··· 1275 1275 return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags); 1276 1276 } 1277 1277 1278 1278 + static struct dma_async_tx_descriptor * 1279 1279 + edma_prep_dma_interleaved(struct dma_chan *chan, 1280 1280 + struct dma_interleaved_template *xt, 1281 1281 + unsigned long tx_flags) 1282 1282 + { 1283 1283 + struct device *dev = chan->device->dev; 1284 1284 + struct edma_chan *echan = to_edma_chan(chan); 1285 1285 + struct edmacc_param *param; 1286 1286 + struct edma_desc *edesc; 1287 1287 + size_t src_icg, dst_icg; 1288 1288 + int src_bidx, dst_bidx; 1289 1289 + 1290 1290 + /* Slave mode is not supported */ 1291 1291 + if (is_slave_direction(xt->dir)) 1292 1292 + return NULL; 1293 1293 + 1294 1294 + if (xt->frame_size != 1 || xt->numf == 0) 1295 1295 + return NULL; 1296 1296 + 1297 1297 + if (xt->sgl[0].size > SZ_64K || xt->numf > SZ_64K) 1298 1298 + return NULL; 1299 1299 + 1300 1300 + src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]); 1301 1301 + if (src_icg) { 1302 1302 + src_bidx = src_icg + xt->sgl[0].size; 1303 1303 + } else if (xt->src_inc) { 1304 1304 + src_bidx = xt->sgl[0].size; 1305 1305 + } else { 1306 1306 + dev_err(dev, "%s: SRC constant addressing is not supported\n", 1307 1307 + __func__); 1308 1308 + return NULL; 1309 1309 + } 1310 1310 + 1311 1311 + dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]); 1312 1312 + if (dst_icg) { 1313 1313 + dst_bidx = dst_icg + xt->sgl[0].size; 1314 1314 + } else if (xt->dst_inc) { 1315 1315 + dst_bidx = xt->sgl[0].size; 1316 1316 + } else { 1317 1317 + dev_err(dev, "%s: DST constant addressing is not supported\n", 1318 1318 + __func__); 1319 1319 + return NULL; 1320 1320 + } 1321 1321 + 1322 1322 + if (src_bidx > SZ_64K || dst_bidx > SZ_64K) 1323 1323 + return NULL; 1324 1324 + 1325 1325 + edesc = kzalloc(struct_size(edesc, pset, 1), GFP_ATOMIC); 1326 1326 + if (!edesc) 1327 1327 + return NULL; 1328 1328 + 1329 1329 + edesc->direction = DMA_MEM_TO_MEM; 1330 1330 + edesc->echan = echan; 1331 1331 + edesc->pset_nr = 1; 1332 1332 + 1333 1333 + param = &edesc->pset[0].param; 1334 1334 + 1335 1335 + param->src = xt->src_start; 1336 1336 + param->dst = xt->dst_start; 1337 1337 + param->a_b_cnt = xt->numf << 16 | xt->sgl[0].size; 1338 1338 + param->ccnt = 1; 1339 1339 + param->src_dst_bidx = (dst_bidx << 16) | src_bidx; 1340 1340 + param->src_dst_cidx = 0; 1341 1341 + 1342 1342 + param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num)); 1343 1343 + param->opt |= ITCCHEN; 1344 1344 + /* Enable transfer complete interrupt if requested */ 1345 1345 + if (tx_flags & DMA_PREP_INTERRUPT) 1346 1346 + param->opt |= TCINTEN; 1347 1347 + else 1348 1348 + edesc->polled = true; 1349 1349 + 1350 1350 + return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags); 1351 1351 + } 1352 1352 + 1278 1353 static struct dma_async_tx_descriptor *edma_prep_dma_cyclic( 1279 1354 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 1280 1355 size_t period_len, enum dma_transfer_direction direction, ··· 1992 1917 "Legacy memcpy is enabled, things might not work\n"); 1993 1918 1994 1919 dma_cap_set(DMA_MEMCPY, s_ddev->cap_mask); 1920 1920 + dma_cap_set(DMA_INTERLEAVE, s_ddev->cap_mask); 1995 1921 s_ddev->device_prep_dma_memcpy = edma_prep_dma_memcpy; 1922 1922 + s_ddev->device_prep_interleaved_dma = edma_prep_dma_interleaved; 1996 1923 s_ddev->directions = BIT(DMA_MEM_TO_MEM); 1997 1924 } 1998 1925 ··· 2030 1953 2031 1954 dma_cap_zero(m_ddev->cap_mask); 2032 1955 dma_cap_set(DMA_MEMCPY, m_ddev->cap_mask); 1956 1956 + dma_cap_set(DMA_INTERLEAVE, m_ddev->cap_mask); 2033 1957 2034 1958 m_ddev->device_prep_dma_memcpy = edma_prep_dma_memcpy; 1959 1959 + m_ddev->device_prep_interleaved_dma = edma_prep_dma_interleaved; 2035 1960 m_ddev->device_alloc_chan_resources = edma_alloc_chan_resources; 2036 1961 m_ddev->device_free_chan_resources = edma_free_chan_resources; 2037 1962 m_ddev->device_issue_pending = edma_issue_pending;

+16 -2

drivers/dma/ti/k3-udma-glue.c

reviewed

··· 32 32 bool epib; 33 33 u32 psdata_size; 34 34 u32 swdata_size; 35 35 + u32 atype; 35 36 }; 36 37 37 38 struct k3_udma_glue_tx_channel { ··· 122 121 return -ENOENT; 123 122 124 123 thread_id = dma_spec.args[0]; 124 124 + if (dma_spec.args_count == 2) { 125 125 + if (dma_spec.args[1] > 2) { 126 126 + dev_err(common->dev, "Invalid channel atype: %u\n", 127 127 + dma_spec.args[1]); 128 128 + ret = -EINVAL; 129 129 + goto out_put_spec; 130 130 + } 131 131 + common->atype = dma_spec.args[1]; 132 132 + } 125 133 126 134 if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) { 127 135 ret = -EINVAL; ··· 212 202 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | 213 203 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | 214 204 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | 215 215 - TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID; 205 205 + TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | 206 206 + TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID; 216 207 req.nav_id = tisci_rm->tisci_dev_id; 217 208 req.index = tx_chn->udma_tchan_id; 218 209 if (tx_chn->tx_pause_on_err) ··· 227 216 req.tx_supr_tdpkt = 1; 228 217 req.tx_fetch_size = tx_chn->common.hdesc_size >> 2; 229 218 req.txcq_qnum = k3_ringacc_get_ring_id(tx_chn->ringtxcq); 219 219 + req.tx_atype = tx_chn->common.atype; 230 220 231 221 return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req); 232 222 } ··· 514 502 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | 515 503 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | 516 504 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | 517 517 - TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID; 505 505 + TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID | 506 506 + TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID; 518 507 519 508 req.nav_id = tisci_rm->tisci_dev_id; 520 509 req.index = rx_chn->udma_rchan_id; ··· 532 519 req.flowid_cnt = rx_chn->flow_num; 533 520 } 534 521 req.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; 522 522 + req.rx_atype = rx_chn->common.atype; 535 523 536 524 ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req); 537 525 if (ret)

+105 -8

drivers/dma/ti/k3-udma.c

reviewed

··· 149 149 150 150 struct udma_chan *channels; 151 151 u32 psil_base; 152 152 + u32 atype; 152 153 }; 153 154 154 155 struct udma_desc { ··· 193 192 u32 hdesc_size; /* Size of a packet descriptor in packet mode */ 194 193 bool notdpkt; /* Suppress sending TDC packet */ 195 194 int remote_thread_id; 195 195 + u32 atype; 196 196 u32 src_thread; 197 197 u32 dst_thread; 198 198 enum psil_endpoint_type ep_type; ··· 1571 1569 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID | \ 1572 1570 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID | \ 1573 1571 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID | \ 1574 1574 - TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID) 1572 1572 + TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID | \ 1573 1573 + TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID) 1575 1574 1576 1575 #define TISCI_RCHAN_VALID_PARAMS ( \ 1577 1576 TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID | \ ··· 1582 1579 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID | \ 1583 1580 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID | \ 1584 1581 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID | \ 1585 1585 - TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID) 1582 1582 + TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID | \ 1583 1583 + TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID) 1586 1584 1587 1585 static int udma_tisci_m2m_channel_config(struct udma_chan *uc) 1588 1586 { ··· 1605 1601 req_tx.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; 1606 1602 req_tx.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; 1607 1603 req_tx.txcq_qnum = tc_ring; 1604 1604 + req_tx.tx_atype = ud->atype; 1608 1605 1609 1606 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); 1610 1607 if (ret) { ··· 1619 1614 req_rx.rx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; 1620 1615 req_rx.rxcq_qnum = tc_ring; 1621 1616 req_rx.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; 1617 1617 + req_rx.rx_atype = ud->atype; 1622 1618 1623 1619 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); 1624 1620 if (ret) ··· 1655 1649 req_tx.tx_supr_tdpkt = uc->config.notdpkt; 1656 1650 req_tx.tx_fetch_size = fetch_size >> 2; 1657 1651 req_tx.txcq_qnum = tc_ring; 1652 1652 + req_tx.tx_atype = uc->config.atype; 1658 1653 1659 1654 ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); 1660 1655 if (ret) ··· 1692 1685 req_rx.rx_fetch_size = fetch_size >> 2; 1693 1686 req_rx.rxcq_qnum = rx_ring; 1694 1687 req_rx.rx_chan_type = mode; 1688 1688 + req_rx.rx_atype = uc->config.atype; 1695 1689 1696 1690 ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); 1697 1691 if (ret) { ··· 3071 3063 3072 3064 static struct platform_driver udma_driver; 3073 3065 3066 3066 + struct udma_filter_param { 3067 3067 + int remote_thread_id; 3068 3068 + u32 atype; 3069 3069 + }; 3070 3070 + 3074 3071 static bool udma_dma_filter_fn(struct dma_chan *chan, void *param) 3075 3072 { 3076 3073 struct udma_chan_config *ucc; 3077 3074 struct psil_endpoint_config *ep_config; 3075 3075 + struct udma_filter_param *filter_param; 3078 3076 struct udma_chan *uc; 3079 3077 struct udma_dev *ud; 3080 3080 - u32 *args; 3081 3078 3082 3079 if (chan->device->dev->driver != &udma_driver.driver) 3083 3080 return false; ··· 3090 3077 uc = to_udma_chan(chan); 3091 3078 ucc = &uc->config; 3092 3079 ud = uc->ud; 3093 3093 - args = param; 3080 3080 + filter_param = param; 3094 3081 3095 3095 - ucc->remote_thread_id = args[0]; 3082 3082 + if (filter_param->atype > 2) { 3083 3083 + dev_err(ud->dev, "Invalid channel atype: %u\n", 3084 3084 + filter_param->atype); 3085 3085 + return false; 3086 3086 + } 3087 3087 + 3088 3088 + ucc->remote_thread_id = filter_param->remote_thread_id; 3089 3089 + ucc->atype = filter_param->atype; 3096 3090 3097 3091 if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET) 3098 3092 ucc->dir = DMA_MEM_TO_DEV; ··· 3112 3092 ucc->remote_thread_id); 3113 3093 ucc->dir = DMA_MEM_TO_MEM; 3114 3094 ucc->remote_thread_id = -1; 3095 3095 + ucc->atype = 0; 3115 3096 return false; 3116 3097 } 3117 3098 ··· 3151 3130 { 3152 3131 struct udma_dev *ud = ofdma->of_dma_data; 3153 3132 dma_cap_mask_t mask = ud->ddev.cap_mask; 3133 3133 + struct udma_filter_param filter_param; 3154 3134 struct dma_chan *chan; 3155 3135 3156 3156 - if (dma_spec->args_count != 1) 3136 3136 + if (dma_spec->args_count != 1 && dma_spec->args_count != 2) 3157 3137 return NULL; 3158 3138 3159 3159 - chan = __dma_request_channel(&mask, udma_dma_filter_fn, 3160 3160 - &dma_spec->args[0], ofdma->of_node); 3139 3139 + filter_param.remote_thread_id = dma_spec->args[0]; 3140 3140 + if (dma_spec->args_count == 2) 3141 3141 + filter_param.atype = dma_spec->args[1]; 3142 3142 + else 3143 3143 + filter_param.atype = 0; 3144 3144 + 3145 3145 + chan = __dma_request_channel(&mask, udma_dma_filter_fn, &filter_param, 3146 3146 + ofdma->of_node); 3161 3147 if (!chan) { 3162 3148 dev_err(ud->dev, "get channel fail in %s.\n", __func__); 3163 3149 return ERR_PTR(-EINVAL); ··· 3501 3473 return 0; 3502 3474 } 3503 3475 3476 3476 + #ifdef CONFIG_DEBUG_FS 3477 3477 + static void udma_dbg_summary_show_chan(struct seq_file *s, 3478 3478 + struct dma_chan *chan) 3479 3479 + { 3480 3480 + struct udma_chan *uc = to_udma_chan(chan); 3481 3481 + struct udma_chan_config *ucc = &uc->config; 3482 3482 + 3483 3483 + seq_printf(s, " %-13s| %s", dma_chan_name(chan), 3484 3484 + chan->dbg_client_name ?: "in-use"); 3485 3485 + seq_printf(s, " (%s, ", dmaengine_get_direction_text(uc->config.dir)); 3486 3486 + 3487 3487 + switch (uc->config.dir) { 3488 3488 + case DMA_MEM_TO_MEM: 3489 3489 + seq_printf(s, "chan%d pair [0x%04x -> 0x%04x], ", uc->tchan->id, 3490 3490 + ucc->src_thread, ucc->dst_thread); 3491 3491 + break; 3492 3492 + case DMA_DEV_TO_MEM: 3493 3493 + seq_printf(s, "rchan%d [0x%04x -> 0x%04x], ", uc->rchan->id, 3494 3494 + ucc->src_thread, ucc->dst_thread); 3495 3495 + break; 3496 3496 + case DMA_MEM_TO_DEV: 3497 3497 + seq_printf(s, "tchan%d [0x%04x -> 0x%04x], ", uc->tchan->id, 3498 3498 + ucc->src_thread, ucc->dst_thread); 3499 3499 + break; 3500 3500 + default: 3501 3501 + seq_printf(s, ")\n"); 3502 3502 + return; 3503 3503 + } 3504 3504 + 3505 3505 + if (ucc->ep_type == PSIL_EP_NATIVE) { 3506 3506 + seq_printf(s, "PSI-L Native"); 3507 3507 + if (ucc->metadata_size) { 3508 3508 + seq_printf(s, "[%s", ucc->needs_epib ? " EPIB" : ""); 3509 3509 + if (ucc->psd_size) 3510 3510 + seq_printf(s, " PSDsize:%u", ucc->psd_size); 3511 3511 + seq_printf(s, " ]"); 3512 3512 + } 3513 3513 + } else { 3514 3514 + seq_printf(s, "PDMA"); 3515 3515 + if (ucc->enable_acc32 || ucc->enable_burst) 3516 3516 + seq_printf(s, "[%s%s ]", 3517 3517 + ucc->enable_acc32 ? " ACC32" : "", 3518 3518 + ucc->enable_burst ? " BURST" : ""); 3519 3519 + } 3520 3520 + 3521 3521 + seq_printf(s, ", %s)\n", ucc->pkt_mode ? "Packet mode" : "TR mode"); 3522 3522 + } 3523 3523 + 3524 3524 + static void udma_dbg_summary_show(struct seq_file *s, 3525 3525 + struct dma_device *dma_dev) 3526 3526 + { 3527 3527 + struct dma_chan *chan; 3528 3528 + 3529 3529 + list_for_each_entry(chan, &dma_dev->channels, device_node) { 3530 3530 + if (chan->client_count) 3531 3531 + udma_dbg_summary_show_chan(s, chan); 3532 3532 + } 3533 3533 + } 3534 3534 + #endif /* CONFIG_DEBUG_FS */ 3535 3535 + 3504 3536 #define TI_UDMAC_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 3505 3537 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 3506 3538 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ ··· 3607 3519 return ret; 3608 3520 } 3609 3521 3522 3522 + ret = of_property_read_u32(navss_node, "ti,udma-atype", &ud->atype); 3523 3523 + if (!ret && ud->atype > 2) { 3524 3524 + dev_err(dev, "Invalid atype: %u\n", ud->atype); 3525 3525 + return -EINVAL; 3526 3526 + } 3527 3527 + 3610 3528 ud->tisci_rm.tisci_udmap_ops = &ud->tisci_rm.tisci->ops.rm_udmap_ops; 3611 3529 ud->tisci_rm.tisci_psil_ops = &ud->tisci_rm.tisci->ops.rm_psil_ops; 3612 3530 ··· 3647 3553 ud->ddev.device_resume = udma_resume; 3648 3554 ud->ddev.device_terminate_all = udma_terminate_all; 3649 3555 ud->ddev.device_synchronize = udma_synchronize; 3556 3556 + #ifdef CONFIG_DEBUG_FS 3557 3557 + ud->ddev.dbg_summary_show = udma_dbg_summary_show; 3558 3558 + #endif 3650 3559 3651 3560 ud->ddev.device_free_chan_resources = udma_free_chan_resources; 3652 3561 ud->ddev.src_addr_widths = TI_UDMAC_BUSWIDTHS;

+1 -1

drivers/dma/ti/omap-dma.c

reviewed

··· 124 124 uint32_t csdp; /* CSDP value */ 125 125 126 126 unsigned sglen; 127 127 - struct omap_sg sg[0]; 127 127 + struct omap_sg sg[]; 128 128 }; 129 129 130 130 enum {

+1 -1

drivers/dma/uniphier-mdmac.c

reviewed

··· 68 68 struct dma_device ddev; 69 69 struct clk *clk; 70 70 void __iomem *reg_base; 71 71 - struct uniphier_mdmac_chan channels[0]; 71 71 + struct uniphier_mdmac_chan channels[]; 72 72 }; 73 73 74 74 static struct uniphier_mdmac_chan *

+609

drivers/dma/uniphier-xdmac.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* 3 3 + * External DMA controller driver for UniPhier SoCs 4 4 + * Copyright 2019 Socionext Inc. 5 5 + * Author: Kunihiko Hayashi <hayashi.kunihiko@socionext.com> 6 6 + */ 7 7 + 8 8 + #include <linux/bitops.h> 9 9 + #include <linux/bitfield.h> 10 10 + #include <linux/iopoll.h> 11 11 + #include <linux/module.h> 12 12 + #include <linux/of.h> 13 13 + #include <linux/of_dma.h> 14 14 + #include <linux/platform_device.h> 15 15 + 16 16 + #include "dmaengine.h" 17 17 + #include "virt-dma.h" 18 18 + 19 19 + #define XDMAC_CH_WIDTH 0x100 20 20 + 21 21 + #define XDMAC_TFA 0x08 22 22 + #define XDMAC_TFA_MCNT_MASK GENMASK(23, 16) 23 23 + #define XDMAC_TFA_MASK GENMASK(5, 0) 24 24 + #define XDMAC_SADM 0x10 25 25 + #define XDMAC_SADM_STW_MASK GENMASK(25, 24) 26 26 + #define XDMAC_SADM_SAM BIT(4) 27 27 + #define XDMAC_SADM_SAM_FIXED XDMAC_SADM_SAM 28 28 + #define XDMAC_SADM_SAM_INC 0 29 29 + #define XDMAC_DADM 0x14 30 30 + #define XDMAC_DADM_DTW_MASK XDMAC_SADM_STW_MASK 31 31 + #define XDMAC_DADM_DAM XDMAC_SADM_SAM 32 32 + #define XDMAC_DADM_DAM_FIXED XDMAC_SADM_SAM_FIXED 33 33 + #define XDMAC_DADM_DAM_INC XDMAC_SADM_SAM_INC 34 34 + #define XDMAC_EXSAD 0x18 35 35 + #define XDMAC_EXDAD 0x1c 36 36 + #define XDMAC_SAD 0x20 37 37 + #define XDMAC_DAD 0x24 38 38 + #define XDMAC_ITS 0x28 39 39 + #define XDMAC_ITS_MASK GENMASK(25, 0) 40 40 + #define XDMAC_TNUM 0x2c 41 41 + #define XDMAC_TNUM_MASK GENMASK(15, 0) 42 42 + #define XDMAC_TSS 0x30 43 43 + #define XDMAC_TSS_REQ BIT(0) 44 44 + #define XDMAC_IEN 0x34 45 45 + #define XDMAC_IEN_ERRIEN BIT(1) 46 46 + #define XDMAC_IEN_ENDIEN BIT(0) 47 47 + #define XDMAC_STAT 0x40 48 48 + #define XDMAC_STAT_TENF BIT(0) 49 49 + #define XDMAC_IR 0x44 50 50 + #define XDMAC_IR_ERRF BIT(1) 51 51 + #define XDMAC_IR_ENDF BIT(0) 52 52 + #define XDMAC_ID 0x48 53 53 + #define XDMAC_ID_ERRIDF BIT(1) 54 54 + #define XDMAC_ID_ENDIDF BIT(0) 55 55 + 56 56 + #define XDMAC_MAX_CHANS 16 57 57 + #define XDMAC_INTERVAL_CLKS 20 58 58 + #define XDMAC_MAX_WORDS XDMAC_TNUM_MASK 59 59 + 60 60 + /* cut lower bit for maintain alignment of maximum transfer size */ 61 61 + #define XDMAC_MAX_WORD_SIZE (XDMAC_ITS_MASK & ~GENMASK(3, 0)) 62 62 + 63 63 + #define UNIPHIER_XDMAC_BUSWIDTHS \ 64 64 + (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 65 65 + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 66 66 + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ 67 67 + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) 68 68 + 69 69 + struct uniphier_xdmac_desc_node { 70 70 + dma_addr_t src; 71 71 + dma_addr_t dst; 72 72 + u32 burst_size; 73 73 + u32 nr_burst; 74 74 + }; 75 75 + 76 76 + struct uniphier_xdmac_desc { 77 77 + struct virt_dma_desc vd; 78 78 + 79 79 + unsigned int nr_node; 80 80 + unsigned int cur_node; 81 81 + enum dma_transfer_direction dir; 82 82 + struct uniphier_xdmac_desc_node nodes[]; 83 83 + }; 84 84 + 85 85 + struct uniphier_xdmac_chan { 86 86 + struct virt_dma_chan vc; 87 87 + struct uniphier_xdmac_device *xdev; 88 88 + struct uniphier_xdmac_desc *xd; 89 89 + void __iomem *reg_ch_base; 90 90 + struct dma_slave_config sconfig; 91 91 + int id; 92 92 + unsigned int req_factor; 93 93 + }; 94 94 + 95 95 + struct uniphier_xdmac_device { 96 96 + struct dma_device ddev; 97 97 + void __iomem *reg_base; 98 98 + int nr_chans; 99 99 + struct uniphier_xdmac_chan channels[]; 100 100 + }; 101 101 + 102 102 + static struct uniphier_xdmac_chan * 103 103 + to_uniphier_xdmac_chan(struct virt_dma_chan *vc) 104 104 + { 105 105 + return container_of(vc, struct uniphier_xdmac_chan, vc); 106 106 + } 107 107 + 108 108 + static struct uniphier_xdmac_desc * 109 109 + to_uniphier_xdmac_desc(struct virt_dma_desc *vd) 110 110 + { 111 111 + return container_of(vd, struct uniphier_xdmac_desc, vd); 112 112 + } 113 113 + 114 114 + /* xc->vc.lock must be held by caller */ 115 115 + static struct uniphier_xdmac_desc * 116 116 + uniphier_xdmac_next_desc(struct uniphier_xdmac_chan *xc) 117 117 + { 118 118 + struct virt_dma_desc *vd; 119 119 + 120 120 + vd = vchan_next_desc(&xc->vc); 121 121 + if (!vd) 122 122 + return NULL; 123 123 + 124 124 + list_del(&vd->node); 125 125 + 126 126 + return to_uniphier_xdmac_desc(vd); 127 127 + } 128 128 + 129 129 + /* xc->vc.lock must be held by caller */ 130 130 + static void uniphier_xdmac_chan_start(struct uniphier_xdmac_chan *xc, 131 131 + struct uniphier_xdmac_desc *xd) 132 132 + { 133 133 + u32 src_mode, src_addr, src_width; 134 134 + u32 dst_mode, dst_addr, dst_width; 135 135 + u32 val, its, tnum; 136 136 + enum dma_slave_buswidth buswidth; 137 137 + 138 138 + src_addr = xd->nodes[xd->cur_node].src; 139 139 + dst_addr = xd->nodes[xd->cur_node].dst; 140 140 + its = xd->nodes[xd->cur_node].burst_size; 141 141 + tnum = xd->nodes[xd->cur_node].nr_burst; 142 142 + 143 143 + /* 144 144 + * The width of MEM side must be 4 or 8 bytes, that does not 145 145 + * affect that of DEV side and transfer size. 146 146 + */ 147 147 + if (xd->dir == DMA_DEV_TO_MEM) { 148 148 + src_mode = XDMAC_SADM_SAM_FIXED; 149 149 + buswidth = xc->sconfig.src_addr_width; 150 150 + } else { 151 151 + src_mode = XDMAC_SADM_SAM_INC; 152 152 + buswidth = DMA_SLAVE_BUSWIDTH_8_BYTES; 153 153 + } 154 154 + src_width = FIELD_PREP(XDMAC_SADM_STW_MASK, __ffs(buswidth)); 155 155 + 156 156 + if (xd->dir == DMA_MEM_TO_DEV) { 157 157 + dst_mode = XDMAC_DADM_DAM_FIXED; 158 158 + buswidth = xc->sconfig.dst_addr_width; 159 159 + } else { 160 160 + dst_mode = XDMAC_DADM_DAM_INC; 161 161 + buswidth = DMA_SLAVE_BUSWIDTH_8_BYTES; 162 162 + } 163 163 + dst_width = FIELD_PREP(XDMAC_DADM_DTW_MASK, __ffs(buswidth)); 164 164 + 165 165 + /* setup transfer factor */ 166 166 + val = FIELD_PREP(XDMAC_TFA_MCNT_MASK, XDMAC_INTERVAL_CLKS); 167 167 + val |= FIELD_PREP(XDMAC_TFA_MASK, xc->req_factor); 168 168 + writel(val, xc->reg_ch_base + XDMAC_TFA); 169 169 + 170 170 + /* setup the channel */ 171 171 + writel(lower_32_bits(src_addr), xc->reg_ch_base + XDMAC_SAD); 172 172 + writel(upper_32_bits(src_addr), xc->reg_ch_base + XDMAC_EXSAD); 173 173 + 174 174 + writel(lower_32_bits(dst_addr), xc->reg_ch_base + XDMAC_DAD); 175 175 + writel(upper_32_bits(dst_addr), xc->reg_ch_base + XDMAC_EXDAD); 176 176 + 177 177 + src_mode |= src_width; 178 178 + dst_mode |= dst_width; 179 179 + writel(src_mode, xc->reg_ch_base + XDMAC_SADM); 180 180 + writel(dst_mode, xc->reg_ch_base + XDMAC_DADM); 181 181 + 182 182 + writel(its, xc->reg_ch_base + XDMAC_ITS); 183 183 + writel(tnum, xc->reg_ch_base + XDMAC_TNUM); 184 184 + 185 185 + /* enable interrupt */ 186 186 + writel(XDMAC_IEN_ENDIEN | XDMAC_IEN_ERRIEN, 187 187 + xc->reg_ch_base + XDMAC_IEN); 188 188 + 189 189 + /* start XDMAC */ 190 190 + val = readl(xc->reg_ch_base + XDMAC_TSS); 191 191 + val |= XDMAC_TSS_REQ; 192 192 + writel(val, xc->reg_ch_base + XDMAC_TSS); 193 193 + } 194 194 + 195 195 + /* xc->vc.lock must be held by caller */ 196 196 + static int uniphier_xdmac_chan_stop(struct uniphier_xdmac_chan *xc) 197 197 + { 198 198 + u32 val; 199 199 + 200 200 + /* disable interrupt */ 201 201 + val = readl(xc->reg_ch_base + XDMAC_IEN); 202 202 + val &= ~(XDMAC_IEN_ENDIEN | XDMAC_IEN_ERRIEN); 203 203 + writel(val, xc->reg_ch_base + XDMAC_IEN); 204 204 + 205 205 + /* stop XDMAC */ 206 206 + val = readl(xc->reg_ch_base + XDMAC_TSS); 207 207 + val &= ~XDMAC_TSS_REQ; 208 208 + writel(0, xc->reg_ch_base + XDMAC_TSS); 209 209 + 210 210 + /* wait until transfer is stopped */ 211 211 + return readl_poll_timeout(xc->reg_ch_base + XDMAC_STAT, val, 212 212 + !(val & XDMAC_STAT_TENF), 100, 1000); 213 213 + } 214 214 + 215 215 + /* xc->vc.lock must be held by caller */ 216 216 + static void uniphier_xdmac_start(struct uniphier_xdmac_chan *xc) 217 217 + { 218 218 + struct uniphier_xdmac_desc *xd; 219 219 + 220 220 + xd = uniphier_xdmac_next_desc(xc); 221 221 + if (xd) 222 222 + uniphier_xdmac_chan_start(xc, xd); 223 223 + 224 224 + /* set desc to chan regardless of xd is null */ 225 225 + xc->xd = xd; 226 226 + } 227 227 + 228 228 + static void uniphier_xdmac_chan_irq(struct uniphier_xdmac_chan *xc) 229 229 + { 230 230 + u32 stat; 231 231 + int ret; 232 232 + 233 233 + spin_lock(&xc->vc.lock); 234 234 + 235 235 + stat = readl(xc->reg_ch_base + XDMAC_ID); 236 236 + 237 237 + if (stat & XDMAC_ID_ERRIDF) { 238 238 + ret = uniphier_xdmac_chan_stop(xc); 239 239 + if (ret) 240 240 + dev_err(xc->xdev->ddev.dev, 241 241 + "DMA transfer error with aborting issue\n"); 242 242 + else 243 243 + dev_err(xc->xdev->ddev.dev, 244 244 + "DMA transfer error\n"); 245 245 + 246 246 + } else if ((stat & XDMAC_ID_ENDIDF) && xc->xd) { 247 247 + xc->xd->cur_node++; 248 248 + if (xc->xd->cur_node >= xc->xd->nr_node) { 249 249 + vchan_cookie_complete(&xc->xd->vd); 250 250 + uniphier_xdmac_start(xc); 251 251 + } else { 252 252 + uniphier_xdmac_chan_start(xc, xc->xd); 253 253 + } 254 254 + } 255 255 + 256 256 + /* write bits to clear */ 257 257 + writel(stat, xc->reg_ch_base + XDMAC_IR); 258 258 + 259 259 + spin_unlock(&xc->vc.lock); 260 260 + } 261 261 + 262 262 + static irqreturn_t uniphier_xdmac_irq_handler(int irq, void *dev_id) 263 263 + { 264 264 + struct uniphier_xdmac_device *xdev = dev_id; 265 265 + int i; 266 266 + 267 267 + for (i = 0; i < xdev->nr_chans; i++) 268 268 + uniphier_xdmac_chan_irq(&xdev->channels[i]); 269 269 + 270 270 + return IRQ_HANDLED; 271 271 + } 272 272 + 273 273 + static void uniphier_xdmac_free_chan_resources(struct dma_chan *chan) 274 274 + { 275 275 + vchan_free_chan_resources(to_virt_chan(chan)); 276 276 + } 277 277 + 278 278 + static struct dma_async_tx_descriptor * 279 279 + uniphier_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst, 280 280 + dma_addr_t src, size_t len, unsigned long flags) 281 281 + { 282 282 + struct virt_dma_chan *vc = to_virt_chan(chan); 283 283 + struct uniphier_xdmac_desc *xd; 284 284 + unsigned int nr; 285 285 + size_t burst_size, tlen; 286 286 + int i; 287 287 + 288 288 + if (len > XDMAC_MAX_WORD_SIZE * XDMAC_MAX_WORDS) 289 289 + return NULL; 290 290 + 291 291 + nr = 1 + len / XDMAC_MAX_WORD_SIZE; 292 292 + 293 293 + xd = kzalloc(struct_size(xd, nodes, nr), GFP_NOWAIT); 294 294 + if (!xd) 295 295 + return NULL; 296 296 + 297 297 + for (i = 0; i < nr; i++) { 298 298 + burst_size = min_t(size_t, len, XDMAC_MAX_WORD_SIZE); 299 299 + xd->nodes[i].src = src; 300 300 + xd->nodes[i].dst = dst; 301 301 + xd->nodes[i].burst_size = burst_size; 302 302 + xd->nodes[i].nr_burst = len / burst_size; 303 303 + tlen = rounddown(len, burst_size); 304 304 + src += tlen; 305 305 + dst += tlen; 306 306 + len -= tlen; 307 307 + } 308 308 + 309 309 + xd->dir = DMA_MEM_TO_MEM; 310 310 + xd->nr_node = nr; 311 311 + xd->cur_node = 0; 312 312 + 313 313 + return vchan_tx_prep(vc, &xd->vd, flags); 314 314 + } 315 315 + 316 316 + static struct dma_async_tx_descriptor * 317 317 + uniphier_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 318 318 + unsigned int sg_len, 319 319 + enum dma_transfer_direction direction, 320 320 + unsigned long flags, void *context) 321 321 + { 322 322 + struct virt_dma_chan *vc = to_virt_chan(chan); 323 323 + struct uniphier_xdmac_chan *xc = to_uniphier_xdmac_chan(vc); 324 324 + struct uniphier_xdmac_desc *xd; 325 325 + struct scatterlist *sg; 326 326 + enum dma_slave_buswidth buswidth; 327 327 + u32 maxburst; 328 328 + int i; 329 329 + 330 330 + if (!is_slave_direction(direction)) 331 331 + return NULL; 332 332 + 333 333 + if (direction == DMA_DEV_TO_MEM) { 334 334 + buswidth = xc->sconfig.src_addr_width; 335 335 + maxburst = xc->sconfig.src_maxburst; 336 336 + } else { 337 337 + buswidth = xc->sconfig.dst_addr_width; 338 338 + maxburst = xc->sconfig.dst_maxburst; 339 339 + } 340 340 + 341 341 + if (!maxburst) 342 342 + maxburst = 1; 343 343 + if (maxburst > xc->xdev->ddev.max_burst) { 344 344 + dev_err(xc->xdev->ddev.dev, 345 345 + "Exceed maximum number of burst words\n"); 346 346 + return NULL; 347 347 + } 348 348 + 349 349 + xd = kzalloc(struct_size(xd, nodes, sg_len), GFP_NOWAIT); 350 350 + if (!xd) 351 351 + return NULL; 352 352 + 353 353 + for_each_sg(sgl, sg, sg_len, i) { 354 354 + xd->nodes[i].src = (direction == DMA_DEV_TO_MEM) 355 355 + ? xc->sconfig.src_addr : sg_dma_address(sg); 356 356 + xd->nodes[i].dst = (direction == DMA_MEM_TO_DEV) 357 357 + ? xc->sconfig.dst_addr : sg_dma_address(sg); 358 358 + xd->nodes[i].burst_size = maxburst * buswidth; 359 359 + xd->nodes[i].nr_burst = 360 360 + sg_dma_len(sg) / xd->nodes[i].burst_size; 361 361 + 362 362 + /* 363 363 + * Currently transfer that size doesn't align the unit size 364 364 + * (the number of burst words * bus-width) is not allowed, 365 365 + * because the driver does not support the way to transfer 366 366 + * residue size. As a matter of fact, in order to transfer 367 367 + * arbitrary size, 'src_maxburst' or 'dst_maxburst' of 368 368 + * dma_slave_config must be 1. 369 369 + */ 370 370 + if (sg_dma_len(sg) % xd->nodes[i].burst_size) { 371 371 + dev_err(xc->xdev->ddev.dev, 372 372 + "Unaligned transfer size: %d", sg_dma_len(sg)); 373 373 + kfree(xd); 374 374 + return NULL; 375 375 + } 376 376 + 377 377 + if (xd->nodes[i].nr_burst > XDMAC_MAX_WORDS) { 378 378 + dev_err(xc->xdev->ddev.dev, 379 379 + "Exceed maximum transfer size"); 380 380 + kfree(xd); 381 381 + return NULL; 382 382 + } 383 383 + } 384 384 + 385 385 + xd->dir = direction; 386 386 + xd->nr_node = sg_len; 387 387 + xd->cur_node = 0; 388 388 + 389 389 + return vchan_tx_prep(vc, &xd->vd, flags); 390 390 + } 391 391 + 392 392 + static int uniphier_xdmac_slave_config(struct dma_chan *chan, 393 393 + struct dma_slave_config *config) 394 394 + { 395 395 + struct virt_dma_chan *vc = to_virt_chan(chan); 396 396 + struct uniphier_xdmac_chan *xc = to_uniphier_xdmac_chan(vc); 397 397 + 398 398 + memcpy(&xc->sconfig, config, sizeof(*config)); 399 399 + 400 400 + return 0; 401 401 + } 402 402 + 403 403 + static int uniphier_xdmac_terminate_all(struct dma_chan *chan) 404 404 + { 405 405 + struct virt_dma_chan *vc = to_virt_chan(chan); 406 406 + struct uniphier_xdmac_chan *xc = to_uniphier_xdmac_chan(vc); 407 407 + unsigned long flags; 408 408 + int ret = 0; 409 409 + LIST_HEAD(head); 410 410 + 411 411 + spin_lock_irqsave(&vc->lock, flags); 412 412 + 413 413 + if (xc->xd) { 414 414 + vchan_terminate_vdesc(&xc->xd->vd); 415 415 + xc->xd = NULL; 416 416 + ret = uniphier_xdmac_chan_stop(xc); 417 417 + } 418 418 + 419 419 + vchan_get_all_descriptors(vc, &head); 420 420 + 421 421 + spin_unlock_irqrestore(&vc->lock, flags); 422 422 + 423 423 + vchan_dma_desc_free_list(vc, &head); 424 424 + 425 425 + return ret; 426 426 + } 427 427 + 428 428 + static void uniphier_xdmac_synchronize(struct dma_chan *chan) 429 429 + { 430 430 + vchan_synchronize(to_virt_chan(chan)); 431 431 + } 432 432 + 433 433 + static void uniphier_xdmac_issue_pending(struct dma_chan *chan) 434 434 + { 435 435 + struct virt_dma_chan *vc = to_virt_chan(chan); 436 436 + struct uniphier_xdmac_chan *xc = to_uniphier_xdmac_chan(vc); 437 437 + unsigned long flags; 438 438 + 439 439 + spin_lock_irqsave(&vc->lock, flags); 440 440 + 441 441 + if (vchan_issue_pending(vc) && !xc->xd) 442 442 + uniphier_xdmac_start(xc); 443 443 + 444 444 + spin_unlock_irqrestore(&vc->lock, flags); 445 445 + } 446 446 + 447 447 + static void uniphier_xdmac_desc_free(struct virt_dma_desc *vd) 448 448 + { 449 449 + kfree(to_uniphier_xdmac_desc(vd)); 450 450 + } 451 451 + 452 452 + static void uniphier_xdmac_chan_init(struct uniphier_xdmac_device *xdev, 453 453 + int ch) 454 454 + { 455 455 + struct uniphier_xdmac_chan *xc = &xdev->channels[ch]; 456 456 + 457 457 + xc->xdev = xdev; 458 458 + xc->reg_ch_base = xdev->reg_base + XDMAC_CH_WIDTH * ch; 459 459 + xc->vc.desc_free = uniphier_xdmac_desc_free; 460 460 + 461 461 + vchan_init(&xc->vc, &xdev->ddev); 462 462 + } 463 463 + 464 464 + static struct dma_chan *of_dma_uniphier_xlate(struct of_phandle_args *dma_spec, 465 465 + struct of_dma *ofdma) 466 466 + { 467 467 + struct uniphier_xdmac_device *xdev = ofdma->of_dma_data; 468 468 + int chan_id = dma_spec->args[0]; 469 469 + 470 470 + if (chan_id >= xdev->nr_chans) 471 471 + return NULL; 472 472 + 473 473 + xdev->channels[chan_id].id = chan_id; 474 474 + xdev->channels[chan_id].req_factor = dma_spec->args[1]; 475 475 + 476 476 + return dma_get_slave_channel(&xdev->channels[chan_id].vc.chan); 477 477 + } 478 478 + 479 479 + static int uniphier_xdmac_probe(struct platform_device *pdev) 480 480 + { 481 481 + struct uniphier_xdmac_device *xdev; 482 482 + struct device *dev = &pdev->dev; 483 483 + struct dma_device *ddev; 484 484 + int irq; 485 485 + int nr_chans; 486 486 + int i, ret; 487 487 + 488 488 + if (of_property_read_u32(dev->of_node, "dma-channels", &nr_chans)) 489 489 + return -EINVAL; 490 490 + if (nr_chans > XDMAC_MAX_CHANS) 491 491 + nr_chans = XDMAC_MAX_CHANS; 492 492 + 493 493 + xdev = devm_kzalloc(dev, struct_size(xdev, channels, nr_chans), 494 494 + GFP_KERNEL); 495 495 + if (!xdev) 496 496 + return -ENOMEM; 497 497 + 498 498 + xdev->nr_chans = nr_chans; 499 499 + xdev->reg_base = devm_platform_ioremap_resource(pdev, 0); 500 500 + if (IS_ERR(xdev->reg_base)) 501 501 + return PTR_ERR(xdev->reg_base); 502 502 + 503 503 + ddev = &xdev->ddev; 504 504 + ddev->dev = dev; 505 505 + dma_cap_zero(ddev->cap_mask); 506 506 + dma_cap_set(DMA_MEMCPY, ddev->cap_mask); 507 507 + dma_cap_set(DMA_SLAVE, ddev->cap_mask); 508 508 + ddev->src_addr_widths = UNIPHIER_XDMAC_BUSWIDTHS; 509 509 + ddev->dst_addr_widths = UNIPHIER_XDMAC_BUSWIDTHS; 510 510 + ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) | 511 511 + BIT(DMA_MEM_TO_MEM); 512 512 + ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 513 513 + ddev->max_burst = XDMAC_MAX_WORDS; 514 514 + ddev->device_free_chan_resources = uniphier_xdmac_free_chan_resources; 515 515 + ddev->device_prep_dma_memcpy = uniphier_xdmac_prep_dma_memcpy; 516 516 + ddev->device_prep_slave_sg = uniphier_xdmac_prep_slave_sg; 517 517 + ddev->device_config = uniphier_xdmac_slave_config; 518 518 + ddev->device_terminate_all = uniphier_xdmac_terminate_all; 519 519 + ddev->device_synchronize = uniphier_xdmac_synchronize; 520 520 + ddev->device_tx_status = dma_cookie_status; 521 521 + ddev->device_issue_pending = uniphier_xdmac_issue_pending; 522 522 + INIT_LIST_HEAD(&ddev->channels); 523 523 + 524 524 + for (i = 0; i < nr_chans; i++) 525 525 + uniphier_xdmac_chan_init(xdev, i); 526 526 + 527 527 + irq = platform_get_irq(pdev, 0); 528 528 + if (irq < 0) 529 529 + return irq; 530 530 + 531 531 + ret = devm_request_irq(dev, irq, uniphier_xdmac_irq_handler, 532 532 + IRQF_SHARED, "xdmac", xdev); 533 533 + if (ret) { 534 534 + dev_err(dev, "Failed to request IRQ\n"); 535 535 + return ret; 536 536 + } 537 537 + 538 538 + ret = dma_async_device_register(ddev); 539 539 + if (ret) { 540 540 + dev_err(dev, "Failed to register XDMA device\n"); 541 541 + return ret; 542 542 + } 543 543 + 544 544 + ret = of_dma_controller_register(dev->of_node, 545 545 + of_dma_uniphier_xlate, xdev); 546 546 + if (ret) { 547 547 + dev_err(dev, "Failed to register XDMA controller\n"); 548 548 + goto out_unregister_dmac; 549 549 + } 550 550 + 551 551 + platform_set_drvdata(pdev, xdev); 552 552 + 553 553 + dev_info(&pdev->dev, "UniPhier XDMAC driver (%d channels)\n", 554 554 + nr_chans); 555 555 + 556 556 + return 0; 557 557 + 558 558 + out_unregister_dmac: 559 559 + dma_async_device_unregister(ddev); 560 560 + 561 561 + return ret; 562 562 + } 563 563 + 564 564 + static int uniphier_xdmac_remove(struct platform_device *pdev) 565 565 + { 566 566 + struct uniphier_xdmac_device *xdev = platform_get_drvdata(pdev); 567 567 + struct dma_device *ddev = &xdev->ddev; 568 568 + struct dma_chan *chan; 569 569 + int ret; 570 570 + 571 571 + /* 572 572 + * Before reaching here, almost all descriptors have been freed by the 573 573 + * ->device_free_chan_resources() hook. However, each channel might 574 574 + * be still holding one descriptor that was on-flight at that moment. 575 575 + * Terminate it to make sure this hardware is no longer running. Then, 576 576 + * free the channel resources once again to avoid memory leak. 577 577 + */ 578 578 + list_for_each_entry(chan, &ddev->channels, device_node) { 579 579 + ret = dmaengine_terminate_sync(chan); 580 580 + if (ret) 581 581 + return ret; 582 582 + uniphier_xdmac_free_chan_resources(chan); 583 583 + } 584 584 + 585 585 + of_dma_controller_free(pdev->dev.of_node); 586 586 + dma_async_device_unregister(ddev); 587 587 + 588 588 + return 0; 589 589 + } 590 590 + 591 591 + static const struct of_device_id uniphier_xdmac_match[] = { 592 592 + { .compatible = "socionext,uniphier-xdmac" }, 593 593 + { /* sentinel */ } 594 594 + }; 595 595 + MODULE_DEVICE_TABLE(of, uniphier_xdmac_match); 596 596 + 597 597 + static struct platform_driver uniphier_xdmac_driver = { 598 598 + .probe = uniphier_xdmac_probe, 599 599 + .remove = uniphier_xdmac_remove, 600 600 + .driver = { 601 601 + .name = "uniphier-xdmac", 602 602 + .of_match_table = uniphier_xdmac_match, 603 603 + }, 604 604 + }; 605 605 + module_platform_driver(uniphier_xdmac_driver); 606 606 + 607 607 + MODULE_AUTHOR("Kunihiko Hayashi <hayashi.kunihiko@socionext.com>"); 608 608 + MODULE_DESCRIPTION("UniPhier external DMA controller driver"); 609 609 + MODULE_LICENSE("GPL v2");

+34 -31

drivers/dma/xilinx/xilinx_dma.c

reviewed

··· 125 125 #define XILINX_VDMA_ENABLE_VERTICAL_FLIP BIT(0) 126 126 127 127 /* HW specific definitions */ 128 128 - #define XILINX_DMA_MAX_CHANS_PER_DEVICE 0x20 128 128 + #define XILINX_MCDMA_MAX_CHANS_PER_DEVICE 0x20 129 129 + #define XILINX_DMA_MAX_CHANS_PER_DEVICE 0x2 130 130 + #define XILINX_CDMA_MAX_CHANS_PER_DEVICE 0x1 129 131 130 132 #define XILINX_DMA_DMAXR_ALL_IRQ_MASK \ 131 133 (XILINX_DMA_DMASR_FRM_CNT_IRQ | \ ··· 470 468 struct clk **tx_clk, struct clk **txs_clk, 471 469 struct clk **rx_clk, struct clk **rxs_clk); 472 470 irqreturn_t (*irq_handler)(int irq, void *data); 471 471 + const int max_channels; 473 472 }; 474 473 475 474 /** ··· 488 485 * @txs_clk: DMA mm2s stream clock 489 486 * @rx_clk: DMA s2mm clock 490 487 * @rxs_clk: DMA s2mm stream clock 491 491 - * @nr_channels: Number of channels DMA device supports 492 492 - * @chan_id: DMA channel identifier 488 488 + * @s2mm_chan_id: DMA s2mm channel identifier 489 489 + * @mm2s_chan_id: DMA mm2s channel identifier 493 490 * @max_buffer_len: Max buffer length 494 494 - * @s2mm_index: S2MM channel index 495 491 */ 496 492 struct xilinx_dma_device { 497 493 void __iomem *regs; 498 494 struct device *dev; 499 495 struct dma_device common; 500 500 - struct xilinx_dma_chan *chan[XILINX_DMA_MAX_CHANS_PER_DEVICE]; 496 496 + struct xilinx_dma_chan *chan[XILINX_MCDMA_MAX_CHANS_PER_DEVICE]; 501 497 u32 flush_on_fsync; 502 498 bool ext_addr; 503 499 struct platform_device *pdev; ··· 506 504 struct clk *txs_clk; 507 505 struct clk *rx_clk; 508 506 struct clk *rxs_clk; 509 509 - u32 nr_channels; 510 510 - u32 chan_id; 507 507 + u32 s2mm_chan_id; 508 508 + u32 mm2s_chan_id; 511 509 u32 max_buffer_len; 512 512 - u32 s2mm_index; 513 510 }; 514 511 515 512 /* Macros */ ··· 1746 1745 return IRQ_NONE; 1747 1746 1748 1747 if (chan->direction == DMA_DEV_TO_MEM) 1749 1749 - chan_offset = chan->xdev->s2mm_index; 1748 1748 + chan_offset = chan->xdev->dma_config->max_channels / 2; 1750 1749 1751 1750 chan_offset = chan_offset + (chan_id - 1); 1752 1751 chan = chan->xdev->chan[chan_offset]; ··· 2405 2404 u32 reg; 2406 2405 int err; 2407 2406 2408 2408 - if (chan->cyclic) 2409 2409 - xilinx_dma_chan_reset(chan); 2410 2410 - 2411 2411 - err = chan->stop_transfer(chan); 2412 2412 - if (err) { 2413 2413 - dev_err(chan->dev, "Cannot stop channel %p: %x\n", 2414 2414 - chan, dma_ctrl_read(chan, XILINX_DMA_REG_DMASR)); 2415 2415 - chan->err = true; 2407 2407 + if (!chan->cyclic) { 2408 2408 + err = chan->stop_transfer(chan); 2409 2409 + if (err) { 2410 2410 + dev_err(chan->dev, "Cannot stop channel %p: %x\n", 2411 2411 + chan, dma_ctrl_read(chan, 2412 2412 + XILINX_DMA_REG_DMASR)); 2413 2413 + chan->err = true; 2414 2414 + } 2416 2415 } 2417 2416 2417 2417 + xilinx_dma_chan_reset(chan); 2418 2418 /* Remove and free all of the descriptors in the lists */ 2419 2419 xilinx_dma_free_descriptors(chan); 2420 2420 chan->idle = true; ··· 2732 2730 * 2733 2731 * @xdev: Driver specific device structure 2734 2732 * @node: Device node 2735 2735 - * @chan_id: DMA Channel id 2736 2733 * 2737 2734 * Return: '0' on success and failure value on error 2738 2735 */ 2739 2736 static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, 2740 2740 - struct device_node *node, int chan_id) 2737 2737 + struct device_node *node) 2741 2738 { 2742 2739 struct xilinx_dma_chan *chan; 2743 2740 bool has_dre = false; ··· 2788 2787 of_device_is_compatible(node, "xlnx,axi-dma-mm2s-channel") || 2789 2788 of_device_is_compatible(node, "xlnx,axi-cdma-channel")) { 2790 2789 chan->direction = DMA_MEM_TO_DEV; 2791 2791 - chan->id = chan_id; 2792 2792 - chan->tdest = chan_id; 2790 2790 + chan->id = xdev->mm2s_chan_id++; 2791 2791 + chan->tdest = chan->id; 2793 2792 2794 2793 chan->ctrl_offset = XILINX_DMA_MM2S_CTRL_OFFSET; 2795 2794 if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { ··· 2805 2804 of_device_is_compatible(node, 2806 2805 "xlnx,axi-dma-s2mm-channel")) { 2807 2806 chan->direction = DMA_DEV_TO_MEM; 2808 2808 - chan->id = chan_id; 2809 2809 - xdev->s2mm_index = xdev->nr_channels; 2810 2810 - chan->tdest = chan_id - xdev->nr_channels; 2807 2807 + chan->id = xdev->s2mm_chan_id++; 2808 2808 + chan->tdest = chan->id - xdev->dma_config->max_channels / 2; 2811 2809 chan->has_vflip = of_property_read_bool(node, 2812 2810 "xlnx,enable-vert-flip"); 2813 2811 if (chan->has_vflip) { ··· 2908 2908 dev_warn(xdev->dev, "missing dma-channels property\n"); 2909 2909 2910 2910 for (i = 0; i < nr_channels; i++) 2911 2911 - xilinx_dma_chan_probe(xdev, node, xdev->chan_id++); 2912 2912 - 2913 2913 - xdev->nr_channels += nr_channels; 2911 2911 + xilinx_dma_chan_probe(xdev, node); 2914 2912 2915 2913 return 0; 2916 2914 } ··· 2926 2928 struct xilinx_dma_device *xdev = ofdma->of_dma_data; 2927 2929 int chan_id = dma_spec->args[0]; 2928 2930 2929 2929 - if (chan_id >= xdev->nr_channels || !xdev->chan[chan_id]) 2931 2931 + if (chan_id >= xdev->dma_config->max_channels || !xdev->chan[chan_id]) 2930 2932 return NULL; 2931 2933 2932 2934 return dma_get_slave_channel(&xdev->chan[chan_id]->common); ··· 2936 2938 .dmatype = XDMA_TYPE_AXIDMA, 2937 2939 .clk_init = axidma_clk_init, 2938 2940 .irq_handler = xilinx_dma_irq_handler, 2941 2941 + .max_channels = XILINX_DMA_MAX_CHANS_PER_DEVICE, 2939 2942 }; 2940 2943 2941 2944 static const struct xilinx_dma_config aximcdma_config = { 2942 2945 .dmatype = XDMA_TYPE_AXIMCDMA, 2943 2946 .clk_init = axidma_clk_init, 2944 2947 .irq_handler = xilinx_mcdma_irq_handler, 2948 2948 + .max_channels = XILINX_MCDMA_MAX_CHANS_PER_DEVICE, 2945 2949 }; 2946 2950 static const struct xilinx_dma_config axicdma_config = { 2947 2951 .dmatype = XDMA_TYPE_CDMA, 2948 2952 .clk_init = axicdma_clk_init, 2949 2953 .irq_handler = xilinx_dma_irq_handler, 2954 2954 + .max_channels = XILINX_CDMA_MAX_CHANS_PER_DEVICE, 2950 2955 }; 2951 2956 2952 2957 static const struct xilinx_dma_config axivdma_config = { 2953 2958 .dmatype = XDMA_TYPE_VDMA, 2954 2959 .clk_init = axivdma_clk_init, 2955 2960 .irq_handler = xilinx_dma_irq_handler, 2961 2961 + .max_channels = XILINX_DMA_MAX_CHANS_PER_DEVICE, 2956 2962 }; 2957 2963 2958 2964 static const struct of_device_id xilinx_dma_of_ids[] = { ··· 3013 3011 3014 3012 /* Retrieve the DMA engine properties from the device tree */ 3015 3013 xdev->max_buffer_len = GENMASK(XILINX_DMA_MAX_TRANS_LEN_MAX - 1, 0); 3014 3014 + xdev->s2mm_chan_id = xdev->dma_config->max_channels / 2; 3016 3015 3017 3016 if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA || 3018 3017 xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) { ··· 3107 3104 } 3108 3105 3109 3106 if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { 3110 3110 - for (i = 0; i < xdev->nr_channels; i++) 3107 3107 + for (i = 0; i < xdev->dma_config->max_channels; i++) 3111 3108 if (xdev->chan[i]) 3112 3109 xdev->chan[i]->num_frms = num_frames; 3113 3110 } ··· 3137 3134 disable_clks: 3138 3135 xdma_disable_allclks(xdev); 3139 3136 error: 3140 3140 - for (i = 0; i < xdev->nr_channels; i++) 3137 3137 + for (i = 0; i < xdev->dma_config->max_channels; i++) 3141 3138 if (xdev->chan[i]) 3142 3139 xilinx_dma_chan_remove(xdev->chan[i]); 3143 3140 ··· 3159 3156 3160 3157 dma_async_device_unregister(&xdev->common); 3161 3158 3162 3162 - for (i = 0; i < xdev->nr_channels; i++) 3159 3159 + for (i = 0; i < xdev->dma_config->max_channels; i++) 3163 3160 if (xdev->chan[i]) 3164 3161 xilinx_dma_chan_remove(xdev->chan[i]); 3165 3162

+36 -32

include/linux/dmaengine.h

reviewed

··· 300 300 * @chan_id: channel ID for sysfs 301 301 * @dev: class device for sysfs 302 302 * @name: backlink name for sysfs 303 303 + * @dbg_client_name: slave name for debugfs in format: 304 304 + * dev_name(requester's dev):channel name, for example: "2b00000.mcasp:tx" 303 305 * @device_node: used to add this to the device chan list 304 306 * @local: per-cpu pointer to a struct dma_chan_percpu 305 307 * @client_count: how many clients are using this channel ··· 320 318 int chan_id; 321 319 struct dma_chan_dev *dev; 322 320 const char *name; 321 321 + #ifdef CONFIG_DEBUG_FS 322 322 + char *dbg_client_name; 323 323 + #endif 323 324 324 325 struct list_head device_node; 325 326 struct dma_chan_percpu __percpu *local; ··· 623 618 624 619 static inline void dma_descriptor_unmap(struct dma_async_tx_descriptor *tx) 625 620 { 626 626 - if (tx->unmap) { 627 627 - dmaengine_unmap_put(tx->unmap); 628 628 - tx->unmap = NULL; 629 629 - } 621 621 + if (!tx->unmap) 622 622 + return; 623 623 + 624 624 + dmaengine_unmap_put(tx->unmap); 625 625 + tx->unmap = NULL; 630 626 } 631 627 632 628 #ifndef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH ··· 811 805 * called and there are no further references to this structure. This 812 806 * must be implemented to free resources however many existing drivers 813 807 * do not and are therefore not safe to unbind while in use. 814 814 - * 808 808 + * @dbg_summary_show: optional routine to show contents in debugfs; default code 809 809 + * will be used when this is omitted, but custom code can show extra, 810 810 + * controller specific information. 815 811 */ 816 812 struct dma_device { 817 813 struct kref ref; ··· 899 891 struct dma_tx_state *txstate); 900 892 void (*device_issue_pending)(struct dma_chan *chan); 901 893 void (*device_release)(struct dma_device *dev); 894 894 + /* debugfs support */ 895 895 + #ifdef CONFIG_DEBUG_FS 896 896 + void (*dbg_summary_show)(struct seq_file *s, struct dma_device *dev); 897 897 + struct dentry *dbg_dev_root; 898 898 + #endif 902 899 }; 903 900 904 901 static inline int dmaengine_slave_config(struct dma_chan *chan, ··· 1168 1155 static inline bool dmaengine_check_align(enum dmaengine_alignment align, 1169 1156 size_t off1, size_t off2, size_t len) 1170 1157 { 1171 1171 - size_t mask; 1172 1172 - 1173 1173 - if (!align) 1174 1174 - return true; 1175 1175 - mask = (1 << align) - 1; 1176 1176 - if (mask & (off1 | off2 | len)) 1177 1177 - return false; 1178 1178 - return true; 1158 1158 + return !(((1 << align) - 1) & (off1 | off2 | len)); 1179 1159 } 1180 1160 1181 1161 static inline bool is_dma_copy_aligned(struct dma_device *dev, size_t off1, ··· 1242 1236 { 1243 1237 if (dma_dev_has_pq_continue(dma) || !dmaf_continue(flags)) 1244 1238 return dma_dev_to_maxpq(dma); 1245 1245 - else if (dmaf_p_disabled_continue(flags)) 1239 1239 + if (dmaf_p_disabled_continue(flags)) 1246 1240 return dma_dev_to_maxpq(dma) - 1; 1247 1247 - else if (dmaf_continue(flags)) 1241 1241 + if (dmaf_continue(flags)) 1248 1242 return dma_dev_to_maxpq(dma) - 3; 1249 1243 BUG(); 1250 1244 } ··· 1255 1249 if (inc) { 1256 1250 if (dir_icg) 1257 1251 return dir_icg; 1258 1258 - else if (sgl) 1252 1252 + if (sgl) 1259 1253 return icg; 1260 1254 } 1261 1255 ··· 1421 1415 static inline void 1422 1416 dma_set_tx_state(struct dma_tx_state *st, dma_cookie_t last, dma_cookie_t used, u32 residue) 1423 1417 { 1424 1424 - if (st) { 1425 1425 - st->last = last; 1426 1426 - st->used = used; 1427 1427 - st->residue = residue; 1428 1428 - } 1418 1418 + if (!st) 1419 1419 + return; 1420 1420 + 1421 1421 + st->last = last; 1422 1422 + st->used = used; 1423 1423 + st->residue = residue; 1429 1424 } 1430 1425 1431 1426 #ifdef CONFIG_DMA_ENGINE ··· 1503 1496 if (ret) 1504 1497 return ret; 1505 1498 1506 1506 - if (caps.descriptor_reuse) { 1507 1507 - tx->flags |= DMA_CTRL_REUSE; 1508 1508 - return 0; 1509 1509 - } else { 1499 1499 + if (!caps.descriptor_reuse) 1510 1500 return -EPERM; 1511 1511 - } 1501 1501 + 1502 1502 + tx->flags |= DMA_CTRL_REUSE; 1503 1503 + return 0; 1512 1504 } 1513 1505 1514 1506 static inline void dmaengine_desc_clear_reuse(struct dma_async_tx_descriptor *tx) ··· 1523 1517 static inline int dmaengine_desc_free(struct dma_async_tx_descriptor *desc) 1524 1518 { 1525 1519 /* this is supported for reusable desc, so check that */ 1526 1526 - if (dmaengine_desc_test_reuse(desc)) 1527 1527 - return desc->desc_free(desc); 1528 1528 - else 1520 1520 + if (!dmaengine_desc_test_reuse(desc)) 1529 1521 return -EPERM; 1522 1522 + 1523 1523 + return desc->desc_free(desc); 1530 1524 } 1531 1525 1532 1526 /* --- DMA device --- */ ··· 1572 1566 case DMA_DEV_TO_DEV: 1573 1567 return "DEV_TO_DEV"; 1574 1568 default: 1575 1575 - break; 1569 1569 + return "invalid"; 1576 1570 } 1577 1577 - 1578 1578 - return "invalid"; 1579 1571 } 1580 1572 #endif /* DMAENGINE_H */

+5 -16

include/uapi/linux/idxd.h

reviewed

··· 83 83 #define DSA_COMP_STATUS_MASK 0x7f 84 84 #define DSA_COMP_STATUS_WRITE 0x80 85 85 86 86 - struct dsa_batch_desc { 87 87 - uint32_t pasid:20; 88 88 - uint32_t rsvd:11; 89 89 - uint32_t priv:1; 90 90 - uint32_t flags:24; 91 91 - uint32_t opcode:8; 92 92 - uint64_t completion_addr; 93 93 - uint64_t desc_list_addr; 94 94 - uint64_t rsvd1; 95 95 - uint32_t desc_count; 96 96 - uint16_t interrupt_handle; 97 97 - uint16_t rsvd2; 98 98 - uint8_t rsvd3[24]; 99 99 - } __attribute__((packed)); 100 100 - 101 86 struct dsa_hw_desc { 102 87 uint32_t pasid:20; 103 88 uint32_t rsvd:11; ··· 94 109 uint64_t src_addr; 95 110 uint64_t rdback_addr; 96 111 uint64_t pattern; 112 112 + uint64_t desc_list_addr; 97 113 }; 98 114 union { 99 115 uint64_t dst_addr; ··· 102 116 uint64_t src2_addr; 103 117 uint64_t comp_pattern; 104 118 }; 105 105 - uint32_t xfer_size; 119 119 + union { 120 120 + uint32_t xfer_size; 121 121 + uint32_t desc_count; 122 122 + }; 106 123 uint16_t int_handle; 107 124 uint16_t rsvd1; 108 125 union {