dmaengine: dma-axi-dmac: Gracefully terminate HW cyclic transfers

Add support for gracefully terminating hardware cyclic DMA transfers when
a new transfer is queued and is flagged with DMA_PREP_LOAD_EOT. Without
this, cyclic transfers would continue indefinitely until we brute force
it with .device_terminate_all().

When a new descriptor is queued while a cyclic transfer is active, mark
the cyclic transfer for termination. For hardware with scatter-gather
support, modify the last segment flags to trigger end-of-transfer. For
non-SG hardware, clear the CYCLIC flag to allow natural completion.

Older IP core versions (pre-4.6.a) can prefetch data when clearing the
CYCLIC flag, causing corruption in the next transfer. Work around this
by disabling and re-enabling the core to flush prefetched data.

The cyclic_eot flag tracks transfers marked for termination, preventing
new transfers from starting until the cyclic one completes. Non-EOT
transfers submitted after cyclic transfers are discarded with a warning.

Also note that for hardware cyclic transfers not using SG, we need to
make sure that chan->next_desc is also set to NULL (so we can look at
possible EOT transfers) and we also need to move the queue check to
after axi_dmac_get_next_desc() because with hardware based cyclic
transfers we might get the queue marked as full and hence we would not
be able to check for cyclic termination.

Signed-off-by: Nuno Sá <nuno.sa@analog.com>
Link: https://patch.msgid.link/20260303-axi-dac-cyclic-support-v2-5-0db27b4be95a@analog.com
Signed-off-by: Vinod Koul <vkoul@kernel.org>

authored by

Nuno Sá and committed by

Vinod Koul 3 months ago f1d201e7 ca3bf200

+91 -13

1 changed file

expand all

drivers

dma

dma-axi-dmac.c

+91 -13

drivers/dma/dma-axi-dmac.c

··· 134 134 struct axi_dmac_chan *chan; 135 135 136 136 bool cyclic; 137 + bool cyclic_eot; 137 138 bool have_partial_xfer; 138 139 139 140 unsigned int num_submitted; ··· 163 162 bool hw_cyclic; 164 163 bool hw_2d; 165 164 bool hw_sg; 165 + bool hw_cyclic_hotfix; 166 166 }; 167 167 168 168 struct axi_dmac { ··· 229 227 return true; 230 228 } 231 229 230 + static struct axi_dmac_desc *axi_dmac_active_desc(struct axi_dmac_chan *chan) 231 + { 232 + return list_first_entry_or_null(&chan->active_descs, 233 + struct axi_dmac_desc, vdesc.node); 234 + } 235 + 232 236 static struct axi_dmac_desc *axi_dmac_get_next_desc(struct axi_dmac *dmac, 233 237 struct axi_dmac_chan *chan) 234 238 { 239 + struct axi_dmac_desc *active = axi_dmac_active_desc(chan); 235 240 struct virt_dma_desc *vdesc; 236 241 struct axi_dmac_desc *desc; 242 + unsigned int val; 243 + 244 + /* 245 + * Just play safe and ignore any SOF if we have an active cyclic transfer 246 + * flagged to end. We'll start it as soon as the current cyclic one ends. 247 + */ 248 + if (active && active->cyclic_eot) 249 + return NULL; 237 250 238 251 /* 239 252 * It means a SW cyclic transfer is in place so we should just return ··· 262 245 if (!vdesc) 263 246 return NULL; 264 247 248 + if (active && active->cyclic && !(vdesc->tx.flags & DMA_PREP_LOAD_EOT)) { 249 + struct device *dev = chan_to_axi_dmac(chan)->dma_dev.dev; 250 + 251 + dev_warn(dev, "Discarding non EOT transfer after cyclic\n"); 252 + list_del(&vdesc->node); 253 + return NULL; 254 + } 255 + 265 256 list_move_tail(&vdesc->node, &chan->active_descs); 266 257 desc = to_axi_dmac_desc(vdesc); 267 258 chan->next_desc = desc; 268 259 269 - return desc; 260 + if (!active || !active->cyclic) 261 + return desc; 262 + 263 + active->cyclic_eot = true; 264 + 265 + if (chan->hw_sg) { 266 + unsigned long flags = AXI_DMAC_HW_FLAG_IRQ | AXI_DMAC_HW_FLAG_LAST; 267 + /* 268 + * Let's then stop the current cyclic transfer by making sure we 269 + * get an EOT interrupt and to open the cyclic loop by marking 270 + * the last segment. 271 + */ 272 + active->sg[active->num_sgs - 1].hw->flags = flags; 273 + return NULL; 274 + } 275 + 276 + /* 277 + * Clear the cyclic bit if there's no Scatter-Gather HW so that we get 278 + * at the end of the transfer. 279 + */ 280 + val = axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS); 281 + val &= ~AXI_DMAC_FLAG_CYCLIC; 282 + axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, val); 283 + 284 + return NULL; 270 285 } 271 286 272 287 static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) ··· 309 260 unsigned int flags = 0; 310 261 unsigned int val; 311 262 312 - val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); 313 - if (val) /* Queue is full, wait for the next SOT IRQ */ 314 - return; 315 - 316 263 desc = axi_dmac_get_next_desc(dmac, chan); 317 264 if (!desc) 265 + return; 266 + 267 + val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); 268 + if (val) /* Queue is full, wait for the next SOT IRQ */ 318 269 return; 319 270 320 271 sg = &desc->sg[desc->num_submitted]; ··· 358 309 * call, enable hw cyclic mode to avoid unnecessary interrupts. 359 310 */ 360 311 if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback) { 361 - if (chan->hw_sg) 312 + if (chan->hw_sg) { 362 313 desc->sg[desc->num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_IRQ; 363 - else if (desc->num_sgs == 1) 314 + } else if (desc->num_sgs == 1) { 315 + chan->next_desc = NULL; 364 316 flags |= AXI_DMAC_FLAG_CYCLIC; 317 + } 365 318 } 366 319 367 320 if (chan->hw_partial_xfer) ··· 379 328 } 380 329 axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, flags); 381 330 axi_dmac_write(dmac, AXI_DMAC_REG_START_TRANSFER, 1); 382 - } 383 - 384 - static struct axi_dmac_desc *axi_dmac_active_desc(struct axi_dmac_chan *chan) 385 - { 386 - return list_first_entry_or_null(&chan->active_descs, 387 - struct axi_dmac_desc, vdesc.node); 388 331 } 389 332 390 333 static inline unsigned int axi_dmac_total_sg_bytes(struct axi_dmac_chan *chan, ··· 470 425 /* wrap around */ 471 426 active->num_completed = 0; 472 427 428 + if (active->cyclic_eot) { 429 + /* 430 + * It means an HW cyclic transfer was marked to stop. And we 431 + * know we have something to schedule, so start the next 432 + * transfer now the cyclic one is done. 433 + */ 434 + list_del(&active->vdesc.node); 435 + vchan_cookie_complete(&active->vdesc); 436 + 437 + if (chan->hw_cyclic_hotfix) { 438 + struct axi_dmac *dmac = chan_to_axi_dmac(chan); 439 + /* 440 + * In older IP cores, ending a cyclic transfer by clearing 441 + * the CYCLIC flag does not guarantee a graceful end. 442 + * It can happen that some data (of the next frame) is 443 + * already prefetched and will be wrongly visible in the 444 + * next transfer. To workaround this, we need to reenable 445 + * the core so everything is flushed. Newer cores handles 446 + * this correctly and do not require this "hotfix". The 447 + * SG IP also does not require this. 448 + */ 449 + dev_dbg(dev, "HW cyclic hotfix\n"); 450 + axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, 0); 451 + axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, AXI_DMAC_CTRL_ENABLE); 452 + } 453 + 454 + return true; 455 + } 456 + 473 457 vdesc = vchan_next_desc(&chan->vchan); 474 458 if (!vdesc) 475 459 return false; ··· 534 460 if (chan->hw_sg) { 535 461 if (active->cyclic) { 536 462 vchan_cyclic_callback(&active->vdesc); 463 + start_next = axi_dmac_handle_cyclic_eot(chan, active); 537 464 } else { 538 465 list_del(&active->vdesc.node); 539 466 vchan_cookie_complete(&active->vdesc); ··· 1177 1102 } else { 1178 1103 chan->length_align_mask = chan->address_align_mask; 1179 1104 } 1105 + 1106 + if (version < ADI_AXI_PCORE_VER(4, 6, 0) && !chan->hw_sg) 1107 + chan->hw_cyclic_hotfix = true; 1180 1108 1181 1109 return 0; 1182 1110 }

Configure Feed

Configure Feed