The driver currently uses a worker thread to free processed
descriptors. Under high load (e.g., CRYPTO_MANAGER_EXTRA_TESTS), the
descriptor allocation rate can significantly outpace the worker
thread's execution rate.
This leads to false resource exhaustion where dma_alloc_coherent()
fails even though many descriptors are waiting in the purge queue to
be freed.
Remove the lazy freeing mechanism (desc_to_purge list and worker) and
instead free the descriptors immediately in the completion callback.
This eliminates the latency gap between hardware completion and
software reclaim, preventing the pool exhaustion during stress tests.
Signed-off-by: Sai Sree Kartheek Adivi <s-adivi@ti.com>
---
drivers/dma/ti/k3-udma-common.c | 40 +++------------------------------
drivers/dma/ti/k3-udma-v2.c | 2 --
drivers/dma/ti/k3-udma.c | 2 --
drivers/dma/ti/k3-udma.h | 3 ---
4 files changed, 3 insertions(+), 44 deletions(-)
diff --git a/drivers/dma/ti/k3-udma-common.c b/drivers/dma/ti/k3-udma-common.c
index 05b2b6b962a06..f9da00298b60f 100644
--- a/drivers/dma/ti/k3-udma-common.c
+++ b/drivers/dma/ti/k3-udma-common.c
@@ -95,32 +95,6 @@ void udma_free_hwdesc(struct udma_chan *uc, struct udma_desc *d)
}
}
-void udma_purge_desc_work(struct work_struct *work)
-{
- struct udma_dev *ud = container_of(work, typeof(*ud), purge_work);
- struct virt_dma_desc *vd, *_vd;
- unsigned long flags;
- LIST_HEAD(head);
-
- spin_lock_irqsave(&ud->lock, flags);
- list_splice_tail_init(&ud->desc_to_purge, &head);
- spin_unlock_irqrestore(&ud->lock, flags);
-
- list_for_each_entry_safe(vd, _vd, &head, node) {
- struct udma_chan *uc = to_udma_chan(vd->tx.chan);
- struct udma_desc *d = to_udma_desc(&vd->tx);
-
- udma_free_hwdesc(uc, d);
- list_del(&vd->node);
- kfree(d);
- }
-
- /* If more to purge, schedule the work again */
- if (!list_empty(&ud->desc_to_purge))
- schedule_work(&ud->purge_work);
-}
-EXPORT_SYMBOL_GPL(udma_purge_desc_work);
-
void udma_desc_free(struct virt_dma_desc *vd)
{
struct udma_dev *ud = to_udma_dev(vd->tx.chan->device);
@@ -131,17 +105,9 @@ void udma_desc_free(struct virt_dma_desc *vd)
if (uc->terminated_desc == d)
uc->terminated_desc = NULL;
- if (uc->use_dma_pool) {
- udma_free_hwdesc(uc, d);
- kfree(d);
- return;
- }
-
- spin_lock_irqsave(&ud->lock, flags);
- list_add_tail(&vd->node, &ud->desc_to_purge);
- spin_unlock_irqrestore(&ud->lock, flags);
-
- schedule_work(&ud->purge_work);
+ udma_free_hwdesc(uc, d);
+ kfree(d);
+ return;
}
EXPORT_SYMBOL_GPL(udma_desc_free);
diff --git a/drivers/dma/ti/k3-udma-v2.c b/drivers/dma/ti/k3-udma-v2.c
index 6761a079025ba..d33382cc0356a 100644
--- a/drivers/dma/ti/k3-udma-v2.c
+++ b/drivers/dma/ti/k3-udma-v2.c
@@ -1320,14 +1320,12 @@ static int udma_v2_probe(struct platform_device *pdev)
ud->psil_base = ud->match_data->psil_base;
INIT_LIST_HEAD(&ud->ddev.channels);
- INIT_LIST_HEAD(&ud->desc_to_purge);
ch_count = setup_resources(ud);
if (ch_count <= 0)
return ch_count;
spin_lock_init(&ud->lock);
- INIT_WORK(&ud->purge_work, udma_purge_desc_work);
ud->desc_align = 64;
if (ud->desc_align < dma_get_cache_alignment())
diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c
index a8d01d955651a..34d458b4a0dbc 100644
--- a/drivers/dma/ti/k3-udma.c
+++ b/drivers/dma/ti/k3-udma.c
@@ -2704,14 +2704,12 @@ static int udma_probe(struct platform_device *pdev)
ud->psil_base = ud->match_data->psil_base;
INIT_LIST_HEAD(&ud->ddev.channels);
- INIT_LIST_HEAD(&ud->desc_to_purge);
ch_count = setup_resources(ud);
if (ch_count <= 0)
return ch_count;
spin_lock_init(&ud->lock);
- INIT_WORK(&ud->purge_work, udma_purge_desc_work);
ud->desc_align = 64;
if (ud->desc_align < dma_get_cache_alignment())
diff --git a/drivers/dma/ti/k3-udma.h b/drivers/dma/ti/k3-udma.h
index 3ae2400e67990..67de9feb9906b 100644
--- a/drivers/dma/ti/k3-udma.h
+++ b/drivers/dma/ti/k3-udma.h
@@ -353,8 +353,6 @@ struct udma_dev {
struct k3_ringacc *ringacc;
- struct work_struct purge_work;
- struct list_head desc_to_purge;
spinlock_t lock;
struct udma_rx_flush rx_flush;
@@ -596,7 +594,6 @@ static inline void udma_fetch_epib(struct udma_chan *uc, struct udma_desc *d)
struct udma_desc *udma_udma_desc_from_paddr(struct udma_chan *uc,
dma_addr_t paddr);
void udma_free_hwdesc(struct udma_chan *uc, struct udma_desc *d);
-void udma_purge_desc_work(struct work_struct *work);
void udma_desc_free(struct virt_dma_desc *vd);
bool udma_desc_is_rx_flush(struct udma_chan *uc, dma_addr_t addr);
bool udma_is_desc_really_done(struct udma_chan *uc, struct udma_desc *d);
--
2.34.1