Some DesignWare-based endpoints integrate an eDMA engine that can be
programmed by the host via MMIO. The upcoming NTB transport remote-eDMA
backend relies on this capability, but there is currently no upstream
test coverage for the end-to-end control and data path.

Extend pci-epf-test with an optional remote eDMA test backend (built when
CONFIG_DW_EDMA is enabled).

- Reserve a spare BAR and expose a small 'pcitest_edma_info' header at
  BAR offset 0. The header carries a magic/version and describes the
  endpoint eDMA register window, per-direction linked-list (LL)
  locations and an endpoint test buffer.
- Map the eDMA registers and LL locations into that BAR using BAR
  subrange mappings (address-match inbound iATU).

To run this extra testing, two new endpoint commands are added:
  * COMMAND_REMOTE_EDMA_SETUP
  * COMMAND_REMOTE_EDMA_CHECKSUM

When the former command is received, the endpoint prepares for the
remote eDMA transfer. The CHECKSUM command is useful for Host-to-EP
transfer testing, as the endpoint side is not expected to receive the
DMA completion interrupt directly. Instead, the host asks the endpoint
to compute a CRC32 over the transferred data.

This backend is exercised by the host-side pci_endpoint_test driver via a
new UAPI flag.

Signed-off-by: Koichiro Den <den@valinux.co.jp>
---
 drivers/pci/endpoint/functions/pci-epf-test.c | 477 ++++++++++++++++++
 1 file changed, 477 insertions(+)

diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c
index e560c3becebb..eea10bddcd2a 100644
--- a/drivers/pci/endpoint/functions/pci-epf-test.c
+++ b/drivers/pci/endpoint/functions/pci-epf-test.c
@@ -10,6 +10,7 @@
 #include <linux/delay.h>
 #include <linux/dmaengine.h>
 #include <linux/io.h>
+#include <linux/iommu.h>
 #include <linux/module.h>
 #include <linux/msi.h>
 #include <linux/slab.h>
@@ -33,6 +34,8 @@
 #define COMMAND_COPY			BIT(5)
 #define COMMAND_ENABLE_DOORBELL		BIT(6)
 #define COMMAND_DISABLE_DOORBELL	BIT(7)
+#define COMMAND_REMOTE_EDMA_SETUP	BIT(8)
+#define COMMAND_REMOTE_EDMA_CHECKSUM	BIT(9)
 
 #define STATUS_READ_SUCCESS		BIT(0)
 #define STATUS_READ_FAIL		BIT(1)
@@ -48,6 +51,10 @@
 #define STATUS_DOORBELL_ENABLE_FAIL	BIT(11)
 #define STATUS_DOORBELL_DISABLE_SUCCESS BIT(12)
 #define STATUS_DOORBELL_DISABLE_FAIL	BIT(13)
+#define STATUS_REMOTE_EDMA_SETUP_SUCCESS	BIT(14)
+#define STATUS_REMOTE_EDMA_SETUP_FAIL		BIT(15)
+#define STATUS_REMOTE_EDMA_CHECKSUM_SUCCESS	BIT(16)
+#define STATUS_REMOTE_EDMA_CHECKSUM_FAIL	BIT(17)
 
 #define FLAG_USE_DMA			BIT(0)
 
@@ -77,6 +84,9 @@ struct pci_epf_test {
 	bool			dma_private;
 	const struct pci_epc_features *epc_features;
 	struct pci_epf_bar	db_bar;
+
+	/* For extended tests that rely on vendor-specific features */
+	void *data;
 };
 
 struct pci_epf_test_reg {
@@ -117,6 +127,454 @@ static enum pci_barno pci_epf_test_next_free_bar(struct pci_epf_test *epf_test)
 	return bar;
 }
 
+#if IS_REACHABLE(CONFIG_DW_EDMA)
+#include <linux/dma/edma.h>
+
+#define PCITEST_EDMA_INFO_MAGIC		0x414d4445U /* 'EDMA' */
+#define PCITEST_EDMA_INFO_VERSION	0x00010000U
+#define PCITEST_EDMA_TEST_BUF_SIZE	(1024 * 1024)
+
+struct pci_epf_test_edma {
+	/* Remote eDMA test resources */
+	bool			enabled;
+	enum pci_barno		bar;
+	void			*info;
+	size_t			total_size;
+	void			*test_buf;
+	dma_addr_t		test_buf_phys;
+	size_t			test_buf_size;
+
+	/* DW eDMA specifics */
+	phys_addr_t		reg_phys;
+	size_t			reg_submap_sz;
+	unsigned long		reg_iova;
+	size_t			reg_iova_sz;
+	phys_addr_t		ll_rd_phys;
+	size_t			ll_rd_sz_aligned;
+	phys_addr_t		ll_wr_phys;
+	size_t			ll_wr_sz_aligned;
+};
+
+struct pcitest_edma_info {
+	__le32 magic;
+	__le32 version;
+
+	__le32 reg_off;
+	__le32 reg_size;
+
+	__le64 ll_rd_phys;
+	__le32 ll_rd_off;
+	__le32 ll_rd_size;
+
+	__le64 ll_wr_phys;
+	__le32 ll_wr_off;
+	__le32 ll_wr_size;
+
+	__le64 test_buf_phys;
+	__le32 test_buf_size;
+};
+
+static bool pci_epf_test_bar_is_reserved(struct pci_epf_test *test,
+					 enum pci_barno barno)
+{
+	struct pci_epf_test_edma *edma = test->data;
+
+	if (!edma)
+		return false;
+
+	return barno == edma->bar;
+}
+
+static void pci_epf_test_clear_submaps(struct pci_epf_bar *bar)
+{
+	kfree(bar->submap);
+	bar->submap = NULL;
+	bar->num_submap = 0;
+}
+
+static int pci_epf_test_add_submap(struct pci_epf_bar *bar, phys_addr_t phys,
+				   size_t size)
+{
+	struct pci_epf_bar_submap *submap, *new;
+
+	new = krealloc_array(bar->submap, bar->num_submap + 1, sizeof(*new),
+			     GFP_KERNEL);
+	if (!new)
+		return -ENOMEM;
+
+	bar->submap = new;
+	submap = &bar->submap[bar->num_submap];
+	submap->phys_addr = phys;
+	submap->size = size;
+	bar->num_submap++;
+
+	return 0;
+}
+
+static void pci_epf_test_clean_remote_edma(struct pci_epf_test *test)
+{
+	struct pci_epf_test_edma *edma = test->data;
+	struct pci_epf *epf = test->epf;
+	struct pci_epc *epc = epf->epc;
+	struct device *dev = epc->dev.parent;
+	struct iommu_domain *dom;
+	struct pci_epf_bar *bar;
+	enum pci_barno barno;
+
+	if (!edma)
+		return;
+
+	barno = edma->bar;
+	if (barno == NO_BAR)
+		return;
+
+	bar = &epf->bar[barno];
+
+	dom = iommu_get_domain_for_dev(dev);
+	if (dom && edma->reg_iova_sz) {
+		iommu_unmap(dom, edma->reg_iova, edma->reg_iova_sz);
+		edma->reg_iova = 0;
+		edma->reg_iova_sz = 0;
+	}
+
+	if (edma->test_buf) {
+		dma_free_coherent(dev, edma->test_buf_size,
+				  edma->test_buf,
+				  edma->test_buf_phys);
+		edma->test_buf = NULL;
+		edma->test_buf_phys = 0;
+		edma->test_buf_size = 0;
+	}
+
+	if (edma->info) {
+		pci_epf_free_space(epf, edma->info, barno, PRIMARY_INTERFACE);
+		edma->info = NULL;
+	}
+
+	pci_epf_test_clear_submaps(bar);
+	pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no, bar);
+
+	edma->bar = NO_BAR;
+	edma->enabled = false;
+}
+
+static int pci_epf_test_init_remote_edma(struct pci_epf_test *test)
+{
+	const struct pci_epc_features *epc_features = test->epc_features;
+	struct pci_epf_test_edma *edma;
+	struct pci_epf *epf = test->epf;
+	struct pci_epc *epc = epf->epc;
+	struct pcitest_edma_info *info;
+	struct device *dev = epc->dev.parent;
+	struct dw_edma_region region;
+	struct iommu_domain *dom;
+	size_t reg_sz_aligned, ll_rd_sz_aligned, ll_wr_sz_aligned;
+	phys_addr_t phys, ll_rd_phys, ll_wr_phys;
+	size_t ll_rd_size, ll_wr_size;
+	resource_size_t reg_size;
+	unsigned long iova;
+	size_t off, size;
+	int ret;
+
+	if (!test->dma_chan_tx || !test->dma_chan_rx)
+		return -ENODEV;
+
+	edma = devm_kzalloc(&epf->dev, sizeof(*edma), GFP_KERNEL);
+	if (!edma)
+		return -ENOMEM;
+	test->data = edma;
+
+	edma->bar = pci_epf_test_next_free_bar(test);
+	if (edma->bar == NO_BAR) {
+		dev_err(&epf->dev, "No spare BAR for remote eDMA (remote eDMA disabled)\n");
+		ret = -ENOSPC;
+		goto err;
+	}
+
+	ret = dw_edma_get_reg_window(epc, &edma->reg_phys, &reg_size);
+	if (ret) {
+		dev_err(dev, "failed to get edma reg window: %d\n", ret);
+		goto err;
+	}
+	dom = iommu_get_domain_for_dev(dev);
+	if (dom) {
+		phys = edma->reg_phys & PAGE_MASK;
+		size = PAGE_ALIGN(reg_size + edma->reg_phys - phys);
+		iova = phys;
+
+		ret = iommu_map(dom, iova, phys, size,
+				IOMMU_READ | IOMMU_WRITE | IOMMU_MMIO,
+				GFP_KERNEL);
+		if (ret) {
+			dev_err(dev, "failed to direct map eDMA reg: %d\n", ret);
+			goto err;
+		}
+		edma->reg_iova = iova;
+		edma->reg_iova_sz = size;
+	}
+
+	/* Get LL location addresses and sizes */
+	ret = dw_edma_chan_get_ll_region(test->dma_chan_rx, &region);
+	if (ret) {
+		dev_err(dev, "failed to get edma ll region for rx: %d\n", ret);
+		goto err;
+	}
+	ll_rd_phys = region.paddr;
+	ll_rd_size = region.sz;
+
+	ret = dw_edma_chan_get_ll_region(test->dma_chan_tx, &region);
+	if (ret) {
+		dev_err(dev, "failed to get edma ll region for tx: %d\n", ret);
+		goto err;
+	}
+	ll_wr_phys = region.paddr;
+	ll_wr_size = region.sz;
+
+	edma->test_buf_size = PCITEST_EDMA_TEST_BUF_SIZE;
+	edma->test_buf = dma_alloc_coherent(dev, edma->test_buf_size,
+					    &edma->test_buf_phys, GFP_KERNEL);
+	if (!edma->test_buf) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	reg_sz_aligned = PAGE_ALIGN(reg_size);
+	ll_rd_sz_aligned = PAGE_ALIGN(ll_rd_size);
+	ll_wr_sz_aligned = PAGE_ALIGN(ll_wr_size);
+	edma->total_size = PAGE_SIZE + reg_sz_aligned + ll_rd_sz_aligned +
+			   ll_wr_sz_aligned;
+	size = roundup_pow_of_two(edma->total_size);
+
+	info = pci_epf_alloc_space(epf, size, edma->bar,
+				   epc_features, PRIMARY_INTERFACE);
+	if (!info) {
+		ret = -ENOMEM;
+		goto err;
+	}
+	memset(info, 0, size);
+
+	off = PAGE_SIZE;
+	info->magic = cpu_to_le32(PCITEST_EDMA_INFO_MAGIC);
+	info->version = cpu_to_le32(PCITEST_EDMA_INFO_VERSION);
+
+	info->reg_off = cpu_to_le32(off);
+	info->reg_size = cpu_to_le32(reg_size);
+	off += reg_sz_aligned;
+
+	info->ll_rd_phys = cpu_to_le64(ll_rd_phys);
+	info->ll_rd_off = cpu_to_le32(off);
+	info->ll_rd_size = cpu_to_le32(ll_rd_size);
+	off += ll_rd_sz_aligned;
+
+	info->ll_wr_phys = cpu_to_le64(ll_wr_phys);
+	info->ll_wr_off = cpu_to_le32(off);
+	info->ll_wr_size = cpu_to_le32(ll_wr_size);
+	off += ll_wr_sz_aligned;
+
+	info->test_buf_phys = cpu_to_le64(edma->test_buf_phys);
+	info->test_buf_size = cpu_to_le32(edma->test_buf_size);
+
+	edma->info = info;
+	edma->reg_submap_sz = reg_sz_aligned;
+	edma->ll_rd_phys = ll_rd_phys;
+	edma->ll_wr_phys = ll_wr_phys;
+	edma->ll_rd_sz_aligned = ll_rd_sz_aligned;
+	edma->ll_wr_sz_aligned = ll_wr_sz_aligned;
+
+	ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no,
+			      &epf->bar[edma->bar]);
+	if (ret) {
+		dev_err(dev,
+			"failed to init BAR%d for remote eDMA: %d\n",
+			edma->bar, ret);
+		goto err;
+	}
+	dev_info(dev, "BAR%d initialized for remote eDMA\n", edma->bar);
+
+	return 0;
+
+err:
+	pci_epf_test_clean_remote_edma(test);
+	devm_kfree(&epf->dev, edma);
+	test->data = NULL;
+	return ret;
+}
+
+static int pci_epf_test_map_remote_edma(struct pci_epf_test *test)
+{
+	struct pci_epf_test_edma *edma = test->data;
+	struct pcitest_edma_info *info;
+	struct pci_epf *epf = test->epf;
+	struct pci_epc *epc = epf->epc;
+	struct pci_epf_bar *bar;
+	enum pci_barno barno;
+	struct device *dev = epc->dev.parent;
+	int ret;
+
+	if (!edma)
+		return -ENODEV;
+
+	info = edma->info;
+	barno = edma->bar;
+
+	if (barno == NO_BAR)
+		return -ENOSPC;
+	if (!info || !edma->test_buf)
+		return -ENODEV;
+
+	bar = &epf->bar[barno];
+	pci_epf_test_clear_submaps(bar);
+
+	ret = pci_epf_test_add_submap(bar, bar->phys_addr, PAGE_SIZE);
+	if (ret)
+		return ret;
+
+	ret = pci_epf_test_add_submap(bar, edma->reg_phys, edma->reg_submap_sz);
+	if (ret)
+		goto err_submap;
+
+	ret = pci_epf_test_add_submap(bar, edma->ll_rd_phys,
+				      edma->ll_rd_sz_aligned);
+	if (ret)
+		goto err_submap;
+
+	ret = pci_epf_test_add_submap(bar, edma->ll_wr_phys,
+				      edma->ll_wr_sz_aligned);
+	if (ret)
+		goto err_submap;
+
+	if (bar->size > edma->total_size) {
+		ret = pci_epf_test_add_submap(bar, 0,
+					      bar->size - edma->total_size);
+		if (ret)
+			goto err_submap;
+	}
+
+	ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, bar);
+	if (ret) {
+		dev_err(dev, "failed to map BAR%d: %d\n", barno, ret);
+		goto err_submap;
+	}
+
+	/*
+	 * Endpoint-local interrupts must be ignored even if the host fails to
+	 * mask them.
+	 */
+	ret = dw_edma_chan_irq_config(test->dma_chan_tx, DW_EDMA_CH_IRQ_REMOTE);
+	if (ret) {
+		dev_err(dev, "failed to set irq mode for tx channel: %d\n",
+			ret);
+		goto err_bar;
+	}
+	ret = dw_edma_chan_irq_config(test->dma_chan_rx, DW_EDMA_CH_IRQ_REMOTE);
+	if (ret) {
+		dev_err(dev, "failed to set irq mode for rx channel: %d\n",
+			ret);
+		goto err_bar;
+	}
+
+	return 0;
+err_bar:
+	pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no, &epf->bar[barno]);
+err_submap:
+	pci_epf_test_clear_submaps(bar);
+	return ret;
+}
+
+static void pci_epf_test_remote_edma_setup(struct pci_epf_test *epf_test,
+					   struct pci_epf_test_reg *reg)
+{
+	struct pci_epf_test_edma *edma = epf_test->data;
+	size_t size = le32_to_cpu(reg->size);
+	void *buf;
+	int ret;
+
+	if (!edma || !edma->test_buf || size > edma->test_buf_size) {
+		reg->status = cpu_to_le32(STATUS_REMOTE_EDMA_SETUP_FAIL);
+		return;
+	}
+
+	buf = edma->test_buf;
+
+	if (!edma->enabled) {
+		/* NB. Currently DW eDMA is the only supported backend */
+		ret = pci_epf_test_map_remote_edma(epf_test);
+		if (ret) {
+			WRITE_ONCE(reg->status,
+				   cpu_to_le32(STATUS_REMOTE_EDMA_SETUP_FAIL));
+			return;
+		}
+		edma->enabled = true;
+	}
+
+	/* Populate the test buffer with random data */
+	get_random_bytes(buf, size);
+	reg->checksum = cpu_to_le32(crc32_le(~0, buf, size));
+
+	WRITE_ONCE(reg->status, cpu_to_le32(STATUS_REMOTE_EDMA_SETUP_SUCCESS));
+}
+
+static void pci_epf_test_remote_edma_checksum(struct pci_epf_test *epf_test,
+					      struct pci_epf_test_reg *reg)
+{
+	struct pci_epf_test_edma *edma = epf_test->data;
+	u32 status = le32_to_cpu(reg->status);
+	size_t size;
+	void *addr;
+	u32 crc32;
+
+	size = le32_to_cpu(reg->size);
+	if (!edma || !edma->test_buf || size > edma->test_buf_size) {
+		status |= STATUS_REMOTE_EDMA_CHECKSUM_FAIL;
+		reg->status = cpu_to_le32(status);
+		return;
+	}
+
+	addr = edma->test_buf;
+	crc32 = crc32_le(~0, addr, size);
+	status |= STATUS_REMOTE_EDMA_CHECKSUM_SUCCESS;
+
+	reg->checksum = cpu_to_le32(crc32);
+	reg->status = cpu_to_le32(status);
+}
+
+static void pci_epf_test_reset_dma_chan(struct dma_chan *chan)
+{
+	dw_edma_chan_irq_config(chan, DW_EDMA_CH_IRQ_DEFAULT);
+}
+#else
+static bool pci_epf_test_bar_is_reserved(struct pci_epf_test *test,
+					 enum pci_barno barno)
+{
+	return false;
+}
+
+static void pci_epf_test_clean_remote_edma(struct pci_epf_test *test)
+{
+}
+
+static int pci_epf_test_init_remote_edma(struct pci_epf_test *test)
+{
+	return -EOPNOTSUPP;
+}
+
+static void pci_epf_test_remote_edma_setup(struct pci_epf_test *epf_test,
+					   struct pci_epf_test_reg *reg)
+{
+	reg->status = cpu_to_le32(STATUS_REMOTE_EDMA_SETUP_FAIL);
+}
+
+static void pci_epf_test_remote_edma_checksum(struct pci_epf_test *epf_test,
+					      struct pci_epf_test_reg *reg)
+{
+	reg->status = cpu_to_le32(STATUS_REMOTE_EDMA_CHECKSUM_FAIL);
+}
+
+static void pci_epf_test_reset_dma_chan(struct dma_chan *chan)
+{
+}
+#endif
+
 static void pci_epf_test_dma_callback(void *param)
 {
 	struct pci_epf_test *epf_test = param;
@@ -168,6 +626,8 @@ static int pci_epf_test_data_transfer(struct pci_epf_test *epf_test,
 		return -EINVAL;
 	}
 
+	pci_epf_test_reset_dma_chan(chan);
+
 	if (epf_test->dma_private) {
 		sconf.direction = dir;
 		if (dir == DMA_MEM_TO_DEV)
@@ -870,6 +1330,14 @@ static void pci_epf_test_cmd_handler(struct work_struct *work)
 		pci_epf_test_disable_doorbell(epf_test, reg);
 		pci_epf_test_raise_irq(epf_test, reg);
 		break;
+	case COMMAND_REMOTE_EDMA_SETUP:
+		pci_epf_test_remote_edma_setup(epf_test, reg);
+		pci_epf_test_raise_irq(epf_test, reg);
+		break;
+	case COMMAND_REMOTE_EDMA_CHECKSUM:
+		pci_epf_test_remote_edma_checksum(epf_test, reg);
+		pci_epf_test_raise_irq(epf_test, reg);
+		break;
 	default:
 		dev_err(dev, "Invalid command 0x%x\n", command);
 		break;
@@ -961,6 +1429,10 @@ static int pci_epf_test_epc_init(struct pci_epf *epf)
 	if (ret)
 		epf_test->dma_supported = false;
 
+	ret = pci_epf_test_init_remote_edma(epf_test);
+	if (ret && ret != -EOPNOTSUPP)
+		dev_warn(dev, "Remote eDMA setup failed\n");
+
 	if (epf->vfunc_no <= 1) {
 		ret = pci_epc_write_header(epc, epf->func_no, epf->vfunc_no, header);
 		if (ret) {
@@ -1007,6 +1479,7 @@ static void pci_epf_test_epc_deinit(struct pci_epf *epf)
 	struct pci_epf_test *epf_test = epf_get_drvdata(epf);
 
 	cancel_delayed_work_sync(&epf_test->cmd_handler);
+	pci_epf_test_clean_remote_edma(epf_test);
 	pci_epf_test_clean_dma_chan(epf_test);
 	pci_epf_test_clear_bar(epf);
 }
@@ -1076,6 +1549,9 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf)
 		if (bar == test_reg_bar)
 			continue;
 
+		if (pci_epf_test_bar_is_reserved(epf_test, bar))
+			continue;
+
 		if (epc_features->bar[bar].type == BAR_FIXED)
 			test_reg_size = epc_features->bar[bar].fixed_size;
 		else
@@ -1146,6 +1622,7 @@ static void pci_epf_test_unbind(struct pci_epf *epf)
 
 	cancel_delayed_work_sync(&epf_test->cmd_handler);
 	if (epc->init_complete) {
+		pci_epf_test_clean_remote_edma(epf_test);
 		pci_epf_test_clean_dma_chan(epf_test);
 		pci_epf_test_clear_bar(epf);
 	}
-- 
2.51.0

On Sun, Jan 18, 2026 at 10:54:38PM +0900, Koichiro Den wrote:
> Some DesignWare-based endpoints integrate an eDMA engine that can be
> programmed by the host via MMIO. The upcoming NTB transport remote-eDMA
> backend relies on this capability, but there is currently no upstream
> test coverage for the end-to-end control and data path.
>
> Extend pci-epf-test with an optional remote eDMA test backend (built when
> CONFIG_DW_EDMA is enabled).
>
> - Reserve a spare BAR and expose a small 'pcitest_edma_info' header at
>   BAR offset 0. The header carries a magic/version and describes the
>   endpoint eDMA register window, per-direction linked-list (LL)
>   locations and an endpoint test buffer.
> - Map the eDMA registers and LL locations into that BAR using BAR
>   subrange mappings (address-match inbound iATU).
>
> To run this extra testing, two new endpoint commands are added:
>   * COMMAND_REMOTE_EDMA_SETUP
>   * COMMAND_REMOTE_EDMA_CHECKSUM
>
> When the former command is received, the endpoint prepares for the
> remote eDMA transfer. The CHECKSUM command is useful for Host-to-EP
> transfer testing, as the endpoint side is not expected to receive the
> DMA completion interrupt directly. Instead, the host asks the endpoint
> to compute a CRC32 over the transferred data.
>
> This backend is exercised by the host-side pci_endpoint_test driver via a
> new UAPI flag.
>
> Signed-off-by: Koichiro Den <den@valinux.co.jp>
> ---
>  drivers/pci/endpoint/functions/pci-epf-test.c | 477 ++++++++++++++++++

This patch should be combined into your submap patches, which is one user
of submap.

Frank

>  1 file changed, 477 insertions(+)
>
> diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c
> index e560c3becebb..eea10bddcd2a 100644
> --- a/drivers/pci/endpoint/functions/pci-epf-test.c
> +++ b/drivers/pci/endpoint/functions/pci-epf-test.c
> @@ -10,6 +10,7 @@
>  #include <linux/delay.h>
>  #include <linux/dmaengine.h>
>  #include <linux/io.h>
> +#include <linux/iommu.h>
>  #include <linux/module.h>
>  #include <linux/msi.h>
>  #include <linux/slab.h>
> @@ -33,6 +34,8 @@
>  #define COMMAND_COPY			BIT(5)
>  #define COMMAND_ENABLE_DOORBELL		BIT(6)
>  #define COMMAND_DISABLE_DOORBELL	BIT(7)
> +#define COMMAND_REMOTE_EDMA_SETUP	BIT(8)
> +#define COMMAND_REMOTE_EDMA_CHECKSUM	BIT(9)
>
>  #define STATUS_READ_SUCCESS		BIT(0)
>  #define STATUS_READ_FAIL		BIT(1)
> @@ -48,6 +51,10 @@
>  #define STATUS_DOORBELL_ENABLE_FAIL	BIT(11)
>  #define STATUS_DOORBELL_DISABLE_SUCCESS BIT(12)
>  #define STATUS_DOORBELL_DISABLE_FAIL	BIT(13)
> +#define STATUS_REMOTE_EDMA_SETUP_SUCCESS	BIT(14)
> +#define STATUS_REMOTE_EDMA_SETUP_FAIL		BIT(15)
> +#define STATUS_REMOTE_EDMA_CHECKSUM_SUCCESS	BIT(16)
> +#define STATUS_REMOTE_EDMA_CHECKSUM_FAIL	BIT(17)
>
>  #define FLAG_USE_DMA			BIT(0)
>
> @@ -77,6 +84,9 @@ struct pci_epf_test {
>  	bool			dma_private;
>  	const struct pci_epc_features *epc_features;
>  	struct pci_epf_bar	db_bar;
> +
> +	/* For extended tests that rely on vendor-specific features */
> +	void *data;
>  };
>
>  struct pci_epf_test_reg {
> @@ -117,6 +127,454 @@ static enum pci_barno pci_epf_test_next_free_bar(struct pci_epf_test *epf_test)
>  	return bar;
>  }
>
> +#if IS_REACHABLE(CONFIG_DW_EDMA)
> +#include <linux/dma/edma.h>
> +
> +#define PCITEST_EDMA_INFO_MAGIC		0x414d4445U /* 'EDMA' */
> +#define PCITEST_EDMA_INFO_VERSION	0x00010000U
> +#define PCITEST_EDMA_TEST_BUF_SIZE	(1024 * 1024)
> +
> +struct pci_epf_test_edma {
> +	/* Remote eDMA test resources */
> +	bool			enabled;
> +	enum pci_barno		bar;
> +	void			*info;
> +	size_t			total_size;
> +	void			*test_buf;
> +	dma_addr_t		test_buf_phys;
> +	size_t			test_buf_size;
> +
> +	/* DW eDMA specifics */
> +	phys_addr_t		reg_phys;
> +	size_t			reg_submap_sz;
> +	unsigned long		reg_iova;
> +	size_t			reg_iova_sz;
> +	phys_addr_t		ll_rd_phys;
> +	size_t			ll_rd_sz_aligned;
> +	phys_addr_t		ll_wr_phys;
> +	size_t			ll_wr_sz_aligned;
> +};
> +
> +struct pcitest_edma_info {
> +	__le32 magic;
> +	__le32 version;
> +
> +	__le32 reg_off;
> +	__le32 reg_size;
> +
> +	__le64 ll_rd_phys;
> +	__le32 ll_rd_off;
> +	__le32 ll_rd_size;
> +
> +	__le64 ll_wr_phys;
> +	__le32 ll_wr_off;
> +	__le32 ll_wr_size;
> +
> +	__le64 test_buf_phys;
> +	__le32 test_buf_size;
> +};
> +
> +static bool pci_epf_test_bar_is_reserved(struct pci_epf_test *test,
> +					 enum pci_barno barno)
> +{
> +	struct pci_epf_test_edma *edma = test->data;
> +
> +	if (!edma)
> +		return false;
> +
> +	return barno == edma->bar;
> +}
> +
> +static void pci_epf_test_clear_submaps(struct pci_epf_bar *bar)
> +{
> +	kfree(bar->submap);
> +	bar->submap = NULL;
> +	bar->num_submap = 0;
> +}
> +
> +static int pci_epf_test_add_submap(struct pci_epf_bar *bar, phys_addr_t phys,
> +				   size_t size)
> +{
> +	struct pci_epf_bar_submap *submap, *new;
> +
> +	new = krealloc_array(bar->submap, bar->num_submap + 1, sizeof(*new),
> +			     GFP_KERNEL);
> +	if (!new)
> +		return -ENOMEM;
> +
> +	bar->submap = new;
> +	submap = &bar->submap[bar->num_submap];
> +	submap->phys_addr = phys;
> +	submap->size = size;
> +	bar->num_submap++;
> +
> +	return 0;
> +}
> +
> +static void pci_epf_test_clean_remote_edma(struct pci_epf_test *test)
> +{
> +	struct pci_epf_test_edma *edma = test->data;
> +	struct pci_epf *epf = test->epf;
> +	struct pci_epc *epc = epf->epc;
> +	struct device *dev = epc->dev.parent;
> +	struct iommu_domain *dom;
> +	struct pci_epf_bar *bar;
> +	enum pci_barno barno;
> +
> +	if (!edma)
> +		return;
> +
> +	barno = edma->bar;
> +	if (barno == NO_BAR)
> +		return;
> +
> +	bar = &epf->bar[barno];
> +
> +	dom = iommu_get_domain_for_dev(dev);
> +	if (dom && edma->reg_iova_sz) {
> +		iommu_unmap(dom, edma->reg_iova, edma->reg_iova_sz);
> +		edma->reg_iova = 0;
> +		edma->reg_iova_sz = 0;
> +	}
> +
> +	if (edma->test_buf) {
> +		dma_free_coherent(dev, edma->test_buf_size,
> +				  edma->test_buf,
> +				  edma->test_buf_phys);
> +		edma->test_buf = NULL;
> +		edma->test_buf_phys = 0;
> +		edma->test_buf_size = 0;
> +	}
> +
> +	if (edma->info) {
> +		pci_epf_free_space(epf, edma->info, barno, PRIMARY_INTERFACE);
> +		edma->info = NULL;
> +	}
> +
> +	pci_epf_test_clear_submaps(bar);
> +	pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no, bar);
> +
> +	edma->bar = NO_BAR;
> +	edma->enabled = false;
> +}
> +
> +static int pci_epf_test_init_remote_edma(struct pci_epf_test *test)
> +{
> +	const struct pci_epc_features *epc_features = test->epc_features;
> +	struct pci_epf_test_edma *edma;
> +	struct pci_epf *epf = test->epf;
> +	struct pci_epc *epc = epf->epc;
> +	struct pcitest_edma_info *info;
> +	struct device *dev = epc->dev.parent;
> +	struct dw_edma_region region;
> +	struct iommu_domain *dom;
> +	size_t reg_sz_aligned, ll_rd_sz_aligned, ll_wr_sz_aligned;
> +	phys_addr_t phys, ll_rd_phys, ll_wr_phys;
> +	size_t ll_rd_size, ll_wr_size;
> +	resource_size_t reg_size;
> +	unsigned long iova;
> +	size_t off, size;
> +	int ret;
> +
> +	if (!test->dma_chan_tx || !test->dma_chan_rx)
> +		return -ENODEV;
> +
> +	edma = devm_kzalloc(&epf->dev, sizeof(*edma), GFP_KERNEL);
> +	if (!edma)
> +		return -ENOMEM;
> +	test->data = edma;
> +
> +	edma->bar = pci_epf_test_next_free_bar(test);
> +	if (edma->bar == NO_BAR) {
> +		dev_err(&epf->dev, "No spare BAR for remote eDMA (remote eDMA disabled)\n");
> +		ret = -ENOSPC;
> +		goto err;
> +	}
> +
> +	ret = dw_edma_get_reg_window(epc, &edma->reg_phys, &reg_size);
> +	if (ret) {
> +		dev_err(dev, "failed to get edma reg window: %d\n", ret);
> +		goto err;
> +	}
> +	dom = iommu_get_domain_for_dev(dev);
> +	if (dom) {
> +		phys = edma->reg_phys & PAGE_MASK;
> +		size = PAGE_ALIGN(reg_size + edma->reg_phys - phys);
> +		iova = phys;
> +
> +		ret = iommu_map(dom, iova, phys, size,
> +				IOMMU_READ | IOMMU_WRITE | IOMMU_MMIO,
> +				GFP_KERNEL);
> +		if (ret) {
> +			dev_err(dev, "failed to direct map eDMA reg: %d\n", ret);
> +			goto err;
> +		}
> +		edma->reg_iova = iova;
> +		edma->reg_iova_sz = size;
> +	}
> +
> +	/* Get LL location addresses and sizes */
> +	ret = dw_edma_chan_get_ll_region(test->dma_chan_rx, &region);
> +	if (ret) {
> +		dev_err(dev, "failed to get edma ll region for rx: %d\n", ret);
> +		goto err;
> +	}
> +	ll_rd_phys = region.paddr;
> +	ll_rd_size = region.sz;
> +
> +	ret = dw_edma_chan_get_ll_region(test->dma_chan_tx, &region);
> +	if (ret) {
> +		dev_err(dev, "failed to get edma ll region for tx: %d\n", ret);
> +		goto err;
> +	}
> +	ll_wr_phys = region.paddr;
> +	ll_wr_size = region.sz;
> +
> +	edma->test_buf_size = PCITEST_EDMA_TEST_BUF_SIZE;
> +	edma->test_buf = dma_alloc_coherent(dev, edma->test_buf_size,
> +					    &edma->test_buf_phys, GFP_KERNEL);
> +	if (!edma->test_buf) {
> +		ret = -ENOMEM;
> +		goto err;
> +	}
> +
> +	reg_sz_aligned = PAGE_ALIGN(reg_size);
> +	ll_rd_sz_aligned = PAGE_ALIGN(ll_rd_size);
> +	ll_wr_sz_aligned = PAGE_ALIGN(ll_wr_size);
> +	edma->total_size = PAGE_SIZE + reg_sz_aligned + ll_rd_sz_aligned +
> +			   ll_wr_sz_aligned;
> +	size = roundup_pow_of_two(edma->total_size);
> +
> +	info = pci_epf_alloc_space(epf, size, edma->bar,
> +				   epc_features, PRIMARY_INTERFACE);
> +	if (!info) {
> +		ret = -ENOMEM;
> +		goto err;
> +	}
> +	memset(info, 0, size);
> +
> +	off = PAGE_SIZE;
> +	info->magic = cpu_to_le32(PCITEST_EDMA_INFO_MAGIC);
> +	info->version = cpu_to_le32(PCITEST_EDMA_INFO_VERSION);
> +
> +	info->reg_off = cpu_to_le32(off);
> +	info->reg_size = cpu_to_le32(reg_size);
> +	off += reg_sz_aligned;
> +
> +	info->ll_rd_phys = cpu_to_le64(ll_rd_phys);
> +	info->ll_rd_off = cpu_to_le32(off);
> +	info->ll_rd_size = cpu_to_le32(ll_rd_size);
> +	off += ll_rd_sz_aligned;
> +
> +	info->ll_wr_phys = cpu_to_le64(ll_wr_phys);
> +	info->ll_wr_off = cpu_to_le32(off);
> +	info->ll_wr_size = cpu_to_le32(ll_wr_size);
> +	off += ll_wr_sz_aligned;
> +
> +	info->test_buf_phys = cpu_to_le64(edma->test_buf_phys);
> +	info->test_buf_size = cpu_to_le32(edma->test_buf_size);
> +
> +	edma->info = info;
> +	edma->reg_submap_sz = reg_sz_aligned;
> +	edma->ll_rd_phys = ll_rd_phys;
> +	edma->ll_wr_phys = ll_wr_phys;
> +	edma->ll_rd_sz_aligned = ll_rd_sz_aligned;
> +	edma->ll_wr_sz_aligned = ll_wr_sz_aligned;
> +
> +	ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no,
> +			      &epf->bar[edma->bar]);
> +	if (ret) {
> +		dev_err(dev,
> +			"failed to init BAR%d for remote eDMA: %d\n",
> +			edma->bar, ret);
> +		goto err;
> +	}
> +	dev_info(dev, "BAR%d initialized for remote eDMA\n", edma->bar);
> +
> +	return 0;
> +
> +err:
> +	pci_epf_test_clean_remote_edma(test);
> +	devm_kfree(&epf->dev, edma);
> +	test->data = NULL;
> +	return ret;
> +}
> +
> +static int pci_epf_test_map_remote_edma(struct pci_epf_test *test)
> +{
> +	struct pci_epf_test_edma *edma = test->data;
> +	struct pcitest_edma_info *info;
> +	struct pci_epf *epf = test->epf;
> +	struct pci_epc *epc = epf->epc;
> +	struct pci_epf_bar *bar;
> +	enum pci_barno barno;
> +	struct device *dev = epc->dev.parent;
> +	int ret;
> +
> +	if (!edma)
> +		return -ENODEV;
> +
> +	info = edma->info;
> +	barno = edma->bar;
> +
> +	if (barno == NO_BAR)
> +		return -ENOSPC;
> +	if (!info || !edma->test_buf)
> +		return -ENODEV;
> +
> +	bar = &epf->bar[barno];
> +	pci_epf_test_clear_submaps(bar);
> +
> +	ret = pci_epf_test_add_submap(bar, bar->phys_addr, PAGE_SIZE);
> +	if (ret)
> +		return ret;
> +
> +	ret = pci_epf_test_add_submap(bar, edma->reg_phys, edma->reg_submap_sz);
> +	if (ret)
> +		goto err_submap;
> +
> +	ret = pci_epf_test_add_submap(bar, edma->ll_rd_phys,
> +				      edma->ll_rd_sz_aligned);
> +	if (ret)
> +		goto err_submap;
> +
> +	ret = pci_epf_test_add_submap(bar, edma->ll_wr_phys,
> +				      edma->ll_wr_sz_aligned);
> +	if (ret)
> +		goto err_submap;
> +
> +	if (bar->size > edma->total_size) {
> +		ret = pci_epf_test_add_submap(bar, 0,
> +					      bar->size - edma->total_size);
> +		if (ret)
> +			goto err_submap;
> +	}
> +
> +	ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, bar);
> +	if (ret) {
> +		dev_err(dev, "failed to map BAR%d: %d\n", barno, ret);
> +		goto err_submap;
> +	}
> +
> +	/*
> +	 * Endpoint-local interrupts must be ignored even if the host fails to
> +	 * mask them.
> +	 */
> +	ret = dw_edma_chan_irq_config(test->dma_chan_tx, DW_EDMA_CH_IRQ_REMOTE);
> +	if (ret) {
> +		dev_err(dev, "failed to set irq mode for tx channel: %d\n",
> +			ret);
> +		goto err_bar;
> +	}
> +	ret = dw_edma_chan_irq_config(test->dma_chan_rx, DW_EDMA_CH_IRQ_REMOTE);
> +	if (ret) {
> +		dev_err(dev, "failed to set irq mode for rx channel: %d\n",
> +			ret);
> +		goto err_bar;
> +	}
> +
> +	return 0;
> +err_bar:
> +	pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no, &epf->bar[barno]);
> +err_submap:
> +	pci_epf_test_clear_submaps(bar);
> +	return ret;
> +}
> +
> +static void pci_epf_test_remote_edma_setup(struct pci_epf_test *epf_test,
> +					   struct pci_epf_test_reg *reg)
> +{
> +	struct pci_epf_test_edma *edma = epf_test->data;
> +	size_t size = le32_to_cpu(reg->size);
> +	void *buf;
> +	int ret;
> +
> +	if (!edma || !edma->test_buf || size > edma->test_buf_size) {
> +		reg->status = cpu_to_le32(STATUS_REMOTE_EDMA_SETUP_FAIL);
> +		return;
> +	}
> +
> +	buf = edma->test_buf;
> +
> +	if (!edma->enabled) {
> +		/* NB. Currently DW eDMA is the only supported backend */
> +		ret = pci_epf_test_map_remote_edma(epf_test);
> +		if (ret) {
> +			WRITE_ONCE(reg->status,
> +				   cpu_to_le32(STATUS_REMOTE_EDMA_SETUP_FAIL));
> +			return;
> +		}
> +		edma->enabled = true;
> +	}
> +
> +	/* Populate the test buffer with random data */
> +	get_random_bytes(buf, size);
> +	reg->checksum = cpu_to_le32(crc32_le(~0, buf, size));
> +
> +	WRITE_ONCE(reg->status, cpu_to_le32(STATUS_REMOTE_EDMA_SETUP_SUCCESS));
> +}
> +
> +static void pci_epf_test_remote_edma_checksum(struct pci_epf_test *epf_test,
> +					      struct pci_epf_test_reg *reg)
> +{
> +	struct pci_epf_test_edma *edma = epf_test->data;
> +	u32 status = le32_to_cpu(reg->status);
> +	size_t size;
> +	void *addr;
> +	u32 crc32;
> +
> +	size = le32_to_cpu(reg->size);
> +	if (!edma || !edma->test_buf || size > edma->test_buf_size) {
> +		status |= STATUS_REMOTE_EDMA_CHECKSUM_FAIL;
> +		reg->status = cpu_to_le32(status);
> +		return;
> +	}
> +
> +	addr = edma->test_buf;
> +	crc32 = crc32_le(~0, addr, size);
> +	status |= STATUS_REMOTE_EDMA_CHECKSUM_SUCCESS;
> +
> +	reg->checksum = cpu_to_le32(crc32);
> +	reg->status = cpu_to_le32(status);
> +}
> +
> +static void pci_epf_test_reset_dma_chan(struct dma_chan *chan)
> +{
> +	dw_edma_chan_irq_config(chan, DW_EDMA_CH_IRQ_DEFAULT);
> +}
> +#else
> +static bool pci_epf_test_bar_is_reserved(struct pci_epf_test *test,
> +					 enum pci_barno barno)
> +{
> +	return false;
> +}
> +
> +static void pci_epf_test_clean_remote_edma(struct pci_epf_test *test)
> +{
> +}
> +
> +static int pci_epf_test_init_remote_edma(struct pci_epf_test *test)
> +{
> +	return -EOPNOTSUPP;
> +}
> +
> +static void pci_epf_test_remote_edma_setup(struct pci_epf_test *epf_test,
> +					   struct pci_epf_test_reg *reg)
> +{
> +	reg->status = cpu_to_le32(STATUS_REMOTE_EDMA_SETUP_FAIL);
> +}
> +
> +static void pci_epf_test_remote_edma_checksum(struct pci_epf_test *epf_test,
> +					      struct pci_epf_test_reg *reg)
> +{
> +	reg->status = cpu_to_le32(STATUS_REMOTE_EDMA_CHECKSUM_FAIL);
> +}
> +
> +static void pci_epf_test_reset_dma_chan(struct dma_chan *chan)
> +{
> +}
> +#endif
> +
>  static void pci_epf_test_dma_callback(void *param)
>  {
>  	struct pci_epf_test *epf_test = param;
> @@ -168,6 +626,8 @@ static int pci_epf_test_data_transfer(struct pci_epf_test *epf_test,
>  		return -EINVAL;
>  	}
>
> +	pci_epf_test_reset_dma_chan(chan);
> +
>  	if (epf_test->dma_private) {
>  		sconf.direction = dir;
>  		if (dir == DMA_MEM_TO_DEV)
> @@ -870,6 +1330,14 @@ static void pci_epf_test_cmd_handler(struct work_struct *work)
>  		pci_epf_test_disable_doorbell(epf_test, reg);
>  		pci_epf_test_raise_irq(epf_test, reg);
>  		break;
> +	case COMMAND_REMOTE_EDMA_SETUP:
> +		pci_epf_test_remote_edma_setup(epf_test, reg);
> +		pci_epf_test_raise_irq(epf_test, reg);
> +		break;
> +	case COMMAND_REMOTE_EDMA_CHECKSUM:
> +		pci_epf_test_remote_edma_checksum(epf_test, reg);
> +		pci_epf_test_raise_irq(epf_test, reg);
> +		break;
>  	default:
>  		dev_err(dev, "Invalid command 0x%x\n", command);
>  		break;
> @@ -961,6 +1429,10 @@ static int pci_epf_test_epc_init(struct pci_epf *epf)
>  	if (ret)
>  		epf_test->dma_supported = false;
>
> +	ret = pci_epf_test_init_remote_edma(epf_test);
> +	if (ret && ret != -EOPNOTSUPP)
> +		dev_warn(dev, "Remote eDMA setup failed\n");
> +
>  	if (epf->vfunc_no <= 1) {
>  		ret = pci_epc_write_header(epc, epf->func_no, epf->vfunc_no, header);
>  		if (ret) {
> @@ -1007,6 +1479,7 @@ static void pci_epf_test_epc_deinit(struct pci_epf *epf)
>  	struct pci_epf_test *epf_test = epf_get_drvdata(epf);
>
>  	cancel_delayed_work_sync(&epf_test->cmd_handler);
> +	pci_epf_test_clean_remote_edma(epf_test);
>  	pci_epf_test_clean_dma_chan(epf_test);
>  	pci_epf_test_clear_bar(epf);
>  }
> @@ -1076,6 +1549,9 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf)
>  		if (bar == test_reg_bar)
>  			continue;
>
> +		if (pci_epf_test_bar_is_reserved(epf_test, bar))
> +			continue;
> +
>  		if (epc_features->bar[bar].type == BAR_FIXED)
>  			test_reg_size = epc_features->bar[bar].fixed_size;
>  		else
> @@ -1146,6 +1622,7 @@ static void pci_epf_test_unbind(struct pci_epf *epf)
>
>  	cancel_delayed_work_sync(&epf_test->cmd_handler);
>  	if (epc->init_complete) {
> +		pci_epf_test_clean_remote_edma(epf_test);
>  		pci_epf_test_clean_dma_chan(epf_test);
>  		pci_epf_test_clear_bar(epf);
>  	}
> --
> 2.51.0
>

On Mon, Jan 19, 2026 at 03:47:27PM -0500, Frank Li wrote:
> On Sun, Jan 18, 2026 at 10:54:38PM +0900, Koichiro Den wrote:
> > Some DesignWare-based endpoints integrate an eDMA engine that can be
> > programmed by the host via MMIO. The upcoming NTB transport remote-eDMA
> > backend relies on this capability, but there is currently no upstream
> > test coverage for the end-to-end control and data path.
> >
> > Extend pci-epf-test with an optional remote eDMA test backend (built when
> > CONFIG_DW_EDMA is enabled).
> >
> > - Reserve a spare BAR and expose a small 'pcitest_edma_info' header at
> >   BAR offset 0. The header carries a magic/version and describes the
> >   endpoint eDMA register window, per-direction linked-list (LL)
> >   locations and an endpoint test buffer.
> > - Map the eDMA registers and LL locations into that BAR using BAR
> >   subrange mappings (address-match inbound iATU).
> >
> > To run this extra testing, two new endpoint commands are added:
> >   * COMMAND_REMOTE_EDMA_SETUP
> >   * COMMAND_REMOTE_EDMA_CHECKSUM
> >
> > When the former command is received, the endpoint prepares for the
> > remote eDMA transfer. The CHECKSUM command is useful for Host-to-EP
> > transfer testing, as the endpoint side is not expected to receive the
> > DMA completion interrupt directly. Instead, the host asks the endpoint
> > to compute a CRC32 over the transferred data.
> >
> > This backend is exercised by the host-side pci_endpoint_test driver via a
> > new UAPI flag.
> >
> > Signed-off-by: Koichiro Den <den@valinux.co.jp>
> > ---
> >  drivers/pci/endpoint/functions/pci-epf-test.c | 477 ++++++++++++++++++
> 
> This patch should be combined into your submap patches, which is one user
> of submap.

Thanks for the comment, and my apologies for the delayed response to this.

The pci endpoint test case addition depends on both of the following prerequisites:

1) [PATCH v9 0/5] PCI: endpoint: BAR subrange mapping support
   https://lore.kernel.org/all/20260122084909.2390865-1-den@valinux.co.jp/

2) A not-yet-submitted series for Patch 01-05, as described in the "Patch
   layout" section of the cover letter:
   https://lore.kernel.org/all/20260118135440.1958279-1-den@valinux.co.jp/

   [...]
   1. dw-edma / DesignWare EP helpers needed for remote embedded-DMA (export
      register/LL windows, IRQ routing control, etc.)

      Patch 01 : dmaengine: dw-edma: Export helper to get integrated register window
      Patch 02 : dmaengine: dw-edma: Add per-channel interrupt routing control
      Patch 03 : dmaengine: dw-edma: Poll completion when local IRQ handling is disabled
      Patch 04 : dmaengine: dw-edma: Add notify-only channels support
      Patch 05 : dmaengine: dw-edma: Add a helper to query linked-list region
   [...]

   I plan to submit these patches shortly, perhaps as a single series, once the
   design discussion in the following thread is resolved:
   https://lore.kernel.org/all/2bcksnyuxj33bjctjombrstfvjrcdtap6i3v6xhfxtqjmbdkwm@jcaoy2iuh5pr/
   Thank you for reviewing that discussion as well.

Given that (1) precedes (2), it should be reasonable to include the PCI
endpoint test case additions (Patchs 35-38) as part of the series in (2).

Kind regards,
Koichiro

> 
> Frank
> 
> >  1 file changed, 477 insertions(+)
> >
> > diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c
> > index e560c3becebb..eea10bddcd2a 100644
> > --- a/drivers/pci/endpoint/functions/pci-epf-test.c
> > +++ b/drivers/pci/endpoint/functions/pci-epf-test.c
> > @@ -10,6 +10,7 @@
> >  #include <linux/delay.h>
> >  #include <linux/dmaengine.h>
> >  #include <linux/io.h>
> > +#include <linux/iommu.h>
> >  #include <linux/module.h>
> >  #include <linux/msi.h>
> >  #include <linux/slab.h>
> > @@ -33,6 +34,8 @@
> >  #define COMMAND_COPY			BIT(5)
> >  #define COMMAND_ENABLE_DOORBELL		BIT(6)
> >  #define COMMAND_DISABLE_DOORBELL	BIT(7)
> > +#define COMMAND_REMOTE_EDMA_SETUP	BIT(8)
> > +#define COMMAND_REMOTE_EDMA_CHECKSUM	BIT(9)
> >
> >  #define STATUS_READ_SUCCESS		BIT(0)
> >  #define STATUS_READ_FAIL		BIT(1)
> > @@ -48,6 +51,10 @@
> >  #define STATUS_DOORBELL_ENABLE_FAIL	BIT(11)
> >  #define STATUS_DOORBELL_DISABLE_SUCCESS BIT(12)
> >  #define STATUS_DOORBELL_DISABLE_FAIL	BIT(13)
> > +#define STATUS_REMOTE_EDMA_SETUP_SUCCESS	BIT(14)
> > +#define STATUS_REMOTE_EDMA_SETUP_FAIL		BIT(15)
> > +#define STATUS_REMOTE_EDMA_CHECKSUM_SUCCESS	BIT(16)
> > +#define STATUS_REMOTE_EDMA_CHECKSUM_FAIL	BIT(17)
> >
> >  #define FLAG_USE_DMA			BIT(0)
> >
> > @@ -77,6 +84,9 @@ struct pci_epf_test {
> >  	bool			dma_private;
> >  	const struct pci_epc_features *epc_features;
> >  	struct pci_epf_bar	db_bar;
> > +
> > +	/* For extended tests that rely on vendor-specific features */
> > +	void *data;
> >  };
> >
> >  struct pci_epf_test_reg {
> > @@ -117,6 +127,454 @@ static enum pci_barno pci_epf_test_next_free_bar(struct pci_epf_test *epf_test)
> >  	return bar;
> >  }
> >
> > +#if IS_REACHABLE(CONFIG_DW_EDMA)
> > +#include <linux/dma/edma.h>
> > +
> > +#define PCITEST_EDMA_INFO_MAGIC		0x414d4445U /* 'EDMA' */
> > +#define PCITEST_EDMA_INFO_VERSION	0x00010000U
> > +#define PCITEST_EDMA_TEST_BUF_SIZE	(1024 * 1024)
> > +
> > +struct pci_epf_test_edma {
> > +	/* Remote eDMA test resources */
> > +	bool			enabled;
> > +	enum pci_barno		bar;
> > +	void			*info;
> > +	size_t			total_size;
> > +	void			*test_buf;
> > +	dma_addr_t		test_buf_phys;
> > +	size_t			test_buf_size;
> > +
> > +	/* DW eDMA specifics */
> > +	phys_addr_t		reg_phys;
> > +	size_t			reg_submap_sz;
> > +	unsigned long		reg_iova;
> > +	size_t			reg_iova_sz;
> > +	phys_addr_t		ll_rd_phys;
> > +	size_t			ll_rd_sz_aligned;
> > +	phys_addr_t		ll_wr_phys;
> > +	size_t			ll_wr_sz_aligned;
> > +};
> > +
> > +struct pcitest_edma_info {
> > +	__le32 magic;
> > +	__le32 version;
> > +
> > +	__le32 reg_off;
> > +	__le32 reg_size;
> > +
> > +	__le64 ll_rd_phys;
> > +	__le32 ll_rd_off;
> > +	__le32 ll_rd_size;
> > +
> > +	__le64 ll_wr_phys;
> > +	__le32 ll_wr_off;
> > +	__le32 ll_wr_size;
> > +
> > +	__le64 test_buf_phys;
> > +	__le32 test_buf_size;
> > +};
> > +
> > +static bool pci_epf_test_bar_is_reserved(struct pci_epf_test *test,
> > +					 enum pci_barno barno)
> > +{
> > +	struct pci_epf_test_edma *edma = test->data;
> > +
> > +	if (!edma)
> > +		return false;
> > +
> > +	return barno == edma->bar;
> > +}
> > +
> > +static void pci_epf_test_clear_submaps(struct pci_epf_bar *bar)
> > +{
> > +	kfree(bar->submap);
> > +	bar->submap = NULL;
> > +	bar->num_submap = 0;
> > +}
> > +
> > +static int pci_epf_test_add_submap(struct pci_epf_bar *bar, phys_addr_t phys,
> > +				   size_t size)
> > +{
> > +	struct pci_epf_bar_submap *submap, *new;
> > +
> > +	new = krealloc_array(bar->submap, bar->num_submap + 1, sizeof(*new),
> > +			     GFP_KERNEL);
> > +	if (!new)
> > +		return -ENOMEM;
> > +
> > +	bar->submap = new;
> > +	submap = &bar->submap[bar->num_submap];
> > +	submap->phys_addr = phys;
> > +	submap->size = size;
> > +	bar->num_submap++;
> > +
> > +	return 0;
> > +}
> > +
> > +static void pci_epf_test_clean_remote_edma(struct pci_epf_test *test)
> > +{
> > +	struct pci_epf_test_edma *edma = test->data;
> > +	struct pci_epf *epf = test->epf;
> > +	struct pci_epc *epc = epf->epc;
> > +	struct device *dev = epc->dev.parent;
> > +	struct iommu_domain *dom;
> > +	struct pci_epf_bar *bar;
> > +	enum pci_barno barno;
> > +
> > +	if (!edma)
> > +		return;
> > +
> > +	barno = edma->bar;
> > +	if (barno == NO_BAR)
> > +		return;
> > +
> > +	bar = &epf->bar[barno];
> > +
> > +	dom = iommu_get_domain_for_dev(dev);
> > +	if (dom && edma->reg_iova_sz) {
> > +		iommu_unmap(dom, edma->reg_iova, edma->reg_iova_sz);
> > +		edma->reg_iova = 0;
> > +		edma->reg_iova_sz = 0;
> > +	}
> > +
> > +	if (edma->test_buf) {
> > +		dma_free_coherent(dev, edma->test_buf_size,
> > +				  edma->test_buf,
> > +				  edma->test_buf_phys);
> > +		edma->test_buf = NULL;
> > +		edma->test_buf_phys = 0;
> > +		edma->test_buf_size = 0;
> > +	}
> > +
> > +	if (edma->info) {
> > +		pci_epf_free_space(epf, edma->info, barno, PRIMARY_INTERFACE);
> > +		edma->info = NULL;
> > +	}
> > +
> > +	pci_epf_test_clear_submaps(bar);
> > +	pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no, bar);
> > +
> > +	edma->bar = NO_BAR;
> > +	edma->enabled = false;
> > +}
> > +
> > +static int pci_epf_test_init_remote_edma(struct pci_epf_test *test)
> > +{
> > +	const struct pci_epc_features *epc_features = test->epc_features;
> > +	struct pci_epf_test_edma *edma;
> > +	struct pci_epf *epf = test->epf;
> > +	struct pci_epc *epc = epf->epc;
> > +	struct pcitest_edma_info *info;
> > +	struct device *dev = epc->dev.parent;
> > +	struct dw_edma_region region;
> > +	struct iommu_domain *dom;
> > +	size_t reg_sz_aligned, ll_rd_sz_aligned, ll_wr_sz_aligned;
> > +	phys_addr_t phys, ll_rd_phys, ll_wr_phys;
> > +	size_t ll_rd_size, ll_wr_size;
> > +	resource_size_t reg_size;
> > +	unsigned long iova;
> > +	size_t off, size;
> > +	int ret;
> > +
> > +	if (!test->dma_chan_tx || !test->dma_chan_rx)
> > +		return -ENODEV;
> > +
> > +	edma = devm_kzalloc(&epf->dev, sizeof(*edma), GFP_KERNEL);
> > +	if (!edma)
> > +		return -ENOMEM;
> > +	test->data = edma;
> > +
> > +	edma->bar = pci_epf_test_next_free_bar(test);
> > +	if (edma->bar == NO_BAR) {
> > +		dev_err(&epf->dev, "No spare BAR for remote eDMA (remote eDMA disabled)\n");
> > +		ret = -ENOSPC;
> > +		goto err;
> > +	}
> > +
> > +	ret = dw_edma_get_reg_window(epc, &edma->reg_phys, &reg_size);
> > +	if (ret) {
> > +		dev_err(dev, "failed to get edma reg window: %d\n", ret);
> > +		goto err;
> > +	}
> > +	dom = iommu_get_domain_for_dev(dev);
> > +	if (dom) {
> > +		phys = edma->reg_phys & PAGE_MASK;
> > +		size = PAGE_ALIGN(reg_size + edma->reg_phys - phys);
> > +		iova = phys;
> > +
> > +		ret = iommu_map(dom, iova, phys, size,
> > +				IOMMU_READ | IOMMU_WRITE | IOMMU_MMIO,
> > +				GFP_KERNEL);
> > +		if (ret) {
> > +			dev_err(dev, "failed to direct map eDMA reg: %d\n", ret);
> > +			goto err;
> > +		}
> > +		edma->reg_iova = iova;
> > +		edma->reg_iova_sz = size;
> > +	}
> > +
> > +	/* Get LL location addresses and sizes */
> > +	ret = dw_edma_chan_get_ll_region(test->dma_chan_rx, &region);
> > +	if (ret) {
> > +		dev_err(dev, "failed to get edma ll region for rx: %d\n", ret);
> > +		goto err;
> > +	}
> > +	ll_rd_phys = region.paddr;
> > +	ll_rd_size = region.sz;
> > +
> > +	ret = dw_edma_chan_get_ll_region(test->dma_chan_tx, &region);
> > +	if (ret) {
> > +		dev_err(dev, "failed to get edma ll region for tx: %d\n", ret);
> > +		goto err;
> > +	}
> > +	ll_wr_phys = region.paddr;
> > +	ll_wr_size = region.sz;
> > +
> > +	edma->test_buf_size = PCITEST_EDMA_TEST_BUF_SIZE;
> > +	edma->test_buf = dma_alloc_coherent(dev, edma->test_buf_size,
> > +					    &edma->test_buf_phys, GFP_KERNEL);
> > +	if (!edma->test_buf) {
> > +		ret = -ENOMEM;
> > +		goto err;
> > +	}
> > +
> > +	reg_sz_aligned = PAGE_ALIGN(reg_size);
> > +	ll_rd_sz_aligned = PAGE_ALIGN(ll_rd_size);
> > +	ll_wr_sz_aligned = PAGE_ALIGN(ll_wr_size);
> > +	edma->total_size = PAGE_SIZE + reg_sz_aligned + ll_rd_sz_aligned +
> > +			   ll_wr_sz_aligned;
> > +	size = roundup_pow_of_two(edma->total_size);
> > +
> > +	info = pci_epf_alloc_space(epf, size, edma->bar,
> > +				   epc_features, PRIMARY_INTERFACE);
> > +	if (!info) {
> > +		ret = -ENOMEM;
> > +		goto err;
> > +	}
> > +	memset(info, 0, size);
> > +
> > +	off = PAGE_SIZE;
> > +	info->magic = cpu_to_le32(PCITEST_EDMA_INFO_MAGIC);
> > +	info->version = cpu_to_le32(PCITEST_EDMA_INFO_VERSION);
> > +
> > +	info->reg_off = cpu_to_le32(off);
> > +	info->reg_size = cpu_to_le32(reg_size);
> > +	off += reg_sz_aligned;
> > +
> > +	info->ll_rd_phys = cpu_to_le64(ll_rd_phys);
> > +	info->ll_rd_off = cpu_to_le32(off);
> > +	info->ll_rd_size = cpu_to_le32(ll_rd_size);
> > +	off += ll_rd_sz_aligned;
> > +
> > +	info->ll_wr_phys = cpu_to_le64(ll_wr_phys);
> > +	info->ll_wr_off = cpu_to_le32(off);
> > +	info->ll_wr_size = cpu_to_le32(ll_wr_size);
> > +	off += ll_wr_sz_aligned;
> > +
> > +	info->test_buf_phys = cpu_to_le64(edma->test_buf_phys);
> > +	info->test_buf_size = cpu_to_le32(edma->test_buf_size);
> > +
> > +	edma->info = info;
> > +	edma->reg_submap_sz = reg_sz_aligned;
> > +	edma->ll_rd_phys = ll_rd_phys;
> > +	edma->ll_wr_phys = ll_wr_phys;
> > +	edma->ll_rd_sz_aligned = ll_rd_sz_aligned;
> > +	edma->ll_wr_sz_aligned = ll_wr_sz_aligned;
> > +
> > +	ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no,
> > +			      &epf->bar[edma->bar]);
> > +	if (ret) {
> > +		dev_err(dev,
> > +			"failed to init BAR%d for remote eDMA: %d\n",
> > +			edma->bar, ret);
> > +		goto err;
> > +	}
> > +	dev_info(dev, "BAR%d initialized for remote eDMA\n", edma->bar);
> > +
> > +	return 0;
> > +
> > +err:
> > +	pci_epf_test_clean_remote_edma(test);
> > +	devm_kfree(&epf->dev, edma);
> > +	test->data = NULL;
> > +	return ret;
> > +}
> > +
> > +static int pci_epf_test_map_remote_edma(struct pci_epf_test *test)
> > +{
> > +	struct pci_epf_test_edma *edma = test->data;
> > +	struct pcitest_edma_info *info;
> > +	struct pci_epf *epf = test->epf;
> > +	struct pci_epc *epc = epf->epc;
> > +	struct pci_epf_bar *bar;
> > +	enum pci_barno barno;
> > +	struct device *dev = epc->dev.parent;
> > +	int ret;
> > +
> > +	if (!edma)
> > +		return -ENODEV;
> > +
> > +	info = edma->info;
> > +	barno = edma->bar;
> > +
> > +	if (barno == NO_BAR)
> > +		return -ENOSPC;
> > +	if (!info || !edma->test_buf)
> > +		return -ENODEV;
> > +
> > +	bar = &epf->bar[barno];
> > +	pci_epf_test_clear_submaps(bar);
> > +
> > +	ret = pci_epf_test_add_submap(bar, bar->phys_addr, PAGE_SIZE);
> > +	if (ret)
> > +		return ret;
> > +
> > +	ret = pci_epf_test_add_submap(bar, edma->reg_phys, edma->reg_submap_sz);
> > +	if (ret)
> > +		goto err_submap;
> > +
> > +	ret = pci_epf_test_add_submap(bar, edma->ll_rd_phys,
> > +				      edma->ll_rd_sz_aligned);
> > +	if (ret)
> > +		goto err_submap;
> > +
> > +	ret = pci_epf_test_add_submap(bar, edma->ll_wr_phys,
> > +				      edma->ll_wr_sz_aligned);
> > +	if (ret)
> > +		goto err_submap;
> > +
> > +	if (bar->size > edma->total_size) {
> > +		ret = pci_epf_test_add_submap(bar, 0,
> > +					      bar->size - edma->total_size);
> > +		if (ret)
> > +			goto err_submap;
> > +	}
> > +
> > +	ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, bar);
> > +	if (ret) {
> > +		dev_err(dev, "failed to map BAR%d: %d\n", barno, ret);
> > +		goto err_submap;
> > +	}
> > +
> > +	/*
> > +	 * Endpoint-local interrupts must be ignored even if the host fails to
> > +	 * mask them.
> > +	 */
> > +	ret = dw_edma_chan_irq_config(test->dma_chan_tx, DW_EDMA_CH_IRQ_REMOTE);
> > +	if (ret) {
> > +		dev_err(dev, "failed to set irq mode for tx channel: %d\n",
> > +			ret);
> > +		goto err_bar;
> > +	}
> > +	ret = dw_edma_chan_irq_config(test->dma_chan_rx, DW_EDMA_CH_IRQ_REMOTE);
> > +	if (ret) {
> > +		dev_err(dev, "failed to set irq mode for rx channel: %d\n",
> > +			ret);
> > +		goto err_bar;
> > +	}
> > +
> > +	return 0;
> > +err_bar:
> > +	pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no, &epf->bar[barno]);
> > +err_submap:
> > +	pci_epf_test_clear_submaps(bar);
> > +	return ret;
> > +}
> > +
> > +static void pci_epf_test_remote_edma_setup(struct pci_epf_test *epf_test,
> > +					   struct pci_epf_test_reg *reg)
> > +{
> > +	struct pci_epf_test_edma *edma = epf_test->data;
> > +	size_t size = le32_to_cpu(reg->size);
> > +	void *buf;
> > +	int ret;
> > +
> > +	if (!edma || !edma->test_buf || size > edma->test_buf_size) {
> > +		reg->status = cpu_to_le32(STATUS_REMOTE_EDMA_SETUP_FAIL);
> > +		return;
> > +	}
> > +
> > +	buf = edma->test_buf;
> > +
> > +	if (!edma->enabled) {
> > +		/* NB. Currently DW eDMA is the only supported backend */
> > +		ret = pci_epf_test_map_remote_edma(epf_test);
> > +		if (ret) {
> > +			WRITE_ONCE(reg->status,
> > +				   cpu_to_le32(STATUS_REMOTE_EDMA_SETUP_FAIL));
> > +			return;
> > +		}
> > +		edma->enabled = true;
> > +	}
> > +
> > +	/* Populate the test buffer with random data */
> > +	get_random_bytes(buf, size);
> > +	reg->checksum = cpu_to_le32(crc32_le(~0, buf, size));
> > +
> > +	WRITE_ONCE(reg->status, cpu_to_le32(STATUS_REMOTE_EDMA_SETUP_SUCCESS));
> > +}
> > +
> > +static void pci_epf_test_remote_edma_checksum(struct pci_epf_test *epf_test,
> > +					      struct pci_epf_test_reg *reg)
> > +{
> > +	struct pci_epf_test_edma *edma = epf_test->data;
> > +	u32 status = le32_to_cpu(reg->status);
> > +	size_t size;
> > +	void *addr;
> > +	u32 crc32;
> > +
> > +	size = le32_to_cpu(reg->size);
> > +	if (!edma || !edma->test_buf || size > edma->test_buf_size) {
> > +		status |= STATUS_REMOTE_EDMA_CHECKSUM_FAIL;
> > +		reg->status = cpu_to_le32(status);
> > +		return;
> > +	}
> > +
> > +	addr = edma->test_buf;
> > +	crc32 = crc32_le(~0, addr, size);
> > +	status |= STATUS_REMOTE_EDMA_CHECKSUM_SUCCESS;
> > +
> > +	reg->checksum = cpu_to_le32(crc32);
> > +	reg->status = cpu_to_le32(status);
> > +}
> > +
> > +static void pci_epf_test_reset_dma_chan(struct dma_chan *chan)
> > +{
> > +	dw_edma_chan_irq_config(chan, DW_EDMA_CH_IRQ_DEFAULT);
> > +}
> > +#else
> > +static bool pci_epf_test_bar_is_reserved(struct pci_epf_test *test,
> > +					 enum pci_barno barno)
> > +{
> > +	return false;
> > +}
> > +
> > +static void pci_epf_test_clean_remote_edma(struct pci_epf_test *test)
> > +{
> > +}
> > +
> > +static int pci_epf_test_init_remote_edma(struct pci_epf_test *test)
> > +{
> > +	return -EOPNOTSUPP;
> > +}
> > +
> > +static void pci_epf_test_remote_edma_setup(struct pci_epf_test *epf_test,
> > +					   struct pci_epf_test_reg *reg)
> > +{
> > +	reg->status = cpu_to_le32(STATUS_REMOTE_EDMA_SETUP_FAIL);
> > +}
> > +
> > +static void pci_epf_test_remote_edma_checksum(struct pci_epf_test *epf_test,
> > +					      struct pci_epf_test_reg *reg)
> > +{
> > +	reg->status = cpu_to_le32(STATUS_REMOTE_EDMA_CHECKSUM_FAIL);
> > +}
> > +
> > +static void pci_epf_test_reset_dma_chan(struct dma_chan *chan)
> > +{
> > +}
> > +#endif
> > +
> >  static void pci_epf_test_dma_callback(void *param)
> >  {
> >  	struct pci_epf_test *epf_test = param;
> > @@ -168,6 +626,8 @@ static int pci_epf_test_data_transfer(struct pci_epf_test *epf_test,
> >  		return -EINVAL;
> >  	}
> >
> > +	pci_epf_test_reset_dma_chan(chan);
> > +
> >  	if (epf_test->dma_private) {
> >  		sconf.direction = dir;
> >  		if (dir == DMA_MEM_TO_DEV)
> > @@ -870,6 +1330,14 @@ static void pci_epf_test_cmd_handler(struct work_struct *work)
> >  		pci_epf_test_disable_doorbell(epf_test, reg);
> >  		pci_epf_test_raise_irq(epf_test, reg);
> >  		break;
> > +	case COMMAND_REMOTE_EDMA_SETUP:
> > +		pci_epf_test_remote_edma_setup(epf_test, reg);
> > +		pci_epf_test_raise_irq(epf_test, reg);
> > +		break;
> > +	case COMMAND_REMOTE_EDMA_CHECKSUM:
> > +		pci_epf_test_remote_edma_checksum(epf_test, reg);
> > +		pci_epf_test_raise_irq(epf_test, reg);
> > +		break;
> >  	default:
> >  		dev_err(dev, "Invalid command 0x%x\n", command);
> >  		break;
> > @@ -961,6 +1429,10 @@ static int pci_epf_test_epc_init(struct pci_epf *epf)
> >  	if (ret)
> >  		epf_test->dma_supported = false;
> >
> > +	ret = pci_epf_test_init_remote_edma(epf_test);
> > +	if (ret && ret != -EOPNOTSUPP)
> > +		dev_warn(dev, "Remote eDMA setup failed\n");
> > +
> >  	if (epf->vfunc_no <= 1) {
> >  		ret = pci_epc_write_header(epc, epf->func_no, epf->vfunc_no, header);
> >  		if (ret) {
> > @@ -1007,6 +1479,7 @@ static void pci_epf_test_epc_deinit(struct pci_epf *epf)
> >  	struct pci_epf_test *epf_test = epf_get_drvdata(epf);
> >
> >  	cancel_delayed_work_sync(&epf_test->cmd_handler);
> > +	pci_epf_test_clean_remote_edma(epf_test);
> >  	pci_epf_test_clean_dma_chan(epf_test);
> >  	pci_epf_test_clear_bar(epf);
> >  }
> > @@ -1076,6 +1549,9 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf)
> >  		if (bar == test_reg_bar)
> >  			continue;
> >
> > +		if (pci_epf_test_bar_is_reserved(epf_test, bar))
> > +			continue;
> > +
> >  		if (epc_features->bar[bar].type == BAR_FIXED)
> >  			test_reg_size = epc_features->bar[bar].fixed_size;
> >  		else
> > @@ -1146,6 +1622,7 @@ static void pci_epf_test_unbind(struct pci_epf *epf)
> >
> >  	cancel_delayed_work_sync(&epf_test->cmd_handler);
> >  	if (epc->init_complete) {
> > +		pci_epf_test_clean_remote_edma(epf_test);
> >  		pci_epf_test_clean_dma_chan(epf_test);
> >  		pci_epf_test_clear_bar(epf);
> >  	}
> > --
> > 2.51.0
> >