From: Jason Gunthorpe <jgg@nvidia.com>
Create a new data structure to hold an array of invalidations that need to
be performed for the domain based on what masters are attached, to replace
the single smmu pointer and linked list of masters in the current design.
Each array entry holds one of the invalidation actions - S1_ASID, S2_VMID,
ATS or their variant with information to feed invalidation commands to HW.
It is structured so that multiple SMMUs can participate in the same array,
removing one key limitation of the current system.
To maximize performance, a sorted array is used as the data structure. It
allows grouping SYNCs together to parallelize invalidations. For instance,
it will group all the ATS entries after the ASID/VMID entry, so they will
all be pushed to the PCI devices in parallel with one SYNC.
To minimize the locking cost on the invalidation fast path (reader of the
invalidation array), the array is managed with RCU.
Provide a set of APIs to add/delete entries to/from an array, which cover
cannot-fail attach cases, e.g. attaching to arm_smmu_blocked_domain. Also
add kunit coverage for those APIs.
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Co-developed-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Nicolin Chen <nicolinc@nvidia.com>
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 79 +++++++
.../iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c | 93 ++++++++
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 212 ++++++++++++++++++
3 files changed, 384 insertions(+)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 96a23ca633cb6..34fcc1a930e6a 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -649,6 +649,82 @@ struct arm_smmu_cmdq_batch {
int num;
};
+/*
+ * The order here also determines the sequence in which commands are sent to the
+ * command queue. E.g. TLBI must be done before ATC_INV.
+ */
+enum arm_smmu_inv_type {
+ INV_TYPE_S1_ASID,
+ INV_TYPE_S2_VMID,
+ INV_TYPE_S2_VMID_S1_CLEAR,
+ INV_TYPE_ATS,
+ INV_TYPE_ATS_FULL,
+};
+
+struct arm_smmu_inv {
+ struct arm_smmu_device *smmu;
+ u8 type;
+ u8 size_opcode;
+ u8 nsize_opcode;
+ u32 id; /* ASID or VMID or SID */
+ union {
+ size_t pgsize; /* ARM_SMMU_FEAT_RANGE_INV */
+ u32 ssid; /* INV_TYPE_ATS */
+ };
+
+ refcount_t users; /* users=0 to mark as a trash to be purged */
+};
+
+/**
+ * struct arm_smmu_invs - Per-domain invalidation array
+ * @num_invs: number of invalidations in the flexible array
+ * @rcu: rcu head for kfree_rcu()
+ * @inv: flexible invalidation array
+ *
+ * The arm_smmu_invs is an RCU data structure. During a ->attach_dev callback,
+ * arm_smmu_invs_merge(), arm_smmu_invs_unref() and arm_smmu_invs_purge() will
+ * be used to allocate a new copy of an old array for addition and deletion in
+ * the old domain's and new domain's invs arrays.
+ *
+ * The arm_smmu_invs_unref() mutates a given array, by internally reducing the
+ * users counts of some given entries. This exists to support a no-fail routine
+ * like attaching to an IOMMU_DOMAIN_BLOCKED. And it could pair with a followup
+ * arm_smmu_invs_purge() call to generate a new clean array.
+ *
+ * Concurrent invalidation thread will push every invalidation described in the
+ * array into the command queue for each invalidation event. It is designed like
+ * this to optimize the invalidation fast path by avoiding locks.
+ *
+ * A domain can be shared across SMMU instances. When an instance gets removed,
+ * it would delete all the entries that belong to that SMMU instance. Then, a
+ * synchronize_rcu() would have to be called to sync the array, to prevent any
+ * concurrent invalidation thread accessing the old array from issuing commands
+ * to the command queue of a removed SMMU instance.
+ */
+struct arm_smmu_invs {
+ size_t num_invs;
+ struct rcu_head rcu;
+ struct arm_smmu_inv inv[];
+};
+
+static inline struct arm_smmu_invs *arm_smmu_invs_alloc(size_t num_invs)
+{
+ struct arm_smmu_invs *new_invs;
+
+ new_invs = kzalloc(struct_size(new_invs, inv, num_invs), GFP_KERNEL);
+ if (!new_invs)
+ return ERR_PTR(-ENOMEM);
+ new_invs->num_invs = num_invs;
+ return new_invs;
+}
+
+struct arm_smmu_invs *arm_smmu_invs_merge(struct arm_smmu_invs *invs,
+ struct arm_smmu_invs *to_merge);
+size_t arm_smmu_invs_unref(struct arm_smmu_invs *invs,
+ struct arm_smmu_invs *to_unref);
+struct arm_smmu_invs *arm_smmu_invs_purge(struct arm_smmu_invs *invs,
+ size_t num_dels);
+
struct arm_smmu_evtq {
struct arm_smmu_queue q;
struct iopf_queue *iopf;
@@ -875,6 +951,8 @@ struct arm_smmu_domain {
struct iommu_domain domain;
+ struct arm_smmu_invs __rcu *invs;
+
/* List of struct arm_smmu_master_domain */
struct list_head devices;
spinlock_t devices_lock;
@@ -956,6 +1034,7 @@ struct arm_smmu_domain *arm_smmu_domain_alloc(void);
static inline void arm_smmu_domain_free(struct arm_smmu_domain *smmu_domain)
{
+ kfree_rcu(smmu_domain->invs, rcu);
kfree(smmu_domain);
}
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c
index d2671bfd37981..417a2b5ea2024 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c
@@ -567,6 +567,98 @@ static void arm_smmu_v3_write_cd_test_sva_release(struct kunit *test)
NUM_EXPECTED_SYNCS(2));
}
+static void arm_smmu_v3_invs_test_verify(struct kunit *test,
+ struct arm_smmu_invs *invs, int num,
+ const int *ids, const int *users)
+{
+ KUNIT_EXPECT_EQ(test, invs->num_invs, num);
+ while (num--) {
+ KUNIT_EXPECT_EQ(test, invs->inv[num].id, ids[num]);
+ KUNIT_EXPECT_EQ(test, refcount_read(&invs->inv[num].users),
+ users[num]);
+ }
+}
+
+static struct arm_smmu_invs invs1 = {
+ .num_invs = 3,
+ .inv = { { .type = INV_TYPE_S2_VMID, .id = 1, },
+ { .type = INV_TYPE_S2_VMID, .id = 2, },
+ { .type = INV_TYPE_S2_VMID, .id = 3, }, },
+};
+
+static struct arm_smmu_invs invs2 = {
+ .num_invs = 3,
+ .inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */
+ { .type = INV_TYPE_ATS, .id = 4, },
+ { .type = INV_TYPE_ATS, .id = 5, }, },
+};
+
+static struct arm_smmu_invs invs3 = {
+ .num_invs = 3,
+ .inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */
+ { .type = INV_TYPE_ATS, .id = 5, }, /* recover a trash */
+ { .type = INV_TYPE_ATS, .id = 6, }, },
+};
+
+static void arm_smmu_v3_invs_test(struct kunit *test)
+{
+ const int results1[2][3] = { { 1, 2, 3, }, { 1, 1, 1, }, };
+ const int results2[2][5] = { { 1, 2, 3, 4, 5, }, { 2, 1, 1, 1, 1, }, };
+ const int results3[2][5] = { { 1, 2, 3, 4, 5, }, { 1, 1, 1, 0, 0, }, };
+ const int results4[2][5] = { { 1, 2, 3, 5, 6, }, { 2, 1, 1, 1, 1, }, };
+ const int results5[2][5] = { { 1, 2, 3, 5, 6, }, { 1, 0, 0, 1, 1, }, };
+ const int results6[2][5] = { { 1, 2, 3, 5, 6, }, { 0, 0, 0, 0, 0, }, };
+ struct arm_smmu_invs *test_a, *test_b;
+ size_t num_dels;
+
+ /* New array */
+ test_a = arm_smmu_invs_alloc(0);
+ KUNIT_EXPECT_EQ(test, test_a->num_invs, 0);
+
+ /* Test1: merge invs1 (new array) */
+ test_b = arm_smmu_invs_merge(test_a, &invs1);
+ kfree(test_a);
+ arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results1[0]),
+ results1[0], results1[1]);
+
+ /* Test2: merge invs2 (new array) */
+ test_a = arm_smmu_invs_merge(test_b, &invs2);
+ kfree(test_b);
+ arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results2[0]),
+ results2[0], results2[1]);
+
+ /* Test3: unref invs2 (same array) */
+ num_dels = arm_smmu_invs_unref(test_a, &invs2);
+ arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results3[0]),
+ results3[0], results3[1]);
+ KUNIT_EXPECT_EQ(test, num_dels, 2);
+
+ /* Test4: merge invs3 (new array) */
+ test_b = arm_smmu_invs_merge(test_a, &invs3);
+ kfree(test_a);
+ arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results4[0]),
+ results4[0], results4[1]);
+
+ /* Test5: unref invs1 (same array) */
+ num_dels = arm_smmu_invs_unref(test_b, &invs1);
+ arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results5[0]),
+ results5[0], results5[1]);
+ KUNIT_EXPECT_EQ(test, num_dels, 2);
+
+ /* Test6: unref invs3 (same array) */
+ num_dels = arm_smmu_invs_unref(test_b, &invs3);
+ arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results6[0]),
+ results6[0], results6[1]);
+ KUNIT_EXPECT_EQ(test, num_dels, 5);
+
+ /* Test7: purge test_b (new array) */
+ test_a = arm_smmu_invs_purge(test_b, num_dels);
+ kfree(test_b);
+ KUNIT_EXPECT_EQ(test, test_a->num_invs, 0);
+
+ kfree(test_a);
+}
+
static struct kunit_case arm_smmu_v3_test_cases[] = {
KUNIT_CASE(arm_smmu_v3_write_ste_test_bypass_to_abort),
KUNIT_CASE(arm_smmu_v3_write_ste_test_abort_to_bypass),
@@ -590,6 +682,7 @@ static struct kunit_case arm_smmu_v3_test_cases[] = {
KUNIT_CASE(arm_smmu_v3_write_ste_test_s2_to_s1_stall),
KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_clear),
KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_release),
+ KUNIT_CASE(arm_smmu_v3_invs_test),
{},
};
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 08af5f2d1235a..83d842bd88817 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -26,6 +26,7 @@
#include <linux/pci.h>
#include <linux/pci-ats.h>
#include <linux/platform_device.h>
+#include <linux/sort.h>
#include <linux/string_choices.h>
#include <kunit/visibility.h>
#include <uapi/linux/iommufd.h>
@@ -1033,6 +1034,209 @@ void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid)
arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
}
+static int arm_smmu_invs_cmp(const void *_l, const void *_r)
+{
+ const struct arm_smmu_inv *l = _l;
+ const struct arm_smmu_inv *r = _r;
+
+ if (l->smmu != r->smmu)
+ return cmp_int((uintptr_t)l->smmu, (uintptr_t)r->smmu);
+ if (l->type != r->type)
+ return cmp_int(l->type, r->type);
+ return cmp_int(l->id, r->id);
+}
+
+/*
+ * Merge compare of two sorted arrays items. If one side is past the end of the
+ * array, return the other side to let it run out the iteration.
+ */
+static inline int
+arm_smmu_invs_merge_cmp(const struct arm_smmu_invs *l, size_t l_idx,
+ const struct arm_smmu_invs *r, size_t r_idx)
+{
+ if (l_idx != l->num_invs && r_idx != r->num_invs)
+ return arm_smmu_invs_cmp(&l->inv[l_idx], &r->inv[r_idx]);
+ if (l_idx != l->num_invs)
+ return -1;
+ return 1;
+}
+
+/**
+ * arm_smmu_invs_merge() - Merge @to_merge into @invs and generate a new array
+ * @invs: the base invalidation array
+ * @to_merge: an array of invlidations to merge
+ *
+ * Return: a newly allocated array on success, or ERR_PTR
+ *
+ * This function must be locked and serialized with arm_smmu_invs_unref() and
+ * arm_smmu_invs_purge(), but do not lockdep on any lock for KUNIT test.
+ *
+ * Either @invs or @to_merge must be sorted itself. This ensures the returned
+ * array will be sorted as well.
+ *
+ * Caller is resposible for freeing the @invs and the returned new one.
+ *
+ * Entries marked as trash will be purged in the returned array.
+ */
+VISIBLE_IF_KUNIT
+struct arm_smmu_invs *arm_smmu_invs_merge(struct arm_smmu_invs *invs,
+ struct arm_smmu_invs *to_merge)
+{
+ struct arm_smmu_invs *new_invs;
+ struct arm_smmu_inv *new;
+ size_t num_adds = 0;
+ size_t num_dels = 0;
+ size_t i, j;
+
+ for (i = j = 0; i != invs->num_invs || j != to_merge->num_invs;) {
+ int cmp = arm_smmu_invs_merge_cmp(invs, i, to_merge, j);
+
+ if (cmp < 0) {
+ /* no found in to_merge, leave alone but delete trash */
+ if (!refcount_read(&invs->inv[i].users))
+ num_dels++;
+ i++;
+ } else if (cmp == 0) {
+ /* same item */
+ i++;
+ j++;
+ } else {
+ /* unique to to_merge */
+ num_adds++;
+ j++;
+ }
+ }
+
+ new_invs = arm_smmu_invs_alloc(invs->num_invs - num_dels + num_adds);
+ if (IS_ERR(new_invs))
+ return new_invs;
+
+ new = new_invs->inv;
+ for (i = j = 0; i != invs->num_invs || j != to_merge->num_invs;) {
+ int cmp = arm_smmu_invs_merge_cmp(invs, i, to_merge, j);
+
+ if (cmp <= 0 && !refcount_read(&invs->inv[i].users)) {
+ i++;
+ continue;
+ }
+
+ if (cmp < 0) {
+ *new = invs->inv[i];
+ i++;
+ } else if (cmp == 0) {
+ *new = invs->inv[i];
+ refcount_inc(&new->users);
+ i++;
+ j++;
+ } else {
+ *new = to_merge->inv[j];
+ refcount_set(&new->users, 1);
+ j++;
+ }
+ new++;
+ }
+
+ WARN_ON(new != new_invs->inv + new_invs->num_invs);
+
+ return new_invs;
+}
+EXPORT_SYMBOL_IF_KUNIT(arm_smmu_invs_merge);
+
+/**
+ * arm_smmu_invs_unref() - Find in @invs for all entries in @to_unref, decrease
+ * the user counts without deletions
+ * @invs: the base invalidation array
+ * @to_unref: an array of invlidations to decrease their user counts
+ *
+ * Return: the number of trash entries in the array, for arm_smmu_invs_purge()
+ *
+ * This function will not fail. Any entry with users=0 will be marked as trash.
+ * All trash entries will remain in the @invs until being completely deleted by
+ * the next arm_smmu_invs_merge() or an arm_smmu_invs_purge() function call.
+ *
+ * This function must be locked and serialized with arm_smmu_invs_merge() and
+ * arm_smmu_invs_purge(), but do not lockdep on any lock for KUNIT test.
+ *
+ * Note that the @invs->num_invs will not be updated, even if the actual number
+ * of invalidations are decreased. Readers should take the read lock to iterate
+ * each entry and check its users counter until @inv->num_invs.
+ */
+VISIBLE_IF_KUNIT
+size_t arm_smmu_invs_unref(struct arm_smmu_invs *invs,
+ struct arm_smmu_invs *to_unref)
+{
+ size_t num_dels = 0;
+ size_t i, j;
+
+ for (i = j = 0; i != invs->num_invs || j != to_unref->num_invs;) {
+ int cmp;
+
+ if (!refcount_read(&invs->inv[i].users)) {
+ num_dels++;
+ i++;
+ continue;
+ }
+
+ cmp = arm_smmu_invs_merge_cmp(invs, i, to_unref, j);
+ if (cmp < 0) {
+ /* not found in to_unref, leave alone */
+ i++;
+ } else if (cmp == 0) {
+ /* same item */
+ if (refcount_dec_and_test(&invs->inv[i].users))
+ num_dels++;
+ i++;
+ j++;
+ } else {
+ /* item in to_unref is not in invs or already a trash */
+ WARN_ON(true);
+ j++;
+ }
+ }
+ return num_dels;
+}
+EXPORT_SYMBOL_IF_KUNIT(arm_smmu_invs_unref);
+
+/**
+ * arm_smmu_invs_purge() - Purge all the trash entries in the @invs
+ * @invs: the base invalidation array
+ * @num_dels: expected number of trash entries, typically the return value from
+ * a prior arm_smmu_invs_unref() call
+ *
+ * Return: a newly allocated array on success removing all the trash entries, or
+ * NULL on failure
+ *
+ * This function must be locked and serialized with arm_smmu_invs_merge() and
+ * arm_smmu_invs_unref(), but do not lockdep on any lock for KUNIT test.
+ *
+ * Caller is resposible for freeing the @invs and the returned new one.
+ */
+VISIBLE_IF_KUNIT
+struct arm_smmu_invs *arm_smmu_invs_purge(struct arm_smmu_invs *invs,
+ size_t num_dels)
+{
+ struct arm_smmu_invs *new_invs;
+ size_t i, j;
+
+ if (WARN_ON(invs->num_invs < num_dels))
+ return NULL;
+
+ new_invs = arm_smmu_invs_alloc(invs->num_invs - num_dels);
+ if (IS_ERR(new_invs))
+ return NULL;
+
+ for (i = j = 0; i != invs->num_invs; i++) {
+ if (!refcount_read(&invs->inv[i].users))
+ continue;
+ new_invs->inv[j] = invs->inv[i];
+ j++;
+ }
+
+ WARN_ON(j != new_invs->num_invs);
+ return new_invs;
+}
+EXPORT_SYMBOL_IF_KUNIT(arm_smmu_invs_purge);
+
/*
* Based on the value of ent report which bits of the STE the HW will access. It
* would be nice if this was complete according to the spec, but minimally it
@@ -2468,13 +2672,21 @@ static bool arm_smmu_enforce_cache_coherency(struct iommu_domain *domain)
struct arm_smmu_domain *arm_smmu_domain_alloc(void)
{
struct arm_smmu_domain *smmu_domain;
+ struct arm_smmu_invs *new_invs;
smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
if (!smmu_domain)
return ERR_PTR(-ENOMEM);
+ new_invs = arm_smmu_invs_alloc(0);
+ if (IS_ERR(new_invs)) {
+ kfree(smmu_domain);
+ return ERR_CAST(new_invs);
+ }
+
INIT_LIST_HEAD(&smmu_domain->devices);
spin_lock_init(&smmu_domain->devices_lock);
+ rcu_assign_pointer(smmu_domain->invs, new_invs);
return smmu_domain;
}
--
2.43.0Hi Nicolin,
kernel test robot noticed the following build errors:
[auto build test ERROR on soc/for-next]
[also build test ERROR on linus/master v6.17-rc5 next-20250909]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Nicolin-Chen/iommu-arm-smmu-v3-Clear-cmds-num-after-arm_smmu_cmdq_batch_submit/20250909-073052
base: https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git for-next
patch link: https://lore.kernel.org/r/80310b98efa4bd7e95d7b3ca302f40d4d69e59c5.1757373449.git.nicolinc%40nvidia.com
patch subject: [PATCH rfcv2 4/8] iommu/arm-smmu-v3: Introduce a per-domain arm_smmu_invs array
config: arm64-randconfig-003-20250909 (https://download.01.org/0day-ci/archive/20250909/202509092020.mxUyqGcN-lkp@intel.com/config)
compiler: aarch64-linux-gcc (GCC) 11.5.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250909/202509092020.mxUyqGcN-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202509092020.mxUyqGcN-lkp@intel.com/
All errors (new ones prefixed by >>):
>> drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c:1082:23: error: static declaration of 'arm_smmu_invs_merge' follows non-static declaration
1082 | struct arm_smmu_invs *arm_smmu_invs_merge(struct arm_smmu_invs *invs,
| ^~~~~~~~~~~~~~~~~~~
In file included from drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c:34:
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h:721:23: note: previous declaration of 'arm_smmu_invs_merge' with type 'struct arm_smmu_invs *(struct arm_smmu_invs *, struct arm_smmu_invs *)'
721 | struct arm_smmu_invs *arm_smmu_invs_merge(struct arm_smmu_invs *invs,
| ^~~~~~~~~~~~~~~~~~~
>> drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c:1165:8: error: static declaration of 'arm_smmu_invs_unref' follows non-static declaration
1165 | size_t arm_smmu_invs_unref(struct arm_smmu_invs *invs,
| ^~~~~~~~~~~~~~~~~~~
In file included from drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c:34:
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h:723:8: note: previous declaration of 'arm_smmu_invs_unref' with type 'size_t(struct arm_smmu_invs *, struct arm_smmu_invs *)' {aka 'long unsigned int(struct arm_smmu_invs *, struct arm_smmu_invs *)'}
723 | size_t arm_smmu_invs_unref(struct arm_smmu_invs *invs,
| ^~~~~~~~~~~~~~~~~~~
>> drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c:1215:23: error: static declaration of 'arm_smmu_invs_purge' follows non-static declaration
1215 | struct arm_smmu_invs *arm_smmu_invs_purge(struct arm_smmu_invs *invs,
| ^~~~~~~~~~~~~~~~~~~
In file included from drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c:34:
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h:725:23: note: previous declaration of 'arm_smmu_invs_purge' with type 'struct arm_smmu_invs *(struct arm_smmu_invs *, size_t)' {aka 'struct arm_smmu_invs *(struct arm_smmu_invs *, long unsigned int)'}
725 | struct arm_smmu_invs *arm_smmu_invs_purge(struct arm_smmu_invs *invs,
| ^~~~~~~~~~~~~~~~~~~
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c:1215:23: warning: 'arm_smmu_invs_purge' defined but not used [-Wunused-function]
1215 | struct arm_smmu_invs *arm_smmu_invs_purge(struct arm_smmu_invs *invs,
| ^~~~~~~~~~~~~~~~~~~
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c:1165:8: warning: 'arm_smmu_invs_unref' defined but not used [-Wunused-function]
1165 | size_t arm_smmu_invs_unref(struct arm_smmu_invs *invs,
| ^~~~~~~~~~~~~~~~~~~
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c:1082:23: warning: 'arm_smmu_invs_merge' defined but not used [-Wunused-function]
1082 | struct arm_smmu_invs *arm_smmu_invs_merge(struct arm_smmu_invs *invs,
| ^~~~~~~~~~~~~~~~~~~
vim +/arm_smmu_invs_merge +1082 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
1063
1064 /**
1065 * arm_smmu_invs_merge() - Merge @to_merge into @invs and generate a new array
1066 * @invs: the base invalidation array
1067 * @to_merge: an array of invlidations to merge
1068 *
1069 * Return: a newly allocated array on success, or ERR_PTR
1070 *
1071 * This function must be locked and serialized with arm_smmu_invs_unref() and
1072 * arm_smmu_invs_purge(), but do not lockdep on any lock for KUNIT test.
1073 *
1074 * Either @invs or @to_merge must be sorted itself. This ensures the returned
1075 * array will be sorted as well.
1076 *
1077 * Caller is resposible for freeing the @invs and the returned new one.
1078 *
1079 * Entries marked as trash will be purged in the returned array.
1080 */
1081 VISIBLE_IF_KUNIT
> 1082 struct arm_smmu_invs *arm_smmu_invs_merge(struct arm_smmu_invs *invs,
1083 struct arm_smmu_invs *to_merge)
1084 {
1085 struct arm_smmu_invs *new_invs;
1086 struct arm_smmu_inv *new;
1087 size_t num_adds = 0;
1088 size_t num_dels = 0;
1089 size_t i, j;
1090
1091 for (i = j = 0; i != invs->num_invs || j != to_merge->num_invs;) {
1092 int cmp = arm_smmu_invs_merge_cmp(invs, i, to_merge, j);
1093
1094 if (cmp < 0) {
1095 /* no found in to_merge, leave alone but delete trash */
1096 if (!refcount_read(&invs->inv[i].users))
1097 num_dels++;
1098 i++;
1099 } else if (cmp == 0) {
1100 /* same item */
1101 i++;
1102 j++;
1103 } else {
1104 /* unique to to_merge */
1105 num_adds++;
1106 j++;
1107 }
1108 }
1109
1110 new_invs = arm_smmu_invs_alloc(invs->num_invs - num_dels + num_adds);
1111 if (IS_ERR(new_invs))
1112 return new_invs;
1113
1114 new = new_invs->inv;
1115 for (i = j = 0; i != invs->num_invs || j != to_merge->num_invs;) {
1116 int cmp = arm_smmu_invs_merge_cmp(invs, i, to_merge, j);
1117
1118 if (cmp <= 0 && !refcount_read(&invs->inv[i].users)) {
1119 i++;
1120 continue;
1121 }
1122
1123 if (cmp < 0) {
1124 *new = invs->inv[i];
1125 i++;
1126 } else if (cmp == 0) {
1127 *new = invs->inv[i];
1128 refcount_inc(&new->users);
1129 i++;
1130 j++;
1131 } else {
1132 *new = to_merge->inv[j];
1133 refcount_set(&new->users, 1);
1134 j++;
1135 }
1136 new++;
1137 }
1138
1139 WARN_ON(new != new_invs->inv + new_invs->num_invs);
1140
1141 return new_invs;
1142 }
1143 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_invs_merge);
1144
1145 /**
1146 * arm_smmu_invs_unref() - Find in @invs for all entries in @to_unref, decrease
1147 * the user counts without deletions
1148 * @invs: the base invalidation array
1149 * @to_unref: an array of invlidations to decrease their user counts
1150 *
1151 * Return: the number of trash entries in the array, for arm_smmu_invs_purge()
1152 *
1153 * This function will not fail. Any entry with users=0 will be marked as trash.
1154 * All trash entries will remain in the @invs until being completely deleted by
1155 * the next arm_smmu_invs_merge() or an arm_smmu_invs_purge() function call.
1156 *
1157 * This function must be locked and serialized with arm_smmu_invs_merge() and
1158 * arm_smmu_invs_purge(), but do not lockdep on any lock for KUNIT test.
1159 *
1160 * Note that the @invs->num_invs will not be updated, even if the actual number
1161 * of invalidations are decreased. Readers should take the read lock to iterate
1162 * each entry and check its users counter until @inv->num_invs.
1163 */
1164 VISIBLE_IF_KUNIT
> 1165 size_t arm_smmu_invs_unref(struct arm_smmu_invs *invs,
1166 struct arm_smmu_invs *to_unref)
1167 {
1168 size_t num_dels = 0;
1169 size_t i, j;
1170
1171 for (i = j = 0; i != invs->num_invs || j != to_unref->num_invs;) {
1172 int cmp;
1173
1174 if (!refcount_read(&invs->inv[i].users)) {
1175 num_dels++;
1176 i++;
1177 continue;
1178 }
1179
1180 cmp = arm_smmu_invs_merge_cmp(invs, i, to_unref, j);
1181 if (cmp < 0) {
1182 /* not found in to_unref, leave alone */
1183 i++;
1184 } else if (cmp == 0) {
1185 /* same item */
1186 if (refcount_dec_and_test(&invs->inv[i].users))
1187 num_dels++;
1188 i++;
1189 j++;
1190 } else {
1191 /* item in to_unref is not in invs or already a trash */
1192 WARN_ON(true);
1193 j++;
1194 }
1195 }
1196 return num_dels;
1197 }
1198 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_invs_unref);
1199
1200 /**
1201 * arm_smmu_invs_purge() - Purge all the trash entries in the @invs
1202 * @invs: the base invalidation array
1203 * @num_dels: expected number of trash entries, typically the return value from
1204 * a prior arm_smmu_invs_unref() call
1205 *
1206 * Return: a newly allocated array on success removing all the trash entries, or
1207 * NULL on failure
1208 *
1209 * This function must be locked and serialized with arm_smmu_invs_merge() and
1210 * arm_smmu_invs_unref(), but do not lockdep on any lock for KUNIT test.
1211 *
1212 * Caller is resposible for freeing the @invs and the returned new one.
1213 */
1214 VISIBLE_IF_KUNIT
> 1215 struct arm_smmu_invs *arm_smmu_invs_purge(struct arm_smmu_invs *invs,
1216 size_t num_dels)
1217 {
1218 struct arm_smmu_invs *new_invs;
1219 size_t i, j;
1220
1221 if (WARN_ON(invs->num_invs < num_dels))
1222 return NULL;
1223
1224 new_invs = arm_smmu_invs_alloc(invs->num_invs - num_dels);
1225 if (IS_ERR(new_invs))
1226 return NULL;
1227
1228 for (i = j = 0; i != invs->num_invs; i++) {
1229 if (!refcount_read(&invs->inv[i].users))
1230 continue;
1231 new_invs->inv[j] = invs->inv[i];
1232 j++;
1233 }
1234
1235 WARN_ON(j != new_invs->num_invs);
1236 return new_invs;
1237 }
1238 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_invs_purge);
1239
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
© 2016 - 2025 Red Hat, Inc.