MADV_DONTNEED[_LOCKED] internal logic can be invoked with batched tlb
flushes. Update vector_madvise(), which is called for
process_madvise(), to use that in the way, by passing an mmu_gather
object that it initializes before starting the internal works, and
flushing the gathered tlb entries at once after all the internal works
are done.
Signed-off-by: SeongJae Park <sj@kernel.org>
---
mm/madvise.c | 40 ++++++++++++++++++++++++++++++++++++++--
1 file changed, 38 insertions(+), 2 deletions(-)
diff --git a/mm/madvise.c b/mm/madvise.c
index 767c5d21ee75..efa4184d6cf5 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -905,6 +905,7 @@ static bool madvise_dontneed_free_valid_vma(struct vm_area_struct *vma,
struct madvise_behavior {
int behavior;
+ struct mmu_gather *tlb;
};
static long madvise_dontneed_free(struct vm_area_struct *vma,
@@ -964,7 +965,8 @@ static long madvise_dontneed_free(struct vm_area_struct *vma,
}
if (behavior == MADV_DONTNEED || behavior == MADV_DONTNEED_LOCKED)
- return madvise_dontneed_single_vma(NULL, vma, start, end);
+ return madvise_dontneed_single_vma(
+ madv_behavior->tlb, vma, start, end);
else if (behavior == MADV_FREE)
return madvise_free_single_vma(NULL, vma, start, end);
else
@@ -1802,19 +1804,50 @@ SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
return do_madvise(current->mm, start, len_in, behavior);
}
+static bool vector_madvise_batch_tlb_flush(int behavior)
+{
+ switch (behavior) {
+ case MADV_DONTNEED:
+ case MADV_DONTNEED_LOCKED:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static void vector_madvise_init_tlb(struct madvise_behavior *madv_behavior,
+ struct mm_struct *mm)
+{
+ if (!vector_madvise_batch_tlb_flush(madv_behavior->behavior))
+ return;
+ tlb_gather_mmu(madv_behavior->tlb, mm);
+}
+
+static void vector_madvise_finish_tlb(struct madvise_behavior *madv_behavior)
+{
+ if (!vector_madvise_batch_tlb_flush(madv_behavior->behavior))
+ return;
+ tlb_finish_mmu(madv_behavior->tlb);
+}
+
/* Perform an madvise operation over a vector of addresses and lengths. */
static ssize_t vector_madvise(struct mm_struct *mm, struct iov_iter *iter,
int behavior)
{
ssize_t ret = 0;
size_t total_len;
- struct madvise_behavior madv_behavior = {.behavior = behavior};
+ struct mmu_gather tlb;
+ struct madvise_behavior madv_behavior = {
+ .behavior = behavior,
+ .tlb = &tlb,
+ };
total_len = iov_iter_count(iter);
ret = madvise_lock(mm, behavior);
if (ret)
return ret;
+ vector_madvise_init_tlb(&madv_behavior, mm);
while (iov_iter_count(iter)) {
unsigned long start = (unsigned long)iter_iov_addr(iter);
@@ -1843,14 +1876,17 @@ static ssize_t vector_madvise(struct mm_struct *mm, struct iov_iter *iter,
}
/* Drop and reacquire lock to unwind race. */
+ vector_madvise_finish_tlb(&madv_behavior);
madvise_unlock(mm, behavior);
madvise_lock(mm, behavior);
+ vector_madvise_init_tlb(&madv_behavior, mm);
continue;
}
if (ret < 0)
break;
iov_iter_advance(iter, iter_iov_len(iter));
}
+ vector_madvise_finish_tlb(&madv_behavior);
madvise_unlock(mm, behavior);
ret = (total_len - iov_iter_count(iter)) ? : ret;
--
2.39.5