From: Barry Song <baohua@kernel.org>
dcache_by_myline_op ensures completion of the data cache operations for a
region, while dcache_by_myline_op_nosync only issues them without waiting.
This enables deferred synchronization so completion for multiple regions
can be handled together later.
Cc: Leon Romanovsky <leon@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Robin Murphy <robin.murphy@arm.com>
Cc: Ada Couprie Diaz <ada.coupriediaz@arm.com>
Cc: Ard Biesheuvel <ardb@kernel.org>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Anshuman Khandual <anshuman.khandual@arm.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Tangquan Zheng <zhengtangquan@oppo.com>
Signed-off-by: Barry Song <baohua@kernel.org>
---
arch/arm64/include/asm/assembler.h | 24 +++++++++++++++++++-----
arch/arm64/kernel/relocate_kernel.S | 3 ++-
2 files changed, 21 insertions(+), 6 deletions(-)
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index f0ca7196f6fa..b408ed61866f 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -371,14 +371,13 @@ alternative_endif
* [start, end) with dcache line size explicitly provided.
*
* op: operation passed to dc instruction
- * domain: domain used in dsb instruction
* start: starting virtual address of the region
* end: end virtual address of the region
* linesz: dcache line size
* fixup: optional label to branch to on user fault
* Corrupts: start, end, tmp
*/
- .macro dcache_by_myline_op op, domain, start, end, linesz, tmp, fixup
+ .macro raw_dcache_by_myline_op op, start, end, linesz, tmp, fixup
sub \tmp, \linesz, #1
bic \start, \start, \tmp
.Ldcache_op\@:
@@ -402,14 +401,13 @@ alternative_endif
add \start, \start, \linesz
cmp \start, \end
b.lo .Ldcache_op\@
- dsb \domain
_cond_uaccess_extable .Ldcache_op\@, \fixup
.endm
/*
* Macro to perform a data cache maintenance for the interval
- * [start, end)
+ * [start, end) and wait for completion
*
* op: operation passed to dc instruction
* domain: domain used in dsb instruction
@@ -420,7 +418,23 @@ alternative_endif
*/
.macro dcache_by_line_op op, domain, start, end, tmp1, tmp2, fixup
dcache_line_size \tmp1, \tmp2
- dcache_by_myline_op \op, \domain, \start, \end, \tmp1, \tmp2, \fixup
+ raw_dcache_by_myline_op \op, \start, \end, \tmp1, \tmp2, \fixup
+ dsb \domain
+ .endm
+
+/*
+ * Macro to perform a data cache maintenance for the interval
+ * [start, end) without waiting for completion
+ *
+ * op: operation passed to dc instruction
+ * start: starting virtual address of the region
+ * end: end virtual address of the region
+ * fixup: optional label to branch to on user fault
+ * Corrupts: start, end, tmp1, tmp2
+ */
+ .macro dcache_by_line_op_nosync op, start, end, tmp1, tmp2, fixup
+ dcache_line_size \tmp1, \tmp2
+ raw_dcache_by_myline_op \op, \start, \end, \tmp1, \tmp2, \fixup
.endm
/*
diff --git a/arch/arm64/kernel/relocate_kernel.S b/arch/arm64/kernel/relocate_kernel.S
index 413f899e4ac6..71938eb3a3a3 100644
--- a/arch/arm64/kernel/relocate_kernel.S
+++ b/arch/arm64/kernel/relocate_kernel.S
@@ -64,7 +64,8 @@ SYM_CODE_START(arm64_relocate_new_kernel)
mov x19, x13
copy_page x13, x12, x1, x2, x3, x4, x5, x6, x7, x8
add x1, x19, #PAGE_SIZE
- dcache_by_myline_op civac, sy, x19, x1, x15, x20
+ raw_dcache_by_myline_op civac, x19, x1, x15, x20
+ dsb sy
b .Lnext
.Ltest_indirection:
tbz x16, IND_INDIRECTION_BIT, .Ltest_destination
--
2.43.0On Sat, Dec 27, 2025 at 11:52:41AM +1300, Barry Song wrote: > From: Barry Song <baohua@kernel.org> > > dcache_by_myline_op ensures completion of the data cache operations for a > region, while dcache_by_myline_op_nosync only issues them without waiting. > This enables deferred synchronization so completion for multiple regions > can be handled together later. > > Cc: Leon Romanovsky <leon@kernel.org> > Cc: Catalin Marinas <catalin.marinas@arm.com> > Cc: Will Deacon <will@kernel.org> > Cc: Marek Szyprowski <m.szyprowski@samsung.com> > Cc: Robin Murphy <robin.murphy@arm.com> > Cc: Ada Couprie Diaz <ada.coupriediaz@arm.com> > Cc: Ard Biesheuvel <ardb@kernel.org> > Cc: Marc Zyngier <maz@kernel.org> > Cc: Anshuman Khandual <anshuman.khandual@arm.com> > Cc: Ryan Roberts <ryan.roberts@arm.com> > Cc: Suren Baghdasaryan <surenb@google.com> > Cc: Tangquan Zheng <zhengtangquan@oppo.com> > Signed-off-by: Barry Song <baohua@kernel.org> > --- > arch/arm64/include/asm/assembler.h | 24 +++++++++++++++++++----- > arch/arm64/kernel/relocate_kernel.S | 3 ++- > 2 files changed, 21 insertions(+), 6 deletions(-) > > diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h > index f0ca7196f6fa..b408ed61866f 100644 > --- a/arch/arm64/include/asm/assembler.h > +++ b/arch/arm64/include/asm/assembler.h > @@ -371,14 +371,13 @@ alternative_endif > * [start, end) with dcache line size explicitly provided. > * > * op: operation passed to dc instruction > - * domain: domain used in dsb instruction > * start: starting virtual address of the region > * end: end virtual address of the region > * linesz: dcache line size > * fixup: optional label to branch to on user fault > * Corrupts: start, end, tmp > */ > - .macro dcache_by_myline_op op, domain, start, end, linesz, tmp, fixup > + .macro raw_dcache_by_myline_op op, start, end, linesz, tmp, fixup > sub \tmp, \linesz, #1 > bic \start, \start, \tmp > .Ldcache_op\@: > @@ -402,14 +401,13 @@ alternative_endif > add \start, \start, \linesz > cmp \start, \end > b.lo .Ldcache_op\@ > - dsb \domain Naming nit, but I'd prefer this to be dcache_by_myline_op_nosync() for consistency with the other macros that you're adding. The 'raw' prefix is used by raw_dcache_line_size() to indicate that we're getting the value from the underlying hardware register. > > _cond_uaccess_extable .Ldcache_op\@, \fixup > .endm > > /* > * Macro to perform a data cache maintenance for the interval > - * [start, end) > + * [start, end) and wait for completion > * > * op: operation passed to dc instruction > * domain: domain used in dsb instruction > @@ -420,7 +418,23 @@ alternative_endif > */ > .macro dcache_by_line_op op, domain, start, end, tmp1, tmp2, fixup > dcache_line_size \tmp1, \tmp2 > - dcache_by_myline_op \op, \domain, \start, \end, \tmp1, \tmp2, \fixup > + raw_dcache_by_myline_op \op, \start, \end, \tmp1, \tmp2, \fixup > + dsb \domain > + .endm This could just be dcache_by_line_op_nosync() + dsb. Will
[...] > > */ > > - .macro dcache_by_myline_op op, domain, start, end, linesz, tmp, fixup > > + .macro raw_dcache_by_myline_op op, start, end, linesz, tmp, fixup > > sub \tmp, \linesz, #1 > > bic \start, \start, \tmp > > .Ldcache_op\@: > > @@ -402,14 +401,13 @@ alternative_endif > > add \start, \start, \linesz > > cmp \start, \end > > b.lo .Ldcache_op\@ > > - dsb \domain > > Naming nit, but I'd prefer this to be dcache_by_myline_op_nosync() for > consistency with the other macros that you're adding. The 'raw' prefix > is used by raw_dcache_line_size() to indicate that we're getting the > value from the underlying hardware register. Ok. thanks! > > > > > _cond_uaccess_extable .Ldcache_op\@, \fixup > > .endm > > > > /* > > * Macro to perform a data cache maintenance for the interval > > - * [start, end) > > + * [start, end) and wait for completion > > * > > * op: operation passed to dc instruction > > * domain: domain used in dsb instruction > > @@ -420,7 +418,23 @@ alternative_endif > > */ > > .macro dcache_by_line_op op, domain, start, end, tmp1, tmp2, fixup > > dcache_line_size \tmp1, \tmp2 > > - dcache_by_myline_op \op, \domain, \start, \end, \tmp1, \tmp2, \fixup > > + raw_dcache_by_myline_op \op, \start, \end, \tmp1, \tmp2, \fixup > > + dsb \domain > > + .endm > > This could just be dcache_by_line_op_nosync() + dsb. Ok. thanks! Best Regards Barry
© 2016 - 2026 Red Hat, Inc.