On 2/2/26 16:56, Kiryl Shutsemau wrote:
> The compound_head() function is a hot path. For example, the zap path
> calls it for every leaf page table entry.
>
> Rewrite the helper function in a branchless manner to eliminate the risk
> of CPU branch misprediction.
>
> Signed-off-by: Kiryl Shutsemau <kas@kernel.org>
> Reviewed-by: Muchun Song <muchun.song@linux.dev>
> Reviewed-by: Zi Yan <ziy@nvidia.com>
> ---
> include/linux/page-flags.h | 27 +++++++++++++++++----------
> 1 file changed, 17 insertions(+), 10 deletions(-)
>
> diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
> index 1aaa604f4b9b..16384cb6f962 100644
> --- a/include/linux/page-flags.h
> +++ b/include/linux/page-flags.h
> @@ -224,25 +224,32 @@ static __always_inline bool compound_info_has_mask(void)
> static __always_inline unsigned long _compound_head(const struct page *page)
> {
> unsigned long info = READ_ONCE(page->compound_info);
> + unsigned long mask;
> +
> + if (!compound_info_has_mask()) {
> + /* Bit 0 encodes PageTail() */
> + if (info & 1)
> + return info - 1;
>
> - /* Bit 0 encodes PageTail() */
> - if (!(info & 1))
> return (unsigned long)page;
> -
> - /*
> - * If compound_info_has_mask() is false, the rest of compound_info is
> - * the pointer to the head page.
> - */
> - if (!compound_info_has_mask())
> - return info - 1;
> + }
>
> /*
> * If compoun_info_has_mask() is true the rest of the info encodes
> * the mask that converts the address of the tail page to the head page.
> *
> * No need to clear bit 0 in the mask as 'page' always has it clear.
> + *
> + * Let's do it in a branchless manner.
> */
> - return (unsigned long)page & info;
> +
> + /* Non-tail: -1UL, Tail: 0 */
> + mask = (info & 1) - 1;
> +
> + /* Non-tail: -1UL, Tail: info */
> + mask |= info;
> +
> + return (unsigned long)page & mask;
> }
>
> #define compound_head(page) ((typeof(page))_compound_head(page))
Nice!
Acked-by: David Hildenbrand (Arm) <david@kernel.org>
--
Cheers,
David