The compound_head() function is a hot path. For example, the zap path
calls it for every leaf page table entry.

Rewrite the helper function in a branchless manner to eliminate the risk
of CPU branch misprediction.

Signed-off-by: Kiryl Shutsemau <kas@kernel.org>
Reviewed-by: Muchun Song <muchun.song@linux.dev>
---
 include/linux/page-flags.h | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index f89702e101e8..95ac963b78af 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -224,25 +224,32 @@ static __always_inline bool compound_info_has_mask(void)
 static __always_inline unsigned long _compound_head(const struct page *page)
 {
 	unsigned long info = READ_ONCE(page->compound_info);
+	unsigned long mask;
+
+	if (!compound_info_has_mask()) {
+		/* Bit 0 encodes PageTail() */
+		if (info & 1)
+			return info - 1;
 
-	/* Bit 0 encodes PageTail() */
-	if (!(info & 1))
 		return (unsigned long)page;
-
-	/*
-	 * If compound_info_has_mask() is false, the rest of compound_info is
-	 * the pointer to the head page.
-	 */
-	if (!compound_info_has_mask())
-		return info - 1;
+	}
 
 	/*
 	 * If compoun_info_has_mask() is true the rest of the info encodes
 	 * the mask that converts the address of the tail page to the head page.
 	 *
 	 * No need to clear bit 0 in the mask as 'page' always has it clear.
+	 *
+	 * Let's do it in a branchless manner.
 	 */
-	return (unsigned long)page & info;
+
+	/* Non-tail: -1UL, Tail: 0 */
+	mask = (info & 1) - 1;
+
+	/* Non-tail: -1UL, Tail: info */
+	mask |= info;
+
+	return (unsigned long)page & mask;
 }
 
 #define compound_head(page)	((typeof(page))_compound_head(page))
-- 
2.51.2

On 21 Jan 2026, at 11:22, Kiryl Shutsemau wrote:

> The compound_head() function is a hot path. For example, the zap path
> calls it for every leaf page table entry.
>
> Rewrite the helper function in a branchless manner to eliminate the risk
> of CPU branch misprediction.
>
> Signed-off-by: Kiryl Shutsemau <kas@kernel.org>
> Reviewed-by: Muchun Song <muchun.song@linux.dev>
> ---
>  include/linux/page-flags.h | 27 +++++++++++++++++----------
>  1 file changed, 17 insertions(+), 10 deletions(-)

LGTM. And it fixed the issue in patch 10.

Reviewed-by: Zi Yan <ziy@nvidia.com>

>
> diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
> index f89702e101e8..95ac963b78af 100644
> --- a/include/linux/page-flags.h
> +++ b/include/linux/page-flags.h
> @@ -224,25 +224,32 @@ static __always_inline bool compound_info_has_mask(void)
>  static __always_inline unsigned long _compound_head(const struct page *page)
>  {
>  	unsigned long info = READ_ONCE(page->compound_info);
> +	unsigned long mask;
> +
> +	if (!compound_info_has_mask()) {
> +		/* Bit 0 encodes PageTail() */
> +		if (info & 1)
> +			return info - 1;
>
> -	/* Bit 0 encodes PageTail() */
> -	if (!(info & 1))
>  		return (unsigned long)page;

Here, info & 1 check now is guarded by compound_info_has_mask(), so the
issue I mentioned in Patch 10 is gone.

> -
> -	/*
> -	 * If compound_info_has_mask() is false, the rest of compound_info is
> -	 * the pointer to the head page.
> -	 */
> -	if (!compound_info_has_mask())
> -		return info - 1;
> +	}


Best Regards,
Yan, Zi