During page isolation, the original migratetype is overwritten, since
MIGRATE_* are enums and stored in pageblock bitmaps. Change
MIGRATE_ISOLATE to be stored a standalone bit, PB_migrate_isolate, like
PB_migrate_skip, so that migratetype is not lost during pageblock
isolation.
Signed-off-by: Zi Yan <ziy@nvidia.com>
---
include/linux/page-isolation.h | 16 ++++++++++++++++
include/linux/pageblock-flags.h | 13 ++++++++++++-
mm/page_alloc.c | 33 +++++++++++++++++++++++++++++++--
3 files changed, 59 insertions(+), 3 deletions(-)
diff --git a/include/linux/page-isolation.h b/include/linux/page-isolation.h
index 277d8d92980c..fc021d3f95ca 100644
--- a/include/linux/page-isolation.h
+++ b/include/linux/page-isolation.h
@@ -11,6 +11,12 @@ static inline bool is_migrate_isolate(int migratetype)
{
return migratetype == MIGRATE_ISOLATE;
}
+#define get_pageblock_isolate(page) \
+ get_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate)
+#define clear_pageblock_isolate(page) \
+ clear_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate)
+#define set_pageblock_isolate(page) \
+ set_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate)
#else
static inline bool is_migrate_isolate_page(struct page *page)
{
@@ -20,6 +26,16 @@ static inline bool is_migrate_isolate(int migratetype)
{
return false;
}
+static inline bool get_pageblock_isolate(struct page *page)
+{
+ return false;
+}
+static inline void clear_pageblock_isolate(struct page *page)
+{
+}
+static inline void set_pageblock_isolate(struct page *page)
+{
+}
#endif
#define MEMORY_OFFLINE 0x1
diff --git a/include/linux/pageblock-flags.h b/include/linux/pageblock-flags.h
index 3acbb271a29a..f2f8540b95ca 100644
--- a/include/linux/pageblock-flags.h
+++ b/include/linux/pageblock-flags.h
@@ -20,7 +20,13 @@ enum pageblock_bits {
PB_migrate_end = PB_migrate + PB_migratetype_bits - 1,
/* 3 bits required for migrate types */
PB_migrate_skip,/* If set the block is skipped by compaction */
-
+#ifdef CONFIG_MEMORY_ISOLATION
+ /*
+ * Pageblock isolation is represented with a separate bit, so that
+ * the migratetype of a block is not overwritten by isolation.
+ */
+ PB_migrate_isolate, /* If set the block is isolated */
+#endif
/*
* Assume the bits will always align on a word. If this assumption
* changes then get/set pageblock needs updating.
@@ -32,6 +38,11 @@ enum pageblock_bits {
#define MIGRATETYPE_MASK ((1UL << (PB_migrate_end + 1)) - 1)
+#ifdef CONFIG_MEMORY_ISOLATION
+#define MIGRATETYPE_AND_ISO_MASK \
+ (((1UL << (PB_migrate_end + 1)) - 1) | BIT(PB_migrate_isolate))
+#endif
+
#if defined(CONFIG_HUGETLB_PAGE)
#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 0207164fcaf6..b2c623699461 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -360,8 +360,14 @@ get_pfnblock_bitmap_bitidx(const struct page *page, unsigned long pfn,
unsigned long *bitmap;
unsigned long word_bitidx;
+#ifdef CONFIG_MEMORY_ISOLATION
+ BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 8);
+ /* extra one for MIGRATE_ISOLATE */
+ BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits) + 1);
+#else
BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits));
+#endif
VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
bitmap = get_pageblock_bitmap(page, pfn);
@@ -435,7 +441,20 @@ bool get_pfnblock_bit(const struct page *page, unsigned long pfn,
__always_inline enum migratetype
get_pfnblock_migratetype(const struct page *page, unsigned long pfn)
{
- return __get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK);
+ unsigned long mask = MIGRATETYPE_MASK;
+ unsigned long flags;
+
+#ifdef CONFIG_MEMORY_ISOLATION
+ mask = MIGRATETYPE_AND_ISO_MASK;
+#endif
+
+ flags = __get_pfnblock_flags_mask(page, pfn, mask);
+
+#ifdef CONFIG_MEMORY_ISOLATION
+ if (flags & BIT(PB_migrate_isolate))
+ return MIGRATE_ISOLATE;
+#endif
+ return flags & MIGRATETYPE_MASK;
}
/**
@@ -513,12 +532,22 @@ void clear_pfnblock_bit(const struct page *page, unsigned long pfn,
__always_inline void set_pageblock_migratetype(struct page *page,
enum migratetype migratetype)
{
+ unsigned long mask = MIGRATETYPE_MASK;
+
if (unlikely(page_group_by_mobility_disabled &&
migratetype < MIGRATE_PCPTYPES))
migratetype = MIGRATE_UNMOVABLE;
+#ifdef CONFIG_MEMORY_ISOLATION
+ if (migratetype == MIGRATE_ISOLATE) {
+ set_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate);
+ return;
+ }
+ /* change mask to clear PB_migrate_isolate if it is set */
+ mask = MIGRATETYPE_AND_ISO_MASK;
+#endif
__set_pfnblock_flags_mask(page, page_to_pfn(page),
- (unsigned long)migratetype, MIGRATETYPE_MASK);
+ (unsigned long)migratetype, mask);
}
#ifdef CONFIG_DEBUG_VM
--
2.47.2On 5/23/25 21:12, Zi Yan wrote:
> During page isolation, the original migratetype is overwritten, since
> MIGRATE_* are enums and stored in pageblock bitmaps. Change
> MIGRATE_ISOLATE to be stored a standalone bit, PB_migrate_isolate, like
> PB_migrate_skip, so that migratetype is not lost during pageblock
> isolation.
>
> Signed-off-by: Zi Yan <ziy@nvidia.com>
<snip>
> #define MEMORY_OFFLINE 0x1
> diff --git a/include/linux/pageblock-flags.h b/include/linux/pageblock-flags.h
> index 3acbb271a29a..f2f8540b95ca 100644
> --- a/include/linux/pageblock-flags.h
> +++ b/include/linux/pageblock-flags.h
> @@ -20,7 +20,13 @@ enum pageblock_bits {
> PB_migrate_end = PB_migrate + PB_migratetype_bits - 1,
> /* 3 bits required for migrate types */
> PB_migrate_skip,/* If set the block is skipped by compaction */
> -
> +#ifdef CONFIG_MEMORY_ISOLATION
> + /*
> + * Pageblock isolation is represented with a separate bit, so that
> + * the migratetype of a block is not overwritten by isolation.
> + */
> + PB_migrate_isolate, /* If set the block is isolated */
> +#endif
> /*
> * Assume the bits will always align on a word. If this assumption
> * changes then get/set pageblock needs updating.
> @@ -32,6 +38,11 @@ enum pageblock_bits {
>
> #define MIGRATETYPE_MASK ((1UL << (PB_migrate_end + 1)) - 1)
>
> +#ifdef CONFIG_MEMORY_ISOLATION
> +#define MIGRATETYPE_AND_ISO_MASK \
> + (((1UL << (PB_migrate_end + 1)) - 1) | BIT(PB_migrate_isolate))
> +#endif
I think if there was:
#else
#define MIGRATETYPE_AND_ISO_MASK MIGRATETYPE_MASK
#endif
you could avoid some #ifdef code later.
> #if defined(CONFIG_HUGETLB_PAGE)
>
> #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
> diff --git a/mm/page_alloc.c b/mm/page_alloc.c
> index 0207164fcaf6..b2c623699461 100644
> --- a/mm/page_alloc.c
> +++ b/mm/page_alloc.c
> @@ -360,8 +360,14 @@ get_pfnblock_bitmap_bitidx(const struct page *page, unsigned long pfn,
> unsigned long *bitmap;
> unsigned long word_bitidx;
>
> +#ifdef CONFIG_MEMORY_ISOLATION
> + BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 8);
> + /* extra one for MIGRATE_ISOLATE */
> + BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits) + 1);
This implicitly assumes MIGRATE_ISOLATE is the last of migratetypes so we
can actually need less PB_migratetype_bits if we stop encoding it within
them anymore, but there's nothing enforcing that (not even as a comment)?
> +#else
> BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
> BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits));
> +#endif
> VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
>
> bitmap = get_pageblock_bitmap(page, pfn);
> @@ -435,7 +441,20 @@ bool get_pfnblock_bit(const struct page *page, unsigned long pfn,
> __always_inline enum migratetype
> get_pfnblock_migratetype(const struct page *page, unsigned long pfn)
> {
> - return __get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK);
> + unsigned long mask = MIGRATETYPE_MASK;
E.g. with my suggestion above you could use MIGRATETYPE_AND_ISO_MASK here.
> + unsigned long flags;
> +
> +#ifdef CONFIG_MEMORY_ISOLATION
> + mask = MIGRATETYPE_AND_ISO_MASK;
> +#endif
And drop this.
> + flags = __get_pfnblock_flags_mask(page, pfn, mask);
> +
> +#ifdef CONFIG_MEMORY_ISOLATION
> + if (flags & BIT(PB_migrate_isolate))
> + return MIGRATE_ISOLATE;
> +#endif
> + return flags & MIGRATETYPE_MASK;
> }
>
> /**
> @@ -513,12 +532,22 @@ void clear_pfnblock_bit(const struct page *page, unsigned long pfn,
> __always_inline void set_pageblock_migratetype(struct page *page,
> enum migratetype migratetype)
> {
> + unsigned long mask = MIGRATETYPE_MASK;
> +
> if (unlikely(page_group_by_mobility_disabled &&
> migratetype < MIGRATE_PCPTYPES))
> migratetype = MIGRATE_UNMOVABLE;
>
> +#ifdef CONFIG_MEMORY_ISOLATION
> + if (migratetype == MIGRATE_ISOLATE) {
> + set_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate);
> + return;
> + }
> + /* change mask to clear PB_migrate_isolate if it is set */
> + mask = MIGRATETYPE_AND_ISO_MASK;
> +#endif
> __set_pfnblock_flags_mask(page, page_to_pfn(page),
> - (unsigned long)migratetype, MIGRATETYPE_MASK);
> + (unsigned long)migratetype, mask);
This could just pass MIGRATETYPE_AND_ISO_MASK here.
> }
>
> #ifdef CONFIG_DEBUG_VM
On 27 May 2025, at 6:11, Vlastimil Babka wrote:
> On 5/23/25 21:12, Zi Yan wrote:
>> During page isolation, the original migratetype is overwritten, since
>> MIGRATE_* are enums and stored in pageblock bitmaps. Change
>> MIGRATE_ISOLATE to be stored a standalone bit, PB_migrate_isolate, like
>> PB_migrate_skip, so that migratetype is not lost during pageblock
>> isolation.
>>
>> Signed-off-by: Zi Yan <ziy@nvidia.com>
>
> <snip>
>
>> #define MEMORY_OFFLINE 0x1
>> diff --git a/include/linux/pageblock-flags.h b/include/linux/pageblock-flags.h
>> index 3acbb271a29a..f2f8540b95ca 100644
>> --- a/include/linux/pageblock-flags.h
>> +++ b/include/linux/pageblock-flags.h
>> @@ -20,7 +20,13 @@ enum pageblock_bits {
>> PB_migrate_end = PB_migrate + PB_migratetype_bits - 1,
>> /* 3 bits required for migrate types */
>> PB_migrate_skip,/* If set the block is skipped by compaction */
>> -
>> +#ifdef CONFIG_MEMORY_ISOLATION
>> + /*
>> + * Pageblock isolation is represented with a separate bit, so that
>> + * the migratetype of a block is not overwritten by isolation.
>> + */
>> + PB_migrate_isolate, /* If set the block is isolated */
>> +#endif
>> /*
>> * Assume the bits will always align on a word. If this assumption
>> * changes then get/set pageblock needs updating.
>> @@ -32,6 +38,11 @@ enum pageblock_bits {
>>
>> #define MIGRATETYPE_MASK ((1UL << (PB_migrate_end + 1)) - 1)
>>
>> +#ifdef CONFIG_MEMORY_ISOLATION
>> +#define MIGRATETYPE_AND_ISO_MASK \
>> + (((1UL << (PB_migrate_end + 1)) - 1) | BIT(PB_migrate_isolate))
>> +#endif
>
> I think if there was:
>
> #else
> #define MIGRATETYPE_AND_ISO_MASK MIGRATETYPE_MASK
> #endif
>
> you could avoid some #ifdef code later.
Sure. Will do.
>
>> #if defined(CONFIG_HUGETLB_PAGE)
>>
>> #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
>> diff --git a/mm/page_alloc.c b/mm/page_alloc.c
>> index 0207164fcaf6..b2c623699461 100644
>> --- a/mm/page_alloc.c
>> +++ b/mm/page_alloc.c
>> @@ -360,8 +360,14 @@ get_pfnblock_bitmap_bitidx(const struct page *page, unsigned long pfn,
>> unsigned long *bitmap;
>> unsigned long word_bitidx;
>>
>> +#ifdef CONFIG_MEMORY_ISOLATION
>> + BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 8);
>> + /* extra one for MIGRATE_ISOLATE */
>> + BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits) + 1);
>
> This implicitly assumes MIGRATE_ISOLATE is the last of migratetypes so we
> can actually need less PB_migratetype_bits if we stop encoding it within
> them anymore, but there's nothing enforcing that (not even as a comment)?’
You point is valid. How about adding __MIGRATE_TYPES right before
MIGRATE_ISOLATE in enum migratetype and using
BUILD_BUG_ON(__MIGRATE_TYPES > (1 << PB_migratetype_bits));
here outside of #ifdef.
The next step of cleanup, as discussed with David, is to remove
MIGRATE_ISOLATE, so that all MIGRATE_TYPES are stored in
PB_migratetype_bits. And {get,set}_pfnblock_migratetype()
gives
struct pageblock_info {
enum migratetype migratetype;
bool isolated;
};
>
>> +#else
>> BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
>> BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits));
>> +#endif
>> VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
>>
>> bitmap = get_pageblock_bitmap(page, pfn);
>> @@ -435,7 +441,20 @@ bool get_pfnblock_bit(const struct page *page, unsigned long pfn,
>> __always_inline enum migratetype
>> get_pfnblock_migratetype(const struct page *page, unsigned long pfn)
>> {
>> - return __get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK);
>> + unsigned long mask = MIGRATETYPE_MASK;
>
> E.g. with my suggestion above you could use MIGRATETYPE_AND_ISO_MASK here.
>
>> + unsigned long flags;
>> +
>> +#ifdef CONFIG_MEMORY_ISOLATION
>> + mask = MIGRATETYPE_AND_ISO_MASK;
>> +#endif
>
> And drop this.
Sure. Will do.
>
>> + flags = __get_pfnblock_flags_mask(page, pfn, mask);
>> +
>> +#ifdef CONFIG_MEMORY_ISOLATION
>> + if (flags & BIT(PB_migrate_isolate))
>> + return MIGRATE_ISOLATE;
>> +#endif
>> + return flags & MIGRATETYPE_MASK;
>> }
>>
>> /**
>> @@ -513,12 +532,22 @@ void clear_pfnblock_bit(const struct page *page, unsigned long pfn,
>> __always_inline void set_pageblock_migratetype(struct page *page,
>> enum migratetype migratetype)
>> {
>> + unsigned long mask = MIGRATETYPE_MASK;
>> +
>> if (unlikely(page_group_by_mobility_disabled &&
>> migratetype < MIGRATE_PCPTYPES))
>> migratetype = MIGRATE_UNMOVABLE;
>>
>> +#ifdef CONFIG_MEMORY_ISOLATION
>> + if (migratetype == MIGRATE_ISOLATE) {
>> + set_pfnblock_bit(page, page_to_pfn(page), PB_migrate_isolate);
>> + return;
>> + }
>> + /* change mask to clear PB_migrate_isolate if it is set */
>> + mask = MIGRATETYPE_AND_ISO_MASK;
>> +#endif
>> __set_pfnblock_flags_mask(page, page_to_pfn(page),
>> - (unsigned long)migratetype, MIGRATETYPE_MASK);
>> + (unsigned long)migratetype, mask);
>
> This could just pass MIGRATETYPE_AND_ISO_MASK here.
Yep.
>
>> }
>>
>> #ifdef CONFIG_DEBUG_VM
Best Regards,
Yan, Zi
© 2016 - 2025 Red Hat, Inc.