Later, we need to add the right amount of references, which should be
the number of borrower domains, to the owner domain. Since we only have
get_page() to increment the page reference by 1, a loop is needed per
page, which is inefficient and time-consuming.
To save the loop time, this commit introduces a set of new helpers
put_page_nr() and get_page_nr() to increment/drop the page reference by nr.
Signed-off-by: Penny Zheng <penny.zheng@arm.com>
---
v3 changes:
- check overflow with "n"
- remove spurious change
- bring back the check that we enter the loop only when count_info is
greater than 0
---
v2 change:
- new commit
---
xen/arch/arm/include/asm/mm.h | 4 ++++
xen/arch/arm/mm.c | 36 ++++++++++++++++++++++++++---------
2 files changed, 31 insertions(+), 9 deletions(-)
diff --git a/xen/arch/arm/include/asm/mm.h b/xen/arch/arm/include/asm/mm.h
index 424aaf2823..c737d51e4d 100644
--- a/xen/arch/arm/include/asm/mm.h
+++ b/xen/arch/arm/include/asm/mm.h
@@ -347,6 +347,10 @@ void free_init_memory(void);
int guest_physmap_mark_populate_on_demand(struct domain *d, unsigned long gfn,
unsigned int order);
+extern bool get_page_nr(struct page_info *page, const struct domain *domain,
+ unsigned long nr);
+extern void put_page_nr(struct page_info *page, unsigned long nr);
+
extern void put_page_type(struct page_info *page);
static inline void put_page_and_type(struct page_info *page)
{
diff --git a/xen/arch/arm/mm.c b/xen/arch/arm/mm.c
index 7b1f2f4906..a9461e07aa 100644
--- a/xen/arch/arm/mm.c
+++ b/xen/arch/arm/mm.c
@@ -1537,7 +1537,8 @@ long arch_memory_op(int op, XEN_GUEST_HANDLE_PARAM(void) arg)
return 0;
}
-struct domain *page_get_owner_and_reference(struct page_info *page)
+static struct domain *page_get_owner_and_nr_reference(struct page_info *page,
+ unsigned long nr)
{
unsigned long x, y = page->count_info;
struct domain *owner;
@@ -1548,10 +1549,10 @@ struct domain *page_get_owner_and_reference(struct page_info *page)
* Count == 0: Page is not allocated, so we cannot take a reference.
* Count == -1: Reference count would wrap, which is invalid.
*/
- if ( unlikely(((x + 1) & PGC_count_mask) <= 1) )
+ if ( unlikely(((x + nr) & PGC_count_mask) <= 1) )
return NULL;
}
- while ( (y = cmpxchg(&page->count_info, x, x + 1)) != x );
+ while ( (y = cmpxchg(&page->count_info, x, x + nr)) != x );
owner = page_get_owner(page);
ASSERT(owner);
@@ -1559,14 +1560,20 @@ struct domain *page_get_owner_and_reference(struct page_info *page)
return owner;
}
-void put_page(struct page_info *page)
+struct domain *page_get_owner_and_reference(struct page_info *page)
+{
+ return page_get_owner_and_nr_reference(page, 1);
+}
+
+void put_page_nr(struct page_info *page, unsigned long nr)
{
unsigned long nx, x, y = page->count_info;
do {
- ASSERT((y & PGC_count_mask) != 0);
+ ASSERT(((y & PGC_count_mask) != 0) &&
+ (((y - nr) & PGC_count_mask) >= 0));
x = y;
- nx = x - 1;
+ nx = x - nr;
}
while ( unlikely((y = cmpxchg(&page->count_info, x, nx)) != x) );
@@ -1576,19 +1583,30 @@ void put_page(struct page_info *page)
}
}
-bool get_page(struct page_info *page, const struct domain *domain)
+void put_page(struct page_info *page)
{
- const struct domain *owner = page_get_owner_and_reference(page);
+ put_page_nr(page, 1);
+}
+
+bool get_page_nr(struct page_info *page, const struct domain *domain,
+ unsigned long nr)
+{
+ const struct domain *owner = page_get_owner_and_nr_reference(page, nr);
if ( likely(owner == domain) )
return true;
if ( owner != NULL )
- put_page(page);
+ put_page_nr(page, nr);
return false;
}
+bool get_page(struct page_info *page, const struct domain *domain)
+{
+ return get_page_nr(page, domain, 1);
+}
+
/* Common code requires get_page_type and put_page_type.
* We don't care about typecounts so we just do the minimum to make it
* happy. */
--
2.25.1
On 12/05/2022 10:11, Penny Zheng wrote:
> Later, we need to add the right amount of references, which should be
> the number of borrower domains, to the owner domain. Since we only have
> get_page() to increment the page reference by 1, a loop is needed per
> page, which is inefficient and time-consuming.
>
> To save the loop time, this commit introduces a set of new helpers
> put_page_nr() and get_page_nr() to increment/drop the page reference by nr.
>
> Signed-off-by: Penny Zheng <penny.zheng@arm.com>
> ---
> v3 changes:
> - check overflow with "n"
> - remove spurious change
> - bring back the check that we enter the loop only when count_info is
> greater than 0
> ---
> v2 change:
> - new commit
> ---
> xen/arch/arm/include/asm/mm.h | 4 ++++
> xen/arch/arm/mm.c | 36 ++++++++++++++++++++++++++---------
> 2 files changed, 31 insertions(+), 9 deletions(-)
>
> diff --git a/xen/arch/arm/include/asm/mm.h b/xen/arch/arm/include/asm/mm.h
> index 424aaf2823..c737d51e4d 100644
> --- a/xen/arch/arm/include/asm/mm.h
> +++ b/xen/arch/arm/include/asm/mm.h
> @@ -347,6 +347,10 @@ void free_init_memory(void);
> int guest_physmap_mark_populate_on_demand(struct domain *d, unsigned long gfn,
> unsigned int order);
>
> +extern bool get_page_nr(struct page_info *page, const struct domain *domain,
> + unsigned long nr);
> +extern void put_page_nr(struct page_info *page, unsigned long nr);
> +
> extern void put_page_type(struct page_info *page);
> static inline void put_page_and_type(struct page_info *page)
> {
> diff --git a/xen/arch/arm/mm.c b/xen/arch/arm/mm.c
> index 7b1f2f4906..a9461e07aa 100644
> --- a/xen/arch/arm/mm.c
> +++ b/xen/arch/arm/mm.c
> @@ -1537,7 +1537,8 @@ long arch_memory_op(int op, XEN_GUEST_HANDLE_PARAM(void) arg)
> return 0;
> }
>
> -struct domain *page_get_owner_and_reference(struct page_info *page)
> +static struct domain *page_get_owner_and_nr_reference(struct page_info *page,
> + unsigned long nr)
> {
> unsigned long x, y = page->count_info;
> struct domain *owner;
> @@ -1548,10 +1549,10 @@ struct domain *page_get_owner_and_reference(struct page_info *page)
> * Count == 0: Page is not allocated, so we cannot take a reference.
> * Count == -1: Reference count would wrap, which is invalid.
> */
> - if ( unlikely(((x + 1) & PGC_count_mask) <= 1) )
> + if ( unlikely(((x + nr) & PGC_count_mask) <= 1) )
This check looks wrong to me. You want to make sure that the right
equation return is at least equal to n otherwise.
Furthermore, I think we need to restrict 'nr' to PGC_count_mask to fully
catch any overflow.
Before the loop, the code would look like:
/* Restrict nr to avoid "double" overflow */
if ( nr >= PGC_count_mask )
{
ASSERT_UNREACHABLE();
return NULL;
}
The check in the loop would look like:
if ( unlikely((x + nr) & PGC_count_mask) <= n )
That said, it might be easier to read the overflow check if we do:
count = x & PGC_count_mask;
if ( !count || ((PGC_count_mask - count) <= n) )
I haven't measured and check which of the two options would result to
better code and performance (get_page() is often called).
> return NULL;
> }
> - while ( (y = cmpxchg(&page->count_info, x, x + 1)) != x );
> + while ( (y = cmpxchg(&page->count_info, x, x + nr)) != x );
>
> owner = page_get_owner(page);
> ASSERT(owner);
> @@ -1559,14 +1560,20 @@ struct domain *page_get_owner_and_reference(struct page_info *page)
> return owner;
> }
>
> -void put_page(struct page_info *page)
> +struct domain *page_get_owner_and_reference(struct page_info *page)
> +{
> + return page_get_owner_and_nr_reference(page, 1);
> +}
> +
> +void put_page_nr(struct page_info *page, unsigned long nr)
> {
> unsigned long nx, x, y = page->count_info;
>
> do {
> - ASSERT((y & PGC_count_mask) != 0);
> + ASSERT(((y & PGC_count_mask) != 0) &&
> + (((y - nr) & PGC_count_mask) >= 0));
I think there are a potential underflow here if 'y' is smaller than
'nr'. But on v2, Stefano suggest to use ASSERT((y & PGC_count_mask) >=
nr); which I think is sufficient here.
> x = y;
> - nx = x - 1;
> + nx = x - nr;
> }
> while ( unlikely((y = cmpxchg(&page->count_info, x, nx)) != x) );
>
> @@ -1576,19 +1583,30 @@ void put_page(struct page_info *page)
> }
> }
>
> -bool get_page(struct page_info *page, const struct domain *domain)
> +void put_page(struct page_info *page)
> {
> - const struct domain *owner = page_get_owner_and_reference(page);
> + put_page_nr(page, 1);
> +}
> +
> +bool get_page_nr(struct page_info *page, const struct domain *domain,
> + unsigned long nr)
> +{
> + const struct domain *owner = page_get_owner_and_nr_reference(page, nr);
>
> if ( likely(owner == domain) )
> return true;
>
> if ( owner != NULL )
> - put_page(page);
> + put_page_nr(page, nr);
>
> return false;
> }
>
> +bool get_page(struct page_info *page, const struct domain *domain)
> +{
> + return get_page_nr(page, domain, 1);
> +}
> +
> /* Common code requires get_page_type and put_page_type.
> * We don't care about typecounts so we just do the minimum to make it
> * happy. */
Cheers,
--
Julien Grall
Hi Julien
> -----Original Message-----
> From: Julien Grall <julien@xen.org>
> Sent: Thursday, May 12, 2022 6:14 PM
> To: Penny Zheng <Penny.Zheng@arm.com>; xen-devel@lists.xenproject.org
> Cc: Wei Chen <Wei.Chen@arm.com>; Stefano Stabellini
> <sstabellini@kernel.org>; Bertrand Marquis <Bertrand.Marquis@arm.com>;
> Volodymyr Babchuk <Volodymyr_Babchuk@epam.com>
> Subject: Re: [PATCH v3 4/8] xen/arm: introduce put_page_nr and get_page_nr
>
>
>
> On 12/05/2022 10:11, Penny Zheng wrote:
> > Later, we need to add the right amount of references, which should be
> > the number of borrower domains, to the owner domain. Since we only
> > have
> > get_page() to increment the page reference by 1, a loop is needed per
> > page, which is inefficient and time-consuming.
> >
> > To save the loop time, this commit introduces a set of new helpers
> > put_page_nr() and get_page_nr() to increment/drop the page reference by
> nr.
> >
> > Signed-off-by: Penny Zheng <penny.zheng@arm.com>
> > ---
> > v3 changes:
> > - check overflow with "n"
> > - remove spurious change
> > - bring back the check that we enter the loop only when count_info is
> > greater than 0
> > ---
> > v2 change:
> > - new commit
> > ---
> > xen/arch/arm/include/asm/mm.h | 4 ++++
> > xen/arch/arm/mm.c | 36 ++++++++++++++++++++++++++---------
> > 2 files changed, 31 insertions(+), 9 deletions(-)
> >
> > diff --git a/xen/arch/arm/include/asm/mm.h
> > b/xen/arch/arm/include/asm/mm.h index 424aaf2823..c737d51e4d 100644
> > --- a/xen/arch/arm/include/asm/mm.h
> > +++ b/xen/arch/arm/include/asm/mm.h
> > @@ -347,6 +347,10 @@ void free_init_memory(void);
> > int guest_physmap_mark_populate_on_demand(struct domain *d,
> unsigned long gfn,
> > unsigned int order);
> >
> > +extern bool get_page_nr(struct page_info *page, const struct domain
> *domain,
> > + unsigned long nr); extern void
> > +put_page_nr(struct page_info *page, unsigned long nr);
> > +
> > extern void put_page_type(struct page_info *page);
> > static inline void put_page_and_type(struct page_info *page)
> > {
> > diff --git a/xen/arch/arm/mm.c b/xen/arch/arm/mm.c index
> > 7b1f2f4906..a9461e07aa 100644
> > --- a/xen/arch/arm/mm.c
> > +++ b/xen/arch/arm/mm.c
> > @@ -1537,7 +1537,8 @@ long arch_memory_op(int op,
> XEN_GUEST_HANDLE_PARAM(void) arg)
> > return 0;
> > }
> >
> > -struct domain *page_get_owner_and_reference(struct page_info *page)
> > +static struct domain *page_get_owner_and_nr_reference(struct page_info
> *page,
> > + unsigned long
> > +nr)
> > {
> > unsigned long x, y = page->count_info;
> > struct domain *owner;
> > @@ -1548,10 +1549,10 @@ struct domain
> *page_get_owner_and_reference(struct page_info *page)
> > * Count == 0: Page is not allocated, so we cannot take a reference.
> > * Count == -1: Reference count would wrap, which is invalid.
> > */
> > - if ( unlikely(((x + 1) & PGC_count_mask) <= 1) )
> > + if ( unlikely(((x + nr) & PGC_count_mask) <= 1) )
>
> This check looks wrong to me. You want to make sure that the right equation
> return is at least equal to n otherwise.
>
Right, right, I haven't considered thoroughly! A thousand thanks for the following
detailed explanation~
> Furthermore, I think we need to restrict 'nr' to PGC_count_mask to fully catch
> any overflow.
>
> Before the loop, the code would look like:
>
> /* Restrict nr to avoid "double" overflow */ if ( nr >= PGC_count_mask ) {
> ASSERT_UNREACHABLE();
> return NULL;
> }
>
> The check in the loop would look like:
>
> if ( unlikely((x + nr) & PGC_count_mask) <= n )
>
> That said, it might be easier to read the overflow check if we do:
>
> count = x & PGC_count_mask;
> if ( !count || ((PGC_count_mask - count) <= n) )
>
> I haven't measured and check which of the two options would result to better
> code and performance (get_page() is often called).
>
Correct me if I understand wrongly:
IMO, only option two is actually catching any overflow? Let (PGC_count_mask - count) <= nr
stay in the loop, not before the loop like option 1, to cover the changeable page->count_info.
> > return NULL;
> > }
> > - while ( (y = cmpxchg(&page->count_info, x, x + 1)) != x );
> > + while ( (y = cmpxchg(&page->count_info, x, x + nr)) != x );
> >
> > owner = page_get_owner(page);
> > ASSERT(owner);
> > @@ -1559,14 +1560,20 @@ struct domain
> *page_get_owner_and_reference(struct page_info *page)
> > return owner;
> > }
> >
> > -void put_page(struct page_info *page)
> > +struct domain *page_get_owner_and_reference(struct page_info *page) {
> > + return page_get_owner_and_nr_reference(page, 1); }
> > +
> > +void put_page_nr(struct page_info *page, unsigned long nr)
> > {
> > unsigned long nx, x, y = page->count_info;
> >
> > do {
> > - ASSERT((y & PGC_count_mask) != 0);
> > + ASSERT(((y & PGC_count_mask) != 0) &&
> > + (((y - nr) & PGC_count_mask) >= 0));
>
> I think there are a potential underflow here if 'y' is smaller than 'nr'. But on v2,
> Stefano suggest to use ASSERT((y & PGC_count_mask) >= nr); which I think is
> sufficient here.
>
Oh, understood. thanks for clarification~
> > x = y;
> > - nx = x - 1;
> > + nx = x - nr;
> > }
> > while ( unlikely((y = cmpxchg(&page->count_info, x, nx)) != x)
> > );
> >
> > @@ -1576,19 +1583,30 @@ void put_page(struct page_info *page)
> > }
> > }
> >
> > -bool get_page(struct page_info *page, const struct domain *domain)
> > +void put_page(struct page_info *page)
> > {
> > - const struct domain *owner = page_get_owner_and_reference(page);
> > + put_page_nr(page, 1);
> > +}
> > +
> > +bool get_page_nr(struct page_info *page, const struct domain *domain,
> > + unsigned long nr)
> > +{
> > + const struct domain *owner =
> > +page_get_owner_and_nr_reference(page, nr);
> >
> > if ( likely(owner == domain) )
> > return true;
> >
> > if ( owner != NULL )
> > - put_page(page);
> > + put_page_nr(page, nr);
> >
> > return false;
> > }
> >
> > +bool get_page(struct page_info *page, const struct domain *domain) {
> > + return get_page_nr(page, domain, 1); }
> > +
> > /* Common code requires get_page_type and put_page_type.
> > * We don't care about typecounts so we just do the minimum to make it
> > * happy. */
>
> Cheers,
>
> --
> Julien Grall
On 13/05/2022 03:22, Penny Zheng wrote:
> Hi Julien
Hi Penny,
>> -----Original Message-----
>> From: Julien Grall <julien@xen.org>
>> Sent: Thursday, May 12, 2022 6:14 PM
>> To: Penny Zheng <Penny.Zheng@arm.com>; xen-devel@lists.xenproject.org
>> Cc: Wei Chen <Wei.Chen@arm.com>; Stefano Stabellini
>> <sstabellini@kernel.org>; Bertrand Marquis <Bertrand.Marquis@arm.com>;
>> Volodymyr Babchuk <Volodymyr_Babchuk@epam.com>
>> Subject: Re: [PATCH v3 4/8] xen/arm: introduce put_page_nr and get_page_nr
>>
>>
>>
>> On 12/05/2022 10:11, Penny Zheng wrote:
>>> Later, we need to add the right amount of references, which should be
>>> the number of borrower domains, to the owner domain. Since we only
>>> have
>>> get_page() to increment the page reference by 1, a loop is needed per
>>> page, which is inefficient and time-consuming.
>>>
>>> To save the loop time, this commit introduces a set of new helpers
>>> put_page_nr() and get_page_nr() to increment/drop the page reference by
>> nr.
>>>
>>> Signed-off-by: Penny Zheng <penny.zheng@arm.com>
>>> ---
>>> v3 changes:
>>> - check overflow with "n"
>>> - remove spurious change
>>> - bring back the check that we enter the loop only when count_info is
>>> greater than 0
>>> ---
>>> v2 change:
>>> - new commit
>>> ---
>>> xen/arch/arm/include/asm/mm.h | 4 ++++
>>> xen/arch/arm/mm.c | 36 ++++++++++++++++++++++++++---------
>>> 2 files changed, 31 insertions(+), 9 deletions(-)
>>>
>>> diff --git a/xen/arch/arm/include/asm/mm.h
>>> b/xen/arch/arm/include/asm/mm.h index 424aaf2823..c737d51e4d 100644
>>> --- a/xen/arch/arm/include/asm/mm.h
>>> +++ b/xen/arch/arm/include/asm/mm.h
>>> @@ -347,6 +347,10 @@ void free_init_memory(void);
>>> int guest_physmap_mark_populate_on_demand(struct domain *d,
>> unsigned long gfn,
>>> unsigned int order);
>>>
>>> +extern bool get_page_nr(struct page_info *page, const struct domain
>> *domain,
>>> + unsigned long nr); extern void
>>> +put_page_nr(struct page_info *page, unsigned long nr);
>>> +
>>> extern void put_page_type(struct page_info *page);
>>> static inline void put_page_and_type(struct page_info *page)
>>> {
>>> diff --git a/xen/arch/arm/mm.c b/xen/arch/arm/mm.c index
>>> 7b1f2f4906..a9461e07aa 100644
>>> --- a/xen/arch/arm/mm.c
>>> +++ b/xen/arch/arm/mm.c
>>> @@ -1537,7 +1537,8 @@ long arch_memory_op(int op,
>> XEN_GUEST_HANDLE_PARAM(void) arg)
>>> return 0;
>>> }
>>>
>>> -struct domain *page_get_owner_and_reference(struct page_info *page)
>>> +static struct domain *page_get_owner_and_nr_reference(struct page_info
>> *page,
>>> + unsigned long
>>> +nr)
>>> {
>>> unsigned long x, y = page->count_info;
>>> struct domain *owner;
>>> @@ -1548,10 +1549,10 @@ struct domain
>> *page_get_owner_and_reference(struct page_info *page)
>>> * Count == 0: Page is not allocated, so we cannot take a reference.
>>> * Count == -1: Reference count would wrap, which is invalid.
>>> */
>>> - if ( unlikely(((x + 1) & PGC_count_mask) <= 1) )
>>> + if ( unlikely(((x + nr) & PGC_count_mask) <= 1) )
>>
>> This check looks wrong to me. You want to make sure that the right equation
>> return is at least equal to n otherwise.
>>
>
> Right, right, I haven't considered thoroughly! A thousand thanks for the following
> detailed explanation~
>
>> Furthermore, I think we need to restrict 'nr' to PGC_count_mask to fully catch
>> any overflow.
>>
>> Before the loop, the code would look like:
>>
>> /* Restrict nr to avoid "double" overflow */ if ( nr >= PGC_count_mask ) {
>> ASSERT_UNREACHABLE();
>> return NULL;
>> }
>>
>> The check in the loop would look like:
>>
>> if ( unlikely((x + nr) & PGC_count_mask) <= n )
>>
>> That said, it might be easier to read the overflow check if we do:
>>
>> count = x & PGC_count_mask;
>> if ( !count || ((PGC_count_mask - count) <= n) )
>>
>> I haven't measured and check which of the two options would result to better
>> code and performance (get_page() is often called).
>>
>
> Correct me if I understand wrongly:
> IMO, only option two is actually catching any overflow? Let (PGC_count_mask - count) <= nr
> stay in the loop, not before the loop like option 1, to cover the changeable page->count_info.
Both option should catch the overflow. In option 1, this was 2 part check:
if ( nr >= PGC_count_mask )
...
do
{
if ( unlikely((x + nr) & PGC_count_mask <= n )
return NULL;
...
} while (...);
Cheers,
--
Julien Grall
© 2016 - 2026 Red Hat, Inc.