Also update different helpes to use NODE_DATA()->memtier. Since
node specific memtier can change based on the reassignment of
NUMA node to a different memory tiers, accessing NODE_DATA()->memtier
needs to happen under an rcu read lock or memory_tier_lock.

Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
---
 include/linux/mmzone.h |  3 +++
 mm/memory-tiers.c      | 38 ++++++++++++++++++++++++++++++++------
 2 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index aab70355d64f..353812495a70 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -928,6 +928,9 @@ typedef struct pglist_data {
 	/* Per-node vmstats */
 	struct per_cpu_nodestat __percpu *per_cpu_nodestats;
 	atomic_long_t		vm_stat[NR_VM_NODE_STAT_ITEMS];
+#ifdef CONFIG_NUMA
+	struct memory_tier __rcu *memtier;
+#endif
 } pg_data_t;
 
 #define node_present_pages(nid)	(NODE_DATA(nid)->node_present_pages)
diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c
index 02e514e87d5c..3778ac6a44a1 100644
--- a/mm/memory-tiers.c
+++ b/mm/memory-tiers.c
@@ -5,6 +5,7 @@
 #include <linux/kobject.h>
 #include <linux/memory.h>
 #include <linux/random.h>
+#include <linux/mmzone.h>
 #include <linux/memory-tiers.h>
 
 #include "internal.h"
@@ -137,12 +138,18 @@ static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memty
 
 static struct memory_tier *__node_get_memory_tier(int node)
 {
-	struct memory_dev_type *memtype;
+	pg_data_t *pgdat;
 
-	memtype = node_memory_types[node];
-	if (memtype && node_isset(node, memtype->nodes))
-		return memtype->memtier;
-	return NULL;
+	pgdat = NODE_DATA(node);
+	if (!pgdat)
+		return NULL;
+	/*
+	 * Since we hold memory_tier_lock, we can avoid
+	 * RCU read locks when accessing the details. No
+	 * parallel updates are possible here.
+	 */
+	return rcu_dereference_check(pgdat->memtier,
+				     lockdep_is_held(&memory_tier_lock));
 }
 
 #ifdef CONFIG_MIGRATION
@@ -295,6 +302,8 @@ static struct memory_tier *set_node_memory_tier(int node)
 {
 	struct memory_tier *memtier;
 	struct memory_dev_type *memtype;
+	pg_data_t *pgdat = NODE_DATA(node);
+
 
 	lockdep_assert_held_once(&memory_tier_lock);
 
@@ -307,6 +316,8 @@ static struct memory_tier *set_node_memory_tier(int node)
 	memtype = node_memory_types[node];
 	node_set(node, memtype->nodes);
 	memtier = find_create_memory_tier(memtype);
+	if (!IS_ERR(memtier))
+		rcu_assign_pointer(pgdat->memtier, memtier);
 	return memtier;
 }
 
@@ -319,12 +330,25 @@ static void destroy_memory_tier(struct memory_tier *memtier)
 static bool clear_node_memory_tier(int node)
 {
 	bool cleared = false;
+	pg_data_t *pgdat;
 	struct memory_tier *memtier;
 
+	pgdat = NODE_DATA(node);
+	if (!pgdat)
+		return false;
+
+	/*
+	 * Make sure that anybody looking at NODE_DATA who finds
+	 * a valid memtier finds memory_dev_types with nodes still
+	 * linked to the memtier. We achieve this by waiting for
+	 * rcu read section to finish using synchronize_rcu.
+	 */
 	memtier = __node_get_memory_tier(node);
 	if (memtier) {
 		struct memory_dev_type *memtype;
 
+		rcu_assign_pointer(pgdat->memtier, NULL);
+		synchronize_rcu();
 		memtype = node_memory_types[node];
 		node_clear(node, memtype->nodes);
 		if (nodes_empty(memtype->nodes)) {
@@ -422,8 +446,10 @@ static int __init memory_tier_init(void)
 		panic("%s() failed to register memory tier: %ld\n",
 		      __func__, PTR_ERR(memtier));
 
-	for_each_node_state(node, N_MEMORY)
+	for_each_node_state(node, N_MEMORY) {
 		__init_node_memory_type(node, default_dram_type);
+		rcu_assign_pointer(NODE_DATA(node)->memtier, memtier);
+	}
 
 	mutex_unlock(&memory_tier_lock);
 #ifdef CONFIG_MIGRATION
-- 
2.37.1

"Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com> writes:

> Also update different helpes to use NODE_DATA()->memtier. Since
> node specific memtier can change based on the reassignment of
> NUMA node to a different memory tiers, accessing NODE_DATA()->memtier
> needs to happen under an rcu read lock or memory_tier_lock.
>
> Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
> ---
>  include/linux/mmzone.h |  3 +++
>  mm/memory-tiers.c      | 38 ++++++++++++++++++++++++++++++++------
>  2 files changed, 35 insertions(+), 6 deletions(-)
>
> diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
> index aab70355d64f..353812495a70 100644
> --- a/include/linux/mmzone.h
> +++ b/include/linux/mmzone.h
> @@ -928,6 +928,9 @@ typedef struct pglist_data {
>  	/* Per-node vmstats */
>  	struct per_cpu_nodestat __percpu *per_cpu_nodestats;
>  	atomic_long_t		vm_stat[NR_VM_NODE_STAT_ITEMS];
> +#ifdef CONFIG_NUMA
> +	struct memory_tier __rcu *memtier;
> +#endif
>  } pg_data_t;
>  
>  #define node_present_pages(nid)	(NODE_DATA(nid)->node_present_pages)
> diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c
> index 02e514e87d5c..3778ac6a44a1 100644
> --- a/mm/memory-tiers.c
> +++ b/mm/memory-tiers.c
> @@ -5,6 +5,7 @@
>  #include <linux/kobject.h>
>  #include <linux/memory.h>
>  #include <linux/random.h>
> +#include <linux/mmzone.h>
>  #include <linux/memory-tiers.h>
>  
>  #include "internal.h"
> @@ -137,12 +138,18 @@ static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memty
>  
>  static struct memory_tier *__node_get_memory_tier(int node)
>  {
> -	struct memory_dev_type *memtype;
> +	pg_data_t *pgdat;
>  
> -	memtype = node_memory_types[node];
> -	if (memtype && node_isset(node, memtype->nodes))
> -		return memtype->memtier;
> -	return NULL;

After adding pgdat->memtier, it appears there's unnecessary to keep
memtype->memtier?

Best Regards,
Huang, Ying

> +	pgdat = NODE_DATA(node);
> +	if (!pgdat)
> +		return NULL;
> +	/*
> +	 * Since we hold memory_tier_lock, we can avoid
> +	 * RCU read locks when accessing the details. No
> +	 * parallel updates are possible here.
> +	 */
> +	return rcu_dereference_check(pgdat->memtier,
> +				     lockdep_is_held(&memory_tier_lock));
>  }
>  
>  #ifdef CONFIG_MIGRATION
> @@ -295,6 +302,8 @@ static struct memory_tier *set_node_memory_tier(int node)
>  {
>  	struct memory_tier *memtier;
>  	struct memory_dev_type *memtype;
> +	pg_data_t *pgdat = NODE_DATA(node);
> +
>  
>  	lockdep_assert_held_once(&memory_tier_lock);
>  
> @@ -307,6 +316,8 @@ static struct memory_tier *set_node_memory_tier(int node)
>  	memtype = node_memory_types[node];
>  	node_set(node, memtype->nodes);
>  	memtier = find_create_memory_tier(memtype);
> +	if (!IS_ERR(memtier))
> +		rcu_assign_pointer(pgdat->memtier, memtier);
>  	return memtier;
>  }
>  
> @@ -319,12 +330,25 @@ static void destroy_memory_tier(struct memory_tier *memtier)
>  static bool clear_node_memory_tier(int node)
>  {
>  	bool cleared = false;
> +	pg_data_t *pgdat;
>  	struct memory_tier *memtier;
>  
> +	pgdat = NODE_DATA(node);
> +	if (!pgdat)
> +		return false;
> +
> +	/*
> +	 * Make sure that anybody looking at NODE_DATA who finds
> +	 * a valid memtier finds memory_dev_types with nodes still
> +	 * linked to the memtier. We achieve this by waiting for
> +	 * rcu read section to finish using synchronize_rcu.
> +	 */
>  	memtier = __node_get_memory_tier(node);
>  	if (memtier) {
>  		struct memory_dev_type *memtype;
>  
> +		rcu_assign_pointer(pgdat->memtier, NULL);
> +		synchronize_rcu();
>  		memtype = node_memory_types[node];
>  		node_clear(node, memtype->nodes);
>  		if (nodes_empty(memtype->nodes)) {
> @@ -422,8 +446,10 @@ static int __init memory_tier_init(void)
>  		panic("%s() failed to register memory tier: %ld\n",
>  		      __func__, PTR_ERR(memtier));
>  
> -	for_each_node_state(node, N_MEMORY)
> +	for_each_node_state(node, N_MEMORY) {
>  		__init_node_memory_type(node, default_dram_type);
> +		rcu_assign_pointer(NODE_DATA(node)->memtier, memtier);
> +	}
>  
>  	mutex_unlock(&memory_tier_lock);
>  #ifdef CONFIG_MIGRATION

On 8/9/22 10:51 AM, Huang, Ying wrote:
> "Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com> writes:
> 
>> Also update different helpes to use NODE_DATA()->memtier. Since
>> node specific memtier can change based on the reassignment of
>> NUMA node to a different memory tiers, accessing NODE_DATA()->memtier
>> needs to happen under an rcu read lock or memory_tier_lock.
>>
>> Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
>> ---
>>  include/linux/mmzone.h |  3 +++
>>  mm/memory-tiers.c      | 38 ++++++++++++++++++++++++++++++++------
>>  2 files changed, 35 insertions(+), 6 deletions(-)
>>
>> diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
>> index aab70355d64f..353812495a70 100644
>> --- a/include/linux/mmzone.h
>> +++ b/include/linux/mmzone.h
>> @@ -928,6 +928,9 @@ typedef struct pglist_data {
>>  	/* Per-node vmstats */
>>  	struct per_cpu_nodestat __percpu *per_cpu_nodestats;
>>  	atomic_long_t		vm_stat[NR_VM_NODE_STAT_ITEMS];
>> +#ifdef CONFIG_NUMA
>> +	struct memory_tier __rcu *memtier;
>> +#endif
>>  } pg_data_t;
>>  
>>  #define node_present_pages(nid)	(NODE_DATA(nid)->node_present_pages)
>> diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c
>> index 02e514e87d5c..3778ac6a44a1 100644
>> --- a/mm/memory-tiers.c
>> +++ b/mm/memory-tiers.c
>> @@ -5,6 +5,7 @@
>>  #include <linux/kobject.h>
>>  #include <linux/memory.h>
>>  #include <linux/random.h>
>> +#include <linux/mmzone.h>
>>  #include <linux/memory-tiers.h>
>>  
>>  #include "internal.h"
>> @@ -137,12 +138,18 @@ static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memty
>>  
>>  static struct memory_tier *__node_get_memory_tier(int node)
>>  {
>> -	struct memory_dev_type *memtype;
>> +	pg_data_t *pgdat;
>>  
>> -	memtype = node_memory_types[node];
>> -	if (memtype && node_isset(node, memtype->nodes))
>> -		return memtype->memtier;
>> -	return NULL;
> 
> After adding pgdat->memtier, it appears there's unnecessary to keep
> memtype->memtier?
> 

It do simplify find_create_memory_tier() where I use if (memtype->memtier) 
to check whether the memtype is already added to a memory tier. I could switch
that to list_empty(memtype->tier_sibiling). But I felt the current one is much
cleaner
.
-aneesh

Aneesh Kumar K V <aneesh.kumar@linux.ibm.com> writes:

> On 8/9/22 10:51 AM, Huang, Ying wrote:
>> "Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com> writes:
>> 
>>> Also update different helpes to use NODE_DATA()->memtier. Since
>>> node specific memtier can change based on the reassignment of
>>> NUMA node to a different memory tiers, accessing NODE_DATA()->memtier
>>> needs to happen under an rcu read lock or memory_tier_lock.
>>>
>>> Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
>>> ---
>>>  include/linux/mmzone.h |  3 +++
>>>  mm/memory-tiers.c      | 38 ++++++++++++++++++++++++++++++++------
>>>  2 files changed, 35 insertions(+), 6 deletions(-)
>>>
>>> diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
>>> index aab70355d64f..353812495a70 100644
>>> --- a/include/linux/mmzone.h
>>> +++ b/include/linux/mmzone.h
>>> @@ -928,6 +928,9 @@ typedef struct pglist_data {
>>>  	/* Per-node vmstats */
>>>  	struct per_cpu_nodestat __percpu *per_cpu_nodestats;
>>>  	atomic_long_t		vm_stat[NR_VM_NODE_STAT_ITEMS];
>>> +#ifdef CONFIG_NUMA
>>> +	struct memory_tier __rcu *memtier;
>>> +#endif
>>>  } pg_data_t;
>>>  
>>>  #define node_present_pages(nid)	(NODE_DATA(nid)->node_present_pages)
>>> diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c
>>> index 02e514e87d5c..3778ac6a44a1 100644
>>> --- a/mm/memory-tiers.c
>>> +++ b/mm/memory-tiers.c
>>> @@ -5,6 +5,7 @@
>>>  #include <linux/kobject.h>
>>>  #include <linux/memory.h>
>>>  #include <linux/random.h>
>>> +#include <linux/mmzone.h>
>>>  #include <linux/memory-tiers.h>
>>>  
>>>  #include "internal.h"
>>> @@ -137,12 +138,18 @@ static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memty
>>>  
>>>  static struct memory_tier *__node_get_memory_tier(int node)
>>>  {
>>> -	struct memory_dev_type *memtype;
>>> +	pg_data_t *pgdat;
>>>  
>>> -	memtype = node_memory_types[node];
>>> -	if (memtype && node_isset(node, memtype->nodes))
>>> -		return memtype->memtier;
>>> -	return NULL;
>> 
>> After adding pgdat->memtier, it appears there's unnecessary to keep
>> memtype->memtier?
>> 
>
> It do simplify find_create_memory_tier() where I use if (memtype->memtier) 
> to check whether the memtype is already added to a memory tier. I could switch
> that to list_empty(memtype->tier_sibiling). But I felt the current one is much
> cleaner

I prefer "list_empty(memtype->tier_sibiling)".  But I will let you to
decide.

Best Regards,
Huang, Ying

"Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com> writes:

> Also update different helpes to use NODE_DATA()->memtier. Since
> node specific memtier can change based on the reassignment of
> NUMA node to a different memory tiers, accessing NODE_DATA()->memtier
> needs to happen under an rcu read lock or memory_tier_lock.
>
> Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
> ---
>  include/linux/mmzone.h |  3 +++
>  mm/memory-tiers.c      | 38 ++++++++++++++++++++++++++++++++------
>  2 files changed, 35 insertions(+), 6 deletions(-)
>
> diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
> index aab70355d64f..353812495a70 100644
> --- a/include/linux/mmzone.h
> +++ b/include/linux/mmzone.h
> @@ -928,6 +928,9 @@ typedef struct pglist_data {
>  	/* Per-node vmstats */
>  	struct per_cpu_nodestat __percpu *per_cpu_nodestats;
>  	atomic_long_t		vm_stat[NR_VM_NODE_STAT_ITEMS];
> +#ifdef CONFIG_NUMA
> +	struct memory_tier __rcu *memtier;
> +#endif
>  } pg_data_t;
>  
>  #define node_present_pages(nid)	(NODE_DATA(nid)->node_present_pages)
> diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c
> index 02e514e87d5c..3778ac6a44a1 100644
> --- a/mm/memory-tiers.c
> +++ b/mm/memory-tiers.c
> @@ -5,6 +5,7 @@
>  #include <linux/kobject.h>
>  #include <linux/memory.h>
>  #include <linux/random.h>
> +#include <linux/mmzone.h>
>  #include <linux/memory-tiers.h>
>  
>  #include "internal.h"
> @@ -137,12 +138,18 @@ static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memty
>  
>  static struct memory_tier *__node_get_memory_tier(int node)
>  {
> -	struct memory_dev_type *memtype;
> +	pg_data_t *pgdat;
>  
> -	memtype = node_memory_types[node];
> -	if (memtype && node_isset(node, memtype->nodes))
> -		return memtype->memtier;
> -	return NULL;
> +	pgdat = NODE_DATA(node);
> +	if (!pgdat)
> +		return NULL;
> +	/*
> +	 * Since we hold memory_tier_lock, we can avoid
> +	 * RCU read locks when accessing the details. No
> +	 * parallel updates are possible here.
> +	 */
> +	return rcu_dereference_check(pgdat->memtier,
> +				     lockdep_is_held(&memory_tier_lock));
>  }
>  
>  #ifdef CONFIG_MIGRATION
> @@ -295,6 +302,8 @@ static struct memory_tier *set_node_memory_tier(int node)
>  {
>  	struct memory_tier *memtier;
>  	struct memory_dev_type *memtype;
> +	pg_data_t *pgdat = NODE_DATA(node);
> +
>  
>  	lockdep_assert_held_once(&memory_tier_lock);
>  
> @@ -307,6 +316,8 @@ static struct memory_tier *set_node_memory_tier(int node)
>  	memtype = node_memory_types[node];
>  	node_set(node, memtype->nodes);
>  	memtier = find_create_memory_tier(memtype);
> +	if (!IS_ERR(memtier))
> +		rcu_assign_pointer(pgdat->memtier, memtier);
>  	return memtier;
>  }
>  
> @@ -319,12 +330,25 @@ static void destroy_memory_tier(struct memory_tier *memtier)
>  static bool clear_node_memory_tier(int node)
>  {
>  	bool cleared = false;
> +	pg_data_t *pgdat;
>  	struct memory_tier *memtier;
>  
> +	pgdat = NODE_DATA(node);
> +	if (!pgdat)
> +		return false;
> +
> +	/*
> +	 * Make sure that anybody looking at NODE_DATA who finds
> +	 * a valid memtier finds memory_dev_types with nodes still
> +	 * linked to the memtier. We achieve this by waiting for
> +	 * rcu read section to finish using synchronize_rcu.
> +	 */

The synchronize_rcu() is also needed because we may free the memory
tier.

Best Regards,
Huang, Ying

>  	memtier = __node_get_memory_tier(node);
>  	if (memtier) {
>  		struct memory_dev_type *memtype;
>  
> +		rcu_assign_pointer(pgdat->memtier, NULL);
> +		synchronize_rcu();
>  		memtype = node_memory_types[node];
>  		node_clear(node, memtype->nodes);
>  		if (nodes_empty(memtype->nodes)) {
> @@ -422,8 +446,10 @@ static int __init memory_tier_init(void)
>  		panic("%s() failed to register memory tier: %ld\n",
>  		      __func__, PTR_ERR(memtier));
>  
> -	for_each_node_state(node, N_MEMORY)
> +	for_each_node_state(node, N_MEMORY) {
>  		__init_node_memory_type(node, default_dram_type);
> +		rcu_assign_pointer(NODE_DATA(node)->memtier, memtier);
> +	}
>  
>  	mutex_unlock(&memory_tier_lock);
>  #ifdef CONFIG_MIGRATION