Series comparison

-[PATCH 0/2] Implement numa node notifier
+[PATCH v2 0/3] Implement numa node notifier
+v1 -> v2:
+  - Remove status_change_nid_normal and the code that
+    deals with it (David & Vlastimil)
+  - Remove slab_mem_offline_callback (David & Vlastimil)
+  - Change the order of canceling the notifiers
+    in {online,offline}_pages (Vlastimil)
+  - Fix up a couple of whitespaces (Jonathan Cameron)
+  - Add RBs-by
 Memory notifier is a tool that allow consumers to get notified whenever
 memory gets onlined or offlined in the system.
 Currently, there are 10 consumers of that, but 5 out of those 10 consumers
-are only interested in getting notifications when a numa node has changed its
+are only interested in getting notifications when a numa node changes its
-state.
+memory state.
 That means going from memoryless to memory-aware of vice versa.
 Which means that for every {online,offline}_pages operation they get
 notified even though the numa node might not have changed its state.
-The first patch implements a numa node notifier that does just that, and have
+While we are doing this, remove status_change_nid_normal, as the only
 current user (slub) does not really need it.
 This allows us to further simplify and clean up the code.
 The first patch gets rid of status_change_nid_normal in slub.
 The second patch implements a numa node notifier that does just that, and have
 those consumers register in there, so they get notified only when they are
 interested.
-The second patch replaces 'status_change_normal{_normal}' fields within
+The third patch replaces 'status_change_nid{_normal}' fields within
 memory_notify with a 'nid', as that is only what we need for memory
 notifer and update the only user of it (page_ext).
 Consumers that are only interested in numa node states change are:
  - memory-tier
  - slub
  - cpuset
  - hmat
  - cxl
+Oscar Salvador (3):
+  mm,slub: Do not special case N_NORMAL nodes for slab_nodes
+  mm,memory_hotplug: Implement numa node notifier
+  mm,memory_hotplug: Rename status_change_nid parameter in memory_notify
-Oscar Salvador (2):
+ drivers/acpi/numa/hmat.c  |   6 +-
-  mm,memory_hotplug: Implement numa node notifier
+ drivers/base/node.c       |  19 +++++++
-  mm,memory_hotplug: Replace status_change_nid parameter in
+ drivers/cxl/core/region.c |  14 ++---
-    memory_notify
+ drivers/cxl/cxl.h         |   4 +-
+ include/linux/memory.h    |  38 ++++++++++++-
- drivers/acpi/numa/hmat.c  |  6 +--
+ kernel/cgroup/cpuset.c    |   2 +-
- drivers/base/node.c       | 19 +++++++++
+ mm/memory-tiers.c         |   8 +--
- drivers/cxl/core/region.c | 14 +++----
+ mm/memory_hotplug.c       | 117 +++++++++++++++++++++-----------------
- drivers/cxl/cxl.h         |  4 +-
+ mm/page_ext.c             |  12 +---
- include/linux/memory.h    | 37 ++++++++++++++++++
+ mm/slub.c                 |  41 ++-----------
- kernel/cgroup/cpuset.c    |  2 +-
+files changed, 145 insertions(+), 116 deletions(-)
  mm/memory-tiers.c         |  8 ++--
  mm/memory_hotplug.c       | 82 +++++++++++++++++++++++++++++----------
  mm/page_ext.c             | 12 +-----
  mm/slub.c                 | 22 +++++------
 files changed, 146 insertions(+), 60 deletions(-)
 --
 .49.0

-New patch
+[PATCH v2 1/3] mm,slub: Do not special case N_NORMAL nodes for slab_nodes
+Currently, slab_mem_going_going_callback() checks whether the node has
+N_NORMAL memory in order to be set in slab_nodes.
+While it is true that gettind rid of that enforcing would mean
+ending up with movables nodes in slab_nodes, the memory waste that comes
+with that is negligible.
+So stop checking for status_change_nid_normal and just use status_change_nid
+instead which works for both types of memory.
+Also, once we allocate the kmem_cache_node cache  for the node in
+slab_mem_online_callback(), we never deallocate it in
+slab_mem_off_callback() when the node goes memoryless, so we can just
+get rid of it.
+The only side effect is that we will stop clearing the node from slab_nodes.
+Signed-off-by: Oscar Salvador <osalvador@suse.de>
+---
+ mm/slub.c | 30 +-----------------------------
+file changed, 1 insertion(+), 29 deletions(-)
+diff --git a/mm/slub.c b/mm/slub.c
+index XXXXXXX..XXXXXXX 100644
+--- a/mm/slub.c
++++ b/mm/slub.c
+@@ -XXX,XX +XXX,XX @@ static int slab_mem_going_offline_callback(void *arg)
+     return 0;
+ }
+-static void slab_mem_offline_callback(void *arg)
+-{
+-    struct memory_notify *marg = arg;
+-    int offline_node;
+-
+-    offline_node = marg->status_change_nid_normal;
+-
+-    /*
+-     * If the node still has available memory. we need kmem_cache_node
+-     * for it yet.
+-     */
+-    if (offline_node < 0)
+-        return;
+-
+-    mutex_lock(&slab_mutex);
+-    node_clear(offline_node, slab_nodes);
+-    /*
+-     * We no longer free kmem_cache_node structures here, as it would be
+-     * racy with all get_node() users, and infeasible to protect them with
+-     * slab_mutex.
+-     */
+-    mutex_unlock(&slab_mutex);
+-}
+-
+ static int slab_mem_going_online_callback(void *arg)
+ {
+     struct kmem_cache_node *n;
+     struct kmem_cache *s;
+     struct memory_notify *marg = arg;
+-    int nid = marg->status_change_nid_normal;
++    int nid = marg->status_change_nid;
+     int ret = 0;
+     /*
+@@ -XXX,XX +XXX,XX @@ static int slab_memory_callback(struct notifier_block *self,
+     case MEM_GOING_OFFLINE:
+         ret = slab_mem_going_offline_callback(arg);
+         break;
+-    case MEM_OFFLINE:
+-    case MEM_CANCEL_ONLINE:
+-        slab_mem_offline_callback(arg);
+-        break;
+     case MEM_ONLINE:
+     case MEM_CANCEL_OFFLINE:
+         break;
+--
+.49.0

-[PATCH 1/2] mm,memory_hotplug: Implement numa node notifier
+[PATCH v2 2/3] mm,memory_hotplug: Implement numa node notifier
-There are at least four consumers of hotplug_memory_notifier that what they
+There are at least five consumers of hotplug_memory_notifier that what they
 really are interested in is whether any numa node changed its state, e.g: going
-from being memory aware to becoming memoryless.
+from being memory aware to becoming memoryless and vice versa.
 Implement a specific notifier for numa nodes when their state gets changed,
 and have those consumers that only care about numa node state changes use it.
 Signed-off-by: Oscar Salvador <osalvador@suse.de>
+Reviewed-by: Harry Yoo <harry.yoo@oracle.com>
+Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
 ---
- drivers/acpi/numa/hmat.c  |  6 +--
+ drivers/acpi/numa/hmat.c  |   6 +-
- drivers/base/node.c       | 19 +++++++++
+ drivers/base/node.c       |  19 +++++++
- drivers/cxl/core/region.c | 14 +++----
+ drivers/cxl/core/region.c |  14 ++---
- drivers/cxl/cxl.h         |  4 +-
+ drivers/cxl/cxl.h         |   4 +-
- include/linux/memory.h    | 38 ++++++++++++++++++
+ include/linux/memory.h    |  38 ++++++++++++-
- kernel/cgroup/cpuset.c    |  2 +-
+ kernel/cgroup/cpuset.c    |   2 +-
- mm/memory-tiers.c         |  8 ++--
+ mm/memory-tiers.c         |   8 +--
- mm/memory_hotplug.c       | 84 +++++++++++++++++++++++++++++----------
+ mm/memory_hotplug.c       | 117 +++++++++++++++++++++-----------------
- mm/slub.c                 | 22 +++++-----
+ mm/slub.c                 |  13 ++---
-files changed, 148 insertions(+), 49 deletions(-)
+files changed, 144 insertions(+), 77 deletions(-)
 diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c
 index XXXXXXX..XXXXXXX 100644
 --- a/drivers/acpi/numa/hmat.c
 +++ b/drivers/acpi/numa/hmat.c
 ...
 +
  struct memory_notify {
      /*
       * The altmap_start_pfn and altmap_nr_pages fields are designated for
 @@ -XXX,XX +XXX,XX @@ struct memory_notify {
+     unsigned long altmap_nr_pages;
+     unsigned long start_pfn;
+     unsigned long nr_pages;
+-    int status_change_nid_normal;
++    int status_change_nid;
++};
++
++struct node_notify {
      int status_change_nid;
  };
-+struct node_notify {
-+    int status_change_nid_normal;
-+    int status_change_nid;
-+};
-+
- struct notifier_block;
- struct mem_section;
 @@ -XXX,XX +XXX,XX @@ static inline int hotplug_memory_notifier(notifier_fn_t fn, int pri)
  {
      return 0;
  }
 ...
 -    struct zone *zone, struct memory_notify *arg)
 +    struct zone *zone, struct node_notify *arg)
  {
      int nid = zone_to_nid(zone);
-@@ -XXX,XX +XXX,XX @@ static void node_states_check_changes_online(unsigned long nr_pages,
+     arg->status_change_nid = NUMA_NO_NODE;
-         arg->status_change_nid_normal = nid;
+-    arg->status_change_nid_normal = NUMA_NO_NODE;
- }
      if (!node_state(nid, N_MEMORY))
          arg->status_change_nid = nid;
 -    if (zone_idx(zone) <= ZONE_NORMAL && !node_state(nid, N_NORMAL_MEMORY))
 -        arg->status_change_nid_normal = nid;
  }
 -static void node_states_set_node(int node, struct memory_notify *arg)
 +static void node_states_set_node(int node, struct node_notify *arg)
  {
-     if (arg->status_change_nid_normal >= 0)
+-    if (arg->status_change_nid_normal >= 0)
-         node_set_state(node, N_NORMAL_MEMORY);
+-        node_set_state(node, N_NORMAL_MEMORY);
 -
      if (arg->status_change_nid >= 0)
          node_set_state(node, N_MEMORY);
  }
 @@ -XXX,XX +XXX,XX @@ int online_pages(unsigned long pfn, unsigned long nr_pages,
      int need_zonelists_rebuild = 0;
      const int nid = zone_to_nid(zone);
      int ret;
 -    struct memory_notify arg;
 ...
 -    arg.nr_pages = nr_pages;
 -    node_states_check_changes_online(nr_pages, zone, &arg);
 +    mem_arg.start_pfn = pfn;
 +    mem_arg.nr_pages = nr_pages;
 +    node_states_check_changes_online(nr_pages, zone, &node_arg);
++
 -    ret = memory_notify(MEM_GOING_ONLINE, &arg);
 +    if (node_arg.status_change_nid >= 0) {
 +        /* Node is becoming memory aware. Notify consumers */
 +        cancel_node_notifier_on_err = true;
 +        ret = node_notify(NODE_BECOMING_MEM_AWARE, &node_arg);
 +        ret = notifier_to_errno(ret);
 +        if (ret)
 +            goto failed_addition;
 +    }
-+
 -    ret = memory_notify(MEM_GOING_ONLINE, &arg);
 +    cancel_mem_notifier_on_err = true;
 +    mem_arg.status_change_nid = node_arg.status_change_nid;
-+    mem_arg.status_change_nid_normal = node_arg.status_change_nid_normal;
 +    ret = memory_notify(MEM_GOING_ONLINE, &mem_arg);
      ret = notifier_to_errno(ret);
      if (ret)
          goto failed_addition;
 @@ -XXX,XX +XXX,XX @@ int online_pages(unsigned long pfn, unsigned long nr_pages,
 ...
  failed_addition:
      pr_debug("online_pages [mem %#010llx-%#010llx] failed\n",
           (unsigned long long) pfn << PAGE_SHIFT,
           (((unsigned long long) pfn + nr_pages) << PAGE_SHIFT) - 1);
 -    memory_notify(MEM_CANCEL_ONLINE, &arg);
++    if (cancel_mem_notifier_on_err)
++        memory_notify(MEM_CANCEL_ONLINE, &mem_arg);
 +    if (cancel_node_notifier_on_err)
 +        node_notify(NODE_CANCEL_MEM_AWARE, &node_arg);
-+    if (cancel_mem_notifier_on_err)
-+        memory_notify(MEM_CANCEL_ONLINE, &mem_arg);
      remove_pfn_range_from_zone(zone, pfn, nr_pages);
      return ret;
  }
 @@ -XXX,XX +XXX,XX @@ early_param("movable_node", cmdline_parse_movable_node);
 ...
 -        struct zone *zone, struct memory_notify *arg)
 +        struct zone *zone, struct node_notify *arg)
  {
      struct pglist_data *pgdat = zone->zone_pgdat;
      unsigned long present_pages = 0;
-@@ -XXX,XX +XXX,XX @@ static void node_states_check_changes_offline(unsigned long nr_pages,
+     enum zone_type zt;
      arg->status_change_nid = NUMA_NO_NODE;
 -    arg->status_change_nid_normal = NUMA_NO_NODE;
      /*
 -     * Check whether node_states[N_NORMAL_MEMORY] will be changed.
 -     * If the memory to be offline is within the range
 -     * [0..ZONE_NORMAL], and it is the last present memory there,
 -     * the zones in that range will become empty after the offlining,
 -     * thus we can determine that we need to clear the node from
 -     * node_states[N_NORMAL_MEMORY].
 +     * Here we count the possible pages within the range [0..ZONE_MOVABLE].
 +     * If after having accounted all the pages, we see that the nr_pages to
 +     * be offlined is over or equal to the accounted pages, we know that the
 +     * node will become empty, ans so, we can clear it for N_MEMORY.
       */
 -    for (zt = 0; zt <= ZONE_NORMAL; zt++)
 +    for (zt = 0; zt <= ZONE_MOVABLE; zt++)
          present_pages += pgdat->node_zones[zt].present_pages;
 -    if (zone_idx(zone) <= ZONE_NORMAL && nr_pages >= present_pages)
 -        arg->status_change_nid_normal = zone_to_nid(zone);
 -
 -    /*
 -     * We have accounted the pages from [0..ZONE_NORMAL); ZONE_HIGHMEM
 -     * does not apply as we don't support 32bit.
 -     * Here we count the possible pages from ZONE_MOVABLE.
 -     * If after having accounted all the pages, we see that the nr_pages
 -     * to be offlined is over or equal to the accounted pages,
 -     * we know that the node will become empty, and so, we can clear
 -     * it for N_MEMORY as well.
 -     */
 -    present_pages += pgdat->node_zones[ZONE_MOVABLE].present_pages;
      if (nr_pages >= present_pages)
          arg->status_change_nid = zone_to_nid(zone);
  }
 -static void node_states_clear_node(int node, struct memory_notify *arg)
 +static void node_states_clear_node(int node, struct node_notify *arg)
  {
-     if (arg->status_change_nid_normal >= 0)
+-    if (arg->status_change_nid_normal >= 0)
-         node_clear_state(node, N_NORMAL_MEMORY);
+-        node_clear_state(node, N_NORMAL_MEMORY);
 -
      if (arg->status_change_nid >= 0)
          node_clear_state(node, N_MEMORY);
  }
 @@ -XXX,XX +XXX,XX @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
      unsigned long pfn, managed_pages, system_ram_pages = 0;
      const int node = zone_to_nid(zone);
      unsigned long flags;
 -    struct memory_notify arg;
 ...
 +    }
 -    ret = memory_notify(MEM_GOING_OFFLINE, &arg);
 +    cancel_mem_notifier_on_err = true;
 +    mem_arg.status_change_nid = node_arg.status_change_nid;
-+    mem_arg.status_change_nid_normal = node_arg.status_change_nid_normal;
 +    ret = memory_notify(MEM_GOING_OFFLINE, &mem_arg);
      ret = notifier_to_errno(ret);
      if (ret) {
          reason = "notifier failure";
 @@ -XXX,XX +XXX,XX @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
 ...
 -    if (arg.status_change_nid >= 0) {
 +    if (node_arg.status_change_nid >= 0) {
          kcompactd_stop(node);
          kswapd_stop(node);
-+        /*Node went memoryless. Notifiy interested consumers */
++        /* Node went memoryless. Notifiy interested consumers */
 +        node_notify(NODE_BECAME_MEMORYLESS, &node_arg);
      }
      writeback_set_ratelimit();
 ...
  failed_removal_isolated:
      /* pushback to free area */
      undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
 -    memory_notify(MEM_CANCEL_OFFLINE, &arg);
++    if (cancel_mem_notifier_on_err)
++        memory_notify(MEM_CANCEL_OFFLINE, &mem_arg);
 +    if (cancel_node_notifier_on_err)
 +        node_notify(NODE_CANCEL_MEMORYLESS, &node_arg);
-+    if (cancel_mem_notifier_on_err)
-+        memory_notify(MEM_CANCEL_OFFLINE, &mem_arg);
  failed_removal_pcplists_disabled:
      lru_cache_enable();
      zone_pcp_enable(zone);
 diff --git a/mm/slub.c b/mm/slub.c
 index XXXXXXX..XXXXXXX 100644
 --- a/mm/slub.c
 +++ b/mm/slub.c
-@@ -XXX,XX +XXX,XX @@ static int slab_mem_going_offline_callback(void *arg)
- static void slab_mem_offline_callback(void *arg)
- {
--    struct memory_notify *marg = arg;
-+    struct node_notify *narg = arg;
-     int offline_node;
--    offline_node = marg->status_change_nid_normal;
-+    offline_node = narg->status_change_nid_normal;
-     /*
-      * If the node still has available memory. we need kmem_cache_node
 @@ -XXX,XX +XXX,XX @@ static int slab_mem_going_online_callback(void *arg)
  {
      struct kmem_cache_node *n;
      struct kmem_cache *s;
 -    struct memory_notify *marg = arg;
--    int nid = marg->status_change_nid_normal;
+-    int nid = marg->status_change_nid;
 +    struct node_notify *narg = arg;
-+    int nid = narg->status_change_nid_normal;
++    int nid = narg->status_change_nid;
      int ret = 0;
      /*
 @@ -XXX,XX +XXX,XX @@ static int slab_memory_callback(struct notifier_block *self,
      int ret = 0;
 ...
          break;
 -    case MEM_GOING_OFFLINE:
 +    case NODE_BECOMING_MEMORYLESS:
          ret = slab_mem_going_offline_callback(arg);
          break;
--    case MEM_OFFLINE:
--    case MEM_CANCEL_ONLINE:
-+    case NODE_BECAME_MEMORYLESS:
-+    case NODE_CANCEL_MEM_AWARE:
-         slab_mem_offline_callback(arg);
-         break;
 -    case MEM_ONLINE:
 -    case MEM_CANCEL_OFFLINE:
-+    case NODE_BECAME_MEM_AWARE:
+-        break;
 +    case NODE_CANCEL_MEMORYLESS:
          break;
      }
      if (ret)
+         ret = notifier_from_errno(ret);
 @@ -XXX,XX +XXX,XX @@ void __init kmem_cache_init(void)
              sizeof(struct kmem_cache_node),
              SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0);
 -    hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
 +    hotplug_node_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
      /* Able to allocate the per node structures */
      slab_state = PARTIAL;
 --
 .49.0

-[PATCH 2/2] mm,memory_hotplug: Replace status_change_nid parameter in memory_notify
+[PATCH v2 3/3] mm,memory_hotplug: Rename status_change_nid parameter in memory_notify
-memory notify consumers are only interested in which node the memory we
+The 'status_change_nid' field was used to track changes in the memory
-are adding belongs to, so replace current status_change_nid{_normal} fields
+state of a numa node, but that funcionality has been decoupled from
-with only one that specifies the node.
+memory_notify and moved to node_notify.
 Current consumers of memory_notify are only interested in which node the
 memory we are adding belongs to, so rename current 'status_change_nid'
 to 'nid'.
 Signed-off-by: Oscar Salvador <osalvador@suse.de>
+Reviewed-by: Harry Yoo <harry.yoo@oracle.com>
+Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
 ---
- include/linux/memory.h |  3 +--
+ include/linux/memory.h |  2 +-
- mm/memory_hotplug.c    |  6 ++----
+ mm/memory_hotplug.c    |  4 ++--
  mm/page_ext.c          | 12 +-----------
-files changed, 4 insertions(+), 17 deletions(-)
+files changed, 4 insertions(+), 14 deletions(-)
 diff --git a/include/linux/memory.h b/include/linux/memory.h
 index XXXXXXX..XXXXXXX 100644
 --- a/include/linux/memory.h
 +++ b/include/linux/memory.h
 @@ -XXX,XX +XXX,XX @@ struct memory_notify {
      unsigned long altmap_nr_pages;
      unsigned long start_pfn;
      unsigned long nr_pages;
--    int status_change_nid_normal;
 -    int status_change_nid;
 +    int nid;
  };
  struct node_notify {
 ...
 @@ -XXX,XX +XXX,XX @@ int online_pages(unsigned long pfn, unsigned long nr_pages,
      }
      cancel_mem_notifier_on_err = true;
 -    mem_arg.status_change_nid = node_arg.status_change_nid;
--    mem_arg.status_change_nid_normal = node_arg.status_change_nid_normal;
      ret = memory_notify(MEM_GOING_ONLINE, &mem_arg);
      ret = notifier_to_errno(ret);
      if (ret)
 @@ -XXX,XX +XXX,XX @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
 ...
 @@ -XXX,XX +XXX,XX @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
      }
      cancel_mem_notifier_on_err = true;
 -    mem_arg.status_change_nid = node_arg.status_change_nid;
--    mem_arg.status_change_nid_normal = node_arg.status_change_nid_normal;
      ret = memory_notify(MEM_GOING_OFFLINE, &mem_arg);
      ret = notifier_to_errno(ret);
      if (ret) {
 diff --git a/mm/page_ext.c b/mm/page_ext.c
 index XXXXXXX..XXXXXXX 100644
 ...

Memory notifier is a tool that allow consumers to get notified whenever
memory gets onlined or offlined in the system.
Currently, there are 10 consumers of that, but 5 out of those 10 consumers
are only interested in getting notifications when a numa node has changed its
state.
That means going from memoryless to memory-aware of vice versa.

Which means that for every {online,offline}_pages operation they get
notified even though the numa node might not have changed its state.

The first patch implements a numa node notifier that does just that, and have
those consumers register in there, so they get notified only when they are
interested.

The second patch replaces 'status_change_normal{_normal}' fields within
memory_notify with a 'nid', as that is only what we need for memory
notifer and update the only user of it (page_ext).

Consumers that are only interested in numa node states change are:

- memory-tier
 - slub
 - cpuset
 - hmat
 - cxl

Oscar Salvador (2):
  mm,memory_hotplug: Implement numa node notifier
  mm,memory_hotplug: Replace status_change_nid parameter in
    memory_notify

-- 
2.49.0

There are at least four consumers of hotplug_memory_notifier that what they
really are interested in is whether any numa node changed its state, e.g: going
from being memory aware to becoming memoryless.

Implement a specific notifier for numa nodes when their state gets changed,
and have those consumers that only care about numa node state changes use it.

Signed-off-by: Oscar Salvador <osalvador@suse.de>
---
 drivers/acpi/numa/hmat.c  |  6 +--
 drivers/base/node.c       | 19 +++++++++
 drivers/cxl/core/region.c | 14 +++----
 drivers/cxl/cxl.h         |  4 +-
 include/linux/memory.h    | 38 ++++++++++++++++++
 kernel/cgroup/cpuset.c    |  2 +-
 mm/memory-tiers.c         |  8 ++--
 mm/memory_hotplug.c       | 84 +++++++++++++++++++++++++++++----------
 mm/slub.c                 | 22 +++++-----
 9 files changed, 148 insertions(+), 49 deletions(-)

diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c
index XXXXXXX..XXXXXXX 100644
--- a/drivers/acpi/numa/hmat.c
+++ b/drivers/acpi/numa/hmat.c
@@ -XXX,XX +XXX,XX @@ static int hmat_callback(struct notifier_block *self,
 			 unsigned long action, void *arg)
 {
 	struct memory_target *target;
-	struct memory_notify *mnb = arg;
+	struct node_notify *mnb = arg;
 	int pxm, nid = mnb->status_change_nid;
 
-	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
+	if (nid == NUMA_NO_NODE || action != NODE_BECAME_MEM_AWARE)
 		return NOTIFY_OK;
 
 	pxm = node_to_pxm(nid);
@@ -XXX,XX +XXX,XX @@ static __init int hmat_init(void)
 	hmat_register_targets();
 
 	/* Keep the table and structures if the notifier may use them */
-	if (hotplug_memory_notifier(hmat_callback, HMAT_CALLBACK_PRI))
+	if (hotplug_node_notifier(hmat_callback, HMAT_CALLBACK_PRI))
 		goto out_put;
 
 	if (!hmat_set_default_dram_perf())
diff --git a/drivers/base/node.c b/drivers/base/node.c
index XXXXXXX..XXXXXXX 100644
--- a/drivers/base/node.c
+++ b/drivers/base/node.c
@@ -XXX,XX +XXX,XX @@ static const struct attribute_group *node_access_node_groups[] = {
 	NULL,
 };
 
+static BLOCKING_NOTIFIER_HEAD(node_chain);
+
+int register_node_notifier(struct notifier_block *nb)
+{
+	return blocking_notifier_chain_register(&node_chain, nb);
+}
+EXPORT_SYMBOL(register_node_notifier);
+
+void unregister_node_notifier(struct notifier_block *nb)
+{
+	blocking_notifier_chain_unregister(&node_chain, nb);
+}
+EXPORT_SYMBOL(unregister_node_notifier);
+
+int node_notify(unsigned long val, void *v)
+{
+	return blocking_notifier_call_chain(&node_chain, val, v);
+}
+
 static void node_remove_accesses(struct node *node)
 {
 	struct node_access_nodes *c, *cnext;
diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
index XXXXXXX..XXXXXXX 100644
--- a/drivers/cxl/core/region.c
+++ b/drivers/cxl/core/region.c
@@ -XXX,XX +XXX,XX @@ static int cxl_region_perf_attrs_callback(struct notifier_block *nb,
 					  unsigned long action, void *arg)
 {
 	struct cxl_region *cxlr = container_of(nb, struct cxl_region,
-					       memory_notifier);
-	struct memory_notify *mnb = arg;
+					       node_notifier);
+	struct node_notify *mnb = arg;
 	int nid = mnb->status_change_nid;
 	int region_nid;
 
-	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
+	if (nid == NUMA_NO_NODE || action != NODE_BECAME_MEM_AWARE)
 		return NOTIFY_DONE;
 
 	/*
@@ -XXX,XX +XXX,XX @@ static void shutdown_notifiers(void *_cxlr)
 {
 	struct cxl_region *cxlr = _cxlr;
 
-	unregister_memory_notifier(&cxlr->memory_notifier);
+	unregister_node_notifier(&cxlr->node_notifier);
 	unregister_mt_adistance_algorithm(&cxlr->adist_notifier);
 }
 
@@ -XXX,XX +XXX,XX @@ static int cxl_region_probe(struct device *dev)
 	if (rc)
 		return rc;
 
-	cxlr->memory_notifier.notifier_call = cxl_region_perf_attrs_callback;
-	cxlr->memory_notifier.priority = CXL_CALLBACK_PRI;
-	register_memory_notifier(&cxlr->memory_notifier);
+	cxlr->node_notifier.notifier_call = cxl_region_perf_attrs_callback;
+	cxlr->node_notifier.priority = CXL_CALLBACK_PRI;
+	register_node_notifier(&cxlr->node_notifier);
 
 	cxlr->adist_notifier.notifier_call = cxl_region_calculate_adistance;
 	cxlr->adist_notifier.priority = 100;
diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index XXXXXXX..XXXXXXX 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -XXX,XX +XXX,XX @@ struct cxl_region_params {
  * @flags: Region state flags
  * @params: active + config params for the region
  * @coord: QoS access coordinates for the region
- * @memory_notifier: notifier for setting the access coordinates to node
+ * @node_notifier: notifier for setting the access coordinates to node
  * @adist_notifier: notifier for calculating the abstract distance of node
  */
 struct cxl_region {
@@ -XXX,XX +XXX,XX @@ struct cxl_region {
 	unsigned long flags;
 	struct cxl_region_params params;
 	struct access_coordinate coord[ACCESS_COORDINATE_MAX];
-	struct notifier_block memory_notifier;
+	struct notifier_block node_notifier;
 	struct notifier_block adist_notifier;
 };
 
diff --git a/include/linux/memory.h b/include/linux/memory.h
index XXXXXXX..XXXXXXX 100644
--- a/include/linux/memory.h
+++ b/include/linux/memory.h
@@ -XXX,XX +XXX,XX @@ int set_memory_block_size_order(unsigned int order);
 #define	MEM_PREPARE_ONLINE	(1<<6)
 #define	MEM_FINISH_OFFLINE	(1<<7)
 
+/* These states are used for numa node notifiers */
+#define NODE_BECOMING_MEM_AWARE		(1<<0)
+#define NODE_BECAME_MEM_AWARE		(1<<1)
+#define NODE_BECOMING_MEMORYLESS	(1<<2)
+#define NODE_BECAME_MEMORYLESS		(1<<3)
+#define NODE_CANCEL_MEM_AWARE		(1<<4)
+#define NODE_CANCEL_MEMORYLESS		(1<<5)
+
 struct memory_notify {
 	/*
 	 * The altmap_start_pfn and altmap_nr_pages fields are designated for
@@ -XXX,XX +XXX,XX @@ struct memory_notify {
 	int status_change_nid;
 };
 
+struct node_notify {
+	int status_change_nid_normal;
+	int status_change_nid;
+};
+
 struct notifier_block;
 struct mem_section;
 
@@ -XXX,XX +XXX,XX @@ static inline int hotplug_memory_notifier(notifier_fn_t fn, int pri)
 {
 	return 0;
 }
+
+static inline int register_node_notifier(struct notifier_block *nb)
+{
+	return 0;
+}
+static inline void unregister_node_notifier(struct notifier_block *nb)
+{
+}
+static inline int node_notify(unsigned long val, void *v)
+{
+	return 0;
+}
+static inline int hotplug_node_notifier(notifier_fn_t fn, int pri)
+{
+	return 0;
+}
 #else /* CONFIG_MEMORY_HOTPLUG */
 extern int register_memory_notifier(struct notifier_block *nb);
+extern int register_node_notifier(struct notifier_block *nb);
 extern void unregister_memory_notifier(struct notifier_block *nb);
+extern void unregister_node_notifier(struct notifier_block *nb);
 int create_memory_block_devices(unsigned long start, unsigned long size,
 				struct vmem_altmap *altmap,
 				struct memory_group *group);
 void remove_memory_block_devices(unsigned long start, unsigned long size);
 extern void memory_dev_init(void);
 extern int memory_notify(unsigned long val, void *v);
+extern int node_notify(unsigned long val, void *v);
 extern struct memory_block *find_memory_block(unsigned long section_nr);
 typedef int (*walk_memory_blocks_func_t)(struct memory_block *, void *);
 extern int walk_memory_blocks(unsigned long start, unsigned long size,
@@ -XXX,XX +XXX,XX @@ int walk_dynamic_memory_groups(int nid, walk_memory_groups_func_t func,
 	register_memory_notifier(&fn##_mem_nb);			\
 })
 
+#define hotplug_node_notifier(fn, pri) ({		\
+	static __meminitdata struct notifier_block fn##_node_nb =\
+		{ .notifier_call = fn, .priority = pri };\
+	register_node_notifier(&fn##_node_nb);			\
+})
+
 #ifdef CONFIG_NUMA
 void memory_block_add_nid(struct memory_block *mem, int nid,
 			  enum meminit_context context);
diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
index XXXXXXX..XXXXXXX 100644
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -XXX,XX +XXX,XX @@ void __init cpuset_init_smp(void)
 	cpumask_copy(top_cpuset.effective_cpus, cpu_active_mask);
 	top_cpuset.effective_mems = node_states[N_MEMORY];
 
-	hotplug_memory_notifier(cpuset_track_online_nodes, CPUSET_CALLBACK_PRI);
+	hotplug_node_notifier(cpuset_track_online_nodes, CPUSET_CALLBACK_PRI);
 
 	cpuset_migrate_mm_wq = alloc_ordered_workqueue("cpuset_migrate_mm", 0);
 	BUG_ON(!cpuset_migrate_mm_wq);
diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c
index XXXXXXX..XXXXXXX 100644
--- a/mm/memory-tiers.c
+++ b/mm/memory-tiers.c
@@ -XXX,XX +XXX,XX @@ static int __meminit memtier_hotplug_callback(struct notifier_block *self,
 					      unsigned long action, void *_arg)
 {
 	struct memory_tier *memtier;
-	struct memory_notify *arg = _arg;
+	struct node_notify *arg = _arg;
 
 	/*
 	 * Only update the node migration order when a node is
@@ -XXX,XX +XXX,XX @@ static int __meminit memtier_hotplug_callback(struct notifier_block *self,
 		return notifier_from_errno(0);
 
 	switch (action) {
-	case MEM_OFFLINE:
+	case NODE_BECAME_MEMORYLESS:
 		mutex_lock(&memory_tier_lock);
 		if (clear_node_memory_tier(arg->status_change_nid))
 			establish_demotion_targets();
 		mutex_unlock(&memory_tier_lock);
 		break;
-	case MEM_ONLINE:
+	case NODE_BECAME_MEM_AWARE:
 		mutex_lock(&memory_tier_lock);
 		memtier = set_node_memory_tier(arg->status_change_nid);
 		if (!IS_ERR(memtier))
@@ -XXX,XX +XXX,XX @@ static int __init memory_tier_init(void)
 	nodes_and(default_dram_nodes, node_states[N_MEMORY],
 		  node_states[N_CPU]);
 
-	hotplug_memory_notifier(memtier_hotplug_callback, MEMTIER_HOTPLUG_PRI);
+	hotplug_node_notifier(memtier_hotplug_callback, MEMTIER_HOTPLUG_PRI);
 	return 0;
 }
 subsys_initcall(memory_tier_init);
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index XXXXXXX..XXXXXXX 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -XXX,XX +XXX,XX @@ static void online_pages_range(unsigned long start_pfn, unsigned long nr_pages)
 
 /* check which state of node_states will be changed when online memory */
 static void node_states_check_changes_online(unsigned long nr_pages,
-	struct zone *zone, struct memory_notify *arg)
+	struct zone *zone, struct node_notify *arg)
 {
 	int nid = zone_to_nid(zone);
 
@@ -XXX,XX +XXX,XX @@ static void node_states_check_changes_online(unsigned long nr_pages,
 		arg->status_change_nid_normal = nid;
 }
 
-static void node_states_set_node(int node, struct memory_notify *arg)
+static void node_states_set_node(int node, struct node_notify *arg)
 {
 	if (arg->status_change_nid_normal >= 0)
 		node_set_state(node, N_NORMAL_MEMORY);
@@ -XXX,XX +XXX,XX @@ int online_pages(unsigned long pfn, unsigned long nr_pages,
 	int need_zonelists_rebuild = 0;
 	const int nid = zone_to_nid(zone);
 	int ret;
-	struct memory_notify arg;
+	struct memory_notify mem_arg;
+	struct node_notify node_arg;
+	bool cancel_mem_notifier_on_err = false, cancel_node_notifier_on_err = false;
 
 	/*
 	 * {on,off}lining is constrained to full memory sections (or more
@@ -XXX,XX +XXX,XX @@ int online_pages(unsigned long pfn, unsigned long nr_pages,
 	/* associate pfn range with the zone */
 	move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_ISOLATE);
 
-	arg.start_pfn = pfn;
-	arg.nr_pages = nr_pages;
-	node_states_check_changes_online(nr_pages, zone, &arg);
+	mem_arg.start_pfn = pfn;
+	mem_arg.nr_pages = nr_pages;
+	node_states_check_changes_online(nr_pages, zone, &node_arg);
 
-	ret = memory_notify(MEM_GOING_ONLINE, &arg);
+	if (node_arg.status_change_nid >= 0) {
+		/* Node is becoming memory aware. Notify consumers */
+		cancel_node_notifier_on_err = true;
+		ret = node_notify(NODE_BECOMING_MEM_AWARE, &node_arg);
+		ret = notifier_to_errno(ret);
+		if (ret)
+			goto failed_addition;
+	}
+
+	cancel_mem_notifier_on_err = true;
+	mem_arg.status_change_nid = node_arg.status_change_nid;
+	mem_arg.status_change_nid_normal = node_arg.status_change_nid_normal;
+	ret = memory_notify(MEM_GOING_ONLINE, &mem_arg);
 	ret = notifier_to_errno(ret);
 	if (ret)
 		goto failed_addition;
@@ -XXX,XX +XXX,XX @@ int online_pages(unsigned long pfn, unsigned long nr_pages,
 	online_pages_range(pfn, nr_pages);
 	adjust_present_page_count(pfn_to_page(pfn), group, nr_pages);
 
-	node_states_set_node(nid, &arg);
+	node_states_set_node(nid, &node_arg);
 	if (need_zonelists_rebuild)
 		build_all_zonelists(NULL);
 
@@ -XXX,XX +XXX,XX @@ int online_pages(unsigned long pfn, unsigned long nr_pages,
 	kswapd_run(nid);
 	kcompactd_run(nid);
 
+	if (node_arg.status_change_nid >= 0)
+		/*
+		 * Node went from memoryless to have memory. Notifiy interested
+		 * consumers
+		 */
+		node_notify(NODE_BECAME_MEM_AWARE, &node_arg);
+
 	writeback_set_ratelimit();
 
-	memory_notify(MEM_ONLINE, &arg);
+	memory_notify(MEM_ONLINE, &mem_arg);
 	return 0;
 
 failed_addition:
 	pr_debug("online_pages [mem %#010llx-%#010llx] failed\n",
 		 (unsigned long long) pfn << PAGE_SHIFT,
 		 (((unsigned long long) pfn + nr_pages) << PAGE_SHIFT) - 1);
-	memory_notify(MEM_CANCEL_ONLINE, &arg);
+	if (cancel_node_notifier_on_err)
+		node_notify(NODE_CANCEL_MEM_AWARE, &node_arg);
+	if (cancel_mem_notifier_on_err)
+		memory_notify(MEM_CANCEL_ONLINE, &mem_arg);
 	remove_pfn_range_from_zone(zone, pfn, nr_pages);
 	return ret;
 }
@@ -XXX,XX +XXX,XX @@ early_param("movable_node", cmdline_parse_movable_node);
 
 /* check which state of node_states will be changed when offline memory */
 static void node_states_check_changes_offline(unsigned long nr_pages,
-		struct zone *zone, struct memory_notify *arg)
+		struct zone *zone, struct node_notify *arg)
 {
 	struct pglist_data *pgdat = zone->zone_pgdat;
 	unsigned long present_pages = 0;
@@ -XXX,XX +XXX,XX @@ static void node_states_check_changes_offline(unsigned long nr_pages,
 		arg->status_change_nid = zone_to_nid(zone);
 }
 
-static void node_states_clear_node(int node, struct memory_notify *arg)
+static void node_states_clear_node(int node, struct node_notify *arg)
 {
 	if (arg->status_change_nid_normal >= 0)
 		node_clear_state(node, N_NORMAL_MEMORY);
@@ -XXX,XX +XXX,XX @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
 	unsigned long pfn, managed_pages, system_ram_pages = 0;
 	const int node = zone_to_nid(zone);
 	unsigned long flags;
-	struct memory_notify arg;
+	struct memory_notify mem_arg;
+	struct node_notify node_arg;
+	bool cancel_mem_notifier_on_err = false, cancel_node_notifier_on_err = false;
 	char *reason;
 	int ret;
 
@@ -XXX,XX +XXX,XX @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
 		goto failed_removal_pcplists_disabled;
 	}
 
-	arg.start_pfn = start_pfn;
-	arg.nr_pages = nr_pages;
-	node_states_check_changes_offline(nr_pages, zone, &arg);
+	mem_arg.start_pfn = start_pfn;
+	mem_arg.nr_pages = nr_pages;
+	node_states_check_changes_offline(nr_pages, zone, &node_arg);
+
+	if (node_arg.status_change_nid >= 0) {
+		cancel_node_notifier_on_err = true;
+		ret = node_notify(NODE_BECOMING_MEMORYLESS, &node_arg);
+		ret = notifier_to_errno(ret);
+		if (ret)
+			goto failed_removal_isolated;
+	}
 
-	ret = memory_notify(MEM_GOING_OFFLINE, &arg);
+	cancel_mem_notifier_on_err = true;
+	mem_arg.status_change_nid = node_arg.status_change_nid;
+	mem_arg.status_change_nid_normal = node_arg.status_change_nid_normal;
+	ret = memory_notify(MEM_GOING_OFFLINE, &mem_arg);
 	ret = notifier_to_errno(ret);
 	if (ret) {
 		reason = "notifier failure";
@@ -XXX,XX +XXX,XX @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
 	 * Make sure to mark the node as memory-less before rebuilding the zone
 	 * list. Otherwise this node would still appear in the fallback lists.
 	 */
-	node_states_clear_node(node, &arg);
+	node_states_clear_node(node, &node_arg);
 	if (!populated_zone(zone)) {
 		zone_pcp_reset(zone);
 		build_all_zonelists(NULL);
 	}
 
-	if (arg.status_change_nid >= 0) {
+	if (node_arg.status_change_nid >= 0) {
 		kcompactd_stop(node);
 		kswapd_stop(node);
+		/*Node went memoryless. Notifiy interested consumers */
+		node_notify(NODE_BECAME_MEMORYLESS, &node_arg);
 	}
 
 	writeback_set_ratelimit();
 
-	memory_notify(MEM_OFFLINE, &arg);
+	memory_notify(MEM_OFFLINE, &mem_arg);
 	remove_pfn_range_from_zone(zone, start_pfn, nr_pages);
 	return 0;
 
 failed_removal_isolated:
 	/* pushback to free area */
 	undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
-	memory_notify(MEM_CANCEL_OFFLINE, &arg);
+	if (cancel_node_notifier_on_err)
+		node_notify(NODE_CANCEL_MEMORYLESS, &node_arg);
+	if (cancel_mem_notifier_on_err)
+		memory_notify(MEM_CANCEL_OFFLINE, &mem_arg);
 failed_removal_pcplists_disabled:
 	lru_cache_enable();
 	zone_pcp_enable(zone);
diff --git a/mm/slub.c b/mm/slub.c
index XXXXXXX..XXXXXXX 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -XXX,XX +XXX,XX @@ static int slab_mem_going_offline_callback(void *arg)
 
 static void slab_mem_offline_callback(void *arg)
 {
-	struct memory_notify *marg = arg;
+	struct node_notify *narg = arg;
 	int offline_node;
 
-	offline_node = marg->status_change_nid_normal;
+	offline_node = narg->status_change_nid_normal;
 
 	/*
 	 * If the node still has available memory. we need kmem_cache_node
@@ -XXX,XX +XXX,XX @@ static int slab_mem_going_online_callback(void *arg)
 {
 	struct kmem_cache_node *n;
 	struct kmem_cache *s;
-	struct memory_notify *marg = arg;
-	int nid = marg->status_change_nid_normal;
+	struct node_notify *narg = arg;
+	int nid = narg->status_change_nid_normal;
 	int ret = 0;
 
 	/*
@@ -XXX,XX +XXX,XX @@ static int slab_memory_callback(struct notifier_block *self,
 	int ret = 0;
 
 	switch (action) {
-	case MEM_GOING_ONLINE:
+	case NODE_BECOMING_MEM_AWARE:
 		ret = slab_mem_going_online_callback(arg);
 		break;
-	case MEM_GOING_OFFLINE:
+	case NODE_BECOMING_MEMORYLESS:
 		ret = slab_mem_going_offline_callback(arg);
 		break;
-	case MEM_OFFLINE:
-	case MEM_CANCEL_ONLINE:
+	case NODE_BECAME_MEMORYLESS:
+	case NODE_CANCEL_MEM_AWARE:
 		slab_mem_offline_callback(arg);
 		break;
-	case MEM_ONLINE:
-	case MEM_CANCEL_OFFLINE:
+	case NODE_BECAME_MEM_AWARE:
+	case NODE_CANCEL_MEMORYLESS:
 		break;
 	}
 	if (ret)
@@ -XXX,XX +XXX,XX @@ void __init kmem_cache_init(void)
 			sizeof(struct kmem_cache_node),
 			SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0);
 
-	hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
+	hotplug_node_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
 
 	/* Able to allocate the per node structures */
 	slab_state = PARTIAL;
-- 
2.49.0

memory notify consumers are only interested in which node the memory we
are adding belongs to, so replace current status_change_nid{_normal} fields
with only one that specifies the node.

Signed-off-by: Oscar Salvador <osalvador@suse.de>
---
 include/linux/memory.h |  3 +--
 mm/memory_hotplug.c    |  6 ++----
 mm/page_ext.c          | 12 +-----------
 3 files changed, 4 insertions(+), 17 deletions(-)

diff --git a/include/linux/memory.h b/include/linux/memory.h
index XXXXXXX..XXXXXXX 100644
--- a/include/linux/memory.h
+++ b/include/linux/memory.h
@@ -XXX,XX +XXX,XX @@ struct memory_notify {
 	unsigned long altmap_nr_pages;
 	unsigned long start_pfn;
 	unsigned long nr_pages;
-	int status_change_nid_normal;
-	int status_change_nid;
+	int nid;
 };
 
 struct node_notify {
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index XXXXXXX..XXXXXXX 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -XXX,XX +XXX,XX @@ int online_pages(unsigned long pfn, unsigned long nr_pages,
 
 	mem_arg.start_pfn = pfn;
 	mem_arg.nr_pages = nr_pages;
+	mem_arg.nid = nid;
 	node_states_check_changes_online(nr_pages, zone, &node_arg);
 
 	if (node_arg.status_change_nid >= 0) {
@@ -XXX,XX +XXX,XX @@ int online_pages(unsigned long pfn, unsigned long nr_pages,
 	}
 
 	cancel_mem_notifier_on_err = true;
-	mem_arg.status_change_nid = node_arg.status_change_nid;
-	mem_arg.status_change_nid_normal = node_arg.status_change_nid_normal;
 	ret = memory_notify(MEM_GOING_ONLINE, &mem_arg);
 	ret = notifier_to_errno(ret);
 	if (ret)
@@ -XXX,XX +XXX,XX @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
 
 	mem_arg.start_pfn = start_pfn;
 	mem_arg.nr_pages = nr_pages;
+	mem_arg.nid = node;
 	node_states_check_changes_offline(nr_pages, zone, &node_arg);
 
 	if (node_arg.status_change_nid >= 0) {
@@ -XXX,XX +XXX,XX @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
 	}
 
 	cancel_mem_notifier_on_err = true;
-	mem_arg.status_change_nid = node_arg.status_change_nid;
-	mem_arg.status_change_nid_normal = node_arg.status_change_nid_normal;
 	ret = memory_notify(MEM_GOING_OFFLINE, &mem_arg);
 	ret = notifier_to_errno(ret);
 	if (ret) {
diff --git a/mm/page_ext.c b/mm/page_ext.c
index XXXXXXX..XXXXXXX 100644
--- a/mm/page_ext.c
+++ b/mm/page_ext.c
@@ -XXX,XX +XXX,XX @@ static int __meminit online_page_ext(unsigned long start_pfn,
 	start = SECTION_ALIGN_DOWN(start_pfn);
 	end = SECTION_ALIGN_UP(start_pfn + nr_pages);
 
-	if (nid == NUMA_NO_NODE) {
-		/*
-		 * In this case, "nid" already exists and contains valid memory.
-		 * "start_pfn" passed to us is a pfn which is an arg for
-		 * online__pages(), and start_pfn should exist.
-		 */
-		nid = pfn_to_nid(start_pfn);
-		VM_BUG_ON(!node_online(nid));
-	}
-
 	for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION)
 		fail = init_section_page_ext(pfn, nid);
 	if (!fail)
@@ -XXX,XX +XXX,XX @@ static int __meminit page_ext_callback(struct notifier_block *self,
 	switch (action) {
 	case MEM_GOING_ONLINE:
 		ret = online_page_ext(mn->start_pfn,
-				   mn->nr_pages, mn->status_change_nid);
+				   mn->nr_pages, mn->nid);
 		break;
 	case MEM_OFFLINE:
 		offline_page_ext(mn->start_pfn,
-- 
2.49.0

v1 -> v2:
  - Remove status_change_nid_normal and the code that
    deals with it (David & Vlastimil)
  - Remove slab_mem_offline_callback (David & Vlastimil)
  - Change the order of canceling the notifiers
    in {online,offline}_pages (Vlastimil)
  - Fix up a couple of whitespaces (Jonathan Cameron)
  - Add RBs-by

Memory notifier is a tool that allow consumers to get notified whenever
memory gets onlined or offlined in the system.
Currently, there are 10 consumers of that, but 5 out of those 10 consumers
are only interested in getting notifications when a numa node changes its
memory state.
That means going from memoryless to memory-aware of vice versa.

Which means that for every {online,offline}_pages operation they get
notified even though the numa node might not have changed its state.

While we are doing this, remove status_change_nid_normal, as the only
current user (slub) does not really need it.
This allows us to further simplify and clean up the code.

The first patch gets rid of status_change_nid_normal in slub.
The second patch implements a numa node notifier that does just that, and have
those consumers register in there, so they get notified only when they are
interested.

The third patch replaces 'status_change_nid{_normal}' fields within
memory_notify with a 'nid', as that is only what we need for memory
notifer and update the only user of it (page_ext).

Consumers that are only interested in numa node states change are:

- memory-tier
 - slub
 - cpuset
 - hmat
 - cxl

Oscar Salvador (3):
  mm,slub: Do not special case N_NORMAL nodes for slab_nodes
  mm,memory_hotplug: Implement numa node notifier
  mm,memory_hotplug: Rename status_change_nid parameter in memory_notify

-- 
2.49.0

Currently, slab_mem_going_going_callback() checks whether the node has
N_NORMAL memory in order to be set in slab_nodes.
While it is true that gettind rid of that enforcing would mean
ending up with movables nodes in slab_nodes, the memory waste that comes
with that is negligible.

So stop checking for status_change_nid_normal and just use status_change_nid
instead which works for both types of memory.

Also, once we allocate the kmem_cache_node cache  for the node in
slab_mem_online_callback(), we never deallocate it in
slab_mem_off_callback() when the node goes memoryless, so we can just
get rid of it.

The only side effect is that we will stop clearing the node from slab_nodes.

Signed-off-by: Oscar Salvador <osalvador@suse.de>
---
 mm/slub.c | 30 +-----------------------------
 1 file changed, 1 insertion(+), 29 deletions(-)

diff --git a/mm/slub.c b/mm/slub.c
index XXXXXXX..XXXXXXX 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -XXX,XX +XXX,XX @@ static int slab_mem_going_offline_callback(void *arg)
 	return 0;
 }
 
-static void slab_mem_offline_callback(void *arg)
-{
-	struct memory_notify *marg = arg;
-	int offline_node;
-
-	offline_node = marg->status_change_nid_normal;
-
-	/*
-	 * If the node still has available memory. we need kmem_cache_node
-	 * for it yet.
-	 */
-	if (offline_node < 0)
-		return;
-
-	mutex_lock(&slab_mutex);
-	node_clear(offline_node, slab_nodes);
-	/*
-	 * We no longer free kmem_cache_node structures here, as it would be
-	 * racy with all get_node() users, and infeasible to protect them with
-	 * slab_mutex.
-	 */
-	mutex_unlock(&slab_mutex);
-}
-
 static int slab_mem_going_online_callback(void *arg)
 {
 	struct kmem_cache_node *n;
 	struct kmem_cache *s;
 	struct memory_notify *marg = arg;
-	int nid = marg->status_change_nid_normal;
+	int nid = marg->status_change_nid;
 	int ret = 0;
 
 	/*
@@ -XXX,XX +XXX,XX @@ static int slab_memory_callback(struct notifier_block *self,
 	case MEM_GOING_OFFLINE:
 		ret = slab_mem_going_offline_callback(arg);
 		break;
-	case MEM_OFFLINE:
-	case MEM_CANCEL_ONLINE:
-		slab_mem_offline_callback(arg);
-		break;
 	case MEM_ONLINE:
 	case MEM_CANCEL_OFFLINE:
 		break;
-- 
2.49.0

There are at least five consumers of hotplug_memory_notifier that what they
really are interested in is whether any numa node changed its state, e.g: going
from being memory aware to becoming memoryless and vice versa.

Implement a specific notifier for numa nodes when their state gets changed,
and have those consumers that only care about numa node state changes use it.

Signed-off-by: Oscar Salvador <osalvador@suse.de>
Reviewed-by: Harry Yoo <harry.yoo@oracle.com>
Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
---
 drivers/acpi/numa/hmat.c  |   6 +-
 drivers/base/node.c       |  19 +++++++
 drivers/cxl/core/region.c |  14 ++---
 drivers/cxl/cxl.h         |   4 +-
 include/linux/memory.h    |  38 ++++++++++++-
 kernel/cgroup/cpuset.c    |   2 +-
 mm/memory-tiers.c         |   8 +--
 mm/memory_hotplug.c       | 117 +++++++++++++++++++++-----------------
 mm/slub.c                 |  13 ++---
 9 files changed, 144 insertions(+), 77 deletions(-)

diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c
index XXXXXXX..XXXXXXX 100644
--- a/drivers/acpi/numa/hmat.c
+++ b/drivers/acpi/numa/hmat.c
@@ -XXX,XX +XXX,XX @@ static int hmat_callback(struct notifier_block *self,
 			 unsigned long action, void *arg)
 {
 	struct memory_target *target;
-	struct memory_notify *mnb = arg;
+	struct node_notify *mnb = arg;
 	int pxm, nid = mnb->status_change_nid;
 
-	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
+	if (nid == NUMA_NO_NODE || action != NODE_BECAME_MEM_AWARE)
 		return NOTIFY_OK;
 
 	pxm = node_to_pxm(nid);
@@ -XXX,XX +XXX,XX @@ static __init int hmat_init(void)
 	hmat_register_targets();
 
 	/* Keep the table and structures if the notifier may use them */
-	if (hotplug_memory_notifier(hmat_callback, HMAT_CALLBACK_PRI))
+	if (hotplug_node_notifier(hmat_callback, HMAT_CALLBACK_PRI))
 		goto out_put;
 
 	if (!hmat_set_default_dram_perf())
diff --git a/drivers/base/node.c b/drivers/base/node.c
index XXXXXXX..XXXXXXX 100644
--- a/drivers/base/node.c
+++ b/drivers/base/node.c
@@ -XXX,XX +XXX,XX @@ static const struct attribute_group *node_access_node_groups[] = {
 	NULL,
 };
 
+static BLOCKING_NOTIFIER_HEAD(node_chain);
+
+int register_node_notifier(struct notifier_block *nb)
+{
+	return blocking_notifier_chain_register(&node_chain, nb);
+}
+EXPORT_SYMBOL(register_node_notifier);
+
+void unregister_node_notifier(struct notifier_block *nb)
+{
+	blocking_notifier_chain_unregister(&node_chain, nb);
+}
+EXPORT_SYMBOL(unregister_node_notifier);
+
+int node_notify(unsigned long val, void *v)
+{
+	return blocking_notifier_call_chain(&node_chain, val, v);
+}
+
 static void node_remove_accesses(struct node *node)
 {
 	struct node_access_nodes *c, *cnext;
diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
index XXXXXXX..XXXXXXX 100644
--- a/drivers/cxl/core/region.c
+++ b/drivers/cxl/core/region.c
@@ -XXX,XX +XXX,XX @@ static int cxl_region_perf_attrs_callback(struct notifier_block *nb,
 					  unsigned long action, void *arg)
 {
 	struct cxl_region *cxlr = container_of(nb, struct cxl_region,
-					       memory_notifier);
-	struct memory_notify *mnb = arg;
+					       node_notifier);
+	struct node_notify *mnb = arg;
 	int nid = mnb->status_change_nid;
 	int region_nid;
 
-	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
+	if (nid == NUMA_NO_NODE || action != NODE_BECAME_MEM_AWARE)
 		return NOTIFY_DONE;
 
 	/*
@@ -XXX,XX +XXX,XX @@ static void shutdown_notifiers(void *_cxlr)
 {
 	struct cxl_region *cxlr = _cxlr;
 
-	unregister_memory_notifier(&cxlr->memory_notifier);
+	unregister_node_notifier(&cxlr->node_notifier);
 	unregister_mt_adistance_algorithm(&cxlr->adist_notifier);
 }
 
@@ -XXX,XX +XXX,XX @@ static int cxl_region_probe(struct device *dev)
 	if (rc)
 		return rc;
 
-	cxlr->memory_notifier.notifier_call = cxl_region_perf_attrs_callback;
-	cxlr->memory_notifier.priority = CXL_CALLBACK_PRI;
-	register_memory_notifier(&cxlr->memory_notifier);
+	cxlr->node_notifier.notifier_call = cxl_region_perf_attrs_callback;
+	cxlr->node_notifier.priority = CXL_CALLBACK_PRI;
+	register_node_notifier(&cxlr->node_notifier);
 
 	cxlr->adist_notifier.notifier_call = cxl_region_calculate_adistance;
 	cxlr->adist_notifier.priority = 100;
diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index XXXXXXX..XXXXXXX 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -XXX,XX +XXX,XX @@ struct cxl_region_params {
  * @flags: Region state flags
  * @params: active + config params for the region
  * @coord: QoS access coordinates for the region
- * @memory_notifier: notifier for setting the access coordinates to node
+ * @node_notifier: notifier for setting the access coordinates to node
  * @adist_notifier: notifier for calculating the abstract distance of node
  */
 struct cxl_region {
@@ -XXX,XX +XXX,XX @@ struct cxl_region {
 	unsigned long flags;
 	struct cxl_region_params params;
 	struct access_coordinate coord[ACCESS_COORDINATE_MAX];
-	struct notifier_block memory_notifier;
+	struct notifier_block node_notifier;
 	struct notifier_block adist_notifier;
 };
 
diff --git a/include/linux/memory.h b/include/linux/memory.h
index XXXXXXX..XXXXXXX 100644
--- a/include/linux/memory.h
+++ b/include/linux/memory.h
@@ -XXX,XX +XXX,XX @@ int set_memory_block_size_order(unsigned int order);
 #define	MEM_PREPARE_ONLINE	(1<<6)
 #define	MEM_FINISH_OFFLINE	(1<<7)
 
+/* These states are used for numa node notifiers */
+#define NODE_BECOMING_MEM_AWARE		(1<<0)
+#define NODE_BECAME_MEM_AWARE		(1<<1)
+#define NODE_BECOMING_MEMORYLESS	(1<<2)
+#define NODE_BECAME_MEMORYLESS		(1<<3)
+#define NODE_CANCEL_MEM_AWARE		(1<<4)
+#define NODE_CANCEL_MEMORYLESS		(1<<5)
+
 struct memory_notify {
 	/*
 	 * The altmap_start_pfn and altmap_nr_pages fields are designated for
@@ -XXX,XX +XXX,XX @@ struct memory_notify {
 	unsigned long altmap_nr_pages;
 	unsigned long start_pfn;
 	unsigned long nr_pages;
-	int status_change_nid_normal;
+	int status_change_nid;
+};
+
+struct node_notify {
 	int status_change_nid;
 };
 
@@ -XXX,XX +XXX,XX @@ static inline int hotplug_memory_notifier(notifier_fn_t fn, int pri)
 {
 	return 0;
 }
+
+static inline int register_node_notifier(struct notifier_block *nb)
+{
+	return 0;
+}
+static inline void unregister_node_notifier(struct notifier_block *nb)
+{
+}
+static inline int node_notify(unsigned long val, void *v)
+{
+	return 0;
+}
+static inline int hotplug_node_notifier(notifier_fn_t fn, int pri)
+{
+	return 0;
+}
 #else /* CONFIG_MEMORY_HOTPLUG */
 extern int register_memory_notifier(struct notifier_block *nb);
+extern int register_node_notifier(struct notifier_block *nb);
 extern void unregister_memory_notifier(struct notifier_block *nb);
+extern void unregister_node_notifier(struct notifier_block *nb);
 int create_memory_block_devices(unsigned long start, unsigned long size,
 				struct vmem_altmap *altmap,
 				struct memory_group *group);
 void remove_memory_block_devices(unsigned long start, unsigned long size);
 extern void memory_dev_init(void);
 extern int memory_notify(unsigned long val, void *v);
+extern int node_notify(unsigned long val, void *v);
 extern struct memory_block *find_memory_block(unsigned long section_nr);
 typedef int (*walk_memory_blocks_func_t)(struct memory_block *, void *);
 extern int walk_memory_blocks(unsigned long start, unsigned long size,
@@ -XXX,XX +XXX,XX @@ int walk_dynamic_memory_groups(int nid, walk_memory_groups_func_t func,
 	register_memory_notifier(&fn##_mem_nb);			\
 })
 
+#define hotplug_node_notifier(fn, pri) ({		\
+	static __meminitdata struct notifier_block fn##_node_nb =\
+		{ .notifier_call = fn, .priority = pri };\
+	register_node_notifier(&fn##_node_nb);			\
+})
+
 #ifdef CONFIG_NUMA
 void memory_block_add_nid(struct memory_block *mem, int nid,
 			  enum meminit_context context);
diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
index XXXXXXX..XXXXXXX 100644
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -XXX,XX +XXX,XX @@ void __init cpuset_init_smp(void)
 	cpumask_copy(top_cpuset.effective_cpus, cpu_active_mask);
 	top_cpuset.effective_mems = node_states[N_MEMORY];
 
-	hotplug_memory_notifier(cpuset_track_online_nodes, CPUSET_CALLBACK_PRI);
+	hotplug_node_notifier(cpuset_track_online_nodes, CPUSET_CALLBACK_PRI);
 
 	cpuset_migrate_mm_wq = alloc_ordered_workqueue("cpuset_migrate_mm", 0);
 	BUG_ON(!cpuset_migrate_mm_wq);
diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c
index XXXXXXX..XXXXXXX 100644
--- a/mm/memory-tiers.c
+++ b/mm/memory-tiers.c
@@ -XXX,XX +XXX,XX @@ static int __meminit memtier_hotplug_callback(struct notifier_block *self,
 					      unsigned long action, void *_arg)
 {
 	struct memory_tier *memtier;
-	struct memory_notify *arg = _arg;
+	struct node_notify *arg = _arg;
 
 	/*
 	 * Only update the node migration order when a node is
@@ -XXX,XX +XXX,XX @@ static int __meminit memtier_hotplug_callback(struct notifier_block *self,
 		return notifier_from_errno(0);
 
 	switch (action) {
-	case MEM_OFFLINE:
+	case NODE_BECAME_MEMORYLESS:
 		mutex_lock(&memory_tier_lock);
 		if (clear_node_memory_tier(arg->status_change_nid))
 			establish_demotion_targets();
 		mutex_unlock(&memory_tier_lock);
 		break;
-	case MEM_ONLINE:
+	case NODE_BECAME_MEM_AWARE:
 		mutex_lock(&memory_tier_lock);
 		memtier = set_node_memory_tier(arg->status_change_nid);
 		if (!IS_ERR(memtier))
@@ -XXX,XX +XXX,XX @@ static int __init memory_tier_init(void)
 	nodes_and(default_dram_nodes, node_states[N_MEMORY],
 		  node_states[N_CPU]);
 
-	hotplug_memory_notifier(memtier_hotplug_callback, MEMTIER_HOTPLUG_PRI);
+	hotplug_node_notifier(memtier_hotplug_callback, MEMTIER_HOTPLUG_PRI);
 	return 0;
 }
 subsys_initcall(memory_tier_init);
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index XXXXXXX..XXXXXXX 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -XXX,XX +XXX,XX @@ static void online_pages_range(unsigned long start_pfn, unsigned long nr_pages)
 
 /* check which state of node_states will be changed when online memory */
 static void node_states_check_changes_online(unsigned long nr_pages,
-	struct zone *zone, struct memory_notify *arg)
+	struct zone *zone, struct node_notify *arg)
 {
 	int nid = zone_to_nid(zone);
 
 	arg->status_change_nid = NUMA_NO_NODE;
-	arg->status_change_nid_normal = NUMA_NO_NODE;
 
 	if (!node_state(nid, N_MEMORY))
 		arg->status_change_nid = nid;
-	if (zone_idx(zone) <= ZONE_NORMAL && !node_state(nid, N_NORMAL_MEMORY))
-		arg->status_change_nid_normal = nid;
 }
 
-static void node_states_set_node(int node, struct memory_notify *arg)
+static void node_states_set_node(int node, struct node_notify *arg)
 {
-	if (arg->status_change_nid_normal >= 0)
-		node_set_state(node, N_NORMAL_MEMORY);
-
 	if (arg->status_change_nid >= 0)
 		node_set_state(node, N_MEMORY);
 }
@@ -XXX,XX +XXX,XX @@ int online_pages(unsigned long pfn, unsigned long nr_pages,
 	int need_zonelists_rebuild = 0;
 	const int nid = zone_to_nid(zone);
 	int ret;
-	struct memory_notify arg;
+	struct memory_notify mem_arg;
+	struct node_notify node_arg;
+	bool cancel_mem_notifier_on_err = false, cancel_node_notifier_on_err = false;
 
 	/*
 	 * {on,off}lining is constrained to full memory sections (or more
@@ -XXX,XX +XXX,XX @@ int online_pages(unsigned long pfn, unsigned long nr_pages,
 	/* associate pfn range with the zone */
 	move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_ISOLATE);
 
-	arg.start_pfn = pfn;
-	arg.nr_pages = nr_pages;
-	node_states_check_changes_online(nr_pages, zone, &arg);
+	mem_arg.start_pfn = pfn;
+	mem_arg.nr_pages = nr_pages;
+	node_states_check_changes_online(nr_pages, zone, &node_arg);
+
+	if (node_arg.status_change_nid >= 0) {
+		/* Node is becoming memory aware. Notify consumers */
+		cancel_node_notifier_on_err = true;
+		ret = node_notify(NODE_BECOMING_MEM_AWARE, &node_arg);
+		ret = notifier_to_errno(ret);
+		if (ret)
+			goto failed_addition;
+	}
 
-	ret = memory_notify(MEM_GOING_ONLINE, &arg);
+	cancel_mem_notifier_on_err = true;
+	mem_arg.status_change_nid = node_arg.status_change_nid;
+	ret = memory_notify(MEM_GOING_ONLINE, &mem_arg);
 	ret = notifier_to_errno(ret);
 	if (ret)
 		goto failed_addition;
@@ -XXX,XX +XXX,XX @@ int online_pages(unsigned long pfn, unsigned long nr_pages,
 	online_pages_range(pfn, nr_pages);
 	adjust_present_page_count(pfn_to_page(pfn), group, nr_pages);
 
-	node_states_set_node(nid, &arg);
+	node_states_set_node(nid, &node_arg);
 	if (need_zonelists_rebuild)
 		build_all_zonelists(NULL);
 
@@ -XXX,XX +XXX,XX @@ int online_pages(unsigned long pfn, unsigned long nr_pages,
 	kswapd_run(nid);
 	kcompactd_run(nid);
 
+	if (node_arg.status_change_nid >= 0)
+		/*
+		 * Node went from memoryless to have memory. Notifiy interested
+		 * consumers
+		 */
+		node_notify(NODE_BECAME_MEM_AWARE, &node_arg);
+
 	writeback_set_ratelimit();
 
-	memory_notify(MEM_ONLINE, &arg);
+	memory_notify(MEM_ONLINE, &mem_arg);
 	return 0;
 
 failed_addition:
 	pr_debug("online_pages [mem %#010llx-%#010llx] failed\n",
 		 (unsigned long long) pfn << PAGE_SHIFT,
 		 (((unsigned long long) pfn + nr_pages) << PAGE_SHIFT) - 1);
-	memory_notify(MEM_CANCEL_ONLINE, &arg);
+	if (cancel_mem_notifier_on_err)
+		memory_notify(MEM_CANCEL_ONLINE, &mem_arg);
+	if (cancel_node_notifier_on_err)
+		node_notify(NODE_CANCEL_MEM_AWARE, &node_arg);
 	remove_pfn_range_from_zone(zone, pfn, nr_pages);
 	return ret;
 }
@@ -XXX,XX +XXX,XX @@ early_param("movable_node", cmdline_parse_movable_node);
 
 /* check which state of node_states will be changed when offline memory */
 static void node_states_check_changes_offline(unsigned long nr_pages,
-		struct zone *zone, struct memory_notify *arg)
+		struct zone *zone, struct node_notify *arg)
 {
 	struct pglist_data *pgdat = zone->zone_pgdat;
 	unsigned long present_pages = 0;
 	enum zone_type zt;
 
 	arg->status_change_nid = NUMA_NO_NODE;
-	arg->status_change_nid_normal = NUMA_NO_NODE;
 
 	/*
-	 * Check whether node_states[N_NORMAL_MEMORY] will be changed.
-	 * If the memory to be offline is within the range
-	 * [0..ZONE_NORMAL], and it is the last present memory there,
-	 * the zones in that range will become empty after the offlining,
-	 * thus we can determine that we need to clear the node from
-	 * node_states[N_NORMAL_MEMORY].
+	 * Here we count the possible pages within the range [0..ZONE_MOVABLE].
+	 * If after having accounted all the pages, we see that the nr_pages to
+	 * be offlined is over or equal to the accounted pages, we know that the
+	 * node will become empty, ans so, we can clear it for N_MEMORY.
 	 */
-	for (zt = 0; zt <= ZONE_NORMAL; zt++)
+	for (zt = 0; zt <= ZONE_MOVABLE; zt++)
 		present_pages += pgdat->node_zones[zt].present_pages;
-	if (zone_idx(zone) <= ZONE_NORMAL && nr_pages >= present_pages)
-		arg->status_change_nid_normal = zone_to_nid(zone);
-
-	/*
-	 * We have accounted the pages from [0..ZONE_NORMAL); ZONE_HIGHMEM
-	 * does not apply as we don't support 32bit.
-	 * Here we count the possible pages from ZONE_MOVABLE.
-	 * If after having accounted all the pages, we see that the nr_pages
-	 * to be offlined is over or equal to the accounted pages,
-	 * we know that the node will become empty, and so, we can clear
-	 * it for N_MEMORY as well.
-	 */
-	present_pages += pgdat->node_zones[ZONE_MOVABLE].present_pages;
 
 	if (nr_pages >= present_pages)
 		arg->status_change_nid = zone_to_nid(zone);
 }
 
-static void node_states_clear_node(int node, struct memory_notify *arg)
+static void node_states_clear_node(int node, struct node_notify *arg)
 {
-	if (arg->status_change_nid_normal >= 0)
-		node_clear_state(node, N_NORMAL_MEMORY);
-
 	if (arg->status_change_nid >= 0)
 		node_clear_state(node, N_MEMORY);
 }
@@ -XXX,XX +XXX,XX @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
 	unsigned long pfn, managed_pages, system_ram_pages = 0;
 	const int node = zone_to_nid(zone);
 	unsigned long flags;
-	struct memory_notify arg;
+	struct memory_notify mem_arg;
+	struct node_notify node_arg;
+	bool cancel_mem_notifier_on_err = false, cancel_node_notifier_on_err = false;
 	char *reason;
 	int ret;
 
@@ -XXX,XX +XXX,XX @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
 		goto failed_removal_pcplists_disabled;
 	}
 
-	arg.start_pfn = start_pfn;
-	arg.nr_pages = nr_pages;
-	node_states_check_changes_offline(nr_pages, zone, &arg);
+	mem_arg.start_pfn = start_pfn;
+	mem_arg.nr_pages = nr_pages;
+	node_states_check_changes_offline(nr_pages, zone, &node_arg);
+
+	if (node_arg.status_change_nid >= 0) {
+		cancel_node_notifier_on_err = true;
+		ret = node_notify(NODE_BECOMING_MEMORYLESS, &node_arg);
+		ret = notifier_to_errno(ret);
+		if (ret)
+			goto failed_removal_isolated;
+	}
 
-	ret = memory_notify(MEM_GOING_OFFLINE, &arg);
+	cancel_mem_notifier_on_err = true;
+	mem_arg.status_change_nid = node_arg.status_change_nid;
+	ret = memory_notify(MEM_GOING_OFFLINE, &mem_arg);
 	ret = notifier_to_errno(ret);
 	if (ret) {
 		reason = "notifier failure";
@@ -XXX,XX +XXX,XX @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
 	 * Make sure to mark the node as memory-less before rebuilding the zone
 	 * list. Otherwise this node would still appear in the fallback lists.
 	 */
-	node_states_clear_node(node, &arg);
+	node_states_clear_node(node, &node_arg);
 	if (!populated_zone(zone)) {
 		zone_pcp_reset(zone);
 		build_all_zonelists(NULL);
 	}
 
-	if (arg.status_change_nid >= 0) {
+	if (node_arg.status_change_nid >= 0) {
 		kcompactd_stop(node);
 		kswapd_stop(node);
+		/* Node went memoryless. Notifiy interested consumers */
+		node_notify(NODE_BECAME_MEMORYLESS, &node_arg);
 	}
 
 	writeback_set_ratelimit();
 
-	memory_notify(MEM_OFFLINE, &arg);
+	memory_notify(MEM_OFFLINE, &mem_arg);
 	remove_pfn_range_from_zone(zone, start_pfn, nr_pages);
 	return 0;
 
 failed_removal_isolated:
 	/* pushback to free area */
 	undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
-	memory_notify(MEM_CANCEL_OFFLINE, &arg);
+	if (cancel_mem_notifier_on_err)
+		memory_notify(MEM_CANCEL_OFFLINE, &mem_arg);
+	if (cancel_node_notifier_on_err)
+		node_notify(NODE_CANCEL_MEMORYLESS, &node_arg);
 failed_removal_pcplists_disabled:
 	lru_cache_enable();
 	zone_pcp_enable(zone);
diff --git a/mm/slub.c b/mm/slub.c
index XXXXXXX..XXXXXXX 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -XXX,XX +XXX,XX @@ static int slab_mem_going_online_callback(void *arg)
 {
 	struct kmem_cache_node *n;
 	struct kmem_cache *s;
-	struct memory_notify *marg = arg;
-	int nid = marg->status_change_nid;
+	struct node_notify *narg = arg;
+	int nid = narg->status_change_nid;
 	int ret = 0;
 
 	/*
@@ -XXX,XX +XXX,XX @@ static int slab_memory_callback(struct notifier_block *self,
 	int ret = 0;
 
 	switch (action) {
-	case MEM_GOING_ONLINE:
+	case NODE_BECOMING_MEM_AWARE:
 		ret = slab_mem_going_online_callback(arg);
 		break;
-	case MEM_GOING_OFFLINE:
+	case NODE_BECOMING_MEMORYLESS:
 		ret = slab_mem_going_offline_callback(arg);
 		break;
-	case MEM_ONLINE:
-	case MEM_CANCEL_OFFLINE:
-		break;
 	}
 	if (ret)
 		ret = notifier_from_errno(ret);
@@ -XXX,XX +XXX,XX @@ void __init kmem_cache_init(void)
 			sizeof(struct kmem_cache_node),
 			SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0);
 
-	hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
+	hotplug_node_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
 
 	/* Able to allocate the per node structures */
 	slab_state = PARTIAL;
-- 
2.49.0

The 'status_change_nid' field was used to track changes in the memory
state of a numa node, but that funcionality has been decoupled from
memory_notify and moved to node_notify.
Current consumers of memory_notify are only interested in which node the
memory we are adding belongs to, so rename current 'status_change_nid'
to 'nid'.

Signed-off-by: Oscar Salvador <osalvador@suse.de>
Reviewed-by: Harry Yoo <harry.yoo@oracle.com>
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
---
 include/linux/memory.h |  2 +-
 mm/memory_hotplug.c    |  4 ++--
 mm/page_ext.c          | 12 +-----------
 3 files changed, 4 insertions(+), 14 deletions(-)

diff --git a/include/linux/memory.h b/include/linux/memory.h
index XXXXXXX..XXXXXXX 100644
--- a/include/linux/memory.h
+++ b/include/linux/memory.h
@@ -XXX,XX +XXX,XX @@ struct memory_notify {
 	unsigned long altmap_nr_pages;
 	unsigned long start_pfn;
 	unsigned long nr_pages;
-	int status_change_nid;
+	int nid;
 };
 
 struct node_notify {
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index XXXXXXX..XXXXXXX 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -XXX,XX +XXX,XX @@ int online_pages(unsigned long pfn, unsigned long nr_pages,
 
 	mem_arg.start_pfn = pfn;
 	mem_arg.nr_pages = nr_pages;
+	mem_arg.nid = nid;
 	node_states_check_changes_online(nr_pages, zone, &node_arg);
 
 	if (node_arg.status_change_nid >= 0) {
@@ -XXX,XX +XXX,XX @@ int online_pages(unsigned long pfn, unsigned long nr_pages,
 	}
 
 	cancel_mem_notifier_on_err = true;
-	mem_arg.status_change_nid = node_arg.status_change_nid;
 	ret = memory_notify(MEM_GOING_ONLINE, &mem_arg);
 	ret = notifier_to_errno(ret);
 	if (ret)
@@ -XXX,XX +XXX,XX @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
 
 	mem_arg.start_pfn = start_pfn;
 	mem_arg.nr_pages = nr_pages;
+	mem_arg.nid = node;
 	node_states_check_changes_offline(nr_pages, zone, &node_arg);
 
 	if (node_arg.status_change_nid >= 0) {
@@ -XXX,XX +XXX,XX @@ int offline_pages(unsigned long start_pfn, unsigned long nr_pages,
 	}
 
 	cancel_mem_notifier_on_err = true;
-	mem_arg.status_change_nid = node_arg.status_change_nid;
 	ret = memory_notify(MEM_GOING_OFFLINE, &mem_arg);
 	ret = notifier_to_errno(ret);
 	if (ret) {
diff --git a/mm/page_ext.c b/mm/page_ext.c
index XXXXXXX..XXXXXXX 100644
--- a/mm/page_ext.c
+++ b/mm/page_ext.c
@@ -XXX,XX +XXX,XX @@ static int __meminit online_page_ext(unsigned long start_pfn,
 	start = SECTION_ALIGN_DOWN(start_pfn);
 	end = SECTION_ALIGN_UP(start_pfn + nr_pages);
 
-	if (nid == NUMA_NO_NODE) {
-		/*
-		 * In this case, "nid" already exists and contains valid memory.
-		 * "start_pfn" passed to us is a pfn which is an arg for
-		 * online__pages(), and start_pfn should exist.
-		 */
-		nid = pfn_to_nid(start_pfn);
-		VM_BUG_ON(!node_online(nid));
-	}
-
 	for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION)
 		fail = init_section_page_ext(pfn, nid);
 	if (!fail)
@@ -XXX,XX +XXX,XX @@ static int __meminit page_ext_callback(struct notifier_block *self,
 	switch (action) {
 	case MEM_GOING_ONLINE:
 		ret = online_page_ext(mn->start_pfn,
-				   mn->nr_pages, mn->status_change_nid);
+				   mn->nr_pages, mn->nid);
 		break;
 	case MEM_OFFLINE:
 		offline_page_ext(mn->start_pfn,
-- 
2.49.0