"struct sg_lb_stats_prop" is a container around "sg_lb_stats" to help
propagate the load balancing stats up the sched domain hierarchy. Embed
the same in "sched_domain_shared" for concurrent load balancing
instances to reuse the statistics collected for domains below.

Signed-off-by: K Prateek Nayak <kprateek.nayak@amd.com>
---
 include/linux/sched/topology.h |  9 +++++----
 kernel/sched/sched.h           | 11 +++++++++++
 kernel/sched/topology.c        | 26 +++++++++++++++++++++++---
 3 files changed, 39 insertions(+), 7 deletions(-)

diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h
index 7f3dbafe1817..a16d7d9dd9d3 100644
--- a/include/linux/sched/topology.h
+++ b/include/linux/sched/topology.h
@@ -78,10 +78,11 @@ extern int sched_domain_level_max;
 struct sched_group;
 
 struct sched_domain_shared {
-	atomic_t	ref;
-	atomic_t	nr_busy_cpus;
-	int		has_idle_cores;
-	int		nr_idle_scan;
+	atomic_t		ref;
+	atomic_t		nr_busy_cpus;
+	int			has_idle_cores;
+	int			nr_idle_scan;
+	void			*private;	/* lb stats propagation field */
 };
 
 struct sched_domain {
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 9372a75ab3cf..391c4180eeb3 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2170,6 +2170,17 @@ struct sg_lb_stats {
 #endif
 };
 
+/*
+ * sg_lb_stats_prop - Load balancer stats propagation container.
+ * This is embedded in sg->shared->private and is used to propagate
+ * sched_domain load balancing statistics up the hierarchy.
+ */
+struct sg_lb_stats_prop {
+	raw_spinlock_t          stats_lock;	/* Lock for updating the cached stats */
+	unsigned long		last_update;	/* Time when stats was last updated (jiffies) */
+	struct sg_lb_stats	sg_stats;	/* Cached sched_group stats */
+};
+
 static inline struct cpumask *sched_group_span(struct sched_group *sg)
 {
 	return to_cpumask(sg->cpumask);
diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index 508ee8aa492b..aeb55f66e8d6 100644
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -621,10 +621,19 @@ static void link_sg_shared(struct sched_group *sg, struct sched_domain_shared *s
 	atomic_inc(&sds->ref);
 }
 
+static void free_sched_domain_shared(struct sched_domain_shared *sd_shared)
+{
+	if (!sd_shared)
+		return;
+
+	kfree(sd_shared->private);
+	kfree(sd_shared);
+}
+
 static void free_sg_shared(struct sched_group *sg)
 {
 	if (sg->shared && atomic_dec_and_test(&sg->shared->ref))
-		kfree(sg->shared);
+		free_sched_domain_shared(sg->shared);
 
 	sg->shared = NULL;
 }
@@ -661,7 +670,7 @@ static void destroy_sched_domain(struct sched_domain *sd)
 	free_sched_groups(sd->groups, 1);
 
 	if (sd->shared && atomic_dec_and_test(&sd->shared->ref))
-		kfree(sd->shared);
+		free_sched_domain_shared(sd->shared);
 	kfree(sd);
 }
 
@@ -2273,6 +2282,7 @@ static int __sdt_alloc(const struct cpumask *cpu_map)
 			struct sched_domain_shared *sds;
 			struct sched_group *sg;
 			struct sched_group_capacity *sgc;
+			struct sg_lb_stats_prop *sg_stats;
 
 			sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(),
 					GFP_KERNEL, cpu_to_node(j));
@@ -2288,6 +2298,16 @@ static int __sdt_alloc(const struct cpumask *cpu_map)
 
 			*per_cpu_ptr(sdd->sds, j) = sds;
 
+			sg_stats = kzalloc_node(sizeof(struct sg_lb_stats_prop),
+					GFP_KERNEL, cpu_to_node(j));
+
+			if (!sg_stats)
+				return -ENOMEM;
+
+			raw_spin_lock_init(&sg_stats->stats_lock);
+			sg_stats->last_update = 0;
+			sds->private = (void *)sg_stats;
+
 			sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
 					GFP_KERNEL, cpu_to_node(j));
 			if (!sg)
@@ -2332,7 +2352,7 @@ static void __sdt_free(const struct cpumask *cpu_map)
 			}
 
 			if (sdd->sds)
-				kfree(*per_cpu_ptr(sdd->sds, j));
+				free_sched_domain_shared(*per_cpu_ptr(sdd->sds, j));
 			if (sdd->sg)
 				kfree(*per_cpu_ptr(sdd->sg, j));
 			if (sdd->sgc)
-- 
2.43.0