From: Tim Chen <tim.c.chen@linux.intel.com>
Prepare for indexing arrays that track in each run queue: the number
of tasks preferring current LLC and each of the other LLC.
The reason to introduce LLC index is because the per LLC-scope data
is needed to do cache aware load balancing. However, the native lld_id
is usually the first CPU of that LLC domain, which is not continuous,
which might waste the space if the per LLC-scope data is stored
in an array (in current implementation).
In the future, this LLC index could be removed after
the native llc_id is used as the key to search into xarray based
array.
Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
---
include/linux/sched.h | 3 +++
kernel/sched/fair.c | 12 ++++++++++++
kernel/sched/sched.h | 2 ++
kernel/sched/topology.c | 29 +++++++++++++++++++++++++++++
4 files changed, 46 insertions(+)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 02ff8b8be25b..81d92e8097f5 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -809,6 +809,9 @@ struct kmap_ctrl {
#endif
};
+/* XXX need fix to not use magic number */
+#define MAX_LLC 64
+
struct task_struct {
#ifdef CONFIG_THREAD_INFO_IN_TASK
/*
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 3128dbcf0a36..f5075d287c51 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1183,6 +1183,18 @@ static int llc_id(int cpu)
return per_cpu(sd_llc_id, cpu);
}
+/*
+ * continuous index.
+ * TBD: replace by xarray with key llc_id()
+ */
+static inline int llc_idx(int cpu)
+{
+ if (cpu < 0)
+ return -1;
+
+ return per_cpu(sd_llc_idx, cpu);
+}
+
void mm_init_sched(struct mm_struct *mm, struct mm_sched __percpu *_pcpu_sched)
{
unsigned long epoch;
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 83552aab74fb..c37c74dfce25 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2056,6 +2056,7 @@ static inline struct sched_domain *lowest_flag_domain(int cpu, int flag)
DECLARE_PER_CPU(struct sched_domain __rcu *, sd_llc);
DECLARE_PER_CPU(int, sd_llc_size);
DECLARE_PER_CPU(int, sd_llc_id);
+DECLARE_PER_CPU(int, sd_llc_idx);
DECLARE_PER_CPU(int, sd_share_id);
DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared);
DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa);
@@ -2064,6 +2065,7 @@ DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);
extern struct static_key_false sched_asym_cpucapacity;
extern struct static_key_false sched_cluster_active;
+extern int max_llcs;
static __always_inline bool sched_asym_cpucap_active(void)
{
diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index b958fe48e020..91a2b7f65fee 100644
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -657,6 +657,7 @@ static void destroy_sched_domains(struct sched_domain *sd)
DEFINE_PER_CPU(struct sched_domain __rcu *, sd_llc);
DEFINE_PER_CPU(int, sd_llc_size);
DEFINE_PER_CPU(int, sd_llc_id);
+DEFINE_PER_CPU(int, sd_llc_idx);
DEFINE_PER_CPU(int, sd_share_id);
DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared);
DEFINE_PER_CPU(struct sched_domain __rcu *, sd_numa);
@@ -666,6 +667,25 @@ DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);
DEFINE_STATIC_KEY_FALSE(sched_asym_cpucapacity);
DEFINE_STATIC_KEY_FALSE(sched_cluster_active);
+int max_llcs = -1;
+
+static void update_llc_idx(int cpu)
+{
+#ifdef CONFIG_SCHED_CACHE
+ int idx = -1, llc_id = -1;
+
+ llc_id = per_cpu(sd_llc_id, cpu);
+ idx = per_cpu(sd_llc_idx, llc_id);
+
+ if (idx < 0) {
+ idx = max_llcs++;
+ BUG_ON(idx > MAX_LLC);
+ per_cpu(sd_llc_idx, llc_id) = idx;
+ }
+ per_cpu(sd_llc_idx, cpu) = idx;
+#endif
+}
+
static void update_top_cache_domain(int cpu)
{
struct sched_domain_shared *sds = NULL;
@@ -684,6 +704,7 @@ static void update_top_cache_domain(int cpu)
per_cpu(sd_llc_size, cpu) = size;
per_cpu(sd_llc_id, cpu) = id;
rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds);
+ update_llc_idx(cpu);
sd = lowest_flag_domain(cpu, SD_CLUSTER);
if (sd)
@@ -2456,6 +2477,14 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
bool has_asym = false;
bool has_cluster = false;
+#ifdef CONFIG_SCHED_CACHE
+ if (max_llcs < 0) {
+ for_each_possible_cpu(i)
+ per_cpu(sd_llc_idx, i) = -1;
+ max_llcs = 0;
+ }
+#endif
+
if (WARN_ON(cpumask_empty(cpu_map)))
goto error;
--
2.25.1
Hi Chen Yu, I tested the patch set on AmpereOne CPU with 192 cores. With certain firmware setting, each core has its own L1/L2 cache. But *no* cores share LLC (L3). So *no* schedule domain has flag 'SD_SHARE_LLC'. With this topology: per_cpu(sd_llc_id, cpu) is actually the cpu id (0-191). And kernel bug will be triggered at: 'BUG_ON(idx > MAX_LLC)' Please see details bellow. The bug will disappear if setting 'MAX_LLC' to 192. But I think we might disable CAS(cache aware scheduling) if no domain has 'SD_SHARE_LLC'. On 8/9/2025 1:03 PM, Chen Yu wrote: > From: Tim Chen <tim.c.chen@linux.intel.com> > > Prepare for indexing arrays that track in each run queue: the number > of tasks preferring current LLC and each of the other LLC. > > The reason to introduce LLC index is because the per LLC-scope data > is needed to do cache aware load balancing. However, the native lld_id > is usually the first CPU of that LLC domain, which is not continuous, > which might waste the space if the per LLC-scope data is stored > in an array (in current implementation). > > In the future, this LLC index could be removed after > the native llc_id is used as the key to search into xarray based > array. > > Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com> > --- > include/linux/sched.h | 3 +++ > kernel/sched/fair.c | 12 ++++++++++++ > kernel/sched/sched.h | 2 ++ > kernel/sched/topology.c | 29 +++++++++++++++++++++++++++++ > 4 files changed, 46 insertions(+) > > diff --git a/include/linux/sched.h b/include/linux/sched.h > index 02ff8b8be25b..81d92e8097f5 100644 > --- a/include/linux/sched.h > +++ b/include/linux/sched.h > @@ -809,6 +809,9 @@ struct kmap_ctrl { > #endif > }; > > +/* XXX need fix to not use magic number */ > +#define MAX_LLC 64 The bug will disappear if setting 'MAX_LLC' to 192. > + > struct task_struct { > #ifdef CONFIG_THREAD_INFO_IN_TASK > /* > diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c > index 3128dbcf0a36..f5075d287c51 100644 > --- a/kernel/sched/fair.c > +++ b/kernel/sched/fair.c > @@ -1183,6 +1183,18 @@ static int llc_id(int cpu) > return per_cpu(sd_llc_id, cpu); > } > > +/* > + * continuous index. > + * TBD: replace by xarray with key llc_id() > + */ > +static inline int llc_idx(int cpu) > +{ > + if (cpu < 0) > + return -1; > + > + return per_cpu(sd_llc_idx, cpu); > +} > + > void mm_init_sched(struct mm_struct *mm, struct mm_sched __percpu *_pcpu_sched) > { > unsigned long epoch; > diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h > index 83552aab74fb..c37c74dfce25 100644 > --- a/kernel/sched/sched.h > +++ b/kernel/sched/sched.h > @@ -2056,6 +2056,7 @@ static inline struct sched_domain *lowest_flag_domain(int cpu, int flag) > DECLARE_PER_CPU(struct sched_domain __rcu *, sd_llc); > DECLARE_PER_CPU(int, sd_llc_size); > DECLARE_PER_CPU(int, sd_llc_id); > +DECLARE_PER_CPU(int, sd_llc_idx); > DECLARE_PER_CPU(int, sd_share_id); > DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared); > DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa); > @@ -2064,6 +2065,7 @@ DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity); > > extern struct static_key_false sched_asym_cpucapacity; > extern struct static_key_false sched_cluster_active; > +extern int max_llcs; > > static __always_inline bool sched_asym_cpucap_active(void) > { > diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c > index b958fe48e020..91a2b7f65fee 100644 > --- a/kernel/sched/topology.c > +++ b/kernel/sched/topology.c > @@ -657,6 +657,7 @@ static void destroy_sched_domains(struct sched_domain *sd) > DEFINE_PER_CPU(struct sched_domain __rcu *, sd_llc); > DEFINE_PER_CPU(int, sd_llc_size); > DEFINE_PER_CPU(int, sd_llc_id); > +DEFINE_PER_CPU(int, sd_llc_idx); > DEFINE_PER_CPU(int, sd_share_id); > DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared); > DEFINE_PER_CPU(struct sched_domain __rcu *, sd_numa); > @@ -666,6 +667,25 @@ DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity); > DEFINE_STATIC_KEY_FALSE(sched_asym_cpucapacity); > DEFINE_STATIC_KEY_FALSE(sched_cluster_active); > > +int max_llcs = -1; > + > +static void update_llc_idx(int cpu) > +{ > +#ifdef CONFIG_SCHED_CACHE > + int idx = -1, llc_id = -1; > + > + llc_id = per_cpu(sd_llc_id, cpu); > + idx = per_cpu(sd_llc_idx, llc_id); > + > + if (idx < 0) { > + idx = max_llcs++; > + BUG_ON(idx > MAX_LLC); [ 2.793048] ------------[ cut here ]------------ [ 2.797737] kernel BUG at kernel/sched/topology.c:682! [ 2.802957] Internal error: Oops - BUG: 00000000f2000800 [#1] SMP [snip] [ 2.917450] update_top_cache_domain+0x248/0x260 (P) [ 2.922493] cpu_attach_domain+0x170/0x250 [ 2.926653] build_sched_domains+0x640/0x7b0 [ 2.930989] sched_init_domains+0xbc/0xd8 [ 2.935062] sched_init_smp+0x48/0xd0 [ 2.938778] kernel_init_freeable+0xf4/0x148 [ 2.943115] kernel_init+0x2c/0x150 [ 2.946658] ret_from_fork+0x10/0x20 [ 2.950288] Code: d2800001 17ffffc7 d2800000 17ffffdf (d4210000) [ 2.956481] ---[ end trace 0000000000000000 ]--- [ 2.961170] Kernel panic - not syncing: Oops - BUG: Fatal exception [ 2.967540] SMP: stopping secondary CPUs [ 2.971529] ---[ end Kernel panic - not syncing: Oops - BUG: Fatal exception ]--- > + per_cpu(sd_llc_idx, llc_id) = idx; > + } > + per_cpu(sd_llc_idx, cpu) = idx; > +#endif > +} > + > static void update_top_cache_domain(int cpu) > { > struct sched_domain_shared *sds = NULL; > @@ -684,6 +704,7 @@ static void update_top_cache_domain(int cpu) > per_cpu(sd_llc_size, cpu) = size; > per_cpu(sd_llc_id, cpu) = id; > rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds); > + update_llc_idx(cpu); > > sd = lowest_flag_domain(cpu, SD_CLUSTER); > if (sd) > @@ -2456,6 +2477,14 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att > bool has_asym = false; > bool has_cluster = false; > > +#ifdef CONFIG_SCHED_CACHE > + if (max_llcs < 0) { > + for_each_possible_cpu(i) > + per_cpu(sd_llc_idx, i) = -1; > + max_llcs = 0; > + } > +#endif > + > if (WARN_ON(cpumask_empty(cpu_map))) > goto error; > A draft patch like bellow can fix the kernel BUG: 1) Do not call update_llc_idx() if domain has no SD_SHARE_LLC 2) Disable CAS if domain has no SD_SHARE_LLC diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 8483c02b4d28..cde9b6cdb1de 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -704,7 +704,8 @@ static void update_top_cache_domain(int cpu) per_cpu(sd_llc_size, cpu) = size; per_cpu(sd_llc_id, cpu) = id; rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds); - update_llc_idx(cpu); + if (sd) + update_llc_idx(cpu); sd = lowest_flag_domain(cpu, SD_CLUSTER); if (sd) @@ -2476,6 +2477,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att int i, ret = -ENOMEM; bool has_asym = false; bool has_cluster = false; + bool has_llc = false; bool llc_has_parent_sd = false; unsigned int multi_llcs_node = 1; @@ -2621,6 +2623,9 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att if (lowest_flag_domain(i, SD_CLUSTER)) has_cluster = true; + + if (highest_flag_domain(i, SD_SHARE_LLC)) + has_llc = true; } rcu_read_unlock(); @@ -2631,7 +2636,8 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att static_branch_inc_cpuslocked(&sched_cluster_active); #ifdef CONFIG_SCHED_CACHE - if (llc_has_parent_sd && multi_llcs_node && !sched_asym_cpucap_active()) + if (has_llc && llc_has_parent_sd && multi_llcs_node && + !sched_asym_cpucap_active()) static_branch_inc_cpuslocked(&sched_cache_present); #endif Thanks, -adam
Hi Adam, On 9/26/2025 2:14 PM, Adam Li wrote: > Hi Chen Yu, > > I tested the patch set on AmpereOne CPU with 192 cores. > With certain firmware setting, each core has its own L1/L2 cache. > But *no* cores share LLC (L3). So *no* schedule domain > has flag 'SD_SHARE_LLC'. > Good catch! And many thanks for your detailed testing and analysis. Is this issue triggered with CONFIG_SCHED_CLUSTER disabled? > With this topology: > per_cpu(sd_llc_id, cpu) is actually the cpu id (0-191). > > And kernel bug will be triggered at: > 'BUG_ON(idx > MAX_LLC)' > Yes, the sd_llc_idx thing is a bit tricky - we want to use it to index into the static array struct sg_lb_stat.nr_pref_llc, and we have to limit its range. A better approach would be to dynamically allocate the buffer, so we could get rid of the 'idx > MAX_LLC' check, but that might complicate the code. > Please see details bellow. > > The bug will disappear if setting 'MAX_LLC' to 192. > But I think we might disable CAS(cache aware scheduling) > if no domain has 'SD_SHARE_LLC'. > I agree with you. Simply disabling cache-aware scheduling if there is no SD_SHARE_LLC would be simpler. > On 8/9/2025 1:03 PM, Chen Yu wrote: > A draft patch like bellow can fix the kernel BUG: > 1) Do not call update_llc_idx() if domain has no SD_SHARE_LLC > 2) Disable CAS if domain has no SD_SHARE_LLC > > diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c > index 8483c02b4d28..cde9b6cdb1de 100644 > --- a/kernel/sched/topology.c > +++ b/kernel/sched/topology.c > @@ -704,7 +704,8 @@ static void update_top_cache_domain(int cpu) > per_cpu(sd_llc_size, cpu) = size; > per_cpu(sd_llc_id, cpu) = id; > rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds); > - update_llc_idx(cpu); > + if (sd) > + update_llc_idx(cpu); > OK, that make sense. > sd = lowest_flag_domain(cpu, SD_CLUSTER); > if (sd) > @@ -2476,6 +2477,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att > int i, ret = -ENOMEM; > bool has_asym = false; > bool has_cluster = false; > + bool has_llc = false; > bool llc_has_parent_sd = false; > unsigned int multi_llcs_node = 1; > > @@ -2621,6 +2623,9 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att > > if (lowest_flag_domain(i, SD_CLUSTER)) > has_cluster = true; > + > + if (highest_flag_domain(i, SD_SHARE_LLC)) > + has_llc = true; > } > rcu_read_unlock(); > > @@ -2631,7 +2636,8 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att > static_branch_inc_cpuslocked(&sched_cluster_active); > > #ifdef CONFIG_SCHED_CACHE > - if (llc_has_parent_sd && multi_llcs_node && !sched_asym_cpucap_active()) > + if (has_llc && llc_has_parent_sd && multi_llcs_node && multi_llcs_node will be false if there is no SD_SHARE_LLC domain on the platform, so I suppose we don’t have to introduce has_llc? multi_llcs is set to true iff there are more than 1 SD_SHARE_LLC domains under its SD_SHARE_LLC parent domain. thanks, Chenyu > + !sched_asym_cpucap_active()) > static_branch_inc_cpuslocked(&sched_cache_present); > #endif > > > Thanks, > -adam
On 9/26/2025 9:51 PM, Chen, Yu C wrote: > Hi Adam, > > On 9/26/2025 2:14 PM, Adam Li wrote: >> Hi Chen Yu, >> >> I tested the patch set on AmpereOne CPU with 192 cores. >> With certain firmware setting, each core has its own L1/L2 cache. >> But *no* cores share LLC (L3). So *no* schedule domain >> has flag 'SD_SHARE_LLC'. >> > > Good catch! And many thanks for your detailed testing and > analysis. > > Is this issue triggered with CONFIG_SCHED_CLUSTER disabled? > Yes. With CONFIG_SCHED_CLUSTER enabled this issue will not be triggered. The maximum sd_llc_idx will be less than MAX_LLC(64) since we have 24 (192/8) cluster domains. >> With this topology: >> per_cpu(sd_llc_id, cpu) is actually the cpu id (0-191). >> >> And kernel bug will be triggered at: >> 'BUG_ON(idx > MAX_LLC)' >> > > Yes, the sd_llc_idx thing is a bit tricky - we want to use it to > index into the static array struct sg_lb_stat.nr_pref_llc, and > we have to limit its range. A better approach would be to > dynamically allocate the buffer, so we could get rid of the > 'idx > MAX_LLC' check, but that might complicate the code. > >> Please see details bellow. >> >> The bug will disappear if setting 'MAX_LLC' to 192. >> But I think we might disable CAS(cache aware scheduling) >> if no domain has 'SD_SHARE_LLC'. >> > > I agree with you. Simply disabling cache-aware scheduling > if there is no SD_SHARE_LLC would be simpler. > >> On 8/9/2025 1:03 PM, Chen Yu wrote: >> A draft patch like bellow can fix the kernel BUG: >> 1) Do not call update_llc_idx() if domain has no SD_SHARE_LLC >> 2) Disable CAS if domain has no SD_SHARE_LLC >> >> diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c >> index 8483c02b4d28..cde9b6cdb1de 100644 >> --- a/kernel/sched/topology.c >> +++ b/kernel/sched/topology.c >> @@ -704,7 +704,8 @@ static void update_top_cache_domain(int cpu) >> per_cpu(sd_llc_size, cpu) = size; >> per_cpu(sd_llc_id, cpu) = id; >> rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds); >> - update_llc_idx(cpu); >> + if (sd) >> + update_llc_idx(cpu); >> > > OK, that make sense. > >> sd = lowest_flag_domain(cpu, SD_CLUSTER); >> if (sd) >> @@ -2476,6 +2477,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att >> int i, ret = -ENOMEM; >> bool has_asym = false; >> bool has_cluster = false; >> + bool has_llc = false; >> bool llc_has_parent_sd = false; >> unsigned int multi_llcs_node = 1; >> >> @@ -2621,6 +2623,9 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att >> >> if (lowest_flag_domain(i, SD_CLUSTER)) >> has_cluster = true; >> + >> + if (highest_flag_domain(i, SD_SHARE_LLC)) >> + has_llc = true; >> } >> rcu_read_unlock(); >> >> @@ -2631,7 +2636,8 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att >> static_branch_inc_cpuslocked(&sched_cluster_active); >> >> #ifdef CONFIG_SCHED_CACHE >> - if (llc_has_parent_sd && multi_llcs_node && !sched_asym_cpucap_active()) >> + if (has_llc && llc_has_parent_sd && multi_llcs_node && > > multi_llcs_node will be false if there is no SD_SHARE_LLC domain on the > platform, so I suppose we don’t have to introduce has_llc? > multi_llcs is set to true iff there are more than 1 SD_SHARE_LLC domains under its > SD_SHARE_LLC parent domain. > If there is *no* SD_SHARE_LLC domain, my test shows 'multi_llcs_node' is still 1 (true). Looks it is because the default value of 'multi_llcs_node' is 1. build_sched_domains(): unsigned int multi_llcs_node = 1; And this condition is always false since we have no SD_SHARE_LLC domain, therefore 'multi_llcs_node' will not be changed: if (!(sd->flags & SD_SHARE_LLC) && child && (child->flags & SD_SHARE_LLC)) Thanks, -adam
On 9/29/2025 6:43 PM, Adam Li wrote: > On 9/26/2025 9:51 PM, Chen, Yu C wrote: >> Hi Adam, >> >> On 9/26/2025 2:14 PM, Adam Li wrote: [snip] >>> #ifdef CONFIG_SCHED_CACHE >>> - if (llc_has_parent_sd && multi_llcs_node && !sched_asym_cpucap_active()) >>> + if (has_llc && llc_has_parent_sd && multi_llcs_node && >> >> multi_llcs_node will be false if there is no SD_SHARE_LLC domain on the >> platform, so I suppose we don’t have to introduce has_llc? >> multi_llcs is set to true iff there are more than 1 SD_SHARE_LLC domains under its >> SD_SHARE_LLC parent domain. >> > > If there is *no* SD_SHARE_LLC domain, my test shows 'multi_llcs_node' is still 1 (true). > > Looks it is because the default value of 'multi_llcs_node' is 1. > > build_sched_domains(): > unsigned int multi_llcs_node = 1; > > And this condition is always false since we have no SD_SHARE_LLC domain, > therefore 'multi_llcs_node' will not be changed: > > if (!(sd->flags & SD_SHARE_LLC) && child && > (child->flags & SD_SHARE_LLC)) > I see. I was looking at the v5 internal version where multi_llcs was initialized to 0. Let me double check and refine the logic. Thanks! Chenyu > Thanks, > -adam
© 2016 - 2025 Red Hat, Inc.