[PATCH v5 07/40] x86/resctrl: Add max_bw to struct resctrl_membw

James Morse posted 40 patches 1 year, 4 months ago
There is a newer version of this series
[PATCH v5 07/40] x86/resctrl: Add max_bw to struct resctrl_membw
Posted by James Morse 1 year, 4 months ago
__rdt_get_mem_config_amd() and __get_mem_config_intel() both use
the default_ctrl property as a maximum value. This is because the
MBA schema works differently between these platforms. Doing this
complicates determining whether the default_ctrl property belongs
to the arch code, or can be derived from the schema format.

Add a max_bw property for x86 platforms to specify their maximum
MBA bandwidth. This isn't needed for other schema formats.

This will allow the default_ctrl to be generated from the schema
properties when it is needed.

Signed-off-by: James Morse <james.morse@arm.com>
Tested-by: Carl Worth <carl@os.amperecomputing.com> # arm64
Tested-by: Shaopeng Tan <tan.shaopeng@jp.fujitsu.com>
Reviewed-by: Shaopeng Tan <tan.shaopeng@jp.fujitsu.com>
---
Changes since v2:
 * This patch is new.
---
 arch/x86/kernel/cpu/resctrl/core.c        | 3 +++
 arch/x86/kernel/cpu/resctrl/ctrlmondata.c | 9 +++++----
 include/linux/resctrl.h                   | 2 ++
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c
index 4c16e58c4a1b..e79807a8f060 100644
--- a/arch/x86/kernel/cpu/resctrl/core.c
+++ b/arch/x86/kernel/cpu/resctrl/core.c
@@ -212,6 +212,7 @@ static bool __get_mem_config_intel(struct rdt_resource *r)
 	hw_res->num_closid = edx.split.cos_max + 1;
 	max_delay = eax.split.max_delay + 1;
 	r->default_ctrl = MAX_MBA_BW;
+	r->membw.max_bw = MAX_MBA_BW;
 	r->membw.arch_needs_linear = true;
 	if (ecx & MBA_IS_LINEAR) {
 		r->membw.delay_linear = true;
@@ -248,6 +249,8 @@ static bool __rdt_get_mem_config_amd(struct rdt_resource *r)
 	cpuid_count(0x80000020, subleaf, &eax, &ebx, &ecx, &edx);
 	hw_res->num_closid = edx + 1;
 	r->default_ctrl = 1 << eax;
+	r->schema_fmt = RESCTRL_SCHEMA_RANGE;
+	r->membw.max_bw = 1 << eax;
 
 	/* AMD does not use delay */
 	r->membw.delay_linear = false;
diff --git a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
index 8d1bdfe89692..56c41bfd07e4 100644
--- a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
+++ b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
@@ -57,10 +57,10 @@ static bool bw_validate(char *buf, unsigned long *data, struct rdt_resource *r)
 		return false;
 	}
 
-	if ((bw < r->membw.min_bw || bw > r->default_ctrl) &&
+	if ((bw < r->membw.min_bw || bw > r->membw.max_bw) &&
 	    !is_mba_sc(r)) {
 		rdt_last_cmd_printf("MB value %ld out of range [%d,%d]\n", bw,
-				    r->membw.min_bw, r->default_ctrl);
+				    r->membw.min_bw, r->membw.max_bw);
 		return false;
 	}
 
@@ -108,8 +108,9 @@ static int parse_bw(struct rdt_parse_data *data, struct resctrl_schema *s,
  */
 static bool cbm_validate(char *buf, u32 *data, struct rdt_resource *r)
 {
-	unsigned long first_bit, zero_bit, val;
+	u32 supported_bits = BIT_MASK(r->cache.cbm_len + 1) - 1;
 	unsigned int cbm_len = r->cache.cbm_len;
+	unsigned long first_bit, zero_bit, val;
 	int ret;
 
 	ret = kstrtoul(buf, 16, &val);
@@ -118,7 +119,7 @@ static bool cbm_validate(char *buf, u32 *data, struct rdt_resource *r)
 		return false;
 	}
 
-	if ((r->cache.min_cbm_bits > 0 && val == 0) || val > r->default_ctrl) {
+	if ((r->cache.min_cbm_bits > 0 && val == 0) || val > supported_bits) {
 		rdt_last_cmd_puts("Mask out of range\n");
 		return false;
 	}
diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h
index 0f61673c9165..b66cd977b658 100644
--- a/include/linux/resctrl.h
+++ b/include/linux/resctrl.h
@@ -165,6 +165,7 @@ enum membw_throttle_mode {
 /**
  * struct resctrl_membw - Memory bandwidth allocation related data
  * @min_bw:		Minimum memory bandwidth percentage user can request
+ * @max_bw:		Maximum memory bandwidth value, used as the reset value
  * @bw_gran:		Granularity at which the memory bandwidth is allocated
  * @delay_linear:	True if memory B/W delay is in linear scale
  * @arch_needs_linear:	True if we can't configure non-linear resources
@@ -175,6 +176,7 @@ enum membw_throttle_mode {
  */
 struct resctrl_membw {
 	u32				min_bw;
+	u32				max_bw;
 	u32				bw_gran;
 	u32				delay_linear;
 	bool				arch_needs_linear;
-- 
2.39.2
Re: [PATCH v5 07/40] x86/resctrl: Add max_bw to struct resctrl_membw
Posted by Reinette Chatre 1 year, 3 months ago
Hi James,

On 10/4/24 11:03 AM, James Morse wrote:
> __rdt_get_mem_config_amd() and __get_mem_config_intel() both use
> the default_ctrl property as a maximum value. This is because the
> MBA schema works differently between these platforms. Doing this

The schema works differently but they can still use the same property
as maximum, is that a problem?

> complicates determining whether the default_ctrl property belongs
> to the arch code, or can be derived from the schema format.

So instead of Intel and AMD both using default_ctrl as a maximum this patch
introduces a new max_bw with both using that as maximum instead.
Unclear how this change fixes the unclear complication.

> 
> Add a max_bw property for x86 platforms to specify their maximum
> MBA bandwidth. This isn't needed for other schema formats.

It is not clear to me how replacing one value with a new value that is
used in exactly the same way addresses the initial complaint of complication.

> 
> This will allow the default_ctrl to be generated from the schema
> properties when it is needed.
> 
> Signed-off-by: James Morse <james.morse@arm.com>
> Tested-by: Carl Worth <carl@os.amperecomputing.com> # arm64
> Tested-by: Shaopeng Tan <tan.shaopeng@jp.fujitsu.com>
> Reviewed-by: Shaopeng Tan <tan.shaopeng@jp.fujitsu.com>
> ---
> Changes since v2:
>  * This patch is new.
> ---
>  arch/x86/kernel/cpu/resctrl/core.c        | 3 +++
>  arch/x86/kernel/cpu/resctrl/ctrlmondata.c | 9 +++++----
>  include/linux/resctrl.h                   | 2 ++
>  3 files changed, 10 insertions(+), 4 deletions(-)
> 
> diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c
> index 4c16e58c4a1b..e79807a8f060 100644
> --- a/arch/x86/kernel/cpu/resctrl/core.c
> +++ b/arch/x86/kernel/cpu/resctrl/core.c
> @@ -212,6 +212,7 @@ static bool __get_mem_config_intel(struct rdt_resource *r)
>  	hw_res->num_closid = edx.split.cos_max + 1;
>  	max_delay = eax.split.max_delay + 1;
>  	r->default_ctrl = MAX_MBA_BW;
> +	r->membw.max_bw = MAX_MBA_BW;
>  	r->membw.arch_needs_linear = true;
>  	if (ecx & MBA_IS_LINEAR) {
>  		r->membw.delay_linear = true;
> @@ -248,6 +249,8 @@ static bool __rdt_get_mem_config_amd(struct rdt_resource *r)
>  	cpuid_count(0x80000020, subleaf, &eax, &ebx, &ecx, &edx);
>  	hw_res->num_closid = edx + 1;
>  	r->default_ctrl = 1 << eax;
> +	r->schema_fmt = RESCTRL_SCHEMA_RANGE;

Stray change?

> +	r->membw.max_bw = 1 << eax;
>  
>  	/* AMD does not use delay */
>  	r->membw.delay_linear = false;
> diff --git a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
> index 8d1bdfe89692..56c41bfd07e4 100644
> --- a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
> +++ b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
> @@ -57,10 +57,10 @@ static bool bw_validate(char *buf, unsigned long *data, struct rdt_resource *r)
>  		return false;
>  	}
>  
> -	if ((bw < r->membw.min_bw || bw > r->default_ctrl) &&
> +	if ((bw < r->membw.min_bw || bw > r->membw.max_bw) &&
>  	    !is_mba_sc(r)) {
>  		rdt_last_cmd_printf("MB value %ld out of range [%d,%d]\n", bw,
> -				    r->membw.min_bw, r->default_ctrl);
> +				    r->membw.min_bw, r->membw.max_bw);
>  		return false;
>  	}
>  
> @@ -108,8 +108,9 @@ static int parse_bw(struct rdt_parse_data *data, struct resctrl_schema *s,
>   */
>  static bool cbm_validate(char *buf, u32 *data, struct rdt_resource *r)
>  {
> -	unsigned long first_bit, zero_bit, val;
> +	u32 supported_bits = BIT_MASK(r->cache.cbm_len + 1) - 1;

Same issue as V4:
https://lore.kernel.org/all/ca528ebd-fb76-40cd-a495-88c2de443cd8@intel.com/

>  	unsigned int cbm_len = r->cache.cbm_len;
> +	unsigned long first_bit, zero_bit, val;
>  	int ret;
>  
>  	ret = kstrtoul(buf, 16, &val);
> @@ -118,7 +119,7 @@ static bool cbm_validate(char *buf, u32 *data, struct rdt_resource *r)
>  		return false;
>  	}
>  
> -	if ((r->cache.min_cbm_bits > 0 && val == 0) || val > r->default_ctrl) {
> +	if ((r->cache.min_cbm_bits > 0 && val == 0) || val > supported_bits) {
>  		rdt_last_cmd_puts("Mask out of range\n");
>  		return false;
>  	}

The above two changes have nothing to do with memory bandwidth. They are unrelated
to the changelog.

> diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h
> index 0f61673c9165..b66cd977b658 100644
> --- a/include/linux/resctrl.h
> +++ b/include/linux/resctrl.h
> @@ -165,6 +165,7 @@ enum membw_throttle_mode {
>  /**
>   * struct resctrl_membw - Memory bandwidth allocation related data
>   * @min_bw:		Minimum memory bandwidth percentage user can request
> + * @max_bw:		Maximum memory bandwidth value, used as the reset value
>   * @bw_gran:		Granularity at which the memory bandwidth is allocated
>   * @delay_linear:	True if memory B/W delay is in linear scale
>   * @arch_needs_linear:	True if we can't configure non-linear resources
> @@ -175,6 +176,7 @@ enum membw_throttle_mode {
>   */
>  struct resctrl_membw {
>  	u32				min_bw;
> +	u32				max_bw;
>  	u32				bw_gran;
>  	u32				delay_linear;
>  	bool				arch_needs_linear;

Reinette
Re: [PATCH v5 07/40] x86/resctrl: Add max_bw to struct resctrl_membw
Posted by James Morse 1 year, 1 month ago
Hi Reinette,

On 23/10/2024 22:14, Reinette Chatre wrote:
> On 10/4/24 11:03 AM, James Morse wrote:
>> __rdt_get_mem_config_amd() and __get_mem_config_intel() both use
>> the default_ctrl property as a maximum value. This is because the
>> MBA schema works differently between these platforms. Doing this

> The schema works differently but they can still use the same property
> as maximum, is that a problem?

I think its a problem for user-space - but its an existing problem.
Today resctrl uses the default as the maximum. This makes that property explicit.


>> complicates determining whether the default_ctrl property belongs
>> to the arch code, or can be derived from the schema format.
> 
> So instead of Intel and AMD both using default_ctrl as a maximum this patch
> introduces a new max_bw with both using that as maximum instead.
> Unclear how this change fixes the unclear complication.

Is the default value something that can be determine from the schema format?

Previously, no - because the default value is where the per-platform maximum bandwidth is
stashed. By making that explicit, we can drop the spoon feeding the architecture code has
to do to tell resctrl that the maximum bitmap is all-ones, and the maximum percentage is
100. Bandwidth is always going to be weird like this, so it makes sense to special case it.

I'll add some form of this text to the commit message.


>> Add a max_bw property for x86 platforms to specify their maximum
>> MBA bandwidth. This isn't needed for other schema formats.
> 
> It is not clear to me how replacing one value with a new value that is
> used in exactly the same way addresses the initial complaint of complication.
> 
>>
>> This will allow the default_ctrl to be generated from the schema
>> properties when it is needed.

>> diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c
>> index 4c16e58c4a1b..e79807a8f060 100644
>> --- a/arch/x86/kernel/cpu/resctrl/core.c
>> +++ b/arch/x86/kernel/cpu/resctrl/core.c

>> @@ -248,6 +249,8 @@ static bool __rdt_get_mem_config_amd(struct rdt_resource *r)
>>  	cpuid_count(0x80000020, subleaf, &eax, &ebx, &ecx, &edx);
>>  	hw_res->num_closid = edx + 1;
>>  	r->default_ctrl = 1 << eax;
>> +	r->schema_fmt = RESCTRL_SCHEMA_RANGE;
> 
> Stray change?

Yes, when these were different values, AMD had to override the value in the table.
Since they got merged back together, its the same.

Thanks!


>> +	r->membw.max_bw = 1 << eax;
>>  
>>  	/* AMD does not use delay */
>>  	r->membw.delay_linear = false;
>> diff --git a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
>> index 8d1bdfe89692..56c41bfd07e4 100644
>> --- a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
>> +++ b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c

>> @@ -108,8 +108,9 @@ static int parse_bw(struct rdt_parse_data *data, struct resctrl_schema *s,
>>   */
>>  static bool cbm_validate(char *buf, u32 *data, struct rdt_resource *r)
>>  {
>> -	unsigned long first_bit, zero_bit, val;
>> +	u32 supported_bits = BIT_MASK(r->cache.cbm_len + 1) - 1;
> 
> Same issue as V4:
> https://lore.kernel.org/all/ca528ebd-fb76-40cd-a495-88c2de443cd8@intel.com/

Huh. I was sure I'd fixed that when you first pointed it out.
Sorry about that!


>> @@ -118,7 +119,7 @@ static bool cbm_validate(char *buf, u32 *data, struct rdt_resource *r)
>>  		return false;
>>  	}
>>  
>> -	if ((r->cache.min_cbm_bits > 0 && val == 0) || val > r->default_ctrl) {
>> +	if ((r->cache.min_cbm_bits > 0 && val == 0) || val > supported_bits) {
>>  		rdt_last_cmd_puts("Mask out of range\n");
>>  		return false;
>>  	}
> 
> The above two changes have nothing to do with memory bandwidth. They are unrelated
> to the changelog.

Yes, these should be in the next patch.


Thanks!

James
Re: [PATCH v5 07/40] x86/resctrl: Add max_bw to struct resctrl_membw
Posted by Tony Luck 1 year, 4 months ago
On Fri, Oct 04, 2024 at 06:03:14PM +0000, James Morse wrote:
> diff --git a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
> index 8d1bdfe89692..56c41bfd07e4 100644
> --- a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
> +++ b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
> @@ -57,10 +57,10 @@ static bool bw_validate(char *buf, unsigned long *data, struct rdt_resource *r)
>  		return false;
>  	}
>  
> -	if ((bw < r->membw.min_bw || bw > r->default_ctrl) &&
> +	if ((bw < r->membw.min_bw || bw > r->membw.max_bw) &&
>  	    !is_mba_sc(r)) {
>  		rdt_last_cmd_printf("MB value %ld out of range [%d,%d]\n", bw,
> -				    r->membw.min_bw, r->default_ctrl);
> +				    r->membw.min_bw, r->membw.max_bw);
>  		return false;
>  	}
>  

Heads up. There is a patch to this function in the TIP x86/urgent
branch. So will likely go into v6.12-rc3. So this patch will need
to refactor on top of:

https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git/commit/?h=x86/urgent&id=2b5648416e47933939dc310c4ea1e29404f35630

-Tony