[PATCH v9 25/31] fs/resctrl: Move allocation/free of closid_num_dirty_rmid

Tony Luck posted 31 patches 1 month ago
There is a newer version of this series
[PATCH v9 25/31] fs/resctrl: Move allocation/free of closid_num_dirty_rmid
Posted by Tony Luck 1 month ago
closid_num_dirty_rmid[] is allocated in dom_data_init() during resctrl
initialization and freed by dom_data_exit() during resctrl exit giving
it the same life cycle as rmid_ptrs[].

Move closid_num_dirty_rmid[] allocaction/free out to
resctrl_l3_mon_resource_init() and resctrl_l3_mon_resource_exit() in
preparation for rmid_ptrs[] to be allocated on resctrl mount in support
of the new telemetry events.

Signed-off-by: Tony Luck <tony.luck@intel.com>
---
 fs/resctrl/monitor.c | 56 ++++++++++++++++++++++----------------------
 1 file changed, 28 insertions(+), 28 deletions(-)

diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c
index 33432a7f56da..d5b96aca5d03 100644
--- a/fs/resctrl/monitor.c
+++ b/fs/resctrl/monitor.c
@@ -805,36 +805,14 @@ void mbm_setup_overflow_handler(struct rdt_l3_mon_domain *dom, unsigned long del
 static int dom_data_init(struct rdt_resource *r)
 {
 	u32 idx_limit = resctrl_arch_system_num_rmid_idx();
-	u32 num_closid = resctrl_arch_get_num_closid(r);
 	struct rmid_entry *entry = NULL;
 	int err = 0, i;
 	u32 idx;
 
 	mutex_lock(&rdtgroup_mutex);
-	if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) {
-		u32 *tmp;
-
-		/*
-		 * If the architecture hasn't provided a sanitised value here,
-		 * this may result in larger arrays than necessary. Resctrl will
-		 * use a smaller system wide value based on the resources in
-		 * use.
-		 */
-		tmp = kcalloc(num_closid, sizeof(*tmp), GFP_KERNEL);
-		if (!tmp) {
-			err = -ENOMEM;
-			goto out_unlock;
-		}
-
-		closid_num_dirty_rmid = tmp;
-	}
 
 	rmid_ptrs = kcalloc(idx_limit, sizeof(struct rmid_entry), GFP_KERNEL);
 	if (!rmid_ptrs) {
-		if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) {
-			kfree(closid_num_dirty_rmid);
-			closid_num_dirty_rmid = NULL;
-		}
 		err = -ENOMEM;
 		goto out_unlock;
 	}
@@ -870,11 +848,6 @@ static void dom_data_exit(struct rdt_resource *r)
 	if (!r->mon_capable)
 		goto out_unlock;
 
-	if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) {
-		kfree(closid_num_dirty_rmid);
-		closid_num_dirty_rmid = NULL;
-	}
-
 	kfree(rmid_ptrs);
 	rmid_ptrs = NULL;
 
@@ -957,9 +930,31 @@ int resctrl_l3_mon_resource_init(void)
 	if (!r->mon_capable)
 		return 0;
 
+	if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) {
+		u32 num_closid = resctrl_arch_get_num_closid(r);
+		u32 *tmp;
+
+		/*
+		 * If the architecture hasn't provided a sanitised value here,
+		 * this may result in larger arrays than necessary. Resctrl will
+		 * use a smaller system wide value based on the resources in
+		 * use.
+		 */
+		tmp = kcalloc(num_closid, sizeof(*tmp), GFP_KERNEL);
+		if (!tmp)
+			return -ENOMEM;
+
+		closid_num_dirty_rmid = tmp;
+	}
+
 	ret = dom_data_init(r);
-	if (ret)
+	if (ret) {
+		if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) {
+			kfree(closid_num_dirty_rmid);
+			closid_num_dirty_rmid = NULL;
+		}
 		return ret;
+	}
 
 	if (resctrl_arch_is_evt_configurable(QOS_L3_MBM_TOTAL_EVENT_ID)) {
 		mon_event_all[QOS_L3_MBM_TOTAL_EVENT_ID].configurable = true;
@@ -984,5 +979,10 @@ void resctrl_l3_mon_resource_exit(void)
 {
 	struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3);
 
+	if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) {
+		kfree(closid_num_dirty_rmid);
+		closid_num_dirty_rmid = NULL;
+	}
+
 	dom_data_exit(r);
 }
-- 
2.50.1
Re: [PATCH v9 25/31] fs/resctrl: Move allocation/free of closid_num_dirty_rmid
Posted by Reinette Chatre 3 weeks, 1 day ago
Hi Tony,

In subject:
closid_num_dirty_rmid -> closid_num_dirty_rmid[]

On 8/29/25 12:33 PM, Tony Luck wrote:
> closid_num_dirty_rmid[] is allocated in dom_data_init() during resctrl
> initialization and freed by dom_data_exit() during resctrl exit giving
> it the same life cycle as rmid_ptrs[].
> 
> Move closid_num_dirty_rmid[] allocaction/free out to
> resctrl_l3_mon_resource_init() and resctrl_l3_mon_resource_exit() in
> preparation for rmid_ptrs[] to be allocated on resctrl mount in support
> of the new telemetry events.
> 
> Signed-off-by: Tony Luck <tony.luck@intel.com>
> ---
>  fs/resctrl/monitor.c | 56 ++++++++++++++++++++++----------------------
>  1 file changed, 28 insertions(+), 28 deletions(-)
> 
> diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c
> index 33432a7f56da..d5b96aca5d03 100644
> --- a/fs/resctrl/monitor.c
> +++ b/fs/resctrl/monitor.c
> @@ -805,36 +805,14 @@ void mbm_setup_overflow_handler(struct rdt_l3_mon_domain *dom, unsigned long del
>  static int dom_data_init(struct rdt_resource *r)
>  {
>  	u32 idx_limit = resctrl_arch_system_num_rmid_idx();
> -	u32 num_closid = resctrl_arch_get_num_closid(r);
>  	struct rmid_entry *entry = NULL;
>  	int err = 0, i;
>  	u32 idx;
>  
>  	mutex_lock(&rdtgroup_mutex);

mutex is held during original allocation code below ...

> -	if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) {
> -		u32 *tmp;
> -
> -		/*
> -		 * If the architecture hasn't provided a sanitised value here,
> -		 * this may result in larger arrays than necessary. Resctrl will
> -		 * use a smaller system wide value based on the resources in
> -		 * use.
> -		 */
> -		tmp = kcalloc(num_closid, sizeof(*tmp), GFP_KERNEL);
> -		if (!tmp) {
> -			err = -ENOMEM;
> -			goto out_unlock;
> -		}
> -
> -		closid_num_dirty_rmid = tmp;
> -	}
>  
>  	rmid_ptrs = kcalloc(idx_limit, sizeof(struct rmid_entry), GFP_KERNEL);
>  	if (!rmid_ptrs) {
> -		if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) {
> -			kfree(closid_num_dirty_rmid);
> -			closid_num_dirty_rmid = NULL;
> -		}
>  		err = -ENOMEM;
>  		goto out_unlock;
>  	}
> @@ -870,11 +848,6 @@ static void dom_data_exit(struct rdt_resource *r)
>  	if (!r->mon_capable)
>  		goto out_unlock;
>  
> -	if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) {
> -		kfree(closid_num_dirty_rmid);
> -		closid_num_dirty_rmid = NULL;
> -	}
> -

mutex is held in original free code  ...

>  	kfree(rmid_ptrs);
>  	rmid_ptrs = NULL;
>  
> @@ -957,9 +930,31 @@ int resctrl_l3_mon_resource_init(void)
>  	if (!r->mon_capable)
>  		return 0;
>  
> +	if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) {
> +		u32 num_closid = resctrl_arch_get_num_closid(r);
> +		u32 *tmp;
> +
> +		/*
> +		 * If the architecture hasn't provided a sanitised value here,
> +		 * this may result in larger arrays than necessary. Resctrl will
> +		 * use a smaller system wide value based on the resources in
> +		 * use.
> +		 */
> +		tmp = kcalloc(num_closid, sizeof(*tmp), GFP_KERNEL);
> +		if (!tmp)
> +			return -ENOMEM;
> +
> +		closid_num_dirty_rmid = tmp;
> +	}

mutex no longer held during allocation ... no mention in changelog why this is ok?

> +
>  	ret = dom_data_init(r);
> -	if (ret)
> +	if (ret) {
> +		if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) {
> +			kfree(closid_num_dirty_rmid);
> +			closid_num_dirty_rmid = NULL;
> +		}
>  		return ret;
> +	}
>  
>  	if (resctrl_arch_is_evt_configurable(QOS_L3_MBM_TOTAL_EVENT_ID)) {
>  		mon_event_all[QOS_L3_MBM_TOTAL_EVENT_ID].configurable = true;
> @@ -984,5 +979,10 @@ void resctrl_l3_mon_resource_exit(void)
>  {
>  	struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3);
>  

If you also use the "if (!r->mon_capable)" check as is done in original code
then resctrl_l3_mon_resource_exit() will be symmetrical with
resctrl_l3_mon_resource_init(). Since resctrl_l3_mon_resource_exit() is
_always_ called, whether monitoring is enabled or not, adding this check
will also make the code easier to follow and not require that reader knows
how closid_num_dirty_rmid[] is connected.
Another benefit of keeping this check is that it forces the function to
remain L3 resource specific (not just by name) and not seemingly become generic
later in series ("fs/resctrl: Move RMID initialization to first mount")

> +	if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) {
> +		kfree(closid_num_dirty_rmid);
> +		closid_num_dirty_rmid = NULL;
> +	}

... now freed without mutex without mention in changelog?

> +
>  	dom_data_exit(r);
>  }

Reinette
Re: [PATCH v9 25/31] fs/resctrl: Move allocation/free of closid_num_dirty_rmid
Posted by Luck, Tony 3 weeks ago
On Wed, Sep 10, 2025 at 10:55:23AM -0700, Reinette Chatre wrote:
> Hi Tony,
> 
> In subject:
> closid_num_dirty_rmid -> closid_num_dirty_rmid[]
> 
> On 8/29/25 12:33 PM, Tony Luck wrote:
> > closid_num_dirty_rmid[] is allocated in dom_data_init() during resctrl
> > initialization and freed by dom_data_exit() during resctrl exit giving
> > it the same life cycle as rmid_ptrs[].
> > 
> > Move closid_num_dirty_rmid[] allocaction/free out to
> > resctrl_l3_mon_resource_init() and resctrl_l3_mon_resource_exit() in
> > preparation for rmid_ptrs[] to be allocated on resctrl mount in support
> > of the new telemetry events.
> > 
> > Signed-off-by: Tony Luck <tony.luck@intel.com>
> > ---
> >  fs/resctrl/monitor.c | 56 ++++++++++++++++++++++----------------------
> >  1 file changed, 28 insertions(+), 28 deletions(-)
> > 
> > diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c
> > index 33432a7f56da..d5b96aca5d03 100644
> > --- a/fs/resctrl/monitor.c
> > +++ b/fs/resctrl/monitor.c
> > @@ -805,36 +805,14 @@ void mbm_setup_overflow_handler(struct rdt_l3_mon_domain *dom, unsigned long del
> >  static int dom_data_init(struct rdt_resource *r)
> >  {
> >  	u32 idx_limit = resctrl_arch_system_num_rmid_idx();
> > -	u32 num_closid = resctrl_arch_get_num_closid(r);
> >  	struct rmid_entry *entry = NULL;
> >  	int err = 0, i;
> >  	u32 idx;
> >  
> >  	mutex_lock(&rdtgroup_mutex);
> 
> mutex is held during original allocation code below ...
> 
> > -	if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) {
> > -		u32 *tmp;
> > -
> > -		/*
> > -		 * If the architecture hasn't provided a sanitised value here,
> > -		 * this may result in larger arrays than necessary. Resctrl will
> > -		 * use a smaller system wide value based on the resources in
> > -		 * use.
> > -		 */
> > -		tmp = kcalloc(num_closid, sizeof(*tmp), GFP_KERNEL);
> > -		if (!tmp) {
> > -			err = -ENOMEM;
> > -			goto out_unlock;
> > -		}
> > -
> > -		closid_num_dirty_rmid = tmp;
> > -	}

James,

While refactoring code, I missed moving the mutex_lock(&rdtgroup_mutex);
that was protecting the allocation of "closid_num_dirty_rmid[]" ... and
Reinette caught this change.

But looking at the code, I'm not at all sure what protection is needed
for the allocation/free of this array. The current calling sequence for
allocation is:

Linux late_init
    resctrl_arch_late_init()
	resctrl_init()
	    resctrl_l3_mon_resource_init
		 dom_data_init()

which doesn't appear to provide any scope for other CPUs to come in and
start using closid_num_dirty_rmid[]

The free path also seems safe too, as all resctrl functions should
be shutdown before calling:

resctrl_arch_exit()
    resctrl_exit()
	resctrl_l3_mon_resource_exit()
	    dom_data_exit()

and if they were not, holding the rdtgroup_mutex around:

	kfree(closid_num_dirty_rmid);
	closid_num_dirty_rmid = NULL;

would do nothing to prevent some still active resctrl function
from tripping over a NULL pointer.


So, unless I'm missing something, I'm planning to address Reinette's
find by documenting inmy commit message that holding rdtgroup_mutex
has always been unnecessary, so it is dropped as part of this refactor.

-Tony
Re: [PATCH v9 25/31] fs/resctrl: Move allocation/free of closid_num_dirty_rmid
Posted by Luck, Tony 2 weeks, 6 days ago
On Thu, Sep 11, 2025 at 03:26:44PM -0700, Luck, Tony wrote:
> James,
> 
> While refactoring code, I missed moving the mutex_lock(&rdtgroup_mutex);
> that was protecting the allocation of "closid_num_dirty_rmid[]" ... and
> Reinette caught this change.
> 
> But looking at the code, I'm not at all sure what protection is needed
> for the allocation/free of this array. The current calling sequence for
> allocation is:
> 
> Linux late_init
>     resctrl_arch_late_init()
> 	resctrl_init()
> 	    resctrl_l3_mon_resource_init
> 		 dom_data_init()
> 
> which doesn't appear to provide any scope for other CPUs to come in and
> start using closid_num_dirty_rmid[]
> 
> The free path also seems safe too, as all resctrl functions should
> be shutdown before calling:
> 
> resctrl_arch_exit()
>     resctrl_exit()
> 	resctrl_l3_mon_resource_exit()
> 	    dom_data_exit()
> 
> and if they were not, holding the rdtgroup_mutex around:
> 
> 	kfree(closid_num_dirty_rmid);
> 	closid_num_dirty_rmid = NULL;
> 
> would do nothing to prevent some still active resctrl function
> from tripping over a NULL pointer.
> 
> 
> So, unless I'm missing something, I'm planning to address Reinette's
> find by documenting inmy commit message that holding rdtgroup_mutex
> has always been unnecessary, so it is dropped as part of this refactor.

James,

Cancel this request. Reinette pointed me at the discussions last year
about allocation[1] and free[2] where you descibed how memory ordering on
ARM might be a concern and the mutex makes it obvious that the access to
closid_num_dirty_rmid is safe.

-Tony

[1] https://lore.kernel.org/lkml/20b566d9-448b-5367-b4db-593466e7a2f8@arm.com/
[2] https://lore.kernel.org/lkml/52f81c45-efa7-42c7-86f4-fc1084b1d57a@redhat.com/