[v2] workqueue: add cmdline parameter `workqueue_unbound_cpus` to further constrain wq_unbound_cpumask at boot time

[PATCH v2] workqueue: add cmdline parameter `workqueue_unbound_cpus` to further constrain wq_unbound_cpumask at boot time

Posted by tiozhang 2 years, 7 months ago

Motivation of doing this is to better improve boot times for devices when
we want to prevent our workqueue works from running on some specific CPUs,
e,g, some CPUs are busy with interrupts.

Signed-off-by: tiozhang <tiozhang@didiglobal.com>
---
 .../admin-guide/kernel-parameters.txt         |  8 ++++++
 kernel/workqueue.c                            | 25 +++++++++++++++++++
 2 files changed, 33 insertions(+)

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index a465d5242774..7f2fe8c60d5c 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -6780,6 +6780,14 @@
 			disables both lockup detectors. Default is 10
 			seconds.
 
+	workqueue_unbound_cpus=
+			[KNL,SMP]
+			Format: <cpu-list>
+			Specify to constrain one or some CPUs to use in
+			unbound workqueues.
+			By default, all online CPUs are available for
+			unbound workqueues.
+
 	workqueue.watchdog_thresh=
 			If CONFIG_WQ_WATCHDOG is configured, workqueue can
 			warn stall conditions and dump internal state to
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 7cd5f5e7e0a1..1475f8f560cc 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -329,6 +329,9 @@ static bool workqueue_freezing;		/* PL: have wqs started freezing? */
 /* PL: allowable cpus for unbound wqs and work items */
 static cpumask_var_t wq_unbound_cpumask;
 
+/* for further constrain wq_unbound_cpumask by cmdline parameter*/
+static cpumask_var_t wq_cmdline_cpumask;
+
 /* CPU where unbound work was last round robin scheduled from this CPU */
 static DEFINE_PER_CPU(int, wq_rr_cpu_last);
 
@@ -6006,6 +6009,10 @@ void __init workqueue_init_early(void)
 	cpumask_copy(wq_unbound_cpumask, housekeeping_cpumask(HK_TYPE_WQ));
 	cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, housekeeping_cpumask(HK_TYPE_DOMAIN));
 
+	if (!cpumask_empty(wq_cmdline_cpumask))
+		cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, wq_cmdline_cpumask);
+	free_bootmem_cpumask_var(wq_cmdline_cpumask);
+
 	pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
 
 	/* initialize CPU pools */
@@ -6129,3 +6136,21 @@ void __init workqueue_init(void)
  */
 void __warn_flushing_systemwide_wq(void) { }
 EXPORT_SYMBOL(__warn_flushing_systemwide_wq);
+
+
+static int __init workqueue_unbound_cpus_setup(char *str)
+{
+	cpumask_var_t cpumask;
+
+	alloc_bootmem_cpumask_var(&wq_cmdline_cpumask);
+	alloc_bootmem_cpumask_var(&cpumask);
+	if (cpulist_parse(str, cpumask) < 0)
+		pr_warn("workqueue_unbound_cpus: incorrect CPU range\n");
+	else
+		cpumask_copy(wq_cmdline_cpumask, cpumask);
+
+	free_bootmem_cpumask_var(cpumask);
+
+	return 0;
+}
+__setup("workqueue_unbound_cpus=", workqueue_unbound_cpus_setup);
-- 
2.17.1

[PATCH v3] workqueue: add cmdline parameter `workqueue_unbound_cpus` to further constrain wq_unbound_cpumask at boot time

Posted by tiozhang 2 years, 7 months ago

Motivation of doing this is to better improve boot times for devices when
we want to prevent our workqueue works from running on some specific CPUs,
e,g, some CPUs are busy with interrupts.

Signed-off-by: tiozhang <tiozhang@didiglobal.com>
---
 .../admin-guide/kernel-parameters.txt         |  8 +++++++
 kernel/workqueue.c                            | 24 +++++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index a465d5242774..7f2fe8c60d5c 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -6780,6 +6780,14 @@
 			disables both lockup detectors. Default is 10
 			seconds.
 
+	workqueue_unbound_cpus=
+			[KNL,SMP]
+			Format: <cpu-list>
+			Specify to constrain one or some CPUs to use in
+			unbound workqueues.
+			By default, all online CPUs are available for
+			unbound workqueues.
+
 	workqueue.watchdog_thresh=
 			If CONFIG_WQ_WATCHDOG is configured, workqueue can
 			warn stall conditions and dump internal state to
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 7cd5f5e7e0a1..c247725b0873 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -329,6 +329,9 @@ static bool workqueue_freezing;		/* PL: have wqs started freezing? */
 /* PL: allowable cpus for unbound wqs and work items */
 static cpumask_var_t wq_unbound_cpumask;
 
+/* for further constrain wq_unbound_cpumask by cmdline parameter*/
+static cpumask_var_t wq_cmdline_cpumask;
+
 /* CPU where unbound work was last round robin scheduled from this CPU */
 static DEFINE_PER_CPU(int, wq_rr_cpu_last);
 
@@ -6006,6 +6009,10 @@ void __init workqueue_init_early(void)
 	cpumask_copy(wq_unbound_cpumask, housekeeping_cpumask(HK_TYPE_WQ));
 	cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, housekeeping_cpumask(HK_TYPE_DOMAIN));
 
+	if (!cpumask_empty(wq_cmdline_cpumask))
+		cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, wq_cmdline_cpumask);
+	free_bootmem_cpumask_var(wq_cmdline_cpumask);
+
 	pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
 
 	/* initialize CPU pools */
@@ -6129,3 +6136,20 @@ void __init workqueue_init(void)
  */
 void __warn_flushing_systemwide_wq(void) { }
 EXPORT_SYMBOL(__warn_flushing_systemwide_wq);
+
+static int __init workqueue_unbound_cpus_setup(char *str)
+{
+	cpumask_var_t cpumask;
+
+	alloc_bootmem_cpumask_var(&wq_cmdline_cpumask);
+	alloc_bootmem_cpumask_var(&cpumask);
+	if (cpulist_parse(str, cpumask) < 0)
+		pr_warn("workqueue_unbound_cpus: incorrect CPU range\n");
+	else
+		cpumask_copy(wq_cmdline_cpumask, cpumask);
+
+	free_bootmem_cpumask_var(cpumask);
+
+	return 0;
+}
+__setup("workqueue_unbound_cpus=", workqueue_unbound_cpus_setup);
-- 
2.17.1

Re: [PATCH v3] workqueue: add cmdline parameter `workqueue_unbound_cpus` to further constrain wq_unbound_cpumask at boot time

Posted by Tejun Heo 2 years, 7 months ago

Hello,

On Sun, Jun 25, 2023 at 11:15:02AM +0800, tiozhang wrote:
> diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
> index a465d5242774..7f2fe8c60d5c 100644
> --- a/Documentation/admin-guide/kernel-parameters.txt
> +++ b/Documentation/admin-guide/kernel-parameters.txt
> @@ -6780,6 +6780,14 @@
>  			disables both lockup detectors. Default is 10
>  			seconds.
>  
> +	workqueue_unbound_cpus=

Please use workqueue.unbound_cpus for consistency.

> +			[KNL,SMP]
> +			Format: <cpu-list>
> +			Specify to constrain one or some CPUs to use in
> +			unbound workqueues.
> +			By default, all online CPUs are available for
> +			unbound workqueues.

and flow the paragraph.

>  	workqueue.watchdog_thresh=
>  			If CONFIG_WQ_WATCHDOG is configured, workqueue can
>  			warn stall conditions and dump internal state to
> diff --git a/kernel/workqueue.c b/kernel/workqueue.c
> index 7cd5f5e7e0a1..c247725b0873 100644
> --- a/kernel/workqueue.c
> +++ b/kernel/workqueue.c
> @@ -329,6 +329,9 @@ static bool workqueue_freezing;		/* PL: have wqs started freezing? */
>  /* PL: allowable cpus for unbound wqs and work items */
>  static cpumask_var_t wq_unbound_cpumask;
>  
> +/* for further constrain wq_unbound_cpumask by cmdline parameter*/
> +static cpumask_var_t wq_cmdline_cpumask;
> +
>  /* CPU where unbound work was last round robin scheduled from this CPU */
>  static DEFINE_PER_CPU(int, wq_rr_cpu_last);
>  
> @@ -6006,6 +6009,10 @@ void __init workqueue_init_early(void)
>  	cpumask_copy(wq_unbound_cpumask, housekeeping_cpumask(HK_TYPE_WQ));
>  	cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, housekeeping_cpumask(HK_TYPE_DOMAIN));
>  
> +	if (!cpumask_empty(wq_cmdline_cpumask))
> +		cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, wq_cmdline_cpumask);
> +	free_bootmem_cpumask_var(wq_cmdline_cpumask);

If workqueue_unbound_cpus_setup() wasn't called during boot because the
parameter wasn't set, the above would end up trying to access and fre
wq_cmdline_cpumask which hasn't been allocated. Let's just use statically
allocated __initdata struct cpumask so that we don't have to worry about
allocation and freeing during early boot.

> +static int __init workqueue_unbound_cpus_setup(char *str)
> +{
> +	cpumask_var_t cpumask;
> +
> +	alloc_bootmem_cpumask_var(&wq_cmdline_cpumask);
> +	alloc_bootmem_cpumask_var(&cpumask);
> +	if (cpulist_parse(str, cpumask) < 0)
> +		pr_warn("workqueue_unbound_cpus: incorrect CPU range\n");
> +	else
> +		cpumask_copy(wq_cmdline_cpumask, cpumask);

You can just parse direclty into wq_cmdline_cpumask and then clear it on
error.

Thanks.

-- 
tejun

[PATCH v4] workqueue: add cmdline parameter `workqueue.unbound_cpus` to further constrain wq_unbound_cpumask at boot time

Posted by tiozhang 2 years, 7 months ago

Motivation of doing this is to better improve boot times for devices when
we want to prevent our workqueue works from running on some specific CPUs,
e,g, some CPUs are busy with interrupts.

Signed-off-by: tiozhang <tiozhang@didiglobal.com>
---
 .../admin-guide/kernel-parameters.txt         |  7 +++++++
 kernel/workqueue.c                            | 20 +++++++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index a465d5242774..a88b133ab09b 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -6780,6 +6780,13 @@
 			disables both lockup detectors. Default is 10
 			seconds.
 
+	workqueue.unbound_cpus=
+			[KNL,SMP] Specify to constrain one or some CPUs
+			to use in unbound workqueues.
+			Format: <cpu-list>
+			By default, all online CPUs are available for
+			unbound workqueues.
+
 	workqueue.watchdog_thresh=
 			If CONFIG_WQ_WATCHDOG is configured, workqueue can
 			warn stall conditions and dump internal state to
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 7cd5f5e7e0a1..29e8254edd63 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -329,6 +329,9 @@ static bool workqueue_freezing;		/* PL: have wqs started freezing? */
 /* PL: allowable cpus for unbound wqs and work items */
 static cpumask_var_t wq_unbound_cpumask;
 
+/* for further constrain wq_unbound_cpumask by cmdline parameter*/
+static struct cpumask wq_cmdline_cpumask __initdata;
+
 /* CPU where unbound work was last round robin scheduled from this CPU */
 static DEFINE_PER_CPU(int, wq_rr_cpu_last);
 
@@ -6006,6 +6009,9 @@ void __init workqueue_init_early(void)
 	cpumask_copy(wq_unbound_cpumask, housekeeping_cpumask(HK_TYPE_WQ));
 	cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, housekeeping_cpumask(HK_TYPE_DOMAIN));
 
+	if (!cpumask_empty(&wq_cmdline_cpumask))
+		cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, &wq_cmdline_cpumask);
+
 	pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
 
 	/* initialize CPU pools */
@@ -6129,3 +6135,17 @@ void __init workqueue_init(void)
  */
 void __warn_flushing_systemwide_wq(void) { }
 EXPORT_SYMBOL(__warn_flushing_systemwide_wq);
+
+static int __init workqueue_unbound_cpus_setup(char *str)
+{
+	int ret;
+
+	ret = cpulist_parse(str, &wq_cmdline_cpumask);
+	if (ret < 0) {
+		cpumask_clear(&wq_cmdline_cpumask);
+		pr_warn("workqueue.unbound_cpus: incorrect CPU range\n");
+	}
+
+	return ret;
+}
+__setup("workqueue.unbound_cpus=", workqueue_unbound_cpus_setup);
-- 
2.17.1

Re: [PATCH v4] workqueue: add cmdline parameter `workqueue.unbound_cpus` to further constrain wq_unbound_cpumask at boot time

Posted by Randy Dunlap 2 years, 7 months ago

Hi--

On 6/28/23 04:18, tiozhang wrote:
> Motivation of doing this is to better improve boot times for devices when
> we want to prevent our workqueue works from running on some specific CPUs,
> e,g, some CPUs are busy with interrupts.
> 
> Signed-off-by: tiozhang <tiozhang@didiglobal.com>
> ---
>  .../admin-guide/kernel-parameters.txt         |  7 +++++++
>  kernel/workqueue.c                            | 20 +++++++++++++++++++
>  2 files changed, 27 insertions(+)
> 

> diff --git a/kernel/workqueue.c b/kernel/workqueue.c
> index 7cd5f5e7e0a1..29e8254edd63 100644
> --- a/kernel/workqueue.c
> +++ b/kernel/workqueue.c

> @@ -6129,3 +6135,17 @@ void __init workqueue_init(void)
>   */
>  void __warn_flushing_systemwide_wq(void) { }
>  EXPORT_SYMBOL(__warn_flushing_systemwide_wq);
> +
> +static int __init workqueue_unbound_cpus_setup(char *str)
> +{
> +	int ret;
> +
> +	ret = cpulist_parse(str, &wq_cmdline_cpumask);
> +	if (ret < 0) {
> +		cpumask_clear(&wq_cmdline_cpumask);
> +		pr_warn("workqueue.unbound_cpus: incorrect CPU range\n");
> +	}
> +
> +	return ret;
> +}
> +__setup("workqueue.unbound_cpus=", workqueue_unbound_cpus_setup);

__setup() functions don't return 0 for success or errno/other values
for error. They return 1 if the parameter is handled and 0 if it is
not handled, as documented in include/linux/init.h.

And "handled" basically means "recognized" as a kernel parameter,
not that the value(s) passed to it are correct.
I.e., they should usually return 0.

-- 
~Randy

[PATCH v5] workqueue: add cmdline parameter `workqueue.unbound_cpus` to further constrain wq_unbound_cpumask at boot time

Posted by tiozhang 2 years, 7 months ago

Motivation of doing this is to better improve boot times for devices when
we want to prevent our workqueue works from running on some specific CPUs,
e,g, some CPUs are busy with interrupts.

Signed-off-by: tiozhang <tiozhang@didiglobal.com>
---
 Documentation/admin-guide/kernel-parameters.txt |  7 +++++++
 kernel/workqueue.c                              | 17 +++++++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index a465d5242774..a88b133ab09b 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -6780,6 +6780,13 @@
 			disables both lockup detectors. Default is 10
 			seconds.
 
+	workqueue.unbound_cpus=
+			[KNL,SMP] Specify to constrain one or some CPUs
+			to use in unbound workqueues.
+			Format: <cpu-list>
+			By default, all online CPUs are available for
+			unbound workqueues.
+
 	workqueue.watchdog_thresh=
 			If CONFIG_WQ_WATCHDOG is configured, workqueue can
 			warn stall conditions and dump internal state to
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 7cd5f5e7e0a1..70145031acdb 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -329,6 +329,9 @@ static bool workqueue_freezing;		/* PL: have wqs started freezing? */
 /* PL: allowable cpus for unbound wqs and work items */
 static cpumask_var_t wq_unbound_cpumask;
 
+/* for further constrain wq_unbound_cpumask by cmdline parameter*/
+static struct cpumask wq_cmdline_cpumask __initdata;
+
 /* CPU where unbound work was last round robin scheduled from this CPU */
 static DEFINE_PER_CPU(int, wq_rr_cpu_last);
 
@@ -6006,6 +6009,9 @@ void __init workqueue_init_early(void)
 	cpumask_copy(wq_unbound_cpumask, housekeeping_cpumask(HK_TYPE_WQ));
 	cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, housekeeping_cpumask(HK_TYPE_DOMAIN));
 
+	if (!cpumask_empty(&wq_cmdline_cpumask))
+		cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, &wq_cmdline_cpumask);
+
 	pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
 
 	/* initialize CPU pools */
@@ -6129,3 +6135,14 @@ void __init workqueue_init(void)
  */
 void __warn_flushing_systemwide_wq(void) { }
 EXPORT_SYMBOL(__warn_flushing_systemwide_wq);
+
+static int __init workqueue_unbound_cpus_setup(char *str)
+{
+	if (cpulist_parse(str, &wq_cmdline_cpumask) < 0) {
+		cpumask_clear(&wq_cmdline_cpumask);
+		pr_warn("workqueue.unbound_cpus: incorrect CPU range, using default\n");
+	}
+
+	return 1;
+}
+__setup("workqueue.unbound_cpus=", workqueue_unbound_cpus_setup);
-- 
2.17.1

Re: [PATCH v5] workqueue: add cmdline parameter `workqueue.unbound_cpus` to further constrain wq_unbound_cpumask at boot time

Posted by Tejun Heo 2 years, 7 months ago

On Thu, Jun 29, 2023 at 11:50:50AM +0800, tiozhang wrote:
> Motivation of doing this is to better improve boot times for devices when
> we want to prevent our workqueue works from running on some specific CPUs,
> e,g, some CPUs are busy with interrupts.
> 
> Signed-off-by: tiozhang <tiozhang@didiglobal.com>

Applied to wq/for-6.6.

Thanks.

-- 
tejun