Introduce a new affinity_context structure for passing cpu affinity information
around in core scheduler code. The relevant functions are modified to use
the new structure. There is no functional change.
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Waiman Long <longman@redhat.com>
---
kernel/sched/core.c | 114 ++++++++++++++++++++++++++--------------
kernel/sched/deadline.c | 7 ++-
kernel/sched/sched.h | 11 ++--
3 files changed, 86 insertions(+), 46 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 84544daf3839..b43b851c0399 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2195,14 +2195,18 @@ void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
#ifdef CONFIG_SMP
static void
-__do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32 flags);
+__do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx);
static int __set_cpus_allowed_ptr(struct task_struct *p,
- const struct cpumask *new_mask,
- u32 flags);
+ struct affinity_context *ctx);
static void migrate_disable_switch(struct rq *rq, struct task_struct *p)
{
+ struct affinity_context ac = {
+ .new_mask = cpumask_of(rq->cpu),
+ .flags = SCA_MIGRATE_DISABLE,
+ };
+
if (likely(!p->migration_disabled))
return;
@@ -2212,7 +2216,7 @@ static void migrate_disable_switch(struct rq *rq, struct task_struct *p)
/*
* Violates locking rules! see comment in __do_set_cpus_allowed().
*/
- __do_set_cpus_allowed(p, cpumask_of(rq->cpu), SCA_MIGRATE_DISABLE);
+ __do_set_cpus_allowed(p, &ac);
}
void migrate_disable(void)
@@ -2234,6 +2238,10 @@ EXPORT_SYMBOL_GPL(migrate_disable);
void migrate_enable(void)
{
struct task_struct *p = current;
+ struct affinity_context ac = {
+ .new_mask = &p->cpus_mask,
+ .flags = SCA_MIGRATE_ENABLE,
+ };
if (p->migration_disabled > 1) {
p->migration_disabled--;
@@ -2249,7 +2257,7 @@ void migrate_enable(void)
*/
preempt_disable();
if (p->cpus_ptr != &p->cpus_mask)
- __set_cpus_allowed_ptr(p, &p->cpus_mask, SCA_MIGRATE_ENABLE);
+ __set_cpus_allowed_ptr(p, &ac);
/*
* Mustn't clear migration_disabled() until cpus_ptr points back at the
* regular cpus_mask, otherwise things that race (eg.
@@ -2529,19 +2537,19 @@ int push_cpu_stop(void *arg)
* sched_class::set_cpus_allowed must do the below, but is not required to
* actually call this function.
*/
-void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask, u32 flags)
+void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx)
{
- if (flags & (SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) {
- p->cpus_ptr = new_mask;
+ if (ctx->flags & (SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) {
+ p->cpus_ptr = ctx->new_mask;
return;
}
- cpumask_copy(&p->cpus_mask, new_mask);
- p->nr_cpus_allowed = cpumask_weight(new_mask);
+ cpumask_copy(&p->cpus_mask, ctx->new_mask);
+ p->nr_cpus_allowed = cpumask_weight(ctx->new_mask);
}
static void
-__do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32 flags)
+__do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx)
{
struct rq *rq = task_rq(p);
bool queued, running;
@@ -2558,7 +2566,7 @@ __do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32
*
* XXX do further audits, this smells like something putrid.
*/
- if (flags & SCA_MIGRATE_DISABLE)
+ if (ctx->flags & SCA_MIGRATE_DISABLE)
SCHED_WARN_ON(!p->on_cpu);
else
lockdep_assert_held(&p->pi_lock);
@@ -2577,7 +2585,7 @@ __do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32
if (running)
put_prev_task(rq, p);
- p->sched_class->set_cpus_allowed(p, new_mask, flags);
+ p->sched_class->set_cpus_allowed(p, ctx);
if (queued)
enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
@@ -2587,7 +2595,12 @@ __do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32
void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
{
- __do_set_cpus_allowed(p, new_mask, 0);
+ struct affinity_context ac = {
+ .new_mask = new_mask,
+ .flags = 0,
+ };
+
+ __do_set_cpus_allowed(p, &ac);
}
int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src,
@@ -2840,8 +2853,7 @@ static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flag
* Called with both p->pi_lock and rq->lock held; drops both before returning.
*/
static int __set_cpus_allowed_ptr_locked(struct task_struct *p,
- const struct cpumask *new_mask,
- u32 flags,
+ struct affinity_context *ctx,
struct rq *rq,
struct rq_flags *rf)
__releases(rq->lock)
@@ -2869,7 +2881,7 @@ static int __set_cpus_allowed_ptr_locked(struct task_struct *p,
cpu_valid_mask = cpu_online_mask;
}
- if (!kthread && !cpumask_subset(new_mask, cpu_allowed_mask)) {
+ if (!kthread && !cpumask_subset(ctx->new_mask, cpu_allowed_mask)) {
ret = -EINVAL;
goto out;
}
@@ -2878,18 +2890,18 @@ static int __set_cpus_allowed_ptr_locked(struct task_struct *p,
* Must re-check here, to close a race against __kthread_bind(),
* sched_setaffinity() is not guaranteed to observe the flag.
*/
- if ((flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) {
+ if ((ctx->flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) {
ret = -EINVAL;
goto out;
}
- if (!(flags & SCA_MIGRATE_ENABLE)) {
- if (cpumask_equal(&p->cpus_mask, new_mask))
+ if (!(ctx->flags & SCA_MIGRATE_ENABLE)) {
+ if (cpumask_equal(&p->cpus_mask, ctx->new_mask))
goto out;
if (WARN_ON_ONCE(p == current &&
is_migration_disabled(p) &&
- !cpumask_test_cpu(task_cpu(p), new_mask))) {
+ !cpumask_test_cpu(task_cpu(p), ctx->new_mask))) {
ret = -EBUSY;
goto out;
}
@@ -2900,15 +2912,15 @@ static int __set_cpus_allowed_ptr_locked(struct task_struct *p,
* for groups of tasks (ie. cpuset), so that load balancing is not
* immediately required to distribute the tasks within their new mask.
*/
- dest_cpu = cpumask_any_and_distribute(cpu_valid_mask, new_mask);
+ dest_cpu = cpumask_any_and_distribute(cpu_valid_mask, ctx->new_mask);
if (dest_cpu >= nr_cpu_ids) {
ret = -EINVAL;
goto out;
}
- __do_set_cpus_allowed(p, new_mask, flags);
+ __do_set_cpus_allowed(p, ctx);
- return affine_move_task(rq, p, rf, dest_cpu, flags);
+ return affine_move_task(rq, p, rf, dest_cpu, ctx->flags);
out:
task_rq_unlock(rq, p, rf);
@@ -2926,22 +2938,27 @@ static int __set_cpus_allowed_ptr_locked(struct task_struct *p,
* call is not atomic; no spinlocks may be held.
*/
static int __set_cpus_allowed_ptr(struct task_struct *p,
- const struct cpumask *new_mask, u32 flags)
+ struct affinity_context *ctx)
{
struct rq_flags rf;
struct rq *rq;
rq = task_rq_lock(p, &rf);
- if (p->user_cpus_ptr && !(flags & SCA_USER) &&
- cpumask_and(rq->scratch_mask, new_mask, p->user_cpus_ptr))
- new_mask = rq->scratch_mask;
+ if (p->user_cpus_ptr && !(ctx->flags & SCA_USER) &&
+ cpumask_and(rq->scratch_mask, ctx->new_mask, p->user_cpus_ptr))
+ ctx->new_mask = rq->scratch_mask;
- return __set_cpus_allowed_ptr_locked(p, new_mask, flags, rq, &rf);
+ return __set_cpus_allowed_ptr_locked(p, ctx, rq, &rf);
}
int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
{
- return __set_cpus_allowed_ptr(p, new_mask, 0);
+ struct affinity_context ac = {
+ .new_mask = new_mask,
+ .flags = 0,
+ };
+
+ return __set_cpus_allowed_ptr(p, &ac);
}
EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
@@ -2958,6 +2975,10 @@ static int restrict_cpus_allowed_ptr(struct task_struct *p,
struct cpumask *new_mask,
const struct cpumask *subset_mask)
{
+ struct affinity_context ac = {
+ .new_mask = new_mask,
+ .flags = 0,
+ };
struct rq_flags rf;
struct rq *rq;
int err;
@@ -2979,7 +3000,7 @@ static int restrict_cpus_allowed_ptr(struct task_struct *p,
goto err_unlock;
}
- return __set_cpus_allowed_ptr_locked(p, new_mask, 0, rq, &rf);
+ return __set_cpus_allowed_ptr_locked(p, &ac, rq, &rf);
err_unlock:
task_rq_unlock(rq, p, &rf);
@@ -3032,7 +3053,7 @@ void force_compatible_cpus_allowed_ptr(struct task_struct *p)
}
static int
-__sched_setaffinity(struct task_struct *p, const struct cpumask *mask, int flags);
+__sched_setaffinity(struct task_struct *p, struct affinity_context *ctx);
/*
* Restore the affinity of a task @p which was previously restricted by a
@@ -3043,13 +3064,17 @@ __sched_setaffinity(struct task_struct *p, const struct cpumask *mask, int flags
*/
void relax_compatible_cpus_allowed_ptr(struct task_struct *p)
{
+ struct affinity_context ac = {
+ .new_mask = task_user_cpus(p),
+ .flags = 0,
+ };
int ret;
/*
* Try to restore the old affinity mask with __sched_setaffinity().
* Cpuset masking will be done there too.
*/
- ret = __sched_setaffinity(p, task_user_cpus(p), 0);
+ ret = __sched_setaffinity(p, &ac);
WARN_ON_ONCE(ret);
}
@@ -8053,7 +8078,7 @@ int dl_task_check_affinity(struct task_struct *p, const struct cpumask *mask)
#endif
static int
-__sched_setaffinity(struct task_struct *p, const struct cpumask *mask, int flags)
+__sched_setaffinity(struct task_struct *p, struct affinity_context *ctx)
{
int retval;
cpumask_var_t cpus_allowed, new_mask;
@@ -8067,13 +8092,16 @@ __sched_setaffinity(struct task_struct *p, const struct cpumask *mask, int flags
}
cpuset_cpus_allowed(p, cpus_allowed);
- cpumask_and(new_mask, mask, cpus_allowed);
+ cpumask_and(new_mask, ctx->new_mask, cpus_allowed);
+
+ ctx->new_mask = new_mask;
+ ctx->flags |= SCA_CHECK;
retval = dl_task_check_affinity(p, new_mask);
if (retval)
goto out_free_new_mask;
again:
- retval = __set_cpus_allowed_ptr(p, new_mask, SCA_CHECK | flags);
+ retval = __set_cpus_allowed_ptr(p, ctx);
if (retval)
goto out_free_new_mask;
@@ -8096,6 +8124,7 @@ __sched_setaffinity(struct task_struct *p, const struct cpumask *mask, int flags
long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
{
+ struct affinity_context ac;
struct cpumask *user_mask;
struct task_struct *p;
int retval;
@@ -8137,8 +8166,12 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
goto out_put_task;
}
cpumask_copy(user_mask, in_mask);
+ ac = (struct affinity_context){
+ .new_mask = in_mask,
+ .flags = SCA_USER,
+ };
- retval = __sched_setaffinity(p, in_mask, SCA_USER);
+ retval = __sched_setaffinity(p, &ac);
/*
* Save in_mask into user_cpus_ptr after a successful
@@ -8936,6 +8969,7 @@ void show_state_filter(unsigned int state_filter)
void __init init_idle(struct task_struct *idle, int cpu)
{
struct rq *rq = cpu_rq(cpu);
+ struct affinity_context ac;
unsigned long flags;
__sched_fork(0, idle);
@@ -8953,13 +8987,17 @@ void __init init_idle(struct task_struct *idle, int cpu)
kthread_set_per_cpu(idle, cpu);
#ifdef CONFIG_SMP
+ ac = (struct affinity_context) {
+ .new_mask = cpumask_of(cpu),
+ .flags = 0,
+ };
/*
* It's possible that init_idle() gets called multiple times on a task,
* in that case do_set_cpus_allowed() will not do the right thing.
*
* And since this is boot we can forgo the serialization.
*/
- set_cpus_allowed_common(idle, cpumask_of(cpu), 0);
+ set_cpus_allowed_common(idle, &ac);
#endif
/*
* We're having a chicken and egg problem, even though we are
diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 0ab79d819a0d..38fa2c3ef7db 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -2486,8 +2486,7 @@ static void task_woken_dl(struct rq *rq, struct task_struct *p)
}
static void set_cpus_allowed_dl(struct task_struct *p,
- const struct cpumask *new_mask,
- u32 flags)
+ struct affinity_context *ctx)
{
struct root_domain *src_rd;
struct rq *rq;
@@ -2502,7 +2501,7 @@ static void set_cpus_allowed_dl(struct task_struct *p,
* update. We already made space for us in the destination
* domain (see cpuset_can_attach()).
*/
- if (!cpumask_intersects(src_rd->span, new_mask)) {
+ if (!cpumask_intersects(src_rd->span, ctx->new_mask)) {
struct dl_bw *src_dl_b;
src_dl_b = dl_bw_of(cpu_of(rq));
@@ -2516,7 +2515,7 @@ static void set_cpus_allowed_dl(struct task_struct *p,
raw_spin_unlock(&src_dl_b->lock);
}
- set_cpus_allowed_common(p, new_mask, flags);
+ set_cpus_allowed_common(p, ctx);
}
/* Assumes rq->lock is held */
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 482b702d65ea..1927c02f68fa 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2157,6 +2157,11 @@ extern const u32 sched_prio_to_wmult[40];
#define RETRY_TASK ((void *)-1UL)
+struct affinity_context {
+ const struct cpumask *new_mask;
+ unsigned int flags;
+};
+
struct sched_class {
#ifdef CONFIG_UCLAMP_TASK
@@ -2185,9 +2190,7 @@ struct sched_class {
void (*task_woken)(struct rq *this_rq, struct task_struct *task);
- void (*set_cpus_allowed)(struct task_struct *p,
- const struct cpumask *newmask,
- u32 flags);
+ void (*set_cpus_allowed)(struct task_struct *p, struct affinity_context *ctx);
void (*rq_online)(struct rq *rq);
void (*rq_offline)(struct rq *rq);
@@ -2301,7 +2304,7 @@ extern void update_group_capacity(struct sched_domain *sd, int cpu);
extern void trigger_load_balance(struct rq *rq);
-extern void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask, u32 flags);
+extern void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx);
static inline struct task_struct *get_push_task(struct rq *rq)
{
--
2.31.1Hi Waiman,
I love your patch! Yet something to improve:
[auto build test ERROR on linux/master]
[also build test ERROR on linus/master v6.0-rc4]
[cannot apply to tip/sched/core tip/master next-20220908]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Waiman-Long/sched-Persistent-user-requested-affinity/20220909-034411
base: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git 506357871c18e06565840d71c2ef9f818e19f460
config: arm-randconfig-r033-20220907 (https://download.01.org/0day-ci/archive/20220909/202209090937.23d2iFjb-lkp@intel.com/config)
compiler: clang version 16.0.0 (https://github.com/llvm/llvm-project 1546df49f5a6d09df78f569e4137ddb365a3e827)
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# install arm cross compiling tool for clang build
# apt-get install binutils-arm-linux-gnueabi
# https://github.com/intel-lab-lkp/linux/commit/039ce835b17cfa35f47f3cb5fd4938e57643cb63
git remote add linux-review https://github.com/intel-lab-lkp/linux
git fetch --no-tags linux-review Waiman-Long/sched-Persistent-user-requested-affinity/20220909-034411
git checkout 039ce835b17cfa35f47f3cb5fd4938e57643cb63
# save the config file
mkdir build_dir && cp config build_dir/.config
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=arm SHELL=/bin/bash kernel/sched/
If you fix the issue, kindly add following tag where applicable
Reported-by: kernel test robot <lkp@intel.com>
All errors (new ones prefixed by >>):
kernel/sched/core.c:6602:35: warning: no previous prototype for function 'schedule_user' [-Wmissing-prototypes]
asmlinkage __visible void __sched schedule_user(void)
^
kernel/sched/core.c:6602:22: note: declare 'static' if the function is not intended to be used outside of this translation unit
asmlinkage __visible void __sched schedule_user(void)
^
static
>> kernel/sched/core.c:8104:40: error: too few arguments to function call, expected 3, have 2
retval = __set_cpus_allowed_ptr(p, ctx);
~~~~~~~~~~~~~~~~~~~~~~ ^
kernel/sched/core.c:3555:19: note: '__set_cpus_allowed_ptr' declared here
static inline int __set_cpus_allowed_ptr(struct task_struct *p,
^
kernel/sched/core.c:8972:26: warning: unused variable 'ac' [-Wunused-variable]
struct affinity_context ac;
^
2 warnings and 1 error generated.
vim +8104 kernel/sched/core.c
8079
8080 static int
8081 __sched_setaffinity(struct task_struct *p, struct affinity_context *ctx)
8082 {
8083 int retval;
8084 cpumask_var_t cpus_allowed, new_mask;
8085
8086 if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL))
8087 return -ENOMEM;
8088
8089 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
8090 retval = -ENOMEM;
8091 goto out_free_cpus_allowed;
8092 }
8093
8094 cpuset_cpus_allowed(p, cpus_allowed);
8095 cpumask_and(new_mask, ctx->new_mask, cpus_allowed);
8096
8097 ctx->new_mask = new_mask;
8098 ctx->flags |= SCA_CHECK;
8099
8100 retval = dl_task_check_affinity(p, new_mask);
8101 if (retval)
8102 goto out_free_new_mask;
8103 again:
> 8104 retval = __set_cpus_allowed_ptr(p, ctx);
8105 if (retval)
8106 goto out_free_new_mask;
8107
8108 cpuset_cpus_allowed(p, cpus_allowed);
8109 if (!cpumask_subset(new_mask, cpus_allowed)) {
8110 /*
8111 * We must have raced with a concurrent cpuset update.
8112 * Just reset the cpumask to the cpuset's cpus_allowed.
8113 */
8114 cpumask_copy(new_mask, cpus_allowed);
8115 goto again;
8116 }
8117
8118 out_free_new_mask:
8119 free_cpumask_var(new_mask);
8120 out_free_cpus_allowed:
8121 free_cpumask_var(cpus_allowed);
8122 return retval;
8123 }
8124
--
0-DAY CI Kernel Test Service
https://01.org/lkp
On 9/8/22 15:41, Waiman Long wrote: > Introduce a new affinity_context structure for passing cpu affinity information > around in core scheduler code. The relevant functions are modified to use > the new structure. There is no functional change. > > Suggested-by: Peter Zijlstra <peterz@infradead.org> > Signed-off-by: Waiman Long <longman@redhat.com> Sorry, this patch has build problem with non-SMP configs as reported by kernel test robot. Will fix that in the next version. Cheers, Longman
© 2016 - 2026 Red Hat, Inc.