In preparation for storing two separate iowait states in there, bump the
size from a 32-bit to a 64-bit size, for 64-bit kernels.

Note that on 32-bit, the number of tasks are limited to 0x8000, which
fits just fine in even half of the existiing 32-bit atomic_t. For 64-bit,
no such limit exists, hence play it safe and make it a 64-bit atomic.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
 kernel/sched/core.c  | 14 +++++++++++++-
 kernel/sched/sched.h |  4 ++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 9bf1b67818d0..ddabf20cd9e0 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -3543,17 +3543,29 @@ static inline bool rq_has_pinned_tasks(struct rq *rq)
 
 static void task_iowait_inc(struct task_struct *p)
 {
+#ifdef CONFIG_64BIT
+	atomic_long_inc(&task_rq(p)->nr_iowait);
+#else
 	atomic_inc(&task_rq(p)->nr_iowait);
+#endif
 }
 
 static void task_iowait_dec(struct task_struct *p)
 {
+#ifdef CONFIG_64BIT
+	atomic_long_dec(&task_rq(p)->nr_iowait);
+#else
 	atomic_dec(&task_rq(p)->nr_iowait);
+#endif
 }
 
 int rq_iowait(struct rq *rq)
 {
+#ifdef CONFIG_64BIT
+	return atomic_long_read(&rq->nr_iowait);
+#else
 	return atomic_read(&rq->nr_iowait);
+#endif
 }
 
 static void
@@ -8372,7 +8384,7 @@ void __init sched_init(void)
 #endif
 #endif /* CONFIG_SMP */
 		hrtick_rq_init(rq);
-		atomic_set(&rq->nr_iowait, 0);
+		atomic_long_set(&rq->nr_iowait, 0);
 		fair_server_init(rq);
 
 #ifdef CONFIG_SCHED_CORE
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index b6b3b565bcb1..6a90c2da1eb3 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1082,7 +1082,11 @@ struct rq {
 	u64			clock_idle_copy;
 #endif
 
+#ifdef CONFIG_64BIT
+	atomic_long_t		nr_iowait;
+#else
 	atomic_t		nr_iowait;
+#endif
 
 #ifdef CONFIG_SCHED_DEBUG
 	u64 last_seen_need_resched_ns;
-- 
2.43.0

hi,

在 2024/8/18 4:45, Jens Axboe 写道:
> In preparation for storing two separate iowait states in there, bump the
> size from a 32-bit to a 64-bit size, for 64-bit kernels.
> 
> Note that on 32-bit, the number of tasks are limited to 0x8000, which
> fits just fine in even half of the existiing 32-bit atomic_t. For 64-bit,
> no such limit exists, hence play it safe and make it a 64-bit atomic.
> 
> Signed-off-by: Jens Axboe <axboe@kernel.dk>
> ---
>  kernel/sched/core.c  | 14 +++++++++++++-
>  kernel/sched/sched.h |  4 ++++
>  2 files changed, 17 insertions(+), 1 deletion(-)
> 
> diff --git a/kernel/sched/core.c b/kernel/sched/core.c
> index 9bf1b67818d0..ddabf20cd9e0 100644
> --- a/kernel/sched/core.c
> +++ b/kernel/sched/core.c
> @@ -3543,17 +3543,29 @@ static inline bool rq_has_pinned_tasks(struct rq *rq)
>  
>  static void task_iowait_inc(struct task_struct *p)
>  {
> +#ifdef CONFIG_64BIT
> +	atomic_long_inc(&task_rq(p)->nr_iowait);
> +#else
>  	atomic_inc(&task_rq(p)->nr_iowait);
> +#endif>  }
>  
>  static void task_iowait_dec(struct task_struct *p)
>  {
> +#ifdef CONFIG_64BIT
> +	atomic_long_dec(&task_rq(p)->nr_iowait);
> +#else
>  	atomic_dec(&task_rq(p)->nr_iowait);
> +#endif
>  }
>  
>  int rq_iowait(struct rq *rq)
>  {
> +#ifdef CONFIG_64BIT
> +	return atomic_long_read(&rq->nr_iowait);
> +#else
>  	return atomic_read(&rq->nr_iowait);
> +#endif
>  }
>  
>  static void
> @@ -8372,7 +8384,7 @@ void __init sched_init(void)
>  #endif
>  #endif /* CONFIG_SMP */
>  		hrtick_rq_init(rq);
> -		atomic_set(&rq->nr_iowait, 0);
> +		atomic_long_set(&rq->nr_iowait, 0);
>  		fair_server_init(rq);
>  
>  #ifdef CONFIG_SCHED_CORE
> diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
> index b6b3b565bcb1..6a90c2da1eb3 100644
> --- a/kernel/sched/sched.h
> +++ b/kernel/sched/sched.h
> @@ -1082,7 +1082,11 @@ struct rq {
>  	u64			clock_idle_copy;
>  #endif
>  
> +#ifdef CONFIG_64BIT
> +	atomic_long_t		nr_iowait;
> +#else
>  	atomic_t		nr_iowait;
> +#endif

When CONFIG_64BIT is not set, the type atomic_long_t as a synonym of the type atomic_t.
Can you directly use atomic_long_t, is right?

Thanks.
Zhang Qiao.

>  
>  #ifdef CONFIG_SCHED_DEBUG
>  	u64 last_seen_need_resched_ns;

On 8/19/24 12:30 AM, Zhang Qiao wrote:
>> diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
>> index b6b3b565bcb1..6a90c2da1eb3 100644
>> --- a/kernel/sched/sched.h
>> +++ b/kernel/sched/sched.h
>> @@ -1082,7 +1082,11 @@ struct rq {
>>  	u64			clock_idle_copy;
>>  #endif
>>  
>> +#ifdef CONFIG_64BIT
>> +	atomic_long_t		nr_iowait;
>> +#else
>>  	atomic_t		nr_iowait;
>> +#endif
> 
> When CONFIG_64BIT is not set, the type atomic_long_t as a synonym of
> the type atomic_t. Can you directly use atomic_long_t, is right?

Ah indeed, thanks! Yeah we can just use atomic_long_t unconditionally,
and that'd kill the ifdefs in sched/core/core.c as well as we can just
define the shift to be half the atomic_long_t size, 4 *
sizeof(atomic_long_t).

I'll respin with that.

-- 
Jens Axboe