include/linux/debugobjects.h | 12 +- lib/debugobjects.c | 849 ++++++++++++++++++++++++------------------- 2 files changed, 486 insertions(+), 375 deletions(-)
Linus,
please pull the latest core/debugobjects branch from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git core-debugobjects-2024-11-18
up to: ff8d523cc452: debugobjects: Track object usage to avoid premature freeing of objects
A set of changes for debugobjects:
- Prevent destroying the kmem_cache on early failure.
Destroying a kmem_cache requires work queues to be set up, but in the
early failure case they are not yet initializated. So rather leak the
cache instead of triggering a BUG.
- Reduce parallel pool fill attempts.
Refilling the object pool requires to take the global pool lock, which
causes a massive performance issue when a large number of CPUs attempt
to refill concurrently. It turns out that it's sufficient to let one
CPU handle the refill from the to free list and in case there are not
enough objects on it to allocate new objects from the kmem cache.
This also splits the free list handling from the actual allocation path
as that yields better results on RT where allocation is restricted to
preemptible code paths. The refill from free list has no such
restrictions.
- Consolidate the global and the per CPU pools to use the same data
structure, so all helper functions can be shared.
- Simplify the object allocation/free logic.
The allocation/free logic is an incomprehensible maze, which tries to
utilize the to free list and the global pool in the best way. This all
can be simplified into a straight forward comprehensible code flow.
- Convert the allocation/free mechanism to batch mode.
Transferring objects from the global pool to the per CPU pools or vice
versa is done by walking the hlist and moving object by object. That
not only increases the pool lock held time, it also dirties up to 17
cache lines.
This can be avoided by storing the pointer to the first object in a
batch of 16 objects in the objects themself and propagate it through
the batch when an object is enqueued into a pool or to a temporary
hlist head on allocation.
This allows to move batches of objects with at max four cache lines
dirtied and reduces the pool lock held time and therefore contention
significantly.
- Improve the object reusage
The current implementation is too agressively freeing unused objects,
which is counterproductive on bursty workloads like a kernel compile.
Address this by:
* increasing the per CPU pool size
* refilling the per CPU pool from the to be freed pool when the per
CPU pool emptied a batch
* keeping track of object usage with a exponentially wheighted
moving average which prevents the work queue callback to free
objects prematuraly.
This combined reduces the allocation/free rate for a full kernel
compile significantly:
kmem_cache_alloc() kmem_cache_free()
Baseline: 380k 330k
Improved: 170k 117k
- A few cleanups and a more cache line friendly layout of debug
information on top.
Thanks,
tglx
------------------>
Thomas Gleixner (22):
debugobjects: Dont destroy kmem cache in init()
debugobjects: Remove pointless hlist initialization
debugobjects: Dont free objects directly on CPU hotplug
debugobjects: Reuse put_objects() on OOM
debugobjects: Remove pointless debug printk
debugobjects: Provide and use free_object_list()
debugobjects: Make debug_objects_enabled bool
debugobjects: Move pools into a datastructure
debugobjects: Use separate list head for boot pool
debugobjects: Rename and tidy up per CPU pools
debugobjects: Move min/max count into pool struct
debugobjects: Rework object allocation
debugobjects: Rework object freeing
debugobjects: Rework free_object_work()
debugobjects: Use static key for boot pool selection
debugobjects: Prepare for batching
debugobjects: Prepare kmem_cache allocations for batching
debugobjects: Implement batch processing
debugobjects: Move pool statistics into global_pool struct
debugobjects: Double the per CPU slots
debugobjects: Refill per CPU pool more agressively
debugobjects: Track object usage to avoid premature freeing of objects
Zhen Lei (3):
debugobjects: Delete a piece of redundant code
debugobjects: Collect newly allocated objects in a list to reduce lock contention
debugobjects: Reduce parallel pool fill attempts
include/linux/debugobjects.h | 12 +-
lib/debugobjects.c | 849 ++++++++++++++++++++++++-------------------
2 files changed, 486 insertions(+), 375 deletions(-)
diff --git a/include/linux/debugobjects.h b/include/linux/debugobjects.h
index 32444686b6ff..8b95545e7924 100644
--- a/include/linux/debugobjects.h
+++ b/include/linux/debugobjects.h
@@ -23,13 +23,17 @@ struct debug_obj_descr;
* @state: tracked object state
* @astate: current active state
* @object: pointer to the real object
+ * @batch_last: pointer to the last hlist node in a batch
* @descr: pointer to an object type specific debug description structure
*/
struct debug_obj {
- struct hlist_node node;
- enum debug_obj_state state;
- unsigned int astate;
- void *object;
+ struct hlist_node node;
+ enum debug_obj_state state;
+ unsigned int astate;
+ union {
+ void *object;
+ struct hlist_node *batch_last;
+ };
const struct debug_obj_descr *descr;
};
diff --git a/lib/debugobjects.c b/lib/debugobjects.c
index 5ce473ad499b..7f50c4480a4e 100644
--- a/lib/debugobjects.c
+++ b/lib/debugobjects.c
@@ -7,25 +7,30 @@
#define pr_fmt(fmt) "ODEBUG: " fmt
+#include <linux/cpu.h>
#include <linux/debugobjects.h>
-#include <linux/interrupt.h>
+#include <linux/debugfs.h>
+#include <linux/hash.h>
+#include <linux/kmemleak.h>
#include <linux/sched.h>
+#include <linux/sched/loadavg.h>
#include <linux/sched/task_stack.h>
#include <linux/seq_file.h>
-#include <linux/debugfs.h>
#include <linux/slab.h>
-#include <linux/hash.h>
-#include <linux/kmemleak.h>
-#include <linux/cpu.h>
+#include <linux/static_key.h>
#define ODEBUG_HASH_BITS 14
#define ODEBUG_HASH_SIZE (1 << ODEBUG_HASH_BITS)
-#define ODEBUG_POOL_SIZE 1024
-#define ODEBUG_POOL_MIN_LEVEL 256
-#define ODEBUG_POOL_PERCPU_SIZE 64
+/* Must be power of two */
#define ODEBUG_BATCH_SIZE 16
+/* Initial values. Must all be a multiple of batch size */
+#define ODEBUG_POOL_SIZE (64 * ODEBUG_BATCH_SIZE)
+#define ODEBUG_POOL_MIN_LEVEL (ODEBUG_POOL_SIZE / 4)
+
+#define ODEBUG_POOL_PERCPU_SIZE (8 * ODEBUG_BATCH_SIZE)
+
#define ODEBUG_CHUNK_SHIFT PAGE_SHIFT
#define ODEBUG_CHUNK_SIZE (1 << ODEBUG_CHUNK_SHIFT)
#define ODEBUG_CHUNK_MASK (~(ODEBUG_CHUNK_SIZE - 1))
@@ -35,7 +40,7 @@
* frequency of 10Hz and about 1024 objects for each freeing operation.
* So it is freeing at most 10k debug objects per second.
*/
-#define ODEBUG_FREE_WORK_MAX 1024
+#define ODEBUG_FREE_WORK_MAX (1024 / ODEBUG_BATCH_SIZE)
#define ODEBUG_FREE_WORK_DELAY DIV_ROUND_UP(HZ, 10)
struct debug_bucket {
@@ -43,16 +48,24 @@ struct debug_bucket {
raw_spinlock_t lock;
};
-/*
- * Debug object percpu free list
- * Access is protected by disabling irq
- */
-struct debug_percpu_free {
- struct hlist_head free_objs;
- int obj_free;
+struct pool_stats {
+ unsigned int cur_used;
+ unsigned int max_used;
+ unsigned int min_fill;
};
-static DEFINE_PER_CPU(struct debug_percpu_free, percpu_obj_pool);
+struct obj_pool {
+ struct hlist_head objects;
+ unsigned int cnt;
+ unsigned int min_cnt;
+ unsigned int max_cnt;
+ struct pool_stats stats;
+} ____cacheline_aligned;
+
+
+static DEFINE_PER_CPU_ALIGNED(struct obj_pool, pool_pcpu) = {
+ .max_cnt = ODEBUG_POOL_PERCPU_SIZE,
+};
static struct debug_bucket obj_hash[ODEBUG_HASH_SIZE];
@@ -60,37 +73,32 @@ static struct debug_obj obj_static_pool[ODEBUG_POOL_SIZE] __initdata;
static DEFINE_RAW_SPINLOCK(pool_lock);
-static HLIST_HEAD(obj_pool);
-static HLIST_HEAD(obj_to_free);
+static struct obj_pool pool_global = {
+ .min_cnt = ODEBUG_POOL_MIN_LEVEL,
+ .max_cnt = ODEBUG_POOL_SIZE,
+ .stats = {
+ .min_fill = ODEBUG_POOL_SIZE,
+ },
+};
-/*
- * Because of the presence of percpu free pools, obj_pool_free will
- * under-count those in the percpu free pools. Similarly, obj_pool_used
- * will over-count those in the percpu free pools. Adjustments will be
- * made at debug_stats_show(). Both obj_pool_min_free and obj_pool_max_used
- * can be off.
- */
-static int __data_racy obj_pool_min_free = ODEBUG_POOL_SIZE;
-static int __data_racy obj_pool_free = ODEBUG_POOL_SIZE;
-static int obj_pool_used;
-static int __data_racy obj_pool_max_used;
+static struct obj_pool pool_to_free = {
+ .max_cnt = UINT_MAX,
+};
+
+static HLIST_HEAD(pool_boot);
+
+static unsigned long avg_usage;
static bool obj_freeing;
-/* The number of objs on the global free list */
-static int obj_nr_tofree;
static int __data_racy debug_objects_maxchain __read_mostly;
static int __data_racy __maybe_unused debug_objects_maxchecked __read_mostly;
static int __data_racy debug_objects_fixups __read_mostly;
static int __data_racy debug_objects_warnings __read_mostly;
-static int __data_racy debug_objects_enabled __read_mostly
+static bool __data_racy debug_objects_enabled __read_mostly
= CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT;
-static int debug_objects_pool_size __ro_after_init
- = ODEBUG_POOL_SIZE;
-static int debug_objects_pool_min_level __ro_after_init
- = ODEBUG_POOL_MIN_LEVEL;
-static const struct debug_obj_descr *descr_test __read_mostly;
-static struct kmem_cache *obj_cache __ro_after_init;
+static const struct debug_obj_descr *descr_test __read_mostly;
+static struct kmem_cache *obj_cache __ro_after_init;
/*
* Track numbers of kmem_cache_alloc()/free() calls done.
@@ -101,19 +109,20 @@ static int __data_racy debug_objects_freed;
static void free_obj_work(struct work_struct *work);
static DECLARE_DELAYED_WORK(debug_obj_work, free_obj_work);
+static DEFINE_STATIC_KEY_FALSE(obj_cache_enabled);
+
static int __init enable_object_debug(char *str)
{
- debug_objects_enabled = 1;
+ debug_objects_enabled = true;
return 0;
}
+early_param("debug_objects", enable_object_debug);
static int __init disable_object_debug(char *str)
{
- debug_objects_enabled = 0;
+ debug_objects_enabled = false;
return 0;
}
-
-early_param("debug_objects", enable_object_debug);
early_param("no_debug_objects", disable_object_debug);
static const char *obj_states[ODEBUG_STATE_MAX] = {
@@ -125,61 +134,280 @@ static const char *obj_states[ODEBUG_STATE_MAX] = {
[ODEBUG_STATE_NOTAVAILABLE] = "not available",
};
-static void fill_pool(void)
+static __always_inline unsigned int pool_count(struct obj_pool *pool)
+{
+ return READ_ONCE(pool->cnt);
+}
+
+static __always_inline bool pool_should_refill(struct obj_pool *pool)
+{
+ return pool_count(pool) < pool->min_cnt;
+}
+
+static __always_inline bool pool_must_refill(struct obj_pool *pool)
+{
+ return pool_count(pool) < pool->min_cnt / 2;
+}
+
+static bool pool_move_batch(struct obj_pool *dst, struct obj_pool *src)
{
- gfp_t gfp = __GFP_HIGH | __GFP_NOWARN;
+ struct hlist_node *last, *next_batch, *first_batch;
struct debug_obj *obj;
- unsigned long flags;
- if (likely(READ_ONCE(obj_pool_free) >= debug_objects_pool_min_level))
+ if (dst->cnt >= dst->max_cnt || !src->cnt)
+ return false;
+
+ first_batch = src->objects.first;
+ obj = hlist_entry(first_batch, typeof(*obj), node);
+ last = obj->batch_last;
+ next_batch = last->next;
+
+ /* Move the next batch to the front of the source pool */
+ src->objects.first = next_batch;
+ if (next_batch)
+ next_batch->pprev = &src->objects.first;
+
+ /* Add the extracted batch to the destination pool */
+ last->next = dst->objects.first;
+ if (last->next)
+ last->next->pprev = &last->next;
+ first_batch->pprev = &dst->objects.first;
+ dst->objects.first = first_batch;
+
+ WRITE_ONCE(src->cnt, src->cnt - ODEBUG_BATCH_SIZE);
+ WRITE_ONCE(dst->cnt, dst->cnt + ODEBUG_BATCH_SIZE);
+ return true;
+}
+
+static bool pool_push_batch(struct obj_pool *dst, struct hlist_head *head)
+{
+ struct hlist_node *last;
+ struct debug_obj *obj;
+
+ if (dst->cnt >= dst->max_cnt)
+ return false;
+
+ obj = hlist_entry(head->first, typeof(*obj), node);
+ last = obj->batch_last;
+
+ hlist_splice_init(head, last, &dst->objects);
+ WRITE_ONCE(dst->cnt, dst->cnt + ODEBUG_BATCH_SIZE);
+ return true;
+}
+
+static bool pool_pop_batch(struct hlist_head *head, struct obj_pool *src)
+{
+ struct hlist_node *last, *next;
+ struct debug_obj *obj;
+
+ if (!src->cnt)
+ return false;
+
+ /* Move the complete list to the head */
+ hlist_move_list(&src->objects, head);
+
+ obj = hlist_entry(head->first, typeof(*obj), node);
+ last = obj->batch_last;
+ next = last->next;
+ /* Disconnect the batch from the list */
+ last->next = NULL;
+
+ /* Move the node after last back to the source pool. */
+ src->objects.first = next;
+ if (next)
+ next->pprev = &src->objects.first;
+
+ WRITE_ONCE(src->cnt, src->cnt - ODEBUG_BATCH_SIZE);
+ return true;
+}
+
+static struct debug_obj *__alloc_object(struct hlist_head *list)
+{
+ struct debug_obj *obj;
+
+ if (unlikely(!list->first))
+ return NULL;
+
+ obj = hlist_entry(list->first, typeof(*obj), node);
+ hlist_del(&obj->node);
+ return obj;
+}
+
+static void pcpu_refill_stats(void)
+{
+ struct pool_stats *stats = &pool_global.stats;
+
+ WRITE_ONCE(stats->cur_used, stats->cur_used + ODEBUG_BATCH_SIZE);
+
+ if (stats->cur_used > stats->max_used)
+ stats->max_used = stats->cur_used;
+
+ if (pool_global.cnt < stats->min_fill)
+ stats->min_fill = pool_global.cnt;
+}
+
+static struct debug_obj *pcpu_alloc(void)
+{
+ struct obj_pool *pcp = this_cpu_ptr(&pool_pcpu);
+
+ lockdep_assert_irqs_disabled();
+
+ for (;;) {
+ struct debug_obj *obj = __alloc_object(&pcp->objects);
+
+ if (likely(obj)) {
+ pcp->cnt--;
+ /*
+ * If this emptied a batch try to refill from the
+ * free pool. Don't do that if this was the top-most
+ * batch as pcpu_free() expects the per CPU pool
+ * to be less than ODEBUG_POOL_PERCPU_SIZE.
+ */
+ if (unlikely(pcp->cnt < (ODEBUG_POOL_PERCPU_SIZE - ODEBUG_BATCH_SIZE) &&
+ !(pcp->cnt % ODEBUG_BATCH_SIZE))) {
+ /*
+ * Don't try to allocate from the regular pool here
+ * to not exhaust it prematurely.
+ */
+ if (pool_count(&pool_to_free)) {
+ guard(raw_spinlock)(&pool_lock);
+ pool_move_batch(pcp, &pool_to_free);
+ pcpu_refill_stats();
+ }
+ }
+ return obj;
+ }
+
+ guard(raw_spinlock)(&pool_lock);
+ if (!pool_move_batch(pcp, &pool_to_free)) {
+ if (!pool_move_batch(pcp, &pool_global))
+ return NULL;
+ }
+ pcpu_refill_stats();
+ }
+}
+
+static void pcpu_free(struct debug_obj *obj)
+{
+ struct obj_pool *pcp = this_cpu_ptr(&pool_pcpu);
+ struct debug_obj *first;
+
+ lockdep_assert_irqs_disabled();
+
+ if (!(pcp->cnt % ODEBUG_BATCH_SIZE)) {
+ obj->batch_last = &obj->node;
+ } else {
+ first = hlist_entry(pcp->objects.first, typeof(*first), node);
+ obj->batch_last = first->batch_last;
+ }
+ hlist_add_head(&obj->node, &pcp->objects);
+ pcp->cnt++;
+
+ /* Pool full ? */
+ if (pcp->cnt < ODEBUG_POOL_PERCPU_SIZE)
return;
+ /* Remove a batch from the per CPU pool */
+ guard(raw_spinlock)(&pool_lock);
+ /* Try to fit the batch into the pool_global first */
+ if (!pool_move_batch(&pool_global, pcp))
+ pool_move_batch(&pool_to_free, pcp);
+ WRITE_ONCE(pool_global.stats.cur_used, pool_global.stats.cur_used - ODEBUG_BATCH_SIZE);
+}
+
+static void free_object_list(struct hlist_head *head)
+{
+ struct hlist_node *tmp;
+ struct debug_obj *obj;
+ int cnt = 0;
+
+ hlist_for_each_entry_safe(obj, tmp, head, node) {
+ hlist_del(&obj->node);
+ kmem_cache_free(obj_cache, obj);
+ cnt++;
+ }
+ debug_objects_freed += cnt;
+}
+
+static void fill_pool_from_freelist(void)
+{
+ static unsigned long state;
+
/*
* Reuse objs from the global obj_to_free list; they will be
* reinitialized when allocating.
- *
- * obj_nr_tofree is checked locklessly; the READ_ONCE() pairs with
- * the WRITE_ONCE() in pool_lock critical sections.
*/
- if (READ_ONCE(obj_nr_tofree)) {
- raw_spin_lock_irqsave(&pool_lock, flags);
- /*
- * Recheck with the lock held as the worker thread might have
- * won the race and freed the global free list already.
- */
- while (obj_nr_tofree && (obj_pool_free < debug_objects_pool_min_level)) {
- obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
- hlist_del(&obj->node);
- WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1);
- hlist_add_head(&obj->node, &obj_pool);
- WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
+ if (!pool_count(&pool_to_free))
+ return;
+
+ /*
+ * Prevent the context from being scheduled or interrupted after
+ * setting the state flag;
+ */
+ guard(irqsave)();
+
+ /*
+ * Avoid lock contention on &pool_lock and avoid making the cache
+ * line exclusive by testing the bit before attempting to set it.
+ */
+ if (test_bit(0, &state) || test_and_set_bit(0, &state))
+ return;
+
+ /* Avoid taking the lock when there is no work to do */
+ while (pool_should_refill(&pool_global) && pool_count(&pool_to_free)) {
+ guard(raw_spinlock)(&pool_lock);
+ /* Move a batch if possible */
+ pool_move_batch(&pool_global, &pool_to_free);
+ }
+ clear_bit(0, &state);
+}
+
+static bool kmem_alloc_batch(struct hlist_head *head, struct kmem_cache *cache, gfp_t gfp)
+{
+ struct hlist_node *last = NULL;
+ struct debug_obj *obj;
+
+ for (int cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) {
+ obj = kmem_cache_zalloc(cache, gfp);
+ if (!obj) {
+ free_object_list(head);
+ return false;
}
- raw_spin_unlock_irqrestore(&pool_lock, flags);
+ debug_objects_allocated++;
+
+ if (!last)
+ last = &obj->node;
+ obj->batch_last = last;
+
+ hlist_add_head(&obj->node, head);
}
+ return true;
+}
+
+static void fill_pool(void)
+{
+ static atomic_t cpus_allocating;
- if (unlikely(!obj_cache))
+ /*
+ * Avoid allocation and lock contention when:
+ * - One other CPU is already allocating
+ * - the global pool has not reached the critical level yet
+ */
+ if (!pool_must_refill(&pool_global) && atomic_read(&cpus_allocating))
return;
- while (READ_ONCE(obj_pool_free) < debug_objects_pool_min_level) {
- struct debug_obj *new[ODEBUG_BATCH_SIZE];
- int cnt;
+ atomic_inc(&cpus_allocating);
+ while (pool_should_refill(&pool_global)) {
+ HLIST_HEAD(head);
- for (cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) {
- new[cnt] = kmem_cache_zalloc(obj_cache, gfp);
- if (!new[cnt])
- break;
- }
- if (!cnt)
- return;
+ if (!kmem_alloc_batch(&head, obj_cache, __GFP_HIGH | __GFP_NOWARN))
+ break;
- raw_spin_lock_irqsave(&pool_lock, flags);
- while (cnt) {
- hlist_add_head(&new[--cnt]->node, &obj_pool);
- debug_objects_allocated++;
- WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
- }
- raw_spin_unlock_irqrestore(&pool_lock, flags);
+ guard(raw_spinlock_irqsave)(&pool_lock);
+ if (!pool_push_batch(&pool_global, &head))
+ pool_push_batch(&pool_to_free, &head);
}
+ atomic_dec(&cpus_allocating);
}
/*
@@ -201,72 +429,37 @@ static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b)
return NULL;
}
-/*
- * Allocate a new object from the hlist
- */
-static struct debug_obj *__alloc_object(struct hlist_head *list)
+static void calc_usage(void)
{
- struct debug_obj *obj = NULL;
+ static DEFINE_RAW_SPINLOCK(avg_lock);
+ static unsigned long avg_period;
+ unsigned long cur, now = jiffies;
- if (list->first) {
- obj = hlist_entry(list->first, typeof(*obj), node);
- hlist_del(&obj->node);
- }
+ if (!time_after_eq(now, READ_ONCE(avg_period)))
+ return;
- return obj;
+ if (!raw_spin_trylock(&avg_lock))
+ return;
+
+ WRITE_ONCE(avg_period, now + msecs_to_jiffies(10));
+ cur = READ_ONCE(pool_global.stats.cur_used) * ODEBUG_FREE_WORK_MAX;
+ WRITE_ONCE(avg_usage, calc_load(avg_usage, EXP_5, cur));
+ raw_spin_unlock(&avg_lock);
}
-static struct debug_obj *
-alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *descr)
+static struct debug_obj *alloc_object(void *addr, struct debug_bucket *b,
+ const struct debug_obj_descr *descr)
{
- struct debug_percpu_free *percpu_pool = this_cpu_ptr(&percpu_obj_pool);
struct debug_obj *obj;
- if (likely(obj_cache)) {
- obj = __alloc_object(&percpu_pool->free_objs);
- if (obj) {
- percpu_pool->obj_free--;
- goto init_obj;
- }
- }
-
- raw_spin_lock(&pool_lock);
- obj = __alloc_object(&obj_pool);
- if (obj) {
- obj_pool_used++;
- WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
-
- /*
- * Looking ahead, allocate one batch of debug objects and
- * put them into the percpu free pool.
- */
- if (likely(obj_cache)) {
- int i;
-
- for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
- struct debug_obj *obj2;
-
- obj2 = __alloc_object(&obj_pool);
- if (!obj2)
- break;
- hlist_add_head(&obj2->node,
- &percpu_pool->free_objs);
- percpu_pool->obj_free++;
- obj_pool_used++;
- WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
- }
- }
-
- if (obj_pool_used > obj_pool_max_used)
- obj_pool_max_used = obj_pool_used;
+ calc_usage();
- if (obj_pool_free < obj_pool_min_free)
- obj_pool_min_free = obj_pool_free;
- }
- raw_spin_unlock(&pool_lock);
+ if (static_branch_likely(&obj_cache_enabled))
+ obj = pcpu_alloc();
+ else
+ obj = __alloc_object(&pool_boot);
-init_obj:
- if (obj) {
+ if (likely(obj)) {
obj->object = addr;
obj->descr = descr;
obj->state = ODEBUG_STATE_NONE;
@@ -276,142 +469,58 @@ alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *d
return obj;
}
-/*
- * workqueue function to free objects.
- *
- * To reduce contention on the global pool_lock, the actual freeing of
- * debug objects will be delayed if the pool_lock is busy.
- */
+/* workqueue function to free objects. */
static void free_obj_work(struct work_struct *work)
{
- struct hlist_node *tmp;
- struct debug_obj *obj;
- unsigned long flags;
- HLIST_HEAD(tofree);
+ static unsigned long last_use_avg;
+ unsigned long cur_used, last_used, delta;
+ unsigned int max_free = 0;
WRITE_ONCE(obj_freeing, false);
- if (!raw_spin_trylock_irqsave(&pool_lock, flags))
- return;
- if (obj_pool_free >= debug_objects_pool_size)
- goto free_objs;
+ /* Rate limit freeing based on current use average */
+ cur_used = READ_ONCE(avg_usage);
+ last_used = last_use_avg;
+ last_use_avg = cur_used;
- /*
- * The objs on the pool list might be allocated before the work is
- * run, so recheck if pool list it full or not, if not fill pool
- * list from the global free list. As it is likely that a workload
- * may be gearing up to use more and more objects, don't free any
- * of them until the next round.
- */
- while (obj_nr_tofree && obj_pool_free < debug_objects_pool_size) {
- obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
- hlist_del(&obj->node);
- hlist_add_head(&obj->node, &obj_pool);
- WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
- WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1);
- }
- raw_spin_unlock_irqrestore(&pool_lock, flags);
- return;
+ if (!pool_count(&pool_to_free))
+ return;
-free_objs:
- /*
- * Pool list is already full and there are still objs on the free
- * list. Move remaining free objs to a temporary list to free the
- * memory outside the pool_lock held region.
- */
- if (obj_nr_tofree) {
- hlist_move_list(&obj_to_free, &tofree);
- debug_objects_freed += obj_nr_tofree;
- WRITE_ONCE(obj_nr_tofree, 0);
+ if (cur_used <= last_used) {
+ delta = (last_used - cur_used) / ODEBUG_FREE_WORK_MAX;
+ max_free = min(delta, ODEBUG_FREE_WORK_MAX);
}
- raw_spin_unlock_irqrestore(&pool_lock, flags);
- hlist_for_each_entry_safe(obj, tmp, &tofree, node) {
- hlist_del(&obj->node);
- kmem_cache_free(obj_cache, obj);
+ for (int cnt = 0; cnt < ODEBUG_FREE_WORK_MAX; cnt++) {
+ HLIST_HEAD(tofree);
+
+ /* Acquire and drop the lock for each batch */
+ scoped_guard(raw_spinlock_irqsave, &pool_lock) {
+ if (!pool_to_free.cnt)
+ return;
+
+ /* Refill the global pool if possible */
+ if (pool_move_batch(&pool_global, &pool_to_free)) {
+ /* Don't free as there seems to be demand */
+ max_free = 0;
+ } else if (max_free) {
+ pool_pop_batch(&tofree, &pool_to_free);
+ max_free--;
+ } else {
+ return;
+ }
+ }
+ free_object_list(&tofree);
}
}
static void __free_object(struct debug_obj *obj)
{
- struct debug_obj *objs[ODEBUG_BATCH_SIZE];
- struct debug_percpu_free *percpu_pool;
- int lookahead_count = 0;
- unsigned long flags;
- bool work;
-
- local_irq_save(flags);
- if (!obj_cache)
- goto free_to_obj_pool;
-
- /*
- * Try to free it into the percpu pool first.
- */
- percpu_pool = this_cpu_ptr(&percpu_obj_pool);
- if (percpu_pool->obj_free < ODEBUG_POOL_PERCPU_SIZE) {
- hlist_add_head(&obj->node, &percpu_pool->free_objs);
- percpu_pool->obj_free++;
- local_irq_restore(flags);
- return;
- }
-
- /*
- * As the percpu pool is full, look ahead and pull out a batch
- * of objects from the percpu pool and free them as well.
- */
- for (; lookahead_count < ODEBUG_BATCH_SIZE; lookahead_count++) {
- objs[lookahead_count] = __alloc_object(&percpu_pool->free_objs);
- if (!objs[lookahead_count])
- break;
- percpu_pool->obj_free--;
- }
-
-free_to_obj_pool:
- raw_spin_lock(&pool_lock);
- work = (obj_pool_free > debug_objects_pool_size) && obj_cache &&
- (obj_nr_tofree < ODEBUG_FREE_WORK_MAX);
- obj_pool_used--;
-
- if (work) {
- WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1);
- hlist_add_head(&obj->node, &obj_to_free);
- if (lookahead_count) {
- WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + lookahead_count);
- obj_pool_used -= lookahead_count;
- while (lookahead_count) {
- hlist_add_head(&objs[--lookahead_count]->node,
- &obj_to_free);
- }
- }
-
- if ((obj_pool_free > debug_objects_pool_size) &&
- (obj_nr_tofree < ODEBUG_FREE_WORK_MAX)) {
- int i;
-
- /*
- * Free one more batch of objects from obj_pool.
- */
- for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
- obj = __alloc_object(&obj_pool);
- hlist_add_head(&obj->node, &obj_to_free);
- WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
- WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1);
- }
- }
- } else {
- WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
- hlist_add_head(&obj->node, &obj_pool);
- if (lookahead_count) {
- WRITE_ONCE(obj_pool_free, obj_pool_free + lookahead_count);
- obj_pool_used -= lookahead_count;
- while (lookahead_count) {
- hlist_add_head(&objs[--lookahead_count]->node,
- &obj_pool);
- }
- }
- }
- raw_spin_unlock(&pool_lock);
- local_irq_restore(flags);
+ guard(irqsave)();
+ if (static_branch_likely(&obj_cache_enabled))
+ pcpu_free(obj);
+ else
+ hlist_add_head(&obj->node, &pool_boot);
}
/*
@@ -421,63 +530,52 @@ static void __free_object(struct debug_obj *obj)
static void free_object(struct debug_obj *obj)
{
__free_object(obj);
- if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) {
+ if (!READ_ONCE(obj_freeing) && pool_count(&pool_to_free)) {
WRITE_ONCE(obj_freeing, true);
schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
}
}
-#ifdef CONFIG_HOTPLUG_CPU
-static int object_cpu_offline(unsigned int cpu)
+static void put_objects(struct hlist_head *list)
{
- struct debug_percpu_free *percpu_pool;
struct hlist_node *tmp;
struct debug_obj *obj;
- unsigned long flags;
- /* Remote access is safe as the CPU is dead already */
- percpu_pool = per_cpu_ptr(&percpu_obj_pool, cpu);
- hlist_for_each_entry_safe(obj, tmp, &percpu_pool->free_objs, node) {
+ /*
+ * Using free_object() puts the objects into reuse or schedules
+ * them for freeing and it get's all the accounting correct.
+ */
+ hlist_for_each_entry_safe(obj, tmp, list, node) {
hlist_del(&obj->node);
- kmem_cache_free(obj_cache, obj);
+ free_object(obj);
}
+}
- raw_spin_lock_irqsave(&pool_lock, flags);
- obj_pool_used -= percpu_pool->obj_free;
- debug_objects_freed += percpu_pool->obj_free;
- raw_spin_unlock_irqrestore(&pool_lock, flags);
-
- percpu_pool->obj_free = 0;
+#ifdef CONFIG_HOTPLUG_CPU
+static int object_cpu_offline(unsigned int cpu)
+{
+ /* Remote access is safe as the CPU is dead already */
+ struct obj_pool *pcp = per_cpu_ptr(&pool_pcpu, cpu);
+ put_objects(&pcp->objects);
+ pcp->cnt = 0;
return 0;
}
#endif
-/*
- * We run out of memory. That means we probably have tons of objects
- * allocated.
- */
+/* Out of memory. Free all objects from hash */
static void debug_objects_oom(void)
{
struct debug_bucket *db = obj_hash;
- struct hlist_node *tmp;
HLIST_HEAD(freelist);
- struct debug_obj *obj;
- unsigned long flags;
- int i;
pr_warn("Out of memory. ODEBUG disabled\n");
- for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
- raw_spin_lock_irqsave(&db->lock, flags);
- hlist_move_list(&db->list, &freelist);
- raw_spin_unlock_irqrestore(&db->lock, flags);
+ for (int i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
+ scoped_guard(raw_spinlock_irqsave, &db->lock)
+ hlist_move_list(&db->list, &freelist);
- /* Now free them */
- hlist_for_each_entry_safe(obj, tmp, &freelist, node) {
- hlist_del(&obj->node);
- free_object(obj);
- }
+ put_objects(&freelist);
}
}
@@ -592,12 +690,24 @@ static struct debug_obj *lookup_object_or_alloc(void *addr, struct debug_bucket
}
/* Out of memory. Do the cleanup outside of the locked region */
- debug_objects_enabled = 0;
+ debug_objects_enabled = false;
return NULL;
}
static void debug_objects_fill_pool(void)
{
+ if (!static_branch_likely(&obj_cache_enabled))
+ return;
+
+ if (likely(!pool_should_refill(&pool_global)))
+ return;
+
+ /* Try reusing objects from obj_to_free_list */
+ fill_pool_from_freelist();
+
+ if (likely(!pool_should_refill(&pool_global)))
+ return;
+
/*
* On RT enabled kernels the pool refill must happen in preemptible
* context -- for !RT kernels we rely on the fact that spinlock_t and
@@ -1007,7 +1117,7 @@ static void __debug_check_no_obj_freed(const void *address, unsigned long size)
debug_objects_maxchecked = objs_checked;
/* Schedule work to actually kmem_cache_free() objects */
- if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) {
+ if (!READ_ONCE(obj_freeing) && pool_count(&pool_to_free)) {
WRITE_ONCE(obj_freeing, true);
schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
}
@@ -1024,23 +1134,33 @@ void debug_check_no_obj_freed(const void *address, unsigned long size)
static int debug_stats_show(struct seq_file *m, void *v)
{
- int cpu, obj_percpu_free = 0;
+ unsigned int cpu, pool_used, pcp_free = 0;
+ /*
+ * pool_global.stats.cur_used is the number of batches currently
+ * handed out to per CPU pools. Convert it to number of objects
+ * and subtract the number of free objects in the per CPU pools.
+ * As this is lockless the number is an estimate.
+ */
for_each_possible_cpu(cpu)
- obj_percpu_free += per_cpu(percpu_obj_pool.obj_free, cpu);
-
- seq_printf(m, "max_chain :%d\n", debug_objects_maxchain);
- seq_printf(m, "max_checked :%d\n", debug_objects_maxchecked);
- seq_printf(m, "warnings :%d\n", debug_objects_warnings);
- seq_printf(m, "fixups :%d\n", debug_objects_fixups);
- seq_printf(m, "pool_free :%d\n", READ_ONCE(obj_pool_free) + obj_percpu_free);
- seq_printf(m, "pool_pcp_free :%d\n", obj_percpu_free);
- seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);
- seq_printf(m, "pool_used :%d\n", obj_pool_used - obj_percpu_free);
- seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);
- seq_printf(m, "on_free_list :%d\n", READ_ONCE(obj_nr_tofree));
- seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);
- seq_printf(m, "objs_freed :%d\n", debug_objects_freed);
+ pcp_free += per_cpu(pool_pcpu.cnt, cpu);
+
+ pool_used = READ_ONCE(pool_global.stats.cur_used);
+ pcp_free = min(pool_used, pcp_free);
+ pool_used -= pcp_free;
+
+ seq_printf(m, "max_chain : %d\n", debug_objects_maxchain);
+ seq_printf(m, "max_checked : %d\n", debug_objects_maxchecked);
+ seq_printf(m, "warnings : %d\n", debug_objects_warnings);
+ seq_printf(m, "fixups : %d\n", debug_objects_fixups);
+ seq_printf(m, "pool_free : %u\n", pool_count(&pool_global) + pcp_free);
+ seq_printf(m, "pool_pcp_free : %u\n", pcp_free);
+ seq_printf(m, "pool_min_free : %u\n", data_race(pool_global.stats.min_fill));
+ seq_printf(m, "pool_used : %u\n", pool_used);
+ seq_printf(m, "pool_max_used : %u\n", data_race(pool_global.stats.max_used));
+ seq_printf(m, "on_free_list : %u\n", pool_count(&pool_to_free));
+ seq_printf(m, "objs_allocated: %d\n", debug_objects_allocated);
+ seq_printf(m, "objs_freed : %d\n", debug_objects_freed);
return 0;
}
DEFINE_SHOW_ATTRIBUTE(debug_stats);
@@ -1194,7 +1314,7 @@ check_results(void *addr, enum debug_obj_state state, int fixups, int warnings)
out:
raw_spin_unlock_irqrestore(&db->lock, flags);
if (res)
- debug_objects_enabled = 0;
+ debug_objects_enabled = false;
return res;
}
@@ -1209,7 +1329,7 @@ static __initconst const struct debug_obj_descr descr_type_test = {
static __initdata struct self_test obj = { .static_init = 0 };
-static void __init debug_objects_selftest(void)
+static bool __init debug_objects_selftest(void)
{
int fixups, oldfixups, warnings, oldwarnings;
unsigned long flags;
@@ -1278,9 +1398,10 @@ static void __init debug_objects_selftest(void)
descr_test = NULL;
local_irq_restore(flags);
+ return debug_objects_enabled;
}
#else
-static inline void debug_objects_selftest(void) { }
+static inline bool debug_objects_selftest(void) { return true; }
#endif
/*
@@ -1295,65 +1416,54 @@ void __init debug_objects_early_init(void)
for (i = 0; i < ODEBUG_HASH_SIZE; i++)
raw_spin_lock_init(&obj_hash[i].lock);
+ /* Keep early boot simple and add everything to the boot list */
for (i = 0; i < ODEBUG_POOL_SIZE; i++)
- hlist_add_head(&obj_static_pool[i].node, &obj_pool);
+ hlist_add_head(&obj_static_pool[i].node, &pool_boot);
}
/*
- * Convert the statically allocated objects to dynamic ones:
+ * Convert the statically allocated objects to dynamic ones.
+ * debug_objects_mem_init() is called early so only one CPU is up and
+ * interrupts are disabled, which means it is safe to replace the active
+ * object references.
*/
-static int __init debug_objects_replace_static_objects(void)
+static bool __init debug_objects_replace_static_objects(struct kmem_cache *cache)
{
struct debug_bucket *db = obj_hash;
struct hlist_node *tmp;
- struct debug_obj *obj, *new;
+ struct debug_obj *obj;
HLIST_HEAD(objects);
- int i, cnt = 0;
+ int i;
- for (i = 0; i < ODEBUG_POOL_SIZE; i++) {
- obj = kmem_cache_zalloc(obj_cache, GFP_KERNEL);
- if (!obj)
+ for (i = 0; i < ODEBUG_POOL_SIZE; i += ODEBUG_BATCH_SIZE) {
+ if (!kmem_alloc_batch(&objects, cache, GFP_KERNEL))
goto free;
- hlist_add_head(&obj->node, &objects);
+ pool_push_batch(&pool_global, &objects);
}
- debug_objects_allocated += i;
-
- /*
- * debug_objects_mem_init() is now called early that only one CPU is up
- * and interrupts have been disabled, so it is safe to replace the
- * active object references.
- */
-
- /* Remove the statically allocated objects from the pool */
- hlist_for_each_entry_safe(obj, tmp, &obj_pool, node)
- hlist_del(&obj->node);
- /* Move the allocated objects to the pool */
- hlist_move_list(&objects, &obj_pool);
+ /* Disconnect the boot pool. */
+ pool_boot.first = NULL;
/* Replace the active object references */
for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
hlist_move_list(&db->list, &objects);
hlist_for_each_entry(obj, &objects, node) {
- new = hlist_entry(obj_pool.first, typeof(*obj), node);
- hlist_del(&new->node);
+ struct debug_obj *new = pcpu_alloc();
+
/* copy object data */
*new = *obj;
hlist_add_head(&new->node, &db->list);
- cnt++;
}
}
-
- pr_debug("%d of %d active objects replaced\n",
- cnt, obj_pool_used);
- return 0;
+ return true;
free:
- hlist_for_each_entry_safe(obj, tmp, &objects, node) {
+ /* Can't use free_object_list() as the cache is not populated yet */
+ hlist_for_each_entry_safe(obj, tmp, &pool_global.objects, node) {
hlist_del(&obj->node);
- kmem_cache_free(obj_cache, obj);
+ kmem_cache_free(cache, obj);
}
- return -ENOMEM;
+ return false;
}
/*
@@ -1364,43 +1474,40 @@ static int __init debug_objects_replace_static_objects(void)
*/
void __init debug_objects_mem_init(void)
{
- int cpu, extras;
+ struct kmem_cache *cache;
+ int extras;
if (!debug_objects_enabled)
return;
- /*
- * Initialize the percpu object pools
- *
- * Initialization is not strictly necessary, but was done for
- * completeness.
- */
- for_each_possible_cpu(cpu)
- INIT_HLIST_HEAD(&per_cpu(percpu_obj_pool.free_objs, cpu));
+ if (!debug_objects_selftest())
+ return;
- obj_cache = kmem_cache_create("debug_objects_cache",
- sizeof (struct debug_obj), 0,
- SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE,
- NULL);
+ cache = kmem_cache_create("debug_objects_cache", sizeof (struct debug_obj), 0,
+ SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE, NULL);
- if (!obj_cache || debug_objects_replace_static_objects()) {
- debug_objects_enabled = 0;
- kmem_cache_destroy(obj_cache);
- pr_warn("out of memory.\n");
+ if (!cache || !debug_objects_replace_static_objects(cache)) {
+ debug_objects_enabled = false;
+ pr_warn("Out of memory.\n");
return;
- } else
- debug_objects_selftest();
-
-#ifdef CONFIG_HOTPLUG_CPU
- cpuhp_setup_state_nocalls(CPUHP_DEBUG_OBJ_DEAD, "object:offline", NULL,
- object_cpu_offline);
-#endif
+ }
/*
- * Increase the thresholds for allocating and freeing objects
- * according to the number of possible CPUs available in the system.
+ * Adjust the thresholds for allocating and freeing objects
+ * according to the number of possible CPUs available in the
+ * system.
*/
extras = num_possible_cpus() * ODEBUG_BATCH_SIZE;
- debug_objects_pool_size += extras;
- debug_objects_pool_min_level += extras;
+ pool_global.max_cnt += extras;
+ pool_global.min_cnt += extras;
+
+ /* Everything worked. Expose the cache */
+ obj_cache = cache;
+ static_branch_enable(&obj_cache_enabled);
+
+#ifdef CONFIG_HOTPLUG_CPU
+ cpuhp_setup_state_nocalls(CPUHP_DEBUG_OBJ_DEAD, "object:offline", NULL,
+ object_cpu_offline);
+#endif
+ return;
}The pull request you sent on Mon, 18 Nov 2024 20:21:47 +0100 (CET): > git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git core-debugobjects-2024-11-18 has been merged into torvalds/linux.git: https://git.kernel.org/torvalds/c/fb1dd1403c7b2219b8c1524c909938bd4b3f401f Thank you! -- Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/prtracker.html
Linus,
please pull the latest timers/core branch from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git timers-core-2024-11-18
up to: cdc905d16b07: posix-timers: Fix spurious warning on double enqueue versus do_exit()
A rather large update for timekeeping and timers:
- The final step to get rid of auto-rearming posix-timers
posix-timers are currently auto-rearmed by the kernel when the signal
of the timer is ignored so that the timer signal can be delivered once
the corresponding signal is unignored.
This requires to throttle the timer to prevent a DoS by small intervals
and keeps the system pointlessly out of low power states for no value.
This is a long standing non-trivial problem due to the lock order of
posix-timer lock and the sighand lock along with life time issues as
the timer and the sigqueue have different life time rules.
Cure this by:
* Embedding the sigqueue into the timer struct to have the same life
time rules. Aside of that this also avoids the lookup of the timer
in the signal delivery and rearm path as it's just a always valid
container_of() now.
* Queuing ignored timer signals onto a seperate ignored list.
* Moving queued timer signals onto the ignored list when the signal is
switched to SIG_IGN before it could be delivered.
* Walking the ignored list when SIG_IGN is lifted and requeue the
signals to the actual signal lists. This allows the signal delivery
code to rearm the timer.
This also required to consolidate the signal delivery rules so they are
consistent across all situations. With that all self test scenarios
finally succeed.
- Core infrastructure for VFS multigrain timestamping
This is required to allow the kernel to use coarse grained time stamps
by default and switch to fine grained time stamps when inode attributes
are actively observed via getattr().
These changes have been provided to the VFS tree as well, so that the
VFS specific infrastructure could be built on top.
- Cleanup and consolidation of the sleep() infrastructure
* Move all sleep and timeout functions into one file
* Rework udelay() and ndelay() into proper documented inline functions
and replace the hardcoded magic numbers by proper defines.
* Rework the fsleep() implementation to take the reality of the timer
wheel granularity on different HZ values into account. Right now the
boundaries are hard coded time ranges which fail to provide the
requested accuracy on different HZ settings.
* Update documentation for all sleep/timeout related functions and fix
up stale documentation links all over the place
* Fixup a few usage sites
- Rework of timekeeping and adjtimex(2) to prepare for multiple PTP clocks
A system can have multiple PTP clocks which are participating in
seperate and independent PTP clock domains. So far the kernel only
considers the PTP clock which is based on CLOCK TAI relevant as that's
the clock which drives the timekeeping adjustments via the various user
space daemons through adjtimex(2).
The non TAI based clock domains are accessible via the file descriptor
based posix clocks, but their usability is very limited. They can't be
accessed fast as they always go all the way out to the hardware and
they cannot be utilized in the kernel itself.
As Time Sensitive Networking (TSN) gains traction it is required to
provide fast user and kernel space access to these clocks.
The approach taken is to utilize the timekeeping and adjtimex(2)
infrastructure to provide this access in a similar way how the kernel
provides access to clock MONOTONIC, REALTIME etc.
Instead of creating a duplicated infrastructure this rework converts
timekeeping and adjtimex(2) into generic functionality which operates
on pointers to data structures instead of using static variables.
This allows to provide time accessors and adjtimex(2) functionality for
the independent PTP clocks in a subsequent step.
- Consolidate hrtimer initialization
hrtimers are set up by initializing the data structure and then
seperately setting the callback function for historical reasons.
That's an extra unnecessary step and makes Rust support less straight
forward than it should be.
Provide a new set of hrtimer_setup*() functions and convert the core
code and a few usage sites of the less frequently used interfaces over.
The bulk of the htimer_init() to hrtimer_setup() conversion is already
prepared and scheduled for the next merge window.
- Drivers:
* Ensure that the global timekeeping clocksource is utilizing the
cluster 0 timer on MIPS multi-cluster systems.
Otherwise CPUs on different clusters use their cluster specific
clocksource which is not guaranteed to be synchronized with other
clusters.
* Mostly boring cleanups, fixes, improvements and code movement
Thanks,
tglx
------------------>
Anna-Maria Behnsen (32):
ntp: Cleanup formatting of code
MAINTAINERS: Add missing file include/linux/delay.h
timers: Move *sleep*() and timeout functions into a separate file
timers: Update schedule_[hr]timeout*() related function descriptions
timers: Rename usleep_idle_range() to usleep_range_idle()
timers: Update function descriptions of sleep/delay related functions
delay: Rework udelay and ndelay
timers: Adjust flseep() to reflect reality
mm/damon/core: Use generic upper bound recommondation for usleep_range()
timers: Add a warning to usleep_range_state() for wrong order of arguments
checkpatch: Remove links to outdated documentation
iopoll/regmap/phy/snd: Fix comment referencing outdated timer documentation
powerpc/rtas: Use fsleep() to minimize additional sleep duration
media: anysee: Fix and remove outdated comment
timers/Documentation: Cleanup delay/sleep documentation
timekeeping: Avoid duplicate leap state update
timekeeping: Move timekeeper_lock into tk_core
timekeeping: Define a struct type for tk_core to make it reusable
timekeeping: Introduce tkd_basic_setup() to make lock and seqcount init reusable
timekeeping: Add struct tk_data as argument to timekeeping_update()
timekeeping: Split out timekeeper update of timekeeping_advanced()
timekeeping: Introduce combined timekeeping action flag
timekeeping: Rework do_settimeofday64() to use shadow_timekeeper
timekeeping: Rework timekeeping_inject_offset() to use shadow_timekeeper
timekeeping: Rework change_clocksource() to use shadow_timekeeper
timekeeping: Rework timekeeping_init() to use shadow_timekeeper
timekeeping: Rework timekeeping_inject_sleeptime64() to use shadow_timekeeper
timekeeping: Rework timekeeping_resume() to use shadow_timekeeper
timekeeping: Rework timekeeping_suspend() to use shadow_timekeeper
timekeeping: Rework do_adjtimex() to use shadow_timekeeper
timekeeping: Remove TK_MIRROR timekeeping_update() action
timekeeping: Merge timekeeping_update_staged() and timekeeping_update()
Dr. David Alan Gilbert (2):
clocksource: Remove unused clocksource_change_rating
clocksource/drivers/dw_apb: Remove unused dw_apb_clockevent functions
Easwar Hariharan (1):
jiffies: Define secs_to_jiffies()
Frederic Weisbecker (11):
clockevents: Improve clockevents_notify_released() comment
clockevents: Shutdown and unregister current clockevents at CPUHP_AP_TICK_DYING
tick: Remove now unneeded low-res tick stop on CPUHP_AP_TICK_DYING
ARM: smp_twd: Remove clockevents shutdown call on offlining
clocksource/drivers/arm_arch_timer: Remove clockevents shutdown call on offlining
clocksource/drivers/arm_global_timer: Remove clockevents shutdown call on offlining
clocksource/drivers/exynos_mct: Remove clockevents shutdown call on offlining
clocksource/drivers/armada-370-xp: Remove clockevents shutdown call on offlining
clocksource/drivers/qcom: Remove clockevents shutdown call on offlining
clocksource/drivers/timer-tegra: Remove clockevents shutdown call on offlining
posix-timers: Fix spurious warning on double enqueue versus do_exit()
Ivaylo Ivanov (1):
dt-bindings: timer: actions,owl-timer: convert to YAML
Javier Carrasco (1):
clocksource/drivers/timer-ti-dm: Fix child node refcount handling
Jeff Layton (3):
timekeeping: Don't use seqcount loop in ktime_mono_to_any() on 64-bit systems
timekeeping: Add interfaces for handling timestamps with a floor value
timekeeping: Add percpu counter for tracking floor swap events
Judith Mendez (1):
clocksource/drivers/timer-ti-dm: Don't fail probe if int not found
Julia Lawall (1):
posix-timers: Replace call_rcu() by kfree_rcu() for simple kmem_cache_free() callback
Mark Brown (1):
clocksource/drivers:sp804: Make user selectable
Miguel Ojeda (2):
time: Partially revert cleanup on msecs_to_jiffies() documentation
time: Fix references to _msecs_to_jiffies() handling of values
Nam Cao (19):
hrtimers: Add missing hrtimer_init() trace points
drm/i915/request: Remove unnecessary modification of hrtimer:: Function
KVM: x86/xen: Initialize hrtimer in kvm_xen_init_vcpu()
_RESEND_PATCH_v2_04_19_wifi_rt2x00_Remove_redundant_hrtimer_init_
io_uring: Remove redundant hrtimer's callback function setup
hrtimers: Introduce hrtimer_setup() to replace hrtimer_init()
hrtimers: Introduce hrtimer_setup_on_stack()
hrtimers: Introduce hrtimer_setup_sleeper_on_stack()
hrtimers: Introduce hrtimer_update_function()
fs/aio: Switch to use hrtimer_setup_sleeper_on_stack()
futex: Switch to use hrtimer_setup_sleeper_on_stack()
net: pktgen: Switch to use hrtimer_setup_sleeper_on_stack()
timers: Switch to use hrtimer_setup_sleeper_on_stack()
wait: Switch to use hrtimer_setup_sleeper_on_stack()
hrtimers: Delete hrtimer_init_sleeper_on_stack()
sched/idle: Switch to use hrtimer_setup_on_stack()
io_uring: Switch to use hrtimer_setup_on_stack()
alarmtimer: Switch to use hrtimer_setup() and hrtimer_setup_on_stack()
hrtimers: Delete hrtimer_init_on_stack()
Paul Burton (1):
clocksource/drivers/mips-gic-timer: Always use cluster 0 counter as clocksource
Rob Herring (Arm) (1):
clocksource/drivers/arm_arch_timer: Use of_property_present() for non-boolean properties
Sergio Paracuellos (1):
clocksource/drivers/ralink: Add Ralink System Tick Counter driver
Tang Bin (1):
clocksource/drivers/gpx: Remove redundant casts
Thomas Gleixner (59):
ntp: Remove unused tick_nsec
ntp: Make tick_usec static
ntp: Clean up comments
ntp: Convert functions with only two states to bool
ntp: Read reference time only once
ntp: Introduce struct ntp_data
ntp: Move tick_length* into ntp_data
ntp: Move tick_stat* into ntp_data
ntp: Move time_offset/constant into ntp_data
ntp: Move time_max/esterror into ntp_data
ntp: Move time_freq/reftime into ntp_data
ntp: Move time_adj/ntp_tick_adj into ntp_data
ntp: Move ntp_next_leap_sec into ntp_data
ntp: Move pps_valid into ntp_data
ntp: Move pps_ft into ntp_data
ntp: Move pps_jitter into ntp_data
ntp: Move pps_fbase into ntp_data
ntp: Move pps_shift/intcnt into ntp_data
ntp: Move pps_freq/stabil into ntp_data
ntp: Move pps monitors into ntp_data
timekeeping: Read NTP tick length only once
timekeeping: Don't stop time readers across hard_pps() update
timekeeping: Abort clocksource change in case of failure
timekeeping: Simplify code in timekeeping_advance()
timekeeping: Reorder struct timekeeper
timekeeping: Move shadow_timekeeper into tk_core
timekeeping: Encapsulate locking/unlocking of timekeeper_lock
timekeeping: Provide timekeeping_restore_shadow()
signal: Confine POSIX_TIMERS properly
signal: Cleanup flush_sigqueue_mask()
posix-timers: Cure si_sys_private race
signal: Allow POSIX timer signals to be dropped
posix-timers: Drop signal if timer has been deleted or reprogrammed
posix-timers: Rename k_itimer:: It_requeue_pending
posix-timers: Add proper state tracking
timers: Add missing READ_ONCE() in __run_timer_base()
timekeeping: Remove CONFIG_DEBUG_TIMEKEEPING
timekeeping: Always check for negative motion
posix-cpu-timers: Correctly update timer status in posix_cpu_timer_del()
posix-timers: Make signal delivery consistent
posix-timers: Make signal overrun accounting sensible
posix-cpu-timers: Cleanup the firing logic
posix-cpu-timers: Use dedicated flag for CPU timer nanosleep
posix-timers: Add a refcount to struct k_itimer
signal: Split up __sigqueue_alloc()
signal: Provide posixtimer_sigqueue_init()
posix-timers: Store PID type in the timer
signal: Refactor send_sigqueue()
signal: Replace resched_timer logic
posix-timers: Embed sigqueue in struct k_itimer
signal: Cleanup unused posix-timer leftovers
posix-timers: Move sequence logic into struct k_itimer
signal: Provide ignored_posix_timers list
posix-timers: Handle ignored list on delete and exit
signal: Handle ignored signals in do_sigaction(action != SIG_IGN)
signal: Queue ignored posixtimers on ignore list
posix-timers: Cleanup SIG_IGN workaround leftovers
alarmtimers: Remove the throttle mechanism from alarm_forward_now()
alarmtimers: Remove return value from alarm functions
Vincent Donnefort (1):
timekeeping: Add the boot clock to system time snapshot
Wang Jinchao (1):
time: Remove '%' from numeric constant in kernel-doc comment
Documentation/dev-tools/checkpatch.rst | 2 -
.../bindings/timer/actions,owl-timer.txt | 21 -
.../bindings/timer/actions,owl-timer.yaml | 107 +++
Documentation/timers/delay_sleep_functions.rst | 121 +++
Documentation/timers/index.rst | 2 +-
Documentation/timers/timers-howto.rst | 115 ---
MAINTAINERS | 4 +-
arch/arm/kernel/smp_twd.c | 1 -
arch/mips/ralink/Kconfig | 7 -
arch/mips/ralink/Makefile | 2 -
arch/powerpc/kernel/rtas.c | 21 +-
arch/riscv/configs/defconfig | 1 -
arch/x86/Kconfig | 1 -
arch/x86/include/asm/timer.h | 2 -
arch/x86/kvm/xen.c | 12 +-
drivers/clocksource/Kconfig | 12 +-
drivers/clocksource/Makefile | 1 +
drivers/clocksource/arm_arch_timer.c | 4 +-
drivers/clocksource/arm_global_timer.c | 1 -
drivers/clocksource/dw_apb_timer.c | 39 -
drivers/clocksource/exynos_mct.c | 1 -
drivers/clocksource/mips-gic-timer.c | 39 +-
drivers/clocksource/timer-armada-370-xp.c | 1 -
drivers/clocksource/timer-gxp.c | 2 +-
drivers/clocksource/timer-qcom.c | 1 -
.../clocksource/timer-ralink.c | 11 +-
drivers/clocksource/timer-tegra.c | 1 -
drivers/clocksource/timer-ti-dm-systimer.c | 8 +-
drivers/clocksource/timer-ti-dm.c | 8 +-
drivers/gpu/drm/i915/i915_request.c | 17 +-
drivers/media/usb/dvb-usb-v2/anysee.c | 17 +-
drivers/net/wireless/ralink/rt2x00/rt2x00usb.c | 2 -
drivers/power/supply/charger-manager.c | 3 +-
fs/aio.c | 2 +-
fs/proc/base.c | 4 +-
fs/timerfd.c | 4 +-
include/asm-generic/delay.h | 96 ++-
include/linux/alarmtimer.h | 10 +-
include/linux/clocksource.h | 1 -
include/linux/delay.h | 79 +-
include/linux/dw_apb_timer.h | 3 -
include/linux/hrtimer.h | 51 +-
include/linux/iopoll.h | 52 +-
include/linux/jiffies.h | 15 +-
include/linux/phy.h | 9 +-
include/linux/posix-timers.h | 72 +-
include/linux/regmap.h | 38 +-
include/linux/sched/signal.h | 4 +-
include/linux/tick.h | 2 -
include/linux/timekeeper_internal.h | 114 +--
include/linux/timekeeping.h | 7 +
include/linux/timex.h | 8 -
include/linux/wait.h | 4 +-
include/uapi/asm-generic/siginfo.h | 2 +-
init/init_task.c | 5 +-
io_uring/io_uring.c | 7 +-
io_uring/timeout.c | 1 -
kernel/cpu.c | 2 -
kernel/fork.c | 1 +
kernel/futex/core.c | 6 +-
kernel/sched/idle.c | 4 +-
kernel/signal.c | 494 +++++++-----
kernel/time/Kconfig | 5 -
kernel/time/Makefile | 2 +-
kernel/time/alarmtimer.c | 96 +--
kernel/time/clockevents.c | 42 +-
kernel/time/clocksource.c | 40 +-
kernel/time/hrtimer.c | 230 ++----
kernel/time/itimer.c | 22 +-
kernel/time/ntp.c | 840 ++++++++++-----------
kernel/time/posix-cpu-timers.c | 72 +-
kernel/time/posix-timers.c | 267 ++++---
kernel/time/posix-timers.h | 8 +-
kernel/time/sleep_timeout.c | 377 +++++++++
kernel/time/tick-internal.h | 3 +-
kernel/time/tick-sched.c | 25 +-
kernel/time/time.c | 20 +-
kernel/time/timekeeping.c | 637 ++++++++--------
kernel/time/timekeeping_debug.c | 13 +
kernel/time/timekeeping_internal.h | 25 +-
kernel/time/timer.c | 195 +----
kernel/time/vsyscall.c | 5 +-
lib/Kconfig.debug | 13 -
mm/damon/core.c | 5 +-
net/bluetooth/hci_event.c | 2 -
net/core/pktgen.c | 2 +-
net/netfilter/xt_IDLETIMER.c | 4 +-
scripts/checkpatch.pl | 10 +-
sound/soc/sof/ops.h | 8 +-
.../testing/selftests/wireguard/qemu/debug.config | 1 -
90 files changed, 2494 insertions(+), 2159 deletions(-)
delete mode 100644 Documentation/devicetree/bindings/timer/actions,owl-timer.txt
create mode 100644 Documentation/devicetree/bindings/timer/actions,owl-timer.yaml
create mode 100644 Documentation/timers/delay_sleep_functions.rst
delete mode 100644 Documentation/timers/timers-howto.rst
rename arch/mips/ralink/cevt-rt3352.c => drivers/clocksource/timer-ralink.c (91%)
create mode 100644 kernel/time/sleep_timeout.c
diff --git a/Documentation/dev-tools/checkpatch.rst b/Documentation/dev-tools/checkpatch.rst
index a9fac978a525..abb3ff682076 100644
--- a/Documentation/dev-tools/checkpatch.rst
+++ b/Documentation/dev-tools/checkpatch.rst
@@ -470,8 +470,6 @@ API usage
usleep_range() should be preferred over udelay(). The proper way of
using usleep_range() is mentioned in the kernel docs.
- See: https://www.kernel.org/doc/html/latest/timers/timers-howto.html#delays-information-on-the-various-kernel-delay-sleep-mechanisms
-
Comments
--------
diff --git a/Documentation/devicetree/bindings/timer/actions,owl-timer.txt b/Documentation/devicetree/bindings/timer/actions,owl-timer.txt
deleted file mode 100644
index 977054f87563..000000000000
--- a/Documentation/devicetree/bindings/timer/actions,owl-timer.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-Actions Semi Owl Timer
-
-Required properties:
-- compatible : "actions,s500-timer" for S500
- "actions,s700-timer" for S700
- "actions,s900-timer" for S900
-- reg : Offset and length of the register set for the device.
-- interrupts : Should contain the interrupts.
-- interrupt-names : Valid names are: "2hz0", "2hz1",
- "timer0", "timer1", "timer2", "timer3"
- See ../resource-names.txt
-
-Example:
-
- timer@b0168000 {
- compatible = "actions,s500-timer";
- reg = <0xb0168000 0x100>;
- interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>,
- <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>;
- interrupt-names = "timer0", "timer1";
- };
diff --git a/Documentation/devicetree/bindings/timer/actions,owl-timer.yaml b/Documentation/devicetree/bindings/timer/actions,owl-timer.yaml
new file mode 100644
index 000000000000..646c554a390a
--- /dev/null
+++ b/Documentation/devicetree/bindings/timer/actions,owl-timer.yaml
@@ -0,0 +1,107 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/timer/actions,owl-timer.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Actions Semi Owl timer
+
+maintainers:
+ - Andreas Färber <afaerber@suse.de>
+
+description:
+ Actions Semi Owl SoCs provide 32bit and 2Hz timers.
+ The 32bit timers support dynamic irq, as well as one-shot mode.
+
+properties:
+ compatible:
+ enum:
+ - actions,s500-timer
+ - actions,s700-timer
+ - actions,s900-timer
+
+ clocks:
+ maxItems: 1
+
+ interrupts:
+ minItems: 1
+ maxItems: 6
+
+ interrupt-names:
+ minItems: 1
+ maxItems: 6
+ items:
+ enum:
+ - 2hz0
+ - 2hz1
+ - timer0
+ - timer1
+ - timer2
+ - timer3
+
+ reg:
+ maxItems: 1
+
+required:
+ - compatible
+ - clocks
+ - interrupts
+ - interrupt-names
+ - reg
+
+allOf:
+ - if:
+ properties:
+ compatible:
+ contains:
+ enum:
+ - actions,s500-timer
+ then:
+ properties:
+ interrupts:
+ minItems: 4
+ maxItems: 4
+ interrupt-names:
+ items:
+ - const: 2hz0
+ - const: 2hz1
+ - const: timer0
+ - const: timer1
+
+ - if:
+ properties:
+ compatible:
+ contains:
+ enum:
+ - actions,s700-timer
+ - actions,s900-timer
+ then:
+ properties:
+ interrupts:
+ minItems: 1
+ maxItems: 1
+ interrupt-names:
+ items:
+ - const: timer1
+
+additionalProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+ #include <dt-bindings/interrupt-controller/irq.h>
+ soc {
+ #address-cells = <1>;
+ #size-cells = <1>;
+ timer@b0168000 {
+ compatible = "actions,s500-timer";
+ reg = <0xb0168000 0x100>;
+ clocks = <&hosc>;
+ interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>;
+ interrupt-names = "2hz0", "2hz1", "timer0", "timer1";
+ };
+ };
+...
diff --git a/Documentation/timers/delay_sleep_functions.rst b/Documentation/timers/delay_sleep_functions.rst
new file mode 100644
index 000000000000..49d603a3f113
--- /dev/null
+++ b/Documentation/timers/delay_sleep_functions.rst
@@ -0,0 +1,121 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Delay and sleep mechanisms
+==========================
+
+This document seeks to answer the common question: "What is the
+RightWay (TM) to insert a delay?"
+
+This question is most often faced by driver writers who have to
+deal with hardware delays and who may not be the most intimately
+familiar with the inner workings of the Linux Kernel.
+
+The following table gives a rough overview about the existing function
+'families' and their limitations. This overview table does not replace the
+reading of the function description before usage!
+
+.. list-table::
+ :widths: 20 20 20 20 20
+ :header-rows: 2
+
+ * -
+ - `*delay()`
+ - `usleep_range*()`
+ - `*sleep()`
+ - `fsleep()`
+ * -
+ - busy-wait loop
+ - hrtimers based
+ - timer list timers based
+ - combines the others
+ * - Usage in atomic Context
+ - yes
+ - no
+ - no
+ - no
+ * - precise on "short intervals"
+ - yes
+ - yes
+ - depends
+ - yes
+ * - precise on "long intervals"
+ - Do not use!
+ - yes
+ - max 12.5% slack
+ - yes
+ * - interruptible variant
+ - no
+ - yes
+ - yes
+ - no
+
+A generic advice for non atomic contexts could be:
+
+#. Use `fsleep()` whenever unsure (as it combines all the advantages of the
+ others)
+#. Use `*sleep()` whenever possible
+#. Use `usleep_range*()` whenever accuracy of `*sleep()` is not sufficient
+#. Use `*delay()` for very, very short delays
+
+Find some more detailed information about the function 'families' in the next
+sections.
+
+`*delay()` family of functions
+------------------------------
+
+These functions use the jiffy estimation of clock speed and will busy wait for
+enough loop cycles to achieve the desired delay. udelay() is the basic
+implementation and ndelay() as well as mdelay() are variants.
+
+These functions are mainly used to add a delay in atomic context. Please make
+sure to ask yourself before adding a delay in atomic context: Is this really
+required?
+
+.. kernel-doc:: include/asm-generic/delay.h
+ :identifiers: udelay ndelay
+
+.. kernel-doc:: include/linux/delay.h
+ :identifiers: mdelay
+
+
+`usleep_range*()` and `*sleep()` family of functions
+----------------------------------------------------
+
+These functions use hrtimers or timer list timers to provide the requested
+sleeping duration. In order to decide which function is the right one to use,
+take some basic information into account:
+
+#. hrtimers are more expensive as they are using an rb-tree (instead of hashing)
+#. hrtimers are more expensive when the requested sleeping duration is the first
+ timer which means real hardware has to be programmed
+#. timer list timers always provide some sort of slack as they are jiffy based
+
+The generic advice is repeated here:
+
+#. Use `fsleep()` whenever unsure (as it combines all the advantages of the
+ others)
+#. Use `*sleep()` whenever possible
+#. Use `usleep_range*()` whenever accuracy of `*sleep()` is not sufficient
+
+First check fsleep() function description and to learn more about accuracy,
+please check msleep() function description.
+
+
+`usleep_range*()`
+~~~~~~~~~~~~~~~~~
+
+.. kernel-doc:: include/linux/delay.h
+ :identifiers: usleep_range usleep_range_idle
+
+.. kernel-doc:: kernel/time/sleep_timeout.c
+ :identifiers: usleep_range_state
+
+
+`*sleep()`
+~~~~~~~~~~
+
+.. kernel-doc:: kernel/time/sleep_timeout.c
+ :identifiers: msleep msleep_interruptible
+
+.. kernel-doc:: include/linux/delay.h
+ :identifiers: ssleep fsleep
diff --git a/Documentation/timers/index.rst b/Documentation/timers/index.rst
index 983f91f8f023..4e88116e4dcf 100644
--- a/Documentation/timers/index.rst
+++ b/Documentation/timers/index.rst
@@ -12,7 +12,7 @@ Timers
hrtimers
no_hz
timekeeping
- timers-howto
+ delay_sleep_functions
.. only:: subproject and html
diff --git a/Documentation/timers/timers-howto.rst b/Documentation/timers/timers-howto.rst
deleted file mode 100644
index ef7a4652ccc9..000000000000
--- a/Documentation/timers/timers-howto.rst
+++ /dev/null
@@ -1,115 +0,0 @@
-===================================================================
-delays - Information on the various kernel delay / sleep mechanisms
-===================================================================
-
-This document seeks to answer the common question: "What is the
-RightWay (TM) to insert a delay?"
-
-This question is most often faced by driver writers who have to
-deal with hardware delays and who may not be the most intimately
-familiar with the inner workings of the Linux Kernel.
-
-
-Inserting Delays
-----------------
-
-The first, and most important, question you need to ask is "Is my
-code in an atomic context?" This should be followed closely by "Does
-it really need to delay in atomic context?" If so...
-
-ATOMIC CONTEXT:
- You must use the `*delay` family of functions. These
- functions use the jiffy estimation of clock speed
- and will busy wait for enough loop cycles to achieve
- the desired delay:
-
- ndelay(unsigned long nsecs)
- udelay(unsigned long usecs)
- mdelay(unsigned long msecs)
-
- udelay is the generally preferred API; ndelay-level
- precision may not actually exist on many non-PC devices.
-
- mdelay is macro wrapper around udelay, to account for
- possible overflow when passing large arguments to udelay.
- In general, use of mdelay is discouraged and code should
- be refactored to allow for the use of msleep.
-
-NON-ATOMIC CONTEXT:
- You should use the `*sleep[_range]` family of functions.
- There are a few more options here, while any of them may
- work correctly, using the "right" sleep function will
- help the scheduler, power management, and just make your
- driver better :)
-
- -- Backed by busy-wait loop:
-
- udelay(unsigned long usecs)
-
- -- Backed by hrtimers:
-
- usleep_range(unsigned long min, unsigned long max)
-
- -- Backed by jiffies / legacy_timers
-
- msleep(unsigned long msecs)
- msleep_interruptible(unsigned long msecs)
-
- Unlike the `*delay` family, the underlying mechanism
- driving each of these calls varies, thus there are
- quirks you should be aware of.
-
-
- SLEEPING FOR "A FEW" USECS ( < ~10us? ):
- * Use udelay
-
- - Why not usleep?
- On slower systems, (embedded, OR perhaps a speed-
- stepped PC!) the overhead of setting up the hrtimers
- for usleep *may* not be worth it. Such an evaluation
- will obviously depend on your specific situation, but
- it is something to be aware of.
-
- SLEEPING FOR ~USECS OR SMALL MSECS ( 10us - 20ms):
- * Use usleep_range
-
- - Why not msleep for (1ms - 20ms)?
- Explained originally here:
- https://lore.kernel.org/r/15327.1186166232@lwn.net
-
- msleep(1~20) may not do what the caller intends, and
- will often sleep longer (~20 ms actual sleep for any
- value given in the 1~20ms range). In many cases this
- is not the desired behavior.
-
- - Why is there no "usleep" / What is a good range?
- Since usleep_range is built on top of hrtimers, the
- wakeup will be very precise (ish), thus a simple
- usleep function would likely introduce a large number
- of undesired interrupts.
-
- With the introduction of a range, the scheduler is
- free to coalesce your wakeup with any other wakeup
- that may have happened for other reasons, or at the
- worst case, fire an interrupt for your upper bound.
-
- The larger a range you supply, the greater a chance
- that you will not trigger an interrupt; this should
- be balanced with what is an acceptable upper bound on
- delay / performance for your specific code path. Exact
- tolerances here are very situation specific, thus it
- is left to the caller to determine a reasonable range.
-
- SLEEPING FOR LARGER MSECS ( 10ms+ )
- * Use msleep or possibly msleep_interruptible
-
- - What's the difference?
- msleep sets the current task to TASK_UNINTERRUPTIBLE
- whereas msleep_interruptible sets the current task to
- TASK_INTERRUPTIBLE before scheduling the sleep. In
- short, the difference is whether the sleep can be ended
- early by a signal. In general, just use msleep unless
- you know you have a need for the interruptible variant.
-
- FLEXIBLE SLEEPING (any delay, uninterruptible)
- * Use fsleep
diff --git a/MAINTAINERS b/MAINTAINERS
index c27f3190737f..3a24287712f1 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -2004,7 +2004,7 @@ F: Documentation/devicetree/bindings/mmc/owl-mmc.yaml
F: Documentation/devicetree/bindings/net/actions,owl-emac.yaml
F: Documentation/devicetree/bindings/pinctrl/actions,*
F: Documentation/devicetree/bindings/power/actions,owl-sps.txt
-F: Documentation/devicetree/bindings/timer/actions,owl-timer.txt
+F: Documentation/devicetree/bindings/timer/actions,owl-timer.yaml
F: arch/arm/boot/dts/actions/
F: arch/arm/mach-actions/
F: arch/arm64/boot/dts/actions/
@@ -10162,10 +10162,12 @@ S: Maintained
T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git timers/core
F: Documentation/timers/
F: include/linux/clockchips.h
+F: include/linux/delay.h
F: include/linux/hrtimer.h
F: include/linux/timer.h
F: kernel/time/clockevents.c
F: kernel/time/hrtimer.c
+F: kernel/time/sleep_timeout.c
F: kernel/time/timer.c
F: kernel/time/timer_list.c
F: kernel/time/timer_migration.*
diff --git a/arch/arm/kernel/smp_twd.c b/arch/arm/kernel/smp_twd.c
index 9a14f721a2b0..42a3706e16a6 100644
--- a/arch/arm/kernel/smp_twd.c
+++ b/arch/arm/kernel/smp_twd.c
@@ -93,7 +93,6 @@ static void twd_timer_stop(void)
{
struct clock_event_device *clk = raw_cpu_ptr(twd_evt);
- twd_shutdown(clk);
disable_percpu_irq(clk->irq);
}
diff --git a/arch/mips/ralink/Kconfig b/arch/mips/ralink/Kconfig
index 08c012a2591f..910d059ec70b 100644
--- a/arch/mips/ralink/Kconfig
+++ b/arch/mips/ralink/Kconfig
@@ -1,13 +1,6 @@
# SPDX-License-Identifier: GPL-2.0
if RALINK
-config CLKEVT_RT3352
- bool
- depends on SOC_RT305X || SOC_MT7620
- default y
- select TIMER_OF
- select CLKSRC_MMIO
-
config RALINK_ILL_ACC
bool
depends on SOC_RT305X
diff --git a/arch/mips/ralink/Makefile b/arch/mips/ralink/Makefile
index 26fabbdea1f1..0c109eae1953 100644
--- a/arch/mips/ralink/Makefile
+++ b/arch/mips/ralink/Makefile
@@ -10,8 +10,6 @@ ifndef CONFIG_MIPS_GIC
obj-y += clk.o timer.o
endif
-obj-$(CONFIG_CLKEVT_RT3352) += cevt-rt3352.o
-
obj-$(CONFIG_RALINK_ILL_ACC) += ill_acc.o
obj-$(CONFIG_IRQ_INTC) += irq.o
diff --git a/arch/powerpc/kernel/rtas.c b/arch/powerpc/kernel/rtas.c
index f7e86e09c49f..d31c9799cab2 100644
--- a/arch/powerpc/kernel/rtas.c
+++ b/arch/powerpc/kernel/rtas.c
@@ -1390,21 +1390,14 @@ bool __ref rtas_busy_delay(int status)
*/
ms = clamp(ms, 1U, 1000U);
/*
- * The delay hint is an order-of-magnitude suggestion, not
- * a minimum. It is fine, possibly even advantageous, for
- * us to pause for less time than hinted. For small values,
- * use usleep_range() to ensure we don't sleep much longer
- * than actually needed.
- *
- * See Documentation/timers/timers-howto.rst for
- * explanation of the threshold used here. In effect we use
- * usleep_range() for 9900 and 9901, msleep() for
- * 9902-9905.
+ * The delay hint is an order-of-magnitude suggestion, not a
+ * minimum. It is fine, possibly even advantageous, for us to
+ * pause for less time than hinted. To make sure pause time will
+ * not be way longer than requested independent of HZ
+ * configuration, use fsleep(). See fsleep() for details of
+ * used sleeping functions.
*/
- if (ms <= 20)
- usleep_range(ms * 100, ms * 1000);
- else
- msleep(ms);
+ fsleep(ms * 1000);
break;
case RTAS_BUSY:
ret = true;
diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig
index 2341393cfac1..26c01b9e3434 100644
--- a/arch/riscv/configs/defconfig
+++ b/arch/riscv/configs/defconfig
@@ -301,7 +301,6 @@ CONFIG_DEBUG_MEMORY_INIT=y
CONFIG_DEBUG_PER_CPU_MAPS=y
CONFIG_SOFTLOCKUP_DETECTOR=y
CONFIG_WQ_WATCHDOG=y
-CONFIG_DEBUG_TIMEKEEPING=y
CONFIG_DEBUG_RT_MUTEXES=y
CONFIG_DEBUG_SPINLOCK=y
CONFIG_DEBUG_MUTEXES=y
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 2852fcd82cbd..53a5eda8219c 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -145,7 +145,6 @@ config X86
select ARCH_HAS_PARANOID_L1D_FLUSH
select BUILDTIME_TABLE_SORT
select CLKEVT_I8253
- select CLOCKSOURCE_VALIDATE_LAST_CYCLE
select CLOCKSOURCE_WATCHDOG
# Word-size accesses may read uninitialized data past the trailing \0
# in strings and cause false KMSAN reports.
diff --git a/arch/x86/include/asm/timer.h b/arch/x86/include/asm/timer.h
index 7365dd4acffb..23baf8c9b34c 100644
--- a/arch/x86/include/asm/timer.h
+++ b/arch/x86/include/asm/timer.h
@@ -6,8 +6,6 @@
#include <linux/interrupt.h>
#include <linux/math64.h>
-#define TICK_SIZE (tick_nsec / 1000)
-
unsigned long long native_sched_clock(void);
extern void recalibrate_cpu_khz(void);
diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c
index 622fe24da910..a909b817b9c0 100644
--- a/arch/x86/kvm/xen.c
+++ b/arch/x86/kvm/xen.c
@@ -263,13 +263,6 @@ static void kvm_xen_stop_timer(struct kvm_vcpu *vcpu)
atomic_set(&vcpu->arch.xen.timer_pending, 0);
}
-static void kvm_xen_init_timer(struct kvm_vcpu *vcpu)
-{
- hrtimer_init(&vcpu->arch.xen.timer, CLOCK_MONOTONIC,
- HRTIMER_MODE_ABS_HARD);
- vcpu->arch.xen.timer.function = xen_timer_callback;
-}
-
static void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, bool atomic)
{
struct kvm_vcpu_xen *vx = &v->arch.xen;
@@ -1070,9 +1063,6 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data)
break;
}
- if (!vcpu->arch.xen.timer.function)
- kvm_xen_init_timer(vcpu);
-
/* Stop the timer (if it's running) before changing the vector */
kvm_xen_stop_timer(vcpu);
vcpu->arch.xen.timer_virq = data->u.timer.port;
@@ -2235,6 +2225,8 @@ void kvm_xen_init_vcpu(struct kvm_vcpu *vcpu)
vcpu->arch.xen.poll_evtchn = 0;
timer_setup(&vcpu->arch.xen.poll_timer, cancel_evtchn_poll, 0);
+ hrtimer_init(&vcpu->arch.xen.timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
+ vcpu->arch.xen.timer.function = xen_timer_callback;
kvm_gpc_init(&vcpu->arch.xen.runstate_cache, vcpu->kvm);
kvm_gpc_init(&vcpu->arch.xen.runstate2_cache, vcpu->kvm);
diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig
index 95dd4660b5b6..487c85259967 100644
--- a/drivers/clocksource/Kconfig
+++ b/drivers/clocksource/Kconfig
@@ -400,7 +400,8 @@ config ARM_GT_INITIAL_PRESCALER_VAL
This affects CPU_FREQ max delta from the initial frequency.
config ARM_TIMER_SP804
- bool "Support for Dual Timer SP804 module" if COMPILE_TEST
+ bool "Support for Dual Timer SP804 module"
+ depends on ARM || ARM64 || COMPILE_TEST
depends on GENERIC_SCHED_CLOCK && HAVE_CLK
select CLKSRC_MMIO
select TIMER_OF if OF
@@ -753,4 +754,13 @@ config EP93XX_TIMER
Enables support for the Cirrus Logic timer block
EP93XX.
+config RALINK_TIMER
+ bool "Ralink System Tick Counter"
+ depends on SOC_RT305X || SOC_MT7620 || COMPILE_TEST
+ select CLKSRC_MMIO
+ select TIMER_OF
+ help
+ Enables support for system tick counter present on
+ Ralink SoCs RT3352 and MT7620.
+
endmenu
diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile
index 22743785299e..43ef16a4efa6 100644
--- a/drivers/clocksource/Makefile
+++ b/drivers/clocksource/Makefile
@@ -91,3 +91,4 @@ obj-$(CONFIG_GOLDFISH_TIMER) += timer-goldfish.o
obj-$(CONFIG_GXP_TIMER) += timer-gxp.o
obj-$(CONFIG_CLKSRC_LOONGSON1_PWM) += timer-loongson1-pwm.o
obj-$(CONFIG_EP93XX_TIMER) += timer-ep93xx.o
+obj-$(CONFIG_RALINK_TIMER) += timer-ralink.o
diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c
index 03733101e231..808f259781fd 100644
--- a/drivers/clocksource/arm_arch_timer.c
+++ b/drivers/clocksource/arm_arch_timer.c
@@ -1179,8 +1179,6 @@ static void arch_timer_stop(struct clock_event_device *clk)
disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
if (arch_timer_has_nonsecure_ppi())
disable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
-
- clk->set_state_shutdown(clk);
}
static int arch_timer_dying_cpu(unsigned int cpu)
@@ -1430,7 +1428,7 @@ static int __init arch_timer_of_init(struct device_node *np)
arch_timers_present |= ARCH_TIMER_TYPE_CP15;
- has_names = of_property_read_bool(np, "interrupt-names");
+ has_names = of_property_present(np, "interrupt-names");
for (i = ARCH_TIMER_PHYS_SECURE_PPI; i < ARCH_TIMER_MAX_TIMER_PPI; i++) {
if (has_names)
diff --git a/drivers/clocksource/arm_global_timer.c b/drivers/clocksource/arm_global_timer.c
index a05cfaab5f84..2d86bbc2764a 100644
--- a/drivers/clocksource/arm_global_timer.c
+++ b/drivers/clocksource/arm_global_timer.c
@@ -195,7 +195,6 @@ static int gt_dying_cpu(unsigned int cpu)
{
struct clock_event_device *clk = this_cpu_ptr(gt_evt);
- gt_clockevent_shutdown(clk);
disable_percpu_irq(clk->irq);
return 0;
}
diff --git a/drivers/clocksource/dw_apb_timer.c b/drivers/clocksource/dw_apb_timer.c
index f5f24a95ee82..3a55ae5fe225 100644
--- a/drivers/clocksource/dw_apb_timer.c
+++ b/drivers/clocksource/dw_apb_timer.c
@@ -68,25 +68,6 @@ static inline void apbt_writel_relaxed(struct dw_apb_timer *timer, u32 val,
writel_relaxed(val, timer->base + offs);
}
-static void apbt_disable_int(struct dw_apb_timer *timer)
-{
- u32 ctrl = apbt_readl(timer, APBTMR_N_CONTROL);
-
- ctrl |= APBTMR_CONTROL_INT;
- apbt_writel(timer, ctrl, APBTMR_N_CONTROL);
-}
-
-/**
- * dw_apb_clockevent_pause() - stop the clock_event_device from running
- *
- * @dw_ced: The APB clock to stop generating events.
- */
-void dw_apb_clockevent_pause(struct dw_apb_clock_event_device *dw_ced)
-{
- disable_irq(dw_ced->timer.irq);
- apbt_disable_int(&dw_ced->timer);
-}
-
static void apbt_eoi(struct dw_apb_timer *timer)
{
apbt_readl_relaxed(timer, APBTMR_N_EOI);
@@ -284,26 +265,6 @@ dw_apb_clockevent_init(int cpu, const char *name, unsigned rating,
return dw_ced;
}
-/**
- * dw_apb_clockevent_resume() - resume a clock that has been paused.
- *
- * @dw_ced: The APB clock to resume.
- */
-void dw_apb_clockevent_resume(struct dw_apb_clock_event_device *dw_ced)
-{
- enable_irq(dw_ced->timer.irq);
-}
-
-/**
- * dw_apb_clockevent_stop() - stop the clock_event_device and release the IRQ.
- *
- * @dw_ced: The APB clock to stop generating the events.
- */
-void dw_apb_clockevent_stop(struct dw_apb_clock_event_device *dw_ced)
-{
- free_irq(dw_ced->timer.irq, &dw_ced->ced);
-}
-
/**
* dw_apb_clockevent_register() - register the clock with the generic layer
*
diff --git a/drivers/clocksource/exynos_mct.c b/drivers/clocksource/exynos_mct.c
index ef8cb1b71be4..e6a02e351d77 100644
--- a/drivers/clocksource/exynos_mct.c
+++ b/drivers/clocksource/exynos_mct.c
@@ -496,7 +496,6 @@ static int exynos4_mct_dying_cpu(unsigned int cpu)
per_cpu_ptr(&percpu_mct_tick, cpu);
struct clock_event_device *evt = &mevt->evt;
- evt->set_state_shutdown(evt);
if (mct_int_type == MCT_INT_SPI) {
if (evt->irq != -1)
disable_irq_nosync(evt->irq);
diff --git a/drivers/clocksource/mips-gic-timer.c b/drivers/clocksource/mips-gic-timer.c
index 110347707ff9..7907b740497a 100644
--- a/drivers/clocksource/mips-gic-timer.c
+++ b/drivers/clocksource/mips-gic-timer.c
@@ -166,6 +166,37 @@ static u64 gic_hpt_read(struct clocksource *cs)
return gic_read_count();
}
+static u64 gic_hpt_read_multicluster(struct clocksource *cs)
+{
+ unsigned int hi, hi2, lo;
+ u64 count;
+
+ mips_cm_lock_other(0, 0, 0, CM_GCR_Cx_OTHER_BLOCK_GLOBAL);
+
+ if (mips_cm_is64) {
+ count = read_gic_redir_counter();
+ goto out;
+ }
+
+ hi = read_gic_redir_counter_32h();
+ while (true) {
+ lo = read_gic_redir_counter_32l();
+
+ /* If hi didn't change then lo didn't wrap & we're done */
+ hi2 = read_gic_redir_counter_32h();
+ if (hi2 == hi)
+ break;
+
+ /* Otherwise, repeat with the latest hi value */
+ hi = hi2;
+ }
+
+ count = (((u64)hi) << 32) + lo;
+out:
+ mips_cm_unlock_other();
+ return count;
+}
+
static struct clocksource gic_clocksource = {
.name = "GIC",
.read = gic_hpt_read,
@@ -203,6 +234,11 @@ static int __init __gic_clocksource_init(void)
gic_clocksource.rating = 200;
gic_clocksource.rating += clamp(gic_frequency / 10000000, 0, 99);
+ if (mips_cps_multicluster_cpus()) {
+ gic_clocksource.read = &gic_hpt_read_multicluster;
+ gic_clocksource.vdso_clock_mode = VDSO_CLOCKMODE_NONE;
+ }
+
ret = clocksource_register_hz(&gic_clocksource, gic_frequency);
if (ret < 0)
pr_warn("Unable to register clocksource\n");
@@ -261,7 +297,8 @@ static int __init gic_clocksource_of_init(struct device_node *node)
* stable CPU frequency or on the platforms with CM3 and CPU frequency
* change performed by the CPC core clocks divider.
*/
- if (mips_cm_revision() >= CM_REV_CM3 || !IS_ENABLED(CONFIG_CPU_FREQ)) {
+ if ((mips_cm_revision() >= CM_REV_CM3 || !IS_ENABLED(CONFIG_CPU_FREQ)) &&
+ !mips_cps_multicluster_cpus()) {
sched_clock_register(mips_cm_is64 ?
gic_read_count_64 : gic_read_count_2x32,
gic_count_width, gic_frequency);
diff --git a/drivers/clocksource/timer-armada-370-xp.c b/drivers/clocksource/timer-armada-370-xp.c
index 6ec565d6939a..54284c1c0651 100644
--- a/drivers/clocksource/timer-armada-370-xp.c
+++ b/drivers/clocksource/timer-armada-370-xp.c
@@ -201,7 +201,6 @@ static int armada_370_xp_timer_dying_cpu(unsigned int cpu)
{
struct clock_event_device *evt = per_cpu_ptr(armada_370_xp_evt, cpu);
- evt->set_state_shutdown(evt);
disable_percpu_irq(evt->irq);
return 0;
}
diff --git a/drivers/clocksource/timer-gxp.c b/drivers/clocksource/timer-gxp.c
index 57aa2e2cce53..48a73c101eb8 100644
--- a/drivers/clocksource/timer-gxp.c
+++ b/drivers/clocksource/timer-gxp.c
@@ -85,7 +85,7 @@ static int __init gxp_timer_init(struct device_node *node)
clk = of_clk_get(node, 0);
if (IS_ERR(clk)) {
- ret = (int)PTR_ERR(clk);
+ ret = PTR_ERR(clk);
pr_err("%pOFn clock not found: %d\n", node, ret);
goto err_free;
}
diff --git a/drivers/clocksource/timer-qcom.c b/drivers/clocksource/timer-qcom.c
index eac4c95c6127..ddb1debe6a6b 100644
--- a/drivers/clocksource/timer-qcom.c
+++ b/drivers/clocksource/timer-qcom.c
@@ -130,7 +130,6 @@ static int msm_local_timer_dying_cpu(unsigned int cpu)
{
struct clock_event_device *evt = per_cpu_ptr(msm_evt, cpu);
- evt->set_state_shutdown(evt);
disable_percpu_irq(evt->irq);
return 0;
}
diff --git a/arch/mips/ralink/cevt-rt3352.c b/drivers/clocksource/timer-ralink.c
similarity index 91%
rename from arch/mips/ralink/cevt-rt3352.c
rename to drivers/clocksource/timer-ralink.c
index 269d4877d120..6ecdb4228f76 100644
--- a/arch/mips/ralink/cevt-rt3352.c
+++ b/drivers/clocksource/timer-ralink.c
@@ -1,7 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0
/*
- * This file is subject to the terms and conditions of the GNU General Public
- * License. See the file "COPYING" in the main directory of this archive
- * for more details.
+ * Ralink System Tick Counter driver present on RT3352 and MT7620 SoCs.
*
* Copyright (C) 2013 by John Crispin <john@phrozen.org>
*/
@@ -16,8 +15,6 @@
#include <linux/of_irq.h>
#include <linux/of_address.h>
-#include <asm/mach-ralink/ralink_regs.h>
-
#define SYSTICK_FREQ (50 * 1000)
#define SYSTICK_CONFIG 0x00
@@ -40,7 +37,7 @@ static int systick_set_oneshot(struct clock_event_device *evt);
static int systick_shutdown(struct clock_event_device *evt);
static int systick_next_event(unsigned long delta,
- struct clock_event_device *evt)
+ struct clock_event_device *evt)
{
struct systick_device *sdev;
u32 count;
@@ -60,7 +57,7 @@ static void systick_event_handler(struct clock_event_device *dev)
static irqreturn_t systick_interrupt(int irq, void *dev_id)
{
- struct clock_event_device *dev = (struct clock_event_device *) dev_id;
+ struct clock_event_device *dev = (struct clock_event_device *)dev_id;
dev->event_handler(dev);
diff --git a/drivers/clocksource/timer-tegra.c b/drivers/clocksource/timer-tegra.c
index e9635c25eef4..35b6ce9deffa 100644
--- a/drivers/clocksource/timer-tegra.c
+++ b/drivers/clocksource/timer-tegra.c
@@ -158,7 +158,6 @@ static int tegra_timer_stop(unsigned int cpu)
{
struct timer_of *to = per_cpu_ptr(&tegra_to, cpu);
- to->clkevt.set_state_shutdown(&to->clkevt);
disable_irq_nosync(to->clkevt.irq);
return 0;
diff --git a/drivers/clocksource/timer-ti-dm-systimer.c b/drivers/clocksource/timer-ti-dm-systimer.c
index c2dcd8d68e45..985a6d08512b 100644
--- a/drivers/clocksource/timer-ti-dm-systimer.c
+++ b/drivers/clocksource/timer-ti-dm-systimer.c
@@ -202,10 +202,10 @@ static bool __init dmtimer_is_preferred(struct device_node *np)
/* Secure gptimer12 is always clocked with a fixed source */
if (!of_property_read_bool(np, "ti,timer-secure")) {
- if (!of_property_read_bool(np, "assigned-clocks"))
+ if (!of_property_present(np, "assigned-clocks"))
return false;
- if (!of_property_read_bool(np, "assigned-clock-parents"))
+ if (!of_property_present(np, "assigned-clock-parents"))
return false;
}
@@ -686,9 +686,9 @@ subsys_initcall(dmtimer_percpu_timer_startup);
static int __init dmtimer_percpu_quirk_init(struct device_node *np, u32 pa)
{
- struct device_node *arm_timer;
+ struct device_node *arm_timer __free(device_node) =
+ of_find_compatible_node(NULL, NULL, "arm,armv7-timer");
- arm_timer = of_find_compatible_node(NULL, NULL, "arm,armv7-timer");
if (of_device_is_available(arm_timer)) {
pr_warn_once("ARM architected timer wrap issue i940 detected\n");
return 0;
diff --git a/drivers/clocksource/timer-ti-dm.c b/drivers/clocksource/timer-ti-dm.c
index b7a34b1a975e..3666d94cc8dd 100644
--- a/drivers/clocksource/timer-ti-dm.c
+++ b/drivers/clocksource/timer-ti-dm.c
@@ -1104,8 +1104,12 @@ static int omap_dm_timer_probe(struct platform_device *pdev)
return -ENOMEM;
timer->irq = platform_get_irq(pdev, 0);
- if (timer->irq < 0)
- return timer->irq;
+ if (timer->irq < 0) {
+ if (of_property_read_bool(dev->of_node, "ti,timer-pwm"))
+ dev_info(dev, "Did not find timer interrupt, timer usable in PWM mode only\n");
+ else
+ return timer->irq;
+ }
timer->io_base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(timer->io_base))
diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c
index 519e096c607c..8f62cfa23fb7 100644
--- a/drivers/gpu/drm/i915/i915_request.c
+++ b/drivers/gpu/drm/i915/i915_request.c
@@ -273,11 +273,6 @@ i915_request_active_engine(struct i915_request *rq,
return ret;
}
-static void __rq_init_watchdog(struct i915_request *rq)
-{
- rq->watchdog.timer.function = NULL;
-}
-
static enum hrtimer_restart __rq_watchdog_expired(struct hrtimer *hrtimer)
{
struct i915_request *rq =
@@ -294,6 +289,14 @@ static enum hrtimer_restart __rq_watchdog_expired(struct hrtimer *hrtimer)
return HRTIMER_NORESTART;
}
+static void __rq_init_watchdog(struct i915_request *rq)
+{
+ struct i915_request_watchdog *wdg = &rq->watchdog;
+
+ hrtimer_init(&wdg->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ wdg->timer.function = __rq_watchdog_expired;
+}
+
static void __rq_arm_watchdog(struct i915_request *rq)
{
struct i915_request_watchdog *wdg = &rq->watchdog;
@@ -304,8 +307,6 @@ static void __rq_arm_watchdog(struct i915_request *rq)
i915_request_get(rq);
- hrtimer_init(&wdg->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
- wdg->timer.function = __rq_watchdog_expired;
hrtimer_start_range_ns(&wdg->timer,
ns_to_ktime(ce->watchdog.timeout_us *
NSEC_PER_USEC),
@@ -317,7 +318,7 @@ static void __rq_cancel_watchdog(struct i915_request *rq)
{
struct i915_request_watchdog *wdg = &rq->watchdog;
- if (wdg->timer.function && hrtimer_try_to_cancel(&wdg->timer) > 0)
+ if (hrtimer_try_to_cancel(&wdg->timer) > 0)
i915_request_put(rq);
}
diff --git a/drivers/media/usb/dvb-usb-v2/anysee.c b/drivers/media/usb/dvb-usb-v2/anysee.c
index 8699846eb416..bea12cdc85e8 100644
--- a/drivers/media/usb/dvb-usb-v2/anysee.c
+++ b/drivers/media/usb/dvb-usb-v2/anysee.c
@@ -46,24 +46,15 @@ static int anysee_ctrl_msg(struct dvb_usb_device *d,
dev_dbg(&d->udev->dev, "%s: >>> %*ph\n", __func__, slen, state->buf);
- /* We need receive one message more after dvb_usb_generic_rw due
- to weird transaction flow, which is 1 x send + 2 x receive. */
+ /*
+ * We need receive one message more after dvb_usbv2_generic_rw_locked()
+ * due to weird transaction flow, which is 1 x send + 2 x receive.
+ */
ret = dvb_usbv2_generic_rw_locked(d, state->buf, sizeof(state->buf),
state->buf, sizeof(state->buf));
if (ret)
goto error_unlock;
- /* TODO FIXME: dvb_usb_generic_rw() fails rarely with error code -32
- * (EPIPE, Broken pipe). Function supports currently msleep() as a
- * parameter but I would not like to use it, since according to
- * Documentation/timers/timers-howto.rst it should not be used such
- * short, under < 20ms, sleeps. Repeating failed message would be
- * better choice as not to add unwanted delays...
- * Fixing that correctly is one of those or both;
- * 1) use repeat if possible
- * 2) add suitable delay
- */
-
/* get answer, retry few times if error returned */
for (i = 0; i < 3; i++) {
/* receive 2nd answer */
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c b/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c
index 8fd22c69855f..a6d50149e0c3 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c
@@ -823,8 +823,6 @@ int rt2x00usb_probe(struct usb_interface *usb_intf,
INIT_WORK(&rt2x00dev->rxdone_work, rt2x00usb_work_rxdone);
INIT_WORK(&rt2x00dev->txdone_work, rt2x00usb_work_txdone);
- hrtimer_init(&rt2x00dev->txstatus_timer, CLOCK_MONOTONIC,
- HRTIMER_MODE_REL);
retval = rt2x00usb_alloc_reg(rt2x00dev);
if (retval)
diff --git a/drivers/power/supply/charger-manager.c b/drivers/power/supply/charger-manager.c
index 96f0a7fbf105..09ec0ecf1486 100644
--- a/drivers/power/supply/charger-manager.c
+++ b/drivers/power/supply/charger-manager.c
@@ -1412,10 +1412,9 @@ static inline struct charger_desc *cm_get_drv_data(struct platform_device *pdev)
return dev_get_platdata(&pdev->dev);
}
-static enum alarmtimer_restart cm_timer_func(struct alarm *alarm, ktime_t now)
+static void cm_timer_func(struct alarm *alarm, ktime_t now)
{
cm_timer_set = false;
- return ALARMTIMER_NORESTART;
}
static int charger_manager_probe(struct platform_device *pdev)
diff --git a/fs/aio.c b/fs/aio.c
index e8920178b50f..a5d331f29943 100644
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -1335,7 +1335,7 @@ static long read_events(struct kioctx *ctx, long min_nr, long nr,
if (until == 0 || ret < 0 || ret >= min_nr)
return ret;
- hrtimer_init_sleeper_on_stack(&t, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ hrtimer_setup_sleeper_on_stack(&t, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
if (until != KTIME_MAX) {
hrtimer_set_expires_range_ns(&t.timer, until, current->timer_slack_ns);
hrtimer_sleeper_start_expires(&t, HRTIMER_MODE_REL);
diff --git a/fs/proc/base.c b/fs/proc/base.c
index b31283d81c52..6a37a43241e4 100644
--- a/fs/proc/base.c
+++ b/fs/proc/base.c
@@ -2553,8 +2553,8 @@ static int show_timer(struct seq_file *m, void *v)
seq_printf(m, "ID: %d\n", timer->it_id);
seq_printf(m, "signal: %d/%px\n",
- timer->sigq->info.si_signo,
- timer->sigq->info.si_value.sival_ptr);
+ timer->sigq.info.si_signo,
+ timer->sigq.info.si_value.sival_ptr);
seq_printf(m, "notify: %s/%s.%d\n",
nstr[notify & ~SIGEV_THREAD_ID],
(notify & SIGEV_THREAD_ID) ? "tid" : "pid",
diff --git a/fs/timerfd.c b/fs/timerfd.c
index 137523e0bb21..f10c99ad5c60 100644
--- a/fs/timerfd.c
+++ b/fs/timerfd.c
@@ -79,13 +79,11 @@ static enum hrtimer_restart timerfd_tmrproc(struct hrtimer *htmr)
return HRTIMER_NORESTART;
}
-static enum alarmtimer_restart timerfd_alarmproc(struct alarm *alarm,
- ktime_t now)
+static void timerfd_alarmproc(struct alarm *alarm, ktime_t now)
{
struct timerfd_ctx *ctx = container_of(alarm, struct timerfd_ctx,
t.alarm);
timerfd_triggered(ctx);
- return ALARMTIMER_NORESTART;
}
/*
diff --git a/include/asm-generic/delay.h b/include/asm-generic/delay.h
index e448ac61430c..76cf237b6e4c 100644
--- a/include/asm-generic/delay.h
+++ b/include/asm-generic/delay.h
@@ -2,6 +2,9 @@
#ifndef __ASM_GENERIC_DELAY_H
#define __ASM_GENERIC_DELAY_H
+#include <linux/math.h>
+#include <vdso/time64.h>
+
/* Undefined functions to get compile-time errors */
extern void __bad_udelay(void);
extern void __bad_ndelay(void);
@@ -12,34 +15,73 @@ extern void __const_udelay(unsigned long xloops);
extern void __delay(unsigned long loops);
/*
- * The weird n/20000 thing suppresses a "comparison is always false due to
- * limited range of data type" warning with non-const 8-bit arguments.
+ * The microseconds/nanosecond delay multiplicators are used to convert a
+ * constant microseconds/nanoseconds value to a value which can be used by the
+ * architectures specific implementation to transform it into loops.
+ */
+#define UDELAY_CONST_MULT ((unsigned long)DIV_ROUND_UP(1ULL << 32, USEC_PER_SEC))
+#define NDELAY_CONST_MULT ((unsigned long)DIV_ROUND_UP(1ULL << 32, NSEC_PER_SEC))
+
+/*
+ * The maximum constant udelay/ndelay value picked out of thin air to prevent
+ * too long constant udelays/ndelays.
*/
+#define DELAY_CONST_MAX 20000
-/* 0x10c7 is 2**32 / 1000000 (rounded up) */
-#define udelay(n) \
- ({ \
- if (__builtin_constant_p(n)) { \
- if ((n) / 20000 >= 1) \
- __bad_udelay(); \
- else \
- __const_udelay((n) * 0x10c7ul); \
- } else { \
- __udelay(n); \
- } \
- })
-
-/* 0x5 is 2**32 / 1000000000 (rounded up) */
-#define ndelay(n) \
- ({ \
- if (__builtin_constant_p(n)) { \
- if ((n) / 20000 >= 1) \
- __bad_ndelay(); \
- else \
- __const_udelay((n) * 5ul); \
- } else { \
- __ndelay(n); \
- } \
- })
+/**
+ * udelay - Inserting a delay based on microseconds with busy waiting
+ * @usec: requested delay in microseconds
+ *
+ * When delaying in an atomic context ndelay(), udelay() and mdelay() are the
+ * only valid variants of delaying/sleeping to go with.
+ *
+ * When inserting delays in non atomic context which are shorter than the time
+ * which is required to queue e.g. an hrtimer and to enter then the scheduler,
+ * it is also valuable to use udelay(). But it is not simple to specify a
+ * generic threshold for this which will fit for all systems. An approximation
+ * is a threshold for all delays up to 10 microseconds.
+ *
+ * When having a delay which is larger than the architecture specific
+ * %MAX_UDELAY_MS value, please make sure mdelay() is used. Otherwise a overflow
+ * risk is given.
+ *
+ * Please note that ndelay(), udelay() and mdelay() may return early for several
+ * reasons (https://lists.openwall.net/linux-kernel/2011/01/09/56):
+ *
+ * #. computed loops_per_jiffy too low (due to the time taken to execute the
+ * timer interrupt.)
+ * #. cache behaviour affecting the time it takes to execute the loop function.
+ * #. CPU clock rate changes.
+ */
+static __always_inline void udelay(unsigned long usec)
+{
+ if (__builtin_constant_p(usec)) {
+ if (usec >= DELAY_CONST_MAX)
+ __bad_udelay();
+ else
+ __const_udelay(usec * UDELAY_CONST_MULT);
+ } else {
+ __udelay(usec);
+ }
+}
+
+/**
+ * ndelay - Inserting a delay based on nanoseconds with busy waiting
+ * @nsec: requested delay in nanoseconds
+ *
+ * See udelay() for basic information about ndelay() and it's variants.
+ */
+static __always_inline void ndelay(unsigned long nsec)
+{
+ if (__builtin_constant_p(nsec)) {
+ if (nsec >= DELAY_CONST_MAX)
+ __bad_udelay();
+ else
+ __const_udelay(nsec * NDELAY_CONST_MULT);
+ } else {
+ __udelay(nsec);
+ }
+}
+#define ndelay(x) ndelay(x)
#endif /* __ASM_GENERIC_DELAY_H */
diff --git a/include/linux/alarmtimer.h b/include/linux/alarmtimer.h
index 05e758b8b894..3ffa5341dce2 100644
--- a/include/linux/alarmtimer.h
+++ b/include/linux/alarmtimer.h
@@ -20,12 +20,6 @@ enum alarmtimer_type {
ALARM_BOOTTIME_FREEZER,
};
-enum alarmtimer_restart {
- ALARMTIMER_NORESTART,
- ALARMTIMER_RESTART,
-};
-
-
#define ALARMTIMER_STATE_INACTIVE 0x00
#define ALARMTIMER_STATE_ENQUEUED 0x01
@@ -42,14 +36,14 @@ enum alarmtimer_restart {
struct alarm {
struct timerqueue_node node;
struct hrtimer timer;
- enum alarmtimer_restart (*function)(struct alarm *, ktime_t now);
+ void (*function)(struct alarm *, ktime_t now);
enum alarmtimer_type type;
int state;
void *data;
};
void alarm_init(struct alarm *alarm, enum alarmtimer_type type,
- enum alarmtimer_restart (*function)(struct alarm *, ktime_t));
+ void (*function)(struct alarm *, ktime_t));
void alarm_start(struct alarm *alarm, ktime_t start);
void alarm_start_relative(struct alarm *alarm, ktime_t start);
void alarm_restart(struct alarm *alarm);
diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h
index d35b677b08fe..ef1b16da6ad5 100644
--- a/include/linux/clocksource.h
+++ b/include/linux/clocksource.h
@@ -215,7 +215,6 @@ static inline s64 clocksource_cyc2ns(u64 cycles, u32 mult, u32 shift)
extern int clocksource_unregister(struct clocksource*);
extern void clocksource_touch_watchdog(void);
-extern void clocksource_change_rating(struct clocksource *cs, int rating);
extern void clocksource_suspend(void);
extern void clocksource_resume(void);
extern struct clocksource * __init clocksource_default_clock(void);
diff --git a/include/linux/delay.h b/include/linux/delay.h
index ff9cda975e30..89866bab100d 100644
--- a/include/linux/delay.h
+++ b/include/linux/delay.h
@@ -6,21 +6,12 @@
* Copyright (C) 1993 Linus Torvalds
*
* Delay routines, using a pre-computed "loops_per_jiffy" value.
- *
- * Please note that ndelay(), udelay() and mdelay() may return early for
- * several reasons:
- * 1. computed loops_per_jiffy too low (due to the time taken to
- * execute the timer interrupt.)
- * 2. cache behaviour affecting the time it takes to execute the
- * loop function.
- * 3. CPU clock rate changes.
- *
- * Please see this thread:
- * https://lists.openwall.net/linux-kernel/2011/01/09/56
+ * Sleep routines using timer list timers or hrtimers.
*/
#include <linux/math.h>
#include <linux/sched.h>
+#include <linux/jiffies.h>
extern unsigned long loops_per_jiffy;
@@ -35,12 +26,21 @@ extern unsigned long loops_per_jiffy;
* The 2nd mdelay() definition ensures GCC will optimize away the
* while loop for the common cases where n <= MAX_UDELAY_MS -- Paul G.
*/
-
#ifndef MAX_UDELAY_MS
#define MAX_UDELAY_MS 5
#endif
#ifndef mdelay
+/**
+ * mdelay - Inserting a delay based on milliseconds with busy waiting
+ * @n: requested delay in milliseconds
+ *
+ * See udelay() for basic information about mdelay() and it's variants.
+ *
+ * Please double check, whether mdelay() is the right way to go or whether a
+ * refactoring of the code is the better variant to be able to use msleep()
+ * instead.
+ */
#define mdelay(n) (\
(__builtin_constant_p(n) && (n)<=MAX_UDELAY_MS) ? udelay((n)*1000) : \
({unsigned long __ms=(n); while (__ms--) udelay(1000);}))
@@ -63,30 +63,75 @@ unsigned long msleep_interruptible(unsigned int msecs);
void usleep_range_state(unsigned long min, unsigned long max,
unsigned int state);
+/**
+ * usleep_range - Sleep for an approximate time
+ * @min: Minimum time in microseconds to sleep
+ * @max: Maximum time in microseconds to sleep
+ *
+ * For basic information please refere to usleep_range_state().
+ *
+ * The task will be in the state TASK_UNINTERRUPTIBLE during the sleep.
+ */
static inline void usleep_range(unsigned long min, unsigned long max)
{
usleep_range_state(min, max, TASK_UNINTERRUPTIBLE);
}
-static inline void usleep_idle_range(unsigned long min, unsigned long max)
+/**
+ * usleep_range_idle - Sleep for an approximate time with idle time accounting
+ * @min: Minimum time in microseconds to sleep
+ * @max: Maximum time in microseconds to sleep
+ *
+ * For basic information please refere to usleep_range_state().
+ *
+ * The sleeping task has the state TASK_IDLE during the sleep to prevent
+ * contribution to the load avarage.
+ */
+static inline void usleep_range_idle(unsigned long min, unsigned long max)
{
usleep_range_state(min, max, TASK_IDLE);
}
+/**
+ * ssleep - wrapper for seconds around msleep
+ * @seconds: Requested sleep duration in seconds
+ *
+ * Please refere to msleep() for detailed information.
+ */
static inline void ssleep(unsigned int seconds)
{
msleep(seconds * 1000);
}
-/* see Documentation/timers/timers-howto.rst for the thresholds */
+static const unsigned int max_slack_shift = 2;
+#define USLEEP_RANGE_UPPER_BOUND ((TICK_NSEC << max_slack_shift) / NSEC_PER_USEC)
+
+/**
+ * fsleep - flexible sleep which autoselects the best mechanism
+ * @usecs: requested sleep duration in microseconds
+ *
+ * flseep() selects the best mechanism that will provide maximum 25% slack
+ * to the requested sleep duration. Therefore it uses:
+ *
+ * * udelay() loop for sleep durations <= 10 microseconds to avoid hrtimer
+ * overhead for really short sleep durations.
+ * * usleep_range() for sleep durations which would lead with the usage of
+ * msleep() to a slack larger than 25%. This depends on the granularity of
+ * jiffies.
+ * * msleep() for all other sleep durations.
+ *
+ * Note: When %CONFIG_HIGH_RES_TIMERS is not set, all sleeps are processed with
+ * the granularity of jiffies and the slack might exceed 25% especially for
+ * short sleep durations.
+ */
static inline void fsleep(unsigned long usecs)
{
if (usecs <= 10)
udelay(usecs);
- else if (usecs <= 20000)
- usleep_range(usecs, 2 * usecs);
+ else if (usecs < USLEEP_RANGE_UPPER_BOUND)
+ usleep_range(usecs, usecs + (usecs >> max_slack_shift));
else
- msleep(DIV_ROUND_UP(usecs, 1000));
+ msleep(DIV_ROUND_UP(usecs, USEC_PER_MSEC));
}
#endif /* defined(_LINUX_DELAY_H) */
diff --git a/include/linux/dw_apb_timer.h b/include/linux/dw_apb_timer.h
index 82ebf9223948..f8811c46b89e 100644
--- a/include/linux/dw_apb_timer.h
+++ b/include/linux/dw_apb_timer.h
@@ -34,9 +34,6 @@ struct dw_apb_clocksource {
};
void dw_apb_clockevent_register(struct dw_apb_clock_event_device *dw_ced);
-void dw_apb_clockevent_pause(struct dw_apb_clock_event_device *dw_ced);
-void dw_apb_clockevent_resume(struct dw_apb_clock_event_device *dw_ced);
-void dw_apb_clockevent_stop(struct dw_apb_clock_event_device *dw_ced);
struct dw_apb_clock_event_device *
dw_apb_clockevent_init(int cpu, const char *name, unsigned rating,
diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h
index aa1e65ccb615..7ef5f7ef31a9 100644
--- a/include/linux/hrtimer.h
+++ b/include/linux/hrtimer.h
@@ -228,32 +228,17 @@ static inline void hrtimer_cancel_wait_running(struct hrtimer *timer)
/* Initialize timers: */
extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock,
enum hrtimer_mode mode);
-extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id,
- enum hrtimer_mode mode);
+extern void hrtimer_setup(struct hrtimer *timer, enum hrtimer_restart (*function)(struct hrtimer *),
+ clockid_t clock_id, enum hrtimer_mode mode);
+extern void hrtimer_setup_on_stack(struct hrtimer *timer,
+ enum hrtimer_restart (*function)(struct hrtimer *),
+ clockid_t clock_id, enum hrtimer_mode mode);
+extern void hrtimer_setup_sleeper_on_stack(struct hrtimer_sleeper *sl, clockid_t clock_id,
+ enum hrtimer_mode mode);
#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
-extern void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t which_clock,
- enum hrtimer_mode mode);
-extern void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
- clockid_t clock_id,
- enum hrtimer_mode mode);
-
extern void destroy_hrtimer_on_stack(struct hrtimer *timer);
#else
-static inline void hrtimer_init_on_stack(struct hrtimer *timer,
- clockid_t which_clock,
- enum hrtimer_mode mode)
-{
- hrtimer_init(timer, which_clock, mode);
-}
-
-static inline void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
- clockid_t clock_id,
- enum hrtimer_mode mode)
-{
- hrtimer_init_sleeper(sl, clock_id, mode);
-}
-
static inline void destroy_hrtimer_on_stack(struct hrtimer *timer) { }
#endif
@@ -337,6 +322,28 @@ static inline int hrtimer_callback_running(struct hrtimer *timer)
return timer->base->running == timer;
}
+/**
+ * hrtimer_update_function - Update the timer's callback function
+ * @timer: Timer to update
+ * @function: New callback function
+ *
+ * Only safe to call if the timer is not enqueued. Can be called in the callback function if the
+ * timer is not enqueued at the same time (see the comments above HRTIMER_STATE_ENQUEUED).
+ */
+static inline void hrtimer_update_function(struct hrtimer *timer,
+ enum hrtimer_restart (*function)(struct hrtimer *))
+{
+ guard(raw_spinlock_irqsave)(&timer->base->cpu_base->lock);
+
+ if (WARN_ON_ONCE(hrtimer_is_queued(timer)))
+ return;
+
+ if (WARN_ON_ONCE(!function))
+ return;
+
+ timer->function = function;
+}
+
/* Forward a hrtimer so it expires after now: */
extern u64
hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval);
diff --git a/include/linux/iopoll.h b/include/linux/iopoll.h
index 19a7b00baff4..91324c331a4b 100644
--- a/include/linux/iopoll.h
+++ b/include/linux/iopoll.h
@@ -19,19 +19,19 @@
* @op: accessor function (takes @args as its arguments)
* @val: Variable to read the value into
* @cond: Break condition (usually involving @val)
- * @sleep_us: Maximum time to sleep between reads in us (0
- * tight-loops). Should be less than ~20ms since usleep_range
- * is used (see Documentation/timers/timers-howto.rst).
+ * @sleep_us: Maximum time to sleep between reads in us (0 tight-loops). Please
+ * read usleep_range() function description for details and
+ * limitations.
* @timeout_us: Timeout in us, 0 means never timeout
* @sleep_before_read: if it is true, sleep @sleep_us before read.
* @args: arguments for @op poll
*
- * Returns 0 on success and -ETIMEDOUT upon a timeout. In either
- * case, the last read value at @args is stored in @val. Must not
- * be called from atomic context if sleep_us or timeout_us are used.
- *
* When available, you'll probably want to use one of the specialized
* macros defined below rather than this macro directly.
+ *
+ * Returns: 0 on success and -ETIMEDOUT upon a timeout. In either
+ * case, the last read value at @args is stored in @val. Must not
+ * be called from atomic context if sleep_us or timeout_us are used.
*/
#define read_poll_timeout(op, val, cond, sleep_us, timeout_us, \
sleep_before_read, args...) \
@@ -64,22 +64,22 @@
* @op: accessor function (takes @args as its arguments)
* @val: Variable to read the value into
* @cond: Break condition (usually involving @val)
- * @delay_us: Time to udelay between reads in us (0 tight-loops). Should
- * be less than ~10us since udelay is used (see
- * Documentation/timers/timers-howto.rst).
+ * @delay_us: Time to udelay between reads in us (0 tight-loops). Please
+ * read udelay() function description for details and
+ * limitations.
* @timeout_us: Timeout in us, 0 means never timeout
* @delay_before_read: if it is true, delay @delay_us before read.
* @args: arguments for @op poll
*
- * Returns 0 on success and -ETIMEDOUT upon a timeout. In either
- * case, the last read value at @args is stored in @val.
- *
* This macro does not rely on timekeeping. Hence it is safe to call even when
* timekeeping is suspended, at the expense of an underestimation of wall clock
* time, which is rather minimal with a non-zero delay_us.
*
* When available, you'll probably want to use one of the specialized
* macros defined below rather than this macro directly.
+ *
+ * Returns: 0 on success and -ETIMEDOUT upon a timeout. In either
+ * case, the last read value at @args is stored in @val.
*/
#define read_poll_timeout_atomic(op, val, cond, delay_us, timeout_us, \
delay_before_read, args...) \
@@ -119,17 +119,17 @@
* @addr: Address to poll
* @val: Variable to read the value into
* @cond: Break condition (usually involving @val)
- * @sleep_us: Maximum time to sleep between reads in us (0
- * tight-loops). Should be less than ~20ms since usleep_range
- * is used (see Documentation/timers/timers-howto.rst).
+ * @sleep_us: Maximum time to sleep between reads in us (0 tight-loops). Please
+ * read usleep_range() function description for details and
+ * limitations.
* @timeout_us: Timeout in us, 0 means never timeout
*
- * Returns 0 on success and -ETIMEDOUT upon a timeout. In either
- * case, the last read value at @addr is stored in @val. Must not
- * be called from atomic context if sleep_us or timeout_us are used.
- *
* When available, you'll probably want to use one of the specialized
* macros defined below rather than this macro directly.
+ *
+ * Returns: 0 on success and -ETIMEDOUT upon a timeout. In either
+ * case, the last read value at @addr is stored in @val. Must not
+ * be called from atomic context if sleep_us or timeout_us are used.
*/
#define readx_poll_timeout(op, addr, val, cond, sleep_us, timeout_us) \
read_poll_timeout(op, val, cond, sleep_us, timeout_us, false, addr)
@@ -140,16 +140,16 @@
* @addr: Address to poll
* @val: Variable to read the value into
* @cond: Break condition (usually involving @val)
- * @delay_us: Time to udelay between reads in us (0 tight-loops). Should
- * be less than ~10us since udelay is used (see
- * Documentation/timers/timers-howto.rst).
+ * @delay_us: Time to udelay between reads in us (0 tight-loops). Please
+ * read udelay() function description for details and
+ * limitations.
* @timeout_us: Timeout in us, 0 means never timeout
*
- * Returns 0 on success and -ETIMEDOUT upon a timeout. In either
- * case, the last read value at @addr is stored in @val.
- *
* When available, you'll probably want to use one of the specialized
* macros defined below rather than this macro directly.
+ *
+ * Returns: 0 on success and -ETIMEDOUT upon a timeout. In either
+ * case, the last read value at @addr is stored in @val.
*/
#define readx_poll_timeout_atomic(op, addr, val, cond, delay_us, timeout_us) \
read_poll_timeout_atomic(op, val, cond, delay_us, timeout_us, false, addr)
diff --git a/include/linux/jiffies.h b/include/linux/jiffies.h
index 1220f0fbe5bf..ed945f42e064 100644
--- a/include/linux/jiffies.h
+++ b/include/linux/jiffies.h
@@ -502,7 +502,7 @@ static inline unsigned long _msecs_to_jiffies(const unsigned int m)
* - all other values are converted to jiffies by either multiplying
* the input value by a factor or dividing it with a factor and
* handling any 32-bit overflows.
- * for the details see __msecs_to_jiffies()
+ * for the details see _msecs_to_jiffies()
*
* msecs_to_jiffies() checks for the passed in value being a constant
* via __builtin_constant_p() allowing gcc to eliminate most of the
@@ -526,6 +526,19 @@ static __always_inline unsigned long msecs_to_jiffies(const unsigned int m)
}
}
+/**
+ * secs_to_jiffies: - convert seconds to jiffies
+ * @_secs: time in seconds
+ *
+ * Conversion is done by simple multiplication with HZ
+ *
+ * secs_to_jiffies() is defined as a macro rather than a static inline
+ * function so it can be used in static initializers.
+ *
+ * Return: jiffies value
+ */
+#define secs_to_jiffies(_secs) ((_secs) * HZ)
+
extern unsigned long __usecs_to_jiffies(const unsigned int u);
#if !(USEC_PER_SEC % HZ)
static inline unsigned long _usecs_to_jiffies(const unsigned int u)
diff --git a/include/linux/phy.h b/include/linux/phy.h
index a98bc91a0cde..504766d4b2d5 100644
--- a/include/linux/phy.h
+++ b/include/linux/phy.h
@@ -1378,12 +1378,13 @@ int phy_read_mmd(struct phy_device *phydev, int devad, u32 regnum);
* @regnum: The register on the MMD to read
* @val: Variable to read the register into
* @cond: Break condition (usually involving @val)
- * @sleep_us: Maximum time to sleep between reads in us (0
- * tight-loops). Should be less than ~20ms since usleep_range
- * is used (see Documentation/timers/timers-howto.rst).
+ * @sleep_us: Maximum time to sleep between reads in us (0 tight-loops). Please
+ * read usleep_range() function description for details and
+ * limitations.
* @timeout_us: Timeout in us, 0 means never timeout
* @sleep_before_read: if it is true, sleep @sleep_us before read.
- * Returns 0 on success and -ETIMEDOUT upon a timeout. In either
+ *
+ * Returns: 0 on success and -ETIMEDOUT upon a timeout. In either
* case, the last read value at @args is stored in @val. Must not
* be called from atomic context if sleep_us or timeout_us are used.
*/
diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h
index 453691710839..f11f10c97bd9 100644
--- a/include/linux/posix-timers.h
+++ b/include/linux/posix-timers.h
@@ -5,12 +5,16 @@
#include <linux/alarmtimer.h>
#include <linux/list.h>
#include <linux/mutex.h>
+#include <linux/pid.h>
#include <linux/posix-timers_types.h>
+#include <linux/rcuref.h>
#include <linux/spinlock.h>
#include <linux/timerqueue.h>
struct kernel_siginfo;
struct task_struct;
+struct sigqueue;
+struct k_itimer;
static inline clockid_t make_process_cpuclock(const unsigned int pid,
const clockid_t clock)
@@ -35,6 +39,8 @@ static inline int clockid_to_fd(const clockid_t clk)
#ifdef CONFIG_POSIX_TIMERS
+#include <linux/signal_types.h>
+
/**
* cpu_timer - Posix CPU timer representation for k_itimer
* @node: timerqueue node to queue in the task/sig
@@ -42,6 +48,7 @@ static inline int clockid_to_fd(const clockid_t clk)
* @pid: Pointer to target task PID
* @elist: List head for the expiry list
* @firing: Timer is currently firing
+ * @nanosleep: Timer is used for nanosleep and is not a regular posix-timer
* @handling: Pointer to the task which handles expiry
*/
struct cpu_timer {
@@ -49,7 +56,8 @@ struct cpu_timer {
struct timerqueue_head *head;
struct pid *pid;
struct list_head elist;
- int firing;
+ bool firing;
+ bool nanosleep;
struct task_struct __rcu *handling;
};
@@ -101,6 +109,12 @@ static inline void posix_cputimers_rt_watchdog(struct posix_cputimers *pct,
pct->bases[CPUCLOCK_SCHED].nextevt = runtime;
}
+void posixtimer_rearm_itimer(struct task_struct *p);
+bool posixtimer_init_sigqueue(struct sigqueue *q);
+void posixtimer_send_sigqueue(struct k_itimer *tmr);
+bool posixtimer_deliver_signal(struct kernel_siginfo *info, struct sigqueue *timer_sigq);
+void posixtimer_free_timer(struct k_itimer *timer);
+
/* Init task static initializer */
#define INIT_CPU_TIMERBASE(b) { \
.nextevt = U64_MAX, \
@@ -122,6 +136,10 @@ struct cpu_timer { };
static inline void posix_cputimers_init(struct posix_cputimers *pct) { }
static inline void posix_cputimers_group_init(struct posix_cputimers *pct,
u64 cpu_limit) { }
+static inline void posixtimer_rearm_itimer(struct task_struct *p) { }
+static inline bool posixtimer_deliver_signal(struct kernel_siginfo *info,
+ struct sigqueue *timer_sigq) { return false; }
+static inline void posixtimer_free_timer(struct k_itimer *timer) { }
#endif
#ifdef CONFIG_POSIX_CPU_TIMERS_TASK_WORK
@@ -132,50 +150,56 @@ static inline void clear_posix_cputimers_work(struct task_struct *p) { }
static inline void posix_cputimers_init_work(void) { }
#endif
-#define REQUEUE_PENDING 1
-
/**
* struct k_itimer - POSIX.1b interval timer structure.
- * @list: List head for binding the timer to signals->posix_timers
+ * @list: List node for binding the timer to tsk::signal::posix_timers
+ * @ignored_list: List node for tracking ignored timers in tsk::signal::ignored_posix_timers
* @t_hash: Entry in the posix timer hash table
* @it_lock: Lock protecting the timer
* @kclock: Pointer to the k_clock struct handling this timer
* @it_clock: The posix timer clock id
* @it_id: The posix timer id for identifying the timer
- * @it_active: Marker that timer is active
+ * @it_status: The status of the timer
+ * @it_sig_periodic: The periodic status at signal delivery
* @it_overrun: The overrun counter for pending signals
* @it_overrun_last: The overrun at the time of the last delivered signal
- * @it_requeue_pending: Indicator that timer waits for being requeued on
- * signal delivery
+ * @it_signal_seq: Sequence count to control signal delivery
+ * @it_sigqueue_seq: The sequence count at the point where the signal was queued
* @it_sigev_notify: The notify word of sigevent struct for signal delivery
* @it_interval: The interval for periodic timers
* @it_signal: Pointer to the creators signal struct
* @it_pid: The pid of the process/task targeted by the signal
* @it_process: The task to wakeup on clock_nanosleep (CPU timers)
- * @sigq: Pointer to preallocated sigqueue
+ * @rcuref: Reference count for life time management
+ * @sigq: Embedded sigqueue
* @it: Union representing the various posix timer type
* internals.
* @rcu: RCU head for freeing the timer.
*/
struct k_itimer {
struct hlist_node list;
+ struct hlist_node ignored_list;
struct hlist_node t_hash;
spinlock_t it_lock;
const struct k_clock *kclock;
clockid_t it_clock;
timer_t it_id;
- int it_active;
+ int it_status;
+ bool it_sig_periodic;
s64 it_overrun;
s64 it_overrun_last;
- int it_requeue_pending;
+ unsigned int it_signal_seq;
+ unsigned int it_sigqueue_seq;
int it_sigev_notify;
+ enum pid_type it_pid_type;
ktime_t it_interval;
struct signal_struct *it_signal;
union {
struct pid *it_pid;
struct task_struct *it_process;
};
- struct sigqueue *sigq;
+ struct sigqueue sigq;
+ rcuref_t rcuref;
union {
struct {
struct hrtimer timer;
@@ -196,5 +220,29 @@ void set_process_cpu_timer(struct task_struct *task, unsigned int clock_idx,
int update_rlimit_cpu(struct task_struct *task, unsigned long rlim_new);
-void posixtimer_rearm(struct kernel_siginfo *info);
+#ifdef CONFIG_POSIX_TIMERS
+static inline void posixtimer_putref(struct k_itimer *tmr)
+{
+ if (rcuref_put(&tmr->rcuref))
+ posixtimer_free_timer(tmr);
+}
+
+static inline void posixtimer_sigqueue_getref(struct sigqueue *q)
+{
+ struct k_itimer *tmr = container_of(q, struct k_itimer, sigq);
+
+ WARN_ON_ONCE(!rcuref_get(&tmr->rcuref));
+}
+
+static inline void posixtimer_sigqueue_putref(struct sigqueue *q)
+{
+ struct k_itimer *tmr = container_of(q, struct k_itimer, sigq);
+
+ posixtimer_putref(tmr);
+}
+#else /* CONFIG_POSIX_TIMERS */
+static inline void posixtimer_sigqueue_getref(struct sigqueue *q) { }
+static inline void posixtimer_sigqueue_putref(struct sigqueue *q) { }
+#endif /* !CONFIG_POSIX_TIMERS */
+
#endif
diff --git a/include/linux/regmap.h b/include/linux/regmap.h
index f9ccad32fc5c..75f162b60ba1 100644
--- a/include/linux/regmap.h
+++ b/include/linux/regmap.h
@@ -106,17 +106,17 @@ struct reg_sequence {
* @addr: Address to poll
* @val: Unsigned integer variable to read the value into
* @cond: Break condition (usually involving @val)
- * @sleep_us: Maximum time to sleep between reads in us (0
- * tight-loops). Should be less than ~20ms since usleep_range
- * is used (see Documentation/timers/timers-howto.rst).
+ * @sleep_us: Maximum time to sleep between reads in us (0 tight-loops). Please
+ * read usleep_range() function description for details and
+ * limitations.
* @timeout_us: Timeout in us, 0 means never timeout
*
- * Returns 0 on success and -ETIMEDOUT upon a timeout or the regmap_read
+ * This is modelled after the readx_poll_timeout macros in linux/iopoll.h.
+ *
+ * Returns: 0 on success and -ETIMEDOUT upon a timeout or the regmap_read
* error return value in case of a error read. In the two former cases,
* the last read value at @addr is stored in @val. Must not be called
* from atomic context if sleep_us or timeout_us are used.
- *
- * This is modelled after the readx_poll_timeout macros in linux/iopoll.h.
*/
#define regmap_read_poll_timeout(map, addr, val, cond, sleep_us, timeout_us) \
({ \
@@ -133,20 +133,20 @@ struct reg_sequence {
* @addr: Address to poll
* @val: Unsigned integer variable to read the value into
* @cond: Break condition (usually involving @val)
- * @delay_us: Time to udelay between reads in us (0 tight-loops).
- * Should be less than ~10us since udelay is used
- * (see Documentation/timers/timers-howto.rst).
+ * @delay_us: Time to udelay between reads in us (0 tight-loops). Please
+ * read udelay() function description for details and
+ * limitations.
* @timeout_us: Timeout in us, 0 means never timeout
*
- * Returns 0 on success and -ETIMEDOUT upon a timeout or the regmap_read
- * error return value in case of a error read. In the two former cases,
- * the last read value at @addr is stored in @val.
- *
* This is modelled after the readx_poll_timeout_atomic macros in linux/iopoll.h.
*
* Note: In general regmap cannot be used in atomic context. If you want to use
* this macro then first setup your regmap for atomic use (flat or no cache
* and MMIO regmap).
+ *
+ * Returns: 0 on success and -ETIMEDOUT upon a timeout or the regmap_read
+ * error return value in case of a error read. In the two former cases,
+ * the last read value at @addr is stored in @val.
*/
#define regmap_read_poll_timeout_atomic(map, addr, val, cond, delay_us, timeout_us) \
({ \
@@ -177,17 +177,17 @@ struct reg_sequence {
* @field: Regmap field to read from
* @val: Unsigned integer variable to read the value into
* @cond: Break condition (usually involving @val)
- * @sleep_us: Maximum time to sleep between reads in us (0
- * tight-loops). Should be less than ~20ms since usleep_range
- * is used (see Documentation/timers/timers-howto.rst).
+ * @sleep_us: Maximum time to sleep between reads in us (0 tight-loops). Please
+ * read usleep_range() function description for details and
+ * limitations.
* @timeout_us: Timeout in us, 0 means never timeout
*
- * Returns 0 on success and -ETIMEDOUT upon a timeout or the regmap_field_read
+ * This is modelled after the readx_poll_timeout macros in linux/iopoll.h.
+ *
+ * Returns: 0 on success and -ETIMEDOUT upon a timeout or the regmap_field_read
* error return value in case of a error read. In the two former cases,
* the last read value at @addr is stored in @val. Must not be called
* from atomic context if sleep_us or timeout_us are used.
- *
- * This is modelled after the readx_poll_timeout macros in linux/iopoll.h.
*/
#define regmap_field_read_poll_timeout(field, val, cond, sleep_us, timeout_us) \
({ \
diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h
index c8ed09ac29ac..d5d03d919df8 100644
--- a/include/linux/sched/signal.h
+++ b/include/linux/sched/signal.h
@@ -138,6 +138,7 @@ struct signal_struct {
/* POSIX.1b Interval Timers */
unsigned int next_posix_timer_id;
struct hlist_head posix_timers;
+ struct hlist_head ignored_posix_timers;
/* ITIMER_REAL timer for the process */
struct hrtimer real_timer;
@@ -338,9 +339,6 @@ extern void force_fatal_sig(int);
extern void force_exit_sig(int);
extern int send_sig(int, struct task_struct *, int);
extern int zap_other_threads(struct task_struct *p);
-extern struct sigqueue *sigqueue_alloc(void);
-extern void sigqueue_free(struct sigqueue *);
-extern int send_sigqueue(struct sigqueue *, struct pid *, enum pid_type);
extern int do_sigaction(int, struct k_sigaction *, struct k_sigaction *);
static inline void clear_notify_signal(void)
diff --git a/include/linux/tick.h b/include/linux/tick.h
index 72744638c5b0..b0c74bfe0600 100644
--- a/include/linux/tick.h
+++ b/include/linux/tick.h
@@ -20,12 +20,10 @@ extern void __init tick_init(void);
extern void tick_suspend_local(void);
/* Should be core only, but XEN resume magic and ARM BL switcher require it */
extern void tick_resume_local(void);
-extern void tick_cleanup_dead_cpu(int cpu);
#else /* CONFIG_GENERIC_CLOCKEVENTS */
static inline void tick_init(void) { }
static inline void tick_suspend_local(void) { }
static inline void tick_resume_local(void) { }
-static inline void tick_cleanup_dead_cpu(int cpu) { }
#endif /* !CONFIG_GENERIC_CLOCKEVENTS */
#if defined(CONFIG_GENERIC_CLOCKEVENTS) && defined(CONFIG_HOTPLUG_CPU)
diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h
index 902c20ef495a..e39d4d563b19 100644
--- a/include/linux/timekeeper_internal.h
+++ b/include/linux/timekeeper_internal.h
@@ -26,7 +26,7 @@
* occupies a single 64byte cache line.
*
* The struct is separate from struct timekeeper as it is also used
- * for a fast NMI safe accessors.
+ * for the fast NMI safe accessors.
*
* @base_real is for the fast NMI safe accessor to allow reading clock
* realtime from any context.
@@ -44,33 +44,38 @@ struct tk_read_base {
/**
* struct timekeeper - Structure holding internal timekeeping values.
- * @tkr_mono: The readout base structure for CLOCK_MONOTONIC
- * @tkr_raw: The readout base structure for CLOCK_MONOTONIC_RAW
- * @xtime_sec: Current CLOCK_REALTIME time in seconds
- * @ktime_sec: Current CLOCK_MONOTONIC time in seconds
- * @wall_to_monotonic: CLOCK_REALTIME to CLOCK_MONOTONIC offset
- * @offs_real: Offset clock monotonic -> clock realtime
- * @offs_boot: Offset clock monotonic -> clock boottime
- * @offs_tai: Offset clock monotonic -> clock tai
- * @tai_offset: The current UTC to TAI offset in seconds
- * @clock_was_set_seq: The sequence number of clock was set events
- * @cs_was_changed_seq: The sequence number of clocksource change events
- * @next_leap_ktime: CLOCK_MONOTONIC time value of a pending leap-second
- * @raw_sec: CLOCK_MONOTONIC_RAW time in seconds
- * @monotonic_to_boot: CLOCK_MONOTONIC to CLOCK_BOOTTIME offset
- * @cycle_interval: Number of clock cycles in one NTP interval
- * @xtime_interval: Number of clock shifted nano seconds in one NTP
- * interval.
- * @xtime_remainder: Shifted nano seconds left over when rounding
- * @cycle_interval
- * @raw_interval: Shifted raw nano seconds accumulated per NTP interval.
- * @ntp_error: Difference between accumulated time and NTP time in ntp
- * shifted nano seconds.
- * @ntp_error_shift: Shift conversion between clock shifted nano seconds and
- * ntp shifted nano seconds.
- * @last_warning: Warning ratelimiter (DEBUG_TIMEKEEPING)
- * @underflow_seen: Underflow warning flag (DEBUG_TIMEKEEPING)
- * @overflow_seen: Overflow warning flag (DEBUG_TIMEKEEPING)
+ * @tkr_mono: The readout base structure for CLOCK_MONOTONIC
+ * @xtime_sec: Current CLOCK_REALTIME time in seconds
+ * @ktime_sec: Current CLOCK_MONOTONIC time in seconds
+ * @wall_to_monotonic: CLOCK_REALTIME to CLOCK_MONOTONIC offset
+ * @offs_real: Offset clock monotonic -> clock realtime
+ * @offs_boot: Offset clock monotonic -> clock boottime
+ * @offs_tai: Offset clock monotonic -> clock tai
+ * @tai_offset: The current UTC to TAI offset in seconds
+ * @tkr_raw: The readout base structure for CLOCK_MONOTONIC_RAW
+ * @raw_sec: CLOCK_MONOTONIC_RAW time in seconds
+ * @clock_was_set_seq: The sequence number of clock was set events
+ * @cs_was_changed_seq: The sequence number of clocksource change events
+ * @monotonic_to_boot: CLOCK_MONOTONIC to CLOCK_BOOTTIME offset
+ * @cycle_interval: Number of clock cycles in one NTP interval
+ * @xtime_interval: Number of clock shifted nano seconds in one NTP
+ * interval.
+ * @xtime_remainder: Shifted nano seconds left over when rounding
+ * @cycle_interval
+ * @raw_interval: Shifted raw nano seconds accumulated per NTP interval.
+ * @next_leap_ktime: CLOCK_MONOTONIC time value of a pending leap-second
+ * @ntp_tick: The ntp_tick_length() value currently being
+ * used. This cached copy ensures we consistently
+ * apply the tick length for an entire tick, as
+ * ntp_tick_length may change mid-tick, and we don't
+ * want to apply that new value to the tick in
+ * progress.
+ * @ntp_error: Difference between accumulated time and NTP time in ntp
+ * shifted nano seconds.
+ * @ntp_error_shift: Shift conversion between clock shifted nano seconds and
+ * ntp shifted nano seconds.
+ * @ntp_err_mult: Multiplication factor for scaled math conversion
+ * @skip_second_overflow: Flag used to avoid updating NTP twice with same second
*
* Note: For timespec(64) based interfaces wall_to_monotonic is what
* we need to add to xtime (or xtime corrected for sub jiffy times)
@@ -88,10 +93,28 @@ struct tk_read_base {
*
* @monotonic_to_boottime is a timespec64 representation of @offs_boot to
* accelerate the VDSO update for CLOCK_BOOTTIME.
+ *
+ * The cacheline ordering of the structure is optimized for in kernel usage of
+ * the ktime_get() and ktime_get_ts64() family of time accessors. Struct
+ * timekeeper is prepended in the core timekeeping code with a sequence count,
+ * which results in the following cacheline layout:
+ *
+ * 0: seqcount, tkr_mono
+ * 1: xtime_sec ... tai_offset
+ * 2: tkr_raw, raw_sec
+ * 3,4: Internal variables
+ *
+ * Cacheline 0,1 contain the data which is used for accessing
+ * CLOCK_MONOTONIC/REALTIME/BOOTTIME/TAI, while cacheline 2 contains the
+ * data for accessing CLOCK_MONOTONIC_RAW. Cacheline 3,4 are internal
+ * variables which are only accessed during timekeeper updates once per
+ * tick.
*/
struct timekeeper {
+ /* Cacheline 0 (together with prepended seqcount of timekeeper core): */
struct tk_read_base tkr_mono;
- struct tk_read_base tkr_raw;
+
+ /* Cacheline 1: */
u64 xtime_sec;
unsigned long ktime_sec;
struct timespec64 wall_to_monotonic;
@@ -99,43 +122,28 @@ struct timekeeper {
ktime_t offs_boot;
ktime_t offs_tai;
s32 tai_offset;
+
+ /* Cacheline 2: */
+ struct tk_read_base tkr_raw;
+ u64 raw_sec;
+
+ /* Cachline 3 and 4 (timekeeping internal variables): */
unsigned int clock_was_set_seq;
u8 cs_was_changed_seq;
- ktime_t next_leap_ktime;
- u64 raw_sec;
+
struct timespec64 monotonic_to_boot;
- /* The following members are for timekeeping internal use */
u64 cycle_interval;
u64 xtime_interval;
s64 xtime_remainder;
u64 raw_interval;
- /* The ntp_tick_length() value currently being used.
- * This cached copy ensures we consistently apply the tick
- * length for an entire tick, as ntp_tick_length may change
- * mid-tick, and we don't want to apply that new value to
- * the tick in progress.
- */
+
+ ktime_t next_leap_ktime;
u64 ntp_tick;
- /* Difference between accumulated time and NTP time in ntp
- * shifted nano seconds. */
s64 ntp_error;
u32 ntp_error_shift;
u32 ntp_err_mult;
- /* Flag used to avoid updating NTP twice with same second */
u32 skip_second_overflow;
-#ifdef CONFIG_DEBUG_TIMEKEEPING
- long last_warning;
- /*
- * These simple flag variables are managed
- * without locks, which is racy, but they are
- * ok since we don't really care about being
- * super precise about how many events were
- * seen, just that a problem was observed.
- */
- int underflow_seen;
- int overflow_seen;
-#endif
};
#ifdef CONFIG_GENERIC_TIME_VSYSCALL
diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h
index fc12a9ba2c88..0e035f675efe 100644
--- a/include/linux/timekeeping.h
+++ b/include/linux/timekeeping.h
@@ -45,6 +45,11 @@ extern void ktime_get_real_ts64(struct timespec64 *tv);
extern void ktime_get_coarse_ts64(struct timespec64 *ts);
extern void ktime_get_coarse_real_ts64(struct timespec64 *ts);
+/* Multigrain timestamp interfaces */
+extern void ktime_get_coarse_real_ts64_mg(struct timespec64 *ts);
+extern void ktime_get_real_ts64_mg(struct timespec64 *ts);
+extern unsigned long timekeeping_get_mg_floor_swaps(void);
+
void getboottime64(struct timespec64 *ts);
/*
@@ -275,6 +280,7 @@ struct ktime_timestamps {
* counter value
* @cycles: Clocksource counter value to produce the system times
* @real: Realtime system time
+ * @boot: Boot time
* @raw: Monotonic raw system time
* @cs_id: Clocksource ID
* @clock_was_set_seq: The sequence number of clock-was-set events
@@ -283,6 +289,7 @@ struct ktime_timestamps {
struct system_time_snapshot {
u64 cycles;
ktime_t real;
+ ktime_t boot;
ktime_t raw;
enum clocksource_ids cs_id;
unsigned int clock_was_set_seq;
diff --git a/include/linux/timex.h b/include/linux/timex.h
index 3871b06bd302..4ee32eff3f22 100644
--- a/include/linux/timex.h
+++ b/include/linux/timex.h
@@ -139,14 +139,6 @@ unsigned long random_get_entropy_fallback(void);
#define MAXSEC 2048 /* max interval between updates (s) */
#define NTP_PHASE_LIMIT ((MAXPHASE / NSEC_PER_USEC) << 5) /* beyond max. dispersion */
-/*
- * kernel variables
- * Note: maximum error = NTP sync distance = dispersion + delay / 2;
- * estimated error = NTP dispersion.
- */
-extern unsigned long tick_usec; /* USER_HZ period (usec) */
-extern unsigned long tick_nsec; /* SHIFTED_HZ period (nsec) */
-
/* Required to safely shift negative values */
#define shift_right(x, s) ({ \
__typeof__(x) __x = (x); \
diff --git a/include/linux/wait.h b/include/linux/wait.h
index 8aa3372f21a0..643b7c7bf376 100644
--- a/include/linux/wait.h
+++ b/include/linux/wait.h
@@ -541,8 +541,8 @@ do { \
int __ret = 0; \
struct hrtimer_sleeper __t; \
\
- hrtimer_init_sleeper_on_stack(&__t, CLOCK_MONOTONIC, \
- HRTIMER_MODE_REL); \
+ hrtimer_setup_sleeper_on_stack(&__t, CLOCK_MONOTONIC, \
+ HRTIMER_MODE_REL); \
if ((timeout) != KTIME_MAX) { \
hrtimer_set_expires_range_ns(&__t.timer, timeout, \
current->timer_slack_ns); \
diff --git a/include/uapi/asm-generic/siginfo.h b/include/uapi/asm-generic/siginfo.h
index b7bc545ec3b2..5a1ca43b5fc6 100644
--- a/include/uapi/asm-generic/siginfo.h
+++ b/include/uapi/asm-generic/siginfo.h
@@ -46,7 +46,7 @@ union __sifields {
__kernel_timer_t _tid; /* timer id */
int _overrun; /* overrun count */
sigval_t _sigval; /* same as below */
- int _sys_private; /* not to be passed to user */
+ int _sys_private; /* Not used by the kernel. Historic leftover. Always 0. */
} _timer;
/* POSIX.1b signals */
diff --git a/init/init_task.c b/init/init_task.c
index 136a8231355a..e557f622bd90 100644
--- a/init/init_task.c
+++ b/init/init_task.c
@@ -30,8 +30,9 @@ static struct signal_struct init_signals = {
.cred_guard_mutex = __MUTEX_INITIALIZER(init_signals.cred_guard_mutex),
.exec_update_lock = __RWSEM_INITIALIZER(init_signals.exec_update_lock),
#ifdef CONFIG_POSIX_TIMERS
- .posix_timers = HLIST_HEAD_INIT,
- .cputimer = {
+ .posix_timers = HLIST_HEAD_INIT,
+ .ignored_posix_timers = HLIST_HEAD_INIT,
+ .cputimer = {
.cputime_atomic = INIT_CPUTIME_ATOMIC,
},
#endif
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index feb61d68dca6..0842aa3f60e7 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -2435,13 +2435,14 @@ static int io_cqring_schedule_timeout(struct io_wait_queue *iowq,
{
ktime_t timeout;
- hrtimer_init_on_stack(&iowq->t, clock_id, HRTIMER_MODE_ABS);
if (iowq->min_timeout) {
timeout = ktime_add_ns(iowq->min_timeout, start_time);
- iowq->t.function = io_cqring_min_timer_wakeup;
+ hrtimer_setup_on_stack(&iowq->t, io_cqring_min_timer_wakeup, clock_id,
+ HRTIMER_MODE_ABS);
} else {
timeout = iowq->timeout;
- iowq->t.function = io_cqring_timer_wakeup;
+ hrtimer_setup_on_stack(&iowq->t, io_cqring_timer_wakeup, clock_id,
+ HRTIMER_MODE_ABS);
}
hrtimer_set_expires_range_ns(&iowq->t, timeout, 0);
diff --git a/io_uring/timeout.c b/io_uring/timeout.c
index 9973876d91b0..2ffe5e1dc68a 100644
--- a/io_uring/timeout.c
+++ b/io_uring/timeout.c
@@ -76,7 +76,6 @@ static void io_timeout_complete(struct io_kiocb *req, struct io_tw_state *ts)
/* re-arm timer */
spin_lock_irq(&ctx->timeout_lock);
list_add(&timeout->list, ctx->timeout_list.prev);
- data->timer.function = io_timeout_fn;
hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
spin_unlock_irq(&ctx->timeout_lock);
return;
diff --git a/kernel/cpu.c b/kernel/cpu.c
index d293d52a3e00..895f3287e3f3 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -1338,8 +1338,6 @@ static int takedown_cpu(unsigned int cpu)
cpuhp_bp_sync_dead(cpu);
- tick_cleanup_dead_cpu(cpu);
-
/*
* Callbacks must be re-integrated right away to the RCU state machine.
* Otherwise an RCU callback could block a further teardown function
diff --git a/kernel/fork.c b/kernel/fork.c
index 60c0b4868fd4..c2bd8367a850 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1864,6 +1864,7 @@ static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
#ifdef CONFIG_POSIX_TIMERS
INIT_HLIST_HEAD(&sig->posix_timers);
+ INIT_HLIST_HEAD(&sig->ignored_posix_timers);
hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
sig->real_timer.function = it_real_fn;
#endif
diff --git a/kernel/futex/core.c b/kernel/futex/core.c
index 136768ae2637..fb7214c7a36f 100644
--- a/kernel/futex/core.c
+++ b/kernel/futex/core.c
@@ -140,9 +140,9 @@ futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout,
if (!time)
return NULL;
- hrtimer_init_sleeper_on_stack(timeout, (flags & FLAGS_CLOCKRT) ?
- CLOCK_REALTIME : CLOCK_MONOTONIC,
- HRTIMER_MODE_ABS);
+ hrtimer_setup_sleeper_on_stack(timeout,
+ (flags & FLAGS_CLOCKRT) ? CLOCK_REALTIME : CLOCK_MONOTONIC,
+ HRTIMER_MODE_ABS);
/*
* If range_ns is 0, calling hrtimer_set_expires_range_ns() is
* effectively the same as calling hrtimer_set_expires().
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index d2f096bb274c..631e42802925 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -399,8 +399,8 @@ void play_idle_precise(u64 duration_ns, u64 latency_ns)
cpuidle_use_deepest_state(latency_ns);
it.done = 0;
- hrtimer_init_on_stack(&it.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
- it.timer.function = idle_inject_timer_fn;
+ hrtimer_setup_on_stack(&it.timer, idle_inject_timer_fn, CLOCK_MONOTONIC,
+ HRTIMER_MODE_REL_HARD);
hrtimer_start(&it.timer, ns_to_ktime(duration_ns),
HRTIMER_MODE_REL_PINNED_HARD);
diff --git a/kernel/signal.c b/kernel/signal.c
index 4344860ffcac..10b464b9d91f 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -59,6 +59,8 @@
#include <asm/cacheflush.h>
#include <asm/syscall.h> /* for syscall_get_* */
+#include "time/posix-timers.h"
+
/*
* SLAB caches for signal bits.
*/
@@ -396,16 +398,9 @@ void task_join_group_stop(struct task_struct *task)
task_set_jobctl_pending(task, mask | JOBCTL_STOP_PENDING);
}
-/*
- * allocate a new signal queue record
- * - this may be called without locks if and only if t == current, otherwise an
- * appropriate lock must be held to stop the target task from exiting
- */
-static struct sigqueue *
-__sigqueue_alloc(int sig, struct task_struct *t, gfp_t gfp_flags,
- int override_rlimit, const unsigned int sigqueue_flags)
+static struct ucounts *sig_get_ucounts(struct task_struct *t, int sig,
+ int override_rlimit)
{
- struct sigqueue *q = NULL;
struct ucounts *ucounts;
long sigpending;
@@ -424,26 +419,53 @@ __sigqueue_alloc(int sig, struct task_struct *t, gfp_t gfp_flags,
if (!sigpending)
return NULL;
- if (override_rlimit || likely(sigpending <= task_rlimit(t, RLIMIT_SIGPENDING))) {
- q = kmem_cache_alloc(sigqueue_cachep, gfp_flags);
- } else {
+ if (unlikely(!override_rlimit && sigpending > task_rlimit(t, RLIMIT_SIGPENDING))) {
+ dec_rlimit_put_ucounts(ucounts, UCOUNT_RLIMIT_SIGPENDING);
print_dropped_signal(sig);
+ return NULL;
}
- if (unlikely(q == NULL)) {
+ return ucounts;
+}
+
+static void __sigqueue_init(struct sigqueue *q, struct ucounts *ucounts,
+ const unsigned int sigqueue_flags)
+{
+ INIT_LIST_HEAD(&q->list);
+ q->flags = sigqueue_flags;
+ q->ucounts = ucounts;
+}
+
+/*
+ * allocate a new signal queue record
+ * - this may be called without locks if and only if t == current, otherwise an
+ * appropriate lock must be held to stop the target task from exiting
+ */
+static struct sigqueue *sigqueue_alloc(int sig, struct task_struct *t, gfp_t gfp_flags,
+ int override_rlimit)
+{
+ struct ucounts *ucounts = sig_get_ucounts(t, sig, override_rlimit);
+ struct sigqueue *q;
+
+ if (!ucounts)
+ return NULL;
+
+ q = kmem_cache_alloc(sigqueue_cachep, gfp_flags);
+ if (!q) {
dec_rlimit_put_ucounts(ucounts, UCOUNT_RLIMIT_SIGPENDING);
- } else {
- INIT_LIST_HEAD(&q->list);
- q->flags = sigqueue_flags;
- q->ucounts = ucounts;
+ return NULL;
}
+
+ __sigqueue_init(q, ucounts, 0);
return q;
}
static void __sigqueue_free(struct sigqueue *q)
{
- if (q->flags & SIGQUEUE_PREALLOC)
+ if (q->flags & SIGQUEUE_PREALLOC) {
+ posixtimer_sigqueue_putref(q);
return;
+ }
if (q->ucounts) {
dec_rlimit_put_ucounts(q->ucounts, UCOUNT_RLIMIT_SIGPENDING);
q->ucounts = NULL;
@@ -478,42 +500,6 @@ void flush_signals(struct task_struct *t)
}
EXPORT_SYMBOL(flush_signals);
-#ifdef CONFIG_POSIX_TIMERS
-static void __flush_itimer_signals(struct sigpending *pending)
-{
- sigset_t signal, retain;
- struct sigqueue *q, *n;
-
- signal = pending->signal;
- sigemptyset(&retain);
-
- list_for_each_entry_safe(q, n, &pending->list, list) {
- int sig = q->info.si_signo;
-
- if (likely(q->info.si_code != SI_TIMER)) {
- sigaddset(&retain, sig);
- } else {
- sigdelset(&signal, sig);
- list_del_init(&q->list);
- __sigqueue_free(q);
- }
- }
-
- sigorsets(&pending->signal, &signal, &retain);
-}
-
-void flush_itimer_signals(void)
-{
- struct task_struct *tsk = current;
- unsigned long flags;
-
- spin_lock_irqsave(&tsk->sighand->siglock, flags);
- __flush_itimer_signals(&tsk->pending);
- __flush_itimer_signals(&tsk->signal->shared_pending);
- spin_unlock_irqrestore(&tsk->sighand->siglock, flags);
-}
-#endif
-
void ignore_signals(struct task_struct *t)
{
int i;
@@ -563,7 +549,7 @@ bool unhandled_signal(struct task_struct *tsk, int sig)
}
static void collect_signal(int sig, struct sigpending *list, kernel_siginfo_t *info,
- bool *resched_timer)
+ struct sigqueue **timer_sigq)
{
struct sigqueue *q, *first = NULL;
@@ -586,12 +572,17 @@ static void collect_signal(int sig, struct sigpending *list, kernel_siginfo_t *i
list_del_init(&first->list);
copy_siginfo(info, &first->info);
- *resched_timer =
- (first->flags & SIGQUEUE_PREALLOC) &&
- (info->si_code == SI_TIMER) &&
- (info->si_sys_private);
-
- __sigqueue_free(first);
+ /*
+ * posix-timer signals are preallocated and freed when the last
+ * reference count is dropped in posixtimer_deliver_signal() or
+ * immediately on timer deletion when the signal is not pending.
+ * Spare the extra round through __sigqueue_free() which is
+ * ignoring preallocated signals.
+ */
+ if (unlikely((first->flags & SIGQUEUE_PREALLOC) && (info->si_code == SI_TIMER)))
+ *timer_sigq = first;
+ else
+ __sigqueue_free(first);
} else {
/*
* Ok, it wasn't in the queue. This must be
@@ -608,12 +599,12 @@ static void collect_signal(int sig, struct sigpending *list, kernel_siginfo_t *i
}
static int __dequeue_signal(struct sigpending *pending, sigset_t *mask,
- kernel_siginfo_t *info, bool *resched_timer)
+ kernel_siginfo_t *info, struct sigqueue **timer_sigq)
{
int sig = next_signal(pending, mask);
if (sig)
- collect_signal(sig, pending, info, resched_timer);
+ collect_signal(sig, pending, info, timer_sigq);
return sig;
}
@@ -625,42 +616,22 @@ static int __dequeue_signal(struct sigpending *pending, sigset_t *mask,
int dequeue_signal(sigset_t *mask, kernel_siginfo_t *info, enum pid_type *type)
{
struct task_struct *tsk = current;
- bool resched_timer = false;
+ struct sigqueue *timer_sigq;
int signr;
lockdep_assert_held(&tsk->sighand->siglock);
+again:
*type = PIDTYPE_PID;
- signr = __dequeue_signal(&tsk->pending, mask, info, &resched_timer);
+ timer_sigq = NULL;
+ signr = __dequeue_signal(&tsk->pending, mask, info, &timer_sigq);
if (!signr) {
*type = PIDTYPE_TGID;
signr = __dequeue_signal(&tsk->signal->shared_pending,
- mask, info, &resched_timer);
-#ifdef CONFIG_POSIX_TIMERS
- /*
- * itimer signal ?
- *
- * itimers are process shared and we restart periodic
- * itimers in the signal delivery path to prevent DoS
- * attacks in the high resolution timer case. This is
- * compliant with the old way of self-restarting
- * itimers, as the SIGALRM is a legacy signal and only
- * queued once. Changing the restart behaviour to
- * restart the timer in the signal dequeue path is
- * reducing the timer noise on heavy loaded !highres
- * systems too.
- */
- if (unlikely(signr == SIGALRM)) {
- struct hrtimer *tmr = &tsk->signal->real_timer;
-
- if (!hrtimer_is_queued(tmr) &&
- tsk->signal->it_real_incr != 0) {
- hrtimer_forward(tmr, tmr->base->get_time(),
- tsk->signal->it_real_incr);
- hrtimer_restart(tmr);
- }
- }
-#endif
+ mask, info, &timer_sigq);
+
+ if (unlikely(signr == SIGALRM))
+ posixtimer_rearm_itimer(tsk);
}
recalc_sigpending();
@@ -682,22 +653,12 @@ int dequeue_signal(sigset_t *mask, kernel_siginfo_t *info, enum pid_type *type)
*/
current->jobctl |= JOBCTL_STOP_DEQUEUED;
}
-#ifdef CONFIG_POSIX_TIMERS
- if (resched_timer) {
- /*
- * Release the siglock to ensure proper locking order
- * of timer locks outside of siglocks. Note, we leave
- * irqs disabled here, since the posix-timers code is
- * about to disable them again anyway.
- */
- spin_unlock(&tsk->sighand->siglock);
- posixtimer_rearm(info);
- spin_lock(&tsk->sighand->siglock);
- /* Don't expose the si_sys_private value to userspace */
- info->si_sys_private = 0;
+ if (IS_ENABLED(CONFIG_POSIX_TIMERS) && unlikely(timer_sigq)) {
+ if (!posixtimer_deliver_signal(info, timer_sigq))
+ goto again;
}
-#endif
+
return signr;
}
EXPORT_SYMBOL_GPL(dequeue_signal);
@@ -772,17 +733,24 @@ void signal_wake_up_state(struct task_struct *t, unsigned int state)
kick_process(t);
}
-/*
- * Remove signals in mask from the pending set and queue.
- * Returns 1 if any signals were found.
- *
- * All callers must be holding the siglock.
- */
-static void flush_sigqueue_mask(sigset_t *mask, struct sigpending *s)
+static inline void posixtimer_sig_ignore(struct task_struct *tsk, struct sigqueue *q);
+
+static void sigqueue_free_ignored(struct task_struct *tsk, struct sigqueue *q)
+{
+ if (likely(!(q->flags & SIGQUEUE_PREALLOC) || q->info.si_code != SI_TIMER))
+ __sigqueue_free(q);
+ else
+ posixtimer_sig_ignore(tsk, q);
+}
+
+/* Remove signals in mask from the pending set and queue. */
+static void flush_sigqueue_mask(struct task_struct *p, sigset_t *mask, struct sigpending *s)
{
struct sigqueue *q, *n;
sigset_t m;
+ lockdep_assert_held(&p->sighand->siglock);
+
sigandsets(&m, mask, &s->signal);
if (sigisemptyset(&m))
return;
@@ -791,7 +759,7 @@ static void flush_sigqueue_mask(sigset_t *mask, struct sigpending *s)
list_for_each_entry_safe(q, n, &s->list, list) {
if (sigismember(mask, q->info.si_signo)) {
list_del_init(&q->list);
- __sigqueue_free(q);
+ sigqueue_free_ignored(p, q);
}
}
}
@@ -916,18 +884,18 @@ static bool prepare_signal(int sig, struct task_struct *p, bool force)
* This is a stop signal. Remove SIGCONT from all queues.
*/
siginitset(&flush, sigmask(SIGCONT));
- flush_sigqueue_mask(&flush, &signal->shared_pending);
+ flush_sigqueue_mask(p, &flush, &signal->shared_pending);
for_each_thread(p, t)
- flush_sigqueue_mask(&flush, &t->pending);
+ flush_sigqueue_mask(p, &flush, &t->pending);
} else if (sig == SIGCONT) {
unsigned int why;
/*
* Remove all stop signals from all queues, wake all threads.
*/
siginitset(&flush, SIG_KERNEL_STOP_MASK);
- flush_sigqueue_mask(&flush, &signal->shared_pending);
+ flush_sigqueue_mask(p, &flush, &signal->shared_pending);
for_each_thread(p, t) {
- flush_sigqueue_mask(&flush, &t->pending);
+ flush_sigqueue_mask(p, &flush, &t->pending);
task_clear_jobctl_pending(t, JOBCTL_STOP_PENDING);
if (likely(!(t->ptrace & PT_SEIZED))) {
t->jobctl &= ~JOBCTL_STOPPED;
@@ -1114,7 +1082,7 @@ static int __send_signal_locked(int sig, struct kernel_siginfo *info,
else
override_rlimit = 0;
- q = __sigqueue_alloc(sig, t, GFP_ATOMIC, override_rlimit, 0);
+ q = sigqueue_alloc(sig, t, GFP_ATOMIC, override_rlimit);
if (q) {
list_add_tail(&q->list, &pending->list);
@@ -1922,112 +1890,242 @@ int kill_pid(struct pid *pid, int sig, int priv)
}
EXPORT_SYMBOL(kill_pid);
+#ifdef CONFIG_POSIX_TIMERS
/*
- * These functions support sending signals using preallocated sigqueue
- * structures. This is needed "because realtime applications cannot
- * afford to lose notifications of asynchronous events, like timer
- * expirations or I/O completions". In the case of POSIX Timers
- * we allocate the sigqueue structure from the timer_create. If this
- * allocation fails we are able to report the failure to the application
- * with an EAGAIN error.
+ * These functions handle POSIX timer signals. POSIX timers use
+ * preallocated sigqueue structs for sending signals.
*/
-struct sigqueue *sigqueue_alloc(void)
+static void __flush_itimer_signals(struct sigpending *pending)
{
- return __sigqueue_alloc(-1, current, GFP_KERNEL, 0, SIGQUEUE_PREALLOC);
+ sigset_t signal, retain;
+ struct sigqueue *q, *n;
+
+ signal = pending->signal;
+ sigemptyset(&retain);
+
+ list_for_each_entry_safe(q, n, &pending->list, list) {
+ int sig = q->info.si_signo;
+
+ if (likely(q->info.si_code != SI_TIMER)) {
+ sigaddset(&retain, sig);
+ } else {
+ sigdelset(&signal, sig);
+ list_del_init(&q->list);
+ __sigqueue_free(q);
+ }
+ }
+
+ sigorsets(&pending->signal, &signal, &retain);
}
-void sigqueue_free(struct sigqueue *q)
+void flush_itimer_signals(void)
{
- spinlock_t *lock = ¤t->sighand->siglock;
- unsigned long flags;
+ struct task_struct *tsk = current;
- if (WARN_ON_ONCE(!(q->flags & SIGQUEUE_PREALLOC)))
- return;
- /*
- * We must hold ->siglock while testing q->list
- * to serialize with collect_signal() or with
- * __exit_signal()->flush_sigqueue().
- */
- spin_lock_irqsave(lock, flags);
- q->flags &= ~SIGQUEUE_PREALLOC;
- /*
- * If it is queued it will be freed when dequeued,
- * like the "regular" sigqueue.
- */
- if (!list_empty(&q->list))
- q = NULL;
- spin_unlock_irqrestore(lock, flags);
+ guard(spinlock_irqsave)(&tsk->sighand->siglock);
+ __flush_itimer_signals(&tsk->pending);
+ __flush_itimer_signals(&tsk->signal->shared_pending);
+}
- if (q)
- __sigqueue_free(q);
+bool posixtimer_init_sigqueue(struct sigqueue *q)
+{
+ struct ucounts *ucounts = sig_get_ucounts(current, -1, 0);
+
+ if (!ucounts)
+ return false;
+ clear_siginfo(&q->info);
+ __sigqueue_init(q, ucounts, SIGQUEUE_PREALLOC);
+ return true;
}
-int send_sigqueue(struct sigqueue *q, struct pid *pid, enum pid_type type)
+static void posixtimer_queue_sigqueue(struct sigqueue *q, struct task_struct *t, enum pid_type type)
{
- int sig = q->info.si_signo;
struct sigpending *pending;
+ int sig = q->info.si_signo;
+
+ signalfd_notify(t, sig);
+ pending = (type != PIDTYPE_PID) ? &t->signal->shared_pending : &t->pending;
+ list_add_tail(&q->list, &pending->list);
+ sigaddset(&pending->signal, sig);
+ complete_signal(sig, t, type);
+}
+
+/*
+ * This function is used by POSIX timers to deliver a timer signal.
+ * Where type is PIDTYPE_PID (such as for timers with SIGEV_THREAD_ID
+ * set), the signal must be delivered to the specific thread (queues
+ * into t->pending).
+ *
+ * Where type is not PIDTYPE_PID, signals must be delivered to the
+ * process. In this case, prefer to deliver to current if it is in
+ * the same thread group as the target process, which avoids
+ * unnecessarily waking up a potentially idle task.
+ */
+static inline struct task_struct *posixtimer_get_target(struct k_itimer *tmr)
+{
+ struct task_struct *t = pid_task(tmr->it_pid, tmr->it_pid_type);
+
+ if (t && tmr->it_pid_type != PIDTYPE_PID && same_thread_group(t, current))
+ t = current;
+ return t;
+}
+
+void posixtimer_send_sigqueue(struct k_itimer *tmr)
+{
+ struct sigqueue *q = &tmr->sigq;
+ int sig = q->info.si_signo;
struct task_struct *t;
unsigned long flags;
- int ret, result;
+ int result;
- if (WARN_ON_ONCE(!(q->flags & SIGQUEUE_PREALLOC)))
- return 0;
- if (WARN_ON_ONCE(q->info.si_code != SI_TIMER))
- return 0;
+ guard(rcu)();
- ret = -1;
- rcu_read_lock();
+ t = posixtimer_get_target(tmr);
+ if (!t)
+ return;
+
+ if (!likely(lock_task_sighand(t, &flags)))
+ return;
/*
- * This function is used by POSIX timers to deliver a timer signal.
- * Where type is PIDTYPE_PID (such as for timers with SIGEV_THREAD_ID
- * set), the signal must be delivered to the specific thread (queues
- * into t->pending).
- *
- * Where type is not PIDTYPE_PID, signals must be delivered to the
- * process. In this case, prefer to deliver to current if it is in
- * the same thread group as the target process, which avoids
- * unnecessarily waking up a potentially idle task.
+ * Update @tmr::sigqueue_seq for posix timer signals with sighand
+ * locked to prevent a race against dequeue_signal().
*/
- t = pid_task(pid, type);
- if (!t)
- goto ret;
- if (type != PIDTYPE_PID && same_thread_group(t, current))
- t = current;
- if (!likely(lock_task_sighand(t, &flags)))
- goto ret;
+ tmr->it_sigqueue_seq = tmr->it_signal_seq;
- ret = 1; /* the signal is ignored */
- result = TRACE_SIGNAL_IGNORED;
- if (!prepare_signal(sig, t, false))
+ /*
+ * Set the signal delivery status under sighand lock, so that the
+ * ignored signal handling can distinguish between a periodic and a
+ * non-periodic timer.
+ */
+ tmr->it_sig_periodic = tmr->it_status == POSIX_TIMER_REQUEUE_PENDING;
+
+ if (!prepare_signal(sig, t, false)) {
+ result = TRACE_SIGNAL_IGNORED;
+
+ if (!list_empty(&q->list)) {
+ /*
+ * If task group is exiting with the signal already pending,
+ * wait for __exit_signal() to do its job. Otherwise if
+ * ignored, it's not supposed to be queued. Try to survive.
+ */
+ WARN_ON_ONCE(!(t->signal->flags & SIGNAL_GROUP_EXIT));
+ goto out;
+ }
+
+ /* Periodic timers with SIG_IGN are queued on the ignored list */
+ if (tmr->it_sig_periodic) {
+ /*
+ * Already queued means the timer was rearmed after
+ * the previous expiry got it on the ignore list.
+ * Nothing to do for that case.
+ */
+ if (hlist_unhashed(&tmr->ignored_list)) {
+ /*
+ * Take a signal reference and queue it on
+ * the ignored list.
+ */
+ posixtimer_sigqueue_getref(q);
+ posixtimer_sig_ignore(t, q);
+ }
+ } else if (!hlist_unhashed(&tmr->ignored_list)) {
+ /*
+ * Covers the case where a timer was periodic and
+ * then the signal was ignored. Later it was rearmed
+ * as oneshot timer. The previous signal is invalid
+ * now, and this oneshot signal has to be dropped.
+ * Remove it from the ignored list and drop the
+ * reference count as the signal is not longer
+ * queued.
+ */
+ hlist_del_init(&tmr->ignored_list);
+ posixtimer_putref(tmr);
+ }
goto out;
+ }
+
+ /* This should never happen and leaks a reference count */
+ if (WARN_ON_ONCE(!hlist_unhashed(&tmr->ignored_list)))
+ hlist_del_init(&tmr->ignored_list);
- ret = 0;
if (unlikely(!list_empty(&q->list))) {
- /*
- * If an SI_TIMER entry is already queue just increment
- * the overrun count.
- */
- q->info.si_overrun++;
+ /* This holds a reference count already */
result = TRACE_SIGNAL_ALREADY_PENDING;
goto out;
}
- q->info.si_overrun = 0;
- signalfd_notify(t, sig);
- pending = (type != PIDTYPE_PID) ? &t->signal->shared_pending : &t->pending;
- list_add_tail(&q->list, &pending->list);
- sigaddset(&pending->signal, sig);
- complete_signal(sig, t, type);
+ posixtimer_sigqueue_getref(q);
+ posixtimer_queue_sigqueue(q, t, tmr->it_pid_type);
result = TRACE_SIGNAL_DELIVERED;
out:
- trace_signal_generate(sig, &q->info, t, type != PIDTYPE_PID, result);
+ trace_signal_generate(sig, &q->info, t, tmr->it_pid_type != PIDTYPE_PID, result);
unlock_task_sighand(t, &flags);
-ret:
- rcu_read_unlock();
- return ret;
}
+static inline void posixtimer_sig_ignore(struct task_struct *tsk, struct sigqueue *q)
+{
+ struct k_itimer *tmr = container_of(q, struct k_itimer, sigq);
+
+ /*
+ * If the timer is marked deleted already or the signal originates
+ * from a non-periodic timer, then just drop the reference
+ * count. Otherwise queue it on the ignored list.
+ */
+ if (tmr->it_signal && tmr->it_sig_periodic)
+ hlist_add_head(&tmr->ignored_list, &tsk->signal->ignored_posix_timers);
+ else
+ posixtimer_putref(tmr);
+}
+
+static void posixtimer_sig_unignore(struct task_struct *tsk, int sig)
+{
+ struct hlist_head *head = &tsk->signal->ignored_posix_timers;
+ struct hlist_node *tmp;
+ struct k_itimer *tmr;
+
+ if (likely(hlist_empty(head)))
+ return;
+
+ /*
+ * Rearming a timer with sighand lock held is not possible due to
+ * lock ordering vs. tmr::it_lock. Just stick the sigqueue back and
+ * let the signal delivery path deal with it whether it needs to be
+ * rearmed or not. This cannot be decided here w/o dropping sighand
+ * lock and creating a loop retry horror show.
+ */
+ hlist_for_each_entry_safe(tmr, tmp , head, ignored_list) {
+ struct task_struct *target;
+
+ /*
+ * tmr::sigq.info.si_signo is immutable, so accessing it
+ * without holding tmr::it_lock is safe.
+ */
+ if (tmr->sigq.info.si_signo != sig)
+ continue;
+
+ hlist_del_init(&tmr->ignored_list);
+
+ /* This should never happen and leaks a reference count */
+ if (WARN_ON_ONCE(!list_empty(&tmr->sigq.list)))
+ continue;
+
+ /*
+ * Get the target for the signal. If target is a thread and
+ * has exited by now, drop the reference count.
+ */
+ guard(rcu)();
+ target = posixtimer_get_target(tmr);
+ if (target)
+ posixtimer_queue_sigqueue(&tmr->sigq, target, tmr->it_pid_type);
+ else
+ posixtimer_putref(tmr);
+ }
+}
+#else /* CONFIG_POSIX_TIMERS */
+static inline void posixtimer_sig_ignore(struct task_struct *tsk, struct sigqueue *q) { }
+static inline void posixtimer_sig_unignore(struct task_struct *tsk, int sig) { }
+#endif /* !CONFIG_POSIX_TIMERS */
+
void do_notify_pidfd(struct task_struct *task)
{
struct pid *pid = task_pid(task);
@@ -4153,8 +4251,8 @@ void kernel_sigaction(int sig, __sighandler_t action)
sigemptyset(&mask);
sigaddset(&mask, sig);
- flush_sigqueue_mask(&mask, ¤t->signal->shared_pending);
- flush_sigqueue_mask(&mask, ¤t->pending);
+ flush_sigqueue_mask(current, &mask, ¤t->signal->shared_pending);
+ flush_sigqueue_mask(current, &mask, ¤t->pending);
recalc_sigpending();
}
spin_unlock_irq(¤t->sighand->siglock);
@@ -4204,6 +4302,8 @@ int do_sigaction(int sig, struct k_sigaction *act, struct k_sigaction *oact)
sigaction_compat_abi(act, oact);
if (act) {
+ bool was_ignored = k->sa.sa_handler == SIG_IGN;
+
sigdelsetmask(&act->sa.sa_mask,
sigmask(SIGKILL) | sigmask(SIGSTOP));
*k = *act;
@@ -4221,9 +4321,11 @@ int do_sigaction(int sig, struct k_sigaction *act, struct k_sigaction *oact)
if (sig_handler_ignored(sig_handler(p, sig), sig)) {
sigemptyset(&mask);
sigaddset(&mask, sig);
- flush_sigqueue_mask(&mask, &p->signal->shared_pending);
+ flush_sigqueue_mask(p, &mask, &p->signal->shared_pending);
for_each_thread(p, t)
- flush_sigqueue_mask(&mask, &t->pending);
+ flush_sigqueue_mask(p, &mask, &t->pending);
+ } else if (was_ignored) {
+ posixtimer_sig_unignore(p, sig);
}
}
diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig
index 8ebb6d5a106b..b0b97a60aaa6 100644
--- a/kernel/time/Kconfig
+++ b/kernel/time/Kconfig
@@ -17,11 +17,6 @@ config ARCH_CLOCKSOURCE_DATA
config ARCH_CLOCKSOURCE_INIT
bool
-# Clocksources require validation of the clocksource against the last
-# cycle update - x86/TSC misfeature
-config CLOCKSOURCE_VALIDATE_LAST_CYCLE
- bool
-
# Timekeeping vsyscall support
config GENERIC_TIME_VSYSCALL
bool
diff --git a/kernel/time/Makefile b/kernel/time/Makefile
index 4af2a264a160..fe0ae82124fe 100644
--- a/kernel/time/Makefile
+++ b/kernel/time/Makefile
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: GPL-2.0
-obj-y += time.o timer.o hrtimer.o
+obj-y += time.o timer.o hrtimer.o sleep_timeout.o
obj-y += timekeeping.o ntp.o clocksource.o jiffies.o timer_list.o
obj-y += timeconv.o timecounter.o alarmtimer.o
diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c
index 8bf888641694..0ddccdff119a 100644
--- a/kernel/time/alarmtimer.c
+++ b/kernel/time/alarmtimer.c
@@ -197,28 +197,15 @@ static enum hrtimer_restart alarmtimer_fired(struct hrtimer *timer)
{
struct alarm *alarm = container_of(timer, struct alarm, timer);
struct alarm_base *base = &alarm_bases[alarm->type];
- unsigned long flags;
- int ret = HRTIMER_NORESTART;
- int restart = ALARMTIMER_NORESTART;
- spin_lock_irqsave(&base->lock, flags);
- alarmtimer_dequeue(base, alarm);
- spin_unlock_irqrestore(&base->lock, flags);
+ scoped_guard (spinlock_irqsave, &base->lock)
+ alarmtimer_dequeue(base, alarm);
if (alarm->function)
- restart = alarm->function(alarm, base->get_ktime());
-
- spin_lock_irqsave(&base->lock, flags);
- if (restart != ALARMTIMER_NORESTART) {
- hrtimer_set_expires(&alarm->timer, alarm->node.expires);
- alarmtimer_enqueue(base, alarm);
- ret = HRTIMER_RESTART;
- }
- spin_unlock_irqrestore(&base->lock, flags);
+ alarm->function(alarm, base->get_ktime());
trace_alarmtimer_fired(alarm, base->get_ktime());
- return ret;
-
+ return HRTIMER_NORESTART;
}
ktime_t alarm_expires_remaining(const struct alarm *alarm)
@@ -334,10 +321,9 @@ static int alarmtimer_resume(struct device *dev)
static void
__alarm_init(struct alarm *alarm, enum alarmtimer_type type,
- enum alarmtimer_restart (*function)(struct alarm *, ktime_t))
+ void (*function)(struct alarm *, ktime_t))
{
timerqueue_init(&alarm->node);
- alarm->timer.function = alarmtimer_fired;
alarm->function = function;
alarm->type = type;
alarm->state = ALARMTIMER_STATE_INACTIVE;
@@ -350,10 +336,10 @@ __alarm_init(struct alarm *alarm, enum alarmtimer_type type,
* @function: callback that is run when the alarm fires
*/
void alarm_init(struct alarm *alarm, enum alarmtimer_type type,
- enum alarmtimer_restart (*function)(struct alarm *, ktime_t))
+ void (*function)(struct alarm *, ktime_t))
{
- hrtimer_init(&alarm->timer, alarm_bases[type].base_clockid,
- HRTIMER_MODE_ABS);
+ hrtimer_setup(&alarm->timer, alarmtimer_fired, alarm_bases[type].base_clockid,
+ HRTIMER_MODE_ABS);
__alarm_init(alarm, type, function);
}
EXPORT_SYMBOL_GPL(alarm_init);
@@ -480,35 +466,11 @@ u64 alarm_forward(struct alarm *alarm, ktime_t now, ktime_t interval)
}
EXPORT_SYMBOL_GPL(alarm_forward);
-static u64 __alarm_forward_now(struct alarm *alarm, ktime_t interval, bool throttle)
+u64 alarm_forward_now(struct alarm *alarm, ktime_t interval)
{
struct alarm_base *base = &alarm_bases[alarm->type];
- ktime_t now = base->get_ktime();
-
- if (IS_ENABLED(CONFIG_HIGH_RES_TIMERS) && throttle) {
- /*
- * Same issue as with posix_timer_fn(). Timers which are
- * periodic but the signal is ignored can starve the system
- * with a very small interval. The real fix which was
- * promised in the context of posix_timer_fn() never
- * materialized, but someone should really work on it.
- *
- * To prevent DOS fake @now to be 1 jiffy out which keeps
- * the overrun accounting correct but creates an
- * inconsistency vs. timer_gettime(2).
- */
- ktime_t kj = NSEC_PER_SEC / HZ;
- if (interval < kj)
- now = ktime_add(now, kj);
- }
-
- return alarm_forward(alarm, now, interval);
-}
-
-u64 alarm_forward_now(struct alarm *alarm, ktime_t interval)
-{
- return __alarm_forward_now(alarm, interval, false);
+ return alarm_forward(alarm, base->get_ktime(), interval);
}
EXPORT_SYMBOL_GPL(alarm_forward_now);
@@ -567,30 +529,12 @@ static enum alarmtimer_type clock2alarm(clockid_t clockid)
*
* Return: whether the timer is to be restarted
*/
-static enum alarmtimer_restart alarm_handle_timer(struct alarm *alarm,
- ktime_t now)
+static void alarm_handle_timer(struct alarm *alarm, ktime_t now)
{
- struct k_itimer *ptr = container_of(alarm, struct k_itimer,
- it.alarm.alarmtimer);
- enum alarmtimer_restart result = ALARMTIMER_NORESTART;
- unsigned long flags;
+ struct k_itimer *ptr = container_of(alarm, struct k_itimer, it.alarm.alarmtimer);
- spin_lock_irqsave(&ptr->it_lock, flags);
-
- if (posix_timer_queue_signal(ptr) && ptr->it_interval) {
- /*
- * Handle ignored signals and rearm the timer. This will go
- * away once we handle ignored signals proper. Ensure that
- * small intervals cannot starve the system.
- */
- ptr->it_overrun += __alarm_forward_now(alarm, ptr->it_interval, true);
- ++ptr->it_requeue_pending;
- ptr->it_active = 1;
- result = ALARMTIMER_RESTART;
- }
- spin_unlock_irqrestore(&ptr->it_lock, flags);
-
- return result;
+ guard(spinlock_irqsave)(&ptr->it_lock);
+ posix_timer_queue_signal(ptr);
}
/**
@@ -751,18 +695,14 @@ static int alarm_timer_create(struct k_itimer *new_timer)
* @now: time at the timer expiration
*
* Wakes up the task that set the alarmtimer
- *
- * Return: ALARMTIMER_NORESTART
*/
-static enum alarmtimer_restart alarmtimer_nsleep_wakeup(struct alarm *alarm,
- ktime_t now)
+static void alarmtimer_nsleep_wakeup(struct alarm *alarm, ktime_t now)
{
struct task_struct *task = alarm->data;
alarm->data = NULL;
if (task)
wake_up_process(task);
- return ALARMTIMER_NORESTART;
}
/**
@@ -814,10 +754,10 @@ static int alarmtimer_do_nsleep(struct alarm *alarm, ktime_t absexp,
static void
alarm_init_on_stack(struct alarm *alarm, enum alarmtimer_type type,
- enum alarmtimer_restart (*function)(struct alarm *, ktime_t))
+ void (*function)(struct alarm *, ktime_t))
{
- hrtimer_init_on_stack(&alarm->timer, alarm_bases[type].base_clockid,
- HRTIMER_MODE_ABS);
+ hrtimer_setup_on_stack(&alarm->timer, alarmtimer_fired, alarm_bases[type].base_clockid,
+ HRTIMER_MODE_ABS);
__alarm_init(alarm, type, function);
}
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c
index 78c7bd64d0dd..f3e831f62906 100644
--- a/kernel/time/clockevents.c
+++ b/kernel/time/clockevents.c
@@ -337,13 +337,21 @@ int clockevents_program_event(struct clock_event_device *dev, ktime_t expires,
}
/*
- * Called after a notify add to make devices available which were
- * released from the notifier call.
+ * Called after a clockevent has been added which might
+ * have replaced a current regular or broadcast device. A
+ * released normal device might be a suitable replacement
+ * for the current broadcast device. Similarly a released
+ * broadcast device might be a suitable replacement for a
+ * normal device.
*/
static void clockevents_notify_released(void)
{
struct clock_event_device *dev;
+ /*
+ * Keep iterating as long as tick_check_new_device()
+ * replaces a device.
+ */
while (!list_empty(&clockevents_released)) {
dev = list_entry(clockevents_released.next,
struct clock_event_device, list);
@@ -610,39 +618,30 @@ void clockevents_resume(void)
#ifdef CONFIG_HOTPLUG_CPU
-# ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
/**
- * tick_offline_cpu - Take CPU out of the broadcast mechanism
+ * tick_offline_cpu - Shutdown all clock events related
+ * to this CPU and take it out of the
+ * broadcast mechanism.
* @cpu: The outgoing CPU
*
- * Called on the outgoing CPU after it took itself offline.
+ * Called by the dying CPU during teardown.
*/
void tick_offline_cpu(unsigned int cpu)
-{
- raw_spin_lock(&clockevents_lock);
- tick_broadcast_offline(cpu);
- raw_spin_unlock(&clockevents_lock);
-}
-# endif
-
-/**
- * tick_cleanup_dead_cpu - Cleanup the tick and clockevents of a dead cpu
- * @cpu: The dead CPU
- */
-void tick_cleanup_dead_cpu(int cpu)
{
struct clock_event_device *dev, *tmp;
- unsigned long flags;
- raw_spin_lock_irqsave(&clockevents_lock, flags);
+ raw_spin_lock(&clockevents_lock);
+ tick_broadcast_offline(cpu);
tick_shutdown(cpu);
+
/*
* Unregister the clock event devices which were
- * released from the users in the notify chain.
+ * released above.
*/
list_for_each_entry_safe(dev, tmp, &clockevents_released, list)
list_del(&dev->list);
+
/*
* Now check whether the CPU has left unused per cpu devices
*/
@@ -654,7 +653,8 @@ void tick_cleanup_dead_cpu(int cpu)
list_del(&dev->list);
}
}
- raw_spin_unlock_irqrestore(&clockevents_lock, flags);
+
+ raw_spin_unlock(&clockevents_lock);
}
#endif
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
index 23336eecb4f4..aab6472853fa 100644
--- a/kernel/time/clocksource.c
+++ b/kernel/time/clocksource.c
@@ -20,6 +20,8 @@
#include "tick-internal.h"
#include "timekeeping_internal.h"
+static void clocksource_enqueue(struct clocksource *cs);
+
static noinline u64 cycles_to_nsec_safe(struct clocksource *cs, u64 start, u64 end)
{
u64 delta = clocksource_delta(end, start, cs->mask);
@@ -171,7 +173,6 @@ static inline void clocksource_watchdog_unlock(unsigned long *flags)
}
static int clocksource_watchdog_kthread(void *data);
-static void __clocksource_change_rating(struct clocksource *cs, int rating);
static void clocksource_watchdog_work(struct work_struct *work)
{
@@ -191,6 +192,13 @@ static void clocksource_watchdog_work(struct work_struct *work)
kthread_run(clocksource_watchdog_kthread, NULL, "kwatchdog");
}
+static void clocksource_change_rating(struct clocksource *cs, int rating)
+{
+ list_del(&cs->list);
+ cs->rating = rating;
+ clocksource_enqueue(cs);
+}
+
static void __clocksource_unstable(struct clocksource *cs)
{
cs->flags &= ~(CLOCK_SOURCE_VALID_FOR_HRES | CLOCK_SOURCE_WATCHDOG);
@@ -697,7 +705,7 @@ static int __clocksource_watchdog_kthread(void)
list_for_each_entry_safe(cs, tmp, &watchdog_list, wd_list) {
if (cs->flags & CLOCK_SOURCE_UNSTABLE) {
list_del_init(&cs->wd_list);
- __clocksource_change_rating(cs, 0);
+ clocksource_change_rating(cs, 0);
select = 1;
}
if (cs->flags & CLOCK_SOURCE_RESELECT) {
@@ -1255,34 +1263,6 @@ int __clocksource_register_scale(struct clocksource *cs, u32 scale, u32 freq)
}
EXPORT_SYMBOL_GPL(__clocksource_register_scale);
-static void __clocksource_change_rating(struct clocksource *cs, int rating)
-{
- list_del(&cs->list);
- cs->rating = rating;
- clocksource_enqueue(cs);
-}
-
-/**
- * clocksource_change_rating - Change the rating of a registered clocksource
- * @cs: clocksource to be changed
- * @rating: new rating
- */
-void clocksource_change_rating(struct clocksource *cs, int rating)
-{
- unsigned long flags;
-
- mutex_lock(&clocksource_mutex);
- clocksource_watchdog_lock(&flags);
- __clocksource_change_rating(cs, rating);
- clocksource_watchdog_unlock(&flags);
-
- clocksource_select();
- clocksource_select_watchdog(false);
- clocksource_suspend_select(false);
- mutex_unlock(&clocksource_mutex);
-}
-EXPORT_SYMBOL(clocksource_change_rating);
-
/*
* Unbind clocksource @cs. Called with clocksource_mutex held
*/
diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c
index cddcd08ea827..55e9ffbcd49a 100644
--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -417,6 +417,11 @@ static inline void debug_hrtimer_init(struct hrtimer *timer)
debug_object_init(timer, &hrtimer_debug_descr);
}
+static inline void debug_hrtimer_init_on_stack(struct hrtimer *timer)
+{
+ debug_object_init_on_stack(timer, &hrtimer_debug_descr);
+}
+
static inline void debug_hrtimer_activate(struct hrtimer *timer,
enum hrtimer_mode mode)
{
@@ -428,28 +433,6 @@ static inline void debug_hrtimer_deactivate(struct hrtimer *timer)
debug_object_deactivate(timer, &hrtimer_debug_descr);
}
-static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
- enum hrtimer_mode mode);
-
-void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id,
- enum hrtimer_mode mode)
-{
- debug_object_init_on_stack(timer, &hrtimer_debug_descr);
- __hrtimer_init(timer, clock_id, mode);
-}
-EXPORT_SYMBOL_GPL(hrtimer_init_on_stack);
-
-static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
- clockid_t clock_id, enum hrtimer_mode mode);
-
-void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
- clockid_t clock_id, enum hrtimer_mode mode)
-{
- debug_object_init_on_stack(&sl->timer, &hrtimer_debug_descr);
- __hrtimer_init_sleeper(sl, clock_id, mode);
-}
-EXPORT_SYMBOL_GPL(hrtimer_init_sleeper_on_stack);
-
void destroy_hrtimer_on_stack(struct hrtimer *timer)
{
debug_object_free(timer, &hrtimer_debug_descr);
@@ -459,6 +442,7 @@ EXPORT_SYMBOL_GPL(destroy_hrtimer_on_stack);
#else
static inline void debug_hrtimer_init(struct hrtimer *timer) { }
+static inline void debug_hrtimer_init_on_stack(struct hrtimer *timer) { }
static inline void debug_hrtimer_activate(struct hrtimer *timer,
enum hrtimer_mode mode) { }
static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }
@@ -472,6 +456,13 @@ debug_init(struct hrtimer *timer, clockid_t clockid,
trace_hrtimer_init(timer, clockid, mode);
}
+static inline void debug_init_on_stack(struct hrtimer *timer, clockid_t clockid,
+ enum hrtimer_mode mode)
+{
+ debug_hrtimer_init_on_stack(timer);
+ trace_hrtimer_init(timer, clockid, mode);
+}
+
static inline void debug_activate(struct hrtimer *timer,
enum hrtimer_mode mode)
{
@@ -1544,6 +1535,11 @@ static inline int hrtimer_clockid_to_base(clockid_t clock_id)
return HRTIMER_BASE_MONOTONIC;
}
+static enum hrtimer_restart hrtimer_dummy_timeout(struct hrtimer *unused)
+{
+ return HRTIMER_NORESTART;
+}
+
static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
enum hrtimer_mode mode)
{
@@ -1580,6 +1576,18 @@ static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
timerqueue_init(&timer->node);
}
+static void __hrtimer_setup(struct hrtimer *timer,
+ enum hrtimer_restart (*function)(struct hrtimer *),
+ clockid_t clock_id, enum hrtimer_mode mode)
+{
+ __hrtimer_init(timer, clock_id, mode);
+
+ if (WARN_ON_ONCE(!function))
+ timer->function = hrtimer_dummy_timeout;
+ else
+ timer->function = function;
+}
+
/**
* hrtimer_init - initialize a timer to the given clock
* @timer: the timer to be initialized
@@ -1600,6 +1608,46 @@ void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
}
EXPORT_SYMBOL_GPL(hrtimer_init);
+/**
+ * hrtimer_setup - initialize a timer to the given clock
+ * @timer: the timer to be initialized
+ * @function: the callback function
+ * @clock_id: the clock to be used
+ * @mode: The modes which are relevant for initialization:
+ * HRTIMER_MODE_ABS, HRTIMER_MODE_REL, HRTIMER_MODE_ABS_SOFT,
+ * HRTIMER_MODE_REL_SOFT
+ *
+ * The PINNED variants of the above can be handed in,
+ * but the PINNED bit is ignored as pinning happens
+ * when the hrtimer is started
+ */
+void hrtimer_setup(struct hrtimer *timer, enum hrtimer_restart (*function)(struct hrtimer *),
+ clockid_t clock_id, enum hrtimer_mode mode)
+{
+ debug_init(timer, clock_id, mode);
+ __hrtimer_setup(timer, function, clock_id, mode);
+}
+EXPORT_SYMBOL_GPL(hrtimer_setup);
+
+/**
+ * hrtimer_setup_on_stack - initialize a timer on stack memory
+ * @timer: The timer to be initialized
+ * @function: the callback function
+ * @clock_id: The clock to be used
+ * @mode: The timer mode
+ *
+ * Similar to hrtimer_setup(), except that this one must be used if struct hrtimer is in stack
+ * memory.
+ */
+void hrtimer_setup_on_stack(struct hrtimer *timer,
+ enum hrtimer_restart (*function)(struct hrtimer *),
+ clockid_t clock_id, enum hrtimer_mode mode)
+{
+ debug_init_on_stack(timer, clock_id, mode);
+ __hrtimer_setup(timer, function, clock_id, mode);
+}
+EXPORT_SYMBOL_GPL(hrtimer_setup_on_stack);
+
/*
* A timer is active, when it is enqueued into the rbtree or the
* callback function is running or it's in the state of being migrated
@@ -1944,7 +1992,7 @@ void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl,
* Make the enqueue delivery mode check work on RT. If the sleeper
* was initialized for hard interrupt delivery, force the mode bit.
* This is a special case for hrtimer_sleepers because
- * hrtimer_init_sleeper() determines the delivery mode on RT so the
+ * __hrtimer_init_sleeper() determines the delivery mode on RT so the
* fiddling with this decision is avoided at the call sites.
*/
if (IS_ENABLED(CONFIG_PREEMPT_RT) && sl->timer.is_hard)
@@ -1987,19 +2035,18 @@ static void __hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
}
/**
- * hrtimer_init_sleeper - initialize sleeper to the given clock
+ * hrtimer_setup_sleeper_on_stack - initialize a sleeper in stack memory
* @sl: sleeper to be initialized
* @clock_id: the clock to be used
* @mode: timer mode abs/rel
*/
-void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id,
- enum hrtimer_mode mode)
+void hrtimer_setup_sleeper_on_stack(struct hrtimer_sleeper *sl,
+ clockid_t clock_id, enum hrtimer_mode mode)
{
- debug_init(&sl->timer, clock_id, mode);
+ debug_init_on_stack(&sl->timer, clock_id, mode);
__hrtimer_init_sleeper(sl, clock_id, mode);
-
}
-EXPORT_SYMBOL_GPL(hrtimer_init_sleeper);
+EXPORT_SYMBOL_GPL(hrtimer_setup_sleeper_on_stack);
int nanosleep_copyout(struct restart_block *restart, struct timespec64 *ts)
{
@@ -2060,8 +2107,7 @@ static long __sched hrtimer_nanosleep_restart(struct restart_block *restart)
struct hrtimer_sleeper t;
int ret;
- hrtimer_init_sleeper_on_stack(&t, restart->nanosleep.clockid,
- HRTIMER_MODE_ABS);
+ hrtimer_setup_sleeper_on_stack(&t, restart->nanosleep.clockid, HRTIMER_MODE_ABS);
hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);
ret = do_nanosleep(&t, HRTIMER_MODE_ABS);
destroy_hrtimer_on_stack(&t.timer);
@@ -2075,7 +2121,7 @@ long hrtimer_nanosleep(ktime_t rqtp, const enum hrtimer_mode mode,
struct hrtimer_sleeper t;
int ret = 0;
- hrtimer_init_sleeper_on_stack(&t, clockid, mode);
+ hrtimer_setup_sleeper_on_stack(&t, clockid, mode);
hrtimer_set_expires_range_ns(&t.timer, rqtp, current->timer_slack_ns);
ret = do_nanosleep(&t, mode);
if (ret != -ERESTART_RESTARTBLOCK)
@@ -2242,123 +2288,3 @@ void __init hrtimers_init(void)
hrtimers_prepare_cpu(smp_processor_id());
open_softirq(HRTIMER_SOFTIRQ, hrtimer_run_softirq);
}
-
-/**
- * schedule_hrtimeout_range_clock - sleep until timeout
- * @expires: timeout value (ktime_t)
- * @delta: slack in expires timeout (ktime_t)
- * @mode: timer mode
- * @clock_id: timer clock to be used
- */
-int __sched
-schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
- const enum hrtimer_mode mode, clockid_t clock_id)
-{
- struct hrtimer_sleeper t;
-
- /*
- * Optimize when a zero timeout value is given. It does not
- * matter whether this is an absolute or a relative time.
- */
- if (expires && *expires == 0) {
- __set_current_state(TASK_RUNNING);
- return 0;
- }
-
- /*
- * A NULL parameter means "infinite"
- */
- if (!expires) {
- schedule();
- return -EINTR;
- }
-
- hrtimer_init_sleeper_on_stack(&t, clock_id, mode);
- hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
- hrtimer_sleeper_start_expires(&t, mode);
-
- if (likely(t.task))
- schedule();
-
- hrtimer_cancel(&t.timer);
- destroy_hrtimer_on_stack(&t.timer);
-
- __set_current_state(TASK_RUNNING);
-
- return !t.task ? 0 : -EINTR;
-}
-EXPORT_SYMBOL_GPL(schedule_hrtimeout_range_clock);
-
-/**
- * schedule_hrtimeout_range - sleep until timeout
- * @expires: timeout value (ktime_t)
- * @delta: slack in expires timeout (ktime_t)
- * @mode: timer mode
- *
- * Make the current task sleep until the given expiry time has
- * elapsed. The routine will return immediately unless
- * the current task state has been set (see set_current_state()).
- *
- * The @delta argument gives the kernel the freedom to schedule the
- * actual wakeup to a time that is both power and performance friendly
- * for regular (non RT/DL) tasks.
- * The kernel give the normal best effort behavior for "@expires+@delta",
- * but may decide to fire the timer earlier, but no earlier than @expires.
- *
- * You can set the task state as follows -
- *
- * %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
- * pass before the routine returns unless the current task is explicitly
- * woken up, (e.g. by wake_up_process()).
- *
- * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
- * delivered to the current task or the current task is explicitly woken
- * up.
- *
- * The current task state is guaranteed to be TASK_RUNNING when this
- * routine returns.
- *
- * Returns 0 when the timer has expired. If the task was woken before the
- * timer expired by a signal (only possible in state TASK_INTERRUPTIBLE) or
- * by an explicit wakeup, it returns -EINTR.
- */
-int __sched schedule_hrtimeout_range(ktime_t *expires, u64 delta,
- const enum hrtimer_mode mode)
-{
- return schedule_hrtimeout_range_clock(expires, delta, mode,
- CLOCK_MONOTONIC);
-}
-EXPORT_SYMBOL_GPL(schedule_hrtimeout_range);
-
-/**
- * schedule_hrtimeout - sleep until timeout
- * @expires: timeout value (ktime_t)
- * @mode: timer mode
- *
- * Make the current task sleep until the given expiry time has
- * elapsed. The routine will return immediately unless
- * the current task state has been set (see set_current_state()).
- *
- * You can set the task state as follows -
- *
- * %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
- * pass before the routine returns unless the current task is explicitly
- * woken up, (e.g. by wake_up_process()).
- *
- * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
- * delivered to the current task or the current task is explicitly woken
- * up.
- *
- * The current task state is guaranteed to be TASK_RUNNING when this
- * routine returns.
- *
- * Returns 0 when the timer has expired. If the task was woken before the
- * timer expired by a signal (only possible in state TASK_INTERRUPTIBLE) or
- * by an explicit wakeup, it returns -EINTR.
- */
-int __sched schedule_hrtimeout(ktime_t *expires,
- const enum hrtimer_mode mode)
-{
- return schedule_hrtimeout_range(expires, 0, mode);
-}
-EXPORT_SYMBOL_GPL(schedule_hrtimeout);
diff --git a/kernel/time/itimer.c b/kernel/time/itimer.c
index 00629e658ca1..876d389b2e21 100644
--- a/kernel/time/itimer.c
+++ b/kernel/time/itimer.c
@@ -151,7 +151,27 @@ COMPAT_SYSCALL_DEFINE2(getitimer, int, which,
#endif
/*
- * The timer is automagically restarted, when interval != 0
+ * Invoked from dequeue_signal() when SIG_ALRM is delivered.
+ *
+ * Restart the ITIMER_REAL timer if it is armed as periodic timer. Doing
+ * this in the signal delivery path instead of self rearming prevents a DoS
+ * with small increments in the high reolution timer case and reduces timer
+ * noise in general.
+ */
+void posixtimer_rearm_itimer(struct task_struct *tsk)
+{
+ struct hrtimer *tmr = &tsk->signal->real_timer;
+
+ if (!hrtimer_is_queued(tmr) && tsk->signal->it_real_incr != 0) {
+ hrtimer_forward(tmr, tmr->base->get_time(),
+ tsk->signal->it_real_incr);
+ hrtimer_restart(tmr);
+ }
+}
+
+/*
+ * Interval timers are restarted in the signal delivery path. See
+ * posixtimer_rearm_itimer().
*/
enum hrtimer_restart it_real_fn(struct hrtimer *timer)
{
diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c
index 802b336f4b8c..b550ebe0f03b 100644
--- a/kernel/time/ntp.c
+++ b/kernel/time/ntp.c
@@ -22,22 +22,79 @@
#include "ntp_internal.h"
#include "timekeeping_internal.h"
-
-/*
- * NTP timekeeping variables:
+/**
+ * struct ntp_data - Structure holding all NTP related state
+ * @tick_usec: USER_HZ period in microseconds
+ * @tick_length: Adjusted tick length
+ * @tick_length_base: Base value for @tick_length
+ * @time_state: State of the clock synchronization
+ * @time_status: Clock status bits
+ * @time_offset: Time adjustment in nanoseconds
+ * @time_constant: PLL time constant
+ * @time_maxerror: Maximum error in microseconds holding the NTP sync distance
+ * (NTP dispersion + delay / 2)
+ * @time_esterror: Estimated error in microseconds holding NTP dispersion
+ * @time_freq: Frequency offset scaled nsecs/secs
+ * @time_reftime: Time at last adjustment in seconds
+ * @time_adjust: Adjustment value
+ * @ntp_tick_adj: Constant boot-param configurable NTP tick adjustment (upscaled)
+ * @ntp_next_leap_sec: Second value of the next pending leapsecond, or TIME64_MAX if no leap
*
- * Note: All of the NTP state is protected by the timekeeping locks.
+ * @pps_valid: PPS signal watchdog counter
+ * @pps_tf: PPS phase median filter
+ * @pps_jitter: PPS current jitter in nanoseconds
+ * @pps_fbase: PPS beginning of the last freq interval
+ * @pps_shift: PPS current interval duration in seconds (shift value)
+ * @pps_intcnt: PPS interval counter
+ * @pps_freq: PPS frequency offset in scaled ns/s
+ * @pps_stabil: PPS current stability in scaled ns/s
+ * @pps_calcnt: PPS monitor: calibration intervals
+ * @pps_jitcnt: PPS monitor: jitter limit exceeded
+ * @pps_stbcnt: PPS monitor: stability limit exceeded
+ * @pps_errcnt: PPS monitor: calibration errors
+ *
+ * Protected by the timekeeping locks.
*/
+struct ntp_data {
+ unsigned long tick_usec;
+ u64 tick_length;
+ u64 tick_length_base;
+ int time_state;
+ int time_status;
+ s64 time_offset;
+ long time_constant;
+ long time_maxerror;
+ long time_esterror;
+ s64 time_freq;
+ time64_t time_reftime;
+ long time_adjust;
+ s64 ntp_tick_adj;
+ time64_t ntp_next_leap_sec;
+#ifdef CONFIG_NTP_PPS
+ int pps_valid;
+ long pps_tf[3];
+ long pps_jitter;
+ struct timespec64 pps_fbase;
+ int pps_shift;
+ int pps_intcnt;
+ s64 pps_freq;
+ long pps_stabil;
+ long pps_calcnt;
+ long pps_jitcnt;
+ long pps_stbcnt;
+ long pps_errcnt;
+#endif
+};
-
-/* USER_HZ period (usecs): */
-unsigned long tick_usec = USER_TICK_USEC;
-
-/* SHIFTED_HZ period (nsecs): */
-unsigned long tick_nsec;
-
-static u64 tick_length;
-static u64 tick_length_base;
+static struct ntp_data tk_ntp_data = {
+ .tick_usec = USER_TICK_USEC,
+ .time_state = TIME_OK,
+ .time_status = STA_UNSYNC,
+ .time_constant = 2,
+ .time_maxerror = NTP_PHASE_LIMIT,
+ .time_esterror = NTP_PHASE_LIMIT,
+ .ntp_next_leap_sec = TIME64_MAX,
+};
#define SECS_PER_DAY 86400
#define MAX_TICKADJ 500LL /* usecs */
@@ -45,46 +102,6 @@ static u64 tick_length_base;
(((MAX_TICKADJ * NSEC_PER_USEC) << NTP_SCALE_SHIFT) / NTP_INTERVAL_FREQ)
#define MAX_TAI_OFFSET 100000
-/*
- * phase-lock loop variables
- */
-
-/*
- * clock synchronization status
- *
- * (TIME_ERROR prevents overwriting the CMOS clock)
- */
-static int time_state = TIME_OK;
-
-/* clock status bits: */
-static int time_status = STA_UNSYNC;
-
-/* time adjustment (nsecs): */
-static s64 time_offset;
-
-/* pll time constant: */
-static long time_constant = 2;
-
-/* maximum error (usecs): */
-static long time_maxerror = NTP_PHASE_LIMIT;
-
-/* estimated error (usecs): */
-static long time_esterror = NTP_PHASE_LIMIT;
-
-/* frequency offset (scaled nsecs/secs): */
-static s64 time_freq;
-
-/* time at last adjustment (secs): */
-static time64_t time_reftime;
-
-static long time_adjust;
-
-/* constant (boot-param configurable) NTP tick adjustment (upscaled) */
-static s64 ntp_tick_adj;
-
-/* second value of the next pending leapsecond, or TIME64_MAX if no leap */
-static time64_t ntp_next_leap_sec = TIME64_MAX;
-
#ifdef CONFIG_NTP_PPS
/*
@@ -101,128 +118,115 @@ static time64_t ntp_next_leap_sec = TIME64_MAX;
intervals to decrease it */
#define PPS_MAXWANDER 100000 /* max PPS freq wander (ns/s) */
-static int pps_valid; /* signal watchdog counter */
-static long pps_tf[3]; /* phase median filter */
-static long pps_jitter; /* current jitter (ns) */
-static struct timespec64 pps_fbase; /* beginning of the last freq interval */
-static int pps_shift; /* current interval duration (s) (shift) */
-static int pps_intcnt; /* interval counter */
-static s64 pps_freq; /* frequency offset (scaled ns/s) */
-static long pps_stabil; /* current stability (scaled ns/s) */
-
/*
- * PPS signal quality monitors
- */
-static long pps_calcnt; /* calibration intervals */
-static long pps_jitcnt; /* jitter limit exceeded */
-static long pps_stbcnt; /* stability limit exceeded */
-static long pps_errcnt; /* calibration errors */
-
-
-/* PPS kernel consumer compensates the whole phase error immediately.
+ * PPS kernel consumer compensates the whole phase error immediately.
* Otherwise, reduce the offset by a fixed factor times the time constant.
*/
-static inline s64 ntp_offset_chunk(s64 offset)
+static inline s64 ntp_offset_chunk(struct ntp_data *ntpdata, s64 offset)
{
- if (time_status & STA_PPSTIME && time_status & STA_PPSSIGNAL)
+ if (ntpdata->time_status & STA_PPSTIME && ntpdata->time_status & STA_PPSSIGNAL)
return offset;
else
- return shift_right(offset, SHIFT_PLL + time_constant);
+ return shift_right(offset, SHIFT_PLL + ntpdata->time_constant);
}
-static inline void pps_reset_freq_interval(void)
+static inline void pps_reset_freq_interval(struct ntp_data *ntpdata)
{
- /* the PPS calibration interval may end
- surprisingly early */
- pps_shift = PPS_INTMIN;
- pps_intcnt = 0;
+ /* The PPS calibration interval may end surprisingly early */
+ ntpdata->pps_shift = PPS_INTMIN;
+ ntpdata->pps_intcnt = 0;
}
/**
* pps_clear - Clears the PPS state variables
+ * @ntpdata: Pointer to ntp data
*/
-static inline void pps_clear(void)
+static inline void pps_clear(struct ntp_data *ntpdata)
{
- pps_reset_freq_interval();
- pps_tf[0] = 0;
- pps_tf[1] = 0;
- pps_tf[2] = 0;
- pps_fbase.tv_sec = pps_fbase.tv_nsec = 0;
- pps_freq = 0;
+ pps_reset_freq_interval(ntpdata);
+ ntpdata->pps_tf[0] = 0;
+ ntpdata->pps_tf[1] = 0;
+ ntpdata->pps_tf[2] = 0;
+ ntpdata->pps_fbase.tv_sec = ntpdata->pps_fbase.tv_nsec = 0;
+ ntpdata->pps_freq = 0;
}
-/* Decrease pps_valid to indicate that another second has passed since
- * the last PPS signal. When it reaches 0, indicate that PPS signal is
- * missing.
+/*
+ * Decrease pps_valid to indicate that another second has passed since the
+ * last PPS signal. When it reaches 0, indicate that PPS signal is missing.
*/
-static inline void pps_dec_valid(void)
+static inline void pps_dec_valid(struct ntp_data *ntpdata)
{
- if (pps_valid > 0)
- pps_valid--;
- else {
- time_status &= ~(STA_PPSSIGNAL | STA_PPSJITTER |
- STA_PPSWANDER | STA_PPSERROR);
- pps_clear();
+ if (ntpdata->pps_valid > 0) {
+ ntpdata->pps_valid--;
+ } else {
+ ntpdata->time_status &= ~(STA_PPSSIGNAL | STA_PPSJITTER |
+ STA_PPSWANDER | STA_PPSERROR);
+ pps_clear(ntpdata);
}
}
-static inline void pps_set_freq(s64 freq)
+static inline void pps_set_freq(struct ntp_data *ntpdata)
{
- pps_freq = freq;
+ ntpdata->pps_freq = ntpdata->time_freq;
}
-static inline int is_error_status(int status)
+static inline bool is_error_status(int status)
{
return (status & (STA_UNSYNC|STA_CLOCKERR))
- /* PPS signal lost when either PPS time or
- * PPS frequency synchronization requested
+ /*
+ * PPS signal lost when either PPS time or PPS frequency
+ * synchronization requested
*/
|| ((status & (STA_PPSFREQ|STA_PPSTIME))
&& !(status & STA_PPSSIGNAL))
- /* PPS jitter exceeded when
- * PPS time synchronization requested */
+ /*
+ * PPS jitter exceeded when PPS time synchronization
+ * requested
+ */
|| ((status & (STA_PPSTIME|STA_PPSJITTER))
== (STA_PPSTIME|STA_PPSJITTER))
- /* PPS wander exceeded or calibration error when
- * PPS frequency synchronization requested
+ /*
+ * PPS wander exceeded or calibration error when PPS
+ * frequency synchronization requested
*/
|| ((status & STA_PPSFREQ)
&& (status & (STA_PPSWANDER|STA_PPSERROR)));
}
-static inline void pps_fill_timex(struct __kernel_timex *txc)
+static inline void pps_fill_timex(struct ntp_data *ntpdata, struct __kernel_timex *txc)
{
- txc->ppsfreq = shift_right((pps_freq >> PPM_SCALE_INV_SHIFT) *
+ txc->ppsfreq = shift_right((ntpdata->pps_freq >> PPM_SCALE_INV_SHIFT) *
PPM_SCALE_INV, NTP_SCALE_SHIFT);
- txc->jitter = pps_jitter;
- if (!(time_status & STA_NANO))
- txc->jitter = pps_jitter / NSEC_PER_USEC;
- txc->shift = pps_shift;
- txc->stabil = pps_stabil;
- txc->jitcnt = pps_jitcnt;
- txc->calcnt = pps_calcnt;
- txc->errcnt = pps_errcnt;
- txc->stbcnt = pps_stbcnt;
+ txc->jitter = ntpdata->pps_jitter;
+ if (!(ntpdata->time_status & STA_NANO))
+ txc->jitter = ntpdata->pps_jitter / NSEC_PER_USEC;
+ txc->shift = ntpdata->pps_shift;
+ txc->stabil = ntpdata->pps_stabil;
+ txc->jitcnt = ntpdata->pps_jitcnt;
+ txc->calcnt = ntpdata->pps_calcnt;
+ txc->errcnt = ntpdata->pps_errcnt;
+ txc->stbcnt = ntpdata->pps_stbcnt;
}
#else /* !CONFIG_NTP_PPS */
-static inline s64 ntp_offset_chunk(s64 offset)
+static inline s64 ntp_offset_chunk(struct ntp_data *ntpdata, s64 offset)
{
- return shift_right(offset, SHIFT_PLL + time_constant);
+ return shift_right(offset, SHIFT_PLL + ntpdata->time_constant);
}
-static inline void pps_reset_freq_interval(void) {}
-static inline void pps_clear(void) {}
-static inline void pps_dec_valid(void) {}
-static inline void pps_set_freq(s64 freq) {}
+static inline void pps_reset_freq_interval(struct ntp_data *ntpdata) {}
+static inline void pps_clear(struct ntp_data *ntpdata) {}
+static inline void pps_dec_valid(struct ntp_data *ntpdata) {}
+static inline void pps_set_freq(struct ntp_data *ntpdata) {}
-static inline int is_error_status(int status)
+static inline bool is_error_status(int status)
{
return status & (STA_UNSYNC|STA_CLOCKERR);
}
-static inline void pps_fill_timex(struct __kernel_timex *txc)
+static inline void pps_fill_timex(struct ntp_data *ntpdata, struct __kernel_timex *txc)
{
/* PPS is not implemented, so these are zero */
txc->ppsfreq = 0;
@@ -237,138 +241,123 @@ static inline void pps_fill_timex(struct __kernel_timex *txc)
#endif /* CONFIG_NTP_PPS */
-
-/**
- * ntp_synced - Returns 1 if the NTP status is not UNSYNC
- *
- */
-static inline int ntp_synced(void)
-{
- return !(time_status & STA_UNSYNC);
-}
-
-
/*
- * NTP methods:
+ * Update tick_length and tick_length_base, based on tick_usec, ntp_tick_adj and
+ * time_freq:
*/
-
-/*
- * Update (tick_length, tick_length_base, tick_nsec), based
- * on (tick_usec, ntp_tick_adj, time_freq):
- */
-static void ntp_update_frequency(void)
+static void ntp_update_frequency(struct ntp_data *ntpdata)
{
- u64 second_length;
- u64 new_base;
+ u64 second_length, new_base, tick_usec = (u64)ntpdata->tick_usec;
- second_length = (u64)(tick_usec * NSEC_PER_USEC * USER_HZ)
- << NTP_SCALE_SHIFT;
+ second_length = (u64)(tick_usec * NSEC_PER_USEC * USER_HZ) << NTP_SCALE_SHIFT;
- second_length += ntp_tick_adj;
- second_length += time_freq;
+ second_length += ntpdata->ntp_tick_adj;
+ second_length += ntpdata->time_freq;
- tick_nsec = div_u64(second_length, HZ) >> NTP_SCALE_SHIFT;
new_base = div_u64(second_length, NTP_INTERVAL_FREQ);
/*
- * Don't wait for the next second_overflow, apply
- * the change to the tick length immediately:
+ * Don't wait for the next second_overflow, apply the change to the
+ * tick length immediately:
*/
- tick_length += new_base - tick_length_base;
- tick_length_base = new_base;
+ ntpdata->tick_length += new_base - ntpdata->tick_length_base;
+ ntpdata->tick_length_base = new_base;
}
-static inline s64 ntp_update_offset_fll(s64 offset64, long secs)
+static inline s64 ntp_update_offset_fll(struct ntp_data *ntpdata, s64 offset64, long secs)
{
- time_status &= ~STA_MODE;
+ ntpdata->time_status &= ~STA_MODE;
if (secs < MINSEC)
return 0;
- if (!(time_status & STA_FLL) && (secs <= MAXSEC))
+ if (!(ntpdata->time_status & STA_FLL) && (secs <= MAXSEC))
return 0;
- time_status |= STA_MODE;
+ ntpdata->time_status |= STA_MODE;
return div64_long(offset64 << (NTP_SCALE_SHIFT - SHIFT_FLL), secs);
}
-static void ntp_update_offset(long offset)
+static void ntp_update_offset(struct ntp_data *ntpdata, long offset)
{
- s64 freq_adj;
- s64 offset64;
- long secs;
+ s64 freq_adj, offset64;
+ long secs, real_secs;
- if (!(time_status & STA_PLL))
+ if (!(ntpdata->time_status & STA_PLL))
return;
- if (!(time_status & STA_NANO)) {
+ if (!(ntpdata->time_status & STA_NANO)) {
/* Make sure the multiplication below won't overflow */
offset = clamp(offset, -USEC_PER_SEC, USEC_PER_SEC);
offset *= NSEC_PER_USEC;
}
- /*
- * Scale the phase adjustment and
- * clamp to the operating range.
- */
+ /* Scale the phase adjustment and clamp to the operating range. */
offset = clamp(offset, -MAXPHASE, MAXPHASE);
/*
* Select how the frequency is to be controlled
* and in which mode (PLL or FLL).
*/
- secs = (long)(__ktime_get_real_seconds() - time_reftime);
- if (unlikely(time_status & STA_FREQHOLD))
+ real_secs = __ktime_get_real_seconds();
+ secs = (long)(real_secs - ntpdata->time_reftime);
+ if (unlikely(ntpdata->time_status & STA_FREQHOLD))
secs = 0;
- time_reftime = __ktime_get_real_seconds();
+ ntpdata->time_reftime = real_secs;
offset64 = offset;
- freq_adj = ntp_update_offset_fll(offset64, secs);
+ freq_adj = ntp_update_offset_fll(ntpdata, offset64, secs);
/*
* Clamp update interval to reduce PLL gain with low
* sampling rate (e.g. intermittent network connection)
* to avoid instability.
*/
- if (unlikely(secs > 1 << (SHIFT_PLL + 1 + time_constant)))
- secs = 1 << (SHIFT_PLL + 1 + time_constant);
+ if (unlikely(secs > 1 << (SHIFT_PLL + 1 + ntpdata->time_constant)))
+ secs = 1 << (SHIFT_PLL + 1 + ntpdata->time_constant);
freq_adj += (offset64 * secs) <<
- (NTP_SCALE_SHIFT - 2 * (SHIFT_PLL + 2 + time_constant));
+ (NTP_SCALE_SHIFT - 2 * (SHIFT_PLL + 2 + ntpdata->time_constant));
- freq_adj = min(freq_adj + time_freq, MAXFREQ_SCALED);
+ freq_adj = min(freq_adj + ntpdata->time_freq, MAXFREQ_SCALED);
- time_freq = max(freq_adj, -MAXFREQ_SCALED);
+ ntpdata->time_freq = max(freq_adj, -MAXFREQ_SCALED);
- time_offset = div_s64(offset64 << NTP_SCALE_SHIFT, NTP_INTERVAL_FREQ);
+ ntpdata->time_offset = div_s64(offset64 << NTP_SCALE_SHIFT, NTP_INTERVAL_FREQ);
}
-/**
- * ntp_clear - Clears the NTP state variables
- */
-void ntp_clear(void)
+static void __ntp_clear(struct ntp_data *ntpdata)
{
- time_adjust = 0; /* stop active adjtime() */
- time_status |= STA_UNSYNC;
- time_maxerror = NTP_PHASE_LIMIT;
- time_esterror = NTP_PHASE_LIMIT;
+ /* Stop active adjtime() */
+ ntpdata->time_adjust = 0;
+ ntpdata->time_status |= STA_UNSYNC;
+ ntpdata->time_maxerror = NTP_PHASE_LIMIT;
+ ntpdata->time_esterror = NTP_PHASE_LIMIT;
- ntp_update_frequency();
+ ntp_update_frequency(ntpdata);
- tick_length = tick_length_base;
- time_offset = 0;
+ ntpdata->tick_length = ntpdata->tick_length_base;
+ ntpdata->time_offset = 0;
- ntp_next_leap_sec = TIME64_MAX;
+ ntpdata->ntp_next_leap_sec = TIME64_MAX;
/* Clear PPS state variables */
- pps_clear();
+ pps_clear(ntpdata);
+}
+
+/**
+ * ntp_clear - Clears the NTP state variables
+ */
+void ntp_clear(void)
+{
+ __ntp_clear(&tk_ntp_data);
}
u64 ntp_tick_length(void)
{
- return tick_length;
+ return tk_ntp_data.tick_length;
}
/**
@@ -379,16 +368,17 @@ u64 ntp_tick_length(void)
*/
ktime_t ntp_get_next_leap(void)
{
+ struct ntp_data *ntpdata = &tk_ntp_data;
ktime_t ret;
- if ((time_state == TIME_INS) && (time_status & STA_INS))
- return ktime_set(ntp_next_leap_sec, 0);
+ if ((ntpdata->time_state == TIME_INS) && (ntpdata->time_status & STA_INS))
+ return ktime_set(ntpdata->ntp_next_leap_sec, 0);
ret = KTIME_MAX;
return ret;
}
/*
- * this routine handles the overflow of the microsecond field
+ * This routine handles the overflow of the microsecond field
*
* The tricky bits of code to handle the accurate clock support
* were provided by Dave Mills (Mills@UDEL.EDU) of NTP fame.
@@ -399,6 +389,7 @@ ktime_t ntp_get_next_leap(void)
*/
int second_overflow(time64_t secs)
{
+ struct ntp_data *ntpdata = &tk_ntp_data;
s64 delta;
int leap = 0;
s32 rem;
@@ -408,87 +399,84 @@ int second_overflow(time64_t secs)
* day, the system clock is set back one second; if in leap-delete
* state, the system clock is set ahead one second.
*/
- switch (time_state) {
+ switch (ntpdata->time_state) {
case TIME_OK:
- if (time_status & STA_INS) {
- time_state = TIME_INS;
+ if (ntpdata->time_status & STA_INS) {
+ ntpdata->time_state = TIME_INS;
div_s64_rem(secs, SECS_PER_DAY, &rem);
- ntp_next_leap_sec = secs + SECS_PER_DAY - rem;
- } else if (time_status & STA_DEL) {
- time_state = TIME_DEL;
+ ntpdata->ntp_next_leap_sec = secs + SECS_PER_DAY - rem;
+ } else if (ntpdata->time_status & STA_DEL) {
+ ntpdata->time_state = TIME_DEL;
div_s64_rem(secs + 1, SECS_PER_DAY, &rem);
- ntp_next_leap_sec = secs + SECS_PER_DAY - rem;
+ ntpdata->ntp_next_leap_sec = secs + SECS_PER_DAY - rem;
}
break;
case TIME_INS:
- if (!(time_status & STA_INS)) {
- ntp_next_leap_sec = TIME64_MAX;
- time_state = TIME_OK;
- } else if (secs == ntp_next_leap_sec) {
+ if (!(ntpdata->time_status & STA_INS)) {
+ ntpdata->ntp_next_leap_sec = TIME64_MAX;
+ ntpdata->time_state = TIME_OK;
+ } else if (secs == ntpdata->ntp_next_leap_sec) {
leap = -1;
- time_state = TIME_OOP;
- printk(KERN_NOTICE
- "Clock: inserting leap second 23:59:60 UTC\n");
+ ntpdata->time_state = TIME_OOP;
+ pr_notice("Clock: inserting leap second 23:59:60 UTC\n");
}
break;
case TIME_DEL:
- if (!(time_status & STA_DEL)) {
- ntp_next_leap_sec = TIME64_MAX;
- time_state = TIME_OK;
- } else if (secs == ntp_next_leap_sec) {
+ if (!(ntpdata->time_status & STA_DEL)) {
+ ntpdata->ntp_next_leap_sec = TIME64_MAX;
+ ntpdata->time_state = TIME_OK;
+ } else if (secs == ntpdata->ntp_next_leap_sec) {
leap = 1;
- ntp_next_leap_sec = TIME64_MAX;
- time_state = TIME_WAIT;
- printk(KERN_NOTICE
- "Clock: deleting leap second 23:59:59 UTC\n");
+ ntpdata->ntp_next_leap_sec = TIME64_MAX;
+ ntpdata->time_state = TIME_WAIT;
+ pr_notice("Clock: deleting leap second 23:59:59 UTC\n");
}
break;
case TIME_OOP:
- ntp_next_leap_sec = TIME64_MAX;
- time_state = TIME_WAIT;
+ ntpdata->ntp_next_leap_sec = TIME64_MAX;
+ ntpdata->time_state = TIME_WAIT;
break;
case TIME_WAIT:
- if (!(time_status & (STA_INS | STA_DEL)))
- time_state = TIME_OK;
+ if (!(ntpdata->time_status & (STA_INS | STA_DEL)))
+ ntpdata->time_state = TIME_OK;
break;
}
-
/* Bump the maxerror field */
- time_maxerror += MAXFREQ / NSEC_PER_USEC;
- if (time_maxerror > NTP_PHASE_LIMIT) {
- time_maxerror = NTP_PHASE_LIMIT;
- time_status |= STA_UNSYNC;
+ ntpdata->time_maxerror += MAXFREQ / NSEC_PER_USEC;
+ if (ntpdata->time_maxerror > NTP_PHASE_LIMIT) {
+ ntpdata->time_maxerror = NTP_PHASE_LIMIT;
+ ntpdata->time_status |= STA_UNSYNC;
}
/* Compute the phase adjustment for the next second */
- tick_length = tick_length_base;
+ ntpdata->tick_length = ntpdata->tick_length_base;
- delta = ntp_offset_chunk(time_offset);
- time_offset -= delta;
- tick_length += delta;
+ delta = ntp_offset_chunk(ntpdata, ntpdata->time_offset);
+ ntpdata->time_offset -= delta;
+ ntpdata->tick_length += delta;
/* Check PPS signal */
- pps_dec_valid();
+ pps_dec_valid(ntpdata);
- if (!time_adjust)
+ if (!ntpdata->time_adjust)
goto out;
- if (time_adjust > MAX_TICKADJ) {
- time_adjust -= MAX_TICKADJ;
- tick_length += MAX_TICKADJ_SCALED;
+ if (ntpdata->time_adjust > MAX_TICKADJ) {
+ ntpdata->time_adjust -= MAX_TICKADJ;
+ ntpdata->tick_length += MAX_TICKADJ_SCALED;
goto out;
}
- if (time_adjust < -MAX_TICKADJ) {
- time_adjust += MAX_TICKADJ;
- tick_length -= MAX_TICKADJ_SCALED;
+ if (ntpdata->time_adjust < -MAX_TICKADJ) {
+ ntpdata->time_adjust += MAX_TICKADJ;
+ ntpdata->tick_length -= MAX_TICKADJ_SCALED;
goto out;
}
- tick_length += (s64)(time_adjust * NSEC_PER_USEC / NTP_INTERVAL_FREQ)
- << NTP_SCALE_SHIFT;
- time_adjust = 0;
+ ntpdata->tick_length += (s64)(ntpdata->time_adjust * NSEC_PER_USEC / NTP_INTERVAL_FREQ)
+ << NTP_SCALE_SHIFT;
+ ntpdata->time_adjust = 0;
out:
return leap;
@@ -611,6 +599,15 @@ static inline int update_rtc(struct timespec64 *to_set, unsigned long *offset_ns
}
#endif
+/**
+ * ntp_synced - Tells whether the NTP status is not UNSYNC
+ * Returns: true if not UNSYNC, false otherwise
+ */
+static inline bool ntp_synced(void)
+{
+ return !(tk_ntp_data.time_status & STA_UNSYNC);
+}
+
/*
* If we have an externally synchronized Linux clock, then update RTC clock
* accordingly every ~11 minutes. Generally RTCs can only store second
@@ -691,162 +688,156 @@ static inline void __init ntp_init_cmos_sync(void) { }
/*
* Propagate a new txc->status value into the NTP state:
*/
-static inline void process_adj_status(const struct __kernel_timex *txc)
+static inline void process_adj_status(struct ntp_data *ntpdata, const struct __kernel_timex *txc)
{
- if ((time_status & STA_PLL) && !(txc->status & STA_PLL)) {
- time_state = TIME_OK;
- time_status = STA_UNSYNC;
- ntp_next_leap_sec = TIME64_MAX;
- /* restart PPS frequency calibration */
- pps_reset_freq_interval();
+ if ((ntpdata->time_status & STA_PLL) && !(txc->status & STA_PLL)) {
+ ntpdata->time_state = TIME_OK;
+ ntpdata->time_status = STA_UNSYNC;
+ ntpdata->ntp_next_leap_sec = TIME64_MAX;
+ /* Restart PPS frequency calibration */
+ pps_reset_freq_interval(ntpdata);
}
/*
* If we turn on PLL adjustments then reset the
* reference time to current time.
*/
- if (!(time_status & STA_PLL) && (txc->status & STA_PLL))
- time_reftime = __ktime_get_real_seconds();
+ if (!(ntpdata->time_status & STA_PLL) && (txc->status & STA_PLL))
+ ntpdata->time_reftime = __ktime_get_real_seconds();
/* only set allowed bits */
- time_status &= STA_RONLY;
- time_status |= txc->status & ~STA_RONLY;
+ ntpdata->time_status &= STA_RONLY;
+ ntpdata->time_status |= txc->status & ~STA_RONLY;
}
-
-static inline void process_adjtimex_modes(const struct __kernel_timex *txc,
+static inline void process_adjtimex_modes(struct ntp_data *ntpdata, const struct __kernel_timex *txc,
s32 *time_tai)
{
if (txc->modes & ADJ_STATUS)
- process_adj_status(txc);
+ process_adj_status(ntpdata, txc);
if (txc->modes & ADJ_NANO)
- time_status |= STA_NANO;
+ ntpdata->time_status |= STA_NANO;
if (txc->modes & ADJ_MICRO)
- time_status &= ~STA_NANO;
+ ntpdata->time_status &= ~STA_NANO;
if (txc->modes & ADJ_FREQUENCY) {
- time_freq = txc->freq * PPM_SCALE;
- time_freq = min(time_freq, MAXFREQ_SCALED);
- time_freq = max(time_freq, -MAXFREQ_SCALED);
- /* update pps_freq */
- pps_set_freq(time_freq);
+ ntpdata->time_freq = txc->freq * PPM_SCALE;
+ ntpdata->time_freq = min(ntpdata->time_freq, MAXFREQ_SCALED);
+ ntpdata->time_freq = max(ntpdata->time_freq, -MAXFREQ_SCALED);
+ /* Update pps_freq */
+ pps_set_freq(ntpdata);
}
if (txc->modes & ADJ_MAXERROR)
- time_maxerror = clamp(txc->maxerror, 0, NTP_PHASE_LIMIT);
+ ntpdata->time_maxerror = clamp(txc->maxerror, 0, NTP_PHASE_LIMIT);
if (txc->modes & ADJ_ESTERROR)
- time_esterror = clamp(txc->esterror, 0, NTP_PHASE_LIMIT);
+ ntpdata->time_esterror = clamp(txc->esterror, 0, NTP_PHASE_LIMIT);
if (txc->modes & ADJ_TIMECONST) {
- time_constant = clamp(txc->constant, 0, MAXTC);
- if (!(time_status & STA_NANO))
- time_constant += 4;
- time_constant = clamp(time_constant, 0, MAXTC);
+ ntpdata->time_constant = clamp(txc->constant, 0, MAXTC);
+ if (!(ntpdata->time_status & STA_NANO))
+ ntpdata->time_constant += 4;
+ ntpdata->time_constant = clamp(ntpdata->time_constant, 0, MAXTC);
}
- if (txc->modes & ADJ_TAI &&
- txc->constant >= 0 && txc->constant <= MAX_TAI_OFFSET)
+ if (txc->modes & ADJ_TAI && txc->constant >= 0 && txc->constant <= MAX_TAI_OFFSET)
*time_tai = txc->constant;
if (txc->modes & ADJ_OFFSET)
- ntp_update_offset(txc->offset);
+ ntp_update_offset(ntpdata, txc->offset);
if (txc->modes & ADJ_TICK)
- tick_usec = txc->tick;
+ ntpdata->tick_usec = txc->tick;
if (txc->modes & (ADJ_TICK|ADJ_FREQUENCY|ADJ_OFFSET))
- ntp_update_frequency();
+ ntp_update_frequency(ntpdata);
}
-
/*
- * adjtimex mainly allows reading (and writing, if superuser) of
+ * adjtimex() mainly allows reading (and writing, if superuser) of
* kernel time-keeping variables. used by xntpd.
*/
int __do_adjtimex(struct __kernel_timex *txc, const struct timespec64 *ts,
s32 *time_tai, struct audit_ntp_data *ad)
{
+ struct ntp_data *ntpdata = &tk_ntp_data;
int result;
if (txc->modes & ADJ_ADJTIME) {
- long save_adjust = time_adjust;
+ long save_adjust = ntpdata->time_adjust;
if (!(txc->modes & ADJ_OFFSET_READONLY)) {
/* adjtime() is independent from ntp_adjtime() */
- time_adjust = txc->offset;
- ntp_update_frequency();
+ ntpdata->time_adjust = txc->offset;
+ ntp_update_frequency(ntpdata);
audit_ntp_set_old(ad, AUDIT_NTP_ADJUST, save_adjust);
- audit_ntp_set_new(ad, AUDIT_NTP_ADJUST, time_adjust);
+ audit_ntp_set_new(ad, AUDIT_NTP_ADJUST, ntpdata->time_adjust);
}
txc->offset = save_adjust;
} else {
/* If there are input parameters, then process them: */
if (txc->modes) {
- audit_ntp_set_old(ad, AUDIT_NTP_OFFSET, time_offset);
- audit_ntp_set_old(ad, AUDIT_NTP_FREQ, time_freq);
- audit_ntp_set_old(ad, AUDIT_NTP_STATUS, time_status);
+ audit_ntp_set_old(ad, AUDIT_NTP_OFFSET, ntpdata->time_offset);
+ audit_ntp_set_old(ad, AUDIT_NTP_FREQ, ntpdata->time_freq);
+ audit_ntp_set_old(ad, AUDIT_NTP_STATUS, ntpdata->time_status);
audit_ntp_set_old(ad, AUDIT_NTP_TAI, *time_tai);
- audit_ntp_set_old(ad, AUDIT_NTP_TICK, tick_usec);
+ audit_ntp_set_old(ad, AUDIT_NTP_TICK, ntpdata->tick_usec);
- process_adjtimex_modes(txc, time_tai);
+ process_adjtimex_modes(ntpdata, txc, time_tai);
- audit_ntp_set_new(ad, AUDIT_NTP_OFFSET, time_offset);
- audit_ntp_set_new(ad, AUDIT_NTP_FREQ, time_freq);
- audit_ntp_set_new(ad, AUDIT_NTP_STATUS, time_status);
+ audit_ntp_set_new(ad, AUDIT_NTP_OFFSET, ntpdata->time_offset);
+ audit_ntp_set_new(ad, AUDIT_NTP_FREQ, ntpdata->time_freq);
+ audit_ntp_set_new(ad, AUDIT_NTP_STATUS, ntpdata->time_status);
audit_ntp_set_new(ad, AUDIT_NTP_TAI, *time_tai);
- audit_ntp_set_new(ad, AUDIT_NTP_TICK, tick_usec);
+ audit_ntp_set_new(ad, AUDIT_NTP_TICK, ntpdata->tick_usec);
}
- txc->offset = shift_right(time_offset * NTP_INTERVAL_FREQ,
- NTP_SCALE_SHIFT);
- if (!(time_status & STA_NANO))
+ txc->offset = shift_right(ntpdata->time_offset * NTP_INTERVAL_FREQ, NTP_SCALE_SHIFT);
+ if (!(ntpdata->time_status & STA_NANO))
txc->offset = (u32)txc->offset / NSEC_PER_USEC;
}
- result = time_state; /* mostly `TIME_OK' */
- /* check for errors */
- if (is_error_status(time_status))
+ result = ntpdata->time_state;
+ if (is_error_status(ntpdata->time_status))
result = TIME_ERROR;
- txc->freq = shift_right((time_freq >> PPM_SCALE_INV_SHIFT) *
+ txc->freq = shift_right((ntpdata->time_freq >> PPM_SCALE_INV_SHIFT) *
PPM_SCALE_INV, NTP_SCALE_SHIFT);
- txc->maxerror = time_maxerror;
- txc->esterror = time_esterror;
- txc->status = time_status;
- txc->constant = time_constant;
+ txc->maxerror = ntpdata->time_maxerror;
+ txc->esterror = ntpdata->time_esterror;
+ txc->status = ntpdata->time_status;
+ txc->constant = ntpdata->time_constant;
txc->precision = 1;
txc->tolerance = MAXFREQ_SCALED / PPM_SCALE;
- txc->tick = tick_usec;
+ txc->tick = ntpdata->tick_usec;
txc->tai = *time_tai;
- /* fill PPS status fields */
- pps_fill_timex(txc);
+ /* Fill PPS status fields */
+ pps_fill_timex(ntpdata, txc);
txc->time.tv_sec = ts->tv_sec;
txc->time.tv_usec = ts->tv_nsec;
- if (!(time_status & STA_NANO))
+ if (!(ntpdata->time_status & STA_NANO))
txc->time.tv_usec = ts->tv_nsec / NSEC_PER_USEC;
/* Handle leapsec adjustments */
- if (unlikely(ts->tv_sec >= ntp_next_leap_sec)) {
- if ((time_state == TIME_INS) && (time_status & STA_INS)) {
+ if (unlikely(ts->tv_sec >= ntpdata->ntp_next_leap_sec)) {
+ if ((ntpdata->time_state == TIME_INS) && (ntpdata->time_status & STA_INS)) {
result = TIME_OOP;
txc->tai++;
txc->time.tv_sec--;
}
- if ((time_state == TIME_DEL) && (time_status & STA_DEL)) {
+ if ((ntpdata->time_state == TIME_DEL) && (ntpdata->time_status & STA_DEL)) {
result = TIME_WAIT;
txc->tai--;
txc->time.tv_sec++;
}
- if ((time_state == TIME_OOP) &&
- (ts->tv_sec == ntp_next_leap_sec)) {
+ if ((ntpdata->time_state == TIME_OOP) && (ts->tv_sec == ntpdata->ntp_next_leap_sec))
result = TIME_WAIT;
- }
}
return result;
@@ -854,17 +845,21 @@ int __do_adjtimex(struct __kernel_timex *txc, const struct timespec64 *ts,
#ifdef CONFIG_NTP_PPS
-/* actually struct pps_normtime is good old struct timespec, but it is
+/*
+ * struct pps_normtime is basically a struct timespec, but it is
* semantically different (and it is the reason why it was invented):
* pps_normtime.nsec has a range of ( -NSEC_PER_SEC / 2, NSEC_PER_SEC / 2 ]
- * while timespec.tv_nsec has a range of [0, NSEC_PER_SEC) */
+ * while timespec.tv_nsec has a range of [0, NSEC_PER_SEC)
+ */
struct pps_normtime {
s64 sec; /* seconds */
long nsec; /* nanoseconds */
};
-/* normalize the timestamp so that nsec is in the
- ( -NSEC_PER_SEC / 2, NSEC_PER_SEC / 2 ] interval */
+/*
+ * Normalize the timestamp so that nsec is in the
+ * [ -NSEC_PER_SEC / 2, NSEC_PER_SEC / 2 ] interval
+ */
static inline struct pps_normtime pps_normalize_ts(struct timespec64 ts)
{
struct pps_normtime norm = {
@@ -880,54 +875,57 @@ static inline struct pps_normtime pps_normalize_ts(struct timespec64 ts)
return norm;
}
-/* get current phase correction and jitter */
-static inline long pps_phase_filter_get(long *jitter)
+/* Get current phase correction and jitter */
+static inline long pps_phase_filter_get(struct ntp_data *ntpdata, long *jitter)
{
- *jitter = pps_tf[0] - pps_tf[1];
+ *jitter = ntpdata->pps_tf[0] - ntpdata->pps_tf[1];
if (*jitter < 0)
*jitter = -*jitter;
/* TODO: test various filters */
- return pps_tf[0];
+ return ntpdata->pps_tf[0];
}
-/* add the sample to the phase filter */
-static inline void pps_phase_filter_add(long err)
+/* Add the sample to the phase filter */
+static inline void pps_phase_filter_add(struct ntp_data *ntpdata, long err)
{
- pps_tf[2] = pps_tf[1];
- pps_tf[1] = pps_tf[0];
- pps_tf[0] = err;
+ ntpdata->pps_tf[2] = ntpdata->pps_tf[1];
+ ntpdata->pps_tf[1] = ntpdata->pps_tf[0];
+ ntpdata->pps_tf[0] = err;
}
-/* decrease frequency calibration interval length.
- * It is halved after four consecutive unstable intervals.
+/*
+ * Decrease frequency calibration interval length. It is halved after four
+ * consecutive unstable intervals.
*/
-static inline void pps_dec_freq_interval(void)
+static inline void pps_dec_freq_interval(struct ntp_data *ntpdata)
{
- if (--pps_intcnt <= -PPS_INTCOUNT) {
- pps_intcnt = -PPS_INTCOUNT;
- if (pps_shift > PPS_INTMIN) {
- pps_shift--;
- pps_intcnt = 0;
+ if (--ntpdata->pps_intcnt <= -PPS_INTCOUNT) {
+ ntpdata->pps_intcnt = -PPS_INTCOUNT;
+ if (ntpdata->pps_shift > PPS_INTMIN) {
+ ntpdata->pps_shift--;
+ ntpdata->pps_intcnt = 0;
}
}
}
-/* increase frequency calibration interval length.
- * It is doubled after four consecutive stable intervals.
+/*
+ * Increase frequency calibration interval length. It is doubled after
+ * four consecutive stable intervals.
*/
-static inline void pps_inc_freq_interval(void)
+static inline void pps_inc_freq_interval(struct ntp_data *ntpdata)
{
- if (++pps_intcnt >= PPS_INTCOUNT) {
- pps_intcnt = PPS_INTCOUNT;
- if (pps_shift < PPS_INTMAX) {
- pps_shift++;
- pps_intcnt = 0;
+ if (++ntpdata->pps_intcnt >= PPS_INTCOUNT) {
+ ntpdata->pps_intcnt = PPS_INTCOUNT;
+ if (ntpdata->pps_shift < PPS_INTMAX) {
+ ntpdata->pps_shift++;
+ ntpdata->pps_intcnt = 0;
}
}
}
-/* update clock frequency based on MONOTONIC_RAW clock PPS signal
+/*
+ * Update clock frequency based on MONOTONIC_RAW clock PPS signal
* timestamps
*
* At the end of the calibration interval the difference between the
@@ -936,90 +934,88 @@ static inline void pps_inc_freq_interval(void)
* too long, the data are discarded.
* Returns the difference between old and new frequency values.
*/
-static long hardpps_update_freq(struct pps_normtime freq_norm)
+static long hardpps_update_freq(struct ntp_data *ntpdata, struct pps_normtime freq_norm)
{
long delta, delta_mod;
s64 ftemp;
- /* check if the frequency interval was too long */
- if (freq_norm.sec > (2 << pps_shift)) {
- time_status |= STA_PPSERROR;
- pps_errcnt++;
- pps_dec_freq_interval();
- printk_deferred(KERN_ERR
- "hardpps: PPSERROR: interval too long - %lld s\n",
- freq_norm.sec);
+ /* Check if the frequency interval was too long */
+ if (freq_norm.sec > (2 << ntpdata->pps_shift)) {
+ ntpdata->time_status |= STA_PPSERROR;
+ ntpdata->pps_errcnt++;
+ pps_dec_freq_interval(ntpdata);
+ printk_deferred(KERN_ERR "hardpps: PPSERROR: interval too long - %lld s\n",
+ freq_norm.sec);
return 0;
}
- /* here the raw frequency offset and wander (stability) is
- * calculated. If the wander is less than the wander threshold
- * the interval is increased; otherwise it is decreased.
+ /*
+ * Here the raw frequency offset and wander (stability) is
+ * calculated. If the wander is less than the wander threshold the
+ * interval is increased; otherwise it is decreased.
*/
ftemp = div_s64(((s64)(-freq_norm.nsec)) << NTP_SCALE_SHIFT,
freq_norm.sec);
- delta = shift_right(ftemp - pps_freq, NTP_SCALE_SHIFT);
- pps_freq = ftemp;
+ delta = shift_right(ftemp - ntpdata->pps_freq, NTP_SCALE_SHIFT);
+ ntpdata->pps_freq = ftemp;
if (delta > PPS_MAXWANDER || delta < -PPS_MAXWANDER) {
- printk_deferred(KERN_WARNING
- "hardpps: PPSWANDER: change=%ld\n", delta);
- time_status |= STA_PPSWANDER;
- pps_stbcnt++;
- pps_dec_freq_interval();
- } else { /* good sample */
- pps_inc_freq_interval();
+ printk_deferred(KERN_WARNING "hardpps: PPSWANDER: change=%ld\n", delta);
+ ntpdata->time_status |= STA_PPSWANDER;
+ ntpdata->pps_stbcnt++;
+ pps_dec_freq_interval(ntpdata);
+ } else {
+ /* Good sample */
+ pps_inc_freq_interval(ntpdata);
}
- /* the stability metric is calculated as the average of recent
- * frequency changes, but is used only for performance
- * monitoring
+ /*
+ * The stability metric is calculated as the average of recent
+ * frequency changes, but is used only for performance monitoring
*/
delta_mod = delta;
if (delta_mod < 0)
delta_mod = -delta_mod;
- pps_stabil += (div_s64(((s64)delta_mod) <<
- (NTP_SCALE_SHIFT - SHIFT_USEC),
- NSEC_PER_USEC) - pps_stabil) >> PPS_INTMIN;
-
- /* if enabled, the system clock frequency is updated */
- if ((time_status & STA_PPSFREQ) != 0 &&
- (time_status & STA_FREQHOLD) == 0) {
- time_freq = pps_freq;
- ntp_update_frequency();
+ ntpdata->pps_stabil += (div_s64(((s64)delta_mod) << (NTP_SCALE_SHIFT - SHIFT_USEC),
+ NSEC_PER_USEC) - ntpdata->pps_stabil) >> PPS_INTMIN;
+
+ /* If enabled, the system clock frequency is updated */
+ if ((ntpdata->time_status & STA_PPSFREQ) && !(ntpdata->time_status & STA_FREQHOLD)) {
+ ntpdata->time_freq = ntpdata->pps_freq;
+ ntp_update_frequency(ntpdata);
}
return delta;
}
-/* correct REALTIME clock phase error against PPS signal */
-static void hardpps_update_phase(long error)
+/* Correct REALTIME clock phase error against PPS signal */
+static void hardpps_update_phase(struct ntp_data *ntpdata, long error)
{
long correction = -error;
long jitter;
- /* add the sample to the median filter */
- pps_phase_filter_add(correction);
- correction = pps_phase_filter_get(&jitter);
+ /* Add the sample to the median filter */
+ pps_phase_filter_add(ntpdata, correction);
+ correction = pps_phase_filter_get(ntpdata, &jitter);
- /* Nominal jitter is due to PPS signal noise. If it exceeds the
+ /*
+ * Nominal jitter is due to PPS signal noise. If it exceeds the
* threshold, the sample is discarded; otherwise, if so enabled,
* the time offset is updated.
*/
- if (jitter > (pps_jitter << PPS_POPCORN)) {
- printk_deferred(KERN_WARNING
- "hardpps: PPSJITTER: jitter=%ld, limit=%ld\n",
- jitter, (pps_jitter << PPS_POPCORN));
- time_status |= STA_PPSJITTER;
- pps_jitcnt++;
- } else if (time_status & STA_PPSTIME) {
- /* correct the time using the phase offset */
- time_offset = div_s64(((s64)correction) << NTP_SCALE_SHIFT,
- NTP_INTERVAL_FREQ);
- /* cancel running adjtime() */
- time_adjust = 0;
+ if (jitter > (ntpdata->pps_jitter << PPS_POPCORN)) {
+ printk_deferred(KERN_WARNING "hardpps: PPSJITTER: jitter=%ld, limit=%ld\n",
+ jitter, (ntpdata->pps_jitter << PPS_POPCORN));
+ ntpdata->time_status |= STA_PPSJITTER;
+ ntpdata->pps_jitcnt++;
+ } else if (ntpdata->time_status & STA_PPSTIME) {
+ /* Correct the time using the phase offset */
+ ntpdata->time_offset = div_s64(((s64)correction) << NTP_SCALE_SHIFT,
+ NTP_INTERVAL_FREQ);
+ /* Cancel running adjtime() */
+ ntpdata->time_adjust = 0;
}
- /* update jitter */
- pps_jitter += (jitter - pps_jitter) >> PPS_INTMIN;
+ /* Update jitter */
+ ntpdata->pps_jitter += (jitter - ntpdata->pps_jitter) >> PPS_INTMIN;
}
/*
@@ -1037,60 +1033,62 @@ static void hardpps_update_phase(long error)
void __hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_ts)
{
struct pps_normtime pts_norm, freq_norm;
+ struct ntp_data *ntpdata = &tk_ntp_data;
pts_norm = pps_normalize_ts(*phase_ts);
- /* clear the error bits, they will be set again if needed */
- time_status &= ~(STA_PPSJITTER | STA_PPSWANDER | STA_PPSERROR);
+ /* Clear the error bits, they will be set again if needed */
+ ntpdata->time_status &= ~(STA_PPSJITTER | STA_PPSWANDER | STA_PPSERROR);
/* indicate signal presence */
- time_status |= STA_PPSSIGNAL;
- pps_valid = PPS_VALID;
+ ntpdata->time_status |= STA_PPSSIGNAL;
+ ntpdata->pps_valid = PPS_VALID;
- /* when called for the first time,
- * just start the frequency interval */
- if (unlikely(pps_fbase.tv_sec == 0)) {
- pps_fbase = *raw_ts;
+ /*
+ * When called for the first time, just start the frequency
+ * interval
+ */
+ if (unlikely(ntpdata->pps_fbase.tv_sec == 0)) {
+ ntpdata->pps_fbase = *raw_ts;
return;
}
- /* ok, now we have a base for frequency calculation */
- freq_norm = pps_normalize_ts(timespec64_sub(*raw_ts, pps_fbase));
-
- /* check that the signal is in the range
- * [1s - MAXFREQ us, 1s + MAXFREQ us], otherwise reject it */
- if ((freq_norm.sec == 0) ||
- (freq_norm.nsec > MAXFREQ * freq_norm.sec) ||
- (freq_norm.nsec < -MAXFREQ * freq_norm.sec)) {
- time_status |= STA_PPSJITTER;
- /* restart the frequency calibration interval */
- pps_fbase = *raw_ts;
+ /* Ok, now we have a base for frequency calculation */
+ freq_norm = pps_normalize_ts(timespec64_sub(*raw_ts, ntpdata->pps_fbase));
+
+ /*
+ * Check that the signal is in the range
+ * [1s - MAXFREQ us, 1s + MAXFREQ us], otherwise reject it
+ */
+ if ((freq_norm.sec == 0) || (freq_norm.nsec > MAXFREQ * freq_norm.sec) ||
+ (freq_norm.nsec < -MAXFREQ * freq_norm.sec)) {
+ ntpdata->time_status |= STA_PPSJITTER;
+ /* Restart the frequency calibration interval */
+ ntpdata->pps_fbase = *raw_ts;
printk_deferred(KERN_ERR "hardpps: PPSJITTER: bad pulse\n");
return;
}
- /* signal is ok */
-
- /* check if the current frequency interval is finished */
- if (freq_norm.sec >= (1 << pps_shift)) {
- pps_calcnt++;
- /* restart the frequency calibration interval */
- pps_fbase = *raw_ts;
- hardpps_update_freq(freq_norm);
+ /* Signal is ok. Check if the current frequency interval is finished */
+ if (freq_norm.sec >= (1 << ntpdata->pps_shift)) {
+ ntpdata->pps_calcnt++;
+ /* Restart the frequency calibration interval */
+ ntpdata->pps_fbase = *raw_ts;
+ hardpps_update_freq(ntpdata, freq_norm);
}
- hardpps_update_phase(pts_norm.nsec);
+ hardpps_update_phase(ntpdata, pts_norm.nsec);
}
#endif /* CONFIG_NTP_PPS */
static int __init ntp_tick_adj_setup(char *str)
{
- int rc = kstrtos64(str, 0, &ntp_tick_adj);
+ int rc = kstrtos64(str, 0, &tk_ntp_data.ntp_tick_adj);
if (rc)
return rc;
- ntp_tick_adj <<= NTP_SCALE_SHIFT;
+ tk_ntp_data.ntp_tick_adj <<= NTP_SCALE_SHIFT;
return 1;
}
diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c
index 6bcee4704059..50e8d04ab661 100644
--- a/kernel/time/posix-cpu-timers.c
+++ b/kernel/time/posix-cpu-timers.c
@@ -453,7 +453,6 @@ static void disarm_timer(struct k_itimer *timer, struct task_struct *p)
struct cpu_timer *ctmr = &timer->it.cpu;
struct posix_cputimer_base *base;
- timer->it_active = 0;
if (!cpu_timer_dequeue(ctmr))
return;
@@ -494,19 +493,28 @@ static int posix_cpu_timer_del(struct k_itimer *timer)
*/
WARN_ON_ONCE(ctmr->head || timerqueue_node_queued(&ctmr->node));
} else {
- if (timer->it.cpu.firing)
+ if (timer->it.cpu.firing) {
+ /*
+ * Prevent signal delivery. The timer cannot be dequeued
+ * because it is on the firing list which is not protected
+ * by sighand->lock. The delivery path is waiting for
+ * the timer lock. So go back, unlock and retry.
+ */
+ timer->it.cpu.firing = false;
ret = TIMER_RETRY;
- else
+ } else {
disarm_timer(timer, p);
-
+ }
unlock_task_sighand(p, &flags);
}
out:
rcu_read_unlock();
- if (!ret)
- put_pid(ctmr->pid);
+ if (!ret) {
+ put_pid(ctmr->pid);
+ timer->it_status = POSIX_TIMER_DISARMED;
+ }
return ret;
}
@@ -560,7 +568,7 @@ static void arm_timer(struct k_itimer *timer, struct task_struct *p)
struct cpu_timer *ctmr = &timer->it.cpu;
u64 newexp = cpu_timer_getexpires(ctmr);
- timer->it_active = 1;
+ timer->it_status = POSIX_TIMER_ARMED;
if (!cpu_timer_enqueue(&base->tqhead, ctmr))
return;
@@ -586,29 +594,20 @@ static void cpu_timer_fire(struct k_itimer *timer)
{
struct cpu_timer *ctmr = &timer->it.cpu;
- timer->it_active = 0;
- if (unlikely(timer->sigq == NULL)) {
+ timer->it_status = POSIX_TIMER_DISARMED;
+
+ if (unlikely(ctmr->nanosleep)) {
/*
* This a special case for clock_nanosleep,
* not a normal timer from sys_timer_create.
*/
wake_up_process(timer->it_process);
cpu_timer_setexpires(ctmr, 0);
- } else if (!timer->it_interval) {
- /*
- * One-shot timer. Clear it as soon as it's fired.
- */
+ } else {
posix_timer_queue_signal(timer);
- cpu_timer_setexpires(ctmr, 0);
- } else if (posix_timer_queue_signal(timer)) {
- /*
- * The signal did not get queued because the signal
- * was ignored, so we won't get any callback to
- * reload the timer. But we need to keep it
- * ticking in case the signal is deliverable next time.
- */
- posix_cpu_timer_rearm(timer);
- ++timer->it_requeue_pending;
+ /* Disable oneshot timers */
+ if (!timer->it_interval)
+ cpu_timer_setexpires(ctmr, 0);
}
}
@@ -667,11 +666,17 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags,
old_expires = cpu_timer_getexpires(ctmr);
if (unlikely(timer->it.cpu.firing)) {
- timer->it.cpu.firing = -1;
+ /*
+ * Prevent signal delivery. The timer cannot be dequeued
+ * because it is on the firing list which is not protected
+ * by sighand->lock. The delivery path is waiting for
+ * the timer lock. So go back, unlock and retry.
+ */
+ timer->it.cpu.firing = false;
ret = TIMER_RETRY;
} else {
cpu_timer_dequeue(ctmr);
- timer->it_active = 0;
+ timer->it_status = POSIX_TIMER_DISARMED;
}
/*
@@ -745,7 +750,7 @@ static void __posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *i
* - Timers which expired, but the signal has not yet been
* delivered
*/
- if (iv && ((timer->it_requeue_pending & REQUEUE_PENDING) || sigev_none))
+ if (iv && timer->it_status != POSIX_TIMER_ARMED)
expires = bump_cpu_timer(timer, now);
else
expires = cpu_timer_getexpires(&timer->it.cpu);
@@ -808,7 +813,7 @@ static u64 collect_timerqueue(struct timerqueue_head *head,
if (++i == MAX_COLLECTED || now < expires)
return expires;
- ctmr->firing = 1;
+ ctmr->firing = true;
/* See posix_cpu_timer_wait_running() */
rcu_assign_pointer(ctmr->handling, current);
cpu_timer_dequeue(ctmr);
@@ -1363,7 +1368,7 @@ static void handle_posix_cpu_timers(struct task_struct *tsk)
* timer call will interfere.
*/
list_for_each_entry_safe(timer, next, &firing, it.cpu.elist) {
- int cpu_firing;
+ bool cpu_firing;
/*
* spin_lock() is sufficient here even independent of the
@@ -1375,13 +1380,13 @@ static void handle_posix_cpu_timers(struct task_struct *tsk)
spin_lock(&timer->it_lock);
list_del_init(&timer->it.cpu.elist);
cpu_firing = timer->it.cpu.firing;
- timer->it.cpu.firing = 0;
+ timer->it.cpu.firing = false;
/*
- * The firing flag is -1 if we collided with a reset
- * of the timer, which already reported this
- * almost-firing as an overrun. So don't generate an event.
+ * If the firing flag is cleared then this raced with a
+ * timer rearm/delete operation. So don't generate an
+ * event.
*/
- if (likely(cpu_firing >= 0))
+ if (likely(cpu_firing))
cpu_timer_fire(timer);
/* See posix_cpu_timer_wait_running() */
rcu_assign_pointer(timer->it.cpu.handling, NULL);
@@ -1478,6 +1483,7 @@ static int do_cpu_nanosleep(const clockid_t which_clock, int flags,
timer.it_overrun = -1;
error = posix_cpu_timer_create(&timer);
timer.it_process = current;
+ timer.it.cpu.nanosleep = true;
if (!error) {
static struct itimerspec64 zero_it;
diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c
index 4576aaed13b2..881a9ce96af7 100644
--- a/kernel/time/posix-timers.c
+++ b/kernel/time/posix-timers.c
@@ -233,11 +233,12 @@ __initcall(init_posix_timers);
* The siginfo si_overrun field and the return value of timer_getoverrun(2)
* are of type int. Clamp the overrun value to INT_MAX
*/
-static inline int timer_overrun_to_int(struct k_itimer *timr, int baseval)
+static inline int timer_overrun_to_int(struct k_itimer *timr)
{
- s64 sum = timr->it_overrun_last + (s64)baseval;
+ if (timr->it_overrun_last > (s64)INT_MAX)
+ return INT_MAX;
- return sum > (s64)INT_MAX ? INT_MAX : (int)sum;
+ return (int)timr->it_overrun_last;
}
static void common_hrtimer_rearm(struct k_itimer *timr)
@@ -249,62 +250,62 @@ static void common_hrtimer_rearm(struct k_itimer *timr)
hrtimer_restart(timer);
}
+static bool __posixtimer_deliver_signal(struct kernel_siginfo *info, struct k_itimer *timr)
+{
+ guard(spinlock)(&timr->it_lock);
+
+ /*
+ * Check if the timer is still alive or whether it got modified
+ * since the signal was queued. In either case, don't rearm and
+ * drop the signal.
+ */
+ if (timr->it_signal_seq != timr->it_sigqueue_seq || WARN_ON_ONCE(!timr->it_signal))
+ return false;
+
+ if (!timr->it_interval || WARN_ON_ONCE(timr->it_status != POSIX_TIMER_REQUEUE_PENDING))
+ return true;
+
+ timr->kclock->timer_rearm(timr);
+ timr->it_status = POSIX_TIMER_ARMED;
+ timr->it_overrun_last = timr->it_overrun;
+ timr->it_overrun = -1LL;
+ ++timr->it_signal_seq;
+ info->si_overrun = timer_overrun_to_int(timr);
+ return true;
+}
+
/*
- * This function is called from the signal delivery code if
- * info->si_sys_private is not zero, which indicates that the timer has to
- * be rearmed. Restart the timer and update info::si_overrun.
+ * This function is called from the signal delivery code. It decides
+ * whether the signal should be dropped and rearms interval timers. The
+ * timer can be unconditionally accessed as there is a reference held on
+ * it.
*/
-void posixtimer_rearm(struct kernel_siginfo *info)
+bool posixtimer_deliver_signal(struct kernel_siginfo *info, struct sigqueue *timer_sigq)
{
- struct k_itimer *timr;
- unsigned long flags;
-
- timr = lock_timer(info->si_tid, &flags);
- if (!timr)
- return;
+ struct k_itimer *timr = container_of(timer_sigq, struct k_itimer, sigq);
+ bool ret;
- if (timr->it_interval && timr->it_requeue_pending == info->si_sys_private) {
- timr->kclock->timer_rearm(timr);
+ /*
+ * Release siglock to ensure proper locking order versus
+ * timr::it_lock. Keep interrupts disabled.
+ */
+ spin_unlock(¤t->sighand->siglock);
- timr->it_active = 1;
- timr->it_overrun_last = timr->it_overrun;
- timr->it_overrun = -1LL;
- ++timr->it_requeue_pending;
+ ret = __posixtimer_deliver_signal(info, timr);
- info->si_overrun = timer_overrun_to_int(timr, info->si_overrun);
- }
+ /* Drop the reference which was acquired when the signal was queued */
+ posixtimer_putref(timr);
- unlock_timer(timr, flags);
+ spin_lock(¤t->sighand->siglock);
+ return ret;
}
-int posix_timer_queue_signal(struct k_itimer *timr)
+void posix_timer_queue_signal(struct k_itimer *timr)
{
- int ret, si_private = 0;
- enum pid_type type;
-
lockdep_assert_held(&timr->it_lock);
- timr->it_active = 0;
- if (timr->it_interval)
- si_private = ++timr->it_requeue_pending;
-
- /*
- * FIXME: if ->sigq is queued we can race with
- * dequeue_signal()->posixtimer_rearm().
- *
- * If dequeue_signal() sees the "right" value of
- * si_sys_private it calls posixtimer_rearm().
- * We re-queue ->sigq and drop ->it_lock().
- * posixtimer_rearm() locks the timer
- * and re-schedules it while ->sigq is pending.
- * Not really bad, but not that we want.
- */
- timr->sigq->info.si_sys_private = si_private;
-
- type = !(timr->it_sigev_notify & SIGEV_THREAD_ID) ? PIDTYPE_TGID : PIDTYPE_PID;
- ret = send_sigqueue(timr->sigq, timr->it_pid, type);
- /* If we failed to send the signal the timer stops. */
- return ret > 0;
+ timr->it_status = timr->it_interval ? POSIX_TIMER_REQUEUE_PENDING : POSIX_TIMER_DISARMED;
+ posixtimer_send_sigqueue(timr);
}
/*
@@ -317,62 +318,10 @@ int posix_timer_queue_signal(struct k_itimer *timr)
static enum hrtimer_restart posix_timer_fn(struct hrtimer *timer)
{
struct k_itimer *timr = container_of(timer, struct k_itimer, it.real.timer);
- enum hrtimer_restart ret = HRTIMER_NORESTART;
- unsigned long flags;
-
- spin_lock_irqsave(&timr->it_lock, flags);
-
- if (posix_timer_queue_signal(timr)) {
- /*
- * The signal was not queued due to SIG_IGN. As a
- * consequence the timer is not going to be rearmed from
- * the signal delivery path. But as a real signal handler
- * can be installed later the timer must be rearmed here.
- */
- if (timr->it_interval != 0) {
- ktime_t now = hrtimer_cb_get_time(timer);
-
- /*
- * FIXME: What we really want, is to stop this
- * timer completely and restart it in case the
- * SIG_IGN is removed. This is a non trivial
- * change to the signal handling code.
- *
- * For now let timers with an interval less than a
- * jiffy expire every jiffy and recheck for a
- * valid signal handler.
- *
- * This avoids interrupt starvation in case of a
- * very small interval, which would expire the
- * timer immediately again.
- *
- * Moving now ahead of time by one jiffy tricks
- * hrtimer_forward() to expire the timer later,
- * while it still maintains the overrun accuracy
- * for the price of a slight inconsistency in the
- * timer_gettime() case. This is at least better
- * than a timer storm.
- *
- * Only required when high resolution timers are
- * enabled as the periodic tick based timers are
- * automatically aligned to the next tick.
- */
- if (IS_ENABLED(CONFIG_HIGH_RES_TIMERS)) {
- ktime_t kj = TICK_NSEC;
-
- if (timr->it_interval < kj)
- now = ktime_add(now, kj);
- }
-
- timr->it_overrun += hrtimer_forward(timer, now, timr->it_interval);
- ret = HRTIMER_RESTART;
- ++timr->it_requeue_pending;
- timr->it_active = 1;
- }
- }
- unlock_timer(timr, flags);
- return ret;
+ guard(spinlock_irqsave)(&timr->it_lock);
+ posix_timer_queue_signal(timr);
+ return HRTIMER_NORESTART;
}
static struct pid *good_sigevent(sigevent_t * event)
@@ -399,32 +348,27 @@ static struct pid *good_sigevent(sigevent_t * event)
}
}
-static struct k_itimer * alloc_posix_timer(void)
+static struct k_itimer *alloc_posix_timer(void)
{
struct k_itimer *tmr = kmem_cache_zalloc(posix_timers_cache, GFP_KERNEL);
if (!tmr)
return tmr;
- if (unlikely(!(tmr->sigq = sigqueue_alloc()))) {
+
+ if (unlikely(!posixtimer_init_sigqueue(&tmr->sigq))) {
kmem_cache_free(posix_timers_cache, tmr);
return NULL;
}
- clear_siginfo(&tmr->sigq->info);
+ rcuref_init(&tmr->rcuref, 1);
return tmr;
}
-static void k_itimer_rcu_free(struct rcu_head *head)
-{
- struct k_itimer *tmr = container_of(head, struct k_itimer, rcu);
-
- kmem_cache_free(posix_timers_cache, tmr);
-}
-
-static void posix_timer_free(struct k_itimer *tmr)
+void posixtimer_free_timer(struct k_itimer *tmr)
{
put_pid(tmr->it_pid);
- sigqueue_free(tmr->sigq);
- call_rcu(&tmr->rcu, k_itimer_rcu_free);
+ if (tmr->sigq.ucounts)
+ dec_rlimit_put_ucounts(tmr->sigq.ucounts, UCOUNT_RLIMIT_SIGPENDING);
+ kfree_rcu(tmr, rcu);
}
static void posix_timer_unhash_and_free(struct k_itimer *tmr)
@@ -432,7 +376,7 @@ static void posix_timer_unhash_and_free(struct k_itimer *tmr)
spin_lock(&hash_lock);
hlist_del_rcu(&tmr->t_hash);
spin_unlock(&hash_lock);
- posix_timer_free(tmr);
+ posixtimer_putref(tmr);
}
static int common_timer_create(struct k_itimer *new_timer)
@@ -467,7 +411,7 @@ static int do_timer_create(clockid_t which_clock, struct sigevent *event,
*/
new_timer_id = posix_timer_add(new_timer);
if (new_timer_id < 0) {
- posix_timer_free(new_timer);
+ posixtimer_free_timer(new_timer);
return new_timer_id;
}
@@ -485,18 +429,23 @@ static int do_timer_create(clockid_t which_clock, struct sigevent *event,
goto out;
}
new_timer->it_sigev_notify = event->sigev_notify;
- new_timer->sigq->info.si_signo = event->sigev_signo;
- new_timer->sigq->info.si_value = event->sigev_value;
+ new_timer->sigq.info.si_signo = event->sigev_signo;
+ new_timer->sigq.info.si_value = event->sigev_value;
} else {
new_timer->it_sigev_notify = SIGEV_SIGNAL;
- new_timer->sigq->info.si_signo = SIGALRM;
- memset(&new_timer->sigq->info.si_value, 0, sizeof(sigval_t));
- new_timer->sigq->info.si_value.sival_int = new_timer->it_id;
+ new_timer->sigq.info.si_signo = SIGALRM;
+ memset(&new_timer->sigq.info.si_value, 0, sizeof(sigval_t));
+ new_timer->sigq.info.si_value.sival_int = new_timer->it_id;
new_timer->it_pid = get_pid(task_tgid(current));
}
- new_timer->sigq->info.si_tid = new_timer->it_id;
- new_timer->sigq->info.si_code = SI_TIMER;
+ if (new_timer->it_sigev_notify & SIGEV_THREAD_ID)
+ new_timer->it_pid_type = PIDTYPE_PID;
+ else
+ new_timer->it_pid_type = PIDTYPE_TGID;
+
+ new_timer->sigq.info.si_tid = new_timer->it_id;
+ new_timer->sigq.info.si_code = SI_TIMER;
if (copy_to_user(created_timer_id, &new_timer_id, sizeof (new_timer_id))) {
error = -EFAULT;
@@ -580,7 +529,14 @@ static struct k_itimer *__lock_timer(timer_t timer_id, unsigned long *flags)
* 1) Set timr::it_signal to NULL with timr::it_lock held
* 2) Release timr::it_lock
* 3) Remove from the hash under hash_lock
- * 4) Call RCU for removal after the grace period
+ * 4) Put the reference count.
+ *
+ * The reference count might not drop to zero if timr::sigq is
+ * queued. In that case the signal delivery or flush will put the
+ * last reference count.
+ *
+ * When the reference count reaches zero, the timer is scheduled
+ * for RCU removal after the grace period.
*
* Holding rcu_read_lock() accross the lookup ensures that
* the timer cannot be freed.
@@ -647,10 +603,10 @@ void common_timer_get(struct k_itimer *timr, struct itimerspec64 *cur_setting)
/* interval timer ? */
if (iv) {
cur_setting->it_interval = ktime_to_timespec64(iv);
- } else if (!timr->it_active) {
+ } else if (timr->it_status == POSIX_TIMER_DISARMED) {
/*
* SIGEV_NONE oneshot timers are never queued and therefore
- * timr->it_active is always false. The check below
+ * timr->it_status is always DISARMED. The check below
* vs. remaining time will handle this case.
*
* For all other timers there is nothing to update here, so
@@ -667,7 +623,7 @@ void common_timer_get(struct k_itimer *timr, struct itimerspec64 *cur_setting)
* is a SIGEV_NONE timer move the expiry time forward by intervals,
* so expiry is > now.
*/
- if (iv && (timr->it_requeue_pending & REQUEUE_PENDING || sig_none))
+ if (iv && timr->it_status != POSIX_TIMER_ARMED)
timr->it_overrun += kc->timer_forward(timr, now);
remaining = kc->timer_remaining(timr, now);
@@ -775,7 +731,7 @@ SYSCALL_DEFINE1(timer_getoverrun, timer_t, timer_id)
if (!timr)
return -EINVAL;
- overrun = timer_overrun_to_int(timr, 0);
+ overrun = timer_overrun_to_int(timr);
unlock_timer(timr, flags);
return overrun;
@@ -867,8 +823,6 @@ void posix_timer_set_common(struct k_itimer *timer, struct itimerspec64 *new_set
else
timer->it_interval = 0;
- /* Prevent reloading in case there is a signal pending */
- timer->it_requeue_pending = (timer->it_requeue_pending + 2) & ~REQUEUE_PENDING;
/* Reset overrun accounting */
timer->it_overrun_last = 0;
timer->it_overrun = -1LL;
@@ -886,8 +840,6 @@ int common_timer_set(struct k_itimer *timr, int flags,
if (old_setting)
common_timer_get(timr, old_setting);
- /* Prevent rearming by clearing the interval */
- timr->it_interval = 0;
/*
* Careful here. On SMP systems the timer expiry function could be
* active and spinning on timr->it_lock.
@@ -895,7 +847,7 @@ int common_timer_set(struct k_itimer *timr, int flags,
if (kc->timer_try_to_cancel(timr) < 0)
return TIMER_RETRY;
- timr->it_active = 0;
+ timr->it_status = POSIX_TIMER_DISARMED;
posix_timer_set_common(timr, new_setting);
/* Keep timer disarmed when it_value is zero */
@@ -908,7 +860,8 @@ int common_timer_set(struct k_itimer *timr, int flags,
sigev_none = timr->it_sigev_notify == SIGEV_NONE;
kc->timer_arm(timr, expires, flags & TIMER_ABSTIME, sigev_none);
- timr->it_active = !sigev_none;
+ if (!sigev_none)
+ timr->it_status = POSIX_TIMER_ARMED;
return 0;
}
@@ -936,6 +889,9 @@ static int do_timer_settime(timer_t timer_id, int tmr_flags,
if (old_spec64)
old_spec64->it_interval = ktime_to_timespec64(timr->it_interval);
+ /* Prevent signal delivery and rearming. */
+ timr->it_signal_seq++;
+
kc = timr->kclock;
if (WARN_ON_ONCE(!kc || !kc->timer_set))
error = -EINVAL;
@@ -1004,17 +960,31 @@ int common_timer_del(struct k_itimer *timer)
{
const struct k_clock *kc = timer->kclock;
- timer->it_interval = 0;
if (kc->timer_try_to_cancel(timer) < 0)
return TIMER_RETRY;
- timer->it_active = 0;
+ timer->it_status = POSIX_TIMER_DISARMED;
return 0;
}
+/*
+ * If the deleted timer is on the ignored list, remove it and
+ * drop the associated reference.
+ */
+static inline void posix_timer_cleanup_ignored(struct k_itimer *tmr)
+{
+ if (!hlist_unhashed(&tmr->ignored_list)) {
+ hlist_del_init(&tmr->ignored_list);
+ posixtimer_putref(tmr);
+ }
+}
+
static inline int timer_delete_hook(struct k_itimer *timer)
{
const struct k_clock *kc = timer->kclock;
+ /* Prevent signal delivery and rearming. */
+ timer->it_signal_seq++;
+
if (WARN_ON_ONCE(!kc || !kc->timer_del))
return -EINVAL;
return kc->timer_del(timer);
@@ -1040,12 +1010,18 @@ SYSCALL_DEFINE1(timer_delete, timer_t, timer_id)
spin_lock(¤t->sighand->siglock);
hlist_del(&timer->list);
- spin_unlock(¤t->sighand->siglock);
+ posix_timer_cleanup_ignored(timer);
/*
* A concurrent lookup could check timer::it_signal lockless. It
* will reevaluate with timer::it_lock held and observe the NULL.
+ *
+ * It must be written with siglock held so that the signal code
+ * observes timer->it_signal == NULL in do_sigaction(SIG_IGN),
+ * which prevents it from moving a pending signal of a deleted
+ * timer to the ignore list.
*/
WRITE_ONCE(timer->it_signal, NULL);
+ spin_unlock(¤t->sighand->siglock);
unlock_timer(timer, flags);
posix_timer_unhash_and_free(timer);
@@ -1091,6 +1067,8 @@ static void itimer_delete(struct k_itimer *timer)
}
hlist_del(&timer->list);
+ posix_timer_cleanup_ignored(timer);
+
/*
* Setting timer::it_signal to NULL is technically not required
* here as nothing can access the timer anymore legitimately via
@@ -1123,6 +1101,19 @@ void exit_itimers(struct task_struct *tsk)
/* The timers are not longer accessible via tsk::signal */
while (!hlist_empty(&timers))
itimer_delete(hlist_entry(timers.first, struct k_itimer, list));
+
+ /*
+ * There should be no timers on the ignored list. itimer_delete() has
+ * mopped them up.
+ */
+ if (!WARN_ON_ONCE(!hlist_empty(&tsk->signal->ignored_posix_timers)))
+ return;
+
+ hlist_move_list(&tsk->signal->ignored_posix_timers, &timers);
+ while (!hlist_empty(&timers)) {
+ posix_timer_cleanup_ignored(hlist_entry(timers.first, struct k_itimer,
+ ignored_list));
+ }
}
SYSCALL_DEFINE2(clock_settime, const clockid_t, which_clock,
diff --git a/kernel/time/posix-timers.h b/kernel/time/posix-timers.h
index 4784ea65f685..61906f0688c1 100644
--- a/kernel/time/posix-timers.h
+++ b/kernel/time/posix-timers.h
@@ -1,6 +1,12 @@
/* SPDX-License-Identifier: GPL-2.0 */
#define TIMER_RETRY 1
+enum posix_timer_state {
+ POSIX_TIMER_DISARMED,
+ POSIX_TIMER_ARMED,
+ POSIX_TIMER_REQUEUE_PENDING,
+};
+
struct k_clock {
int (*clock_getres)(const clockid_t which_clock,
struct timespec64 *tp);
@@ -36,7 +42,7 @@ extern const struct k_clock clock_process;
extern const struct k_clock clock_thread;
extern const struct k_clock alarm_clock;
-int posix_timer_queue_signal(struct k_itimer *timr);
+void posix_timer_queue_signal(struct k_itimer *timr);
void common_timer_get(struct k_itimer *timr, struct itimerspec64 *cur_setting);
int common_timer_set(struct k_itimer *timr, int flags,
diff --git a/kernel/time/sleep_timeout.c b/kernel/time/sleep_timeout.c
new file mode 100644
index 000000000000..dfe939f6e4ec
--- /dev/null
+++ b/kernel/time/sleep_timeout.c
@@ -0,0 +1,377 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Kernel internal schedule timeout and sleeping functions
+ */
+
+#include <linux/delay.h>
+#include <linux/jiffies.h>
+#include <linux/timer.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
+
+#include "tick-internal.h"
+
+/*
+ * Since schedule_timeout()'s timer is defined on the stack, it must store
+ * the target task on the stack as well.
+ */
+struct process_timer {
+ struct timer_list timer;
+ struct task_struct *task;
+};
+
+static void process_timeout(struct timer_list *t)
+{
+ struct process_timer *timeout = from_timer(timeout, t, timer);
+
+ wake_up_process(timeout->task);
+}
+
+/**
+ * schedule_timeout - sleep until timeout
+ * @timeout: timeout value in jiffies
+ *
+ * Make the current task sleep until @timeout jiffies have elapsed.
+ * The function behavior depends on the current task state
+ * (see also set_current_state() description):
+ *
+ * %TASK_RUNNING - the scheduler is called, but the task does not sleep
+ * at all. That happens because sched_submit_work() does nothing for
+ * tasks in %TASK_RUNNING state.
+ *
+ * %TASK_UNINTERRUPTIBLE - at least @timeout jiffies are guaranteed to
+ * pass before the routine returns unless the current task is explicitly
+ * woken up, (e.g. by wake_up_process()).
+ *
+ * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
+ * delivered to the current task or the current task is explicitly woken
+ * up.
+ *
+ * The current task state is guaranteed to be %TASK_RUNNING when this
+ * routine returns.
+ *
+ * Specifying a @timeout value of %MAX_SCHEDULE_TIMEOUT will schedule
+ * the CPU away without a bound on the timeout. In this case the return
+ * value will be %MAX_SCHEDULE_TIMEOUT.
+ *
+ * Returns: 0 when the timer has expired otherwise the remaining time in
+ * jiffies will be returned. In all cases the return value is guaranteed
+ * to be non-negative.
+ */
+signed long __sched schedule_timeout(signed long timeout)
+{
+ struct process_timer timer;
+ unsigned long expire;
+
+ switch (timeout) {
+ case MAX_SCHEDULE_TIMEOUT:
+ /*
+ * These two special cases are useful to be comfortable
+ * in the caller. Nothing more. We could take
+ * MAX_SCHEDULE_TIMEOUT from one of the negative value
+ * but I' d like to return a valid offset (>=0) to allow
+ * the caller to do everything it want with the retval.
+ */
+ schedule();
+ goto out;
+ default:
+ /*
+ * Another bit of PARANOID. Note that the retval will be
+ * 0 since no piece of kernel is supposed to do a check
+ * for a negative retval of schedule_timeout() (since it
+ * should never happens anyway). You just have the printk()
+ * that will tell you if something is gone wrong and where.
+ */
+ if (timeout < 0) {
+ pr_err("%s: wrong timeout value %lx\n", __func__, timeout);
+ dump_stack();
+ __set_current_state(TASK_RUNNING);
+ goto out;
+ }
+ }
+
+ expire = timeout + jiffies;
+
+ timer.task = current;
+ timer_setup_on_stack(&timer.timer, process_timeout, 0);
+ timer.timer.expires = expire;
+ add_timer(&timer.timer);
+ schedule();
+ del_timer_sync(&timer.timer);
+
+ /* Remove the timer from the object tracker */
+ destroy_timer_on_stack(&timer.timer);
+
+ timeout = expire - jiffies;
+
+ out:
+ return timeout < 0 ? 0 : timeout;
+}
+EXPORT_SYMBOL(schedule_timeout);
+
+/*
+ * __set_current_state() can be used in schedule_timeout_*() functions, because
+ * schedule_timeout() calls schedule() unconditionally.
+ */
+
+/**
+ * schedule_timeout_interruptible - sleep until timeout (interruptible)
+ * @timeout: timeout value in jiffies
+ *
+ * See schedule_timeout() for details.
+ *
+ * Task state is set to TASK_INTERRUPTIBLE before starting the timeout.
+ */
+signed long __sched schedule_timeout_interruptible(signed long timeout)
+{
+ __set_current_state(TASK_INTERRUPTIBLE);
+ return schedule_timeout(timeout);
+}
+EXPORT_SYMBOL(schedule_timeout_interruptible);
+
+/**
+ * schedule_timeout_killable - sleep until timeout (killable)
+ * @timeout: timeout value in jiffies
+ *
+ * See schedule_timeout() for details.
+ *
+ * Task state is set to TASK_KILLABLE before starting the timeout.
+ */
+signed long __sched schedule_timeout_killable(signed long timeout)
+{
+ __set_current_state(TASK_KILLABLE);
+ return schedule_timeout(timeout);
+}
+EXPORT_SYMBOL(schedule_timeout_killable);
+
+/**
+ * schedule_timeout_uninterruptible - sleep until timeout (uninterruptible)
+ * @timeout: timeout value in jiffies
+ *
+ * See schedule_timeout() for details.
+ *
+ * Task state is set to TASK_UNINTERRUPTIBLE before starting the timeout.
+ */
+signed long __sched schedule_timeout_uninterruptible(signed long timeout)
+{
+ __set_current_state(TASK_UNINTERRUPTIBLE);
+ return schedule_timeout(timeout);
+}
+EXPORT_SYMBOL(schedule_timeout_uninterruptible);
+
+/**
+ * schedule_timeout_idle - sleep until timeout (idle)
+ * @timeout: timeout value in jiffies
+ *
+ * See schedule_timeout() for details.
+ *
+ * Task state is set to TASK_IDLE before starting the timeout. It is similar to
+ * schedule_timeout_uninterruptible(), except this task will not contribute to
+ * load average.
+ */
+signed long __sched schedule_timeout_idle(signed long timeout)
+{
+ __set_current_state(TASK_IDLE);
+ return schedule_timeout(timeout);
+}
+EXPORT_SYMBOL(schedule_timeout_idle);
+
+/**
+ * schedule_hrtimeout_range_clock - sleep until timeout
+ * @expires: timeout value (ktime_t)
+ * @delta: slack in expires timeout (ktime_t)
+ * @mode: timer mode
+ * @clock_id: timer clock to be used
+ *
+ * Details are explained in schedule_hrtimeout_range() function description as
+ * this function is commonly used.
+ */
+int __sched schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
+ const enum hrtimer_mode mode, clockid_t clock_id)
+{
+ struct hrtimer_sleeper t;
+
+ /*
+ * Optimize when a zero timeout value is given. It does not
+ * matter whether this is an absolute or a relative time.
+ */
+ if (expires && *expires == 0) {
+ __set_current_state(TASK_RUNNING);
+ return 0;
+ }
+
+ /*
+ * A NULL parameter means "infinite"
+ */
+ if (!expires) {
+ schedule();
+ return -EINTR;
+ }
+
+ hrtimer_setup_sleeper_on_stack(&t, clock_id, mode);
+ hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
+ hrtimer_sleeper_start_expires(&t, mode);
+
+ if (likely(t.task))
+ schedule();
+
+ hrtimer_cancel(&t.timer);
+ destroy_hrtimer_on_stack(&t.timer);
+
+ __set_current_state(TASK_RUNNING);
+
+ return !t.task ? 0 : -EINTR;
+}
+EXPORT_SYMBOL_GPL(schedule_hrtimeout_range_clock);
+
+/**
+ * schedule_hrtimeout_range - sleep until timeout
+ * @expires: timeout value (ktime_t)
+ * @delta: slack in expires timeout (ktime_t)
+ * @mode: timer mode
+ *
+ * Make the current task sleep until the given expiry time has
+ * elapsed. The routine will return immediately unless
+ * the current task state has been set (see set_current_state()).
+ *
+ * The @delta argument gives the kernel the freedom to schedule the
+ * actual wakeup to a time that is both power and performance friendly
+ * for regular (non RT/DL) tasks.
+ * The kernel give the normal best effort behavior for "@expires+@delta",
+ * but may decide to fire the timer earlier, but no earlier than @expires.
+ *
+ * You can set the task state as follows -
+ *
+ * %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
+ * pass before the routine returns unless the current task is explicitly
+ * woken up, (e.g. by wake_up_process()).
+ *
+ * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
+ * delivered to the current task or the current task is explicitly woken
+ * up.
+ *
+ * The current task state is guaranteed to be TASK_RUNNING when this
+ * routine returns.
+ *
+ * Returns: 0 when the timer has expired. If the task was woken before the
+ * timer expired by a signal (only possible in state TASK_INTERRUPTIBLE) or
+ * by an explicit wakeup, it returns -EINTR.
+ */
+int __sched schedule_hrtimeout_range(ktime_t *expires, u64 delta,
+ const enum hrtimer_mode mode)
+{
+ return schedule_hrtimeout_range_clock(expires, delta, mode,
+ CLOCK_MONOTONIC);
+}
+EXPORT_SYMBOL_GPL(schedule_hrtimeout_range);
+
+/**
+ * schedule_hrtimeout - sleep until timeout
+ * @expires: timeout value (ktime_t)
+ * @mode: timer mode
+ *
+ * See schedule_hrtimeout_range() for details. @delta argument of
+ * schedule_hrtimeout_range() is set to 0 and has therefore no impact.
+ */
+int __sched schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode)
+{
+ return schedule_hrtimeout_range(expires, 0, mode);
+}
+EXPORT_SYMBOL_GPL(schedule_hrtimeout);
+
+/**
+ * msleep - sleep safely even with waitqueue interruptions
+ * @msecs: Requested sleep duration in milliseconds
+ *
+ * msleep() uses jiffy based timeouts for the sleep duration. Because of the
+ * design of the timer wheel, the maximum additional percentage delay (slack) is
+ * 12.5%. This is only valid for timers which will end up in level 1 or a higher
+ * level of the timer wheel. For explanation of those 12.5% please check the
+ * detailed description about the basics of the timer wheel.
+ *
+ * The slack of timers which will end up in level 0 depends on sleep duration
+ * (msecs) and HZ configuration and can be calculated in the following way (with
+ * the timer wheel design restriction that the slack is not less than 12.5%):
+ *
+ * ``slack = MSECS_PER_TICK / msecs``
+ *
+ * When the allowed slack of the callsite is known, the calculation could be
+ * turned around to find the minimal allowed sleep duration to meet the
+ * constraints. For example:
+ *
+ * * ``HZ=1000`` with ``slack=25%``: ``MSECS_PER_TICK / slack = 1 / (1/4) = 4``:
+ * all sleep durations greater or equal 4ms will meet the constraints.
+ * * ``HZ=1000`` with ``slack=12.5%``: ``MSECS_PER_TICK / slack = 1 / (1/8) = 8``:
+ * all sleep durations greater or equal 8ms will meet the constraints.
+ * * ``HZ=250`` with ``slack=25%``: ``MSECS_PER_TICK / slack = 4 / (1/4) = 16``:
+ * all sleep durations greater or equal 16ms will meet the constraints.
+ * * ``HZ=250`` with ``slack=12.5%``: ``MSECS_PER_TICK / slack = 4 / (1/8) = 32``:
+ * all sleep durations greater or equal 32ms will meet the constraints.
+ *
+ * See also the signal aware variant msleep_interruptible().
+ */
+void msleep(unsigned int msecs)
+{
+ unsigned long timeout = msecs_to_jiffies(msecs);
+
+ while (timeout)
+ timeout = schedule_timeout_uninterruptible(timeout);
+}
+EXPORT_SYMBOL(msleep);
+
+/**
+ * msleep_interruptible - sleep waiting for signals
+ * @msecs: Requested sleep duration in milliseconds
+ *
+ * See msleep() for some basic information.
+ *
+ * The difference between msleep() and msleep_interruptible() is that the sleep
+ * could be interrupted by a signal delivery and then returns early.
+ *
+ * Returns: The remaining time of the sleep duration transformed to msecs (see
+ * schedule_timeout() for details).
+ */
+unsigned long msleep_interruptible(unsigned int msecs)
+{
+ unsigned long timeout = msecs_to_jiffies(msecs);
+
+ while (timeout && !signal_pending(current))
+ timeout = schedule_timeout_interruptible(timeout);
+ return jiffies_to_msecs(timeout);
+}
+EXPORT_SYMBOL(msleep_interruptible);
+
+/**
+ * usleep_range_state - Sleep for an approximate time in a given state
+ * @min: Minimum time in usecs to sleep
+ * @max: Maximum time in usecs to sleep
+ * @state: State of the current task that will be while sleeping
+ *
+ * usleep_range_state() sleeps at least for the minimum specified time but not
+ * longer than the maximum specified amount of time. The range might reduce
+ * power usage by allowing hrtimers to coalesce an already scheduled interrupt
+ * with this hrtimer. In the worst case, an interrupt is scheduled for the upper
+ * bound.
+ *
+ * The sleeping task is set to the specified state before starting the sleep.
+ *
+ * In non-atomic context where the exact wakeup time is flexible, use
+ * usleep_range() or its variants instead of udelay(). The sleep improves
+ * responsiveness by avoiding the CPU-hogging busy-wait of udelay().
+ */
+void __sched usleep_range_state(unsigned long min, unsigned long max, unsigned int state)
+{
+ ktime_t exp = ktime_add_us(ktime_get(), min);
+ u64 delta = (u64)(max - min) * NSEC_PER_USEC;
+
+ if (WARN_ON_ONCE(max < min))
+ delta = 0;
+
+ for (;;) {
+ __set_current_state(state);
+ /* Do not return before the requested sleep time has elapsed */
+ if (!schedule_hrtimeout_range(&exp, delta, HRTIMER_MODE_ABS))
+ break;
+ }
+}
+EXPORT_SYMBOL(usleep_range_state);
diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h
index 5f2105e637bd..faac36de35b9 100644
--- a/kernel/time/tick-internal.h
+++ b/kernel/time/tick-internal.h
@@ -25,6 +25,7 @@ extern int tick_do_timer_cpu __read_mostly;
extern void tick_setup_periodic(struct clock_event_device *dev, int broadcast);
extern void tick_handle_periodic(struct clock_event_device *dev);
extern void tick_check_new_device(struct clock_event_device *dev);
+extern void tick_offline_cpu(unsigned int cpu);
extern void tick_shutdown(unsigned int cpu);
extern void tick_suspend(void);
extern void tick_resume(void);
@@ -142,10 +143,8 @@ static inline bool tick_broadcast_oneshot_available(void) { return tick_oneshot_
#endif /* !(BROADCAST && ONESHOT) */
#if defined(CONFIG_GENERIC_CLOCKEVENTS_BROADCAST) && defined(CONFIG_HOTPLUG_CPU)
-extern void tick_offline_cpu(unsigned int cpu);
extern void tick_broadcast_offline(unsigned int cpu);
#else
-static inline void tick_offline_cpu(unsigned int cpu) { }
static inline void tick_broadcast_offline(unsigned int cpu) { }
#endif
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 753a184c7090..9f90c7333b1d 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -311,14 +311,6 @@ static enum hrtimer_restart tick_nohz_handler(struct hrtimer *timer)
return HRTIMER_RESTART;
}
-static void tick_sched_timer_cancel(struct tick_sched *ts)
-{
- if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES))
- hrtimer_cancel(&ts->sched_timer);
- else if (tick_sched_flag_test(ts, TS_FLAG_NOHZ))
- tick_program_event(KTIME_MAX, 1);
-}
-
#ifdef CONFIG_NO_HZ_FULL
cpumask_var_t tick_nohz_full_mask;
EXPORT_SYMBOL_GPL(tick_nohz_full_mask);
@@ -1055,7 +1047,10 @@ static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu)
* the tick timer.
*/
if (unlikely(expires == KTIME_MAX)) {
- tick_sched_timer_cancel(ts);
+ if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES))
+ hrtimer_cancel(&ts->sched_timer);
+ else
+ tick_program_event(KTIME_MAX, 1);
return;
}
@@ -1604,21 +1599,13 @@ void tick_setup_sched_timer(bool hrtimer)
*/
void tick_sched_timer_dying(int cpu)
{
- struct tick_device *td = &per_cpu(tick_cpu_device, cpu);
struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
- struct clock_event_device *dev = td->evtdev;
ktime_t idle_sleeptime, iowait_sleeptime;
unsigned long idle_calls, idle_sleeps;
/* This must happen before hrtimers are migrated! */
- tick_sched_timer_cancel(ts);
-
- /*
- * If the clockevents doesn't support CLOCK_EVT_STATE_ONESHOT_STOPPED,
- * make sure not to call low-res tick handler.
- */
- if (tick_sched_flag_test(ts, TS_FLAG_NOHZ))
- dev->event_handler = clockevents_handle_noop;
+ if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES))
+ hrtimer_cancel(&ts->sched_timer);
idle_sleeptime = ts->idle_sleeptime;
iowait_sleeptime = ts->iowait_sleeptime;
diff --git a/kernel/time/time.c b/kernel/time/time.c
index 642647f5046b..1b69caa87480 100644
--- a/kernel/time/time.c
+++ b/kernel/time/time.c
@@ -556,9 +556,9 @@ EXPORT_SYMBOL(ns_to_timespec64);
* - all other values are converted to jiffies by either multiplying
* the input value by a factor or dividing it with a factor and
* handling any 32-bit overflows.
- * for the details see __msecs_to_jiffies()
+ * for the details see _msecs_to_jiffies()
*
- * __msecs_to_jiffies() checks for the passed in value being a constant
+ * msecs_to_jiffies() checks for the passed in value being a constant
* via __builtin_constant_p() allowing gcc to eliminate most of the
* code, __msecs_to_jiffies() is called if the value passed does not
* allow constant folding and the actual conversion must be done at
@@ -866,7 +866,7 @@ struct timespec64 timespec64_add_safe(const struct timespec64 lhs,
*
* Handles compat or 32-bit modes.
*
- * Return: %0 on success or negative errno on error
+ * Return: 0 on success or negative errno on error
*/
int get_timespec64(struct timespec64 *ts,
const struct __kernel_timespec __user *uts)
@@ -897,7 +897,7 @@ EXPORT_SYMBOL_GPL(get_timespec64);
* @ts: input &struct timespec64
* @uts: user's &struct __kernel_timespec
*
- * Return: %0 on success or negative errno on error
+ * Return: 0 on success or negative errno on error
*/
int put_timespec64(const struct timespec64 *ts,
struct __kernel_timespec __user *uts)
@@ -944,7 +944,7 @@ static int __put_old_timespec32(const struct timespec64 *ts64,
*
* Handles X86_X32_ABI compatibility conversion.
*
- * Return: %0 on success or negative errno on error
+ * Return: 0 on success or negative errno on error
*/
int get_old_timespec32(struct timespec64 *ts, const void __user *uts)
{
@@ -963,7 +963,7 @@ EXPORT_SYMBOL_GPL(get_old_timespec32);
*
* Handles X86_X32_ABI compatibility conversion.
*
- * Return: %0 on success or negative errno on error
+ * Return: 0 on success or negative errno on error
*/
int put_old_timespec32(const struct timespec64 *ts, void __user *uts)
{
@@ -979,7 +979,7 @@ EXPORT_SYMBOL_GPL(put_old_timespec32);
* @it: destination &struct itimerspec64
* @uit: user's &struct __kernel_itimerspec
*
- * Return: %0 on success or negative errno on error
+ * Return: 0 on success or negative errno on error
*/
int get_itimerspec64(struct itimerspec64 *it,
const struct __kernel_itimerspec __user *uit)
@@ -1002,7 +1002,7 @@ EXPORT_SYMBOL_GPL(get_itimerspec64);
* @it: input &struct itimerspec64
* @uit: user's &struct __kernel_itimerspec
*
- * Return: %0 on success or negative errno on error
+ * Return: 0 on success or negative errno on error
*/
int put_itimerspec64(const struct itimerspec64 *it,
struct __kernel_itimerspec __user *uit)
@@ -1024,7 +1024,7 @@ EXPORT_SYMBOL_GPL(put_itimerspec64);
* @its: destination &struct itimerspec64
* @uits: user's &struct old_itimerspec32
*
- * Return: %0 on success or negative errno on error
+ * Return: 0 on success or negative errno on error
*/
int get_old_itimerspec32(struct itimerspec64 *its,
const struct old_itimerspec32 __user *uits)
@@ -1043,7 +1043,7 @@ EXPORT_SYMBOL_GPL(get_old_itimerspec32);
* @its: input &struct itimerspec64
* @uits: user's &struct old_itimerspec32
*
- * Return: %0 on success or negative errno on error
+ * Return: 0 on success or negative errno on error
*/
int put_old_itimerspec32(const struct itimerspec64 *its,
struct old_itimerspec32 __user *uits)
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 7e6f409bf311..d115adebc418 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -30,8 +30,9 @@
#include "timekeeping_internal.h"
#define TK_CLEAR_NTP (1 << 0)
-#define TK_MIRROR (1 << 1)
-#define TK_CLOCK_WAS_SET (1 << 2)
+#define TK_CLOCK_WAS_SET (1 << 1)
+
+#define TK_UPDATE_ALL (TK_CLEAR_NTP | TK_CLOCK_WAS_SET)
enum timekeeping_adv_mode {
/* Update timekeeper when a tick has passed */
@@ -41,20 +42,18 @@ enum timekeeping_adv_mode {
TK_ADV_FREQ
};
-DEFINE_RAW_SPINLOCK(timekeeper_lock);
-
/*
* The most important data for readout fits into a single 64 byte
* cache line.
*/
-static struct {
+struct tk_data {
seqcount_raw_spinlock_t seq;
struct timekeeper timekeeper;
-} tk_core ____cacheline_aligned = {
- .seq = SEQCNT_RAW_SPINLOCK_ZERO(tk_core.seq, &timekeeper_lock),
-};
+ struct timekeeper shadow_timekeeper;
+ raw_spinlock_t lock;
+} ____cacheline_aligned;
-static struct timekeeper shadow_timekeeper;
+static struct tk_data tk_core;
/* flag for if timekeeping is suspended */
int __read_mostly timekeeping_suspended;
@@ -114,6 +113,36 @@ static struct tk_fast tk_fast_raw ____cacheline_aligned = {
.base[1] = FAST_TK_INIT,
};
+unsigned long timekeeper_lock_irqsave(void)
+{
+ unsigned long flags;
+
+ raw_spin_lock_irqsave(&tk_core.lock, flags);
+ return flags;
+}
+
+void timekeeper_unlock_irqrestore(unsigned long flags)
+{
+ raw_spin_unlock_irqrestore(&tk_core.lock, flags);
+}
+
+/*
+ * Multigrain timestamps require tracking the latest fine-grained timestamp
+ * that has been issued, and never returning a coarse-grained timestamp that is
+ * earlier than that value.
+ *
+ * mg_floor represents the latest fine-grained time that has been handed out as
+ * a file timestamp on the system. This is tracked as a monotonic ktime_t, and
+ * converted to a realtime clock value on an as-needed basis.
+ *
+ * Maintaining mg_floor ensures the multigrain interfaces never issue a
+ * timestamp earlier than one that has been previously issued.
+ *
+ * The exception to this rule is when there is a backward realtime clock jump. If
+ * such an event occurs, a timestamp can appear to be earlier than a previous one.
+ */
+static __cacheline_aligned_in_smp atomic64_t mg_floor;
+
static inline void tk_normalize_xtime(struct timekeeper *tk)
{
while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) {
@@ -161,13 +190,15 @@ static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm)
WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp));
tk->wall_to_monotonic = wtm;
set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec);
- tk->offs_real = timespec64_to_ktime(tmp);
- tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0));
+ /* Paired with READ_ONCE() in ktime_mono_to_any() */
+ WRITE_ONCE(tk->offs_real, timespec64_to_ktime(tmp));
+ WRITE_ONCE(tk->offs_tai, ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0)));
}
static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)
{
- tk->offs_boot = ktime_add(tk->offs_boot, delta);
+ /* Paired with READ_ONCE() in ktime_mono_to_any() */
+ WRITE_ONCE(tk->offs_boot, ktime_add(tk->offs_boot, delta));
/*
* Timespec representation for VDSO update to avoid 64bit division
* on every update.
@@ -184,7 +215,7 @@ static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)
* the tkr's clocksource may change between the read reference, and the
* clock reference passed to the read function. This can cause crashes if
* the wrong clocksource is passed to the wrong read function.
- * This isn't necessary to use when holding the timekeeper_lock or doing
+ * This isn't necessary to use when holding the tk_core.lock or doing
* a read of the fast-timekeeper tkrs (which is protected by its own locking
* and update logic).
*/
@@ -195,97 +226,6 @@ static inline u64 tk_clock_read(const struct tk_read_base *tkr)
return clock->read(clock);
}
-#ifdef CONFIG_DEBUG_TIMEKEEPING
-#define WARNING_FREQ (HZ*300) /* 5 minute rate-limiting */
-
-static void timekeeping_check_update(struct timekeeper *tk, u64 offset)
-{
-
- u64 max_cycles = tk->tkr_mono.clock->max_cycles;
- const char *name = tk->tkr_mono.clock->name;
-
- if (offset > max_cycles) {
- printk_deferred("WARNING: timekeeping: Cycle offset (%lld) is larger than allowed by the '%s' clock's max_cycles value (%lld): time overflow danger\n",
- offset, name, max_cycles);
- printk_deferred(" timekeeping: Your kernel is sick, but tries to cope by capping time updates\n");
- } else {
- if (offset > (max_cycles >> 1)) {
- printk_deferred("INFO: timekeeping: Cycle offset (%lld) is larger than the '%s' clock's 50%% safety margin (%lld)\n",
- offset, name, max_cycles >> 1);
- printk_deferred(" timekeeping: Your kernel is still fine, but is feeling a bit nervous\n");
- }
- }
-
- if (tk->underflow_seen) {
- if (jiffies - tk->last_warning > WARNING_FREQ) {
- printk_deferred("WARNING: Underflow in clocksource '%s' observed, time update ignored.\n", name);
- printk_deferred(" Please report this, consider using a different clocksource, if possible.\n");
- printk_deferred(" Your kernel is probably still fine.\n");
- tk->last_warning = jiffies;
- }
- tk->underflow_seen = 0;
- }
-
- if (tk->overflow_seen) {
- if (jiffies - tk->last_warning > WARNING_FREQ) {
- printk_deferred("WARNING: Overflow in clocksource '%s' observed, time update capped.\n", name);
- printk_deferred(" Please report this, consider using a different clocksource, if possible.\n");
- printk_deferred(" Your kernel is probably still fine.\n");
- tk->last_warning = jiffies;
- }
- tk->overflow_seen = 0;
- }
-}
-
-static inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles);
-
-static inline u64 timekeeping_debug_get_ns(const struct tk_read_base *tkr)
-{
- struct timekeeper *tk = &tk_core.timekeeper;
- u64 now, last, mask, max, delta;
- unsigned int seq;
-
- /*
- * Since we're called holding a seqcount, the data may shift
- * under us while we're doing the calculation. This can cause
- * false positives, since we'd note a problem but throw the
- * results away. So nest another seqcount here to atomically
- * grab the points we are checking with.
- */
- do {
- seq = read_seqcount_begin(&tk_core.seq);
- now = tk_clock_read(tkr);
- last = tkr->cycle_last;
- mask = tkr->mask;
- max = tkr->clock->max_cycles;
- } while (read_seqcount_retry(&tk_core.seq, seq));
-
- delta = clocksource_delta(now, last, mask);
-
- /*
- * Try to catch underflows by checking if we are seeing small
- * mask-relative negative values.
- */
- if (unlikely((~delta & mask) < (mask >> 3)))
- tk->underflow_seen = 1;
-
- /* Check for multiplication overflows */
- if (unlikely(delta > max))
- tk->overflow_seen = 1;
-
- /* timekeeping_cycles_to_ns() handles both under and overflow */
- return timekeeping_cycles_to_ns(tkr, now);
-}
-#else
-static inline void timekeeping_check_update(struct timekeeper *tk, u64 offset)
-{
-}
-static inline u64 timekeeping_debug_get_ns(const struct tk_read_base *tkr)
-{
- BUG();
-}
-#endif
-
/**
* tk_setup_internals - Set up internals to use clocksource clock.
*
@@ -390,19 +330,11 @@ static inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 c
return ((delta * tkr->mult) + tkr->xtime_nsec) >> tkr->shift;
}
-static __always_inline u64 __timekeeping_get_ns(const struct tk_read_base *tkr)
+static __always_inline u64 timekeeping_get_ns(const struct tk_read_base *tkr)
{
return timekeeping_cycles_to_ns(tkr, tk_clock_read(tkr));
}
-static inline u64 timekeeping_get_ns(const struct tk_read_base *tkr)
-{
- if (IS_ENABLED(CONFIG_DEBUG_TIMEKEEPING))
- return timekeeping_debug_get_ns(tkr);
-
- return __timekeeping_get_ns(tkr);
-}
-
/**
* update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper.
* @tkr: Timekeeping readout base from which we take the update
@@ -446,7 +378,7 @@ static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf)
seq = raw_read_seqcount_latch(&tkf->seq);
tkr = tkf->base + (seq & 0x01);
now = ktime_to_ns(tkr->base);
- now += __timekeeping_get_ns(tkr);
+ now += timekeeping_get_ns(tkr);
} while (raw_read_seqcount_latch_retry(&tkf->seq, seq));
return now;
@@ -517,7 +449,7 @@ EXPORT_SYMBOL_GPL(ktime_get_raw_fast_ns);
* timekeeping_inject_sleeptime64()
* __timekeeping_inject_sleeptime(tk, delta);
* timestamp();
- * timekeeping_update(tk, TK_CLEAR_NTP...);
+ * timekeeping_update_staged(tkd, TK_CLEAR_NTP...);
*
* (2) On 32-bit systems, the 64-bit boot offset (tk->offs_boot) may be
* partially updated. Since the tk->offs_boot update is a rare event, this
@@ -562,7 +494,7 @@ static __always_inline u64 __ktime_get_real_fast(struct tk_fast *tkf, u64 *mono)
tkr = tkf->base + (seq & 0x01);
basem = ktime_to_ns(tkr->base);
baser = ktime_to_ns(tkr->base_real);
- delta = __timekeeping_get_ns(tkr);
+ delta = timekeeping_get_ns(tkr);
} while (raw_read_seqcount_latch_retry(&tkf->seq, seq));
if (mono)
@@ -676,13 +608,11 @@ static void update_pvclock_gtod(struct timekeeper *tk, bool was_set)
int pvclock_gtod_register_notifier(struct notifier_block *nb)
{
struct timekeeper *tk = &tk_core.timekeeper;
- unsigned long flags;
int ret;
- raw_spin_lock_irqsave(&timekeeper_lock, flags);
+ guard(raw_spinlock_irqsave)(&tk_core.lock);
ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb);
update_pvclock_gtod(tk, true);
- raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
return ret;
}
@@ -695,14 +625,8 @@ EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier);
*/
int pvclock_gtod_unregister_notifier(struct notifier_block *nb)
{
- unsigned long flags;
- int ret;
-
- raw_spin_lock_irqsave(&timekeeper_lock, flags);
- ret = raw_notifier_chain_unregister(&pvclock_gtod_chain, nb);
- raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
-
- return ret;
+ guard(raw_spinlock_irqsave)(&tk_core.lock);
+ return raw_notifier_chain_unregister(&pvclock_gtod_chain, nb);
}
EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier);
@@ -717,6 +641,18 @@ static inline void tk_update_leap_state(struct timekeeper *tk)
tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real);
}
+/*
+ * Leap state update for both shadow and the real timekeeper
+ * Separate to spare a full memcpy() of the timekeeper.
+ */
+static void tk_update_leap_state_all(struct tk_data *tkd)
+{
+ write_seqcount_begin(&tkd->seq);
+ tk_update_leap_state(&tkd->shadow_timekeeper);
+ tkd->timekeeper.next_leap_ktime = tkd->shadow_timekeeper.next_leap_ktime;
+ write_seqcount_end(&tkd->seq);
+}
+
/*
* Update the ktime_t based scalar nsec members of the timekeeper
*/
@@ -750,9 +686,30 @@ static inline void tk_update_ktime_data(struct timekeeper *tk)
tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC);
}
-/* must hold timekeeper_lock */
-static void timekeeping_update(struct timekeeper *tk, unsigned int action)
+/*
+ * Restore the shadow timekeeper from the real timekeeper.
+ */
+static void timekeeping_restore_shadow(struct tk_data *tkd)
+{
+ lockdep_assert_held(&tkd->lock);
+ memcpy(&tkd->shadow_timekeeper, &tkd->timekeeper, sizeof(tkd->timekeeper));
+}
+
+static void timekeeping_update_from_shadow(struct tk_data *tkd, unsigned int action)
{
+ struct timekeeper *tk = &tk_core.shadow_timekeeper;
+
+ lockdep_assert_held(&tkd->lock);
+
+ /*
+ * Block out readers before running the updates below because that
+ * updates VDSO and other time related infrastructure. Not blocking
+ * the readers might let a reader see time going backwards when
+ * reading from the VDSO after the VDSO update and then reading in
+ * the kernel from the timekeeper before that got updated.
+ */
+ write_seqcount_begin(&tkd->seq);
+
if (action & TK_CLEAR_NTP) {
tk->ntp_error = 0;
ntp_clear();
@@ -770,14 +727,17 @@ static void timekeeping_update(struct timekeeper *tk, unsigned int action)
if (action & TK_CLOCK_WAS_SET)
tk->clock_was_set_seq++;
+
/*
- * The mirroring of the data to the shadow-timekeeper needs
- * to happen last here to ensure we don't over-write the
- * timekeeper structure on the next update with stale data
+ * Update the real timekeeper.
+ *
+ * We could avoid this memcpy() by switching pointers, but that has
+ * the downside that the reader side does not longer benefit from
+ * the cacheline optimized data layout of the timekeeper and requires
+ * another indirection.
*/
- if (action & TK_MIRROR)
- memcpy(&shadow_timekeeper, &tk_core.timekeeper,
- sizeof(tk_core.timekeeper));
+ memcpy(&tkd->timekeeper, tk, sizeof(*tk));
+ write_seqcount_end(&tkd->seq);
}
/**
@@ -930,6 +890,14 @@ ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs)
unsigned int seq;
ktime_t tconv;
+ if (IS_ENABLED(CONFIG_64BIT)) {
+ /*
+ * Paired with WRITE_ONCE()s in tk_set_wall_to_mono() and
+ * tk_update_sleep_time().
+ */
+ return ktime_add(tmono, READ_ONCE(*offset));
+ }
+
do {
seq = read_seqcount_begin(&tk_core.seq);
tconv = ktime_add(tmono, *offset);
@@ -1060,6 +1028,7 @@ void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot)
unsigned int seq;
ktime_t base_raw;
ktime_t base_real;
+ ktime_t base_boot;
u64 nsec_raw;
u64 nsec_real;
u64 now;
@@ -1074,6 +1043,8 @@ void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot)
systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq;
base_real = ktime_add(tk->tkr_mono.base,
tk_core.timekeeper.offs_real);
+ base_boot = ktime_add(tk->tkr_mono.base,
+ tk_core.timekeeper.offs_boot);
base_raw = tk->tkr_raw.base;
nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now);
nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now);
@@ -1081,6 +1052,7 @@ void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot)
systime_snapshot->cycles = now;
systime_snapshot->real = ktime_add_ns(base_real, nsec_real);
+ systime_snapshot->boot = ktime_add_ns(base_boot, nsec_real);
systime_snapshot->raw = ktime_add_ns(base_raw, nsec_raw);
}
EXPORT_SYMBOL_GPL(ktime_get_snapshot);
@@ -1440,45 +1412,35 @@ EXPORT_SYMBOL_GPL(timekeeping_clocksource_has_base);
*/
int do_settimeofday64(const struct timespec64 *ts)
{
- struct timekeeper *tk = &tk_core.timekeeper;
struct timespec64 ts_delta, xt;
- unsigned long flags;
- int ret = 0;
if (!timespec64_valid_settod(ts))
return -EINVAL;
- raw_spin_lock_irqsave(&timekeeper_lock, flags);
- write_seqcount_begin(&tk_core.seq);
-
- timekeeping_forward_now(tk);
+ scoped_guard (raw_spinlock_irqsave, &tk_core.lock) {
+ struct timekeeper *tks = &tk_core.shadow_timekeeper;
- xt = tk_xtime(tk);
- ts_delta = timespec64_sub(*ts, xt);
-
- if (timespec64_compare(&tk->wall_to_monotonic, &ts_delta) > 0) {
- ret = -EINVAL;
- goto out;
- }
+ timekeeping_forward_now(tks);
- tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts_delta));
+ xt = tk_xtime(tks);
+ ts_delta = timespec64_sub(*ts, xt);
- tk_set_xtime(tk, ts);
-out:
- timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
+ if (timespec64_compare(&tks->wall_to_monotonic, &ts_delta) > 0) {
+ timekeeping_restore_shadow(&tk_core);
+ return -EINVAL;
+ }
- write_seqcount_end(&tk_core.seq);
- raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
+ tk_set_wall_to_mono(tks, timespec64_sub(tks->wall_to_monotonic, ts_delta));
+ tk_set_xtime(tks, ts);
+ timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL);
+ }
/* Signal hrtimers about time change */
clock_was_set(CLOCK_SET_WALL);
- if (!ret) {
- audit_tk_injoffset(ts_delta);
- add_device_randomness(ts, sizeof(*ts));
- }
-
- return ret;
+ audit_tk_injoffset(ts_delta);
+ add_device_randomness(ts, sizeof(*ts));
+ return 0;
}
EXPORT_SYMBOL(do_settimeofday64);
@@ -1490,40 +1452,31 @@ EXPORT_SYMBOL(do_settimeofday64);
*/
static int timekeeping_inject_offset(const struct timespec64 *ts)
{
- struct timekeeper *tk = &tk_core.timekeeper;
- unsigned long flags;
- struct timespec64 tmp;
- int ret = 0;
-
if (ts->tv_nsec < 0 || ts->tv_nsec >= NSEC_PER_SEC)
return -EINVAL;
- raw_spin_lock_irqsave(&timekeeper_lock, flags);
- write_seqcount_begin(&tk_core.seq);
-
- timekeeping_forward_now(tk);
-
- /* Make sure the proposed value is valid */
- tmp = timespec64_add(tk_xtime(tk), *ts);
- if (timespec64_compare(&tk->wall_to_monotonic, ts) > 0 ||
- !timespec64_valid_settod(&tmp)) {
- ret = -EINVAL;
- goto error;
- }
+ scoped_guard (raw_spinlock_irqsave, &tk_core.lock) {
+ struct timekeeper *tks = &tk_core.shadow_timekeeper;
+ struct timespec64 tmp;
- tk_xtime_add(tk, ts);
- tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *ts));
+ timekeeping_forward_now(tks);
-error: /* even if we error out, we forwarded the time, so call update */
- timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
+ /* Make sure the proposed value is valid */
+ tmp = timespec64_add(tk_xtime(tks), *ts);
+ if (timespec64_compare(&tks->wall_to_monotonic, ts) > 0 ||
+ !timespec64_valid_settod(&tmp)) {
+ timekeeping_restore_shadow(&tk_core);
+ return -EINVAL;
+ }
- write_seqcount_end(&tk_core.seq);
- raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
+ tk_xtime_add(tks, ts);
+ tk_set_wall_to_mono(tks, timespec64_sub(tks->wall_to_monotonic, *ts));
+ timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL);
+ }
/* Signal hrtimers about time change */
clock_was_set(CLOCK_SET_WALL);
-
- return ret;
+ return 0;
}
/*
@@ -1576,43 +1529,34 @@ static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset)
*/
static int change_clocksource(void *data)
{
- struct timekeeper *tk = &tk_core.timekeeper;
- struct clocksource *new, *old = NULL;
- unsigned long flags;
- bool change = false;
-
- new = (struct clocksource *) data;
+ struct clocksource *new = data, *old = NULL;
/*
- * If the cs is in module, get a module reference. Succeeds
- * for built-in code (owner == NULL) as well.
+ * If the clocksource is in a module, get a module reference.
+ * Succeeds for built-in code (owner == NULL) as well. Abort if the
+ * reference can't be acquired.
*/
- if (try_module_get(new->owner)) {
- if (!new->enable || new->enable(new) == 0)
- change = true;
- else
- module_put(new->owner);
- }
-
- raw_spin_lock_irqsave(&timekeeper_lock, flags);
- write_seqcount_begin(&tk_core.seq);
-
- timekeeping_forward_now(tk);
+ if (!try_module_get(new->owner))
+ return 0;
- if (change) {
- old = tk->tkr_mono.clock;
- tk_setup_internals(tk, new);
+ /* Abort if the device can't be enabled */
+ if (new->enable && new->enable(new) != 0) {
+ module_put(new->owner);
+ return 0;
}
- timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
+ scoped_guard (raw_spinlock_irqsave, &tk_core.lock) {
+ struct timekeeper *tks = &tk_core.shadow_timekeeper;
- write_seqcount_end(&tk_core.seq);
- raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
+ timekeeping_forward_now(tks);
+ old = tks->tkr_mono.clock;
+ tk_setup_internals(tks, new);
+ timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL);
+ }
if (old) {
if (old->disable)
old->disable(old);
-
module_put(old->owner);
}
@@ -1737,6 +1681,12 @@ read_persistent_wall_and_boot_offset(struct timespec64 *wall_time,
*boot_offset = ns_to_timespec64(local_clock());
}
+static __init void tkd_basic_setup(struct tk_data *tkd)
+{
+ raw_spin_lock_init(&tkd->lock);
+ seqcount_raw_spinlock_init(&tkd->seq, &tkd->lock);
+}
+
/*
* Flag reflecting whether timekeeping_resume() has injected sleeptime.
*
@@ -1761,9 +1711,10 @@ static bool persistent_clock_exists;
void __init timekeeping_init(void)
{
struct timespec64 wall_time, boot_offset, wall_to_mono;
- struct timekeeper *tk = &tk_core.timekeeper;
+ struct timekeeper *tks = &tk_core.shadow_timekeeper;
struct clocksource *clock;
- unsigned long flags;
+
+ tkd_basic_setup(&tk_core);
read_persistent_wall_and_boot_offset(&wall_time, &boot_offset);
if (timespec64_valid_settod(&wall_time) &&
@@ -1783,24 +1734,21 @@ void __init timekeeping_init(void)
*/
wall_to_mono = timespec64_sub(boot_offset, wall_time);
- raw_spin_lock_irqsave(&timekeeper_lock, flags);
- write_seqcount_begin(&tk_core.seq);
+ guard(raw_spinlock_irqsave)(&tk_core.lock);
+
ntp_init();
clock = clocksource_default_clock();
if (clock->enable)
clock->enable(clock);
- tk_setup_internals(tk, clock);
+ tk_setup_internals(tks, clock);
- tk_set_xtime(tk, &wall_time);
- tk->raw_sec = 0;
+ tk_set_xtime(tks, &wall_time);
+ tks->raw_sec = 0;
- tk_set_wall_to_mono(tk, wall_to_mono);
+ tk_set_wall_to_mono(tks, wall_to_mono);
- timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
-
- write_seqcount_end(&tk_core.seq);
- raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
+ timekeeping_update_from_shadow(&tk_core, TK_CLOCK_WAS_SET);
}
/* time in seconds when suspend began for persistent clock */
@@ -1878,22 +1826,14 @@ bool timekeeping_rtc_skipsuspend(void)
*/
void timekeeping_inject_sleeptime64(const struct timespec64 *delta)
{
- struct timekeeper *tk = &tk_core.timekeeper;
- unsigned long flags;
-
- raw_spin_lock_irqsave(&timekeeper_lock, flags);
- write_seqcount_begin(&tk_core.seq);
-
- suspend_timing_needed = false;
-
- timekeeping_forward_now(tk);
+ scoped_guard(raw_spinlock_irqsave, &tk_core.lock) {
+ struct timekeeper *tks = &tk_core.shadow_timekeeper;
- __timekeeping_inject_sleeptime(tk, delta);
-
- timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
-
- write_seqcount_end(&tk_core.seq);
- raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
+ suspend_timing_needed = false;
+ timekeeping_forward_now(tks);
+ __timekeeping_inject_sleeptime(tks, delta);
+ timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL);
+ }
/* Signal hrtimers about time change */
clock_was_set(CLOCK_SET_WALL | CLOCK_SET_BOOT);
@@ -1905,20 +1845,19 @@ void timekeeping_inject_sleeptime64(const struct timespec64 *delta)
*/
void timekeeping_resume(void)
{
- struct timekeeper *tk = &tk_core.timekeeper;
- struct clocksource *clock = tk->tkr_mono.clock;
- unsigned long flags;
+ struct timekeeper *tks = &tk_core.shadow_timekeeper;
+ struct clocksource *clock = tks->tkr_mono.clock;
struct timespec64 ts_new, ts_delta;
- u64 cycle_now, nsec;
bool inject_sleeptime = false;
+ u64 cycle_now, nsec;
+ unsigned long flags;
read_persistent_clock64(&ts_new);
clockevents_resume();
clocksource_resume();
- raw_spin_lock_irqsave(&timekeeper_lock, flags);
- write_seqcount_begin(&tk_core.seq);
+ raw_spin_lock_irqsave(&tk_core.lock, flags);
/*
* After system resumes, we need to calculate the suspended time and
@@ -1932,7 +1871,7 @@ void timekeeping_resume(void)
* The less preferred source will only be tried if there is no better
* usable source. The rtc part is handled separately in rtc core code.
*/
- cycle_now = tk_clock_read(&tk->tkr_mono);
+ cycle_now = tk_clock_read(&tks->tkr_mono);
nsec = clocksource_stop_suspend_timing(clock, cycle_now);
if (nsec > 0) {
ts_delta = ns_to_timespec64(nsec);
@@ -1944,18 +1883,17 @@ void timekeeping_resume(void)
if (inject_sleeptime) {
suspend_timing_needed = false;
- __timekeeping_inject_sleeptime(tk, &ts_delta);
+ __timekeeping_inject_sleeptime(tks, &ts_delta);
}
/* Re-base the last cycle value */
- tk->tkr_mono.cycle_last = cycle_now;
- tk->tkr_raw.cycle_last = cycle_now;
+ tks->tkr_mono.cycle_last = cycle_now;
+ tks->tkr_raw.cycle_last = cycle_now;
- tk->ntp_error = 0;
+ tks->ntp_error = 0;
timekeeping_suspended = 0;
- timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
- write_seqcount_end(&tk_core.seq);
- raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
+ timekeeping_update_from_shadow(&tk_core, TK_CLOCK_WAS_SET);
+ raw_spin_unlock_irqrestore(&tk_core.lock, flags);
touch_softlockup_watchdog();
@@ -1967,11 +1905,11 @@ void timekeeping_resume(void)
int timekeeping_suspend(void)
{
- struct timekeeper *tk = &tk_core.timekeeper;
- unsigned long flags;
- struct timespec64 delta, delta_delta;
- static struct timespec64 old_delta;
+ struct timekeeper *tks = &tk_core.shadow_timekeeper;
+ struct timespec64 delta, delta_delta;
+ static struct timespec64 old_delta;
struct clocksource *curr_clock;
+ unsigned long flags;
u64 cycle_now;
read_persistent_clock64(&timekeeping_suspend_time);
@@ -1986,9 +1924,8 @@ int timekeeping_suspend(void)
suspend_timing_needed = true;
- raw_spin_lock_irqsave(&timekeeper_lock, flags);
- write_seqcount_begin(&tk_core.seq);
- timekeeping_forward_now(tk);
+ raw_spin_lock_irqsave(&tk_core.lock, flags);
+ timekeeping_forward_now(tks);
timekeeping_suspended = 1;
/*
@@ -1996,8 +1933,8 @@ int timekeeping_suspend(void)
* just read from the current clocksource. Save this to potentially
* use in suspend timing.
*/
- curr_clock = tk->tkr_mono.clock;
- cycle_now = tk->tkr_mono.cycle_last;
+ curr_clock = tks->tkr_mono.clock;
+ cycle_now = tks->tkr_mono.cycle_last;
clocksource_start_suspend_timing(curr_clock, cycle_now);
if (persistent_clock_exists) {
@@ -2007,7 +1944,7 @@ int timekeeping_suspend(void)
* try to compensate so the difference in system time
* and persistent_clock time stays close to constant.
*/
- delta = timespec64_sub(tk_xtime(tk), timekeeping_suspend_time);
+ delta = timespec64_sub(tk_xtime(tks), timekeeping_suspend_time);
delta_delta = timespec64_sub(delta, old_delta);
if (abs(delta_delta.tv_sec) >= 2) {
/*
@@ -2022,10 +1959,9 @@ int timekeeping_suspend(void)
}
}
- timekeeping_update(tk, TK_MIRROR);
- halt_fast_timekeeper(tk);
- write_seqcount_end(&tk_core.seq);
- raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
+ timekeeping_update_from_shadow(&tk_core, 0);
+ halt_fast_timekeeper(tks);
+ raw_spin_unlock_irqrestore(&tk_core.lock, flags);
tick_suspend();
clocksource_suspend();
@@ -2130,16 +2066,17 @@ static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
*/
static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
{
+ u64 ntp_tl = ntp_tick_length();
u32 mult;
/*
* Determine the multiplier from the current NTP tick length.
* Avoid expensive division when the tick length doesn't change.
*/
- if (likely(tk->ntp_tick == ntp_tick_length())) {
+ if (likely(tk->ntp_tick == ntp_tl)) {
mult = tk->tkr_mono.mult - tk->ntp_err_mult;
} else {
- tk->ntp_tick = ntp_tick_length();
+ tk->ntp_tick = ntp_tl;
mult = div64_u64((tk->ntp_tick >> tk->ntp_error_shift) -
tk->xtime_remainder, tk->cycle_interval);
}
@@ -2278,28 +2215,24 @@ static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset,
*/
static bool timekeeping_advance(enum timekeeping_adv_mode mode)
{
+ struct timekeeper *tk = &tk_core.shadow_timekeeper;
struct timekeeper *real_tk = &tk_core.timekeeper;
- struct timekeeper *tk = &shadow_timekeeper;
- u64 offset;
- int shift = 0, maxshift;
unsigned int clock_set = 0;
- unsigned long flags;
+ int shift = 0, maxshift;
+ u64 offset;
- raw_spin_lock_irqsave(&timekeeper_lock, flags);
+ guard(raw_spinlock_irqsave)(&tk_core.lock);
/* Make sure we're fully resumed: */
if (unlikely(timekeeping_suspended))
- goto out;
+ return false;
offset = clocksource_delta(tk_clock_read(&tk->tkr_mono),
tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
/* Check if there's really nothing to do */
if (offset < real_tk->cycle_interval && mode == TK_ADV_TICK)
- goto out;
-
- /* Do some additional sanity checking */
- timekeeping_check_update(tk, offset);
+ return false;
/*
* With NO_HZ we may have to accumulate many cycle_intervals
@@ -2315,8 +2248,7 @@ static bool timekeeping_advance(enum timekeeping_adv_mode mode)
maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;
shift = min(shift, maxshift);
while (offset >= tk->cycle_interval) {
- offset = logarithmic_accumulation(tk, offset, shift,
- &clock_set);
+ offset = logarithmic_accumulation(tk, offset, shift, &clock_set);
if (offset < tk->cycle_interval<<shift)
shift--;
}
@@ -2330,23 +2262,7 @@ static bool timekeeping_advance(enum timekeeping_adv_mode mode)
*/
clock_set |= accumulate_nsecs_to_secs(tk);
- write_seqcount_begin(&tk_core.seq);
- /*
- * Update the real timekeeper.
- *
- * We could avoid this memcpy by switching pointers, but that
- * requires changes to all other timekeeper usage sites as
- * well, i.e. move the timekeeper pointer getter into the
- * spinlocked/seqcount protected sections. And we trade this
- * memcpy under the tk_core.seq against one before we start
- * updating.
- */
- timekeeping_update(tk, clock_set);
- memcpy(real_tk, tk, sizeof(*tk));
- /* The memcpy must come last. Do not put anything here! */
- write_seqcount_end(&tk_core.seq);
-out:
- raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
+ timekeeping_update_from_shadow(&tk_core, clock_set);
return !!clock_set;
}
@@ -2394,6 +2310,94 @@ void ktime_get_coarse_real_ts64(struct timespec64 *ts)
}
EXPORT_SYMBOL(ktime_get_coarse_real_ts64);
+/**
+ * ktime_get_coarse_real_ts64_mg - return latter of coarse grained time or floor
+ * @ts: timespec64 to be filled
+ *
+ * Fetch the global mg_floor value, convert it to realtime and compare it
+ * to the current coarse-grained time. Fill @ts with whichever is
+ * latest. Note that this is a filesystem-specific interface and should be
+ * avoided outside of that context.
+ */
+void ktime_get_coarse_real_ts64_mg(struct timespec64 *ts)
+{
+ struct timekeeper *tk = &tk_core.timekeeper;
+ u64 floor = atomic64_read(&mg_floor);
+ ktime_t f_real, offset, coarse;
+ unsigned int seq;
+
+ do {
+ seq = read_seqcount_begin(&tk_core.seq);
+ *ts = tk_xtime(tk);
+ offset = tk_core.timekeeper.offs_real;
+ } while (read_seqcount_retry(&tk_core.seq, seq));
+
+ coarse = timespec64_to_ktime(*ts);
+ f_real = ktime_add(floor, offset);
+ if (ktime_after(f_real, coarse))
+ *ts = ktime_to_timespec64(f_real);
+}
+
+/**
+ * ktime_get_real_ts64_mg - attempt to update floor value and return result
+ * @ts: pointer to the timespec to be set
+ *
+ * Get a monotonic fine-grained time value and attempt to swap it into
+ * mg_floor. If that succeeds then accept the new floor value. If it fails
+ * then another task raced in during the interim time and updated the
+ * floor. Since any update to the floor must be later than the previous
+ * floor, either outcome is acceptable.
+ *
+ * Typically this will be called after calling ktime_get_coarse_real_ts64_mg(),
+ * and determining that the resulting coarse-grained timestamp did not effect
+ * a change in ctime. Any more recent floor value would effect a change to
+ * ctime, so there is no need to retry the atomic64_try_cmpxchg() on failure.
+ *
+ * @ts will be filled with the latest floor value, regardless of the outcome of
+ * the cmpxchg. Note that this is a filesystem specific interface and should be
+ * avoided outside of that context.
+ */
+void ktime_get_real_ts64_mg(struct timespec64 *ts)
+{
+ struct timekeeper *tk = &tk_core.timekeeper;
+ ktime_t old = atomic64_read(&mg_floor);
+ ktime_t offset, mono;
+ unsigned int seq;
+ u64 nsecs;
+
+ do {
+ seq = read_seqcount_begin(&tk_core.seq);
+
+ ts->tv_sec = tk->xtime_sec;
+ mono = tk->tkr_mono.base;
+ nsecs = timekeeping_get_ns(&tk->tkr_mono);
+ offset = tk_core.timekeeper.offs_real;
+ } while (read_seqcount_retry(&tk_core.seq, seq));
+
+ mono = ktime_add_ns(mono, nsecs);
+
+ /*
+ * Attempt to update the floor with the new time value. As any
+ * update must be later then the existing floor, and would effect
+ * a change to ctime from the perspective of the current task,
+ * accept the resulting floor value regardless of the outcome of
+ * the swap.
+ */
+ if (atomic64_try_cmpxchg(&mg_floor, &old, mono)) {
+ ts->tv_nsec = 0;
+ timespec64_add_ns(ts, nsecs);
+ timekeeping_inc_mg_floor_swaps();
+ } else {
+ /*
+ * Another task changed mg_floor since "old" was fetched.
+ * "old" has been updated with the latest value of "mg_floor".
+ * That value is newer than the previous floor value, which
+ * is enough to effect a change to ctime. Accept it.
+ */
+ *ts = ktime_to_timespec64(ktime_add(old, offset));
+ }
+}
+
void ktime_get_coarse_ts64(struct timespec64 *ts)
{
struct timekeeper *tk = &tk_core.timekeeper;
@@ -2551,13 +2555,10 @@ EXPORT_SYMBOL_GPL(random_get_entropy_fallback);
*/
int do_adjtimex(struct __kernel_timex *txc)
{
- struct timekeeper *tk = &tk_core.timekeeper;
struct audit_ntp_data ad;
bool offset_set = false;
bool clock_set = false;
struct timespec64 ts;
- unsigned long flags;
- s32 orig_tai, tai;
int ret;
/* Validate the data before disabling interrupts */
@@ -2568,6 +2569,7 @@ int do_adjtimex(struct __kernel_timex *txc)
if (txc->modes & ADJ_SETOFFSET) {
struct timespec64 delta;
+
delta.tv_sec = txc->time.tv_sec;
delta.tv_nsec = txc->time.tv_usec;
if (!(txc->modes & ADJ_NANO))
@@ -2585,21 +2587,21 @@ int do_adjtimex(struct __kernel_timex *txc)
ktime_get_real_ts64(&ts);
add_device_randomness(&ts, sizeof(ts));
- raw_spin_lock_irqsave(&timekeeper_lock, flags);
- write_seqcount_begin(&tk_core.seq);
+ scoped_guard (raw_spinlock_irqsave, &tk_core.lock) {
+ struct timekeeper *tks = &tk_core.shadow_timekeeper;
+ s32 orig_tai, tai;
- orig_tai = tai = tk->tai_offset;
- ret = __do_adjtimex(txc, &ts, &tai, &ad);
+ orig_tai = tai = tks->tai_offset;
+ ret = __do_adjtimex(txc, &ts, &tai, &ad);
- if (tai != orig_tai) {
- __timekeeping_set_tai_offset(tk, tai);
- timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
- clock_set = true;
+ if (tai != orig_tai) {
+ __timekeeping_set_tai_offset(tks, tai);
+ timekeeping_update_from_shadow(&tk_core, TK_CLOCK_WAS_SET);
+ clock_set = true;
+ } else {
+ tk_update_leap_state_all(&tk_core);
+ }
}
- tk_update_leap_state(tk);
-
- write_seqcount_end(&tk_core.seq);
- raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
audit_ntp_log(&ad);
@@ -2623,15 +2625,8 @@ int do_adjtimex(struct __kernel_timex *txc)
*/
void hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_ts)
{
- unsigned long flags;
-
- raw_spin_lock_irqsave(&timekeeper_lock, flags);
- write_seqcount_begin(&tk_core.seq);
-
+ guard(raw_spinlock_irqsave)(&tk_core.lock);
__hardpps(phase_ts, raw_ts);
-
- write_seqcount_end(&tk_core.seq);
- raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
}
EXPORT_SYMBOL(hardpps);
#endif /* CONFIG_NTP_PPS */
diff --git a/kernel/time/timekeeping_debug.c b/kernel/time/timekeeping_debug.c
index b73e8850e58d..badeb222eab9 100644
--- a/kernel/time/timekeeping_debug.c
+++ b/kernel/time/timekeeping_debug.c
@@ -17,6 +17,9 @@
#define NUM_BINS 32
+/* Incremented every time mg_floor is updated */
+DEFINE_PER_CPU(unsigned long, timekeeping_mg_floor_swaps);
+
static unsigned int sleep_time_bin[NUM_BINS] = {0};
static int tk_debug_sleep_time_show(struct seq_file *s, void *data)
@@ -53,3 +56,13 @@ void tk_debug_account_sleep_time(const struct timespec64 *t)
(s64)t->tv_sec, t->tv_nsec / NSEC_PER_MSEC);
}
+unsigned long timekeeping_get_mg_floor_swaps(void)
+{
+ unsigned long sum = 0;
+ int cpu;
+
+ for_each_possible_cpu(cpu)
+ sum += data_race(per_cpu(timekeeping_mg_floor_swaps, cpu));
+
+ return sum;
+}
diff --git a/kernel/time/timekeeping_internal.h b/kernel/time/timekeeping_internal.h
index 4ca2787d1642..63e600e943a7 100644
--- a/kernel/time/timekeeping_internal.h
+++ b/kernel/time/timekeeping_internal.h
@@ -10,12 +10,26 @@
* timekeeping debug functions
*/
#ifdef CONFIG_DEBUG_FS
+
+DECLARE_PER_CPU(unsigned long, timekeeping_mg_floor_swaps);
+
+static inline void timekeeping_inc_mg_floor_swaps(void)
+{
+ this_cpu_inc(timekeeping_mg_floor_swaps);
+}
+
extern void tk_debug_account_sleep_time(const struct timespec64 *t);
+
#else
+
#define tk_debug_account_sleep_time(x)
+
+static inline void timekeeping_inc_mg_floor_swaps(void)
+{
+}
+
#endif
-#ifdef CONFIG_CLOCKSOURCE_VALIDATE_LAST_CYCLE
static inline u64 clocksource_delta(u64 now, u64 last, u64 mask)
{
u64 ret = (now - last) & mask;
@@ -26,14 +40,9 @@ static inline u64 clocksource_delta(u64 now, u64 last, u64 mask)
*/
return ret & ~(mask >> 1) ? 0 : ret;
}
-#else
-static inline u64 clocksource_delta(u64 now, u64 last, u64 mask)
-{
- return (now - last) & mask;
-}
-#endif
/* Semi public for serialization of non timekeeper VDSO updates. */
-extern raw_spinlock_t timekeeper_lock;
+unsigned long timekeeper_lock_irqsave(void);
+void timekeeper_unlock_irqrestore(unsigned long flags);
#endif /* _TIMEKEEPING_INTERNAL_H */
diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index 0fc9d066a7be..a283e524835d 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -37,7 +37,6 @@
#include <linux/tick.h>
#include <linux/kallsyms.h>
#include <linux/irq_work.h>
-#include <linux/sched/signal.h>
#include <linux/sched/sysctl.h>
#include <linux/sched/nohz.h>
#include <linux/sched/debug.h>
@@ -2422,7 +2421,8 @@ static inline void __run_timers(struct timer_base *base)
static void __run_timer_base(struct timer_base *base)
{
- if (time_before(jiffies, base->next_expiry))
+ /* Can race against a remote CPU updating next_expiry under the lock */
+ if (time_before(jiffies, READ_ONCE(base->next_expiry)))
return;
timer_base_lock_expiry(base);
@@ -2526,141 +2526,6 @@ void update_process_times(int user_tick)
run_posix_cpu_timers();
}
-/*
- * Since schedule_timeout()'s timer is defined on the stack, it must store
- * the target task on the stack as well.
- */
-struct process_timer {
- struct timer_list timer;
- struct task_struct *task;
-};
-
-static void process_timeout(struct timer_list *t)
-{
- struct process_timer *timeout = from_timer(timeout, t, timer);
-
- wake_up_process(timeout->task);
-}
-
-/**
- * schedule_timeout - sleep until timeout
- * @timeout: timeout value in jiffies
- *
- * Make the current task sleep until @timeout jiffies have elapsed.
- * The function behavior depends on the current task state
- * (see also set_current_state() description):
- *
- * %TASK_RUNNING - the scheduler is called, but the task does not sleep
- * at all. That happens because sched_submit_work() does nothing for
- * tasks in %TASK_RUNNING state.
- *
- * %TASK_UNINTERRUPTIBLE - at least @timeout jiffies are guaranteed to
- * pass before the routine returns unless the current task is explicitly
- * woken up, (e.g. by wake_up_process()).
- *
- * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
- * delivered to the current task or the current task is explicitly woken
- * up.
- *
- * The current task state is guaranteed to be %TASK_RUNNING when this
- * routine returns.
- *
- * Specifying a @timeout value of %MAX_SCHEDULE_TIMEOUT will schedule
- * the CPU away without a bound on the timeout. In this case the return
- * value will be %MAX_SCHEDULE_TIMEOUT.
- *
- * Returns 0 when the timer has expired otherwise the remaining time in
- * jiffies will be returned. In all cases the return value is guaranteed
- * to be non-negative.
- */
-signed long __sched schedule_timeout(signed long timeout)
-{
- struct process_timer timer;
- unsigned long expire;
-
- switch (timeout)
- {
- case MAX_SCHEDULE_TIMEOUT:
- /*
- * These two special cases are useful to be comfortable
- * in the caller. Nothing more. We could take
- * MAX_SCHEDULE_TIMEOUT from one of the negative value
- * but I' d like to return a valid offset (>=0) to allow
- * the caller to do everything it want with the retval.
- */
- schedule();
- goto out;
- default:
- /*
- * Another bit of PARANOID. Note that the retval will be
- * 0 since no piece of kernel is supposed to do a check
- * for a negative retval of schedule_timeout() (since it
- * should never happens anyway). You just have the printk()
- * that will tell you if something is gone wrong and where.
- */
- if (timeout < 0) {
- printk(KERN_ERR "schedule_timeout: wrong timeout "
- "value %lx\n", timeout);
- dump_stack();
- __set_current_state(TASK_RUNNING);
- goto out;
- }
- }
-
- expire = timeout + jiffies;
-
- timer.task = current;
- timer_setup_on_stack(&timer.timer, process_timeout, 0);
- __mod_timer(&timer.timer, expire, MOD_TIMER_NOTPENDING);
- schedule();
- del_timer_sync(&timer.timer);
-
- /* Remove the timer from the object tracker */
- destroy_timer_on_stack(&timer.timer);
-
- timeout = expire - jiffies;
-
- out:
- return timeout < 0 ? 0 : timeout;
-}
-EXPORT_SYMBOL(schedule_timeout);
-
-/*
- * We can use __set_current_state() here because schedule_timeout() calls
- * schedule() unconditionally.
- */
-signed long __sched schedule_timeout_interruptible(signed long timeout)
-{
- __set_current_state(TASK_INTERRUPTIBLE);
- return schedule_timeout(timeout);
-}
-EXPORT_SYMBOL(schedule_timeout_interruptible);
-
-signed long __sched schedule_timeout_killable(signed long timeout)
-{
- __set_current_state(TASK_KILLABLE);
- return schedule_timeout(timeout);
-}
-EXPORT_SYMBOL(schedule_timeout_killable);
-
-signed long __sched schedule_timeout_uninterruptible(signed long timeout)
-{
- __set_current_state(TASK_UNINTERRUPTIBLE);
- return schedule_timeout(timeout);
-}
-EXPORT_SYMBOL(schedule_timeout_uninterruptible);
-
-/*
- * Like schedule_timeout_uninterruptible(), except this task will not contribute
- * to load average.
- */
-signed long __sched schedule_timeout_idle(signed long timeout)
-{
- __set_current_state(TASK_IDLE);
- return schedule_timeout(timeout);
-}
-EXPORT_SYMBOL(schedule_timeout_idle);
-
#ifdef CONFIG_HOTPLUG_CPU
static void migrate_timer_list(struct timer_base *new_base, struct hlist_head *head)
{
@@ -2757,59 +2622,3 @@ void __init init_timers(void)
posix_cputimers_init_work();
open_softirq(TIMER_SOFTIRQ, run_timer_softirq);
}
-
-/**
- * msleep - sleep safely even with waitqueue interruptions
- * @msecs: Time in milliseconds to sleep for
- */
-void msleep(unsigned int msecs)
-{
- unsigned long timeout = msecs_to_jiffies(msecs);
-
- while (timeout)
- timeout = schedule_timeout_uninterruptible(timeout);
-}
-
-EXPORT_SYMBOL(msleep);
-
-/**
- * msleep_interruptible - sleep waiting for signals
- * @msecs: Time in milliseconds to sleep for
- */
-unsigned long msleep_interruptible(unsigned int msecs)
-{
- unsigned long timeout = msecs_to_jiffies(msecs);
-
- while (timeout && !signal_pending(current))
- timeout = schedule_timeout_interruptible(timeout);
- return jiffies_to_msecs(timeout);
-}
-
-EXPORT_SYMBOL(msleep_interruptible);
-
-/**
- * usleep_range_state - Sleep for an approximate time in a given state
- * @min: Minimum time in usecs to sleep
- * @max: Maximum time in usecs to sleep
- * @state: State of the current task that will be while sleeping
- *
- * In non-atomic context where the exact wakeup time is flexible, use
- * usleep_range_state() instead of udelay(). The sleep improves responsiveness
- * by avoiding the CPU-hogging busy-wait of udelay(), and the range reduces
- * power usage by allowing hrtimers to take advantage of an already-
- * scheduled interrupt instead of scheduling a new one just for this sleep.
- */
-void __sched usleep_range_state(unsigned long min, unsigned long max,
- unsigned int state)
-{
- ktime_t exp = ktime_add_us(ktime_get(), min);
- u64 delta = (u64)(max - min) * NSEC_PER_USEC;
-
- for (;;) {
- __set_current_state(state);
- /* Do not return before the requested sleep time has elapsed */
- if (!schedule_hrtimeout_range(&exp, delta, HRTIMER_MODE_ABS))
- break;
- }
-}
-EXPORT_SYMBOL(usleep_range_state);
diff --git a/kernel/time/vsyscall.c b/kernel/time/vsyscall.c
index 9193d6133e5d..98488b20b594 100644
--- a/kernel/time/vsyscall.c
+++ b/kernel/time/vsyscall.c
@@ -151,9 +151,8 @@ void update_vsyscall_tz(void)
unsigned long vdso_update_begin(void)
{
struct vdso_data *vdata = __arch_get_k_vdso_data();
- unsigned long flags;
+ unsigned long flags = timekeeper_lock_irqsave();
- raw_spin_lock_irqsave(&timekeeper_lock, flags);
vdso_write_begin(vdata);
return flags;
}
@@ -172,5 +171,5 @@ void vdso_update_end(unsigned long flags)
vdso_write_end(vdata);
__arch_sync_vdso_data(vdata);
- raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
+ timekeeper_unlock_irqrestore(flags);
}
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 7315f643817a..14977b9fc254 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1328,19 +1328,6 @@ config SCHEDSTATS
endmenu
-config DEBUG_TIMEKEEPING
- bool "Enable extra timekeeping sanity checking"
- help
- This option will enable additional timekeeping sanity checks
- which may be helpful when diagnosing issues where timekeeping
- problems are suspected.
-
- This may include checks in the timekeeping hotpaths, so this
- option may have a (very small) performance impact to some
- workloads.
-
- If unsure, say N.
-
config DEBUG_PREEMPT
bool "Debug preemptible kernel"
depends on DEBUG_KERNEL && PREEMPTION && TRACE_IRQFLAGS_SUPPORT
diff --git a/mm/damon/core.c b/mm/damon/core.c
index a83f3b736d51..79efd8089d6c 100644
--- a/mm/damon/core.c
+++ b/mm/damon/core.c
@@ -1892,11 +1892,10 @@ static unsigned long damos_wmark_wait_us(struct damos *scheme)
static void kdamond_usleep(unsigned long usecs)
{
- /* See Documentation/timers/timers-howto.rst for the thresholds */
- if (usecs > 20 * USEC_PER_MSEC)
+ if (usecs >= USLEEP_RANGE_UPPER_BOUND)
schedule_timeout_idle(usecs_to_jiffies(usecs));
else
- usleep_idle_range(usecs, usecs + 1);
+ usleep_range_idle(usecs, usecs + 1);
}
/* Returns negative error code if it's not activated but should return */
diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c
index 1c82dcdf6e8f..4bd94d432bcf 100644
--- a/net/bluetooth/hci_event.c
+++ b/net/bluetooth/hci_event.c
@@ -42,8 +42,6 @@
#define ZERO_KEY "\x00\x00\x00\x00\x00\x00\x00\x00" \
"\x00\x00\x00\x00\x00\x00\x00\x00"
-#define secs_to_jiffies(_secs) msecs_to_jiffies((_secs) * 1000)
-
/* Handle HCI Event packets */
static void *hci_ev_skb_pull(struct hci_dev *hdev, struct sk_buff *skb,
diff --git a/net/core/pktgen.c b/net/core/pktgen.c
index 34f68ef74b8f..7e23cacbe66e 100644
--- a/net/core/pktgen.c
+++ b/net/core/pktgen.c
@@ -2285,7 +2285,7 @@ static void spin(struct pktgen_dev *pkt_dev, ktime_t spin_until)
s64 remaining;
struct hrtimer_sleeper t;
- hrtimer_init_sleeper_on_stack(&t, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
+ hrtimer_setup_sleeper_on_stack(&t, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
hrtimer_set_expires(&t.timer, spin_until);
remaining = ktime_to_ns(hrtimer_expires_remaining(&t.timer));
diff --git a/net/netfilter/xt_IDLETIMER.c b/net/netfilter/xt_IDLETIMER.c
index db720efa811d..5514600586a9 100644
--- a/net/netfilter/xt_IDLETIMER.c
+++ b/net/netfilter/xt_IDLETIMER.c
@@ -107,14 +107,12 @@ static void idletimer_tg_expired(struct timer_list *t)
schedule_work(&timer->work);
}
-static enum alarmtimer_restart idletimer_tg_alarmproc(struct alarm *alarm,
- ktime_t now)
+static void idletimer_tg_alarmproc(struct alarm *alarm, ktime_t now)
{
struct idletimer_tg *timer = alarm->data;
pr_debug("alarm %s expired\n", timer->attr.attr.name);
schedule_work(&timer->work);
- return ALARMTIMER_NORESTART;
}
static int idletimer_check_sysfs_name(const char *name, unsigned int size)
diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl
index 4427572b2477..98790fe5115d 100755
--- a/scripts/checkpatch.pl
+++ b/scripts/checkpatch.pl
@@ -6597,11 +6597,11 @@ sub process {
# ignore udelay's < 10, however
if (! ($delay < 10) ) {
CHK("USLEEP_RANGE",
- "usleep_range is preferred over udelay; see Documentation/timers/timers-howto.rst\n" . $herecurr);
+ "usleep_range is preferred over udelay; see function description of usleep_range() and udelay().\n" . $herecurr);
}
if ($delay > 2000) {
WARN("LONG_UDELAY",
- "long udelay - prefer mdelay; see arch/arm/include/asm/delay.h\n" . $herecurr);
+ "long udelay - prefer mdelay; see function description of mdelay().\n" . $herecurr);
}
}
@@ -6609,7 +6609,7 @@ sub process {
if ($line =~ /\bmsleep\s*\((\d+)\);/) {
if ($1 < 20) {
WARN("MSLEEP",
- "msleep < 20ms can sleep for up to 20ms; see Documentation/timers/timers-howto.rst\n" . $herecurr);
+ "msleep < 20ms can sleep for up to 20ms; see function description of msleep().\n" . $herecurr);
}
}
@@ -7077,11 +7077,11 @@ sub process {
my $max = $7;
if ($min eq $max) {
WARN("USLEEP_RANGE",
- "usleep_range should not use min == max args; see Documentation/timers/timers-howto.rst\n" . "$here\n$stat\n");
+ "usleep_range should not use min == max args; see function description of usleep_range().\n" . "$here\n$stat\n");
} elsif ($min =~ /^\d+$/ && $max =~ /^\d+$/ &&
$min > $max) {
WARN("USLEEP_RANGE",
- "usleep_range args reversed, use min then max; see Documentation/timers/timers-howto.rst\n" . "$here\n$stat\n");
+ "usleep_range args reversed, use min then max; see function description of usleep_range().\n" . "$here\n$stat\n");
}
}
diff --git a/sound/soc/sof/ops.h b/sound/soc/sof/ops.h
index 2584621c3b2d..d73644e85b6e 100644
--- a/sound/soc/sof/ops.h
+++ b/sound/soc/sof/ops.h
@@ -597,12 +597,12 @@ snd_sof_is_chain_dma_supported(struct snd_sof_dev *sdev, u32 dai_type)
* @addr: Address to poll
* @val: Variable to read the value into
* @cond: Break condition (usually involving @val)
- * @sleep_us: Maximum time to sleep between reads in us (0
- * tight-loops). Should be less than ~20ms since usleep_range
- * is used (see Documentation/timers/timers-howto.rst).
+ * @sleep_us: Maximum time to sleep between reads in us (0 tight-loops). Please
+ * read usleep_range() function description for details and
+ * limitations.
* @timeout_us: Timeout in us, 0 means never timeout
*
- * Returns 0 on success and -ETIMEDOUT upon a timeout. In either
+ * Returns: 0 on success and -ETIMEDOUT upon a timeout. In either
* case, the last read value at @addr is stored in @val. Must not
* be called from atomic context if sleep_us or timeout_us are used.
*
diff --git a/tools/testing/selftests/wireguard/qemu/debug.config b/tools/testing/selftests/wireguard/qemu/debug.config
index 9d172210e2c6..139fd9aa8b12 100644
--- a/tools/testing/selftests/wireguard/qemu/debug.config
+++ b/tools/testing/selftests/wireguard/qemu/debug.config
@@ -31,7 +31,6 @@ CONFIG_SCHED_DEBUG=y
CONFIG_SCHED_INFO=y
CONFIG_SCHEDSTATS=y
CONFIG_SCHED_STACK_END_CHECK=y
-CONFIG_DEBUG_TIMEKEEPING=y
CONFIG_DEBUG_PREEMPT=y
CONFIG_DEBUG_RT_MUTEXES=y
CONFIG_DEBUG_SPINLOCK=yThe pull request you sent on Mon, 18 Nov 2024 20:21:57 +0100 (CET): > git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git timers-core-2024-11-18 has been merged into torvalds/linux.git: https://git.kernel.org/torvalds/c/bf9aa14fc523d2763fc9a10672a709224e8fcaf4 Thank you! -- Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/prtracker.html
On Mon, 18 Nov 2024 at 11:22, Thomas Gleixner <tglx@linutronix.de> wrote:
>
> - Core infrastructure for VFS multigrain timestamping
>
> This is required to allow the kernel to use coarse grained time stamps
> by default and switch to fine grained time stamps when inode attributes
> are actively observed via getattr().
>
> These changes have been provided to the VFS tree as well, so that the
> VFS specific infrastructure could be built on top.
Bah. Except the vfs tree didn't take it as a shared branch, but
instead cherry-picked the commits and as a result they are duplicate
and caused a (trivial) merge conflict.
Oh well.
Linus
On Tue, Nov 19, 2024 at 04:33:45PM -0800, Linus Torvalds wrote:
> On Mon, 18 Nov 2024 at 11:22, Thomas Gleixner <tglx@linutronix.de> wrote:
> >
>
> > - Core infrastructure for VFS multigrain timestamping
> >
> > This is required to allow the kernel to use coarse grained time stamps
> > by default and switch to fine grained time stamps when inode attributes
> > are actively observed via getattr().
> >
> > These changes have been provided to the VFS tree as well, so that the
> > VFS specific infrastructure could be built on top.
>
> Bah. Except the vfs tree didn't take it as a shared branch, but
> instead cherry-picked the commits and as a result they are duplicate
> and caused a (trivial) merge conflict.
Wait, I'm confused. I definitely pulled that branch the day after Thomas
gave it to me and in my vfs.mgtime branch I clearly see:
commit d7c898a73f875bd205df53074c1d542766171da1
Merge: 8cf0b93919e1 2a15385742c6
Author: Christian Brauner <brauner@kernel.org>
AuthorDate: Mon Oct 7 12:47:19 2024 +0200
Commit: Christian Brauner <brauner@kernel.org>
CommitDate: Thu Oct 10 10:20:57 2024 +0200
Merge tag 'timers-core-for-vfs' of ssh://gitolite.kernel.org/pub/scm/linux/kernel/git/tip/tip into vfs.mgtime
Timekeeping interfaces for consumption by the VFS tree.
Signed-off-by: Christian Brauner <brauner@kernel.org>
Unless I did something odd during the pull?
* Christian Brauner <brauner@kernel.org> wrote:
> On Tue, Nov 19, 2024 at 04:33:45PM -0800, Linus Torvalds wrote:
> > On Mon, 18 Nov 2024 at 11:22, Thomas Gleixner <tglx@linutronix.de> wrote:
> > >
> >
> > > - Core infrastructure for VFS multigrain timestamping
> > >
> > > This is required to allow the kernel to use coarse grained time stamps
> > > by default and switch to fine grained time stamps when inode attributes
> > > are actively observed via getattr().
> > >
> > > These changes have been provided to the VFS tree as well, so that the
> > > VFS specific infrastructure could be built on top.
> >
> > Bah. Except the vfs tree didn't take it as a shared branch, but
> > instead cherry-picked the commits and as a result they are duplicate
> > and caused a (trivial) merge conflict.
>
> Wait, I'm confused. I definitely pulled that branch the day after Thomas
> gave it to me and in my vfs.mgtime branch I clearly see:
>
> commit d7c898a73f875bd205df53074c1d542766171da1
> Merge: 8cf0b93919e1 2a15385742c6
> Author: Christian Brauner <brauner@kernel.org>
> AuthorDate: Mon Oct 7 12:47:19 2024 +0200
> Commit: Christian Brauner <brauner@kernel.org>
> CommitDate: Thu Oct 10 10:20:57 2024 +0200
>
> Merge tag 'timers-core-for-vfs' of ssh://gitolite.kernel.org/pub/scm/linux/kernel/git/tip/tip into vfs.mgtime
>
> Timekeeping interfaces for consumption by the VFS tree.
>
> Signed-off-by: Christian Brauner <brauner@kernel.org>
>
> Unless I did something odd during the pull?
The problem was caused by two commits which got rebased in the VFS
tree:
Commit 1:
ee3283c608df ("timekeeping: Add interfaces for handling timestamps with a floor value")
commit ee3283c608dfa21251b0821d7bb198c7ae3189f6
Author: Jeff Layton <jlayton@kernel.org>
AuthorDate: Wed Oct 2 17:27:16 2024 -0400
Commit: Christian Brauner <brauner@kernel.org>
CommitDate: Thu Oct 10 10:20:46 2024 +0200
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Which is a rebase of Thomas's original commit:
70c8fd00a9bd ("timekeeping: Add interfaces for handling timestamps with a floor value")
commit 70c8fd00a9bd0509bbf7bccd9baea8bbd5ddc756
Author: Jeff Layton <jlayton@kernel.org>
AuthorDate: Wed Oct 2 17:27:16 2024 -0400
Commit: Thomas Gleixner <tglx@linutronix.de>
CommitDate: Sun Oct 6 20:56:07 2024 +0200
And commit 2:
2a15385742c6 ("timekeeping: Add percpu counter for tracking floor swap events")
commit 2a15385742c689a271345dcbb4c28b9c568bc7ce
Author: Jeff Layton <jlayton@kernel.org>
AuthorDate: Wed Oct 2 17:27:17 2024 -0400
Commit: Christian Brauner <brauner@kernel.org>
CommitDate: Thu Oct 10 10:20:46 2024 +0200
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Which is a rebase of Thomas's original commit:
96f9a366ec8a timekeeping: Add percpu counter for tracking floor swap events
commit 96f9a366ec8abe027326d7aab84d64370019f0f1 (tag: timers-core-for-vfs)
Author: Jeff Layton <jlayton@kernel.org>
AuthorDate: Wed Oct 2 17:27:17 2024 -0400
Commit: Thomas Gleixner <tglx@linutronix.de>
CommitDate: Sun Oct 6 20:56:07 2024 +0200
Thanks,
Ingo
On Wed, Nov 20, 2024 at 01:12:28PM +0100, Ingo Molnar wrote:
>
> * Christian Brauner <brauner@kernel.org> wrote:
>
> > On Tue, Nov 19, 2024 at 04:33:45PM -0800, Linus Torvalds wrote:
> > > On Mon, 18 Nov 2024 at 11:22, Thomas Gleixner <tglx@linutronix.de> wrote:
> > > >
> > >
> > > > - Core infrastructure for VFS multigrain timestamping
> > > >
> > > > This is required to allow the kernel to use coarse grained time stamps
> > > > by default and switch to fine grained time stamps when inode attributes
> > > > are actively observed via getattr().
> > > >
> > > > These changes have been provided to the VFS tree as well, so that the
> > > > VFS specific infrastructure could be built on top.
> > >
> > > Bah. Except the vfs tree didn't take it as a shared branch, but
> > > instead cherry-picked the commits and as a result they are duplicate
> > > and caused a (trivial) merge conflict.
> >
> > Wait, I'm confused. I definitely pulled that branch the day after Thomas
> > gave it to me and in my vfs.mgtime branch I clearly see:
> >
> > commit d7c898a73f875bd205df53074c1d542766171da1
> > Merge: 8cf0b93919e1 2a15385742c6
> > Author: Christian Brauner <brauner@kernel.org>
> > AuthorDate: Mon Oct 7 12:47:19 2024 +0200
> > Commit: Christian Brauner <brauner@kernel.org>
> > CommitDate: Thu Oct 10 10:20:57 2024 +0200
> >
> > Merge tag 'timers-core-for-vfs' of ssh://gitolite.kernel.org/pub/scm/linux/kernel/git/tip/tip into vfs.mgtime
> >
> > Timekeeping interfaces for consumption by the VFS tree.
> >
> > Signed-off-by: Christian Brauner <brauner@kernel.org>
> >
> > Unless I did something odd during the pull?
>
> The problem was caused by two commits which got rebased in the VFS
> tree:
>
> Commit 1:
>
> ee3283c608df ("timekeeping: Add interfaces for handling timestamps with a floor value")
>
> commit ee3283c608dfa21251b0821d7bb198c7ae3189f6
> Author: Jeff Layton <jlayton@kernel.org>
> AuthorDate: Wed Oct 2 17:27:16 2024 -0400
> Commit: Christian Brauner <brauner@kernel.org>
> CommitDate: Thu Oct 10 10:20:46 2024 +0200
> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>
> Which is a rebase of Thomas's original commit:
>
> 70c8fd00a9bd ("timekeeping: Add interfaces for handling timestamps with a floor value")
>
> commit 70c8fd00a9bd0509bbf7bccd9baea8bbd5ddc756
> Author: Jeff Layton <jlayton@kernel.org>
> AuthorDate: Wed Oct 2 17:27:16 2024 -0400
> Commit: Thomas Gleixner <tglx@linutronix.de>
> CommitDate: Sun Oct 6 20:56:07 2024 +0200
>
> And commit 2:
>
> 2a15385742c6 ("timekeeping: Add percpu counter for tracking floor swap events")
>
> commit 2a15385742c689a271345dcbb4c28b9c568bc7ce
> Author: Jeff Layton <jlayton@kernel.org>
> AuthorDate: Wed Oct 2 17:27:17 2024 -0400
> Commit: Christian Brauner <brauner@kernel.org>
> CommitDate: Thu Oct 10 10:20:46 2024 +0200
> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>
> Which is a rebase of Thomas's original commit:
>
> 96f9a366ec8a timekeeping: Add percpu counter for tracking floor swap events
>
> commit 96f9a366ec8abe027326d7aab84d64370019f0f1 (tag: timers-core-for-vfs)
> Author: Jeff Layton <jlayton@kernel.org>
> AuthorDate: Wed Oct 2 17:27:17 2024 -0400
> Commit: Thomas Gleixner <tglx@linutronix.de>
> CommitDate: Sun Oct 6 20:56:07 2024 +0200
I was just looking through my reflog and I realised that I did a rebase
onto v6.12-rc2 (v6.12-rc1 had a horribly virtqueue bug that made testing
in a vm a giant pain). Sorry about that I should've noticed that
earlier.
Though I'm confused why -next didn't catch this. When that happens -next
usually reports duplicate commits. Hm... Did I miss the report?
9fed2c0f2f07 (HEAD -> vfs.mgtime, tag: vfs-6.13.mgtime, origin-all/vfs.mgtime) HEAD@{0}: am: fs: reduce pointer chasing in is_mgtime() test
b40508ca5d5c HEAD@{1}: rebase (finish): returning to refs/heads/vfs.mgtime
b40508ca5d5c HEAD@{2}: rebase (reset): 'onto': Merge patch series "timekeeping/fs: multigrain timestamp redux"
d7c898a73f87 HEAD@{3}: rebase (reset): 'onto': Merge tag 'timers-core-for-vfs' of ssh://gitolite.kernel.org/pub/scm/linux/kernel/git/tip/tip into vfs.mgtime
8cf0b93919e1 (tag: v6.12-rc2, master.v6.12-rc2) HEAD@{4}: rebase (reset): 'onto'
* Christian Brauner <brauner@kernel.org> wrote:
> On Wed, Nov 20, 2024 at 01:12:28PM +0100, Ingo Molnar wrote:
> >
> > * Christian Brauner <brauner@kernel.org> wrote:
> >
> > > On Tue, Nov 19, 2024 at 04:33:45PM -0800, Linus Torvalds wrote:
> > > > On Mon, 18 Nov 2024 at 11:22, Thomas Gleixner <tglx@linutronix.de> wrote:
> > > > >
> > > >
> > > > > - Core infrastructure for VFS multigrain timestamping
> > > > >
> > > > > This is required to allow the kernel to use coarse grained time stamps
> > > > > by default and switch to fine grained time stamps when inode attributes
> > > > > are actively observed via getattr().
> > > > >
> > > > > These changes have been provided to the VFS tree as well, so that the
> > > > > VFS specific infrastructure could be built on top.
> > > >
> > > > Bah. Except the vfs tree didn't take it as a shared branch, but
> > > > instead cherry-picked the commits and as a result they are duplicate
> > > > and caused a (trivial) merge conflict.
> > >
> > > Wait, I'm confused. I definitely pulled that branch the day after Thomas
> > > gave it to me and in my vfs.mgtime branch I clearly see:
> > >
> > > commit d7c898a73f875bd205df53074c1d542766171da1
> > > Merge: 8cf0b93919e1 2a15385742c6
> > > Author: Christian Brauner <brauner@kernel.org>
> > > AuthorDate: Mon Oct 7 12:47:19 2024 +0200
> > > Commit: Christian Brauner <brauner@kernel.org>
> > > CommitDate: Thu Oct 10 10:20:57 2024 +0200
> > >
> > > Merge tag 'timers-core-for-vfs' of ssh://gitolite.kernel.org/pub/scm/linux/kernel/git/tip/tip into vfs.mgtime
> > >
> > > Timekeeping interfaces for consumption by the VFS tree.
> > >
> > > Signed-off-by: Christian Brauner <brauner@kernel.org>
> > >
> > > Unless I did something odd during the pull?
> >
> > The problem was caused by two commits which got rebased in the VFS
> > tree:
> >
> > Commit 1:
> >
> > ee3283c608df ("timekeeping: Add interfaces for handling timestamps with a floor value")
> >
> > commit ee3283c608dfa21251b0821d7bb198c7ae3189f6
> > Author: Jeff Layton <jlayton@kernel.org>
> > AuthorDate: Wed Oct 2 17:27:16 2024 -0400
> > Commit: Christian Brauner <brauner@kernel.org>
> > CommitDate: Thu Oct 10 10:20:46 2024 +0200
> > ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> >
> > Which is a rebase of Thomas's original commit:
> >
> > 70c8fd00a9bd ("timekeeping: Add interfaces for handling timestamps with a floor value")
> >
> > commit 70c8fd00a9bd0509bbf7bccd9baea8bbd5ddc756
> > Author: Jeff Layton <jlayton@kernel.org>
> > AuthorDate: Wed Oct 2 17:27:16 2024 -0400
> > Commit: Thomas Gleixner <tglx@linutronix.de>
> > CommitDate: Sun Oct 6 20:56:07 2024 +0200
> >
> > And commit 2:
> >
> > 2a15385742c6 ("timekeeping: Add percpu counter for tracking floor swap events")
> >
> > commit 2a15385742c689a271345dcbb4c28b9c568bc7ce
> > Author: Jeff Layton <jlayton@kernel.org>
> > AuthorDate: Wed Oct 2 17:27:17 2024 -0400
> > Commit: Christian Brauner <brauner@kernel.org>
> > CommitDate: Thu Oct 10 10:20:46 2024 +0200
> > ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> >
> > Which is a rebase of Thomas's original commit:
> >
> > 96f9a366ec8a timekeeping: Add percpu counter for tracking floor swap events
> >
> > commit 96f9a366ec8abe027326d7aab84d64370019f0f1 (tag: timers-core-for-vfs)
> > Author: Jeff Layton <jlayton@kernel.org>
> > AuthorDate: Wed Oct 2 17:27:17 2024 -0400
> > Commit: Thomas Gleixner <tglx@linutronix.de>
> > CommitDate: Sun Oct 6 20:56:07 2024 +0200
>
> I was just looking through my reflog and I realised that I did a rebase
> onto v6.12-rc2 (v6.12-rc1 had a horribly virtqueue bug that made testing
> in a vm a giant pain). Sorry about that I should've noticed that
> earlier.
So, so sh1t happens, but I think there could also be a workflow bug
here: somehow your tooling added your Signed-off-by during the rebase,
to a commit not committed by you originally.
> Though I'm confused why -next didn't catch this. When that happens -next
> usually reports duplicate commits. Hm... Did I miss the report?
I haven't seen duplicate commit warnings from -next for some time - but
it can detect accidental rebases, because without your SOB added -next
would have detected the incorrect SOB chain I believe. So I'd
investigate how your SOB got there. I don't think vanilla rebase is
adding it automatically?
Thanks,
Ingo
On Wed, Nov 20, 2024 at 05:10:37PM +0100, Ingo Molnar wrote:
>
> * Christian Brauner <brauner@kernel.org> wrote:
>
> > On Wed, Nov 20, 2024 at 01:12:28PM +0100, Ingo Molnar wrote:
> > >
> > > * Christian Brauner <brauner@kernel.org> wrote:
> > >
> > > > On Tue, Nov 19, 2024 at 04:33:45PM -0800, Linus Torvalds wrote:
> > > > > On Mon, 18 Nov 2024 at 11:22, Thomas Gleixner <tglx@linutronix.de> wrote:
> > > > > >
> > > > >
> > > > > > - Core infrastructure for VFS multigrain timestamping
> > > > > >
> > > > > > This is required to allow the kernel to use coarse grained time stamps
> > > > > > by default and switch to fine grained time stamps when inode attributes
> > > > > > are actively observed via getattr().
> > > > > >
> > > > > > These changes have been provided to the VFS tree as well, so that the
> > > > > > VFS specific infrastructure could be built on top.
> > > > >
> > > > > Bah. Except the vfs tree didn't take it as a shared branch, but
> > > > > instead cherry-picked the commits and as a result they are duplicate
> > > > > and caused a (trivial) merge conflict.
> > > >
> > > > Wait, I'm confused. I definitely pulled that branch the day after Thomas
> > > > gave it to me and in my vfs.mgtime branch I clearly see:
> > > >
> > > > commit d7c898a73f875bd205df53074c1d542766171da1
> > > > Merge: 8cf0b93919e1 2a15385742c6
> > > > Author: Christian Brauner <brauner@kernel.org>
> > > > AuthorDate: Mon Oct 7 12:47:19 2024 +0200
> > > > Commit: Christian Brauner <brauner@kernel.org>
> > > > CommitDate: Thu Oct 10 10:20:57 2024 +0200
> > > >
> > > > Merge tag 'timers-core-for-vfs' of ssh://gitolite.kernel.org/pub/scm/linux/kernel/git/tip/tip into vfs.mgtime
> > > >
> > > > Timekeeping interfaces for consumption by the VFS tree.
> > > >
> > > > Signed-off-by: Christian Brauner <brauner@kernel.org>
> > > >
> > > > Unless I did something odd during the pull?
> > >
> > > The problem was caused by two commits which got rebased in the VFS
> > > tree:
> > >
> > > Commit 1:
> > >
> > > ee3283c608df ("timekeeping: Add interfaces for handling timestamps with a floor value")
> > >
> > > commit ee3283c608dfa21251b0821d7bb198c7ae3189f6
> > > Author: Jeff Layton <jlayton@kernel.org>
> > > AuthorDate: Wed Oct 2 17:27:16 2024 -0400
> > > Commit: Christian Brauner <brauner@kernel.org>
> > > CommitDate: Thu Oct 10 10:20:46 2024 +0200
> > > ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> > >
> > > Which is a rebase of Thomas's original commit:
> > >
> > > 70c8fd00a9bd ("timekeeping: Add interfaces for handling timestamps with a floor value")
> > >
> > > commit 70c8fd00a9bd0509bbf7bccd9baea8bbd5ddc756
> > > Author: Jeff Layton <jlayton@kernel.org>
> > > AuthorDate: Wed Oct 2 17:27:16 2024 -0400
> > > Commit: Thomas Gleixner <tglx@linutronix.de>
> > > CommitDate: Sun Oct 6 20:56:07 2024 +0200
> > >
> > > And commit 2:
> > >
> > > 2a15385742c6 ("timekeeping: Add percpu counter for tracking floor swap events")
> > >
> > > commit 2a15385742c689a271345dcbb4c28b9c568bc7ce
> > > Author: Jeff Layton <jlayton@kernel.org>
> > > AuthorDate: Wed Oct 2 17:27:17 2024 -0400
> > > Commit: Christian Brauner <brauner@kernel.org>
> > > CommitDate: Thu Oct 10 10:20:46 2024 +0200
> > > ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> > >
> > > Which is a rebase of Thomas's original commit:
> > >
> > > 96f9a366ec8a timekeeping: Add percpu counter for tracking floor swap events
> > >
> > > commit 96f9a366ec8abe027326d7aab84d64370019f0f1 (tag: timers-core-for-vfs)
> > > Author: Jeff Layton <jlayton@kernel.org>
> > > AuthorDate: Wed Oct 2 17:27:17 2024 -0400
> > > Commit: Thomas Gleixner <tglx@linutronix.de>
> > > CommitDate: Sun Oct 6 20:56:07 2024 +0200
> >
> > I was just looking through my reflog and I realised that I did a rebase
> > onto v6.12-rc2 (v6.12-rc1 had a horribly virtqueue bug that made testing
> > in a vm a giant pain). Sorry about that I should've noticed that
> > earlier.
>
> So, so sh1t happens, but I think there could also be a workflow bug
> here: somehow your tooling added your Signed-off-by during the rebase,
> to a commit not committed by you originally.
I see. I assume that happens because I have an ancient
.git/hooks/prepare-commit-msg which contains:
COMMIT_MSG_FILE=$1
COMMIT_SOURCE=$2
SOB=$(git var GIT_COMMITTER_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p')
git interpret-trailers --in-place --trailer "$SOB" "$COMMIT_MSG_FILE"
I honestly have no idea anymore why I added that hook originally...
>
> > Though I'm confused why -next didn't catch this. When that happens -next
> > usually reports duplicate commits. Hm... Did I miss the report?
>
> I haven't seen duplicate commit warnings from -next for some time - but
> it can detect accidental rebases, because without your SOB added -next
> would have detected the incorrect SOB chain I believe. So I'd
> investigate how your SOB got there. I don't think vanilla rebase is
> adding it automatically?
Yes, vanilla rebase doesn't do this.
On Wed, Nov 20, 2024 at 08:44:39PM +0100, Christian Brauner wrote:
> On Wed, Nov 20, 2024 at 05:10:37PM +0100, Ingo Molnar wrote:
> >
> > * Christian Brauner <brauner@kernel.org> wrote:
> >
> > > On Wed, Nov 20, 2024 at 01:12:28PM +0100, Ingo Molnar wrote:
> > > >
> > > > * Christian Brauner <brauner@kernel.org> wrote:
> > > >
> > > > > On Tue, Nov 19, 2024 at 04:33:45PM -0800, Linus Torvalds wrote:
> > > > > > On Mon, 18 Nov 2024 at 11:22, Thomas Gleixner <tglx@linutronix.de> wrote:
> > > > > > >
> > > > > >
> > > > > > > - Core infrastructure for VFS multigrain timestamping
> > > > > > >
> > > > > > > This is required to allow the kernel to use coarse grained time stamps
> > > > > > > by default and switch to fine grained time stamps when inode attributes
> > > > > > > are actively observed via getattr().
> > > > > > >
> > > > > > > These changes have been provided to the VFS tree as well, so that the
> > > > > > > VFS specific infrastructure could be built on top.
> > > > > >
> > > > > > Bah. Except the vfs tree didn't take it as a shared branch, but
> > > > > > instead cherry-picked the commits and as a result they are duplicate
> > > > > > and caused a (trivial) merge conflict.
> > > > >
> > > > > Wait, I'm confused. I definitely pulled that branch the day after Thomas
> > > > > gave it to me and in my vfs.mgtime branch I clearly see:
> > > > >
> > > > > commit d7c898a73f875bd205df53074c1d542766171da1
> > > > > Merge: 8cf0b93919e1 2a15385742c6
> > > > > Author: Christian Brauner <brauner@kernel.org>
> > > > > AuthorDate: Mon Oct 7 12:47:19 2024 +0200
> > > > > Commit: Christian Brauner <brauner@kernel.org>
> > > > > CommitDate: Thu Oct 10 10:20:57 2024 +0200
> > > > >
> > > > > Merge tag 'timers-core-for-vfs' of ssh://gitolite.kernel.org/pub/scm/linux/kernel/git/tip/tip into vfs.mgtime
> > > > >
> > > > > Timekeeping interfaces for consumption by the VFS tree.
> > > > >
> > > > > Signed-off-by: Christian Brauner <brauner@kernel.org>
> > > > >
> > > > > Unless I did something odd during the pull?
> > > >
> > > > The problem was caused by two commits which got rebased in the VFS
> > > > tree:
> > > >
> > > > Commit 1:
> > > >
> > > > ee3283c608df ("timekeeping: Add interfaces for handling timestamps with a floor value")
> > > >
> > > > commit ee3283c608dfa21251b0821d7bb198c7ae3189f6
> > > > Author: Jeff Layton <jlayton@kernel.org>
> > > > AuthorDate: Wed Oct 2 17:27:16 2024 -0400
> > > > Commit: Christian Brauner <brauner@kernel.org>
> > > > CommitDate: Thu Oct 10 10:20:46 2024 +0200
> > > > ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> > > >
> > > > Which is a rebase of Thomas's original commit:
> > > >
> > > > 70c8fd00a9bd ("timekeeping: Add interfaces for handling timestamps with a floor value")
> > > >
> > > > commit 70c8fd00a9bd0509bbf7bccd9baea8bbd5ddc756
> > > > Author: Jeff Layton <jlayton@kernel.org>
> > > > AuthorDate: Wed Oct 2 17:27:16 2024 -0400
> > > > Commit: Thomas Gleixner <tglx@linutronix.de>
> > > > CommitDate: Sun Oct 6 20:56:07 2024 +0200
> > > >
> > > > And commit 2:
> > > >
> > > > 2a15385742c6 ("timekeeping: Add percpu counter for tracking floor swap events")
> > > >
> > > > commit 2a15385742c689a271345dcbb4c28b9c568bc7ce
> > > > Author: Jeff Layton <jlayton@kernel.org>
> > > > AuthorDate: Wed Oct 2 17:27:17 2024 -0400
> > > > Commit: Christian Brauner <brauner@kernel.org>
> > > > CommitDate: Thu Oct 10 10:20:46 2024 +0200
> > > > ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> > > >
> > > > Which is a rebase of Thomas's original commit:
> > > >
> > > > 96f9a366ec8a timekeeping: Add percpu counter for tracking floor swap events
> > > >
> > > > commit 96f9a366ec8abe027326d7aab84d64370019f0f1 (tag: timers-core-for-vfs)
> > > > Author: Jeff Layton <jlayton@kernel.org>
> > > > AuthorDate: Wed Oct 2 17:27:17 2024 -0400
> > > > Commit: Thomas Gleixner <tglx@linutronix.de>
> > > > CommitDate: Sun Oct 6 20:56:07 2024 +0200
> > >
> > > I was just looking through my reflog and I realised that I did a rebase
> > > onto v6.12-rc2 (v6.12-rc1 had a horribly virtqueue bug that made testing
> > > in a vm a giant pain). Sorry about that I should've noticed that
> > > earlier.
> >
> > So, so sh1t happens, but I think there could also be a workflow bug
> > here: somehow your tooling added your Signed-off-by during the rebase,
> > to a commit not committed by you originally.
>
> I see. I assume that happens because I have an ancient
> .git/hooks/prepare-commit-msg which contains:
>
> COMMIT_MSG_FILE=$1
> COMMIT_SOURCE=$2
>
> SOB=$(git var GIT_COMMITTER_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p')
> git interpret-trailers --in-place --trailer "$SOB" "$COMMIT_MSG_FILE"
>
> I honestly have no idea anymore why I added that hook originally...
To bring this to an end: I removed the hook and tested that on rebase no
SoB will be added to the rebased patches. Thanks for pointing me in the
right direction Ingo!
And Thomas, sorry for the confusion and thanks for providing the tag. It
won't happen again.
On Tue, Nov 19, 2024 at 04:33:45PM -0800, Linus Torvalds wrote:
> Bah. Except the vfs tree didn't take it as a shared branch, but
> instead cherry-picked the commits and as a result they are duplicate
> and caused a (trivial) merge conflict.
Yeah, we were looking at that last night with tglx and he sent me a resolution
(see below) and looking at your tree now, it all looks correct.
commit 6a972393d45dbe3df1e57e2360661990d22c8369
Merge: 183ec5f26b2f cdc905d16b07
Author: Thomas Gleixner <tglx@linutronix.de>
Date: Tue Nov 19 20:58:25 2024 +0100
Merge branch 'timers/core' into tmp
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
diff --cc kernel/cpu.c
index c4aaf73dec9e,895f3287e3f3..6e34b52cb5ce
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@@ -1338,9 -1338,6 +1338,8 @@@ static int takedown_cpu(unsigned int cp
cpuhp_bp_sync_dead(cpu);
+ lockdep_cleanup_dead_cpu(cpu, idle_thread_get(cpu));
- tick_cleanup_dead_cpu(cpu);
+
/*
* Callbacks must be re-integrated right away to the RCU state machine.
* Otherwise an RCU callback could block a further teardown function
diff --cc kernel/time/timekeeping.c
index 18752983e834,d115adebc418..0ca85ff4fbb4
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@@ -445,11 -375,11 +377,11 @@@ static __always_inline u64 __ktime_get_
u64 now;
do {
- seq = raw_read_seqcount_latch(&tkf->seq);
+ seq = read_seqcount_latch(&tkf->seq);
tkr = tkf->base + (seq & 0x01);
now = ktime_to_ns(tkr->base);
- now += __timekeeping_get_ns(tkr);
+ now += timekeeping_get_ns(tkr);
- } while (raw_read_seqcount_latch_retry(&tkf->seq, seq));
+ } while (read_seqcount_latch_retry(&tkf->seq, seq));
return now;
}
--
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquette
* Borislav Petkov <bp@alien8.de> wrote:
> On Tue, Nov 19, 2024 at 04:33:45PM -0800, Linus Torvalds wrote:
>
> > Bah. Except the vfs tree didn't take it as a shared branch, but
> > instead cherry-picked the commits and as a result they are
> > duplicate and caused a (trivial) merge conflict.
>
> Yeah, we were looking at that last night with tglx and he sent me a
> resolution (see below) and looking at your tree now, it all looks
> correct.
So there were two conflicts: one caused by the VFS evil rebase, which
caused a duplication of two functions within timekeeping.c.
The other conflict was an interaction with the locking tree due to
overlapping changes, which I resolved in tip:core/merge a couple of
weeks ago for -next:
commit 8446247e3ddaf3b173ef96131793bab3f567bd96
Merge: 2d4cdd3ab714 183ec5f26b2f
Author: Ingo Molnar <mingo@kernel.org>
Date: Wed Nov 6 13:58:08 2024 +0100
Merge branch 'locking/core' into core/merge, to resolve conflict
Conflicts:
kernel/time/timekeeping.c
Signed-off-by: Ingo Molnar <mingo@kernel.org>
AFAICT we resolved it in the same way, the only difference is that
Linus added one more newline for readability.
So we are all good!
Thanks,
Ingo
Linus,
please pull the latest irq/core branch from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git irq-core-2024-11-18
up to: 1f181d1cda56: irqchip/riscv-aplic: Prevent crash when MSI domain is missing
A set of updates for the interrupt subsystem:
- Tree wide:
* Make nr_irqs static to the core code and provide accessor functions
to remove existing and prevent future aliasing problems with local
variables or function arguments of the same name.
- Core code:
* Prevent freeing an interrupt in the devres code which is not managed
by devres in the first place.
* Use seq_put_decimal_ull_width() for decimal values output in
/proc/interrupts which increases performance significantly as it
avoids parsing the format strings over and over.
* Optimize raising the timer and hrtimer soft interrupts by using the
'set bit only' variants instead of the combined version which checks
whether ksoftirqd should be woken up. The latter is a pointless
exercise as both soft interrupts are raised in the context of the
timer interrupt and therefore never wake up ksoftirqd.
* Delegate timer/hrtimer soft interrupt processing to a dedicated thread
on RT.
Timer and hrtimer soft interrupts are always processed in ksoftirqd
on RT enabled kernels. This can lead to high latencies when other
soft interrupts are delegated to ksoftirqd as well.
The separate thread allows to run them seperately under a RT
scheduling policy to reduce the latency overhead.
- Drivers:
* New drivers or extensions of existing drivers to support Renesas
RZ/V2H(P), Aspeed AST27XX, T-HEAD C900 and ATMEL sam9x7 interrupt
chips
* Support for multi-cluster GICs on MIPS.
MIPS CPUs can come with multiple CPU clusters, where each CPU cluster
has its own GIC (Generic Interrupt Controller). This requires to
access the GIC of a remote cluster through a redirect register block.
This is encapsulated into a set of helper functions to keep the
complexity out of the actual code paths which handle the GIC details.
* Support for encrypted guests in the ARM GICV3 ITS driver
The ITS page needs to be shared with the hypervisor and therefore
must be decrypted.
* Small cleanups and fixes all over the place
Thanks,
tglx
------------------>
Bart Van Assche (22):
genirq: Introduce irq_get_nr_irqs() and irq_set_nr_irqs()
ARM: Switch to irq_get_nr_irqs() / irq_set_nr_irqs()
LoongArch: Switch to irq_set_nr_irqs()
powerpc/cell: Switch to irq_get_nr_irqs()
s390/irq: Switch to irq_get_nr_irqs()
x86/acpi: Switch to irq_get_nr_irqs() and irq_set_nr_irqs()
hpet: Switch to irq_get_nr_irqs()
net: 3com: 3c59x: Switch to irq_get_nr_irqs()
net: hamradio: baycom_ser_fdx: Switch to irq_get_nr_irqs()
net: hamradio: scc: Switch to irq_get_nr_irqs()
scsi: aha152x: Switch to irq_get_nr_irqs()
serial: core: Switch to irq_get_nr_irqs()
serial: 8250: Switch to irq_get_nr_irqs()
serial: amba-pl010: Switch to irq_get_nr_irqs()
serial: amba-pl011: Switch to irq_get_nr_irqs()
serial: cpm_uart: Switch to irq_get_nr_irqs()
serial: ucc_uart: Switch to irq_get_nr_irqs()
sh: intc: Switch to irq_get_nr_irqs()
xen/events: Switch to irq_get_nr_irqs()
fs/procfs: Switch to irq_get_nr_irqs()
genirq: Switch to irq_get_nr_irqs()
genirq: Unexport nr_irqs
Chao-ying Fu (1):
irqchip/mips-gic: Setup defaults in each cluster
David Wang (1):
genirq/proc: Use seq_put_decimal_ull_width() for decimal values
Fabrizio Castro (3):
dt-bindings: interrupt-controller: Add Renesas RZ/V2H(P) Interrupt Controller
irqchip: Add RZ/V2H(P) Interrupt Control Unit (ICU) driver
arm64: dts: renesas: r9a09g057: Add ICU node
Gregory CLEMENT (1):
irqchip/mips-gic: Prevent indirect access to clusters without CPU cores
Hari Prasath (1):
irqchip/atmel-aic5: Add support for sam9x7 aic
Hongbo Li (1):
irqchip/sifive-plic: Make use of __assign_bit()
Inochi Amaoto (3):
dt-bindings: interrupt-controller: Add T-HEAD C900 ACLINT SSWI device
irqchip: Add T-HEAD C900 ACLINT SSWI driver
riscv: defconfig: Enable T-HEAD C900 ACLINT SSWI drivers
Kevin Chen (2):
dt-bindings: interrupt-controller: Add support for ASPEED AST27XX INTC
irqchip/aspeed-intc: Add AST27XX INTC support
Nathan Chancellor (1):
irqchip/mips-gic: Fix selection of GENERIC_IRQ_EFFECTIVE_AFF_MASK
Paul Burton (3):
irqchip/mips-gic: Replace open coded online CPU iterations
irqchip/mips-gic: Support multi-cluster in for_each_online_cpu_gic()
irqchip/mips-gic: Multi-cluster support
Rob Herring (Arm) (1):
irqchip/stm32mp-exti: Use of_property_present() for non-boolean properties
Samuel Holland (1):
irqchip/riscv-aplic: Prevent crash when MSI domain is missing
Sebastian Andrzej Siewior (3):
hrtimer: Use __raise_softirq_irqoff() to raise the softirq
timers: Use __raise_softirq_irqoff() to raise the softirq.
softirq: Use a dedicated thread for timer wakeups on PREEMPT_RT.
Steven Price (3):
irqchip/gic-v3-its: Share ITS tables with a non-trusted hypervisor
irqchip/gic-v3-its: Rely on genpool alignment
irqchip/gic-v3-its: Fix over allocation in itt_alloc_pool()
Thorsten Blum (1):
genirq/irqdesc: Use str_enabled_disabled() helper in wakeup_show()
Varshini Rajendran (1):
dt-bindings: interrupt-controller: Add support for sam9x7 aic
Zijun Hu (1):
genirq/devres: Don't free interrupt which is not managed by devres
.../interrupt-controller/aspeed,ast2700-intc.yaml | 86 ++++
.../bindings/interrupt-controller/atmel,aic.yaml | 1 +
.../interrupt-controller/renesas,rzv2h-icu.yaml | 278 +++++++++++
.../thead,c900-aclint-sswi.yaml | 58 +++
arch/arm/kernel/irq.c | 5 +-
arch/arm64/boot/dts/renesas/r9a09g057.dtsi | 90 ++++
arch/loongarch/kernel/irq.c | 4 +-
arch/powerpc/platforms/cell/axon_msi.c | 2 +-
arch/riscv/configs/defconfig | 1 +
arch/s390/kernel/irq.c | 2 +-
arch/x86/kernel/acpi/boot.c | 6 +-
arch/x86/kernel/apic/vector.c | 8 +-
drivers/char/hpet.c | 1 +
drivers/irqchip/Kconfig | 20 +
drivers/irqchip/Makefile | 3 +
drivers/irqchip/irq-aspeed-intc.c | 139 ++++++
drivers/irqchip/irq-atmel-aic5.c | 9 +
drivers/irqchip/irq-gic-v3-its.c | 141 +++++-
drivers/irqchip/irq-mips-gic.c | 269 +++++++++--
drivers/irqchip/irq-renesas-rzv2h.c | 513 +++++++++++++++++++++
drivers/irqchip/irq-riscv-aplic-main.c | 3 +-
drivers/irqchip/irq-riscv-aplic-msi.c | 3 +
drivers/irqchip/irq-sifive-plic.c | 9 +-
drivers/irqchip/irq-stm32mp-exti.c | 3 +-
drivers/irqchip/irq-thead-c900-aclint-sswi.c | 176 +++++++
drivers/net/ethernet/3com/3c59x.c | 2 +-
drivers/net/hamradio/baycom_ser_fdx.c | 1 +
drivers/net/hamradio/scc.c | 4 +-
drivers/scsi/aha152x.c | 2 +-
drivers/sh/intc/virq-debugfs.c | 1 +
drivers/soc/renesas/Kconfig | 1 +
drivers/tty/serial/8250/8250_port.c | 2 +-
drivers/tty/serial/amba-pl010.c | 2 +-
drivers/tty/serial/amba-pl011.c | 2 +-
drivers/tty/serial/cpm_uart.c | 2 +-
drivers/tty/serial/serial_core.c | 2 +-
drivers/tty/serial/ucc_uart.c | 2 +-
drivers/xen/events/events_base.c | 2 +-
fs/proc/interrupts.c | 4 +-
fs/proc/stat.c | 4 +-
include/linux/cpuhotplug.h | 1 +
include/linux/interrupt.h | 47 ++
include/linux/irqnr.h | 36 +-
kernel/irq/devres.c | 3 +-
kernel/irq/irqdesc.c | 30 +-
kernel/irq/irqdomain.c | 2 +-
kernel/irq/proc.c | 12 +-
kernel/rcu/rcutorture.c | 8 +
kernel/softirq.c | 69 ++-
kernel/time/hrtimer.c | 4 +-
kernel/time/tick-sched.c | 2 +-
kernel/time/timer.c | 2 +-
52 files changed, 1945 insertions(+), 134 deletions(-)
create mode 100644 Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2700-intc.yaml
create mode 100644 Documentation/devicetree/bindings/interrupt-controller/renesas,rzv2h-icu.yaml
create mode 100644 Documentation/devicetree/bindings/interrupt-controller/thead,c900-aclint-sswi.yaml
create mode 100644 drivers/irqchip/irq-aspeed-intc.c
create mode 100644 drivers/irqchip/irq-renesas-rzv2h.c
create mode 100644 drivers/irqchip/irq-thead-c900-aclint-sswi.c
diff --git a/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2700-intc.yaml b/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2700-intc.yaml
new file mode 100644
index 000000000000..55636d06a674
--- /dev/null
+++ b/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2700-intc.yaml
@@ -0,0 +1,86 @@
+# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/interrupt-controller/aspeed,ast2700-intc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Aspeed AST2700 Interrupt Controller
+
+description:
+ This interrupt controller hardware is second level interrupt controller that
+ is hooked to a parent interrupt controller. It's useful to combine multiple
+ interrupt sources into 1 interrupt to parent interrupt controller.
+
+maintainers:
+ - Kevin Chen <kevin_chen@aspeedtech.com>
+
+properties:
+ compatible:
+ enum:
+ - aspeed,ast2700-intc-ic
+
+ reg:
+ maxItems: 1
+
+ interrupt-controller: true
+
+ '#interrupt-cells':
+ const: 2
+ description:
+ The first cell is the IRQ number, the second cell is the trigger
+ type as defined in interrupt.txt in this directory.
+
+ interrupts:
+ maxItems: 6
+ description: |
+ Depend to which INTC0 or INTC1 used.
+ INTC0 and INTC1 are two kinds of interrupt controller with enable and raw
+ status registers for use.
+ INTC0 is used to assert GIC if interrupt in INTC1 asserted.
+ INTC1 is used to assert INTC0 if interrupt of modules asserted.
+ +-----+ +-------+ +---------+---module0
+ | GIC |---| INTC0 |--+--| INTC1_0 |---module2
+ | | | | | | |---...
+ +-----+ +-------+ | +---------+---module31
+ |
+ | +---------+---module0
+ +---| INTC1_1 |---module2
+ | | |---...
+ | +---------+---module31
+ ...
+ | +---------+---module0
+ +---| INTC1_5 |---module2
+ | |---...
+ +---------+---module31
+
+
+required:
+ - compatible
+ - reg
+ - interrupt-controller
+ - '#interrupt-cells'
+ - interrupts
+
+additionalProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+ bus {
+ #address-cells = <2>;
+ #size-cells = <2>;
+
+ interrupt-controller@12101b00 {
+ compatible = "aspeed,ast2700-intc-ic";
+ reg = <0 0x12101b00 0 0x10>;
+ #interrupt-cells = <2>;
+ interrupt-controller;
+ interrupts = <GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 193 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 194 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 195 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>;
+ };
+ };
diff --git a/Documentation/devicetree/bindings/interrupt-controller/atmel,aic.yaml b/Documentation/devicetree/bindings/interrupt-controller/atmel,aic.yaml
index d4658fe3867c..d671ed884c9e 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/atmel,aic.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/atmel,aic.yaml
@@ -23,6 +23,7 @@ properties:
- atmel,sama5d3-aic
- atmel,sama5d4-aic
- microchip,sam9x60-aic
+ - microchip,sam9x7-aic
reg:
maxItems: 1
diff --git a/Documentation/devicetree/bindings/interrupt-controller/renesas,rzv2h-icu.yaml b/Documentation/devicetree/bindings/interrupt-controller/renesas,rzv2h-icu.yaml
new file mode 100644
index 000000000000..d7ef4f1323a7
--- /dev/null
+++ b/Documentation/devicetree/bindings/interrupt-controller/renesas,rzv2h-icu.yaml
@@ -0,0 +1,278 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/interrupt-controller/renesas,rzv2h-icu.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Renesas RZ/V2H(P) Interrupt Control Unit
+
+maintainers:
+ - Fabrizio Castro <fabrizio.castro.jz@renesas.com>
+ - Geert Uytterhoeven <geert+renesas@glider.be>
+
+allOf:
+ - $ref: /schemas/interrupt-controller.yaml#
+
+description:
+ The Interrupt Control Unit (ICU) handles external interrupts (NMI, IRQ, and
+ TINT), error interrupts, DMAC requests, GPT interrupts, and internal
+ interrupts.
+
+properties:
+ compatible:
+ const: renesas,r9a09g057-icu # RZ/V2H(P)
+
+ '#interrupt-cells':
+ description: The first cell is the SPI number of the NMI or the
+ PORT_IRQ[0-15] interrupt, as per user manual. The second cell is used to
+ specify the flag.
+ const: 2
+
+ '#address-cells':
+ const: 0
+
+ interrupt-controller: true
+
+ reg:
+ maxItems: 1
+
+ interrupts:
+ minItems: 58
+ items:
+ - description: NMI interrupt
+ - description: PORT_IRQ0 interrupt
+ - description: PORT_IRQ1 interrupt
+ - description: PORT_IRQ2 interrupt
+ - description: PORT_IRQ3 interrupt
+ - description: PORT_IRQ4 interrupt
+ - description: PORT_IRQ5 interrupt
+ - description: PORT_IRQ6 interrupt
+ - description: PORT_IRQ7 interrupt
+ - description: PORT_IRQ8 interrupt
+ - description: PORT_IRQ9 interrupt
+ - description: PORT_IRQ10 interrupt
+ - description: PORT_IRQ11 interrupt
+ - description: PORT_IRQ12 interrupt
+ - description: PORT_IRQ13 interrupt
+ - description: PORT_IRQ14 interrupt
+ - description: PORT_IRQ15 interrupt
+ - description: GPIO interrupt, TINT0
+ - description: GPIO interrupt, TINT1
+ - description: GPIO interrupt, TINT2
+ - description: GPIO interrupt, TINT3
+ - description: GPIO interrupt, TINT4
+ - description: GPIO interrupt, TINT5
+ - description: GPIO interrupt, TINT6
+ - description: GPIO interrupt, TINT7
+ - description: GPIO interrupt, TINT8
+ - description: GPIO interrupt, TINT9
+ - description: GPIO interrupt, TINT10
+ - description: GPIO interrupt, TINT11
+ - description: GPIO interrupt, TINT12
+ - description: GPIO interrupt, TINT13
+ - description: GPIO interrupt, TINT14
+ - description: GPIO interrupt, TINT15
+ - description: GPIO interrupt, TINT16
+ - description: GPIO interrupt, TINT17
+ - description: GPIO interrupt, TINT18
+ - description: GPIO interrupt, TINT19
+ - description: GPIO interrupt, TINT20
+ - description: GPIO interrupt, TINT21
+ - description: GPIO interrupt, TINT22
+ - description: GPIO interrupt, TINT23
+ - description: GPIO interrupt, TINT24
+ - description: GPIO interrupt, TINT25
+ - description: GPIO interrupt, TINT26
+ - description: GPIO interrupt, TINT27
+ - description: GPIO interrupt, TINT28
+ - description: GPIO interrupt, TINT29
+ - description: GPIO interrupt, TINT30
+ - description: GPIO interrupt, TINT31
+ - description: Software interrupt, INTA55_0
+ - description: Software interrupt, INTA55_1
+ - description: Software interrupt, INTA55_2
+ - description: Software interrupt, INTA55_3
+ - description: Error interrupt to CA55
+ - description: GTCCRA compare match/input capture (U0)
+ - description: GTCCRB compare match/input capture (U0)
+ - description: GTCCRA compare match/input capture (U1)
+ - description: GTCCRB compare match/input capture (U1)
+
+ interrupt-names:
+ minItems: 58
+ items:
+ - const: nmi
+ - const: port_irq0
+ - const: port_irq1
+ - const: port_irq2
+ - const: port_irq3
+ - const: port_irq4
+ - const: port_irq5
+ - const: port_irq6
+ - const: port_irq7
+ - const: port_irq8
+ - const: port_irq9
+ - const: port_irq10
+ - const: port_irq11
+ - const: port_irq12
+ - const: port_irq13
+ - const: port_irq14
+ - const: port_irq15
+ - const: tint0
+ - const: tint1
+ - const: tint2
+ - const: tint3
+ - const: tint4
+ - const: tint5
+ - const: tint6
+ - const: tint7
+ - const: tint8
+ - const: tint9
+ - const: tint10
+ - const: tint11
+ - const: tint12
+ - const: tint13
+ - const: tint14
+ - const: tint15
+ - const: tint16
+ - const: tint17
+ - const: tint18
+ - const: tint19
+ - const: tint20
+ - const: tint21
+ - const: tint22
+ - const: tint23
+ - const: tint24
+ - const: tint25
+ - const: tint26
+ - const: tint27
+ - const: tint28
+ - const: tint29
+ - const: tint30
+ - const: tint31
+ - const: int-ca55-0
+ - const: int-ca55-1
+ - const: int-ca55-2
+ - const: int-ca55-3
+ - const: icu-error-ca55
+ - const: gpt-u0-gtciada
+ - const: gpt-u0-gtciadb
+ - const: gpt-u1-gtciada
+ - const: gpt-u1-gtciadb
+
+ clocks:
+ maxItems: 1
+
+ power-domains:
+ maxItems: 1
+
+ resets:
+ maxItems: 1
+
+required:
+ - compatible
+ - reg
+ - '#interrupt-cells'
+ - '#address-cells'
+ - interrupt-controller
+ - interrupts
+ - interrupt-names
+ - clocks
+ - power-domains
+ - resets
+
+unevaluatedProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+ #include <dt-bindings/clock/renesas-cpg-mssr.h>
+
+ icu: interrupt-controller@10400000 {
+ compatible = "renesas,r9a09g057-icu";
+ reg = <0x10400000 0x10000>;
+ #interrupt-cells = <2>;
+ #address-cells = <0>;
+ interrupt-controller;
+ interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 419 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 420 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 421 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 422 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 423 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 424 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 425 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 426 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 427 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 428 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 429 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 430 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 431 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 432 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 433 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 434 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 435 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 436 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 437 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 438 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 439 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 440 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 441 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 442 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 443 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 444 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 445 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 446 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 447 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 448 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 450 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 262 IRQ_TYPE_EDGE_RISING>,
+ <GIC_SPI 263 IRQ_TYPE_EDGE_RISING>,
+ <GIC_SPI 264 IRQ_TYPE_EDGE_RISING>,
+ <GIC_SPI 265 IRQ_TYPE_EDGE_RISING>,
+ <GIC_SPI 266 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 451 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 452 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 453 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 454 IRQ_TYPE_LEVEL_HIGH>;
+ interrupt-names = "nmi",
+ "port_irq0", "port_irq1", "port_irq2",
+ "port_irq3", "port_irq4", "port_irq5",
+ "port_irq6", "port_irq7", "port_irq8",
+ "port_irq9", "port_irq10", "port_irq11",
+ "port_irq12", "port_irq13", "port_irq14",
+ "port_irq15",
+ "tint0", "tint1", "tint2", "tint3",
+ "tint4", "tint5", "tint6", "tint7",
+ "tint8", "tint9", "tint10", "tint11",
+ "tint12", "tint13", "tint14", "tint15",
+ "tint16", "tint17", "tint18", "tint19",
+ "tint20", "tint21", "tint22", "tint23",
+ "tint24", "tint25", "tint26", "tint27",
+ "tint28", "tint29", "tint30", "tint31",
+ "int-ca55-0", "int-ca55-1",
+ "int-ca55-2", "int-ca55-3",
+ "icu-error-ca55",
+ "gpt-u0-gtciada", "gpt-u0-gtciadb",
+ "gpt-u1-gtciada", "gpt-u1-gtciadb";
+ clocks = <&cpg CPG_MOD 0x5>;
+ power-domains = <&cpg>;
+ resets = <&cpg 0x36>;
+ };
diff --git a/Documentation/devicetree/bindings/interrupt-controller/thead,c900-aclint-sswi.yaml b/Documentation/devicetree/bindings/interrupt-controller/thead,c900-aclint-sswi.yaml
new file mode 100644
index 000000000000..8d330906bbbd
--- /dev/null
+++ b/Documentation/devicetree/bindings/interrupt-controller/thead,c900-aclint-sswi.yaml
@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/interrupt-controller/thead,c900-aclint-sswi.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: T-HEAD C900 ACLINT Supervisor-level Software Interrupt Device
+
+maintainers:
+ - Inochi Amaoto <inochiama@outlook.com>
+
+description:
+ The SSWI device is a part of the THEAD ACLINT device. It provides
+ supervisor-level IPI functionality for a set of HARTs on a THEAD
+ platform. It provides a register to set an IPI (SETSSIP) for each
+ HART connected to the SSWI device.
+
+properties:
+ compatible:
+ items:
+ - enum:
+ - sophgo,sg2044-aclint-sswi
+ - const: thead,c900-aclint-sswi
+
+ reg:
+ maxItems: 1
+
+ "#interrupt-cells":
+ const: 0
+
+ interrupt-controller: true
+
+ interrupts-extended:
+ minItems: 1
+ maxItems: 4095
+
+additionalProperties: false
+
+required:
+ - compatible
+ - reg
+ - "#interrupt-cells"
+ - interrupt-controller
+ - interrupts-extended
+
+examples:
+ - |
+ interrupt-controller@94000000 {
+ compatible = "sophgo,sg2044-aclint-sswi", "thead,c900-aclint-sswi";
+ reg = <0x94000000 0x00004000>;
+ #interrupt-cells = <0>;
+ interrupt-controller;
+ interrupts-extended = <&cpu1intc 1>,
+ <&cpu2intc 1>,
+ <&cpu3intc 1>,
+ <&cpu4intc 1>;
+ };
+...
diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c
index dab42d066d06..e1993e28a9ec 100644
--- a/arch/arm/kernel/irq.c
+++ b/arch/arm/kernel/irq.c
@@ -111,7 +111,7 @@ void handle_IRQ(unsigned int irq, struct pt_regs *regs)
* Some hardware gives randomly wrong interrupts. Rather
* than crashing, do something sensible.
*/
- if (unlikely(!irq || irq >= nr_irqs))
+ if (unlikely(!irq || irq >= irq_get_nr_irqs()))
desc = NULL;
else
desc = irq_to_desc(irq);
@@ -151,7 +151,6 @@ void __init init_IRQ(void)
#ifdef CONFIG_SPARSE_IRQ
int __init arch_probe_nr_irqs(void)
{
- nr_irqs = machine_desc->nr_irqs ? machine_desc->nr_irqs : NR_IRQS;
- return nr_irqs;
+ return irq_set_nr_irqs(machine_desc->nr_irqs ? : NR_IRQS);
}
#endif
diff --git a/arch/arm64/boot/dts/renesas/r9a09g057.dtsi b/arch/arm64/boot/dts/renesas/r9a09g057.dtsi
index 1ad5a1b6917f..abcdef3ba5bc 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g057.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g057.dtsi
@@ -90,6 +90,95 @@ soc: soc {
#size-cells = <2>;
ranges;
+ icu: interrupt-controller@10400000 {
+ compatible = "renesas,r9a09g057-icu";
+ reg = <0 0x10400000 0 0x10000>;
+ #interrupt-cells = <2>;
+ #address-cells = <0>;
+ interrupt-controller;
+ interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 419 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 420 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 421 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 422 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 423 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 424 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 425 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 426 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 427 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 428 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 429 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 430 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 431 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 432 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 433 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 434 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 435 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 436 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 437 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 438 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 439 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 440 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 441 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 442 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 443 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 444 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 445 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 446 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 447 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 448 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 450 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 262 IRQ_TYPE_EDGE_RISING>,
+ <GIC_SPI 263 IRQ_TYPE_EDGE_RISING>,
+ <GIC_SPI 264 IRQ_TYPE_EDGE_RISING>,
+ <GIC_SPI 265 IRQ_TYPE_EDGE_RISING>,
+ <GIC_SPI 266 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 451 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 452 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 453 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 454 IRQ_TYPE_LEVEL_HIGH>;
+ interrupt-names = "nmi",
+ "port_irq0", "port_irq1", "port_irq2",
+ "port_irq3", "port_irq4", "port_irq5",
+ "port_irq6", "port_irq7", "port_irq8",
+ "port_irq9", "port_irq10", "port_irq11",
+ "port_irq12", "port_irq13", "port_irq14",
+ "port_irq15",
+ "tint0", "tint1", "tint2", "tint3",
+ "tint4", "tint5", "tint6", "tint7",
+ "tint8", "tint9", "tint10", "tint11",
+ "tint12", "tint13", "tint14", "tint15",
+ "tint16", "tint17", "tint18", "tint19",
+ "tint20", "tint21", "tint22", "tint23",
+ "tint24", "tint25", "tint26", "tint27",
+ "tint28", "tint29", "tint30", "tint31",
+ "int-ca55-0", "int-ca55-1",
+ "int-ca55-2", "int-ca55-3",
+ "icu-error-ca55",
+ "gpt-u0-gtciada", "gpt-u0-gtciadb",
+ "gpt-u1-gtciada", "gpt-u1-gtciadb";
+ clocks = <&cpg CPG_MOD 0x5>;
+ power-domains = <&cpg>;
+ resets = <&cpg 0x36>;
+ };
+
pinctrl: pinctrl@10410000 {
compatible = "renesas,r9a09g057-pinctrl";
reg = <0 0x10410000 0 0x10000>;
@@ -99,6 +188,7 @@ pinctrl: pinctrl@10410000 {
gpio-ranges = <&pinctrl 0 0 96>;
#interrupt-cells = <2>;
interrupt-controller;
+ interrupt-parent = <&icu>;
power-domains = <&cpg>;
resets = <&cpg 0xa5>, <&cpg 0xa6>;
};
diff --git a/arch/loongarch/kernel/irq.c b/arch/loongarch/kernel/irq.c
index d129039b368b..80946cafaec1 100644
--- a/arch/loongarch/kernel/irq.c
+++ b/arch/loongarch/kernel/irq.c
@@ -92,9 +92,9 @@ int __init arch_probe_nr_irqs(void)
int nr_io_pics = bitmap_weight(loongson_sysconf.cores_io_master, NR_CPUS);
if (!cpu_has_avecint)
- nr_irqs = (64 + NR_VECTORS * nr_io_pics);
+ irq_set_nr_irqs(64 + NR_VECTORS * nr_io_pics);
else
- nr_irqs = (64 + NR_VECTORS * (nr_cpu_ids + nr_io_pics));
+ irq_set_nr_irqs(64 + NR_VECTORS * (nr_cpu_ids + nr_io_pics));
return NR_IRQS_LEGACY;
}
diff --git a/arch/powerpc/platforms/cell/axon_msi.c b/arch/powerpc/platforms/cell/axon_msi.c
index 28dc86744cac..d243f7fd8982 100644
--- a/arch/powerpc/platforms/cell/axon_msi.c
+++ b/arch/powerpc/platforms/cell/axon_msi.c
@@ -112,7 +112,7 @@ static void axon_msi_cascade(struct irq_desc *desc)
pr_devel("axon_msi: woff %x roff %x msi %x\n",
write_offset, msic->read_offset, msi);
- if (msi < nr_irqs && irq_get_chip_data(msi) == msic) {
+ if (msi < irq_get_nr_irqs() && irq_get_chip_data(msi) == msic) {
generic_handle_irq(msi);
msic->fifo_virt[idx] = cpu_to_le32(0xffffffff);
} else {
diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig
index 2341393cfac1..5b1d6325df85 100644
--- a/arch/riscv/configs/defconfig
+++ b/arch/riscv/configs/defconfig
@@ -256,6 +256,7 @@ CONFIG_RPMSG_CTRL=y
CONFIG_RPMSG_VIRTIO=y
CONFIG_PM_DEVFREQ=y
CONFIG_IIO=y
+CONFIG_THEAD_C900_ACLINT_SSWI=y
CONFIG_PHY_SUN4I_USB=m
CONFIG_PHY_STARFIVE_JH7110_DPHY_RX=m
CONFIG_PHY_STARFIVE_JH7110_PCIE=m
diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c
index 2639a3d12736..a2c867181b5a 100644
--- a/arch/s390/kernel/irq.c
+++ b/arch/s390/kernel/irq.c
@@ -253,7 +253,7 @@ int show_interrupts(struct seq_file *p, void *v)
seq_putc(p, '\n');
goto out;
}
- if (index < nr_irqs) {
+ if (index < irq_get_nr_irqs()) {
show_msi_interrupt(p, index);
goto out;
}
diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c
index 4efecac49863..3a44a9dc3fb7 100644
--- a/arch/x86/kernel/acpi/boot.c
+++ b/arch/x86/kernel/acpi/boot.c
@@ -1171,7 +1171,8 @@ static int __init acpi_parse_madt_ioapic_entries(void)
}
count = acpi_table_parse_madt(ACPI_MADT_TYPE_INTERRUPT_OVERRIDE,
- acpi_parse_int_src_ovr, nr_irqs);
+ acpi_parse_int_src_ovr,
+ irq_get_nr_irqs());
if (count < 0) {
pr_err("Error parsing interrupt source overrides entry\n");
/* TBD: Cleanup to allow fallback to MPS */
@@ -1191,7 +1192,8 @@ static int __init acpi_parse_madt_ioapic_entries(void)
mp_config_acpi_legacy_irqs();
count = acpi_table_parse_madt(ACPI_MADT_TYPE_NMI_SOURCE,
- acpi_parse_nmi_src, nr_irqs);
+ acpi_parse_nmi_src,
+ irq_get_nr_irqs());
if (count < 0) {
pr_err("Error parsing NMI SRC entry\n");
/* TBD: Cleanup to allow fallback to MPS */
diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c
index 557318145038..736f62812f5c 100644
--- a/arch/x86/kernel/apic/vector.c
+++ b/arch/x86/kernel/apic/vector.c
@@ -712,8 +712,8 @@ int __init arch_probe_nr_irqs(void)
{
int nr;
- if (nr_irqs > (NR_VECTORS * nr_cpu_ids))
- nr_irqs = NR_VECTORS * nr_cpu_ids;
+ if (irq_get_nr_irqs() > NR_VECTORS * nr_cpu_ids)
+ irq_set_nr_irqs(NR_VECTORS * nr_cpu_ids);
nr = (gsi_top + nr_legacy_irqs()) + 8 * nr_cpu_ids;
#if defined(CONFIG_PCI_MSI)
@@ -725,8 +725,8 @@ int __init arch_probe_nr_irqs(void)
else
nr += gsi_top * 16;
#endif
- if (nr < nr_irqs)
- nr_irqs = nr;
+ if (nr < irq_get_nr_irqs())
+ irq_set_nr_irqs(nr);
/*
* We don't know if PIC is present at this point so we need to do
diff --git a/drivers/char/hpet.c b/drivers/char/hpet.c
index e904e476e49a..48fe96ab4649 100644
--- a/drivers/char/hpet.c
+++ b/drivers/char/hpet.c
@@ -162,6 +162,7 @@ static irqreturn_t hpet_interrupt(int irq, void *data)
static void hpet_timer_set_irq(struct hpet_dev *devp)
{
+ const unsigned int nr_irqs = irq_get_nr_irqs();
unsigned long v;
int irq, gsi;
struct hpet_timer __iomem *timer;
diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig
index 341cd9ca5a05..9cac13663871 100644
--- a/drivers/irqchip/Kconfig
+++ b/drivers/irqchip/Kconfig
@@ -265,6 +265,13 @@ config RENESAS_RZG2L_IRQC
Enable support for the Renesas RZ/G2L (and alike SoC) Interrupt Controller
for external devices.
+config RENESAS_RZV2H_ICU
+ bool "Renesas RZ/V2H(P) ICU support" if COMPILE_TEST
+ select GENERIC_IRQ_CHIP
+ select IRQ_DOMAIN_HIERARCHY
+ help
+ Enable support for the Renesas RZ/V2H(P) Interrupt Control Unit (ICU)
+
config SL28CPLD_INTC
bool "Kontron sl28cpld IRQ controller"
depends on MFD_SL28CPLD=y || COMPILE_TEST
@@ -345,6 +352,7 @@ config KEYSTONE_IRQ
config MIPS_GIC
bool
+ select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP
select GENERIC_IRQ_IPI if SMP
select IRQ_DOMAIN_HIERARCHY
select MIPS_CM
@@ -611,6 +619,18 @@ config STARFIVE_JH8100_INTC
If you don't know what to do here, say Y.
+config THEAD_C900_ACLINT_SSWI
+ bool "THEAD C9XX ACLINT S-mode IPI Interrupt Controller"
+ depends on RISCV
+ depends on SMP
+ select IRQ_DOMAIN_HIERARCHY
+ select GENERIC_IRQ_IPI_MUX
+ help
+ This enables support for T-HEAD specific ACLINT SSWI device
+ support.
+
+ If you don't know what to do here, say Y.
+
config EXYNOS_IRQ_COMBINER
bool "Samsung Exynos IRQ combiner support" if COMPILE_TEST
depends on (ARCH_EXYNOS && ARM) || COMPILE_TEST
diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
index e3679ec2b9f7..25e9ad29b8c4 100644
--- a/drivers/irqchip/Makefile
+++ b/drivers/irqchip/Makefile
@@ -51,6 +51,7 @@ obj-$(CONFIG_RENESAS_INTC_IRQPIN) += irq-renesas-intc-irqpin.o
obj-$(CONFIG_RENESAS_IRQC) += irq-renesas-irqc.o
obj-$(CONFIG_RENESAS_RZA1_IRQC) += irq-renesas-rza1.o
obj-$(CONFIG_RENESAS_RZG2L_IRQC) += irq-renesas-rzg2l.o
+obj-$(CONFIG_RENESAS_RZV2H_ICU) += irq-renesas-rzv2h.o
obj-$(CONFIG_VERSATILE_FPGA_IRQ) += irq-versatile-fpga.o
obj-$(CONFIG_ARCH_NSPIRE) += irq-zevio.o
obj-$(CONFIG_ARCH_VT8500) += irq-vt8500.o
@@ -84,6 +85,7 @@ obj-$(CONFIG_MVEBU_SEI) += irq-mvebu-sei.o
obj-$(CONFIG_LS_EXTIRQ) += irq-ls-extirq.o
obj-$(CONFIG_LS_SCFG_MSI) += irq-ls-scfg-msi.o
obj-$(CONFIG_ARCH_ASPEED) += irq-aspeed-vic.o irq-aspeed-i2c-ic.o irq-aspeed-scu-ic.o
+obj-$(CONFIG_ARCH_ASPEED) += irq-aspeed-intc.o
obj-$(CONFIG_STM32MP_EXTI) += irq-stm32mp-exti.o
obj-$(CONFIG_STM32_EXTI) += irq-stm32-exti.o
obj-$(CONFIG_QCOM_IRQ_COMBINER) += qcom-irq-combiner.o
@@ -101,6 +103,7 @@ obj-$(CONFIG_RISCV_APLIC_MSI) += irq-riscv-aplic-msi.o
obj-$(CONFIG_RISCV_IMSIC) += irq-riscv-imsic-state.o irq-riscv-imsic-early.o irq-riscv-imsic-platform.o
obj-$(CONFIG_SIFIVE_PLIC) += irq-sifive-plic.o
obj-$(CONFIG_STARFIVE_JH8100_INTC) += irq-starfive-jh8100-intc.o
+obj-$(CONFIG_THEAD_C900_ACLINT_SSWI) += irq-thead-c900-aclint-sswi.o
obj-$(CONFIG_IMX_IRQSTEER) += irq-imx-irqsteer.o
obj-$(CONFIG_IMX_INTMUX) += irq-imx-intmux.o
obj-$(CONFIG_IMX_MU_MSI) += irq-imx-mu-msi.o
diff --git a/drivers/irqchip/irq-aspeed-intc.c b/drivers/irqchip/irq-aspeed-intc.c
new file mode 100644
index 000000000000..bd3b759b4b2c
--- /dev/null
+++ b/drivers/irqchip/irq-aspeed-intc.c
@@ -0,0 +1,139 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Aspeed Interrupt Controller.
+ *
+ * Copyright (C) 2023 ASPEED Technology Inc.
+ */
+
+#include <linux/bitops.h>
+#include <linux/irq.h>
+#include <linux/irqchip.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/io.h>
+#include <linux/spinlock.h>
+
+#define INTC_INT_ENABLE_REG 0x00
+#define INTC_INT_STATUS_REG 0x04
+#define INTC_IRQS_PER_WORD 32
+
+struct aspeed_intc_ic {
+ void __iomem *base;
+ raw_spinlock_t gic_lock;
+ raw_spinlock_t intc_lock;
+ struct irq_domain *irq_domain;
+};
+
+static void aspeed_intc_ic_irq_handler(struct irq_desc *desc)
+{
+ struct aspeed_intc_ic *intc_ic = irq_desc_get_handler_data(desc);
+ struct irq_chip *chip = irq_desc_get_chip(desc);
+
+ chained_irq_enter(chip, desc);
+
+ scoped_guard(raw_spinlock, &intc_ic->gic_lock) {
+ unsigned long bit, status;
+
+ status = readl(intc_ic->base + INTC_INT_STATUS_REG);
+ for_each_set_bit(bit, &status, INTC_IRQS_PER_WORD) {
+ generic_handle_domain_irq(intc_ic->irq_domain, bit);
+ writel(BIT(bit), intc_ic->base + INTC_INT_STATUS_REG);
+ }
+ }
+
+ chained_irq_exit(chip, desc);
+}
+
+static void aspeed_intc_irq_mask(struct irq_data *data)
+{
+ struct aspeed_intc_ic *intc_ic = irq_data_get_irq_chip_data(data);
+ unsigned int mask = readl(intc_ic->base + INTC_INT_ENABLE_REG) & ~BIT(data->hwirq);
+
+ guard(raw_spinlock)(&intc_ic->intc_lock);
+ writel(mask, intc_ic->base + INTC_INT_ENABLE_REG);
+}
+
+static void aspeed_intc_irq_unmask(struct irq_data *data)
+{
+ struct aspeed_intc_ic *intc_ic = irq_data_get_irq_chip_data(data);
+ unsigned int unmask = readl(intc_ic->base + INTC_INT_ENABLE_REG) | BIT(data->hwirq);
+
+ guard(raw_spinlock)(&intc_ic->intc_lock);
+ writel(unmask, intc_ic->base + INTC_INT_ENABLE_REG);
+}
+
+static struct irq_chip aspeed_intc_chip = {
+ .name = "ASPEED INTC",
+ .irq_mask = aspeed_intc_irq_mask,
+ .irq_unmask = aspeed_intc_irq_unmask,
+};
+
+static int aspeed_intc_ic_map_irq_domain(struct irq_domain *domain, unsigned int irq,
+ irq_hw_number_t hwirq)
+{
+ irq_set_chip_and_handler(irq, &aspeed_intc_chip, handle_level_irq);
+ irq_set_chip_data(irq, domain->host_data);
+
+ return 0;
+}
+
+static const struct irq_domain_ops aspeed_intc_ic_irq_domain_ops = {
+ .map = aspeed_intc_ic_map_irq_domain,
+};
+
+static int __init aspeed_intc_ic_of_init(struct device_node *node,
+ struct device_node *parent)
+{
+ struct aspeed_intc_ic *intc_ic;
+ int irq, i, ret = 0;
+
+ intc_ic = kzalloc(sizeof(*intc_ic), GFP_KERNEL);
+ if (!intc_ic)
+ return -ENOMEM;
+
+ intc_ic->base = of_iomap(node, 0);
+ if (!intc_ic->base) {
+ pr_err("Failed to iomap intc_ic base\n");
+ ret = -ENOMEM;
+ goto err_free_ic;
+ }
+ writel(0xffffffff, intc_ic->base + INTC_INT_STATUS_REG);
+ writel(0x0, intc_ic->base + INTC_INT_ENABLE_REG);
+
+ intc_ic->irq_domain = irq_domain_add_linear(node, INTC_IRQS_PER_WORD,
+ &aspeed_intc_ic_irq_domain_ops, intc_ic);
+ if (!intc_ic->irq_domain) {
+ ret = -ENOMEM;
+ goto err_iounmap;
+ }
+
+ raw_spin_lock_init(&intc_ic->gic_lock);
+ raw_spin_lock_init(&intc_ic->intc_lock);
+
+ /* Check all the irq numbers valid. If not, unmaps all the base and frees the data. */
+ for (i = 0; i < of_irq_count(node); i++) {
+ irq = irq_of_parse_and_map(node, i);
+ if (!irq) {
+ pr_err("Failed to get irq number\n");
+ ret = -EINVAL;
+ goto err_iounmap;
+ }
+ }
+
+ for (i = 0; i < of_irq_count(node); i++) {
+ irq = irq_of_parse_and_map(node, i);
+ irq_set_chained_handler_and_data(irq, aspeed_intc_ic_irq_handler, intc_ic);
+ }
+
+ return 0;
+
+err_iounmap:
+ iounmap(intc_ic->base);
+err_free_ic:
+ kfree(intc_ic);
+ return ret;
+}
+
+IRQCHIP_DECLARE(ast2700_intc_ic, "aspeed,ast2700-intc-ic", aspeed_intc_ic_of_init);
diff --git a/drivers/irqchip/irq-atmel-aic5.c b/drivers/irqchip/irq-atmel-aic5.c
index c0f55dc7b050..e98c2875af9e 100644
--- a/drivers/irqchip/irq-atmel-aic5.c
+++ b/drivers/irqchip/irq-atmel-aic5.c
@@ -319,6 +319,7 @@ static const struct of_device_id aic5_irq_fixups[] __initconst = {
{ .compatible = "atmel,sama5d3", .data = sama5d3_aic_irq_fixup },
{ .compatible = "atmel,sama5d4", .data = sama5d3_aic_irq_fixup },
{ .compatible = "microchip,sam9x60", .data = sam9x60_aic_irq_fixup },
+ { .compatible = "microchip,sam9x7", .data = sam9x60_aic_irq_fixup },
{ /* sentinel */ },
};
@@ -405,3 +406,11 @@ static int __init sam9x60_aic5_of_init(struct device_node *node,
return aic5_of_init(node, parent, NR_SAM9X60_IRQS);
}
IRQCHIP_DECLARE(sam9x60_aic5, "microchip,sam9x60-aic", sam9x60_aic5_of_init);
+
+#define NR_SAM9X7_IRQS 70
+
+static int __init sam9x7_aic5_of_init(struct device_node *node, struct device_node *parent)
+{
+ return aic5_of_init(node, parent, NR_SAM9X7_IRQS);
+}
+IRQCHIP_DECLARE(sam9x7_aic5, "microchip,sam9x7-aic", sam9x7_aic5_of_init);
diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c
index fdec478ba5e7..395961fb2180 100644
--- a/drivers/irqchip/irq-gic-v3-its.c
+++ b/drivers/irqchip/irq-gic-v3-its.c
@@ -12,12 +12,14 @@
#include <linux/crash_dump.h>
#include <linux/delay.h>
#include <linux/efi.h>
+#include <linux/genalloc.h>
#include <linux/interrupt.h>
#include <linux/iommu.h>
#include <linux/iopoll.h>
#include <linux/irqdomain.h>
#include <linux/list.h>
#include <linux/log2.h>
+#include <linux/mem_encrypt.h>
#include <linux/memblock.h>
#include <linux/mm.h>
#include <linux/msi.h>
@@ -27,6 +29,7 @@
#include <linux/of_pci.h>
#include <linux/of_platform.h>
#include <linux/percpu.h>
+#include <linux/set_memory.h>
#include <linux/slab.h>
#include <linux/syscore_ops.h>
@@ -164,6 +167,7 @@ struct its_device {
struct its_node *its;
struct event_lpi_map event_map;
void *itt;
+ u32 itt_sz;
u32 nr_ites;
u32 device_id;
bool shared;
@@ -199,6 +203,87 @@ static DEFINE_IDA(its_vpeid_ida);
#define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base)
#define gic_data_rdist_vlpi_base() (gic_data_rdist_rd_base() + SZ_128K)
+static struct page *its_alloc_pages_node(int node, gfp_t gfp,
+ unsigned int order)
+{
+ struct page *page;
+ int ret = 0;
+
+ page = alloc_pages_node(node, gfp, order);
+
+ if (!page)
+ return NULL;
+
+ ret = set_memory_decrypted((unsigned long)page_address(page),
+ 1 << order);
+ /*
+ * If set_memory_decrypted() fails then we don't know what state the
+ * page is in, so we can't free it. Instead we leak it.
+ * set_memory_decrypted() will already have WARNed.
+ */
+ if (ret)
+ return NULL;
+
+ return page;
+}
+
+static struct page *its_alloc_pages(gfp_t gfp, unsigned int order)
+{
+ return its_alloc_pages_node(NUMA_NO_NODE, gfp, order);
+}
+
+static void its_free_pages(void *addr, unsigned int order)
+{
+ /*
+ * If the memory cannot be encrypted again then we must leak the pages.
+ * set_memory_encrypted() will already have WARNed.
+ */
+ if (set_memory_encrypted((unsigned long)addr, 1 << order))
+ return;
+ free_pages((unsigned long)addr, order);
+}
+
+static struct gen_pool *itt_pool;
+
+static void *itt_alloc_pool(int node, int size)
+{
+ unsigned long addr;
+ struct page *page;
+
+ if (size >= PAGE_SIZE) {
+ page = its_alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, get_order(size));
+
+ return page ? page_address(page) : NULL;
+ }
+
+ do {
+ addr = gen_pool_alloc(itt_pool, size);
+ if (addr)
+ break;
+
+ page = its_alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
+ if (!page)
+ break;
+
+ gen_pool_add(itt_pool, (unsigned long)page_address(page), PAGE_SIZE, node);
+ } while (!addr);
+
+ return (void *)addr;
+}
+
+static void itt_free_pool(void *addr, int size)
+{
+ if (!addr)
+ return;
+
+ if (size >= PAGE_SIZE) {
+ its_free_pages(addr, get_order(size));
+ return;
+ }
+
+ gen_pool_free(itt_pool, (unsigned long)addr, size);
+}
+
/*
* Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we
* always have vSGIs mapped.
@@ -621,7 +706,6 @@ static struct its_collection *its_build_mapd_cmd(struct its_node *its,
u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites);
itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt);
- itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN);
its_encode_cmd(cmd, GITS_CMD_MAPD);
its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id);
@@ -2181,7 +2265,8 @@ static struct page *its_allocate_prop_table(gfp_t gfp_flags)
{
struct page *prop_page;
- prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ));
+ prop_page = its_alloc_pages(gfp_flags,
+ get_order(LPI_PROPBASE_SZ));
if (!prop_page)
return NULL;
@@ -2192,8 +2277,7 @@ static struct page *its_allocate_prop_table(gfp_t gfp_flags)
static void its_free_prop_table(struct page *prop_page)
{
- free_pages((unsigned long)page_address(prop_page),
- get_order(LPI_PROPBASE_SZ));
+ its_free_pages(page_address(prop_page), get_order(LPI_PROPBASE_SZ));
}
static bool gic_check_reserved_range(phys_addr_t addr, unsigned long size)
@@ -2315,7 +2399,7 @@ static int its_setup_baser(struct its_node *its, struct its_baser *baser,
order = get_order(GITS_BASER_PAGES_MAX * psz);
}
- page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order);
+ page = its_alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order);
if (!page)
return -ENOMEM;
@@ -2328,7 +2412,7 @@ static int its_setup_baser(struct its_node *its, struct its_baser *baser,
/* 52bit PA is supported only when PageSize=64K */
if (psz != SZ_64K) {
pr_err("ITS: no 52bit PA support when psz=%d\n", psz);
- free_pages((unsigned long)base, order);
+ its_free_pages(base, order);
return -ENXIO;
}
@@ -2384,7 +2468,7 @@ static int its_setup_baser(struct its_node *its, struct its_baser *baser,
pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n",
&its->phys_base, its_base_type_string[type],
val, tmp);
- free_pages((unsigned long)base, order);
+ its_free_pages(base, order);
return -ENXIO;
}
@@ -2523,8 +2607,7 @@ static void its_free_tables(struct its_node *its)
for (i = 0; i < GITS_BASER_NR_REGS; i++) {
if (its->tables[i].base) {
- free_pages((unsigned long)its->tables[i].base,
- its->tables[i].order);
+ its_free_pages(its->tables[i].base, its->tables[i].order);
its->tables[i].base = NULL;
}
}
@@ -2790,7 +2873,7 @@ static bool allocate_vpe_l2_table(int cpu, u32 id)
/* Allocate memory for 2nd level table */
if (!table[idx]) {
- page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz));
+ page = its_alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz));
if (!page)
return false;
@@ -2909,7 +2992,7 @@ static int allocate_vpe_l1_table(void)
pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n",
np, npg, psz, epp, esz);
- page = alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE));
+ page = its_alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE));
if (!page)
return -ENOMEM;
@@ -2955,8 +3038,7 @@ static struct page *its_allocate_pending_table(gfp_t gfp_flags)
{
struct page *pend_page;
- pend_page = alloc_pages(gfp_flags | __GFP_ZERO,
- get_order(LPI_PENDBASE_SZ));
+ pend_page = its_alloc_pages(gfp_flags | __GFP_ZERO, get_order(LPI_PENDBASE_SZ));
if (!pend_page)
return NULL;
@@ -2968,7 +3050,7 @@ static struct page *its_allocate_pending_table(gfp_t gfp_flags)
static void its_free_pending_table(struct page *pt)
{
- free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ));
+ its_free_pages(page_address(pt), get_order(LPI_PENDBASE_SZ));
}
/*
@@ -3303,8 +3385,8 @@ static bool its_alloc_table_entry(struct its_node *its,
/* Allocate memory for 2nd level table */
if (!table[idx]) {
- page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
- get_order(baser->psz));
+ page = its_alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
+ get_order(baser->psz));
if (!page)
return false;
@@ -3399,15 +3481,18 @@ static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
if (WARN_ON(!is_power_of_2(nvecs)))
nvecs = roundup_pow_of_two(nvecs);
- dev = kzalloc(sizeof(*dev), GFP_KERNEL);
/*
* Even if the device wants a single LPI, the ITT must be
* sized as a power of two (and you need at least one bit...).
*/
nr_ites = max(2, nvecs);
sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1);
- sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1;
- itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node);
+ sz = max(sz, ITS_ITT_ALIGN);
+
+ itt = itt_alloc_pool(its->numa_node, sz);
+
+ dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+
if (alloc_lpis) {
lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis);
if (lpi_map)
@@ -3419,9 +3504,9 @@ static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
lpi_base = 0;
}
- if (!dev || !itt || !col_map || (!lpi_map && alloc_lpis)) {
+ if (!dev || !itt || !col_map || (!lpi_map && alloc_lpis)) {
kfree(dev);
- kfree(itt);
+ itt_free_pool(itt, sz);
bitmap_free(lpi_map);
kfree(col_map);
return NULL;
@@ -3431,6 +3516,7 @@ static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
dev->its = its;
dev->itt = itt;
+ dev->itt_sz = sz;
dev->nr_ites = nr_ites;
dev->event_map.lpi_map = lpi_map;
dev->event_map.col_map = col_map;
@@ -3458,7 +3544,7 @@ static void its_free_device(struct its_device *its_dev)
list_del(&its_dev->entry);
raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);
kfree(its_dev->event_map.col_map);
- kfree(its_dev->itt);
+ itt_free_pool(its_dev->itt, its_dev->itt_sz);
kfree(its_dev);
}
@@ -5116,8 +5202,9 @@ static int __init its_probe_one(struct its_node *its)
}
}
- page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
- get_order(ITS_CMD_QUEUE_SZ));
+ page = its_alloc_pages_node(its->numa_node,
+ GFP_KERNEL | __GFP_ZERO,
+ get_order(ITS_CMD_QUEUE_SZ));
if (!page) {
err = -ENOMEM;
goto out_unmap_sgir;
@@ -5181,7 +5268,7 @@ static int __init its_probe_one(struct its_node *its)
out_free_tables:
its_free_tables(its);
out_free_cmd:
- free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ));
+ its_free_pages(its->cmd_base, get_order(ITS_CMD_QUEUE_SZ));
out_unmap_sgir:
if (its->sgir_base)
iounmap(its->sgir_base);
@@ -5667,6 +5754,10 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists,
bool has_v4_1 = false;
int err;
+ itt_pool = gen_pool_create(get_order(ITS_ITT_ALIGN), -1);
+ if (!itt_pool)
+ return -ENOMEM;
+
gic_rdists = rdists;
lpi_prop_prio = irq_prio;
diff --git a/drivers/irqchip/irq-mips-gic.c b/drivers/irqchip/irq-mips-gic.c
index 76253e864f23..bca8053864b2 100644
--- a/drivers/irqchip/irq-mips-gic.c
+++ b/drivers/irqchip/irq-mips-gic.c
@@ -66,6 +66,87 @@ static struct gic_all_vpes_chip_data {
bool mask;
} gic_all_vpes_chip_data[GIC_NUM_LOCAL_INTRS];
+static int __gic_with_next_online_cpu(int prev)
+{
+ unsigned int cpu;
+
+ /* Discover the next online CPU */
+ cpu = cpumask_next(prev, cpu_online_mask);
+
+ /* If there isn't one, we're done */
+ if (cpu >= nr_cpu_ids)
+ return cpu;
+
+ /*
+ * Move the access lock to the next CPU's GIC local register block.
+ *
+ * Set GIC_VL_OTHER. Since the caller holds gic_lock nothing can
+ * clobber the written value.
+ */
+ write_gic_vl_other(mips_cm_vp_id(cpu));
+
+ return cpu;
+}
+
+static inline void gic_unlock_cluster(void)
+{
+ if (mips_cps_multicluster_cpus())
+ mips_cm_unlock_other();
+}
+
+/**
+ * for_each_online_cpu_gic() - Iterate over online CPUs, access local registers
+ * @cpu: An integer variable to hold the current CPU number
+ * @gic_lock: A pointer to raw spin lock used as a guard
+ *
+ * Iterate over online CPUs & configure the other/redirect register region to
+ * access each CPUs GIC local register block, which can be accessed from the
+ * loop body using read_gic_vo_*() or write_gic_vo_*() accessor functions or
+ * their derivatives.
+ */
+#define for_each_online_cpu_gic(cpu, gic_lock) \
+ guard(raw_spinlock_irqsave)(gic_lock); \
+ for ((cpu) = __gic_with_next_online_cpu(-1); \
+ (cpu) < nr_cpu_ids; \
+ gic_unlock_cluster(), \
+ (cpu) = __gic_with_next_online_cpu(cpu))
+
+/**
+ * gic_irq_lock_cluster() - Lock redirect block access to IRQ's cluster
+ * @d: struct irq_data corresponding to the interrupt we're interested in
+ *
+ * Locks redirect register block access to the global register block of the GIC
+ * within the remote cluster that the IRQ corresponding to @d is affine to,
+ * returning true when this redirect block setup & locking has been performed.
+ *
+ * If @d is affine to the local cluster then no locking is performed and this
+ * function will return false, indicating to the caller that it should access
+ * the local clusters registers without the overhead of indirection through the
+ * redirect block.
+ *
+ * In summary, if this function returns true then the caller should access GIC
+ * registers using redirect register block accessors & then call
+ * mips_cm_unlock_other() when done. If this function returns false then the
+ * caller should trivially access GIC registers in the local cluster.
+ *
+ * Returns true if locking performed, else false.
+ */
+static bool gic_irq_lock_cluster(struct irq_data *d)
+{
+ unsigned int cpu, cl;
+
+ cpu = cpumask_first(irq_data_get_effective_affinity_mask(d));
+ BUG_ON(cpu >= NR_CPUS);
+
+ cl = cpu_cluster(&cpu_data[cpu]);
+ if (cl == cpu_cluster(¤t_cpu_data))
+ return false;
+ if (mips_cps_numcores(cl) == 0)
+ return false;
+ mips_cm_lock_other(cl, 0, 0, CM_GCR_Cx_OTHER_BLOCK_GLOBAL);
+ return true;
+}
+
static void gic_clear_pcpu_masks(unsigned int intr)
{
unsigned int i;
@@ -112,7 +193,12 @@ static void gic_send_ipi(struct irq_data *d, unsigned int cpu)
{
irq_hw_number_t hwirq = GIC_HWIRQ_TO_SHARED(irqd_to_hwirq(d));
- write_gic_wedge(GIC_WEDGE_RW | hwirq);
+ if (gic_irq_lock_cluster(d)) {
+ write_gic_redir_wedge(GIC_WEDGE_RW | hwirq);
+ mips_cm_unlock_other();
+ } else {
+ write_gic_wedge(GIC_WEDGE_RW | hwirq);
+ }
}
int gic_get_c0_compare_int(void)
@@ -180,7 +266,13 @@ static void gic_mask_irq(struct irq_data *d)
{
unsigned int intr = GIC_HWIRQ_TO_SHARED(d->hwirq);
- write_gic_rmask(intr);
+ if (gic_irq_lock_cluster(d)) {
+ write_gic_redir_rmask(intr);
+ mips_cm_unlock_other();
+ } else {
+ write_gic_rmask(intr);
+ }
+
gic_clear_pcpu_masks(intr);
}
@@ -189,7 +281,12 @@ static void gic_unmask_irq(struct irq_data *d)
unsigned int intr = GIC_HWIRQ_TO_SHARED(d->hwirq);
unsigned int cpu;
- write_gic_smask(intr);
+ if (gic_irq_lock_cluster(d)) {
+ write_gic_redir_smask(intr);
+ mips_cm_unlock_other();
+ } else {
+ write_gic_smask(intr);
+ }
gic_clear_pcpu_masks(intr);
cpu = cpumask_first(irq_data_get_effective_affinity_mask(d));
@@ -200,7 +297,12 @@ static void gic_ack_irq(struct irq_data *d)
{
unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
- write_gic_wedge(irq);
+ if (gic_irq_lock_cluster(d)) {
+ write_gic_redir_wedge(irq);
+ mips_cm_unlock_other();
+ } else {
+ write_gic_wedge(irq);
+ }
}
static int gic_set_type(struct irq_data *d, unsigned int type)
@@ -240,9 +342,16 @@ static int gic_set_type(struct irq_data *d, unsigned int type)
break;
}
- change_gic_pol(irq, pol);
- change_gic_trig(irq, trig);
- change_gic_dual(irq, dual);
+ if (gic_irq_lock_cluster(d)) {
+ change_gic_redir_pol(irq, pol);
+ change_gic_redir_trig(irq, trig);
+ change_gic_redir_dual(irq, dual);
+ mips_cm_unlock_other();
+ } else {
+ change_gic_pol(irq, pol);
+ change_gic_trig(irq, trig);
+ change_gic_dual(irq, dual);
+ }
if (trig == GIC_TRIG_EDGE)
irq_set_chip_handler_name_locked(d, &gic_edge_irq_controller,
@@ -260,25 +369,72 @@ static int gic_set_affinity(struct irq_data *d, const struct cpumask *cpumask,
bool force)
{
unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq);
+ unsigned int cpu, cl, old_cpu, old_cl;
unsigned long flags;
- unsigned int cpu;
+ /*
+ * The GIC specifies that we can only route an interrupt to one VP(E),
+ * ie. CPU in Linux parlance, at a time. Therefore we always route to
+ * the first online CPU in the mask.
+ */
cpu = cpumask_first_and(cpumask, cpu_online_mask);
if (cpu >= NR_CPUS)
return -EINVAL;
- /* Assumption : cpumask refers to a single CPU */
- raw_spin_lock_irqsave(&gic_lock, flags);
+ old_cpu = cpumask_first(irq_data_get_effective_affinity_mask(d));
+ old_cl = cpu_cluster(&cpu_data[old_cpu]);
+ cl = cpu_cluster(&cpu_data[cpu]);
- /* Re-route this IRQ */
- write_gic_map_vp(irq, BIT(mips_cm_vp_id(cpu)));
+ raw_spin_lock_irqsave(&gic_lock, flags);
- /* Update the pcpu_masks */
- gic_clear_pcpu_masks(irq);
- if (read_gic_mask(irq))
- set_bit(irq, per_cpu_ptr(pcpu_masks, cpu));
+ /*
+ * If we're moving affinity between clusters, stop routing the
+ * interrupt to any VP(E) in the old cluster.
+ */
+ if (cl != old_cl) {
+ if (gic_irq_lock_cluster(d)) {
+ write_gic_redir_map_vp(irq, 0);
+ mips_cm_unlock_other();
+ } else {
+ write_gic_map_vp(irq, 0);
+ }
+ }
+ /*
+ * Update effective affinity - after this gic_irq_lock_cluster() will
+ * begin operating on the new cluster.
+ */
irq_data_update_effective_affinity(d, cpumask_of(cpu));
+
+ /*
+ * If we're moving affinity between clusters, configure the interrupt
+ * trigger type in the new cluster.
+ */
+ if (cl != old_cl)
+ gic_set_type(d, irqd_get_trigger_type(d));
+
+ /* Route the interrupt to its new VP(E) */
+ if (gic_irq_lock_cluster(d)) {
+ write_gic_redir_map_pin(irq,
+ GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin);
+ write_gic_redir_map_vp(irq, BIT(mips_cm_vp_id(cpu)));
+
+ /* Update the pcpu_masks */
+ gic_clear_pcpu_masks(irq);
+ if (read_gic_redir_mask(irq))
+ set_bit(irq, per_cpu_ptr(pcpu_masks, cpu));
+
+ mips_cm_unlock_other();
+ } else {
+ write_gic_map_pin(irq, GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin);
+ write_gic_map_vp(irq, BIT(mips_cm_vp_id(cpu)));
+
+ /* Update the pcpu_masks */
+ gic_clear_pcpu_masks(irq);
+ if (read_gic_mask(irq))
+ set_bit(irq, per_cpu_ptr(pcpu_masks, cpu));
+ }
+
raw_spin_unlock_irqrestore(&gic_lock, flags);
return IRQ_SET_MASK_OK;
@@ -350,37 +506,33 @@ static struct irq_chip gic_local_irq_controller = {
static void gic_mask_local_irq_all_vpes(struct irq_data *d)
{
struct gic_all_vpes_chip_data *cd;
- unsigned long flags;
int intr, cpu;
+ if (!mips_cps_multicluster_cpus())
+ return;
+
intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
cd = irq_data_get_irq_chip_data(d);
cd->mask = false;
- raw_spin_lock_irqsave(&gic_lock, flags);
- for_each_online_cpu(cpu) {
- write_gic_vl_other(mips_cm_vp_id(cpu));
+ for_each_online_cpu_gic(cpu, &gic_lock)
write_gic_vo_rmask(BIT(intr));
- }
- raw_spin_unlock_irqrestore(&gic_lock, flags);
}
static void gic_unmask_local_irq_all_vpes(struct irq_data *d)
{
struct gic_all_vpes_chip_data *cd;
- unsigned long flags;
int intr, cpu;
+ if (!mips_cps_multicluster_cpus())
+ return;
+
intr = GIC_HWIRQ_TO_LOCAL(d->hwirq);
cd = irq_data_get_irq_chip_data(d);
cd->mask = true;
- raw_spin_lock_irqsave(&gic_lock, flags);
- for_each_online_cpu(cpu) {
- write_gic_vl_other(mips_cm_vp_id(cpu));
+ for_each_online_cpu_gic(cpu, &gic_lock)
write_gic_vo_smask(BIT(intr));
- }
- raw_spin_unlock_irqrestore(&gic_lock, flags);
}
static void gic_all_vpes_irq_cpu_online(void)
@@ -436,11 +588,21 @@ static int gic_shared_irq_domain_map(struct irq_domain *d, unsigned int virq,
unsigned long flags;
data = irq_get_irq_data(virq);
+ irq_data_update_effective_affinity(data, cpumask_of(cpu));
raw_spin_lock_irqsave(&gic_lock, flags);
- write_gic_map_pin(intr, GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin);
- write_gic_map_vp(intr, BIT(mips_cm_vp_id(cpu)));
- irq_data_update_effective_affinity(data, cpumask_of(cpu));
+
+ /* Route the interrupt to its VP(E) */
+ if (gic_irq_lock_cluster(data)) {
+ write_gic_redir_map_pin(intr,
+ GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin);
+ write_gic_redir_map_vp(intr, BIT(mips_cm_vp_id(cpu)));
+ mips_cm_unlock_other();
+ } else {
+ write_gic_map_pin(intr, GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin);
+ write_gic_map_vp(intr, BIT(mips_cm_vp_id(cpu)));
+ }
+
raw_spin_unlock_irqrestore(&gic_lock, flags);
return 0;
@@ -469,7 +631,6 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int virq,
irq_hw_number_t hwirq)
{
struct gic_all_vpes_chip_data *cd;
- unsigned long flags;
unsigned int intr;
int err, cpu;
u32 map;
@@ -533,12 +694,10 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int virq,
if (!gic_local_irq_is_routable(intr))
return -EPERM;
- raw_spin_lock_irqsave(&gic_lock, flags);
- for_each_online_cpu(cpu) {
- write_gic_vl_other(mips_cm_vp_id(cpu));
- write_gic_vo_map(mips_gic_vx_map_reg(intr), map);
+ if (mips_cps_multicluster_cpus()) {
+ for_each_online_cpu_gic(cpu, &gic_lock)
+ write_gic_vo_map(mips_gic_vx_map_reg(intr), map);
}
- raw_spin_unlock_irqrestore(&gic_lock, flags);
return 0;
}
@@ -621,6 +780,9 @@ static int gic_ipi_domain_alloc(struct irq_domain *d, unsigned int virq,
if (ret)
goto error;
+ /* Set affinity to cpu. */
+ irq_data_update_effective_affinity(irq_get_irq_data(virq + i),
+ cpumask_of(cpu));
ret = irq_set_irq_type(virq + i, IRQ_TYPE_EDGE_RISING);
if (ret)
goto error;
@@ -734,7 +896,7 @@ static int gic_cpu_startup(unsigned int cpu)
static int __init gic_of_init(struct device_node *node,
struct device_node *parent)
{
- unsigned int cpu_vec, i, gicconfig;
+ unsigned int cpu_vec, i, gicconfig, cl, nclusters;
unsigned long reserved;
phys_addr_t gic_base;
struct resource res;
@@ -815,11 +977,32 @@ static int __init gic_of_init(struct device_node *node,
board_bind_eic_interrupt = &gic_bind_eic_interrupt;
- /* Setup defaults */
- for (i = 0; i < gic_shared_intrs; i++) {
- change_gic_pol(i, GIC_POL_ACTIVE_HIGH);
- change_gic_trig(i, GIC_TRIG_LEVEL);
- write_gic_rmask(i);
+ /*
+ * Initialise each cluster's GIC shared registers to sane default
+ * values.
+ * Otherwise, the IPI set up will be erased if we move code
+ * to gic_cpu_startup for each cpu.
+ */
+ nclusters = mips_cps_numclusters();
+ for (cl = 0; cl < nclusters; cl++) {
+ if (cl == cpu_cluster(¤t_cpu_data)) {
+ for (i = 0; i < gic_shared_intrs; i++) {
+ change_gic_pol(i, GIC_POL_ACTIVE_HIGH);
+ change_gic_trig(i, GIC_TRIG_LEVEL);
+ write_gic_rmask(i);
+ }
+ } else if (mips_cps_numcores(cl) != 0) {
+ mips_cm_lock_other(cl, 0, 0, CM_GCR_Cx_OTHER_BLOCK_GLOBAL);
+ for (i = 0; i < gic_shared_intrs; i++) {
+ change_gic_redir_pol(i, GIC_POL_ACTIVE_HIGH);
+ change_gic_redir_trig(i, GIC_TRIG_LEVEL);
+ write_gic_redir_rmask(i);
+ }
+ mips_cm_unlock_other();
+
+ } else {
+ pr_warn("No CPU cores on the cluster %d skip it\n", cl);
+ }
}
return cpuhp_setup_state(CPUHP_AP_IRQ_MIPS_GIC_STARTING,
diff --git a/drivers/irqchip/irq-renesas-rzv2h.c b/drivers/irqchip/irq-renesas-rzv2h.c
new file mode 100644
index 000000000000..fe2d29e91026
--- /dev/null
+++ b/drivers/irqchip/irq-renesas-rzv2h.c
@@ -0,0 +1,513 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Renesas RZ/V2H(P) ICU Driver
+ *
+ * Based on irq-renesas-rzg2l.c
+ *
+ * Copyright (C) 2024 Renesas Electronics Corporation.
+ *
+ * Author: Fabrizio Castro <fabrizio.castro.jz@renesas.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/cleanup.h>
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/irqchip.h>
+#include <linux/irqdomain.h>
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+#include <linux/pm_runtime.h>
+#include <linux/reset.h>
+#include <linux/spinlock.h>
+#include <linux/syscore_ops.h>
+
+/* DT "interrupts" indexes */
+#define ICU_IRQ_START 1
+#define ICU_IRQ_COUNT 16
+#define ICU_TINT_START (ICU_IRQ_START + ICU_IRQ_COUNT)
+#define ICU_TINT_COUNT 32
+#define ICU_NUM_IRQ (ICU_TINT_START + ICU_TINT_COUNT)
+
+/* Registers */
+#define ICU_NSCNT 0x00
+#define ICU_NSCLR 0x04
+#define ICU_NITSR 0x08
+#define ICU_ISCTR 0x10
+#define ICU_ISCLR 0x14
+#define ICU_IITSR 0x18
+#define ICU_TSCTR 0x20
+#define ICU_TSCLR 0x24
+#define ICU_TITSR(k) (0x28 + (k) * 4)
+#define ICU_TSSR(k) (0x30 + (k) * 4)
+
+/* NMI */
+#define ICU_NMI_EDGE_FALLING 0
+#define ICU_NMI_EDGE_RISING 1
+
+#define ICU_NSCLR_NCLR BIT(0)
+
+/* IRQ */
+#define ICU_IRQ_LEVEL_LOW 0
+#define ICU_IRQ_EDGE_FALLING 1
+#define ICU_IRQ_EDGE_RISING 2
+#define ICU_IRQ_EDGE_BOTH 3
+
+#define ICU_IITSR_IITSEL_PREP(iitsel, n) ((iitsel) << ((n) * 2))
+#define ICU_IITSR_IITSEL_GET(iitsr, n) (((iitsr) >> ((n) * 2)) & 0x03)
+#define ICU_IITSR_IITSEL_MASK(n) ICU_IITSR_IITSEL_PREP(0x03, n)
+
+/* TINT */
+#define ICU_TINT_EDGE_RISING 0
+#define ICU_TINT_EDGE_FALLING 1
+#define ICU_TINT_LEVEL_HIGH 2
+#define ICU_TINT_LEVEL_LOW 3
+
+#define ICU_TSSR_K(tint_nr) ((tint_nr) / 4)
+#define ICU_TSSR_TSSEL_N(tint_nr) ((tint_nr) % 4)
+#define ICU_TSSR_TSSEL_PREP(tssel, n) ((tssel) << ((n) * 8))
+#define ICU_TSSR_TSSEL_MASK(n) ICU_TSSR_TSSEL_PREP(0x7F, n)
+#define ICU_TSSR_TIEN(n) (BIT(7) << ((n) * 8))
+
+#define ICU_TITSR_K(tint_nr) ((tint_nr) / 16)
+#define ICU_TITSR_TITSEL_N(tint_nr) ((tint_nr) % 16)
+#define ICU_TITSR_TITSEL_PREP(titsel, n) ICU_IITSR_IITSEL_PREP(titsel, n)
+#define ICU_TITSR_TITSEL_MASK(n) ICU_IITSR_IITSEL_MASK(n)
+#define ICU_TITSR_TITSEL_GET(titsr, n) ICU_IITSR_IITSEL_GET(titsr, n)
+
+#define ICU_TINT_EXTRACT_HWIRQ(x) FIELD_GET(GENMASK(15, 0), (x))
+#define ICU_TINT_EXTRACT_GPIOINT(x) FIELD_GET(GENMASK(31, 16), (x))
+#define ICU_PB5_TINT 0x55
+
+/**
+ * struct rzv2h_icu_priv - Interrupt Control Unit controller private data structure.
+ * @base: Controller's base address
+ * @irqchip: Pointer to struct irq_chip
+ * @fwspec: IRQ firmware specific data
+ * @lock: Lock to serialize access to hardware registers
+ */
+struct rzv2h_icu_priv {
+ void __iomem *base;
+ const struct irq_chip *irqchip;
+ struct irq_fwspec fwspec[ICU_NUM_IRQ];
+ raw_spinlock_t lock;
+};
+
+static inline struct rzv2h_icu_priv *irq_data_to_priv(struct irq_data *data)
+{
+ return data->domain->host_data;
+}
+
+static void rzv2h_icu_eoi(struct irq_data *d)
+{
+ struct rzv2h_icu_priv *priv = irq_data_to_priv(d);
+ unsigned int hw_irq = irqd_to_hwirq(d);
+ unsigned int tintirq_nr;
+ u32 bit;
+
+ scoped_guard(raw_spinlock, &priv->lock) {
+ if (hw_irq >= ICU_TINT_START) {
+ tintirq_nr = hw_irq - ICU_TINT_START;
+ bit = BIT(tintirq_nr);
+ if (!irqd_is_level_type(d))
+ writel_relaxed(bit, priv->base + ICU_TSCLR);
+ } else if (hw_irq >= ICU_IRQ_START) {
+ tintirq_nr = hw_irq - ICU_IRQ_START;
+ bit = BIT(tintirq_nr);
+ if (!irqd_is_level_type(d))
+ writel_relaxed(bit, priv->base + ICU_ISCLR);
+ } else {
+ writel_relaxed(ICU_NSCLR_NCLR, priv->base + ICU_NSCLR);
+ }
+ }
+
+ irq_chip_eoi_parent(d);
+}
+
+static void rzv2h_tint_irq_endisable(struct irq_data *d, bool enable)
+{
+ struct rzv2h_icu_priv *priv = irq_data_to_priv(d);
+ unsigned int hw_irq = irqd_to_hwirq(d);
+ u32 tint_nr, tssel_n, k, tssr;
+
+ if (hw_irq < ICU_TINT_START)
+ return;
+
+ tint_nr = hw_irq - ICU_TINT_START;
+ k = ICU_TSSR_K(tint_nr);
+ tssel_n = ICU_TSSR_TSSEL_N(tint_nr);
+
+ guard(raw_spinlock)(&priv->lock);
+ tssr = readl_relaxed(priv->base + ICU_TSSR(k));
+ if (enable)
+ tssr |= ICU_TSSR_TIEN(tssel_n);
+ else
+ tssr &= ~ICU_TSSR_TIEN(tssel_n);
+ writel_relaxed(tssr, priv->base + ICU_TSSR(k));
+}
+
+static void rzv2h_icu_irq_disable(struct irq_data *d)
+{
+ irq_chip_disable_parent(d);
+ rzv2h_tint_irq_endisable(d, false);
+}
+
+static void rzv2h_icu_irq_enable(struct irq_data *d)
+{
+ rzv2h_tint_irq_endisable(d, true);
+ irq_chip_enable_parent(d);
+}
+
+static int rzv2h_nmi_set_type(struct irq_data *d, unsigned int type)
+{
+ struct rzv2h_icu_priv *priv = irq_data_to_priv(d);
+ u32 sense;
+
+ switch (type & IRQ_TYPE_SENSE_MASK) {
+ case IRQ_TYPE_EDGE_FALLING:
+ sense = ICU_NMI_EDGE_FALLING;
+ break;
+
+ case IRQ_TYPE_EDGE_RISING:
+ sense = ICU_NMI_EDGE_RISING;
+ break;
+
+ default:
+ return -EINVAL;
+ }
+
+ writel_relaxed(sense, priv->base + ICU_NITSR);
+
+ return 0;
+}
+
+static void rzv2h_clear_irq_int(struct rzv2h_icu_priv *priv, unsigned int hwirq)
+{
+ unsigned int irq_nr = hwirq - ICU_IRQ_START;
+ u32 isctr, iitsr, iitsel;
+ u32 bit = BIT(irq_nr);
+
+ isctr = readl_relaxed(priv->base + ICU_ISCTR);
+ iitsr = readl_relaxed(priv->base + ICU_IITSR);
+ iitsel = ICU_IITSR_IITSEL_GET(iitsr, irq_nr);
+
+ /*
+ * When level sensing is used, the interrupt flag gets automatically cleared when the
+ * interrupt signal is de-asserted by the source of the interrupt request, therefore clear
+ * the interrupt only for edge triggered interrupts.
+ */
+ if ((isctr & bit) && (iitsel != ICU_IRQ_LEVEL_LOW))
+ writel_relaxed(bit, priv->base + ICU_ISCLR);
+}
+
+static int rzv2h_irq_set_type(struct irq_data *d, unsigned int type)
+{
+ struct rzv2h_icu_priv *priv = irq_data_to_priv(d);
+ unsigned int hwirq = irqd_to_hwirq(d);
+ u32 irq_nr = hwirq - ICU_IRQ_START;
+ u32 iitsr, sense;
+
+ switch (type & IRQ_TYPE_SENSE_MASK) {
+ case IRQ_TYPE_LEVEL_LOW:
+ sense = ICU_IRQ_LEVEL_LOW;
+ break;
+
+ case IRQ_TYPE_EDGE_FALLING:
+ sense = ICU_IRQ_EDGE_FALLING;
+ break;
+
+ case IRQ_TYPE_EDGE_RISING:
+ sense = ICU_IRQ_EDGE_RISING;
+ break;
+
+ case IRQ_TYPE_EDGE_BOTH:
+ sense = ICU_IRQ_EDGE_BOTH;
+ break;
+
+ default:
+ return -EINVAL;
+ }
+
+ guard(raw_spinlock)(&priv->lock);
+ iitsr = readl_relaxed(priv->base + ICU_IITSR);
+ iitsr &= ~ICU_IITSR_IITSEL_MASK(irq_nr);
+ iitsr |= ICU_IITSR_IITSEL_PREP(sense, irq_nr);
+ rzv2h_clear_irq_int(priv, hwirq);
+ writel_relaxed(iitsr, priv->base + ICU_IITSR);
+
+ return 0;
+}
+
+static void rzv2h_clear_tint_int(struct rzv2h_icu_priv *priv, unsigned int hwirq)
+{
+ unsigned int tint_nr = hwirq - ICU_TINT_START;
+ int titsel_n = ICU_TITSR_TITSEL_N(tint_nr);
+ u32 tsctr, titsr, titsel;
+ u32 bit = BIT(tint_nr);
+ int k = tint_nr / 16;
+
+ tsctr = readl_relaxed(priv->base + ICU_TSCTR);
+ titsr = readl_relaxed(priv->base + ICU_TITSR(k));
+ titsel = ICU_TITSR_TITSEL_GET(titsr, titsel_n);
+
+ /*
+ * Writing 1 to the corresponding flag from register ICU_TSCTR only has effect if
+ * TSTATn = 1b and if it's a rising edge or a falling edge interrupt.
+ */
+ if ((tsctr & bit) && ((titsel == ICU_TINT_EDGE_RISING) ||
+ (titsel == ICU_TINT_EDGE_FALLING)))
+ writel_relaxed(bit, priv->base + ICU_TSCLR);
+}
+
+static int rzv2h_tint_set_type(struct irq_data *d, unsigned int type)
+{
+ u32 titsr, titsr_k, titsel_n, tien;
+ struct rzv2h_icu_priv *priv;
+ u32 tssr, tssr_k, tssel_n;
+ unsigned int hwirq;
+ u32 tint, sense;
+ int tint_nr;
+
+ switch (type & IRQ_TYPE_SENSE_MASK) {
+ case IRQ_TYPE_LEVEL_LOW:
+ sense = ICU_TINT_LEVEL_LOW;
+ break;
+
+ case IRQ_TYPE_LEVEL_HIGH:
+ sense = ICU_TINT_LEVEL_HIGH;
+ break;
+
+ case IRQ_TYPE_EDGE_RISING:
+ sense = ICU_TINT_EDGE_RISING;
+ break;
+
+ case IRQ_TYPE_EDGE_FALLING:
+ sense = ICU_TINT_EDGE_FALLING;
+ break;
+
+ default:
+ return -EINVAL;
+ }
+
+ tint = (u32)(uintptr_t)irq_data_get_irq_chip_data(d);
+ if (tint > ICU_PB5_TINT)
+ return -EINVAL;
+
+ priv = irq_data_to_priv(d);
+ hwirq = irqd_to_hwirq(d);
+
+ tint_nr = hwirq - ICU_TINT_START;
+
+ tssr_k = ICU_TSSR_K(tint_nr);
+ tssel_n = ICU_TSSR_TSSEL_N(tint_nr);
+
+ titsr_k = ICU_TITSR_K(tint_nr);
+ titsel_n = ICU_TITSR_TITSEL_N(tint_nr);
+ tien = ICU_TSSR_TIEN(titsel_n);
+
+ guard(raw_spinlock)(&priv->lock);
+
+ tssr = readl_relaxed(priv->base + ICU_TSSR(tssr_k));
+ tssr &= ~(ICU_TSSR_TSSEL_MASK(tssel_n) | tien);
+ tssr |= ICU_TSSR_TSSEL_PREP(tint, tssel_n);
+
+ writel_relaxed(tssr, priv->base + ICU_TSSR(tssr_k));
+
+ titsr = readl_relaxed(priv->base + ICU_TITSR(titsr_k));
+ titsr &= ~ICU_TITSR_TITSEL_MASK(titsel_n);
+ titsr |= ICU_TITSR_TITSEL_PREP(sense, titsel_n);
+
+ writel_relaxed(titsr, priv->base + ICU_TITSR(titsr_k));
+
+ rzv2h_clear_tint_int(priv, hwirq);
+
+ writel_relaxed(tssr | tien, priv->base + ICU_TSSR(tssr_k));
+
+ return 0;
+}
+
+static int rzv2h_icu_set_type(struct irq_data *d, unsigned int type)
+{
+ unsigned int hw_irq = irqd_to_hwirq(d);
+ int ret;
+
+ if (hw_irq >= ICU_TINT_START)
+ ret = rzv2h_tint_set_type(d, type);
+ else if (hw_irq >= ICU_IRQ_START)
+ ret = rzv2h_irq_set_type(d, type);
+ else
+ ret = rzv2h_nmi_set_type(d, type);
+
+ if (ret)
+ return ret;
+
+ return irq_chip_set_type_parent(d, IRQ_TYPE_LEVEL_HIGH);
+}
+
+static const struct irq_chip rzv2h_icu_chip = {
+ .name = "rzv2h-icu",
+ .irq_eoi = rzv2h_icu_eoi,
+ .irq_mask = irq_chip_mask_parent,
+ .irq_unmask = irq_chip_unmask_parent,
+ .irq_disable = rzv2h_icu_irq_disable,
+ .irq_enable = rzv2h_icu_irq_enable,
+ .irq_get_irqchip_state = irq_chip_get_parent_state,
+ .irq_set_irqchip_state = irq_chip_set_parent_state,
+ .irq_retrigger = irq_chip_retrigger_hierarchy,
+ .irq_set_type = rzv2h_icu_set_type,
+ .irq_set_affinity = irq_chip_set_affinity_parent,
+ .flags = IRQCHIP_SET_TYPE_MASKED,
+};
+
+static int rzv2h_icu_alloc(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs,
+ void *arg)
+{
+ struct rzv2h_icu_priv *priv = domain->host_data;
+ unsigned long tint = 0;
+ irq_hw_number_t hwirq;
+ unsigned int type;
+ int ret;
+
+ ret = irq_domain_translate_twocell(domain, arg, &hwirq, &type);
+ if (ret)
+ return ret;
+
+ /*
+ * For TINT interrupts the hwirq and TINT are encoded in
+ * fwspec->param[0].
+ * hwirq is embedded in bits 0-15.
+ * TINT is embedded in bits 16-31.
+ */
+ if (hwirq >= ICU_TINT_START) {
+ tint = ICU_TINT_EXTRACT_GPIOINT(hwirq);
+ hwirq = ICU_TINT_EXTRACT_HWIRQ(hwirq);
+
+ if (hwirq < ICU_TINT_START)
+ return -EINVAL;
+ }
+
+ if (hwirq > (ICU_NUM_IRQ - 1))
+ return -EINVAL;
+
+ ret = irq_domain_set_hwirq_and_chip(domain, virq, hwirq, priv->irqchip,
+ (void *)(uintptr_t)tint);
+ if (ret)
+ return ret;
+
+ return irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, &priv->fwspec[hwirq]);
+}
+
+static const struct irq_domain_ops rzv2h_icu_domain_ops = {
+ .alloc = rzv2h_icu_alloc,
+ .free = irq_domain_free_irqs_common,
+ .translate = irq_domain_translate_twocell,
+};
+
+static int rzv2h_icu_parse_interrupts(struct rzv2h_icu_priv *priv, struct device_node *np)
+{
+ struct of_phandle_args map;
+ unsigned int i;
+ int ret;
+
+ for (i = 0; i < ICU_NUM_IRQ; i++) {
+ ret = of_irq_parse_one(np, i, &map);
+ if (ret)
+ return ret;
+
+ of_phandle_args_to_fwspec(np, map.args, map.args_count, &priv->fwspec[i]);
+ }
+
+ return 0;
+}
+
+static int rzv2h_icu_init(struct device_node *node, struct device_node *parent)
+{
+ struct irq_domain *irq_domain, *parent_domain;
+ struct rzv2h_icu_priv *rzv2h_icu_data;
+ struct platform_device *pdev;
+ struct reset_control *resetn;
+ int ret;
+
+ pdev = of_find_device_by_node(node);
+ if (!pdev)
+ return -ENODEV;
+
+ parent_domain = irq_find_host(parent);
+ if (!parent_domain) {
+ dev_err(&pdev->dev, "cannot find parent domain\n");
+ ret = -ENODEV;
+ goto put_dev;
+ }
+
+ rzv2h_icu_data = devm_kzalloc(&pdev->dev, sizeof(*rzv2h_icu_data), GFP_KERNEL);
+ if (!rzv2h_icu_data) {
+ ret = -ENOMEM;
+ goto put_dev;
+ }
+
+ rzv2h_icu_data->irqchip = &rzv2h_icu_chip;
+
+ rzv2h_icu_data->base = devm_of_iomap(&pdev->dev, pdev->dev.of_node, 0, NULL);
+ if (IS_ERR(rzv2h_icu_data->base)) {
+ ret = PTR_ERR(rzv2h_icu_data->base);
+ goto put_dev;
+ }
+
+ ret = rzv2h_icu_parse_interrupts(rzv2h_icu_data, node);
+ if (ret) {
+ dev_err(&pdev->dev, "cannot parse interrupts: %d\n", ret);
+ goto put_dev;
+ }
+
+ resetn = devm_reset_control_get_exclusive(&pdev->dev, NULL);
+ if (IS_ERR(resetn)) {
+ ret = PTR_ERR(resetn);
+ goto put_dev;
+ }
+
+ ret = reset_control_deassert(resetn);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to deassert resetn pin, %d\n", ret);
+ goto put_dev;
+ }
+
+ pm_runtime_enable(&pdev->dev);
+ ret = pm_runtime_resume_and_get(&pdev->dev);
+ if (ret < 0) {
+ dev_err(&pdev->dev, "pm_runtime_resume_and_get failed: %d\n", ret);
+ goto pm_disable;
+ }
+
+ raw_spin_lock_init(&rzv2h_icu_data->lock);
+
+ irq_domain = irq_domain_add_hierarchy(parent_domain, 0, ICU_NUM_IRQ, node,
+ &rzv2h_icu_domain_ops, rzv2h_icu_data);
+ if (!irq_domain) {
+ dev_err(&pdev->dev, "failed to add irq domain\n");
+ ret = -ENOMEM;
+ goto pm_put;
+ }
+
+ /*
+ * coccicheck complains about a missing put_device call before returning, but it's a false
+ * positive. We still need &pdev->dev after successfully returning from this function.
+ */
+ return 0;
+
+pm_put:
+ pm_runtime_put(&pdev->dev);
+pm_disable:
+ pm_runtime_disable(&pdev->dev);
+ reset_control_assert(resetn);
+put_dev:
+ put_device(&pdev->dev);
+
+ return ret;
+}
+
+IRQCHIP_PLATFORM_DRIVER_BEGIN(rzv2h_icu)
+IRQCHIP_MATCH("renesas,r9a09g057-icu", rzv2h_icu_init)
+IRQCHIP_PLATFORM_DRIVER_END(rzv2h_icu)
+MODULE_AUTHOR("Fabrizio Castro <fabrizio.castro.jz@renesas.com>");
+MODULE_DESCRIPTION("Renesas RZ/V2H(P) ICU Driver");
diff --git a/drivers/irqchip/irq-riscv-aplic-main.c b/drivers/irqchip/irq-riscv-aplic-main.c
index 900e72541db9..93e7c51f944a 100644
--- a/drivers/irqchip/irq-riscv-aplic-main.c
+++ b/drivers/irqchip/irq-riscv-aplic-main.c
@@ -207,7 +207,8 @@ static int aplic_probe(struct platform_device *pdev)
else
rc = aplic_direct_setup(dev, regs);
if (rc)
- dev_err(dev, "failed to setup APLIC in %s mode\n", msi_mode ? "MSI" : "direct");
+ dev_err_probe(dev, rc, "failed to setup APLIC in %s mode\n",
+ msi_mode ? "MSI" : "direct");
#ifdef CONFIG_ACPI
if (!acpi_disabled)
diff --git a/drivers/irqchip/irq-riscv-aplic-msi.c b/drivers/irqchip/irq-riscv-aplic-msi.c
index 945bff28265c..fb8d1838609f 100644
--- a/drivers/irqchip/irq-riscv-aplic-msi.c
+++ b/drivers/irqchip/irq-riscv-aplic-msi.c
@@ -266,6 +266,9 @@ int aplic_msi_setup(struct device *dev, void __iomem *regs)
if (msi_domain)
dev_set_msi_domain(dev, msi_domain);
}
+
+ if (!dev_get_msi_domain(dev))
+ return -EPROBE_DEFER;
}
if (!msi_create_device_irq_domain(dev, MSI_DEFAULT_DOMAIN, &aplic_msi_template,
diff --git a/drivers/irqchip/irq-sifive-plic.c b/drivers/irqchip/irq-sifive-plic.c
index 2f6ef5c495bd..a3b199dd16ef 100644
--- a/drivers/irqchip/irq-sifive-plic.c
+++ b/drivers/irqchip/irq-sifive-plic.c
@@ -251,11 +251,10 @@ static int plic_irq_suspend(void)
priv = per_cpu_ptr(&plic_handlers, smp_processor_id())->priv;
- for (i = 0; i < priv->nr_irqs; i++)
- if (readl(priv->regs + PRIORITY_BASE + i * PRIORITY_PER_ID))
- __set_bit(i, priv->prio_save);
- else
- __clear_bit(i, priv->prio_save);
+ for (i = 0; i < priv->nr_irqs; i++) {
+ __assign_bit(i, priv->prio_save,
+ readl(priv->regs + PRIORITY_BASE + i * PRIORITY_PER_ID));
+ }
for_each_cpu(cpu, cpu_present_mask) {
struct plic_handler *handler = per_cpu_ptr(&plic_handlers, cpu);
diff --git a/drivers/irqchip/irq-stm32mp-exti.c b/drivers/irqchip/irq-stm32mp-exti.c
index 33e0cfdea654..cb83d6cc6113 100644
--- a/drivers/irqchip/irq-stm32mp-exti.c
+++ b/drivers/irqchip/irq-stm32mp-exti.c
@@ -696,8 +696,7 @@ static int stm32mp_exti_probe(struct platform_device *pdev)
if (ret)
return ret;
- if (of_property_read_bool(np, "interrupts-extended"))
- host_data->dt_has_irqs_desc = true;
+ host_data->dt_has_irqs_desc = of_property_present(np, "interrupts-extended");
return 0;
}
diff --git a/drivers/irqchip/irq-thead-c900-aclint-sswi.c b/drivers/irqchip/irq-thead-c900-aclint-sswi.c
new file mode 100644
index 000000000000..b0e366ade427
--- /dev/null
+++ b/drivers/irqchip/irq-thead-c900-aclint-sswi.c
@@ -0,0 +1,176 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2024 Inochi Amaoto <inochiama@gmail.com>
+ */
+
+#define pr_fmt(fmt) "thead-c900-aclint-sswi: " fmt
+#include <linux/cpu.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/irq.h>
+#include <linux/irqchip.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/pci.h>
+#include <linux/spinlock.h>
+#include <linux/smp.h>
+#include <linux/string_choices.h>
+#include <asm/sbi.h>
+#include <asm/vendorid_list.h>
+
+#define THEAD_ACLINT_xSWI_REGISTER_SIZE 4
+
+#define THEAD_C9XX_CSR_SXSTATUS 0x5c0
+#define THEAD_C9XX_SXSTATUS_CLINTEE BIT(17)
+
+static int sswi_ipi_virq __ro_after_init;
+static DEFINE_PER_CPU(void __iomem *, sswi_cpu_regs);
+
+static void thead_aclint_sswi_ipi_send(unsigned int cpu)
+{
+ writel_relaxed(0x1, per_cpu(sswi_cpu_regs, cpu));
+}
+
+static void thead_aclint_sswi_ipi_clear(void)
+{
+ writel_relaxed(0x0, this_cpu_read(sswi_cpu_regs));
+}
+
+static void thead_aclint_sswi_ipi_handle(struct irq_desc *desc)
+{
+ struct irq_chip *chip = irq_desc_get_chip(desc);
+
+ chained_irq_enter(chip, desc);
+
+ csr_clear(CSR_IP, IE_SIE);
+ thead_aclint_sswi_ipi_clear();
+
+ ipi_mux_process();
+
+ chained_irq_exit(chip, desc);
+}
+
+static int thead_aclint_sswi_starting_cpu(unsigned int cpu)
+{
+ enable_percpu_irq(sswi_ipi_virq, irq_get_trigger_type(sswi_ipi_virq));
+
+ return 0;
+}
+
+static int thead_aclint_sswi_dying_cpu(unsigned int cpu)
+{
+ thead_aclint_sswi_ipi_clear();
+
+ disable_percpu_irq(sswi_ipi_virq);
+
+ return 0;
+}
+
+static int __init thead_aclint_sswi_parse_irq(struct fwnode_handle *fwnode,
+ void __iomem *reg)
+{
+ struct of_phandle_args parent;
+ unsigned long hartid;
+ u32 contexts, i;
+ int rc, cpu;
+
+ contexts = of_irq_count(to_of_node(fwnode));
+ if (!(contexts)) {
+ pr_err("%pfwP: no ACLINT SSWI context available\n", fwnode);
+ return -EINVAL;
+ }
+
+ for (i = 0; i < contexts; i++) {
+ rc = of_irq_parse_one(to_of_node(fwnode), i, &parent);
+ if (rc)
+ return rc;
+
+ rc = riscv_of_parent_hartid(parent.np, &hartid);
+ if (rc)
+ return rc;
+
+ if (parent.args[0] != RV_IRQ_SOFT)
+ return -ENOTSUPP;
+
+ cpu = riscv_hartid_to_cpuid(hartid);
+
+ per_cpu(sswi_cpu_regs, cpu) = reg + i * THEAD_ACLINT_xSWI_REGISTER_SIZE;
+ }
+
+ pr_info("%pfwP: register %u CPU%s\n", fwnode, contexts, str_plural(contexts));
+
+ return 0;
+}
+
+static int __init thead_aclint_sswi_probe(struct fwnode_handle *fwnode)
+{
+ struct irq_domain *domain;
+ void __iomem *reg;
+ int virq, rc;
+
+ /* If it is T-HEAD CPU, check whether SSWI is enabled */
+ if (riscv_cached_mvendorid(0) == THEAD_VENDOR_ID &&
+ !(csr_read(THEAD_C9XX_CSR_SXSTATUS) & THEAD_C9XX_SXSTATUS_CLINTEE))
+ return -ENOTSUPP;
+
+ if (!is_of_node(fwnode))
+ return -EINVAL;
+
+ reg = of_iomap(to_of_node(fwnode), 0);
+ if (!reg)
+ return -ENOMEM;
+
+ /* Parse SSWI setting */
+ rc = thead_aclint_sswi_parse_irq(fwnode, reg);
+ if (rc < 0)
+ return rc;
+
+ /* If mulitple SSWI devices are present, do not register irq again */
+ if (sswi_ipi_virq)
+ return 0;
+
+ /* Find riscv intc domain and create IPI irq mapping */
+ domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), DOMAIN_BUS_ANY);
+ if (!domain) {
+ pr_err("%pfwP: Failed to find INTC domain\n", fwnode);
+ return -ENOENT;
+ }
+
+ sswi_ipi_virq = irq_create_mapping(domain, RV_IRQ_SOFT);
+ if (!sswi_ipi_virq) {
+ pr_err("unable to create ACLINT SSWI IRQ mapping\n");
+ return -ENOMEM;
+ }
+
+ /* Register SSWI irq and handler */
+ virq = ipi_mux_create(BITS_PER_BYTE, thead_aclint_sswi_ipi_send);
+ if (virq <= 0) {
+ pr_err("unable to create muxed IPIs\n");
+ irq_dispose_mapping(sswi_ipi_virq);
+ return virq < 0 ? virq : -ENOMEM;
+ }
+
+ irq_set_chained_handler(sswi_ipi_virq, thead_aclint_sswi_ipi_handle);
+
+ cpuhp_setup_state(CPUHP_AP_IRQ_THEAD_ACLINT_SSWI_STARTING,
+ "irqchip/thead-aclint-sswi:starting",
+ thead_aclint_sswi_starting_cpu,
+ thead_aclint_sswi_dying_cpu);
+
+ riscv_ipi_set_virq_range(virq, BITS_PER_BYTE);
+
+ /* Announce that SSWI is providing IPIs */
+ pr_info("providing IPIs using THEAD ACLINT SSWI\n");
+
+ return 0;
+}
+
+static int __init thead_aclint_sswi_early_probe(struct device_node *node,
+ struct device_node *parent)
+{
+ return thead_aclint_sswi_probe(&node->fwnode);
+}
+IRQCHIP_DECLARE(thead_aclint_sswi, "thead,c900-aclint-sswi", thead_aclint_sswi_early_probe);
diff --git a/drivers/net/ethernet/3com/3c59x.c b/drivers/net/ethernet/3com/3c59x.c
index 082388bb6169..790270912913 100644
--- a/drivers/net/ethernet/3com/3c59x.c
+++ b/drivers/net/ethernet/3com/3c59x.c
@@ -1302,7 +1302,7 @@ static int vortex_probe1(struct device *gendev, void __iomem *ioaddr, int irq,
if (print_info)
pr_cont(", IRQ %d\n", dev->irq);
/* Tell them about an invalid IRQ. */
- if (dev->irq <= 0 || dev->irq >= nr_irqs)
+ if (dev->irq <= 0 || dev->irq >= irq_get_nr_irqs())
pr_warn(" *** Warning: IRQ %d is unlikely to work! ***\n",
dev->irq);
diff --git a/drivers/net/hamradio/baycom_ser_fdx.c b/drivers/net/hamradio/baycom_ser_fdx.c
index 646f605e358f..799f8ece7824 100644
--- a/drivers/net/hamradio/baycom_ser_fdx.c
+++ b/drivers/net/hamradio/baycom_ser_fdx.c
@@ -373,6 +373,7 @@ static enum uart ser12_check_uart(unsigned int iobase)
static int ser12_open(struct net_device *dev)
{
+ const unsigned int nr_irqs = irq_get_nr_irqs();
struct baycom_state *bc = netdev_priv(dev);
enum uart u;
diff --git a/drivers/net/hamradio/scc.c b/drivers/net/hamradio/scc.c
index a9184a78650b..c71e52249289 100644
--- a/drivers/net/hamradio/scc.c
+++ b/drivers/net/hamradio/scc.c
@@ -1460,6 +1460,7 @@ scc_start_calibrate(struct scc_channel *scc, int duration, unsigned char pattern
static void z8530_init(void)
{
+ const unsigned int nr_irqs = irq_get_nr_irqs();
struct scc_channel *scc;
int chip, k;
unsigned long flags;
@@ -1735,7 +1736,7 @@ static int scc_net_siocdevprivate(struct net_device *dev,
if (hwcfg.irq == 2) hwcfg.irq = 9;
- if (hwcfg.irq < 0 || hwcfg.irq >= nr_irqs)
+ if (hwcfg.irq < 0 || hwcfg.irq >= irq_get_nr_irqs())
return -EINVAL;
if (!Ivec[hwcfg.irq].used && hwcfg.irq)
@@ -2117,6 +2118,7 @@ static int __init scc_init_driver (void)
static void __exit scc_cleanup_driver(void)
{
+ const unsigned int nr_irqs = irq_get_nr_irqs();
io_port ctrl;
int k;
struct scc_channel *scc;
diff --git a/drivers/scsi/aha152x.c b/drivers/scsi/aha152x.c
index a0fb330b8df5..4276f868cd91 100644
--- a/drivers/scsi/aha152x.c
+++ b/drivers/scsi/aha152x.c
@@ -295,7 +295,7 @@ CMD_INC_RESID(struct scsi_cmnd *cmd, int inc)
#else
#define IRQ_MIN 9
#if defined(__PPC)
-#define IRQ_MAX (nr_irqs-1)
+#define IRQ_MAX (irq_get_nr_irqs()-1)
#else
#define IRQ_MAX 12
#endif
diff --git a/drivers/sh/intc/virq-debugfs.c b/drivers/sh/intc/virq-debugfs.c
index 939915a07d99..5dd8febe6da5 100644
--- a/drivers/sh/intc/virq-debugfs.c
+++ b/drivers/sh/intc/virq-debugfs.c
@@ -18,6 +18,7 @@
static int intc_irq_xlate_show(struct seq_file *m, void *priv)
{
+ const unsigned int nr_irqs = irq_get_nr_irqs();
int i;
seq_printf(m, "%-5s %-7s %-15s\n", "irq", "enum", "chip name");
diff --git a/drivers/soc/renesas/Kconfig b/drivers/soc/renesas/Kconfig
index 5d94c3f31494..9f7fe02310b9 100644
--- a/drivers/soc/renesas/Kconfig
+++ b/drivers/soc/renesas/Kconfig
@@ -347,6 +347,7 @@ config ARCH_R9A09G011
config ARCH_R9A09G057
bool "ARM64 Platform support for RZ/V2H(P)"
+ select RENESAS_RZV2H_ICU
help
This enables support for the Renesas RZ/V2H(P) SoC variants.
diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c
index 3509af7dc52b..0b886c0924da 100644
--- a/drivers/tty/serial/8250/8250_port.c
+++ b/drivers/tty/serial/8250/8250_port.c
@@ -3176,7 +3176,7 @@ static void serial8250_config_port(struct uart_port *port, int flags)
static int
serial8250_verify_port(struct uart_port *port, struct serial_struct *ser)
{
- if (ser->irq >= nr_irqs || ser->irq < 0 ||
+ if (ser->irq >= irq_get_nr_irqs() || ser->irq < 0 ||
ser->baud_base < 9600 || ser->type < PORT_UNKNOWN ||
ser->type >= ARRAY_SIZE(uart_config) || ser->type == PORT_CIRRUS ||
ser->type == PORT_STARTECH)
diff --git a/drivers/tty/serial/amba-pl010.c b/drivers/tty/serial/amba-pl010.c
index eabbf8afc9b5..c3a7fad02ac9 100644
--- a/drivers/tty/serial/amba-pl010.c
+++ b/drivers/tty/serial/amba-pl010.c
@@ -499,7 +499,7 @@ static int pl010_verify_port(struct uart_port *port, struct serial_struct *ser)
int ret = 0;
if (ser->type != PORT_UNKNOWN && ser->type != PORT_AMBA)
ret = -EINVAL;
- if (ser->irq < 0 || ser->irq >= nr_irqs)
+ if (ser->irq < 0 || ser->irq >= irq_get_nr_irqs())
ret = -EINVAL;
if (ser->baud_base < 9600)
ret = -EINVAL;
diff --git a/drivers/tty/serial/amba-pl011.c b/drivers/tty/serial/amba-pl011.c
index 7d0134ecd82f..1c60850030b1 100644
--- a/drivers/tty/serial/amba-pl011.c
+++ b/drivers/tty/serial/amba-pl011.c
@@ -2202,7 +2202,7 @@ static int pl011_verify_port(struct uart_port *port, struct serial_struct *ser)
if (ser->type != PORT_UNKNOWN && ser->type != PORT_AMBA)
ret = -EINVAL;
- if (ser->irq < 0 || ser->irq >= nr_irqs)
+ if (ser->irq < 0 || ser->irq >= irq_get_nr_irqs())
ret = -EINVAL;
if (ser->baud_base < 9600)
ret = -EINVAL;
diff --git a/drivers/tty/serial/cpm_uart.c b/drivers/tty/serial/cpm_uart.c
index a927478f581d..6eb8625de435 100644
--- a/drivers/tty/serial/cpm_uart.c
+++ b/drivers/tty/serial/cpm_uart.c
@@ -631,7 +631,7 @@ static int cpm_uart_verify_port(struct uart_port *port,
if (ser->type != PORT_UNKNOWN && ser->type != PORT_CPM)
ret = -EINVAL;
- if (ser->irq < 0 || ser->irq >= nr_irqs)
+ if (ser->irq < 0 || ser->irq >= irq_get_nr_irqs())
ret = -EINVAL;
if (ser->baud_base < 9600)
ret = -EINVAL;
diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c
index d94d73e45fb6..74fa02b23772 100644
--- a/drivers/tty/serial/serial_core.c
+++ b/drivers/tty/serial/serial_core.c
@@ -919,7 +919,7 @@ static int uart_set_info(struct tty_struct *tty, struct tty_port *port,
if (uport->ops->verify_port)
retval = uport->ops->verify_port(uport, new_info);
- if ((new_info->irq >= nr_irqs) || (new_info->irq < 0) ||
+ if ((new_info->irq >= irq_get_nr_irqs()) || (new_info->irq < 0) ||
(new_info->baud_base < 9600))
retval = -EINVAL;
diff --git a/drivers/tty/serial/ucc_uart.c b/drivers/tty/serial/ucc_uart.c
index 53bb8c5ef499..4eed909468ff 100644
--- a/drivers/tty/serial/ucc_uart.c
+++ b/drivers/tty/serial/ucc_uart.c
@@ -1045,7 +1045,7 @@ static int qe_uart_verify_port(struct uart_port *port,
if (ser->type != PORT_UNKNOWN && ser->type != PORT_CPM)
return -EINVAL;
- if (ser->irq < 0 || ser->irq >= nr_irqs)
+ if (ser->irq < 0 || ser->irq >= irq_get_nr_irqs())
return -EINVAL;
if (ser->baud_base < 9600)
diff --git a/drivers/xen/events/events_base.c b/drivers/xen/events/events_base.c
index 81effbd53dc5..985e155ebe4b 100644
--- a/drivers/xen/events/events_base.c
+++ b/drivers/xen/events/events_base.c
@@ -411,7 +411,7 @@ static evtchn_port_t evtchn_from_irq(unsigned int irq)
{
const struct irq_info *info = NULL;
- if (likely(irq < nr_irqs))
+ if (likely(irq < irq_get_nr_irqs()))
info = info_for_irq(irq);
if (!info)
return 0;
diff --git a/fs/proc/interrupts.c b/fs/proc/interrupts.c
index cb0edc7cbf09..714a22ded8a8 100644
--- a/fs/proc/interrupts.c
+++ b/fs/proc/interrupts.c
@@ -11,13 +11,13 @@
*/
static void *int_seq_start(struct seq_file *f, loff_t *pos)
{
- return (*pos <= nr_irqs) ? pos : NULL;
+ return *pos <= irq_get_nr_irqs() ? pos : NULL;
}
static void *int_seq_next(struct seq_file *f, void *v, loff_t *pos)
{
(*pos)++;
- if (*pos > nr_irqs)
+ if (*pos > irq_get_nr_irqs())
return NULL;
return pos;
}
diff --git a/fs/proc/stat.c b/fs/proc/stat.c
index da60956b2915..8b444e862319 100644
--- a/fs/proc/stat.c
+++ b/fs/proc/stat.c
@@ -76,7 +76,7 @@ static void show_all_irqs(struct seq_file *p)
seq_put_decimal_ull(p, " ", kstat_irqs_usr(i));
next = i + 1;
}
- show_irq_gap(p, nr_irqs - next);
+ show_irq_gap(p, irq_get_nr_irqs() - next);
}
static int show_stat(struct seq_file *p, void *v)
@@ -196,7 +196,7 @@ static int stat_open(struct inode *inode, struct file *file)
unsigned int size = 1024 + 128 * num_online_cpus();
/* minimum size to display an interrupt count : 2 bytes */
- size += 2 * nr_irqs;
+ size += 2 * irq_get_nr_irqs();
return single_open_size(file, show_stat, NULL, size);
}
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 2361ed4d2b15..799052249c7b 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -147,6 +147,7 @@ enum cpuhp_state {
CPUHP_AP_IRQ_EIOINTC_STARTING,
CPUHP_AP_IRQ_AVECINTC_STARTING,
CPUHP_AP_IRQ_SIFIVE_PLIC_STARTING,
+ CPUHP_AP_IRQ_THEAD_ACLINT_SSWI_STARTING,
CPUHP_AP_IRQ_RISCV_IMSIC_STARTING,
CPUHP_AP_IRQ_RISCV_SBI_IPI_STARTING,
CPUHP_AP_ARM_MVEBU_COHERENCY,
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 457151f9f263..8cd9327e4e78 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -616,6 +616,53 @@ extern void __raise_softirq_irqoff(unsigned int nr);
extern void raise_softirq_irqoff(unsigned int nr);
extern void raise_softirq(unsigned int nr);
+/*
+ * With forced-threaded interrupts enabled a raised softirq is deferred to
+ * ksoftirqd unless it can be handled within the threaded interrupt. This
+ * affects timer_list timers and hrtimers which are explicitly marked with
+ * HRTIMER_MODE_SOFT.
+ * With PREEMPT_RT enabled more hrtimers are moved to softirq for processing
+ * which includes all timers which are not explicitly marked HRTIMER_MODE_HARD.
+ * Userspace controlled timers (like the clock_nanosleep() interface) is divided
+ * into two categories: Tasks with elevated scheduling policy including
+ * SCHED_{FIFO|RR|DL} and the remaining scheduling policy. The tasks with the
+ * elevated scheduling policy are woken up directly from the HARDIRQ while all
+ * other wake ups are delayed to softirq and so to ksoftirqd.
+ *
+ * The ksoftirqd runs at SCHED_OTHER policy at which it should remain since it
+ * handles the softirq in an overloaded situation (not handled everything
+ * within its last run).
+ * If the timers are handled at SCHED_OTHER priority then they competes with all
+ * other SCHED_OTHER tasks for CPU resources are possibly delayed.
+ * Moving timers softirqs to a low priority SCHED_FIFO thread instead ensures
+ * that timer are performed before scheduling any SCHED_OTHER thread.
+ */
+DECLARE_PER_CPU(struct task_struct *, ktimerd);
+DECLARE_PER_CPU(unsigned long, pending_timer_softirq);
+void raise_ktimers_thread(unsigned int nr);
+
+static inline unsigned int local_timers_pending_force_th(void)
+{
+ return __this_cpu_read(pending_timer_softirq);
+}
+
+static inline void raise_timer_softirq(unsigned int nr)
+{
+ lockdep_assert_in_irq();
+ if (force_irqthreads())
+ raise_ktimers_thread(nr);
+ else
+ __raise_softirq_irqoff(nr);
+}
+
+static inline unsigned int local_timers_pending(void)
+{
+ if (force_irqthreads())
+ return local_timers_pending_force_th();
+ else
+ return local_softirq_pending();
+}
+
DECLARE_PER_CPU(struct task_struct *, ksoftirqd);
static inline struct task_struct *this_cpu_ksoftirqd(void)
diff --git a/include/linux/irqnr.h b/include/linux/irqnr.h
index 3496baa0b07f..e97206c721a0 100644
--- a/include/linux/irqnr.h
+++ b/include/linux/irqnr.h
@@ -5,30 +5,36 @@
#include <uapi/linux/irqnr.h>
-extern int nr_irqs;
+unsigned int irq_get_nr_irqs(void) __pure;
+unsigned int irq_set_nr_irqs(unsigned int nr);
extern struct irq_desc *irq_to_desc(unsigned int irq);
unsigned int irq_get_next_irq(unsigned int offset);
-# define for_each_irq_desc(irq, desc) \
- for (irq = 0, desc = irq_to_desc(irq); irq < nr_irqs; \
- irq++, desc = irq_to_desc(irq)) \
- if (!desc) \
- ; \
- else
-
+#define for_each_irq_desc(irq, desc) \
+ for (unsigned int __nr_irqs__ = irq_get_nr_irqs(); __nr_irqs__; \
+ __nr_irqs__ = 0) \
+ for (irq = 0, desc = irq_to_desc(irq); irq < __nr_irqs__; \
+ irq++, desc = irq_to_desc(irq)) \
+ if (!desc) \
+ ; \
+ else
# define for_each_irq_desc_reverse(irq, desc) \
- for (irq = nr_irqs - 1, desc = irq_to_desc(irq); irq >= 0; \
- irq--, desc = irq_to_desc(irq)) \
+ for (irq = irq_get_nr_irqs() - 1, desc = irq_to_desc(irq); \
+ irq >= 0; irq--, desc = irq_to_desc(irq)) \
if (!desc) \
; \
else
-# define for_each_active_irq(irq) \
- for (irq = irq_get_next_irq(0); irq < nr_irqs; \
- irq = irq_get_next_irq(irq + 1))
+#define for_each_active_irq(irq) \
+ for (unsigned int __nr_irqs__ = irq_get_nr_irqs(); __nr_irqs__; \
+ __nr_irqs__ = 0) \
+ for (irq = irq_get_next_irq(0); irq < __nr_irqs__; \
+ irq = irq_get_next_irq(irq + 1))
-#define for_each_irq_nr(irq) \
- for (irq = 0; irq < nr_irqs; irq++)
+#define for_each_irq_nr(irq) \
+ for (unsigned int __nr_irqs__ = irq_get_nr_irqs(); __nr_irqs__; \
+ __nr_irqs__ = 0) \
+ for (irq = 0; irq < __nr_irqs__; irq++)
#endif
diff --git a/kernel/irq/devres.c b/kernel/irq/devres.c
index b3e98668f4dd..eb16a58e0322 100644
--- a/kernel/irq/devres.c
+++ b/kernel/irq/devres.c
@@ -141,9 +141,8 @@ void devm_free_irq(struct device *dev, unsigned int irq, void *dev_id)
{
struct irq_devres match_data = { irq, dev_id };
- WARN_ON(devres_destroy(dev, devm_irq_release, devm_irq_match,
+ WARN_ON(devres_release(dev, devm_irq_release, devm_irq_match,
&match_data));
- free_irq(irq, dev_id);
}
EXPORT_SYMBOL(devm_free_irq);
diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c
index 1dee88ba0ae4..0253e77fcd9a 100644
--- a/kernel/irq/irqdesc.c
+++ b/kernel/irq/irqdesc.c
@@ -15,6 +15,7 @@
#include <linux/maple_tree.h>
#include <linux/irqdomain.h>
#include <linux/sysfs.h>
+#include <linux/string_choices.h>
#include "internals.h"
@@ -138,8 +139,30 @@ static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node,
desc_smp_init(desc, node, affinity);
}
-int nr_irqs = NR_IRQS;
-EXPORT_SYMBOL_GPL(nr_irqs);
+static unsigned int nr_irqs = NR_IRQS;
+
+/**
+ * irq_get_nr_irqs() - Number of interrupts supported by the system.
+ */
+unsigned int irq_get_nr_irqs(void)
+{
+ return nr_irqs;
+}
+EXPORT_SYMBOL_GPL(irq_get_nr_irqs);
+
+/**
+ * irq_set_nr_irqs() - Set the number of interrupts supported by the system.
+ * @nr: New number of interrupts.
+ *
+ * Return: @nr.
+ */
+unsigned int irq_set_nr_irqs(unsigned int nr)
+{
+ nr_irqs = nr;
+
+ return nr;
+}
+EXPORT_SYMBOL_GPL(irq_set_nr_irqs);
static DEFINE_MUTEX(sparse_irq_lock);
static struct maple_tree sparse_irqs = MTREE_INIT_EXT(sparse_irqs,
@@ -298,8 +321,7 @@ static ssize_t wakeup_show(struct kobject *kobj,
ssize_t ret = 0;
raw_spin_lock_irq(&desc->lock);
- ret = sprintf(buf, "%s\n",
- irqd_is_wakeup_set(&desc->irq_data) ? "enabled" : "disabled");
+ ret = sprintf(buf, "%s\n", str_enabled_disabled(irqd_is_wakeup_set(&desc->irq_data)));
raw_spin_unlock_irq(&desc->lock);
return ret;
diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c
index e0bff21f30e0..ec6d8e72d980 100644
--- a/kernel/irq/irqdomain.c
+++ b/kernel/irq/irqdomain.c
@@ -1225,7 +1225,7 @@ int irq_domain_alloc_descs(int virq, unsigned int cnt, irq_hw_number_t hwirq,
virq = __irq_alloc_descs(virq, virq, cnt, node, THIS_MODULE,
affinity);
} else {
- hint = hwirq % nr_irqs;
+ hint = hwirq % irq_get_nr_irqs();
if (hint == 0)
hint++;
virq = __irq_alloc_descs(-1, hint, cnt, node, THIS_MODULE,
diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c
index 9081ada81c3d..f36c33bd2da4 100644
--- a/kernel/irq/proc.c
+++ b/kernel/irq/proc.c
@@ -457,11 +457,12 @@ int __weak arch_show_interrupts(struct seq_file *p, int prec)
}
#ifndef ACTUAL_NR_IRQS
-# define ACTUAL_NR_IRQS nr_irqs
+# define ACTUAL_NR_IRQS irq_get_nr_irqs()
#endif
int show_interrupts(struct seq_file *p, void *v)
{
+ const unsigned int nr_irqs = irq_get_nr_irqs();
static int prec;
int i = *(loff_t *) v, j;
@@ -494,9 +495,12 @@ int show_interrupts(struct seq_file *p, void *v)
if (!desc->action || irq_desc_is_chained(desc) || !desc->kstat_irqs)
goto outsparse;
- seq_printf(p, "%*d: ", prec, i);
- for_each_online_cpu(j)
- seq_printf(p, "%10u ", desc->kstat_irqs ? per_cpu(desc->kstat_irqs->cnt, j) : 0);
+ seq_printf(p, "%*d:", prec, i);
+ for_each_online_cpu(j) {
+ unsigned int cnt = desc->kstat_irqs ? per_cpu(desc->kstat_irqs->cnt, j) : 0;
+
+ seq_put_decimal_ull_width(p, " ", cnt, 10);
+ }
raw_spin_lock_irqsave(&desc->lock, flags);
if (desc->irq_data.chip) {
diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index bb75dbf5c800..270c31a1e857 100644
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -2440,6 +2440,14 @@ static int rcutorture_booster_init(unsigned int cpu)
WARN_ON_ONCE(!t);
sp.sched_priority = 2;
sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
+#ifdef CONFIG_IRQ_FORCED_THREADING
+ if (force_irqthreads()) {
+ t = per_cpu(ktimerd, cpu);
+ WARN_ON_ONCE(!t);
+ sp.sched_priority = 2;
+ sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
+ }
+#endif
}
/* Don't allow time recalculation while creating a new task. */
diff --git a/kernel/softirq.c b/kernel/softirq.c
index d082e7840f88..7b525c904462 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -624,6 +624,24 @@ static inline void tick_irq_exit(void)
#endif
}
+#ifdef CONFIG_IRQ_FORCED_THREADING
+DEFINE_PER_CPU(struct task_struct *, ktimerd);
+DEFINE_PER_CPU(unsigned long, pending_timer_softirq);
+
+static void wake_timersd(void)
+{
+ struct task_struct *tsk = __this_cpu_read(ktimerd);
+
+ if (tsk)
+ wake_up_process(tsk);
+}
+
+#else
+
+static inline void wake_timersd(void) { }
+
+#endif
+
static inline void __irq_exit_rcu(void)
{
#ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
@@ -636,6 +654,10 @@ static inline void __irq_exit_rcu(void)
if (!in_interrupt() && local_softirq_pending())
invoke_softirq();
+ if (IS_ENABLED(CONFIG_IRQ_FORCED_THREADING) && force_irqthreads() &&
+ local_timers_pending_force_th() && !(in_nmi() | in_hardirq()))
+ wake_timersd();
+
tick_irq_exit();
}
@@ -971,12 +993,57 @@ static struct smp_hotplug_thread softirq_threads = {
.thread_comm = "ksoftirqd/%u",
};
+#ifdef CONFIG_IRQ_FORCED_THREADING
+static void ktimerd_setup(unsigned int cpu)
+{
+ /* Above SCHED_NORMAL to handle timers before regular tasks. */
+ sched_set_fifo_low(current);
+}
+
+static int ktimerd_should_run(unsigned int cpu)
+{
+ return local_timers_pending_force_th();
+}
+
+void raise_ktimers_thread(unsigned int nr)
+{
+ trace_softirq_raise(nr);
+ __this_cpu_or(pending_timer_softirq, BIT(nr));
+}
+
+static void run_ktimerd(unsigned int cpu)
+{
+ unsigned int timer_si;
+
+ ksoftirqd_run_begin();
+
+ timer_si = local_timers_pending_force_th();
+ __this_cpu_write(pending_timer_softirq, 0);
+ or_softirq_pending(timer_si);
+
+ __do_softirq();
+
+ ksoftirqd_run_end();
+}
+
+static struct smp_hotplug_thread timer_thread = {
+ .store = &ktimerd,
+ .setup = ktimerd_setup,
+ .thread_should_run = ktimerd_should_run,
+ .thread_fn = run_ktimerd,
+ .thread_comm = "ktimers/%u",
+};
+#endif
+
static __init int spawn_ksoftirqd(void)
{
cpuhp_setup_state_nocalls(CPUHP_SOFTIRQ_DEAD, "softirq:dead", NULL,
takeover_tasklets);
BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
-
+#ifdef CONFIG_IRQ_FORCED_THREADING
+ if (force_irqthreads())
+ BUG_ON(smpboot_register_percpu_thread(&timer_thread));
+#endif
return 0;
}
early_initcall(spawn_ksoftirqd);
diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c
index cddcd08ea827..d9911516e743 100644
--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -1811,7 +1811,7 @@ void hrtimer_interrupt(struct clock_event_device *dev)
if (!ktime_before(now, cpu_base->softirq_expires_next)) {
cpu_base->softirq_expires_next = KTIME_MAX;
cpu_base->softirq_activated = 1;
- raise_softirq_irqoff(HRTIMER_SOFTIRQ);
+ raise_timer_softirq(HRTIMER_SOFTIRQ);
}
__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
@@ -1906,7 +1906,7 @@ void hrtimer_run_queues(void)
if (!ktime_before(now, cpu_base->softirq_expires_next)) {
cpu_base->softirq_expires_next = KTIME_MAX;
cpu_base->softirq_activated = 1;
- raise_softirq_irqoff(HRTIMER_SOFTIRQ);
+ raise_timer_softirq(HRTIMER_SOFTIRQ);
}
__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 753a184c7090..976a212cca2e 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -859,7 +859,7 @@ static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
static inline bool local_timer_softirq_pending(void)
{
- return local_softirq_pending() & BIT(TIMER_SOFTIRQ);
+ return local_timers_pending() & BIT(TIMER_SOFTIRQ);
}
/*
diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index 0fc9d066a7be..06f0bc1db6d9 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -2499,7 +2499,7 @@ static void run_local_timers(void)
*/
if (time_after_eq(jiffies, READ_ONCE(base->next_expiry)) ||
(i == BASE_DEF && tmigr_requires_handle_remote())) {
- raise_softirq(TIMER_SOFTIRQ);
+ raise_timer_softirq(TIMER_SOFTIRQ);
return;
}
}The pull request you sent on Mon, 18 Nov 2024 20:21:52 +0100 (CET): > git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git irq-core-2024-11-18 has been merged into torvalds/linux.git: https://git.kernel.org/torvalds/c/5c2b050848337f393011ee7fcd2e9f2663eec40d Thank you! -- Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/prtracker.html
Linus,
please pull the latest timers/vdso branch from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git timers-vdso-2024-11-18
up to: 7fa3c36ea270: x86/vdso: Add missing brackets in switch case
First step of consolidating the VDSO data page handling:
The VDSO data page handling is architecture specific for historical
reasons, but there is no real technical reason to do so.
Aside of that VDSO data has become a dump ground for various mechanisms
and fail to provide a clear separation of the functionalities.
Clean this up by:
* consolidating the VDSO page data by getting rid of architecture
specific warts especially in x86 and PowerPC.
* removing the last includes of header files which are pulling in other
headers outside of the VDSO namespace.
* seperating timekeeping and other VDSO data accordingly.
Further consolidation of the VDSO page handling is done in subsequent
changes scheduled for the next merge window.
This also lays the ground for expanding the VDSO time getters for
independent PTP clocks in a generic way without making every architecture
add support seperately.
Thanks,
tglx
------------------>
Arnd Bergmann (1):
vdso: Change PAGE_MASK to signed on all 32-bit architectures
Nam Cao (1):
vdso: Rename struct arch_vdso_data to arch_vdso_time_data
Thomas Gleixner (1):
x86/vdso: Add missing brackets in switch case
Thomas Weißschuh (36):
vdso: Remove timekeeper argument of __arch_update_vsyscall()
arm: vdso: Remove timekeeper includes
arm64: vdso: Remove timekeeper include
powerpc/vdso: Remove timekeeper includes
riscv: vdso: Remove timekeeper include
s390/vdso: Remove timekeeper includes
x86/vdso: Remove timekeeper include
LoongArch: vdso: Remove timekeeper includes
MIPS: vdso: Remove timekeeper includes
csky/vdso: Remove gettimeofday() and friends from VDSO
csky/vdso: Remove arch_vma_name()
s390/vdso: Drop LBASE_VDSO
arm64: vdso: Drop LBASE_VDSO
arm64: vdso: Use only one single vvar mapping
riscv: vdso: Use only one single vvar mapping
ARM: vdso: Remove assembly for datapage access
LoongArch: vDSO: Use vdso/datapage.h to access vDSO data
MIPS: vdso: Avoid name conflict around "vdso_data"
x86/mm/mmap: Remove arch_vma_name()
x86/vdso: Use __arch_get_vdso_data() to access vdso data
x86/vdso: Place vdso_data at beginning of vvar page
x86/vdso: Access rng data from kernel without vvar
x86/vdso: Allocate vvar page from C code
x86/vdso: Access timens vdso data without vvar.h
x86/vdso: Access rng vdso data without vvar.h
x86/vdso: Move the rng offset to vsyscall.h
x86/vdso: Access vdso data without vvar.h
x86/vdso: Delete vvar.h
x86/vdso: Split virtual clock pages into dedicated mapping
powerpc/vdso: Remove offset comment from 32bit vdso_arch_data
powerpc/procfs: Propagate error of remap_pfn_range()
powerpc/pseries/lparcfg: Fix printing of system_active_processors
powerpc/pseries/lparcfg: Use num_possible_cpus() for potential processors
powerpc: Add kconfig option for the systemcfg page
powerpc: Split systemcfg data out of vdso data page
powerpc: Split systemcfg struct definitions out from vdso
Vincenzo Frascino (3):
drm: i915: Change fault type to unsigned long
vdso: Introduce vdso/page.h
s390: Remove remaining _PAGE_* macros
arch/Kconfig | 2 +-
arch/alpha/include/asm/page.h | 6 +-
arch/arc/include/uapi/asm/page.h | 7 +-
arch/arm/include/asm/page.h | 5 +-
arch/arm/include/asm/vdso/gettimeofday.h | 4 +-
arch/arm/include/asm/vdso/vsyscall.h | 4 -
arch/arm/kernel/vdso.c | 1 -
arch/arm/vdso/Makefile | 2 +-
arch/arm/vdso/datapage.S | 16 ---
arch/arm/vdso/vdso.lds.S | 3 +-
arch/arm64/include/asm/page-def.h | 5 +-
arch/arm64/include/asm/vdso.h | 9 +-
arch/arm64/include/asm/vdso/vsyscall.h | 3 +-
arch/arm64/kernel/vdso.c | 44 +++-----
arch/arm64/kernel/vdso/vdso.lds.S | 2 +-
arch/arm64/kernel/vdso32/vdso.lds.S | 2 +-
arch/csky/Kconfig | 4 -
arch/csky/include/asm/page.h | 8 +-
arch/csky/include/asm/vdso/clocksource.h | 9 --
arch/csky/include/asm/vdso/gettimeofday.h | 114 ---------------------
arch/csky/include/asm/vdso/processor.h | 12 ---
arch/csky/include/asm/vdso/vsyscall.h | 22 ----
arch/csky/kernel/vdso.c | 31 +-----
arch/csky/kernel/vdso/Makefile | 1 -
arch/csky/kernel/vdso/vdso.lds.S | 4 -
arch/csky/kernel/vdso/vgettimeofday.c | 30 ------
arch/hexagon/include/asm/page.h | 4 +-
arch/loongarch/include/asm/page.h | 7 +-
arch/loongarch/include/asm/vdso/getrandom.h | 3 +-
arch/loongarch/include/asm/vdso/gettimeofday.h | 4 +-
arch/loongarch/include/asm/vdso/vdso.h | 18 +---
arch/loongarch/include/asm/vdso/vsyscall.h | 4 -
arch/loongarch/kernel/asm-offsets.c | 9 ++
arch/loongarch/kernel/vdso.c | 1 -
arch/loongarch/vdso/vdso.lds.S | 8 +-
arch/loongarch/vdso/vgetcpu.c | 2 +-
arch/m68k/include/asm/page.h | 6 +-
arch/microblaze/include/asm/page.h | 5 +-
arch/mips/include/asm/page.h | 7 +-
arch/mips/include/asm/vdso/vsyscall.h | 1 -
arch/mips/kernel/vdso.c | 1 -
arch/mips/vdso/genvdso.c | 4 +-
arch/nios2/include/asm/page.h | 7 +-
arch/openrisc/include/asm/page.h | 11 +-
arch/parisc/include/asm/page.h | 4 +-
arch/powerpc/Kconfig | 8 ++
arch/powerpc/include/asm/page.h | 10 +-
arch/powerpc/include/asm/systemcfg.h | 52 ++++++++++
arch/powerpc/include/asm/vdso/vsyscall.h | 4 -
arch/powerpc/include/asm/vdso_datapage.h | 61 +----------
arch/powerpc/kernel/proc_powerpc.c | 37 +++++--
arch/powerpc/kernel/setup-common.c | 5 +-
arch/powerpc/kernel/smp.c | 11 +-
arch/powerpc/kernel/time.c | 5 +-
arch/powerpc/kernel/vdso.c | 20 ----
arch/powerpc/platforms/powernv/smp.c | 5 +-
arch/powerpc/platforms/pseries/hotplug-cpu.c | 5 +-
arch/powerpc/platforms/pseries/lparcfg.c | 5 +-
arch/riscv/Kconfig | 2 +-
arch/riscv/include/asm/page.h | 4 +-
.../riscv/include/asm/vdso/{data.h => time_data.h} | 8 +-
arch/riscv/include/asm/vdso/vsyscall.h | 4 -
arch/riscv/kernel/sys_hwprobe.c | 2 +-
arch/riscv/kernel/vdso.c | 52 +++-------
arch/riscv/kernel/vdso/hwprobe.c | 4 +-
arch/s390/Kconfig | 2 +-
arch/s390/include/asm/page.h | 10 +-
arch/s390/include/asm/pgtable.h | 2 +-
arch/s390/include/asm/vdso.h | 3 -
arch/s390/include/asm/vdso/data.h | 12 ---
arch/s390/include/asm/vdso/time_data.h | 12 +++
arch/s390/include/asm/vdso/vsyscall.h | 5 -
arch/s390/kernel/time.c | 1 -
arch/s390/kernel/vdso32/vdso32.lds.S | 2 +-
arch/s390/kernel/vdso64/vdso64.lds.S | 2 +-
arch/s390/mm/fault.c | 2 +-
arch/s390/mm/gmap.c | 6 +-
arch/s390/mm/pgalloc.c | 4 +-
arch/sh/include/asm/page.h | 6 +-
arch/sparc/include/asm/page_32.h | 4 +-
arch/sparc/include/asm/page_64.h | 4 +-
arch/um/include/asm/page.h | 5 +-
arch/x86/entry/vdso/vdso-layout.lds.S | 20 ++--
arch/x86/entry/vdso/vma.c | 92 +++++++++++------
arch/x86/include/asm/page_types.h | 5 +-
arch/x86/include/asm/vdso/getrandom.h | 10 +-
arch/x86/include/asm/vdso/gettimeofday.h | 12 ++-
arch/x86/include/asm/vdso/vsyscall.h | 15 ++-
arch/x86/include/asm/vvar.h | 71 -------------
arch/x86/kernel/vmlinux.lds.S | 23 -----
arch/x86/mm/mmap.c | 5 -
arch/x86/tools/relocs.c | 1 -
arch/xtensa/include/asm/page.h | 8 +-
drivers/gpu/drm/i915/gt/intel_gt.c | 6 +-
include/asm-generic/vdso/vsyscall.h | 3 +-
include/vdso/datapage.h | 8 +-
include/vdso/page.h | 31 ++++++
kernel/time/vsyscall.c | 2 +-
98 files changed, 372 insertions(+), 782 deletions(-)
delete mode 100644 arch/arm/vdso/datapage.S
delete mode 100644 arch/csky/include/asm/vdso/clocksource.h
delete mode 100644 arch/csky/include/asm/vdso/gettimeofday.h
delete mode 100644 arch/csky/include/asm/vdso/processor.h
delete mode 100644 arch/csky/include/asm/vdso/vsyscall.h
delete mode 100644 arch/csky/kernel/vdso/vgettimeofday.c
create mode 100644 arch/powerpc/include/asm/systemcfg.h
rename arch/riscv/include/asm/vdso/{data.h => time_data.h} (71%)
delete mode 100644 arch/s390/include/asm/vdso/data.h
create mode 100644 arch/s390/include/asm/vdso/time_data.h
delete mode 100644 arch/x86/include/asm/vvar.h
create mode 100644 include/vdso/page.h
diff --git a/arch/Kconfig b/arch/Kconfig
index 8af374ea1adc..7f1ec327b587 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -1530,7 +1530,7 @@ config HAVE_SPARSE_SYSCALL_NR
entries at 4000, 5000 and 6000 locations. This option turns on syscall
related optimizations for a given architecture.
-config ARCH_HAS_VDSO_DATA
+config ARCH_HAS_VDSO_TIME_DATA
bool
config HAVE_STATIC_CALL
diff --git a/arch/alpha/include/asm/page.h b/arch/alpha/include/asm/page.h
index 70419e6be1a3..261af54fd601 100644
--- a/arch/alpha/include/asm/page.h
+++ b/arch/alpha/include/asm/page.h
@@ -4,11 +4,7 @@
#include <linux/const.h>
#include <asm/pal.h>
-
-/* PAGE_SHIFT determines the page size */
-#define PAGE_SHIFT CONFIG_PAGE_SHIFT
-#define PAGE_SIZE (_AC(1,UL) << PAGE_SHIFT)
-#define PAGE_MASK (~(PAGE_SIZE-1))
+#include <vdso/page.h>
#ifndef __ASSEMBLY__
diff --git a/arch/arc/include/uapi/asm/page.h b/arch/arc/include/uapi/asm/page.h
index 7fd9e741b527..4606a326af5c 100644
--- a/arch/arc/include/uapi/asm/page.h
+++ b/arch/arc/include/uapi/asm/page.h
@@ -14,7 +14,7 @@
/* PAGE_SHIFT determines the page size */
#ifdef __KERNEL__
-#define PAGE_SHIFT CONFIG_PAGE_SHIFT
+#include <vdso/page.h>
#else
/*
* Default 8k
@@ -24,11 +24,10 @@
* not available
*/
#define PAGE_SHIFT 13
+#define PAGE_SIZE _BITUL(PAGE_SHIFT) /* Default 8K */
+#define PAGE_MASK (~(PAGE_SIZE-1))
#endif
-#define PAGE_SIZE _BITUL(PAGE_SHIFT) /* Default 8K */
#define PAGE_OFFSET _AC(0x80000000, UL) /* Kernel starts at 2G onwrds */
-#define PAGE_MASK (~(PAGE_SIZE-1))
-
#endif /* _UAPI__ASM_ARC_PAGE_H */
diff --git a/arch/arm/include/asm/page.h b/arch/arm/include/asm/page.h
index 62af9f7f9e96..ef11b721230e 100644
--- a/arch/arm/include/asm/page.h
+++ b/arch/arm/include/asm/page.h
@@ -7,10 +7,7 @@
#ifndef _ASMARM_PAGE_H
#define _ASMARM_PAGE_H
-/* PAGE_SHIFT determines the page size */
-#define PAGE_SHIFT CONFIG_PAGE_SHIFT
-#define PAGE_SIZE (_AC(1,UL) << PAGE_SHIFT)
-#define PAGE_MASK (~((1 << PAGE_SHIFT) - 1))
+#include <vdso/page.h>
#ifndef __ASSEMBLY__
diff --git a/arch/arm/include/asm/vdso/gettimeofday.h b/arch/arm/include/asm/vdso/gettimeofday.h
index 2134cbd5469f..592d3d015ca7 100644
--- a/arch/arm/include/asm/vdso/gettimeofday.h
+++ b/arch/arm/include/asm/vdso/gettimeofday.h
@@ -15,8 +15,6 @@
#define VDSO_HAS_CLOCK_GETRES 1
-extern struct vdso_data *__get_datapage(void);
-
static __always_inline int gettimeofday_fallback(
struct __kernel_old_timeval *_tv,
struct timezone *_tz)
@@ -139,7 +137,7 @@ static __always_inline u64 __arch_get_hw_counter(int clock_mode,
static __always_inline const struct vdso_data *__arch_get_vdso_data(void)
{
- return __get_datapage();
+ return _vdso_data;
}
#endif /* !__ASSEMBLY__ */
diff --git a/arch/arm/include/asm/vdso/vsyscall.h b/arch/arm/include/asm/vdso/vsyscall.h
index 47e41ae8ccd0..705414710dcd 100644
--- a/arch/arm/include/asm/vdso/vsyscall.h
+++ b/arch/arm/include/asm/vdso/vsyscall.h
@@ -4,16 +4,12 @@
#ifndef __ASSEMBLY__
-#include <linux/timekeeper_internal.h>
#include <vdso/datapage.h>
#include <asm/cacheflush.h>
extern struct vdso_data *vdso_data;
extern bool cntvct_ok;
-/*
- * Update the vDSO data page to keep in sync with kernel timekeeping.
- */
static __always_inline
struct vdso_data *__arm_get_k_vdso_data(void)
{
diff --git a/arch/arm/kernel/vdso.c b/arch/arm/kernel/vdso.c
index d499ad461b00..29dd2f3c62fe 100644
--- a/arch/arm/kernel/vdso.c
+++ b/arch/arm/kernel/vdso.c
@@ -14,7 +14,6 @@
#include <linux/of.h>
#include <linux/printk.h>
#include <linux/slab.h>
-#include <linux/timekeeper_internal.h>
#include <linux/vmalloc.h>
#include <asm/arch_timer.h>
#include <asm/barrier.h>
diff --git a/arch/arm/vdso/Makefile b/arch/arm/vdso/Makefile
index 01067a2bc43b..8a306bbec4a0 100644
--- a/arch/arm/vdso/Makefile
+++ b/arch/arm/vdso/Makefile
@@ -5,7 +5,7 @@ include $(srctree)/lib/vdso/Makefile
hostprogs := vdsomunge
-obj-vdso := vgettimeofday.o datapage.o note.o
+obj-vdso := vgettimeofday.o note.o
# Build rules
targets := $(obj-vdso) vdso.so vdso.so.dbg vdso.so.raw vdso.lds
diff --git a/arch/arm/vdso/datapage.S b/arch/arm/vdso/datapage.S
deleted file mode 100644
index 9cd73b725d9f..000000000000
--- a/arch/arm/vdso/datapage.S
+++ /dev/null
@@ -1,16 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#include <linux/linkage.h>
-#include <asm/asm-offsets.h>
-
- .align 2
-.L_vdso_data_ptr:
- .long _start - . - VDSO_DATA_SIZE
-
-ENTRY(__get_datapage)
- .fnstart
- adr r0, .L_vdso_data_ptr
- ldr r1, [r0]
- add r0, r0, r1
- bx lr
- .fnend
-ENDPROC(__get_datapage)
diff --git a/arch/arm/vdso/vdso.lds.S b/arch/arm/vdso/vdso.lds.S
index 165d1d2eb76b..9bfa0f52923c 100644
--- a/arch/arm/vdso/vdso.lds.S
+++ b/arch/arm/vdso/vdso.lds.S
@@ -11,6 +11,7 @@
*/
#include <linux/const.h>
+#include <asm/asm-offsets.h>
#include <asm/page.h>
#include <asm/vdso.h>
@@ -19,7 +20,7 @@ OUTPUT_ARCH(arm)
SECTIONS
{
- PROVIDE(_start = .);
+ PROVIDE(_vdso_data = . - VDSO_DATA_SIZE);
. = SIZEOF_HEADERS;
diff --git a/arch/arm64/include/asm/page-def.h b/arch/arm64/include/asm/page-def.h
index 792e9fe881dc..d402e08442ee 100644
--- a/arch/arm64/include/asm/page-def.h
+++ b/arch/arm64/include/asm/page-def.h
@@ -10,9 +10,6 @@
#include <linux/const.h>
-/* PAGE_SHIFT determines the page size */
-#define PAGE_SHIFT CONFIG_PAGE_SHIFT
-#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
-#define PAGE_MASK (~(PAGE_SIZE-1))
+#include <vdso/page.h>
#endif /* __ASM_PAGE_DEF_H */
diff --git a/arch/arm64/include/asm/vdso.h b/arch/arm64/include/asm/vdso.h
index 4305995c8f82..3e3c3fdb1842 100644
--- a/arch/arm64/include/asm/vdso.h
+++ b/arch/arm64/include/asm/vdso.h
@@ -5,13 +5,6 @@
#ifndef __ASM_VDSO_H
#define __ASM_VDSO_H
-/*
- * Default link address for the vDSO.
- * Since we randomise the VDSO mapping, there's little point in trying
- * to prelink this.
- */
-#define VDSO_LBASE 0x0
-
#define __VVAR_PAGES 2
#ifndef __ASSEMBLY__
@@ -20,7 +13,7 @@
#define VDSO_SYMBOL(base, name) \
({ \
- (void *)(vdso_offset_##name - VDSO_LBASE + (unsigned long)(base)); \
+ (void *)(vdso_offset_##name + (unsigned long)(base)); \
})
extern char vdso_start[], vdso_end[];
diff --git a/arch/arm64/include/asm/vdso/vsyscall.h b/arch/arm64/include/asm/vdso/vsyscall.h
index 5b6d0dd3cef5..eea51946d45a 100644
--- a/arch/arm64/include/asm/vdso/vsyscall.h
+++ b/arch/arm64/include/asm/vdso/vsyscall.h
@@ -6,7 +6,6 @@
#ifndef __ASSEMBLY__
-#include <linux/timekeeper_internal.h>
#include <vdso/datapage.h>
enum vvar_pages {
@@ -37,7 +36,7 @@ struct vdso_rng_data *__arm64_get_k_vdso_rnd_data(void)
#define __arch_get_k_vdso_rng_data __arm64_get_k_vdso_rnd_data
static __always_inline
-void __arm64_update_vsyscall(struct vdso_data *vdata, struct timekeeper *tk)
+void __arm64_update_vsyscall(struct vdso_data *vdata)
{
vdata[CS_HRES_COARSE].mask = VDSO_PRECISION_MASK;
vdata[CS_RAW].mask = VDSO_PRECISION_MASK;
diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c
index 706c9c3a7a50..e8ed8e5b713b 100644
--- a/arch/arm64/kernel/vdso.c
+++ b/arch/arm64/kernel/vdso.c
@@ -19,7 +19,6 @@
#include <linux/signal.h>
#include <linux/slab.h>
#include <linux/time_namespace.h>
-#include <linux/timekeeper_internal.h>
#include <linux/vmalloc.h>
#include <vdso/datapage.h>
#include <vdso/helpers.h>
@@ -39,8 +38,6 @@ struct vdso_abi_info {
const char *vdso_code_start;
const char *vdso_code_end;
unsigned long vdso_pages;
- /* Data Mapping */
- struct vm_special_mapping *dm;
/* Code Mapping */
struct vm_special_mapping *cm;
};
@@ -113,6 +110,8 @@ struct vdso_data *arch_get_vdso_data(void *vvar_page)
return (struct vdso_data *)(vvar_page);
}
+static const struct vm_special_mapping vvar_map;
+
/*
* The vvar mapping contains data for a specific time namespace, so when a task
* changes namespace we must unmap its vvar data for the old namespace.
@@ -129,12 +128,8 @@ int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
mmap_read_lock(mm);
for_each_vma(vmi, vma) {
- if (vma_is_special_mapping(vma, vdso_info[VDSO_ABI_AA64].dm))
- zap_vma_pages(vma);
-#ifdef CONFIG_COMPAT_VDSO
- if (vma_is_special_mapping(vma, vdso_info[VDSO_ABI_AA32].dm))
+ if (vma_is_special_mapping(vma, &vvar_map))
zap_vma_pages(vma);
-#endif
}
mmap_read_unlock(mm);
@@ -176,6 +171,11 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
return vmf_insert_pfn(vma, vmf->address, pfn);
}
+static const struct vm_special_mapping vvar_map = {
+ .name = "[vvar]",
+ .fault = vvar_fault,
+};
+
static int __setup_additional_pages(enum vdso_abi abi,
struct mm_struct *mm,
struct linux_binprm *bprm,
@@ -199,7 +199,7 @@ static int __setup_additional_pages(enum vdso_abi abi,
ret = _install_special_mapping(mm, vdso_base, VVAR_NR_PAGES * PAGE_SIZE,
VM_READ|VM_MAYREAD|VM_PFNMAP,
- vdso_info[abi].dm);
+ &vvar_map);
if (IS_ERR(ret))
goto up_fail;
@@ -229,7 +229,6 @@ static int __setup_additional_pages(enum vdso_abi abi,
enum aarch32_map {
AA32_MAP_VECTORS, /* kuser helpers */
AA32_MAP_SIGPAGE,
- AA32_MAP_VVAR,
AA32_MAP_VDSO,
};
@@ -254,10 +253,6 @@ static struct vm_special_mapping aarch32_vdso_maps[] = {
.pages = &aarch32_sig_page,
.mremap = aarch32_sigpage_mremap,
},
- [AA32_MAP_VVAR] = {
- .name = "[vvar]",
- .fault = vvar_fault,
- },
[AA32_MAP_VDSO] = {
.name = "[vdso]",
.mremap = vdso_mremap,
@@ -307,7 +302,6 @@ static int __init __aarch32_alloc_vdso_pages(void)
if (!IS_ENABLED(CONFIG_COMPAT_VDSO))
return 0;
- vdso_info[VDSO_ABI_AA32].dm = &aarch32_vdso_maps[AA32_MAP_VVAR];
vdso_info[VDSO_ABI_AA32].cm = &aarch32_vdso_maps[AA32_MAP_VDSO];
return __vdso_init(VDSO_ABI_AA32);
@@ -402,26 +396,14 @@ int aarch32_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
}
#endif /* CONFIG_COMPAT */
-enum aarch64_map {
- AA64_MAP_VVAR,
- AA64_MAP_VDSO,
-};
-
-static struct vm_special_mapping aarch64_vdso_maps[] __ro_after_init = {
- [AA64_MAP_VVAR] = {
- .name = "[vvar]",
- .fault = vvar_fault,
- },
- [AA64_MAP_VDSO] = {
- .name = "[vdso]",
- .mremap = vdso_mremap,
- },
+static struct vm_special_mapping aarch64_vdso_map __ro_after_init = {
+ .name = "[vdso]",
+ .mremap = vdso_mremap,
};
static int __init vdso_init(void)
{
- vdso_info[VDSO_ABI_AA64].dm = &aarch64_vdso_maps[AA64_MAP_VVAR];
- vdso_info[VDSO_ABI_AA64].cm = &aarch64_vdso_maps[AA64_MAP_VDSO];
+ vdso_info[VDSO_ABI_AA64].cm = &aarch64_vdso_map;
return __vdso_init(VDSO_ABI_AA64);
}
diff --git a/arch/arm64/kernel/vdso/vdso.lds.S b/arch/arm64/kernel/vdso/vdso.lds.S
index f204a9ddc833..4ec32e86a8da 100644
--- a/arch/arm64/kernel/vdso/vdso.lds.S
+++ b/arch/arm64/kernel/vdso/vdso.lds.S
@@ -25,7 +25,7 @@ SECTIONS
#ifdef CONFIG_TIME_NS
PROVIDE(_timens_data = _vdso_data + PAGE_SIZE);
#endif
- . = VDSO_LBASE + SIZEOF_HEADERS;
+ . = SIZEOF_HEADERS;
.hash : { *(.hash) } :text
.gnu.hash : { *(.gnu.hash) }
diff --git a/arch/arm64/kernel/vdso32/vdso.lds.S b/arch/arm64/kernel/vdso32/vdso.lds.S
index 8d95d7d35057..732702a187e9 100644
--- a/arch/arm64/kernel/vdso32/vdso.lds.S
+++ b/arch/arm64/kernel/vdso32/vdso.lds.S
@@ -22,7 +22,7 @@ SECTIONS
#ifdef CONFIG_TIME_NS
PROVIDE_HIDDEN(_timens_data = _vdso_data + PAGE_SIZE);
#endif
- . = VDSO_LBASE + SIZEOF_HEADERS;
+ . = SIZEOF_HEADERS;
.hash : { *(.hash) } :text
.gnu.hash : { *(.gnu.hash) }
diff --git a/arch/csky/Kconfig b/arch/csky/Kconfig
index 5479707eb5d1..acc431c331b0 100644
--- a/arch/csky/Kconfig
+++ b/arch/csky/Kconfig
@@ -64,9 +64,6 @@ config CSKY
select GENERIC_IRQ_MULTI_HANDLER
select GENERIC_SCHED_CLOCK
select GENERIC_SMP_IDLE_THREAD
- select GENERIC_TIME_VSYSCALL
- select GENERIC_VDSO_32
- select GENERIC_GETTIMEOFDAY
select GX6605S_TIMER if CPU_CK610
select HAVE_ARCH_TRACEHOOK
select HAVE_ARCH_AUDITSYSCALL
@@ -80,7 +77,6 @@ config CSKY
select HAVE_DEBUG_KMEMLEAK
select HAVE_DYNAMIC_FTRACE
select HAVE_DYNAMIC_FTRACE_WITH_REGS
- select HAVE_GENERIC_VDSO
select HAVE_FUNCTION_TRACER
select HAVE_FUNCTION_GRAPH_TRACER
select HAVE_FUNCTION_ERROR_INJECTION
diff --git a/arch/csky/include/asm/page.h b/arch/csky/include/asm/page.h
index 0ca6c408c07f..f8beae295afb 100644
--- a/arch/csky/include/asm/page.h
+++ b/arch/csky/include/asm/page.h
@@ -7,12 +7,8 @@
#include <asm/cache.h>
#include <linux/const.h>
-/*
- * PAGE_SHIFT determines the page size: 4KB
- */
-#define PAGE_SHIFT CONFIG_PAGE_SHIFT
-#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
-#define PAGE_MASK (~(PAGE_SIZE - 1))
+#include <vdso/page.h>
+
#define THREAD_SIZE (PAGE_SIZE * 2)
#define THREAD_MASK (~(THREAD_SIZE - 1))
#define THREAD_SHIFT (PAGE_SHIFT + 1)
diff --git a/arch/csky/include/asm/vdso/clocksource.h b/arch/csky/include/asm/vdso/clocksource.h
deleted file mode 100644
index dfca7b4724b7..000000000000
--- a/arch/csky/include/asm/vdso/clocksource.h
+++ /dev/null
@@ -1,9 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-
-#ifndef __ASM_VDSO_CSKY_CLOCKSOURCE_H
-#define __ASM_VDSO_CSKY_CLOCKSOURCE_H
-
-#define VDSO_ARCH_CLOCKMODES \
- VDSO_CLOCKMODE_ARCHTIMER
-
-#endif /* __ASM_VDSO_CSKY_CLOCKSOURCE_H */
diff --git a/arch/csky/include/asm/vdso/gettimeofday.h b/arch/csky/include/asm/vdso/gettimeofday.h
deleted file mode 100644
index 6c4f1446944f..000000000000
--- a/arch/csky/include/asm/vdso/gettimeofday.h
+++ /dev/null
@@ -1,114 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-
-#ifndef __ASM_VDSO_CSKY_GETTIMEOFDAY_H
-#define __ASM_VDSO_CSKY_GETTIMEOFDAY_H
-
-#ifndef __ASSEMBLY__
-
-#include <asm/barrier.h>
-#include <asm/unistd.h>
-#include <abi/regdef.h>
-#include <uapi/linux/time.h>
-
-#define VDSO_HAS_CLOCK_GETRES 1
-
-static __always_inline
-int gettimeofday_fallback(struct __kernel_old_timeval *_tv,
- struct timezone *_tz)
-{
- register struct __kernel_old_timeval *tv asm("a0") = _tv;
- register struct timezone *tz asm("a1") = _tz;
- register long ret asm("a0");
- register long nr asm(syscallid) = __NR_gettimeofday;
-
- asm volatile ("trap 0\n"
- : "=r" (ret)
- : "r"(tv), "r"(tz), "r"(nr)
- : "memory");
-
- return ret;
-}
-
-static __always_inline
-long clock_gettime_fallback(clockid_t _clkid, struct __kernel_timespec *_ts)
-{
- register clockid_t clkid asm("a0") = _clkid;
- register struct __kernel_timespec *ts asm("a1") = _ts;
- register long ret asm("a0");
- register long nr asm(syscallid) = __NR_clock_gettime64;
-
- asm volatile ("trap 0\n"
- : "=r" (ret)
- : "r"(clkid), "r"(ts), "r"(nr)
- : "memory");
-
- return ret;
-}
-
-static __always_inline
-long clock_gettime32_fallback(clockid_t _clkid, struct old_timespec32 *_ts)
-{
- register clockid_t clkid asm("a0") = _clkid;
- register struct old_timespec32 *ts asm("a1") = _ts;
- register long ret asm("a0");
- register long nr asm(syscallid) = __NR_clock_gettime;
-
- asm volatile ("trap 0\n"
- : "=r" (ret)
- : "r"(clkid), "r"(ts), "r"(nr)
- : "memory");
-
- return ret;
-}
-
-static __always_inline
-int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts)
-{
- register clockid_t clkid asm("a0") = _clkid;
- register struct __kernel_timespec *ts asm("a1") = _ts;
- register long ret asm("a0");
- register long nr asm(syscallid) = __NR_clock_getres_time64;
-
- asm volatile ("trap 0\n"
- : "=r" (ret)
- : "r"(clkid), "r"(ts), "r"(nr)
- : "memory");
-
- return ret;
-}
-
-static __always_inline
-int clock_getres32_fallback(clockid_t _clkid, struct old_timespec32 *_ts)
-{
- register clockid_t clkid asm("a0") = _clkid;
- register struct old_timespec32 *ts asm("a1") = _ts;
- register long ret asm("a0");
- register long nr asm(syscallid) = __NR_clock_getres;
-
- asm volatile ("trap 0\n"
- : "=r" (ret)
- : "r"(clkid), "r"(ts), "r"(nr)
- : "memory");
-
- return ret;
-}
-
-uint64_t csky_pmu_read_cc(void);
-static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
- const struct vdso_data *vd)
-{
-#ifdef CONFIG_CSKY_PMU_V1
- return csky_pmu_read_cc();
-#else
- return 0;
-#endif
-}
-
-static __always_inline const struct vdso_data *__arch_get_vdso_data(void)
-{
- return _vdso_data;
-}
-
-#endif /* !__ASSEMBLY__ */
-
-#endif /* __ASM_VDSO_CSKY_GETTIMEOFDAY_H */
diff --git a/arch/csky/include/asm/vdso/processor.h b/arch/csky/include/asm/vdso/processor.h
deleted file mode 100644
index 39a6b561d0cc..000000000000
--- a/arch/csky/include/asm/vdso/processor.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-
-#ifndef __ASM_VDSO_CSKY_PROCESSOR_H
-#define __ASM_VDSO_CSKY_PROCESSOR_H
-
-#ifndef __ASSEMBLY__
-
-#define cpu_relax() barrier()
-
-#endif /* __ASSEMBLY__ */
-
-#endif /* __ASM_VDSO_CSKY_PROCESSOR_H */
diff --git a/arch/csky/include/asm/vdso/vsyscall.h b/arch/csky/include/asm/vdso/vsyscall.h
deleted file mode 100644
index c276211a7c4d..000000000000
--- a/arch/csky/include/asm/vdso/vsyscall.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-
-#ifndef __ASM_VDSO_CSKY_VSYSCALL_H
-#define __ASM_VDSO_CSKY_VSYSCALL_H
-
-#ifndef __ASSEMBLY__
-
-#include <vdso/datapage.h>
-
-extern struct vdso_data *vdso_data;
-
-static __always_inline struct vdso_data *__csky_get_k_vdso_data(void)
-{
- return vdso_data;
-}
-#define __arch_get_k_vdso_data __csky_get_k_vdso_data
-
-#include <asm-generic/vdso/vsyscall.h>
-
-#endif /* !__ASSEMBLY__ */
-
-#endif /* __ASM_VDSO_CSKY_VSYSCALL_H */
diff --git a/arch/csky/kernel/vdso.c b/arch/csky/kernel/vdso.c
index 5c9ef63c29f1..c54d019d66bc 100644
--- a/arch/csky/kernel/vdso.c
+++ b/arch/csky/kernel/vdso.c
@@ -8,23 +8,19 @@
#include <linux/slab.h>
#include <asm/page.h>
-#include <vdso/datapage.h>
extern char vdso_start[], vdso_end[];
static unsigned int vdso_pages;
static struct page **vdso_pagelist;
-static union vdso_data_store vdso_data_store __page_aligned_data;
-struct vdso_data *vdso_data = vdso_data_store.data;
-
static int __init vdso_init(void)
{
unsigned int i;
vdso_pages = (vdso_end - vdso_start) >> PAGE_SHIFT;
vdso_pagelist =
- kcalloc(vdso_pages + 1, sizeof(struct page *), GFP_KERNEL);
+ kcalloc(vdso_pages, sizeof(struct page *), GFP_KERNEL);
if (unlikely(vdso_pagelist == NULL)) {
pr_err("vdso: pagelist allocation failed\n");
return -ENOMEM;
@@ -36,7 +32,6 @@ static int __init vdso_init(void)
pg = virt_to_page(vdso_start + (i << PAGE_SHIFT));
vdso_pagelist[i] = pg;
}
- vdso_pagelist[i] = virt_to_page(vdso_data);
return 0;
}
@@ -52,11 +47,8 @@ int arch_setup_additional_pages(struct linux_binprm *bprm,
static struct vm_special_mapping vdso_mapping = {
.name = "[vdso]",
};
- static struct vm_special_mapping vvar_mapping = {
- .name = "[vvar]",
- };
- vdso_len = (vdso_pages + 1) << PAGE_SHIFT;
+ vdso_len = vdso_pages << PAGE_SHIFT;
mmap_write_lock(mm);
vdso_base = get_unmapped_area(NULL, 0, vdso_len, 0, 0);
@@ -85,27 +77,8 @@ int arch_setup_additional_pages(struct linux_binprm *bprm,
}
vdso_base += (vdso_pages << PAGE_SHIFT);
- vvar_mapping.pages = &vdso_pagelist[vdso_pages];
- vma = _install_special_mapping(mm, vdso_base, PAGE_SIZE,
- (VM_READ | VM_MAYREAD), &vvar_mapping);
-
- if (IS_ERR(vma)) {
- ret = PTR_ERR(vma);
- mm->context.vdso = NULL;
- goto end;
- }
ret = 0;
end:
mmap_write_unlock(mm);
return ret;
}
-
-const char *arch_vma_name(struct vm_area_struct *vma)
-{
- if (vma->vm_mm && (vma->vm_start == (long)vma->vm_mm->context.vdso))
- return "[vdso]";
- if (vma->vm_mm && (vma->vm_start ==
- (long)vma->vm_mm->context.vdso + PAGE_SIZE))
- return "[vdso_data]";
- return NULL;
-}
diff --git a/arch/csky/kernel/vdso/Makefile b/arch/csky/kernel/vdso/Makefile
index bc2261f5a8d4..069ef0b17fe5 100644
--- a/arch/csky/kernel/vdso/Makefile
+++ b/arch/csky/kernel/vdso/Makefile
@@ -5,7 +5,6 @@ include $(srctree)/lib/vdso/Makefile
# Symbols present in the vdso
vdso-syms += rt_sigreturn
-vdso-syms += vgettimeofday
# Files to link into the vdso
obj-vdso = $(patsubst %, %.o, $(vdso-syms)) note.o
diff --git a/arch/csky/kernel/vdso/vdso.lds.S b/arch/csky/kernel/vdso/vdso.lds.S
index 590a6c79fff7..8d226252d439 100644
--- a/arch/csky/kernel/vdso/vdso.lds.S
+++ b/arch/csky/kernel/vdso/vdso.lds.S
@@ -49,10 +49,6 @@ VERSION
LINUX_5.10 {
global:
__vdso_rt_sigreturn;
- __vdso_clock_gettime;
- __vdso_clock_gettime64;
- __vdso_gettimeofday;
- __vdso_clock_getres;
local: *;
};
}
diff --git a/arch/csky/kernel/vdso/vgettimeofday.c b/arch/csky/kernel/vdso/vgettimeofday.c
deleted file mode 100644
index 55af30e83752..000000000000
--- a/arch/csky/kernel/vdso/vgettimeofday.c
+++ /dev/null
@@ -1,30 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-
-#include <linux/time.h>
-#include <linux/types.h>
-#include <vdso/gettime.h>
-
-extern
-int __vdso_clock_gettime(clockid_t clock,
- struct old_timespec32 *ts)
-{
- return __cvdso_clock_gettime32(clock, ts);
-}
-
-int __vdso_clock_gettime64(clockid_t clock,
- struct __kernel_timespec *ts)
-{
- return __cvdso_clock_gettime(clock, ts);
-}
-
-int __vdso_gettimeofday(struct __kernel_old_timeval *tv,
- struct timezone *tz)
-{
- return __cvdso_gettimeofday(tv, tz);
-}
-
-int __vdso_clock_getres(clockid_t clock_id,
- struct old_timespec32 *res)
-{
- return __cvdso_clock_getres_time32(clock_id, res);
-}
diff --git a/arch/hexagon/include/asm/page.h b/arch/hexagon/include/asm/page.h
index 8a6af57274c2..b01f8df69dd4 100644
--- a/arch/hexagon/include/asm/page.h
+++ b/arch/hexagon/include/asm/page.h
@@ -45,9 +45,7 @@
#define HVM_HUGEPAGE_SIZE 0x5
#endif
-#define PAGE_SHIFT CONFIG_PAGE_SHIFT
-#define PAGE_SIZE (1UL << PAGE_SHIFT)
-#define PAGE_MASK (~((1 << PAGE_SHIFT) - 1))
+#include <vdso/page.h>
#ifdef __KERNEL__
#ifndef __ASSEMBLY__
diff --git a/arch/loongarch/include/asm/page.h b/arch/loongarch/include/asm/page.h
index e85df33f11c7..83f3533e31a4 100644
--- a/arch/loongarch/include/asm/page.h
+++ b/arch/loongarch/include/asm/page.h
@@ -8,12 +8,7 @@
#include <linux/const.h>
#include <asm/addrspace.h>
-/*
- * PAGE_SHIFT determines the page size
- */
-#define PAGE_SHIFT CONFIG_PAGE_SHIFT
-#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
-#define PAGE_MASK (~(PAGE_SIZE - 1))
+#include <vdso/page.h>
#define HPAGE_SHIFT (PAGE_SHIFT + PAGE_SHIFT - 3)
#define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT)
diff --git a/arch/loongarch/include/asm/vdso/getrandom.h b/arch/loongarch/include/asm/vdso/getrandom.h
index 02f36772541b..e80f3c4ac748 100644
--- a/arch/loongarch/include/asm/vdso/getrandom.h
+++ b/arch/loongarch/include/asm/vdso/getrandom.h
@@ -30,8 +30,7 @@ static __always_inline ssize_t getrandom_syscall(void *_buffer, size_t _len, uns
static __always_inline const struct vdso_rng_data *__arch_get_vdso_rng_data(void)
{
- return (const struct vdso_rng_data *)(get_vdso_data() + VVAR_LOONGARCH_PAGES_START *
- PAGE_SIZE + offsetof(struct loongarch_vdso_data, rng_data));
+ return &_loongarch_data.rng_data;
}
#endif /* !__ASSEMBLY__ */
diff --git a/arch/loongarch/include/asm/vdso/gettimeofday.h b/arch/loongarch/include/asm/vdso/gettimeofday.h
index 89e6b222c2f2..7eb3f041af76 100644
--- a/arch/loongarch/include/asm/vdso/gettimeofday.h
+++ b/arch/loongarch/include/asm/vdso/gettimeofday.h
@@ -91,14 +91,14 @@ static inline bool loongarch_vdso_hres_capable(void)
static __always_inline const struct vdso_data *__arch_get_vdso_data(void)
{
- return (const struct vdso_data *)get_vdso_data();
+ return _vdso_data;
}
#ifdef CONFIG_TIME_NS
static __always_inline
const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd)
{
- return (const struct vdso_data *)(get_vdso_data() + VVAR_TIMENS_PAGE_OFFSET * PAGE_SIZE);
+ return _timens_data;
}
#endif
#endif /* !__ASSEMBLY__ */
diff --git a/arch/loongarch/include/asm/vdso/vdso.h b/arch/loongarch/include/asm/vdso/vdso.h
index e31ac7474513..1c183a9b2115 100644
--- a/arch/loongarch/include/asm/vdso/vdso.h
+++ b/arch/loongarch/include/asm/vdso/vdso.h
@@ -48,23 +48,7 @@ enum vvar_pages {
#define VVAR_SIZE (VVAR_NR_PAGES << PAGE_SHIFT)
-static inline unsigned long get_vdso_base(void)
-{
- unsigned long addr;
-
- __asm__(
- " la.pcrel %0, _start\n"
- : "=r" (addr)
- :
- :);
-
- return addr;
-}
-
-static inline unsigned long get_vdso_data(void)
-{
- return get_vdso_base() - VVAR_SIZE;
-}
+extern struct loongarch_vdso_data _loongarch_data __attribute__((visibility("hidden")));
#endif /* __ASSEMBLY__ */
diff --git a/arch/loongarch/include/asm/vdso/vsyscall.h b/arch/loongarch/include/asm/vdso/vsyscall.h
index b1273ce6f140..8987e951d0a9 100644
--- a/arch/loongarch/include/asm/vdso/vsyscall.h
+++ b/arch/loongarch/include/asm/vdso/vsyscall.h
@@ -4,15 +4,11 @@
#ifndef __ASSEMBLY__
-#include <linux/timekeeper_internal.h>
#include <vdso/datapage.h>
extern struct vdso_data *vdso_data;
extern struct vdso_rng_data *vdso_rng_data;
-/*
- * Update the vDSO data page to keep in sync with kernel timekeeping.
- */
static __always_inline
struct vdso_data *__loongarch_get_k_vdso_data(void)
{
diff --git a/arch/loongarch/kernel/asm-offsets.c b/arch/loongarch/kernel/asm-offsets.c
index bee9f7a3108f..049c5c3e370c 100644
--- a/arch/loongarch/kernel/asm-offsets.c
+++ b/arch/loongarch/kernel/asm-offsets.c
@@ -14,6 +14,7 @@
#include <asm/ptrace.h>
#include <asm/processor.h>
#include <asm/ftrace.h>
+#include <vdso/datapage.h>
static void __used output_ptreg_defines(void)
{
@@ -321,3 +322,11 @@ static void __used output_kvm_defines(void)
OFFSET(KVM_GPGD, kvm, arch.pgd);
BLANK();
}
+
+static void __used output_vdso_defines(void)
+{
+ COMMENT("LoongArch vDSO offsets.");
+
+ DEFINE(__VVAR_PAGES, VVAR_NR_PAGES);
+ BLANK();
+}
diff --git a/arch/loongarch/kernel/vdso.c b/arch/loongarch/kernel/vdso.c
index f6fcc52aefae..4d7cb9425dc3 100644
--- a/arch/loongarch/kernel/vdso.c
+++ b/arch/loongarch/kernel/vdso.c
@@ -15,7 +15,6 @@
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/time_namespace.h>
-#include <linux/timekeeper_internal.h>
#include <asm/page.h>
#include <asm/vdso.h>
diff --git a/arch/loongarch/vdso/vdso.lds.S b/arch/loongarch/vdso/vdso.lds.S
index 6b441bde4026..160cfaef2de4 100644
--- a/arch/loongarch/vdso/vdso.lds.S
+++ b/arch/loongarch/vdso/vdso.lds.S
@@ -3,6 +3,8 @@
* Author: Huacai Chen <chenhuacai@loongson.cn>
* Copyright (C) 2020-2022 Loongson Technology Corporation Limited
*/
+#include <asm/page.h>
+#include <generated/asm-offsets.h>
OUTPUT_FORMAT("elf64-loongarch", "elf64-loongarch", "elf64-loongarch")
@@ -10,7 +12,11 @@ OUTPUT_ARCH(loongarch)
SECTIONS
{
- PROVIDE(_start = .);
+ PROVIDE(_vdso_data = . - __VVAR_PAGES * PAGE_SIZE);
+#ifdef CONFIG_TIME_NS
+ PROVIDE(_timens_data = _vdso_data + PAGE_SIZE);
+#endif
+ PROVIDE(_loongarch_data = _vdso_data + 2 * PAGE_SIZE);
. = SIZEOF_HEADERS;
.hash : { *(.hash) } :text
diff --git a/arch/loongarch/vdso/vgetcpu.c b/arch/loongarch/vdso/vgetcpu.c
index 9e445be39763..0db51258b2a7 100644
--- a/arch/loongarch/vdso/vgetcpu.c
+++ b/arch/loongarch/vdso/vgetcpu.c
@@ -21,7 +21,7 @@ static __always_inline int read_cpu_id(void)
static __always_inline const struct vdso_pcpu_data *get_pcpu_data(void)
{
- return (struct vdso_pcpu_data *)(get_vdso_data() + VVAR_LOONGARCH_PAGES_START * PAGE_SIZE);
+ return _loongarch_data.pdata;
}
extern
diff --git a/arch/m68k/include/asm/page.h b/arch/m68k/include/asm/page.h
index 8cfb84b49975..b173ba27d36f 100644
--- a/arch/m68k/include/asm/page.h
+++ b/arch/m68k/include/asm/page.h
@@ -6,10 +6,8 @@
#include <asm/setup.h>
#include <asm/page_offset.h>
-/* PAGE_SHIFT determines the page size */
-#define PAGE_SHIFT CONFIG_PAGE_SHIFT
-#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
-#define PAGE_MASK (~(PAGE_SIZE-1))
+#include <vdso/page.h>
+
#define PAGE_OFFSET (PAGE_OFFSET_RAW)
#ifndef __ASSEMBLY__
diff --git a/arch/microblaze/include/asm/page.h b/arch/microblaze/include/asm/page.h
index 8810f4f1c3b0..d1ec3806edab 100644
--- a/arch/microblaze/include/asm/page.h
+++ b/arch/microblaze/include/asm/page.h
@@ -19,10 +19,7 @@
#ifdef __KERNEL__
-/* PAGE_SHIFT determines the page size */
-#define PAGE_SHIFT CONFIG_PAGE_SHIFT
-#define PAGE_SIZE (ASM_CONST(1) << PAGE_SHIFT)
-#define PAGE_MASK (~(PAGE_SIZE-1))
+#include <vdso/page.h>
#define LOAD_OFFSET ASM_CONST((CONFIG_KERNEL_START-CONFIG_KERNEL_BASE_ADDR))
diff --git a/arch/mips/include/asm/page.h b/arch/mips/include/asm/page.h
index 4609cb0326cf..bc3e3484c1bf 100644
--- a/arch/mips/include/asm/page.h
+++ b/arch/mips/include/asm/page.h
@@ -14,12 +14,7 @@
#include <linux/kernel.h>
#include <asm/mipsregs.h>
-/*
- * PAGE_SHIFT determines the page size
- */
-#define PAGE_SHIFT CONFIG_PAGE_SHIFT
-#define PAGE_SIZE (_AC(1,UL) << PAGE_SHIFT)
-#define PAGE_MASK (~((1 << PAGE_SHIFT) - 1))
+#include <vdso/page.h>
/*
* This is used for calculating the real page sizes
diff --git a/arch/mips/include/asm/vdso/vsyscall.h b/arch/mips/include/asm/vdso/vsyscall.h
index 47168aaf1eff..a4582870aaea 100644
--- a/arch/mips/include/asm/vdso/vsyscall.h
+++ b/arch/mips/include/asm/vdso/vsyscall.h
@@ -4,7 +4,6 @@
#ifndef __ASSEMBLY__
-#include <linux/timekeeper_internal.h>
#include <vdso/datapage.h>
extern struct vdso_data *vdso_data;
diff --git a/arch/mips/kernel/vdso.c b/arch/mips/kernel/vdso.c
index dda36fa26307..4c8e3c0aa210 100644
--- a/arch/mips/kernel/vdso.c
+++ b/arch/mips/kernel/vdso.c
@@ -14,7 +14,6 @@
#include <linux/random.h>
#include <linux/sched.h>
#include <linux/slab.h>
-#include <linux/timekeeper_internal.h>
#include <asm/abi.h>
#include <asm/mips-cps.h>
diff --git a/arch/mips/vdso/genvdso.c b/arch/mips/vdso/genvdso.c
index 09e30eb4be86..d47412ea6e67 100644
--- a/arch/mips/vdso/genvdso.c
+++ b/arch/mips/vdso/genvdso.c
@@ -270,7 +270,7 @@ int main(int argc, char **argv)
/* Write out the stripped VDSO data. */
fprintf(out_file,
- "static unsigned char vdso_data[PAGE_ALIGN(%zu)] __page_aligned_data = {\n\t",
+ "static unsigned char vdso_image_data[PAGE_ALIGN(%zu)] __page_aligned_data = {\n\t",
vdso_size);
for (i = 0; i < vdso_size; i++) {
if (!(i % 10))
@@ -286,7 +286,7 @@ int main(int argc, char **argv)
fprintf(out_file, "struct mips_vdso_image vdso_image%s%s = {\n",
(vdso_name[0]) ? "_" : "", vdso_name);
- fprintf(out_file, "\t.data = vdso_data,\n");
+ fprintf(out_file, "\t.data = vdso_image_data,\n");
fprintf(out_file, "\t.size = PAGE_ALIGN(%zu),\n", vdso_size);
fprintf(out_file, "\t.mapping = {\n");
fprintf(out_file, "\t\t.name = \"[vdso]\",\n");
diff --git a/arch/nios2/include/asm/page.h b/arch/nios2/include/asm/page.h
index 0722f88e63cc..2897ec1b74f6 100644
--- a/arch/nios2/include/asm/page.h
+++ b/arch/nios2/include/asm/page.h
@@ -18,12 +18,7 @@
#include <linux/pfn.h>
#include <linux/const.h>
-/*
- * PAGE_SHIFT determines the page size
- */
-#define PAGE_SHIFT CONFIG_PAGE_SHIFT
-#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
-#define PAGE_MASK (~(PAGE_SIZE - 1))
+#include <vdso/page.h>
/*
* PAGE_OFFSET -- the first address of the first page of memory.
diff --git a/arch/openrisc/include/asm/page.h b/arch/openrisc/include/asm/page.h
index 1d5913f67c31..124a2db4b160 100644
--- a/arch/openrisc/include/asm/page.h
+++ b/arch/openrisc/include/asm/page.h
@@ -15,16 +15,7 @@
#ifndef __ASM_OPENRISC_PAGE_H
#define __ASM_OPENRISC_PAGE_H
-
-/* PAGE_SHIFT determines the page size */
-
-#define PAGE_SHIFT CONFIG_PAGE_SHIFT
-#ifdef __ASSEMBLY__
-#define PAGE_SIZE (1 << PAGE_SHIFT)
-#else
-#define PAGE_SIZE (1UL << PAGE_SHIFT)
-#endif
-#define PAGE_MASK (~(PAGE_SIZE-1))
+#include <vdso/page.h>
#define PAGE_OFFSET 0xc0000000
#define KERNELBASE PAGE_OFFSET
diff --git a/arch/parisc/include/asm/page.h b/arch/parisc/include/asm/page.h
index 4bea2e95798f..6c4836fb5407 100644
--- a/arch/parisc/include/asm/page.h
+++ b/arch/parisc/include/asm/page.h
@@ -4,9 +4,7 @@
#include <linux/const.h>
-#define PAGE_SHIFT CONFIG_PAGE_SHIFT
-#define PAGE_SIZE (_AC(1,UL) << PAGE_SHIFT)
-#define PAGE_MASK (~(PAGE_SIZE-1))
+#include <vdso/page.h>
#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 8094a01974cc..5d348e1f09d7 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -1298,6 +1298,14 @@ config MODULES_SIZE
endmenu
+config PPC64_PROC_SYSTEMCFG
+ def_bool y
+ depends on PPC64 && PROC_FS
+ help
+ This option enables the presence of /proc/ppc64/systemcfg through
+ which the systemcfg page can be accessed.
+ This interface only exists for backwards-compatibility.
+
if PPC64
# This value must have zeroes in the bottom 60 bits otherwise lots will break
config PAGE_OFFSET
diff --git a/arch/powerpc/include/asm/page.h b/arch/powerpc/include/asm/page.h
index 83d0a4fc5f75..af9a2628d1df 100644
--- a/arch/powerpc/include/asm/page.h
+++ b/arch/powerpc/include/asm/page.h
@@ -21,8 +21,7 @@
* page size. When using 64K pages however, whether we are really supporting
* 64K pages in HW or not is irrelevant to those definitions.
*/
-#define PAGE_SHIFT CONFIG_PAGE_SHIFT
-#define PAGE_SIZE (ASM_CONST(1) << PAGE_SHIFT)
+#include <vdso/page.h>
#ifndef __ASSEMBLY__
#ifndef CONFIG_HUGETLB_PAGE
@@ -41,13 +40,6 @@ extern unsigned int hpage_shift;
#define HUGE_MAX_HSTATE (MMU_PAGE_COUNT-1)
#endif
-/*
- * Subtle: (1 << PAGE_SHIFT) is an int, not an unsigned long. So if we
- * assign PAGE_MASK to a larger type it gets extended the way we want
- * (i.e. with 1s in the high bits)
- */
-#define PAGE_MASK (~((1 << PAGE_SHIFT) - 1))
-
/*
* KERNELBASE is the virtual address of the start of the kernel, it's often
* the same as PAGE_OFFSET, but _might not be_.
diff --git a/arch/powerpc/include/asm/systemcfg.h b/arch/powerpc/include/asm/systemcfg.h
new file mode 100644
index 000000000000..2f9b1d6a5c98
--- /dev/null
+++ b/arch/powerpc/include/asm/systemcfg.h
@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#ifndef _SYSTEMCFG_H
+#define _SYSTEMCFG_H
+
+/*
+ * Copyright (C) 2002 Peter Bergner <bergner@vnet.ibm.com>, IBM
+ * Copyright (C) 2005 Benjamin Herrenschmidy <benh@kernel.crashing.org>,
+ * IBM Corp.
+ */
+
+#ifdef CONFIG_PPC64
+
+/*
+ * If the major version changes we are incompatible.
+ * Minor version changes are a hint.
+ */
+#define SYSTEMCFG_MAJOR 1
+#define SYSTEMCFG_MINOR 1
+
+#include <linux/types.h>
+
+struct systemcfg {
+ __u8 eye_catcher[16]; /* Eyecatcher: SYSTEMCFG:PPC64 0x00 */
+ struct { /* Systemcfg version numbers */
+ __u32 major; /* Major number 0x10 */
+ __u32 minor; /* Minor number 0x14 */
+ } version;
+
+ /* Note about the platform flags: it now only contains the lpar
+ * bit. The actual platform number is dead and buried
+ */
+ __u32 platform; /* Platform flags 0x18 */
+ __u32 processor; /* Processor type 0x1C */
+ __u64 processorCount; /* # of physical processors 0x20 */
+ __u64 physicalMemorySize; /* Size of real memory(B) 0x28 */
+ __u64 tb_orig_stamp; /* (NU) Timebase at boot 0x30 */
+ __u64 tb_ticks_per_sec; /* Timebase tics / sec 0x38 */
+ __u64 tb_to_xs; /* (NU) Inverse of TB to 2^20 0x40 */
+ __u64 stamp_xsec; /* (NU) 0x48 */
+ __u64 tb_update_count; /* (NU) Timebase atomicity ctr 0x50 */
+ __u32 tz_minuteswest; /* (NU) Min. west of Greenwich 0x58 */
+ __u32 tz_dsttime; /* (NU) Type of dst correction 0x5C */
+ __u32 dcache_size; /* L1 d-cache size 0x60 */
+ __u32 dcache_line_size; /* L1 d-cache line size 0x64 */
+ __u32 icache_size; /* L1 i-cache size 0x68 */
+ __u32 icache_line_size; /* L1 i-cache line size 0x6C */
+};
+
+extern struct systemcfg *systemcfg;
+
+#endif /* CONFIG_PPC64 */
+#endif /* _SYSTEMCFG_H */
diff --git a/arch/powerpc/include/asm/vdso/vsyscall.h b/arch/powerpc/include/asm/vdso/vsyscall.h
index 92f480d8cc6d..48560a119559 100644
--- a/arch/powerpc/include/asm/vdso/vsyscall.h
+++ b/arch/powerpc/include/asm/vdso/vsyscall.h
@@ -4,12 +4,8 @@
#ifndef __ASSEMBLY__
-#include <linux/timekeeper_internal.h>
#include <asm/vdso_datapage.h>
-/*
- * Update the vDSO data page to keep in sync with kernel timekeeping.
- */
static __always_inline
struct vdso_data *__arch_get_k_vdso_data(void)
{
diff --git a/arch/powerpc/include/asm/vdso_datapage.h b/arch/powerpc/include/asm/vdso_datapage.h
index 248dee138f7b..a9686310be2c 100644
--- a/arch/powerpc/include/asm/vdso_datapage.h
+++ b/arch/powerpc/include/asm/vdso_datapage.h
@@ -9,29 +9,6 @@
* IBM Corp.
*/
-
-/*
- * Note about this structure:
- *
- * This structure was historically called systemcfg and exposed to
- * userland via /proc/ppc64/systemcfg. Unfortunately, this became an
- * ABI issue as some proprietary software started relying on being able
- * to mmap() it, thus we have to keep the base layout at least for a
- * few kernel versions.
- *
- * However, since ppc32 doesn't suffer from this backward handicap,
- * a simpler version of the data structure is used there with only the
- * fields actually used by the vDSO.
- *
- */
-
-/*
- * If the major version changes we are incompatible.
- * Minor version changes are a hint.
- */
-#define SYSTEMCFG_MAJOR 1
-#define SYSTEMCFG_MINOR 1
-
#ifndef __ASSEMBLY__
#include <linux/unistd.h>
@@ -40,41 +17,10 @@
#define SYSCALL_MAP_SIZE ((NR_syscalls + 31) / 32)
-/*
- * So here is the ppc64 backward compatible version
- */
-
#ifdef CONFIG_PPC64
struct vdso_arch_data {
- __u8 eye_catcher[16]; /* Eyecatcher: SYSTEMCFG:PPC64 0x00 */
- struct { /* Systemcfg version numbers */
- __u32 major; /* Major number 0x10 */
- __u32 minor; /* Minor number 0x14 */
- } version;
-
- /* Note about the platform flags: it now only contains the lpar
- * bit. The actual platform number is dead and buried
- */
- __u32 platform; /* Platform flags 0x18 */
- __u32 processor; /* Processor type 0x1C */
- __u64 processorCount; /* # of physical processors 0x20 */
- __u64 physicalMemorySize; /* Size of real memory(B) 0x28 */
- __u64 tb_orig_stamp; /* (NU) Timebase at boot 0x30 */
- __u64 tb_ticks_per_sec; /* Timebase tics / sec 0x38 */
- __u64 tb_to_xs; /* (NU) Inverse of TB to 2^20 0x40 */
- __u64 stamp_xsec; /* (NU) 0x48 */
- __u64 tb_update_count; /* (NU) Timebase atomicity ctr 0x50 */
- __u32 tz_minuteswest; /* (NU) Min. west of Greenwich 0x58 */
- __u32 tz_dsttime; /* (NU) Type of dst correction 0x5C */
- __u32 dcache_size; /* L1 d-cache size 0x60 */
- __u32 dcache_line_size; /* L1 d-cache line size 0x64 */
- __u32 icache_size; /* L1 i-cache size 0x68 */
- __u32 icache_line_size; /* L1 i-cache line size 0x6C */
-
- /* those additional ones don't have to be located anywhere
- * special as they were not part of the original systemcfg
- */
+ __u64 tb_ticks_per_sec; /* Timebase tics / sec */
__u32 dcache_block_size; /* L1 d-cache block size */
__u32 icache_block_size; /* L1 i-cache block size */
__u32 dcache_log_block_size; /* L1 d-cache log block size */
@@ -88,11 +34,8 @@ struct vdso_arch_data {
#else /* CONFIG_PPC64 */
-/*
- * And here is the simpler 32 bits version
- */
struct vdso_arch_data {
- __u64 tb_ticks_per_sec; /* Timebase tics / sec 0x38 */
+ __u64 tb_ticks_per_sec; /* Timebase tics / sec */
__u32 syscall_map[SYSCALL_MAP_SIZE]; /* Map of syscalls */
__u32 compat_syscall_map[0]; /* No compat syscalls on PPC32 */
struct vdso_data data[CS_BASES];
diff --git a/arch/powerpc/kernel/proc_powerpc.c b/arch/powerpc/kernel/proc_powerpc.c
index b109cd7b5d01..3816a2bf2b84 100644
--- a/arch/powerpc/kernel/proc_powerpc.c
+++ b/arch/powerpc/kernel/proc_powerpc.c
@@ -4,6 +4,7 @@
*/
#include <linux/init.h>
+#include <linux/memblock.h>
#include <linux/mm.h>
#include <linux/proc_fs.h>
#include <linux/kernel.h>
@@ -12,9 +13,10 @@
#include <asm/machdep.h>
#include <asm/vdso_datapage.h>
#include <asm/rtas.h>
+#include <asm/systemcfg.h>
#include <linux/uaccess.h>
-#ifdef CONFIG_PPC64
+#ifdef CONFIG_PPC64_PROC_SYSTEMCFG
static loff_t page_map_seek(struct file *file, loff_t off, int whence)
{
@@ -33,10 +35,9 @@ static int page_map_mmap( struct file *file, struct vm_area_struct *vma )
if ((vma->vm_end - vma->vm_start) > PAGE_SIZE)
return -EINVAL;
- remap_pfn_range(vma, vma->vm_start,
- __pa(pde_data(file_inode(file))) >> PAGE_SHIFT,
- PAGE_SIZE, vma->vm_page_prot);
- return 0;
+ return remap_pfn_range(vma, vma->vm_start,
+ __pa(pde_data(file_inode(file))) >> PAGE_SHIFT,
+ PAGE_SIZE, vma->vm_page_prot);
}
static const struct proc_ops page_map_proc_ops = {
@@ -45,13 +46,35 @@ static const struct proc_ops page_map_proc_ops = {
.proc_mmap = page_map_mmap,
};
+static union {
+ struct systemcfg data;
+ u8 page[PAGE_SIZE];
+} systemcfg_data_store __page_aligned_data;
+struct systemcfg *systemcfg = &systemcfg_data_store.data;
static int __init proc_ppc64_init(void)
{
struct proc_dir_entry *pde;
+ strcpy((char *)systemcfg->eye_catcher, "SYSTEMCFG:PPC64");
+ systemcfg->version.major = SYSTEMCFG_MAJOR;
+ systemcfg->version.minor = SYSTEMCFG_MINOR;
+ systemcfg->processor = mfspr(SPRN_PVR);
+ /*
+ * Fake the old platform number for pSeries and add
+ * in LPAR bit if necessary
+ */
+ systemcfg->platform = 0x100;
+ if (firmware_has_feature(FW_FEATURE_LPAR))
+ systemcfg->platform |= 1;
+ systemcfg->physicalMemorySize = memblock_phys_mem_size();
+ systemcfg->dcache_size = ppc64_caches.l1d.size;
+ systemcfg->dcache_line_size = ppc64_caches.l1d.line_size;
+ systemcfg->icache_size = ppc64_caches.l1i.size;
+ systemcfg->icache_line_size = ppc64_caches.l1i.line_size;
+
pde = proc_create_data("powerpc/systemcfg", S_IFREG | 0444, NULL,
- &page_map_proc_ops, vdso_data);
+ &page_map_proc_ops, systemcfg);
if (!pde)
return 1;
proc_set_size(pde, PAGE_SIZE);
@@ -60,7 +83,7 @@ static int __init proc_ppc64_init(void)
}
__initcall(proc_ppc64_init);
-#endif /* CONFIG_PPC64 */
+#endif /* CONFIG_PPC64_PROC_SYSTEMCFG */
/*
* Create the ppc64 and ppc64/rtas directories early. This allows us to
diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c
index 943430077375..0b732d3b283b 100644
--- a/arch/powerpc/kernel/setup-common.c
+++ b/arch/powerpc/kernel/setup-common.c
@@ -67,6 +67,7 @@
#include <asm/cpu_has_feature.h>
#include <asm/kasan.h>
#include <asm/mce.h>
+#include <asm/systemcfg.h>
#include "setup.h"
@@ -560,7 +561,9 @@ void __init smp_setup_cpu_maps(void)
out:
of_node_put(dn);
}
- vdso_data->processorCount = num_present_cpus();
+#endif
+#ifdef CONFIG_PPC64_PROC_SYSTEMCFG
+ systemcfg->processorCount = num_present_cpus();
#endif /* CONFIG_PPC64 */
/* Initialize CPU <=> thread mapping/
diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c
index 4ab9b8cee77a..5ac7084eebc0 100644
--- a/arch/powerpc/kernel/smp.c
+++ b/arch/powerpc/kernel/smp.c
@@ -61,6 +61,7 @@
#include <asm/ftrace.h>
#include <asm/kup.h>
#include <asm/fadump.h>
+#include <asm/systemcfg.h>
#include <trace/events/ipi.h>
@@ -1186,8 +1187,8 @@ int generic_cpu_disable(void)
return -EBUSY;
set_cpu_online(cpu, false);
-#ifdef CONFIG_PPC64
- vdso_data->processorCount--;
+#ifdef CONFIG_PPC64_PROC_SYSTEMCFG
+ systemcfg->processorCount--;
#endif
/* Update affinity of all IRQs previously aimed at this CPU */
irq_migrate_all_off_this_cpu();
@@ -1642,10 +1643,12 @@ void start_secondary(void *unused)
secondary_cpu_time_init();
-#ifdef CONFIG_PPC64
+#ifdef CONFIG_PPC64_PROC_SYSTEMCFG
if (system_state == SYSTEM_RUNNING)
- vdso_data->processorCount++;
+ systemcfg->processorCount++;
+#endif
+#ifdef CONFIG_PPC64
vdso_getcpu_init();
#endif
set_numa_node(numa_cpu_lookup_table[cpu]);
diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c
index 0ff9f038e800..0727332ad86f 100644
--- a/arch/powerpc/kernel/time.c
+++ b/arch/powerpc/kernel/time.c
@@ -71,11 +71,11 @@
#include <asm/vdso_datapage.h>
#include <asm/firmware.h>
#include <asm/mce.h>
+#include <asm/systemcfg.h>
/* powerpc clocksource/clockevent code */
#include <linux/clockchips.h>
-#include <linux/timekeeper_internal.h>
static u64 timebase_read(struct clocksource *);
static struct clocksource clocksource_timebase = {
@@ -951,6 +951,9 @@ void __init time_init(void)
}
vdso_data->tb_ticks_per_sec = tb_ticks_per_sec;
+#ifdef CONFIG_PPC64_PROC_SYSTEMCFG
+ systemcfg->tb_ticks_per_sec = tb_ticks_per_sec;
+#endif
/* initialise and enable the large decrementer (if we have one) */
set_decrementer_max();
diff --git a/arch/powerpc/kernel/vdso.c b/arch/powerpc/kernel/vdso.c
index ee4b9d676cff..924f7f4fa597 100644
--- a/arch/powerpc/kernel/vdso.c
+++ b/arch/powerpc/kernel/vdso.c
@@ -16,7 +16,6 @@
#include <linux/user.h>
#include <linux/elf.h>
#include <linux/security.h>
-#include <linux/memblock.h>
#include <linux/syscalls.h>
#include <linux/time_namespace.h>
#include <vdso/datapage.h>
@@ -349,25 +348,6 @@ static struct page ** __init vdso_setup_pages(void *start, void *end)
static int __init vdso_init(void)
{
#ifdef CONFIG_PPC64
- /*
- * Fill up the "systemcfg" stuff for backward compatibility
- */
- strcpy((char *)vdso_data->eye_catcher, "SYSTEMCFG:PPC64");
- vdso_data->version.major = SYSTEMCFG_MAJOR;
- vdso_data->version.minor = SYSTEMCFG_MINOR;
- vdso_data->processor = mfspr(SPRN_PVR);
- /*
- * Fake the old platform number for pSeries and add
- * in LPAR bit if necessary
- */
- vdso_data->platform = 0x100;
- if (firmware_has_feature(FW_FEATURE_LPAR))
- vdso_data->platform |= 1;
- vdso_data->physicalMemorySize = memblock_phys_mem_size();
- vdso_data->dcache_size = ppc64_caches.l1d.size;
- vdso_data->dcache_line_size = ppc64_caches.l1d.line_size;
- vdso_data->icache_size = ppc64_caches.l1i.size;
- vdso_data->icache_line_size = ppc64_caches.l1i.line_size;
vdso_data->dcache_block_size = ppc64_caches.l1d.block_size;
vdso_data->icache_block_size = ppc64_caches.l1i.block_size;
vdso_data->dcache_log_block_size = ppc64_caches.l1d.log_block_size;
diff --git a/arch/powerpc/platforms/powernv/smp.c b/arch/powerpc/platforms/powernv/smp.c
index 8f14f0581a21..2e9da58195f5 100644
--- a/arch/powerpc/platforms/powernv/smp.c
+++ b/arch/powerpc/platforms/powernv/smp.c
@@ -36,6 +36,7 @@
#include <asm/kexec.h>
#include <asm/reg.h>
#include <asm/powernv.h>
+#include <asm/systemcfg.h>
#include "powernv.h"
@@ -136,7 +137,9 @@ static int pnv_smp_cpu_disable(void)
* the generic fixup_irqs. --BenH.
*/
set_cpu_online(cpu, false);
- vdso_data->processorCount--;
+#ifdef CONFIG_PPC64_PROC_SYSTEMCFG
+ systemcfg->processorCount--;
+#endif
if (cpu == boot_cpuid)
boot_cpuid = cpumask_any(cpu_online_mask);
if (xive_enabled())
diff --git a/arch/powerpc/platforms/pseries/hotplug-cpu.c b/arch/powerpc/platforms/pseries/hotplug-cpu.c
index 6838a0fcda29..bc6926dbf148 100644
--- a/arch/powerpc/platforms/pseries/hotplug-cpu.c
+++ b/arch/powerpc/platforms/pseries/hotplug-cpu.c
@@ -33,6 +33,7 @@
#include <asm/xive.h>
#include <asm/plpar_wrappers.h>
#include <asm/topology.h>
+#include <asm/systemcfg.h>
#include "pseries.h"
@@ -83,7 +84,9 @@ static int pseries_cpu_disable(void)
int cpu = smp_processor_id();
set_cpu_online(cpu, false);
- vdso_data->processorCount--;
+#ifdef CONFIG_PPC64_PROC_SYSTEMCFG
+ systemcfg->processorCount--;
+#endif
/*fix boot_cpuid here*/
if (cpu == boot_cpuid)
diff --git a/arch/powerpc/platforms/pseries/lparcfg.c b/arch/powerpc/platforms/pseries/lparcfg.c
index 62da20f9700a..cc22924f159f 100644
--- a/arch/powerpc/platforms/pseries/lparcfg.c
+++ b/arch/powerpc/platforms/pseries/lparcfg.c
@@ -29,7 +29,6 @@
#include <asm/firmware.h>
#include <asm/rtas.h>
#include <asm/time.h>
-#include <asm/vdso_datapage.h>
#include <asm/vio.h>
#include <asm/mmu.h>
#include <asm/machdep.h>
@@ -530,7 +529,7 @@ static int pseries_lparcfg_data(struct seq_file *m, void *v)
lrdrp = of_get_property(rtas_node, "ibm,lrdr-capacity", NULL);
if (lrdrp == NULL) {
- partition_potential_processors = vdso_data->processorCount;
+ partition_potential_processors = num_possible_cpus();
} else {
partition_potential_processors = be32_to_cpup(lrdrp + 4);
}
@@ -553,7 +552,7 @@ static int pseries_lparcfg_data(struct seq_file *m, void *v)
} else { /* non SPLPAR case */
seq_printf(m, "system_active_processors=%d\n",
- partition_potential_processors);
+ partition_active_processors);
seq_printf(m, "system_potential_processors=%d\n",
partition_potential_processors);
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 62545946ecf4..c278280c134f 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -50,7 +50,7 @@ config RISCV
select ARCH_HAS_SYSCALL_WRAPPER
select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
select ARCH_HAS_UBSAN
- select ARCH_HAS_VDSO_DATA
+ select ARCH_HAS_VDSO_TIME_DATA
select ARCH_KEEP_MEMBLOCK if ACPI
select ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE if 64BIT && MMU
select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX
diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h
index 32d308a3355f..9875399827c7 100644
--- a/arch/riscv/include/asm/page.h
+++ b/arch/riscv/include/asm/page.h
@@ -12,9 +12,7 @@
#include <linux/pfn.h>
#include <linux/const.h>
-#define PAGE_SHIFT CONFIG_PAGE_SHIFT
-#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
-#define PAGE_MASK (~(PAGE_SIZE - 1))
+#include <vdso/page.h>
#define HPAGE_SHIFT PMD_SHIFT
#define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT)
diff --git a/arch/riscv/include/asm/vdso/data.h b/arch/riscv/include/asm/vdso/time_data.h
similarity index 71%
rename from arch/riscv/include/asm/vdso/data.h
rename to arch/riscv/include/asm/vdso/time_data.h
index dc2f76f58b76..dfa65228999b 100644
--- a/arch/riscv/include/asm/vdso/data.h
+++ b/arch/riscv/include/asm/vdso/time_data.h
@@ -1,12 +1,12 @@
/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __RISCV_ASM_VDSO_DATA_H
-#define __RISCV_ASM_VDSO_DATA_H
+#ifndef __RISCV_ASM_VDSO_TIME_DATA_H
+#define __RISCV_ASM_VDSO_TIME_DATA_H
#include <linux/types.h>
#include <vdso/datapage.h>
#include <asm/hwprobe.h>
-struct arch_vdso_data {
+struct arch_vdso_time_data {
/* Stash static answers to the hwprobe queries when all CPUs are selected. */
__u64 all_cpu_hwprobe_values[RISCV_HWPROBE_MAX_KEY + 1];
@@ -14,4 +14,4 @@ struct arch_vdso_data {
__u8 homogeneous_cpus;
};
-#endif /* __RISCV_ASM_VDSO_DATA_H */
+#endif /* __RISCV_ASM_VDSO_TIME_DATA_H */
diff --git a/arch/riscv/include/asm/vdso/vsyscall.h b/arch/riscv/include/asm/vdso/vsyscall.h
index 82fd5d83bd60..e8a9c4b53c0c 100644
--- a/arch/riscv/include/asm/vdso/vsyscall.h
+++ b/arch/riscv/include/asm/vdso/vsyscall.h
@@ -4,14 +4,10 @@
#ifndef __ASSEMBLY__
-#include <linux/timekeeper_internal.h>
#include <vdso/datapage.h>
extern struct vdso_data *vdso_data;
-/*
- * Update the vDSO data page to keep in sync with kernel timekeeping.
- */
static __always_inline struct vdso_data *__riscv_get_k_vdso_data(void)
{
return vdso_data;
diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c
index cea0ca2bf2a2..711a31f27c3d 100644
--- a/arch/riscv/kernel/sys_hwprobe.c
+++ b/arch/riscv/kernel/sys_hwprobe.c
@@ -402,7 +402,7 @@ static int do_riscv_hwprobe(struct riscv_hwprobe __user *pairs,
static int __init init_hwprobe_vdso_data(void)
{
struct vdso_data *vd = __arch_get_k_vdso_data();
- struct arch_vdso_data *avd = &vd->arch_data;
+ struct arch_vdso_time_data *avd = &vd->arch_data;
u64 id_bitsmash = 0;
struct riscv_hwprobe pair;
int key;
diff --git a/arch/riscv/kernel/vdso.c b/arch/riscv/kernel/vdso.c
index 98315b98256d..3ca3ae4277e1 100644
--- a/arch/riscv/kernel/vdso.c
+++ b/arch/riscv/kernel/vdso.c
@@ -23,11 +23,6 @@ enum vvar_pages {
VVAR_NR_PAGES,
};
-enum rv_vdso_map {
- RV_VDSO_MAP_VVAR,
- RV_VDSO_MAP_VDSO,
-};
-
#define VVAR_SIZE (VVAR_NR_PAGES << PAGE_SHIFT)
static union vdso_data_store vdso_data_store __page_aligned_data;
@@ -38,8 +33,6 @@ struct __vdso_info {
const char *vdso_code_start;
const char *vdso_code_end;
unsigned long vdso_pages;
- /* Data Mapping */
- struct vm_special_mapping *dm;
/* Code Mapping */
struct vm_special_mapping *cm;
};
@@ -92,6 +85,8 @@ struct vdso_data *arch_get_vdso_data(void *vvar_page)
return (struct vdso_data *)(vvar_page);
}
+static const struct vm_special_mapping rv_vvar_map;
+
/*
* The vvar mapping contains data for a specific time namespace, so when a task
* changes namespace we must unmap its vvar data for the old namespace.
@@ -108,12 +103,8 @@ int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
mmap_read_lock(mm);
for_each_vma(vmi, vma) {
- if (vma_is_special_mapping(vma, vdso_info.dm))
- zap_vma_pages(vma);
-#ifdef CONFIG_COMPAT
- if (vma_is_special_mapping(vma, compat_vdso_info.dm))
+ if (vma_is_special_mapping(vma, &rv_vvar_map))
zap_vma_pages(vma);
-#endif
}
mmap_read_unlock(mm);
@@ -155,43 +146,34 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
return vmf_insert_pfn(vma, vmf->address, pfn);
}
-static struct vm_special_mapping rv_vdso_maps[] __ro_after_init = {
- [RV_VDSO_MAP_VVAR] = {
- .name = "[vvar]",
- .fault = vvar_fault,
- },
- [RV_VDSO_MAP_VDSO] = {
- .name = "[vdso]",
- .mremap = vdso_mremap,
- },
+static const struct vm_special_mapping rv_vvar_map = {
+ .name = "[vvar]",
+ .fault = vvar_fault,
+};
+
+static struct vm_special_mapping rv_vdso_map __ro_after_init = {
+ .name = "[vdso]",
+ .mremap = vdso_mremap,
};
static struct __vdso_info vdso_info __ro_after_init = {
.name = "vdso",
.vdso_code_start = vdso_start,
.vdso_code_end = vdso_end,
- .dm = &rv_vdso_maps[RV_VDSO_MAP_VVAR],
- .cm = &rv_vdso_maps[RV_VDSO_MAP_VDSO],
+ .cm = &rv_vdso_map,
};
#ifdef CONFIG_COMPAT
-static struct vm_special_mapping rv_compat_vdso_maps[] __ro_after_init = {
- [RV_VDSO_MAP_VVAR] = {
- .name = "[vvar]",
- .fault = vvar_fault,
- },
- [RV_VDSO_MAP_VDSO] = {
- .name = "[vdso]",
- .mremap = vdso_mremap,
- },
+static struct vm_special_mapping rv_compat_vdso_map __ro_after_init = {
+ .name = "[vdso]",
+ .mremap = vdso_mremap,
};
static struct __vdso_info compat_vdso_info __ro_after_init = {
.name = "compat_vdso",
.vdso_code_start = compat_vdso_start,
.vdso_code_end = compat_vdso_end,
- .dm = &rv_compat_vdso_maps[RV_VDSO_MAP_VVAR],
- .cm = &rv_compat_vdso_maps[RV_VDSO_MAP_VDSO],
+ .cm = &rv_compat_vdso_map,
};
#endif
@@ -227,7 +209,7 @@ static int __setup_additional_pages(struct mm_struct *mm,
}
ret = _install_special_mapping(mm, vdso_base, VVAR_SIZE,
- (VM_READ | VM_MAYREAD | VM_PFNMAP), vdso_info->dm);
+ (VM_READ | VM_MAYREAD | VM_PFNMAP), &rv_vvar_map);
if (IS_ERR(ret))
goto up_fail;
diff --git a/arch/riscv/kernel/vdso/hwprobe.c b/arch/riscv/kernel/vdso/hwprobe.c
index 1e926e4b5881..a158c029344f 100644
--- a/arch/riscv/kernel/vdso/hwprobe.c
+++ b/arch/riscv/kernel/vdso/hwprobe.c
@@ -17,7 +17,7 @@ static int riscv_vdso_get_values(struct riscv_hwprobe *pairs, size_t pair_count,
unsigned int flags)
{
const struct vdso_data *vd = __arch_get_vdso_data();
- const struct arch_vdso_data *avd = &vd->arch_data;
+ const struct arch_vdso_time_data *avd = &vd->arch_data;
bool all_cpus = !cpusetsize && !cpus;
struct riscv_hwprobe *p = pairs;
struct riscv_hwprobe *end = pairs + pair_count;
@@ -52,7 +52,7 @@ static int riscv_vdso_get_cpus(struct riscv_hwprobe *pairs, size_t pair_count,
unsigned int flags)
{
const struct vdso_data *vd = __arch_get_vdso_data();
- const struct arch_vdso_data *avd = &vd->arch_data;
+ const struct arch_vdso_time_data *avd = &vd->arch_data;
struct riscv_hwprobe *p = pairs;
struct riscv_hwprobe *end = pairs + pair_count;
unsigned char *c = (unsigned char *)cpus;
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index d339fe4fdedf..8cdd8359e00c 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -88,7 +88,7 @@ config S390
select ARCH_HAS_STRICT_MODULE_RWX
select ARCH_HAS_SYSCALL_WRAPPER
select ARCH_HAS_UBSAN
- select ARCH_HAS_VDSO_DATA
+ select ARCH_HAS_VDSO_TIME_DATA
select ARCH_HAVE_NMI_SAFE_CMPXCHG
select ARCH_INLINE_READ_LOCK
select ARCH_INLINE_READ_LOCK_BH
diff --git a/arch/s390/include/asm/page.h b/arch/s390/include/asm/page.h
index 73e1e03317b4..b7ba87f89761 100644
--- a/arch/s390/include/asm/page.h
+++ b/arch/s390/include/asm/page.h
@@ -11,14 +11,8 @@
#include <linux/const.h>
#include <asm/types.h>
-#define _PAGE_SHIFT CONFIG_PAGE_SHIFT
-#define _PAGE_SIZE (_AC(1, UL) << _PAGE_SHIFT)
-#define _PAGE_MASK (~(_PAGE_SIZE - 1))
-
-/* PAGE_SHIFT determines the page size */
-#define PAGE_SHIFT _PAGE_SHIFT
-#define PAGE_SIZE _PAGE_SIZE
-#define PAGE_MASK _PAGE_MASK
+#include <vdso/page.h>
+
#define PAGE_DEFAULT_ACC _AC(0, UL)
/* storage-protection override */
#define PAGE_SPO_ACC 9
diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index 0ffbaf741955..8b67036edb69 100644
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -338,7 +338,7 @@ static inline int is_module_addr(void *addr)
#define _REGION2_INDEX (0x7ffUL << _REGION2_SHIFT)
#define _REGION3_INDEX (0x7ffUL << _REGION3_SHIFT)
#define _SEGMENT_INDEX (0x7ffUL << _SEGMENT_SHIFT)
-#define _PAGE_INDEX (0xffUL << _PAGE_SHIFT)
+#define _PAGE_INDEX (0xffUL << PAGE_SHIFT)
#define _REGION1_SIZE (1UL << _REGION1_SHIFT)
#define _REGION2_SIZE (1UL << _REGION2_SHIFT)
diff --git a/arch/s390/include/asm/vdso.h b/arch/s390/include/asm/vdso.h
index 91061f0279be..92c73e4d97a9 100644
--- a/arch/s390/include/asm/vdso.h
+++ b/arch/s390/include/asm/vdso.h
@@ -12,9 +12,6 @@ int vdso_getcpu_init(void);
#endif /* __ASSEMBLY__ */
-/* Default link address for the vDSO */
-#define VDSO_LBASE 0
-
#define __VVAR_PAGES 2
#define VDSO_VERSION_STRING LINUX_2.6.29
diff --git a/arch/s390/include/asm/vdso/data.h b/arch/s390/include/asm/vdso/data.h
deleted file mode 100644
index 0e2b40ef69b0..000000000000
--- a/arch/s390/include/asm/vdso/data.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __S390_ASM_VDSO_DATA_H
-#define __S390_ASM_VDSO_DATA_H
-
-#include <linux/types.h>
-
-struct arch_vdso_data {
- __s64 tod_steering_delta;
- __u64 tod_steering_end;
-};
-
-#endif /* __S390_ASM_VDSO_DATA_H */
diff --git a/arch/s390/include/asm/vdso/time_data.h b/arch/s390/include/asm/vdso/time_data.h
new file mode 100644
index 000000000000..8a08752422e6
--- /dev/null
+++ b/arch/s390/include/asm/vdso/time_data.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __S390_ASM_VDSO_TIME_DATA_H
+#define __S390_ASM_VDSO_TIME_DATA_H
+
+#include <linux/types.h>
+
+struct arch_vdso_time_data {
+ __s64 tod_steering_delta;
+ __u64 tod_steering_end;
+};
+
+#endif /* __S390_ASM_VDSO_TIME_DATA_H */
diff --git a/arch/s390/include/asm/vdso/vsyscall.h b/arch/s390/include/asm/vdso/vsyscall.h
index 3c5d5e47814e..3eb576ecd3bd 100644
--- a/arch/s390/include/asm/vdso/vsyscall.h
+++ b/arch/s390/include/asm/vdso/vsyscall.h
@@ -7,7 +7,6 @@
#ifndef __ASSEMBLY__
#include <linux/hrtimer.h>
-#include <linux/timekeeper_internal.h>
#include <vdso/datapage.h>
#include <asm/vdso.h>
@@ -17,10 +16,6 @@ enum vvar_pages {
VVAR_NR_PAGES
};
-/*
- * Update the vDSO data page to keep in sync with kernel timekeeping.
- */
-
static __always_inline struct vdso_data *__s390_get_k_vdso_data(void)
{
return vdso_data;
diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c
index b713effe0579..4fae6c701784 100644
--- a/arch/s390/kernel/time.c
+++ b/arch/s390/kernel/time.c
@@ -36,7 +36,6 @@
#include <linux/profile.h>
#include <linux/timex.h>
#include <linux/notifier.h>
-#include <linux/timekeeper_internal.h>
#include <linux/clockchips.h>
#include <linux/gfp.h>
#include <linux/kprobes.h>
diff --git a/arch/s390/kernel/vdso32/vdso32.lds.S b/arch/s390/kernel/vdso32/vdso32.lds.S
index 65b9513a5a0e..c916c4f73f76 100644
--- a/arch/s390/kernel/vdso32/vdso32.lds.S
+++ b/arch/s390/kernel/vdso32/vdso32.lds.S
@@ -16,7 +16,7 @@ SECTIONS
#ifdef CONFIG_TIME_NS
PROVIDE(_timens_data = _vdso_data + PAGE_SIZE);
#endif
- . = VDSO_LBASE + SIZEOF_HEADERS;
+ . = SIZEOF_HEADERS;
.hash : { *(.hash) } :text
.gnu.hash : { *(.gnu.hash) }
diff --git a/arch/s390/kernel/vdso64/vdso64.lds.S b/arch/s390/kernel/vdso64/vdso64.lds.S
index 753040a4b5ab..ec42b7d9cb53 100644
--- a/arch/s390/kernel/vdso64/vdso64.lds.S
+++ b/arch/s390/kernel/vdso64/vdso64.lds.S
@@ -18,7 +18,7 @@ SECTIONS
#ifdef CONFIG_TIME_NS
PROVIDE(_timens_data = _vdso_data + PAGE_SIZE);
#endif
- . = VDSO_LBASE + SIZEOF_HEADERS;
+ . = SIZEOF_HEADERS;
.hash : { *(.hash) } :text
.gnu.hash : { *(.gnu.hash) }
diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
index ad8b0d6b77ea..12e10269dfcd 100644
--- a/arch/s390/mm/fault.c
+++ b/arch/s390/mm/fault.c
@@ -147,7 +147,7 @@ static void dump_pagetable(unsigned long asce, unsigned long address)
goto out;
table = __va(entry & _SEGMENT_ENTRY_ORIGIN);
}
- table += (address & _PAGE_INDEX) >> _PAGE_SHIFT;
+ table += (address & _PAGE_INDEX) >> PAGE_SHIFT;
if (get_kernel_nofault(entry, table))
goto bad;
pr_cont("P:%016lx ", entry);
diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c
index eb0b51a36be0..346ec059c8bd 100644
--- a/arch/s390/mm/gmap.c
+++ b/arch/s390/mm/gmap.c
@@ -851,7 +851,7 @@ static inline unsigned long *gmap_table_walk(struct gmap *gmap,
if (*table & _REGION_ENTRY_INVALID)
return NULL;
table = __va(*table & _SEGMENT_ENTRY_ORIGIN);
- table += (gaddr & _PAGE_INDEX) >> _PAGE_SHIFT;
+ table += (gaddr & _PAGE_INDEX) >> PAGE_SHIFT;
}
return table;
}
@@ -1317,7 +1317,7 @@ static void gmap_unshadow_page(struct gmap *sg, unsigned long raddr)
table = gmap_table_walk(sg, raddr, 0); /* get page table pointer */
if (!table || *table & _PAGE_INVALID)
return;
- gmap_call_notifier(sg, raddr, raddr + _PAGE_SIZE - 1);
+ gmap_call_notifier(sg, raddr, raddr + PAGE_SIZE - 1);
ptep_unshadow_pte(sg->mm, raddr, (pte_t *) table);
}
@@ -1335,7 +1335,7 @@ static void __gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr,
int i;
BUG_ON(!gmap_is_shadow(sg));
- for (i = 0; i < _PAGE_ENTRIES; i++, raddr += _PAGE_SIZE)
+ for (i = 0; i < _PAGE_ENTRIES; i++, raddr += PAGE_SIZE)
pgt[i] = _PAGE_INVALID;
}
diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c
index f691e0fb66a2..58696a0c4e4a 100644
--- a/arch/s390/mm/pgalloc.c
+++ b/arch/s390/mm/pgalloc.c
@@ -278,7 +278,7 @@ static inline unsigned long base_##NAME##_addr_end(unsigned long addr, \
return (next - 1) < (end - 1) ? next : end; \
}
-BASE_ADDR_END_FUNC(page, _PAGE_SIZE)
+BASE_ADDR_END_FUNC(page, PAGE_SIZE)
BASE_ADDR_END_FUNC(segment, _SEGMENT_SIZE)
BASE_ADDR_END_FUNC(region3, _REGION3_SIZE)
BASE_ADDR_END_FUNC(region2, _REGION2_SIZE)
@@ -302,7 +302,7 @@ static int base_page_walk(unsigned long *origin, unsigned long addr,
if (!alloc)
return 0;
pte = origin;
- pte += (addr & _PAGE_INDEX) >> _PAGE_SHIFT;
+ pte += (addr & _PAGE_INDEX) >> PAGE_SHIFT;
do {
next = base_page_addr_end(addr, end);
*pte = base_lra(addr);
diff --git a/arch/sh/include/asm/page.h b/arch/sh/include/asm/page.h
index f780b467e75d..fc39b8171bfb 100644
--- a/arch/sh/include/asm/page.h
+++ b/arch/sh/include/asm/page.h
@@ -8,10 +8,8 @@
#include <linux/const.h>
-/* PAGE_SHIFT determines the page size */
-#define PAGE_SHIFT CONFIG_PAGE_SHIFT
-#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
-#define PAGE_MASK (~(PAGE_SIZE-1))
+#include <vdso/page.h>
+
#define PTE_MASK PAGE_MASK
#if defined(CONFIG_HUGETLB_PAGE_SIZE_64K)
diff --git a/arch/sparc/include/asm/page_32.h b/arch/sparc/include/asm/page_32.h
index 9977c77374cd..9954254ea569 100644
--- a/arch/sparc/include/asm/page_32.h
+++ b/arch/sparc/include/asm/page_32.h
@@ -11,9 +11,7 @@
#include <linux/const.h>
-#define PAGE_SHIFT CONFIG_PAGE_SHIFT
-#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
-#define PAGE_MASK (~(PAGE_SIZE-1))
+#include <vdso/page.h>
#ifndef __ASSEMBLY__
diff --git a/arch/sparc/include/asm/page_64.h b/arch/sparc/include/asm/page_64.h
index e9bd24821c93..2a68ff5b6eab 100644
--- a/arch/sparc/include/asm/page_64.h
+++ b/arch/sparc/include/asm/page_64.h
@@ -4,9 +4,7 @@
#include <linux/const.h>
-#define PAGE_SHIFT CONFIG_PAGE_SHIFT
-#define PAGE_SIZE (_AC(1,UL) << PAGE_SHIFT)
-#define PAGE_MASK (~(PAGE_SIZE-1))
+#include <vdso/page.h>
/* Flushing for D-cache alias handling is only needed if
* the page size is smaller than 16K.
diff --git a/arch/um/include/asm/page.h b/arch/um/include/asm/page.h
index 9ef9a8aedfa6..834313ecd3d6 100644
--- a/arch/um/include/asm/page.h
+++ b/arch/um/include/asm/page.h
@@ -9,10 +9,7 @@
#include <linux/const.h>
-/* PAGE_SHIFT determines the page size */
-#define PAGE_SHIFT CONFIG_PAGE_SHIFT
-#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
-#define PAGE_MASK (~(PAGE_SIZE-1))
+#include <vdso/page.h>
#ifndef __ASSEMBLY__
diff --git a/arch/x86/entry/vdso/vdso-layout.lds.S b/arch/x86/entry/vdso/vdso-layout.lds.S
index bafa73f09e92..872947c1004c 100644
--- a/arch/x86/entry/vdso/vdso-layout.lds.S
+++ b/arch/x86/entry/vdso/vdso-layout.lds.S
@@ -1,5 +1,6 @@
/* SPDX-License-Identifier: GPL-2.0 */
#include <asm/vdso.h>
+#include <asm/vdso/vsyscall.h>
/*
* Linker script for vDSO. This is an ELF shared object prelinked to
@@ -16,23 +17,16 @@ SECTIONS
* segment.
*/
- vvar_start = . - 4 * PAGE_SIZE;
+ vvar_start = . - __VVAR_PAGES * PAGE_SIZE;
vvar_page = vvar_start;
- /* Place all vvars at the offsets in asm/vvar.h. */
-#define EMIT_VVAR(name, offset) vvar_ ## name = vvar_page + offset;
-#include <asm/vvar.h>
-#undef EMIT_VVAR
+ vdso_rng_data = vvar_page + __VDSO_RND_DATA_OFFSET;
- pvclock_page = vvar_start + PAGE_SIZE;
- hvclock_page = vvar_start + 2 * PAGE_SIZE;
- timens_page = vvar_start + 3 * PAGE_SIZE;
+ timens_page = vvar_start + PAGE_SIZE;
-#undef _ASM_X86_VVAR_H
- /* Place all vvars in timens too at the offsets in asm/vvar.h. */
-#define EMIT_VVAR(name, offset) timens_ ## name = timens_page + offset;
-#include <asm/vvar.h>
-#undef EMIT_VVAR
+ vclock_pages = vvar_start + VDSO_NR_VCLOCK_PAGES * PAGE_SIZE;
+ pvclock_page = vclock_pages + VDSO_PAGE_PVCLOCK_OFFSET * PAGE_SIZE;
+ hvclock_page = vclock_pages + VDSO_PAGE_HVCLOCK_OFFSET * PAGE_SIZE;
. = SIZEOF_HEADERS;
diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c
index b8fed8b8b9cc..bfc7cabf4017 100644
--- a/arch/x86/entry/vdso/vma.c
+++ b/arch/x86/entry/vdso/vma.c
@@ -20,26 +20,20 @@
#include <asm/vgtod.h>
#include <asm/proto.h>
#include <asm/vdso.h>
-#include <asm/vvar.h>
#include <asm/tlb.h>
#include <asm/page.h>
#include <asm/desc.h>
#include <asm/cpufeature.h>
+#include <asm/vdso/vsyscall.h>
#include <clocksource/hyperv_timer.h>
-#undef _ASM_X86_VVAR_H
-#define EMIT_VVAR(name, offset) \
- const size_t name ## _offset = offset;
-#include <asm/vvar.h>
-
struct vdso_data *arch_get_vdso_data(void *vvar_page)
{
- return (struct vdso_data *)(vvar_page + _vdso_data_offset);
+ return (struct vdso_data *)vvar_page;
}
-#undef EMIT_VVAR
-DEFINE_VVAR(struct vdso_data, _vdso_data);
-DEFINE_VVAR_SINGLE(struct vdso_rng_data, _vdso_rng_data);
+static union vdso_data_store vdso_data_store __page_aligned_data;
+struct vdso_data *vdso_data = vdso_data_store.data;
unsigned int vclocks_used __read_mostly;
@@ -154,7 +148,7 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
if (sym_offset == image->sym_vvar_page) {
struct page *timens_page = find_timens_vvar_page(vma);
- pfn = __pa_symbol(&__vvar_page) >> PAGE_SHIFT;
+ pfn = __pa_symbol(vdso_data) >> PAGE_SHIFT;
/*
* If a task belongs to a time namespace then a namespace
@@ -182,32 +176,52 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm,
}
return vmf_insert_pfn(vma, vmf->address, pfn);
- } else if (sym_offset == image->sym_pvclock_page) {
- struct pvclock_vsyscall_time_info *pvti =
- pvclock_get_pvti_cpu0_va();
- if (pvti && vclock_was_used(VDSO_CLOCKMODE_PVCLOCK)) {
- return vmf_insert_pfn_prot(vma, vmf->address,
- __pa(pvti) >> PAGE_SHIFT,
- pgprot_decrypted(vma->vm_page_prot));
- }
- } else if (sym_offset == image->sym_hvclock_page) {
- pfn = hv_get_tsc_pfn();
- if (pfn && vclock_was_used(VDSO_CLOCKMODE_HVCLOCK))
- return vmf_insert_pfn(vma, vmf->address, pfn);
} else if (sym_offset == image->sym_timens_page) {
struct page *timens_page = find_timens_vvar_page(vma);
if (!timens_page)
return VM_FAULT_SIGBUS;
- pfn = __pa_symbol(&__vvar_page) >> PAGE_SHIFT;
+ pfn = __pa_symbol(vdso_data) >> PAGE_SHIFT;
return vmf_insert_pfn(vma, vmf->address, pfn);
}
return VM_FAULT_SIGBUS;
}
+static vm_fault_t vvar_vclock_fault(const struct vm_special_mapping *sm,
+ struct vm_area_struct *vma, struct vm_fault *vmf)
+{
+ switch (vmf->pgoff) {
+#ifdef CONFIG_PARAVIRT_CLOCK
+ case VDSO_PAGE_PVCLOCK_OFFSET:
+ {
+ struct pvclock_vsyscall_time_info *pvti =
+ pvclock_get_pvti_cpu0_va();
+
+ if (pvti && vclock_was_used(VDSO_CLOCKMODE_PVCLOCK))
+ return vmf_insert_pfn_prot(vma, vmf->address,
+ __pa(pvti) >> PAGE_SHIFT,
+ pgprot_decrypted(vma->vm_page_prot));
+ break;
+ }
+#endif /* CONFIG_PARAVIRT_CLOCK */
+#ifdef CONFIG_HYPERV_TIMER
+ case VDSO_PAGE_HVCLOCK_OFFSET:
+ {
+ unsigned long pfn = hv_get_tsc_pfn();
+
+ if (pfn && vclock_was_used(VDSO_CLOCKMODE_HVCLOCK))
+ return vmf_insert_pfn(vma, vmf->address, pfn);
+ break;
+ }
+#endif /* CONFIG_HYPERV_TIMER */
+ }
+
+ return VM_FAULT_SIGBUS;
+}
+
static const struct vm_special_mapping vdso_mapping = {
.name = "[vdso]",
.fault = vdso_fault,
@@ -217,6 +231,10 @@ static const struct vm_special_mapping vvar_mapping = {
.name = "[vvar]",
.fault = vvar_fault,
};
+static const struct vm_special_mapping vvar_vclock_mapping = {
+ .name = "[vvar_vclock]",
+ .fault = vvar_vclock_fault,
+};
/*
* Add vdso and vvar mappings to current process.
@@ -259,7 +277,7 @@ static int map_vdso(const struct vdso_image *image, unsigned long addr)
vma = _install_special_mapping(mm,
addr,
- -image->sym_vvar_start,
+ (__VVAR_PAGES - VDSO_NR_VCLOCK_PAGES) * PAGE_SIZE,
VM_READ|VM_MAYREAD|VM_IO|VM_DONTDUMP|
VM_PFNMAP,
&vvar_mapping);
@@ -267,11 +285,26 @@ static int map_vdso(const struct vdso_image *image, unsigned long addr)
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
do_munmap(mm, text_start, image->size, NULL);
- } else {
- current->mm->context.vdso = (void __user *)text_start;
- current->mm->context.vdso_image = image;
+ goto up_fail;
}
+ vma = _install_special_mapping(mm,
+ addr + (__VVAR_PAGES - VDSO_NR_VCLOCK_PAGES) * PAGE_SIZE,
+ VDSO_NR_VCLOCK_PAGES * PAGE_SIZE,
+ VM_READ|VM_MAYREAD|VM_IO|VM_DONTDUMP|
+ VM_PFNMAP,
+ &vvar_vclock_mapping);
+
+ if (IS_ERR(vma)) {
+ ret = PTR_ERR(vma);
+ do_munmap(mm, text_start, image->size, NULL);
+ do_munmap(mm, addr, image->size, NULL);
+ goto up_fail;
+ }
+
+ current->mm->context.vdso = (void __user *)text_start;
+ current->mm->context.vdso_image = image;
+
up_fail:
mmap_write_unlock(mm);
return ret;
@@ -293,7 +326,8 @@ int map_vdso_once(const struct vdso_image *image, unsigned long addr)
*/
for_each_vma(vmi, vma) {
if (vma_is_special_mapping(vma, &vdso_mapping) ||
- vma_is_special_mapping(vma, &vvar_mapping)) {
+ vma_is_special_mapping(vma, &vvar_mapping) ||
+ vma_is_special_mapping(vma, &vvar_vclock_mapping)) {
mmap_write_unlock(mm);
return -EEXIST;
}
diff --git a/arch/x86/include/asm/page_types.h b/arch/x86/include/asm/page_types.h
index 52f1b4ff0cc1..974688973cf6 100644
--- a/arch/x86/include/asm/page_types.h
+++ b/arch/x86/include/asm/page_types.h
@@ -6,10 +6,7 @@
#include <linux/types.h>
#include <linux/mem_encrypt.h>
-/* PAGE_SHIFT determines the page size */
-#define PAGE_SHIFT CONFIG_PAGE_SHIFT
-#define PAGE_SIZE (_AC(1,UL) << PAGE_SHIFT)
-#define PAGE_MASK (~(PAGE_SIZE-1))
+#include <vdso/page.h>
#define __VIRTUAL_MASK ((1UL << __VIRTUAL_MASK_SHIFT) - 1)
diff --git a/arch/x86/include/asm/vdso/getrandom.h b/arch/x86/include/asm/vdso/getrandom.h
index ff5334ad32a0..2bf9c0e970c3 100644
--- a/arch/x86/include/asm/vdso/getrandom.h
+++ b/arch/x86/include/asm/vdso/getrandom.h
@@ -8,7 +8,6 @@
#ifndef __ASSEMBLY__
#include <asm/unistd.h>
-#include <asm/vvar.h>
/**
* getrandom_syscall - Invoke the getrandom() syscall.
@@ -28,13 +27,14 @@ static __always_inline ssize_t getrandom_syscall(void *buffer, size_t len, unsig
return ret;
}
-#define __vdso_rng_data (VVAR(_vdso_rng_data))
+extern struct vdso_rng_data vdso_rng_data
+ __attribute__((visibility("hidden")));
static __always_inline const struct vdso_rng_data *__arch_get_vdso_rng_data(void)
{
- if (IS_ENABLED(CONFIG_TIME_NS) && __vdso_data->clock_mode == VDSO_CLOCKMODE_TIMENS)
- return (void *)&__vdso_rng_data + ((void *)&__timens_vdso_data - (void *)&__vdso_data);
- return &__vdso_rng_data;
+ if (IS_ENABLED(CONFIG_TIME_NS) && __arch_get_vdso_data()->clock_mode == VDSO_CLOCKMODE_TIMENS)
+ return (void *)&vdso_rng_data + ((void *)&timens_page - (void *)__arch_get_vdso_data());
+ return &vdso_rng_data;
}
#endif /* !__ASSEMBLY__ */
diff --git a/arch/x86/include/asm/vdso/gettimeofday.h b/arch/x86/include/asm/vdso/gettimeofday.h
index b2d2df026f6e..375a34b0f365 100644
--- a/arch/x86/include/asm/vdso/gettimeofday.h
+++ b/arch/x86/include/asm/vdso/gettimeofday.h
@@ -14,14 +14,16 @@
#include <uapi/linux/time.h>
#include <asm/vgtod.h>
-#include <asm/vvar.h>
#include <asm/unistd.h>
#include <asm/msr.h>
#include <asm/pvclock.h>
#include <clocksource/hyperv_timer.h>
-#define __vdso_data (VVAR(_vdso_data))
-#define __timens_vdso_data (TIMENS(_vdso_data))
+extern struct vdso_data vvar_page
+ __attribute__((visibility("hidden")));
+
+extern struct vdso_data timens_page
+ __attribute__((visibility("hidden")));
#define VDSO_HAS_TIME 1
@@ -61,7 +63,7 @@ extern struct ms_hyperv_tsc_page hvclock_page
static __always_inline
const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd)
{
- return __timens_vdso_data;
+ return &timens_page;
}
#endif
@@ -275,7 +277,7 @@ static inline u64 __arch_get_hw_counter(s32 clock_mode,
static __always_inline const struct vdso_data *__arch_get_vdso_data(void)
{
- return __vdso_data;
+ return &vvar_page;
}
static inline bool arch_vdso_clocksource_ok(const struct vdso_data *vd)
diff --git a/arch/x86/include/asm/vdso/vsyscall.h b/arch/x86/include/asm/vdso/vsyscall.h
index 67fedf1698b5..37b4a70559a8 100644
--- a/arch/x86/include/asm/vdso/vsyscall.h
+++ b/arch/x86/include/asm/vdso/vsyscall.h
@@ -2,12 +2,19 @@
#ifndef __ASM_VDSO_VSYSCALL_H
#define __ASM_VDSO_VSYSCALL_H
+#define __VDSO_RND_DATA_OFFSET 640
+#define __VVAR_PAGES 4
+
+#define VDSO_NR_VCLOCK_PAGES 2
+#define VDSO_PAGE_PVCLOCK_OFFSET 0
+#define VDSO_PAGE_HVCLOCK_OFFSET 1
+
#ifndef __ASSEMBLY__
-#include <linux/timekeeper_internal.h>
#include <vdso/datapage.h>
#include <asm/vgtod.h>
-#include <asm/vvar.h>
+
+extern struct vdso_data *vdso_data;
/*
* Update the vDSO data page to keep in sync with kernel timekeeping.
@@ -15,14 +22,14 @@
static __always_inline
struct vdso_data *__x86_get_k_vdso_data(void)
{
- return _vdso_data;
+ return vdso_data;
}
#define __arch_get_k_vdso_data __x86_get_k_vdso_data
static __always_inline
struct vdso_rng_data *__x86_get_k_vdso_rng_data(void)
{
- return &_vdso_rng_data;
+ return (void *)vdso_data + __VDSO_RND_DATA_OFFSET;
}
#define __arch_get_k_vdso_rng_data __x86_get_k_vdso_rng_data
diff --git a/arch/x86/include/asm/vvar.h b/arch/x86/include/asm/vvar.h
deleted file mode 100644
index 9d9af37f7cab..000000000000
--- a/arch/x86/include/asm/vvar.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * vvar.h: Shared vDSO/kernel variable declarations
- * Copyright (c) 2011 Andy Lutomirski
- *
- * A handful of variables are accessible (read-only) from userspace
- * code in the vsyscall page and the vdso. They are declared here.
- * Some other file must define them with DEFINE_VVAR.
- *
- * In normal kernel code, they are used like any other variable.
- * In user code, they are accessed through the VVAR macro.
- *
- * These variables live in a page of kernel data that has an extra RO
- * mapping for userspace. Each variable needs a unique offset within
- * that page; specify that offset with the DECLARE_VVAR macro. (If
- * you mess up, the linker will catch it.)
- */
-
-#ifndef _ASM_X86_VVAR_H
-#define _ASM_X86_VVAR_H
-
-#ifdef EMIT_VVAR
-/*
- * EMIT_VVAR() is used by the kernel linker script to put vvars in the
- * right place. Also, it's used by kernel code to import offsets values.
- */
-#define DECLARE_VVAR(offset, type, name) \
- EMIT_VVAR(name, offset)
-#define DECLARE_VVAR_SINGLE(offset, type, name) \
- EMIT_VVAR(name, offset)
-
-#else
-
-extern char __vvar_page;
-
-#define DECLARE_VVAR(offset, type, name) \
- extern type vvar_ ## name[CS_BASES] \
- __attribute__((visibility("hidden"))); \
- extern type timens_ ## name[CS_BASES] \
- __attribute__((visibility("hidden"))); \
-
-#define DECLARE_VVAR_SINGLE(offset, type, name) \
- extern type vvar_ ## name \
- __attribute__((visibility("hidden"))); \
-
-#define VVAR(name) (vvar_ ## name)
-#define TIMENS(name) (timens_ ## name)
-
-#define DEFINE_VVAR(type, name) \
- type name[CS_BASES] \
- __attribute__((section(".vvar_" #name), aligned(16))) __visible
-
-#define DEFINE_VVAR_SINGLE(type, name) \
- type name \
- __attribute__((section(".vvar_" #name), aligned(16))) __visible
-
-#endif
-
-/* DECLARE_VVAR(offset, type, name) */
-
-DECLARE_VVAR(128, struct vdso_data, _vdso_data)
-
-#if !defined(_SINGLE_DATA)
-#define _SINGLE_DATA
-DECLARE_VVAR_SINGLE(640, struct vdso_rng_data, _vdso_rng_data)
-#endif
-
-#undef DECLARE_VVAR
-#undef DECLARE_VVAR_SINGLE
-
-#endif
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 6726be89b7a6..e7e19842736a 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -193,29 +193,6 @@ SECTIONS
ORC_UNWIND_TABLE
- . = ALIGN(PAGE_SIZE);
- __vvar_page = .;
-
- .vvar : AT(ADDR(.vvar) - LOAD_OFFSET) {
- /* work around gold bug 13023 */
- __vvar_beginning_hack = .;
-
- /* Place all vvars at the offsets in asm/vvar.h. */
-#define EMIT_VVAR(name, offset) \
- . = __vvar_beginning_hack + offset; \
- *(.vvar_ ## name)
-#include <asm/vvar.h>
-#undef EMIT_VVAR
-
- /*
- * Pad the rest of the page with zeros. Otherwise the loader
- * can leave garbage here.
- */
- . = __vvar_beginning_hack + PAGE_SIZE;
- } :data
-
- . = ALIGN(__vvar_page + PAGE_SIZE, PAGE_SIZE);
-
/* Init code and data - will be freed after init */
. = ALIGN(PAGE_SIZE);
.init.begin : AT(ADDR(.init.begin) - LOAD_OFFSET) {
diff --git a/arch/x86/mm/mmap.c b/arch/x86/mm/mmap.c
index a2cabb1c81e1..b8a6ffffb451 100644
--- a/arch/x86/mm/mmap.c
+++ b/arch/x86/mm/mmap.c
@@ -163,11 +163,6 @@ unsigned long get_mmap_base(int is_legacy)
return is_legacy ? mm->mmap_legacy_base : mm->mmap_base;
}
-const char *arch_vma_name(struct vm_area_struct *vma)
-{
- return NULL;
-}
-
/**
* mmap_address_hint_valid - Validate the address hint of mmap
* @addr: Address hint
diff --git a/arch/x86/tools/relocs.c b/arch/x86/tools/relocs.c
index c101bed61940..6afe2e5e9102 100644
--- a/arch/x86/tools/relocs.c
+++ b/arch/x86/tools/relocs.c
@@ -89,7 +89,6 @@ static const char * const sym_regex_kernel[S_NSYMTYPES] = {
"init_per_cpu__.*|"
"__end_rodata_hpage_align|"
#endif
- "__vvar_page|"
"_end)$"
};
diff --git a/arch/xtensa/include/asm/page.h b/arch/xtensa/include/asm/page.h
index 4db56ef052d2..595c1037b738 100644
--- a/arch/xtensa/include/asm/page.h
+++ b/arch/xtensa/include/asm/page.h
@@ -18,13 +18,7 @@
#include <asm/cache.h>
#include <asm/kmem_layout.h>
-/*
- * PAGE_SHIFT determines the page size
- */
-
-#define PAGE_SHIFT CONFIG_PAGE_SHIFT
-#define PAGE_SIZE (__XTENSA_UL_CONST(1) << PAGE_SHIFT)
-#define PAGE_MASK (~(PAGE_SIZE-1))
+#include <vdso/page.h>
#ifdef CONFIG_MMU
#define PAGE_OFFSET XCHAL_KSEG_CACHED_VADDR
diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c
index a6c69a706fd7..bb29f361110e 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt.c
@@ -302,7 +302,7 @@ static void gen6_check_faults(struct intel_gt *gt)
{
struct intel_engine_cs *engine;
enum intel_engine_id id;
- u32 fault;
+ unsigned long fault;
for_each_engine(engine, gt, id) {
fault = GEN6_RING_FAULT_REG_READ(engine);
@@ -310,8 +310,8 @@ static void gen6_check_faults(struct intel_gt *gt)
gt_dbg(gt, "Unexpected fault\n"
"\tAddr: 0x%08lx\n"
"\tAddress space: %s\n"
- "\tSource ID: %d\n"
- "\tType: %d\n",
+ "\tSource ID: %ld\n"
+ "\tType: %ld\n",
fault & PAGE_MASK,
fault & RING_FAULT_GTTSEL_MASK ?
"GGTT" : "PPGTT",
diff --git a/include/asm-generic/vdso/vsyscall.h b/include/asm-generic/vdso/vsyscall.h
index c835607f78ae..01dafd604188 100644
--- a/include/asm-generic/vdso/vsyscall.h
+++ b/include/asm-generic/vdso/vsyscall.h
@@ -12,8 +12,7 @@ static __always_inline struct vdso_data *__arch_get_k_vdso_data(void)
#endif /* __arch_get_k_vdso_data */
#ifndef __arch_update_vsyscall
-static __always_inline void __arch_update_vsyscall(struct vdso_data *vdata,
- struct timekeeper *tk)
+static __always_inline void __arch_update_vsyscall(struct vdso_data *vdata)
{
}
#endif /* __arch_update_vsyscall */
diff --git a/include/vdso/datapage.h b/include/vdso/datapage.h
index b85f24cac3f5..d967baa0cd0c 100644
--- a/include/vdso/datapage.h
+++ b/include/vdso/datapage.h
@@ -19,10 +19,10 @@
#include <vdso/time32.h>
#include <vdso/time64.h>
-#ifdef CONFIG_ARCH_HAS_VDSO_DATA
-#include <asm/vdso/data.h>
+#ifdef CONFIG_ARCH_HAS_VDSO_TIME_DATA
+#include <asm/vdso/time_data.h>
#else
-struct arch_vdso_data {};
+struct arch_vdso_time_data {};
#endif
#define VDSO_BASES (CLOCK_TAI + 1)
@@ -114,7 +114,7 @@ struct vdso_data {
u32 hrtimer_res;
u32 __unused;
- struct arch_vdso_data arch_data;
+ struct arch_vdso_time_data arch_data;
};
/**
diff --git a/include/vdso/page.h b/include/vdso/page.h
new file mode 100644
index 000000000000..710ae2414e68
--- /dev/null
+++ b/include/vdso/page.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __VDSO_PAGE_H
+#define __VDSO_PAGE_H
+
+#include <uapi/linux/const.h>
+
+/*
+ * PAGE_SHIFT determines the page size.
+ *
+ * Note: This definition is required because PAGE_SHIFT is used
+ * in several places throuout the codebase.
+ */
+#define PAGE_SHIFT CONFIG_PAGE_SHIFT
+
+#define PAGE_SIZE (_AC(1,UL) << CONFIG_PAGE_SHIFT)
+
+#if !defined(CONFIG_64BIT)
+/*
+ * Applies only to 32-bit architectures.
+ *
+ * Subtle: (1 << CONFIG_PAGE_SHIFT) is an int, not an unsigned long.
+ * So if we assign PAGE_MASK to a larger type it gets extended the
+ * way we want (i.e. with 1s in the high bits) while masking a
+ * 64-bit value such as phys_addr_t.
+ */
+#define PAGE_MASK (~((1 << CONFIG_PAGE_SHIFT) - 1))
+#else
+#define PAGE_MASK (~(PAGE_SIZE - 1))
+#endif
+
+#endif /* __VDSO_PAGE_H */
diff --git a/kernel/time/vsyscall.c b/kernel/time/vsyscall.c
index 9193d6133e5d..28706a13c222 100644
--- a/kernel/time/vsyscall.c
+++ b/kernel/time/vsyscall.c
@@ -119,7 +119,7 @@ void update_vsyscall(struct timekeeper *tk)
if (clock_mode != VDSO_CLOCKMODE_NONE)
update_vdso_data(vdata, tk);
- __arch_update_vsyscall(vdata, tk);
+ __arch_update_vsyscall(vdata);
vdso_write_end(vdata);
The pull request you sent on Mon, 18 Nov 2024 20:22:02 +0100 (CET): > git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git timers-vdso-2024-11-18 has been merged into torvalds/linux.git: https://git.kernel.org/torvalds/c/035238752319a58244d86facd442c5f40b0e97e2 Thank you! -- Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/prtracker.html
© 2016 - 2026 Red Hat, Inc.