hrtimer.c (8edfb0362e8e52dec2de08fa163af01c9da2c9d0) hrtimer.c (887d9dc989eb0154492e41e7c07492edbb088ba1)
1/*
2 * linux/kernel/hrtimer.c
3 *
4 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
5 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
6 * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
7 *
8 * High-resolution kernel timers

--- 53 unchanged lines hidden (view full) ---

62 * There are more clockids then hrtimer bases. Thus, we index
63 * into the timer bases by the hrtimer_base_type enum. When trying
64 * to reach a base using a clockid, hrtimer_clockid_to_base()
65 * is used to convert from clockid to the proper hrtimer_base_type.
66 */
67DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
68{
69 .lock = __RAW_SPIN_LOCK_UNLOCKED(hrtimer_bases.lock),
1/*
2 * linux/kernel/hrtimer.c
3 *
4 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
5 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
6 * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
7 *
8 * High-resolution kernel timers

--- 53 unchanged lines hidden (view full) ---

62 * There are more clockids then hrtimer bases. Thus, we index
63 * into the timer bases by the hrtimer_base_type enum. When trying
64 * to reach a base using a clockid, hrtimer_clockid_to_base()
65 * is used to convert from clockid to the proper hrtimer_base_type.
66 */
67DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
68{
69 .lock = __RAW_SPIN_LOCK_UNLOCKED(hrtimer_bases.lock),
70 .seq = SEQCNT_ZERO(hrtimer_bases.seq),
70 .clock_base =
71 {
72 {
73 .index = HRTIMER_BASE_MONOTONIC,
74 .clockid = CLOCK_MONOTONIC,
75 .get_time = &ktime_get,
76 },
77 {

--- 28 unchanged lines hidden (view full) ---

106
107/*
108 * Functions and macros which are different for UP/SMP systems are kept in a
109 * single place
110 */
111#ifdef CONFIG_SMP
112
113/*
71 .clock_base =
72 {
73 {
74 .index = HRTIMER_BASE_MONOTONIC,
75 .clockid = CLOCK_MONOTONIC,
76 .get_time = &ktime_get,
77 },
78 {

--- 28 unchanged lines hidden (view full) ---

107
108/*
109 * Functions and macros which are different for UP/SMP systems are kept in a
110 * single place
111 */
112#ifdef CONFIG_SMP
113
114/*
115 * We require the migration_base for lock_hrtimer_base()/switch_hrtimer_base()
116 * such that hrtimer_callback_running() can unconditionally dereference
117 * timer->base->cpu_base
118 */
119static struct hrtimer_cpu_base migration_cpu_base = {
120 .seq = SEQCNT_ZERO(migration_cpu_base),
121 .clock_base = { { .cpu_base = &migration_cpu_base, }, },
122};
123
124#define migration_base migration_cpu_base.clock_base[0]
125
126/*
114 * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock
115 * means that all timers which are tied to this base via timer->base are
116 * locked, and the base itself is locked too.
117 *
118 * So __run_timers/migrate_timers can safely modify all timers which could
119 * be found on the lists/queues.
120 *
121 * When the timer's base is locked, and the timer removed from list, it is
127 * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock
128 * means that all timers which are tied to this base via timer->base are
129 * locked, and the base itself is locked too.
130 *
131 * So __run_timers/migrate_timers can safely modify all timers which could
132 * be found on the lists/queues.
133 *
134 * When the timer's base is locked, and the timer removed from list, it is
122 * possible to set timer->base = NULL and drop the lock: the timer remains
123 * locked.
135 * possible to set timer->base = &migration_base and drop the lock: the timer
136 * remains locked.
124 */
125static
126struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
127 unsigned long *flags)
128{
129 struct hrtimer_clock_base *base;
130
131 for (;;) {
132 base = timer->base;
137 */
138static
139struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
140 unsigned long *flags)
141{
142 struct hrtimer_clock_base *base;
143
144 for (;;) {
145 base = timer->base;
133 if (likely(base != NULL)) {
146 if (likely(base != &migration_base)) {
134 raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
135 if (likely(base == timer->base))
136 return base;
137 /* The timer has migrated to another CPU: */
138 raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags);
139 }
140 cpu_relax();
141 }

--- 47 unchanged lines hidden (view full) ---

189 * the event source in the high resolution case. The softirq
190 * code will take care of this when the timer function has
191 * completed. There is no conflict as we hold the lock until
192 * the timer is enqueued.
193 */
194 if (unlikely(hrtimer_callback_running(timer)))
195 return base;
196
147 raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
148 if (likely(base == timer->base))
149 return base;
150 /* The timer has migrated to another CPU: */
151 raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags);
152 }
153 cpu_relax();
154 }

--- 47 unchanged lines hidden (view full) ---

202 * the event source in the high resolution case. The softirq
203 * code will take care of this when the timer function has
204 * completed. There is no conflict as we hold the lock until
205 * the timer is enqueued.
206 */
207 if (unlikely(hrtimer_callback_running(timer)))
208 return base;
209
197 /* See the comment in lock_timer_base() */
198 timer->base = NULL;
210 /* See the comment in lock_hrtimer_base() */
211 timer->base = &migration_base;
199 raw_spin_unlock(&base->cpu_base->lock);
200 raw_spin_lock(&new_base->cpu_base->lock);
201
202 if (cpu != this_cpu && hrtimer_check_target(timer, new_base)) {
203 cpu = this_cpu;
204 raw_spin_unlock(&new_base->cpu_base->lock);
205 raw_spin_lock(&base->cpu_base->lock);
206 timer->base = base;

--- 626 unchanged lines hidden (view full) ---

833 */
834static int enqueue_hrtimer(struct hrtimer *timer,
835 struct hrtimer_clock_base *base)
836{
837 debug_activate(timer);
838
839 base->cpu_base->active_bases |= 1 << base->index;
840
212 raw_spin_unlock(&base->cpu_base->lock);
213 raw_spin_lock(&new_base->cpu_base->lock);
214
215 if (cpu != this_cpu && hrtimer_check_target(timer, new_base)) {
216 cpu = this_cpu;
217 raw_spin_unlock(&new_base->cpu_base->lock);
218 raw_spin_lock(&base->cpu_base->lock);
219 timer->base = base;

--- 626 unchanged lines hidden (view full) ---

846 */
847static int enqueue_hrtimer(struct hrtimer *timer,
848 struct hrtimer_clock_base *base)
849{
850 debug_activate(timer);
851
852 base->cpu_base->active_bases |= 1 << base->index;
853
841 /*
842 * HRTIMER_STATE_ENQUEUED is or'ed to the current state to preserve the
843 * state of a possibly running callback.
844 */
845 timer->state |= HRTIMER_STATE_ENQUEUED;
854 timer->state = HRTIMER_STATE_ENQUEUED;
846
847 return timerqueue_add(&base->active, &timer->node);
848}
849
850/*
851 * __remove_hrtimer - internal function to remove a timer
852 *
853 * Caller must hold the base lock.

--- 48 unchanged lines hidden (view full) ---

902 * skipped. The interrupt event on this CPU is fired and
903 * reprogramming happens in the interrupt handler. This is a
904 * rare case and less expensive than a smp call.
905 */
906 debug_deactivate(timer);
907 timer_stats_hrtimer_clear_start_info(timer);
908 reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);
909
855
856 return timerqueue_add(&base->active, &timer->node);
857}
858
859/*
860 * __remove_hrtimer - internal function to remove a timer
861 *
862 * Caller must hold the base lock.

--- 48 unchanged lines hidden (view full) ---

911 * skipped. The interrupt event on this CPU is fired and
912 * reprogramming happens in the interrupt handler. This is a
913 * rare case and less expensive than a smp call.
914 */
915 debug_deactivate(timer);
916 timer_stats_hrtimer_clear_start_info(timer);
917 reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);
918
910 if (!restart) {
911 /*
912 * We must preserve the CALLBACK state flag here,
913 * otherwise we could move the timer base in
914 * switch_hrtimer_base.
915 */
916 state &= HRTIMER_STATE_CALLBACK;
917 }
919 if (!restart)
920 state = HRTIMER_STATE_INACTIVE;
921
918 __remove_hrtimer(timer, base, state, reprogram);
919 return 1;
920 }
921 return 0;
922}
923
924/**
925 * hrtimer_start_range_ns - (re)start an hrtimer on the current CPU

--- 184 unchanged lines hidden (view full) ---

1110void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
1111 enum hrtimer_mode mode)
1112{
1113 debug_init(timer, clock_id, mode);
1114 __hrtimer_init(timer, clock_id, mode);
1115}
1116EXPORT_SYMBOL_GPL(hrtimer_init);
1117
922 __remove_hrtimer(timer, base, state, reprogram);
923 return 1;
924 }
925 return 0;
926}
927
928/**
929 * hrtimer_start_range_ns - (re)start an hrtimer on the current CPU

--- 184 unchanged lines hidden (view full) ---

1114void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
1115 enum hrtimer_mode mode)
1116{
1117 debug_init(timer, clock_id, mode);
1118 __hrtimer_init(timer, clock_id, mode);
1119}
1120EXPORT_SYMBOL_GPL(hrtimer_init);
1121
1122/*
1123 * A timer is active, when it is enqueued into the rbtree or the
1124 * callback function is running or it's in the state of being migrated
1125 * to another cpu.
1126 *
1127 * It is important for this function to not return a false negative.
1128 */
1129bool hrtimer_active(const struct hrtimer *timer)
1130{
1131 struct hrtimer_cpu_base *cpu_base;
1132 unsigned int seq;
1133
1134 do {
1135 cpu_base = READ_ONCE(timer->base->cpu_base);
1136 seq = raw_read_seqcount_begin(&cpu_base->seq);
1137
1138 if (timer->state != HRTIMER_STATE_INACTIVE ||
1139 cpu_base->running == timer)
1140 return true;
1141
1142 } while (read_seqcount_retry(&cpu_base->seq, seq) ||
1143 cpu_base != READ_ONCE(timer->base->cpu_base));
1144
1145 return false;
1146}
1147EXPORT_SYMBOL_GPL(hrtimer_active);
1148
1149/*
1150 * The write_seqcount_barrier()s in __run_hrtimer() split the thing into 3
1151 * distinct sections:
1152 *
1153 * - queued: the timer is queued
1154 * - callback: the timer is being ran
1155 * - post: the timer is inactive or (re)queued
1156 *
1157 * On the read side we ensure we observe timer->state and cpu_base->running
1158 * from the same section, if anything changed while we looked at it, we retry.
1159 * This includes timer->base changing because sequence numbers alone are
1160 * insufficient for that.
1161 *
1162 * The sequence numbers are required because otherwise we could still observe
1163 * a false negative if the read side got smeared over multiple consequtive
1164 * __run_hrtimer() invocations.
1165 */
1166
1118static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
1119 struct hrtimer_clock_base *base,
1120 struct hrtimer *timer, ktime_t *now)
1121{
1122 enum hrtimer_restart (*fn)(struct hrtimer *);
1123 int restart;
1124
1167static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
1168 struct hrtimer_clock_base *base,
1169 struct hrtimer *timer, ktime_t *now)
1170{
1171 enum hrtimer_restart (*fn)(struct hrtimer *);
1172 int restart;
1173
1125 WARN_ON(!irqs_disabled());
1174 lockdep_assert_held(&cpu_base->lock);
1126
1127 debug_deactivate(timer);
1175
1176 debug_deactivate(timer);
1128 __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0);
1177 cpu_base->running = timer;
1178
1179 /*
1180 * Separate the ->running assignment from the ->state assignment.
1181 *
1182 * As with a regular write barrier, this ensures the read side in
1183 * hrtimer_active() cannot observe cpu_base->running == NULL &&
1184 * timer->state == INACTIVE.
1185 */
1186 raw_write_seqcount_barrier(&cpu_base->seq);
1187
1188 __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 0);
1129 timer_stats_account_hrtimer(timer);
1130 fn = timer->function;
1131
1132 /*
1133 * Because we run timers from hardirq context, there is no chance
1134 * they get migrated to another cpu, therefore its safe to unlock
1135 * the timer base.
1136 */
1137 raw_spin_unlock(&cpu_base->lock);
1138 trace_hrtimer_expire_entry(timer, now);
1139 restart = fn(timer);
1140 trace_hrtimer_expire_exit(timer);
1141 raw_spin_lock(&cpu_base->lock);
1142
1143 /*
1189 timer_stats_account_hrtimer(timer);
1190 fn = timer->function;
1191
1192 /*
1193 * Because we run timers from hardirq context, there is no chance
1194 * they get migrated to another cpu, therefore its safe to unlock
1195 * the timer base.
1196 */
1197 raw_spin_unlock(&cpu_base->lock);
1198 trace_hrtimer_expire_entry(timer, now);
1199 restart = fn(timer);
1200 trace_hrtimer_expire_exit(timer);
1201 raw_spin_lock(&cpu_base->lock);
1202
1203 /*
1144 * Note: We clear the CALLBACK bit after enqueue_hrtimer and
1204 * Note: We clear the running state after enqueue_hrtimer and
1145 * we do not reprogramm the event hardware. Happens either in
1146 * hrtimer_start_range_ns() or in hrtimer_interrupt()
1147 *
1148 * Note: Because we dropped the cpu_base->lock above,
1149 * hrtimer_start_range_ns() can have popped in and enqueued the timer
1150 * for us already.
1151 */
1152 if (restart != HRTIMER_NORESTART &&
1153 !(timer->state & HRTIMER_STATE_ENQUEUED))
1154 enqueue_hrtimer(timer, base);
1155
1205 * we do not reprogramm the event hardware. Happens either in
1206 * hrtimer_start_range_ns() or in hrtimer_interrupt()
1207 *
1208 * Note: Because we dropped the cpu_base->lock above,
1209 * hrtimer_start_range_ns() can have popped in and enqueued the timer
1210 * for us already.
1211 */
1212 if (restart != HRTIMER_NORESTART &&
1213 !(timer->state & HRTIMER_STATE_ENQUEUED))
1214 enqueue_hrtimer(timer, base);
1215
1156 WARN_ON_ONCE(!(timer->state & HRTIMER_STATE_CALLBACK));
1216 /*
1217 * Separate the ->running assignment from the ->state assignment.
1218 *
1219 * As with a regular write barrier, this ensures the read side in
1220 * hrtimer_active() cannot observe cpu_base->running == NULL &&
1221 * timer->state == INACTIVE.
1222 */
1223 raw_write_seqcount_barrier(&cpu_base->seq);
1157
1224
1158 timer->state &= ~HRTIMER_STATE_CALLBACK;
1225 WARN_ON_ONCE(cpu_base->running != timer);
1226 cpu_base->running = NULL;
1159}
1160
1161static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now)
1162{
1163 struct hrtimer_clock_base *base = cpu_base->clock_base;
1164 unsigned int active = cpu_base->active_bases;
1165
1166 for (; active; base++, active >>= 1) {

--- 547 unchanged lines hidden --- 		1227}
1228
1229static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now)
1230{
1231 struct hrtimer_clock_base *base = cpu_base->clock_base;
1232 unsigned int active = cpu_base->active_bases;
1233
1234 for (; active; base++, active >>= 1) {

--- 547 unchanged lines hidden ---