1 /* 2 * Virtual cpu timer based timer functions. 3 * 4 * Copyright IBM Corp. 2004, 2012 5 * Author(s): Jan Glauber <jan.glauber@de.ibm.com> 6 */ 7 8 #include <linux/kernel_stat.h> 9 #include <linux/notifier.h> 10 #include <linux/kprobes.h> 11 #include <linux/export.h> 12 #include <linux/kernel.h> 13 #include <linux/timex.h> 14 #include <linux/types.h> 15 #include <linux/time.h> 16 #include <linux/cpu.h> 17 #include <linux/smp.h> 18 19 #include <asm/irq_regs.h> 20 #include <asm/cputime.h> 21 #include <asm/vtimer.h> 22 #include <asm/vtime.h> 23 #include <asm/irq.h> 24 #include "entry.h" 25 26 static void virt_timer_expire(void); 27 28 DEFINE_PER_CPU(struct s390_idle_data, s390_idle); 29 30 static LIST_HEAD(virt_timer_list); 31 static DEFINE_SPINLOCK(virt_timer_lock); 32 static atomic64_t virt_timer_current; 33 static atomic64_t virt_timer_elapsed; 34 35 static inline u64 get_vtimer(void) 36 { 37 u64 timer; 38 39 asm volatile("stpt %0" : "=m" (timer)); 40 return timer; 41 } 42 43 static inline void set_vtimer(u64 expires) 44 { 45 u64 timer; 46 47 asm volatile( 48 " stpt %0\n" /* Store current cpu timer value */ 49 " spt %1" /* Set new value imm. afterwards */ 50 : "=m" (timer) : "m" (expires)); 51 S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer; 52 S390_lowcore.last_update_timer = expires; 53 } 54 55 static inline int virt_timer_forward(u64 elapsed) 56 { 57 BUG_ON(!irqs_disabled()); 58 59 if (list_empty(&virt_timer_list)) 60 return 0; 61 elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed); 62 return elapsed >= atomic64_read(&virt_timer_current); 63 } 64 65 /* 66 * Update process times based on virtual cpu times stored by entry.S 67 * to the lowcore fields user_timer, system_timer & steal_clock. 68 */ 69 static int do_account_vtime(struct task_struct *tsk, int hardirq_offset) 70 { 71 struct thread_info *ti = task_thread_info(tsk); 72 u64 timer, clock, user, system, steal; 73 74 timer = S390_lowcore.last_update_timer; 75 clock = S390_lowcore.last_update_clock; 76 asm volatile( 77 " stpt %0\n" /* Store current cpu timer value */ 78 " stck %1" /* Store current tod clock value */ 79 : "=m" (S390_lowcore.last_update_timer), 80 "=m" (S390_lowcore.last_update_clock)); 81 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; 82 S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock; 83 84 user = S390_lowcore.user_timer - ti->user_timer; 85 S390_lowcore.steal_timer -= user; 86 ti->user_timer = S390_lowcore.user_timer; 87 account_user_time(tsk, user, user); 88 89 system = S390_lowcore.system_timer - ti->system_timer; 90 S390_lowcore.steal_timer -= system; 91 ti->system_timer = S390_lowcore.system_timer; 92 account_system_time(tsk, hardirq_offset, system, system); 93 94 steal = S390_lowcore.steal_timer; 95 if ((s64) steal > 0) { 96 S390_lowcore.steal_timer = 0; 97 account_steal_time(steal); 98 } 99 100 return virt_timer_forward(user + system); 101 } 102 103 void vtime_task_switch(struct task_struct *prev) 104 { 105 struct thread_info *ti; 106 107 do_account_vtime(prev, 0); 108 ti = task_thread_info(prev); 109 ti->user_timer = S390_lowcore.user_timer; 110 ti->system_timer = S390_lowcore.system_timer; 111 ti = task_thread_info(current); 112 S390_lowcore.user_timer = ti->user_timer; 113 S390_lowcore.system_timer = ti->system_timer; 114 } 115 116 /* 117 * In s390, accounting pending user time also implies 118 * accounting system time in order to correctly compute 119 * the stolen time accounting. 120 */ 121 void vtime_account_user(struct task_struct *tsk) 122 { 123 if (do_account_vtime(tsk, HARDIRQ_OFFSET)) 124 virt_timer_expire(); 125 } 126 127 /* 128 * Update process times based on virtual cpu times stored by entry.S 129 * to the lowcore fields user_timer, system_timer & steal_clock. 130 */ 131 void vtime_account_irq_enter(struct task_struct *tsk) 132 { 133 struct thread_info *ti = task_thread_info(tsk); 134 u64 timer, system; 135 136 WARN_ON_ONCE(!irqs_disabled()); 137 138 timer = S390_lowcore.last_update_timer; 139 S390_lowcore.last_update_timer = get_vtimer(); 140 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; 141 142 system = S390_lowcore.system_timer - ti->system_timer; 143 S390_lowcore.steal_timer -= system; 144 ti->system_timer = S390_lowcore.system_timer; 145 account_system_time(tsk, 0, system, system); 146 147 virt_timer_forward(system); 148 } 149 EXPORT_SYMBOL_GPL(vtime_account_irq_enter); 150 151 void vtime_account_system(struct task_struct *tsk) 152 __attribute__((alias("vtime_account_irq_enter"))); 153 EXPORT_SYMBOL_GPL(vtime_account_system); 154 155 void __kprobes vtime_stop_cpu(void) 156 { 157 struct s390_idle_data *idle = &__get_cpu_var(s390_idle); 158 unsigned long long idle_time; 159 unsigned long psw_mask; 160 161 trace_hardirqs_on(); 162 163 /* Wait for external, I/O or machine check interrupt. */ 164 psw_mask = PSW_KERNEL_BITS | PSW_MASK_WAIT | PSW_MASK_DAT | 165 PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK; 166 idle->nohz_delay = 0; 167 168 /* Call the assembler magic in entry.S */ 169 psw_idle(idle, psw_mask); 170 171 /* Account time spent with enabled wait psw loaded as idle time. */ 172 idle->sequence++; 173 smp_wmb(); 174 idle_time = idle->clock_idle_exit - idle->clock_idle_enter; 175 idle->clock_idle_enter = idle->clock_idle_exit = 0ULL; 176 idle->idle_time += idle_time; 177 idle->idle_count++; 178 account_idle_time(idle_time); 179 smp_wmb(); 180 idle->sequence++; 181 } 182 183 cputime64_t s390_get_idle_time(int cpu) 184 { 185 struct s390_idle_data *idle = &per_cpu(s390_idle, cpu); 186 unsigned long long now, idle_enter, idle_exit; 187 unsigned int sequence; 188 189 do { 190 now = get_tod_clock(); 191 sequence = ACCESS_ONCE(idle->sequence); 192 idle_enter = ACCESS_ONCE(idle->clock_idle_enter); 193 idle_exit = ACCESS_ONCE(idle->clock_idle_exit); 194 } while ((sequence & 1) || (ACCESS_ONCE(idle->sequence) != sequence)); 195 return idle_enter ? ((idle_exit ?: now) - idle_enter) : 0; 196 } 197 198 /* 199 * Sorted add to a list. List is linear searched until first bigger 200 * element is found. 201 */ 202 static void list_add_sorted(struct vtimer_list *timer, struct list_head *head) 203 { 204 struct vtimer_list *tmp; 205 206 list_for_each_entry(tmp, head, entry) { 207 if (tmp->expires > timer->expires) { 208 list_add_tail(&timer->entry, &tmp->entry); 209 return; 210 } 211 } 212 list_add_tail(&timer->entry, head); 213 } 214 215 /* 216 * Handler for expired virtual CPU timer. 217 */ 218 static void virt_timer_expire(void) 219 { 220 struct vtimer_list *timer, *tmp; 221 unsigned long elapsed; 222 LIST_HEAD(cb_list); 223 224 /* walk timer list, fire all expired timers */ 225 spin_lock(&virt_timer_lock); 226 elapsed = atomic64_read(&virt_timer_elapsed); 227 list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) { 228 if (timer->expires < elapsed) 229 /* move expired timer to the callback queue */ 230 list_move_tail(&timer->entry, &cb_list); 231 else 232 timer->expires -= elapsed; 233 } 234 if (!list_empty(&virt_timer_list)) { 235 timer = list_first_entry(&virt_timer_list, 236 struct vtimer_list, entry); 237 atomic64_set(&virt_timer_current, timer->expires); 238 } 239 atomic64_sub(elapsed, &virt_timer_elapsed); 240 spin_unlock(&virt_timer_lock); 241 242 /* Do callbacks and recharge periodic timers */ 243 list_for_each_entry_safe(timer, tmp, &cb_list, entry) { 244 list_del_init(&timer->entry); 245 timer->function(timer->data); 246 if (timer->interval) { 247 /* Recharge interval timer */ 248 timer->expires = timer->interval + 249 atomic64_read(&virt_timer_elapsed); 250 spin_lock(&virt_timer_lock); 251 list_add_sorted(timer, &virt_timer_list); 252 spin_unlock(&virt_timer_lock); 253 } 254 } 255 } 256 257 void init_virt_timer(struct vtimer_list *timer) 258 { 259 timer->function = NULL; 260 INIT_LIST_HEAD(&timer->entry); 261 } 262 EXPORT_SYMBOL(init_virt_timer); 263 264 static inline int vtimer_pending(struct vtimer_list *timer) 265 { 266 return !list_empty(&timer->entry); 267 } 268 269 static void internal_add_vtimer(struct vtimer_list *timer) 270 { 271 if (list_empty(&virt_timer_list)) { 272 /* First timer, just program it. */ 273 atomic64_set(&virt_timer_current, timer->expires); 274 atomic64_set(&virt_timer_elapsed, 0); 275 list_add(&timer->entry, &virt_timer_list); 276 } else { 277 /* Update timer against current base. */ 278 timer->expires += atomic64_read(&virt_timer_elapsed); 279 if (likely((s64) timer->expires < 280 (s64) atomic64_read(&virt_timer_current))) 281 /* The new timer expires before the current timer. */ 282 atomic64_set(&virt_timer_current, timer->expires); 283 /* Insert new timer into the list. */ 284 list_add_sorted(timer, &virt_timer_list); 285 } 286 } 287 288 static void __add_vtimer(struct vtimer_list *timer, int periodic) 289 { 290 unsigned long flags; 291 292 timer->interval = periodic ? timer->expires : 0; 293 spin_lock_irqsave(&virt_timer_lock, flags); 294 internal_add_vtimer(timer); 295 spin_unlock_irqrestore(&virt_timer_lock, flags); 296 } 297 298 /* 299 * add_virt_timer - add an oneshot virtual CPU timer 300 */ 301 void add_virt_timer(struct vtimer_list *timer) 302 { 303 __add_vtimer(timer, 0); 304 } 305 EXPORT_SYMBOL(add_virt_timer); 306 307 /* 308 * add_virt_timer_int - add an interval virtual CPU timer 309 */ 310 void add_virt_timer_periodic(struct vtimer_list *timer) 311 { 312 __add_vtimer(timer, 1); 313 } 314 EXPORT_SYMBOL(add_virt_timer_periodic); 315 316 static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic) 317 { 318 unsigned long flags; 319 int rc; 320 321 BUG_ON(!timer->function); 322 323 if (timer->expires == expires && vtimer_pending(timer)) 324 return 1; 325 spin_lock_irqsave(&virt_timer_lock, flags); 326 rc = vtimer_pending(timer); 327 if (rc) 328 list_del_init(&timer->entry); 329 timer->interval = periodic ? expires : 0; 330 timer->expires = expires; 331 internal_add_vtimer(timer); 332 spin_unlock_irqrestore(&virt_timer_lock, flags); 333 return rc; 334 } 335 336 /* 337 * returns whether it has modified a pending timer (1) or not (0) 338 */ 339 int mod_virt_timer(struct vtimer_list *timer, u64 expires) 340 { 341 return __mod_vtimer(timer, expires, 0); 342 } 343 EXPORT_SYMBOL(mod_virt_timer); 344 345 /* 346 * returns whether it has modified a pending timer (1) or not (0) 347 */ 348 int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires) 349 { 350 return __mod_vtimer(timer, expires, 1); 351 } 352 EXPORT_SYMBOL(mod_virt_timer_periodic); 353 354 /* 355 * Delete a virtual timer. 356 * 357 * returns whether the deleted timer was pending (1) or not (0) 358 */ 359 int del_virt_timer(struct vtimer_list *timer) 360 { 361 unsigned long flags; 362 363 if (!vtimer_pending(timer)) 364 return 0; 365 spin_lock_irqsave(&virt_timer_lock, flags); 366 list_del_init(&timer->entry); 367 spin_unlock_irqrestore(&virt_timer_lock, flags); 368 return 1; 369 } 370 EXPORT_SYMBOL(del_virt_timer); 371 372 /* 373 * Start the virtual CPU timer on the current CPU. 374 */ 375 void init_cpu_vtimer(void) 376 { 377 /* set initial cpu timer */ 378 set_vtimer(VTIMER_MAX_SLICE); 379 } 380 381 static int s390_nohz_notify(struct notifier_block *self, unsigned long action, 382 void *hcpu) 383 { 384 struct s390_idle_data *idle; 385 long cpu = (long) hcpu; 386 387 idle = &per_cpu(s390_idle, cpu); 388 switch (action & ~CPU_TASKS_FROZEN) { 389 case CPU_DYING: 390 idle->nohz_delay = 0; 391 default: 392 break; 393 } 394 return NOTIFY_OK; 395 } 396 397 void __init vtime_init(void) 398 { 399 /* Enable cpu timer interrupts on the boot cpu. */ 400 init_cpu_vtimer(); 401 cpu_notifier(s390_nohz_notify, 0); 402 } 403