1 /* 2 * Virtual cpu timer based timer functions. 3 * 4 * Copyright IBM Corp. 2004, 2012 5 * Author(s): Jan Glauber <jan.glauber@de.ibm.com> 6 */ 7 8 #include <linux/kernel_stat.h> 9 #include <linux/notifier.h> 10 #include <linux/kprobes.h> 11 #include <linux/export.h> 12 #include <linux/kernel.h> 13 #include <linux/timex.h> 14 #include <linux/types.h> 15 #include <linux/time.h> 16 #include <linux/cpu.h> 17 #include <linux/smp.h> 18 19 #include <asm/irq_regs.h> 20 #include <asm/cputime.h> 21 #include <asm/vtimer.h> 22 #include <asm/irq.h> 23 #include "entry.h" 24 25 static void virt_timer_expire(void); 26 27 DEFINE_PER_CPU(struct s390_idle_data, s390_idle); 28 29 static LIST_HEAD(virt_timer_list); 30 static DEFINE_SPINLOCK(virt_timer_lock); 31 static atomic64_t virt_timer_current; 32 static atomic64_t virt_timer_elapsed; 33 34 static inline u64 get_vtimer(void) 35 { 36 u64 timer; 37 38 asm volatile("stpt %0" : "=m" (timer)); 39 return timer; 40 } 41 42 static inline void set_vtimer(u64 expires) 43 { 44 u64 timer; 45 46 asm volatile( 47 " stpt %0\n" /* Store current cpu timer value */ 48 " spt %1" /* Set new value imm. afterwards */ 49 : "=m" (timer) : "m" (expires)); 50 S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer; 51 S390_lowcore.last_update_timer = expires; 52 } 53 54 static inline int virt_timer_forward(u64 elapsed) 55 { 56 BUG_ON(!irqs_disabled()); 57 58 if (list_empty(&virt_timer_list)) 59 return 0; 60 elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed); 61 return elapsed >= atomic64_read(&virt_timer_current); 62 } 63 64 /* 65 * Update process times based on virtual cpu times stored by entry.S 66 * to the lowcore fields user_timer, system_timer & steal_clock. 67 */ 68 static int do_account_vtime(struct task_struct *tsk, int hardirq_offset) 69 { 70 struct thread_info *ti = task_thread_info(tsk); 71 u64 timer, clock, user, system, steal; 72 73 timer = S390_lowcore.last_update_timer; 74 clock = S390_lowcore.last_update_clock; 75 asm volatile( 76 " stpt %0\n" /* Store current cpu timer value */ 77 " stck %1" /* Store current tod clock value */ 78 : "=m" (S390_lowcore.last_update_timer), 79 "=m" (S390_lowcore.last_update_clock)); 80 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; 81 S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock; 82 83 user = S390_lowcore.user_timer - ti->user_timer; 84 S390_lowcore.steal_timer -= user; 85 ti->user_timer = S390_lowcore.user_timer; 86 account_user_time(tsk, user, user); 87 88 system = S390_lowcore.system_timer - ti->system_timer; 89 S390_lowcore.steal_timer -= system; 90 ti->system_timer = S390_lowcore.system_timer; 91 account_system_time(tsk, hardirq_offset, system, system); 92 93 steal = S390_lowcore.steal_timer; 94 if ((s64) steal > 0) { 95 S390_lowcore.steal_timer = 0; 96 account_steal_time(steal); 97 } 98 99 return virt_timer_forward(user + system); 100 } 101 102 void vtime_task_switch(struct task_struct *prev) 103 { 104 struct thread_info *ti; 105 106 do_account_vtime(prev, 0); 107 ti = task_thread_info(prev); 108 ti->user_timer = S390_lowcore.user_timer; 109 ti->system_timer = S390_lowcore.system_timer; 110 ti = task_thread_info(current); 111 S390_lowcore.user_timer = ti->user_timer; 112 S390_lowcore.system_timer = ti->system_timer; 113 } 114 115 /* 116 * In s390, accounting pending user time also implies 117 * accounting system time in order to correctly compute 118 * the stolen time accounting. 119 */ 120 void vtime_account_user(struct task_struct *tsk) 121 { 122 if (do_account_vtime(tsk, HARDIRQ_OFFSET)) 123 virt_timer_expire(); 124 } 125 126 /* 127 * Update process times based on virtual cpu times stored by entry.S 128 * to the lowcore fields user_timer, system_timer & steal_clock. 129 */ 130 void vtime_account_irq_enter(struct task_struct *tsk) 131 { 132 struct thread_info *ti = task_thread_info(tsk); 133 u64 timer, system; 134 135 WARN_ON_ONCE(!irqs_disabled()); 136 137 timer = S390_lowcore.last_update_timer; 138 S390_lowcore.last_update_timer = get_vtimer(); 139 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; 140 141 system = S390_lowcore.system_timer - ti->system_timer; 142 S390_lowcore.steal_timer -= system; 143 ti->system_timer = S390_lowcore.system_timer; 144 account_system_time(tsk, 0, system, system); 145 146 virt_timer_forward(system); 147 } 148 EXPORT_SYMBOL_GPL(vtime_account_irq_enter); 149 150 void vtime_account_system(struct task_struct *tsk) 151 __attribute__((alias("vtime_account_irq_enter"))); 152 EXPORT_SYMBOL_GPL(vtime_account_system); 153 154 void __kprobes vtime_stop_cpu(void) 155 { 156 struct s390_idle_data *idle = &__get_cpu_var(s390_idle); 157 unsigned long long idle_time; 158 unsigned long psw_mask; 159 160 trace_hardirqs_on(); 161 162 /* Wait for external, I/O or machine check interrupt. */ 163 psw_mask = psw_kernel_bits | PSW_MASK_WAIT | PSW_MASK_DAT | 164 PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK; 165 idle->nohz_delay = 0; 166 167 /* Call the assembler magic in entry.S */ 168 psw_idle(idle, psw_mask); 169 170 /* Account time spent with enabled wait psw loaded as idle time. */ 171 idle->sequence++; 172 smp_wmb(); 173 idle_time = idle->clock_idle_exit - idle->clock_idle_enter; 174 idle->clock_idle_enter = idle->clock_idle_exit = 0ULL; 175 idle->idle_time += idle_time; 176 idle->idle_count++; 177 account_idle_time(idle_time); 178 smp_wmb(); 179 idle->sequence++; 180 } 181 182 cputime64_t s390_get_idle_time(int cpu) 183 { 184 struct s390_idle_data *idle = &per_cpu(s390_idle, cpu); 185 unsigned long long now, idle_enter, idle_exit; 186 unsigned int sequence; 187 188 do { 189 now = get_tod_clock(); 190 sequence = ACCESS_ONCE(idle->sequence); 191 idle_enter = ACCESS_ONCE(idle->clock_idle_enter); 192 idle_exit = ACCESS_ONCE(idle->clock_idle_exit); 193 } while ((sequence & 1) || (idle->sequence != sequence)); 194 return idle_enter ? ((idle_exit ?: now) - idle_enter) : 0; 195 } 196 197 /* 198 * Sorted add to a list. List is linear searched until first bigger 199 * element is found. 200 */ 201 static void list_add_sorted(struct vtimer_list *timer, struct list_head *head) 202 { 203 struct vtimer_list *tmp; 204 205 list_for_each_entry(tmp, head, entry) { 206 if (tmp->expires > timer->expires) { 207 list_add_tail(&timer->entry, &tmp->entry); 208 return; 209 } 210 } 211 list_add_tail(&timer->entry, head); 212 } 213 214 /* 215 * Handler for expired virtual CPU timer. 216 */ 217 static void virt_timer_expire(void) 218 { 219 struct vtimer_list *timer, *tmp; 220 unsigned long elapsed; 221 LIST_HEAD(cb_list); 222 223 /* walk timer list, fire all expired timers */ 224 spin_lock(&virt_timer_lock); 225 elapsed = atomic64_read(&virt_timer_elapsed); 226 list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) { 227 if (timer->expires < elapsed) 228 /* move expired timer to the callback queue */ 229 list_move_tail(&timer->entry, &cb_list); 230 else 231 timer->expires -= elapsed; 232 } 233 if (!list_empty(&virt_timer_list)) { 234 timer = list_first_entry(&virt_timer_list, 235 struct vtimer_list, entry); 236 atomic64_set(&virt_timer_current, timer->expires); 237 } 238 atomic64_sub(elapsed, &virt_timer_elapsed); 239 spin_unlock(&virt_timer_lock); 240 241 /* Do callbacks and recharge periodic timers */ 242 list_for_each_entry_safe(timer, tmp, &cb_list, entry) { 243 list_del_init(&timer->entry); 244 timer->function(timer->data); 245 if (timer->interval) { 246 /* Recharge interval timer */ 247 timer->expires = timer->interval + 248 atomic64_read(&virt_timer_elapsed); 249 spin_lock(&virt_timer_lock); 250 list_add_sorted(timer, &virt_timer_list); 251 spin_unlock(&virt_timer_lock); 252 } 253 } 254 } 255 256 void init_virt_timer(struct vtimer_list *timer) 257 { 258 timer->function = NULL; 259 INIT_LIST_HEAD(&timer->entry); 260 } 261 EXPORT_SYMBOL(init_virt_timer); 262 263 static inline int vtimer_pending(struct vtimer_list *timer) 264 { 265 return !list_empty(&timer->entry); 266 } 267 268 static void internal_add_vtimer(struct vtimer_list *timer) 269 { 270 if (list_empty(&virt_timer_list)) { 271 /* First timer, just program it. */ 272 atomic64_set(&virt_timer_current, timer->expires); 273 atomic64_set(&virt_timer_elapsed, 0); 274 list_add(&timer->entry, &virt_timer_list); 275 } else { 276 /* Update timer against current base. */ 277 timer->expires += atomic64_read(&virt_timer_elapsed); 278 if (likely((s64) timer->expires < 279 (s64) atomic64_read(&virt_timer_current))) 280 /* The new timer expires before the current timer. */ 281 atomic64_set(&virt_timer_current, timer->expires); 282 /* Insert new timer into the list. */ 283 list_add_sorted(timer, &virt_timer_list); 284 } 285 } 286 287 static void __add_vtimer(struct vtimer_list *timer, int periodic) 288 { 289 unsigned long flags; 290 291 timer->interval = periodic ? timer->expires : 0; 292 spin_lock_irqsave(&virt_timer_lock, flags); 293 internal_add_vtimer(timer); 294 spin_unlock_irqrestore(&virt_timer_lock, flags); 295 } 296 297 /* 298 * add_virt_timer - add an oneshot virtual CPU timer 299 */ 300 void add_virt_timer(struct vtimer_list *timer) 301 { 302 __add_vtimer(timer, 0); 303 } 304 EXPORT_SYMBOL(add_virt_timer); 305 306 /* 307 * add_virt_timer_int - add an interval virtual CPU timer 308 */ 309 void add_virt_timer_periodic(struct vtimer_list *timer) 310 { 311 __add_vtimer(timer, 1); 312 } 313 EXPORT_SYMBOL(add_virt_timer_periodic); 314 315 static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic) 316 { 317 unsigned long flags; 318 int rc; 319 320 BUG_ON(!timer->function); 321 322 if (timer->expires == expires && vtimer_pending(timer)) 323 return 1; 324 spin_lock_irqsave(&virt_timer_lock, flags); 325 rc = vtimer_pending(timer); 326 if (rc) 327 list_del_init(&timer->entry); 328 timer->interval = periodic ? expires : 0; 329 timer->expires = expires; 330 internal_add_vtimer(timer); 331 spin_unlock_irqrestore(&virt_timer_lock, flags); 332 return rc; 333 } 334 335 /* 336 * returns whether it has modified a pending timer (1) or not (0) 337 */ 338 int mod_virt_timer(struct vtimer_list *timer, u64 expires) 339 { 340 return __mod_vtimer(timer, expires, 0); 341 } 342 EXPORT_SYMBOL(mod_virt_timer); 343 344 /* 345 * returns whether it has modified a pending timer (1) or not (0) 346 */ 347 int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires) 348 { 349 return __mod_vtimer(timer, expires, 1); 350 } 351 EXPORT_SYMBOL(mod_virt_timer_periodic); 352 353 /* 354 * Delete a virtual timer. 355 * 356 * returns whether the deleted timer was pending (1) or not (0) 357 */ 358 int del_virt_timer(struct vtimer_list *timer) 359 { 360 unsigned long flags; 361 362 if (!vtimer_pending(timer)) 363 return 0; 364 spin_lock_irqsave(&virt_timer_lock, flags); 365 list_del_init(&timer->entry); 366 spin_unlock_irqrestore(&virt_timer_lock, flags); 367 return 1; 368 } 369 EXPORT_SYMBOL(del_virt_timer); 370 371 /* 372 * Start the virtual CPU timer on the current CPU. 373 */ 374 void __cpuinit init_cpu_vtimer(void) 375 { 376 /* set initial cpu timer */ 377 set_vtimer(VTIMER_MAX_SLICE); 378 } 379 380 static int __cpuinit s390_nohz_notify(struct notifier_block *self, 381 unsigned long action, void *hcpu) 382 { 383 struct s390_idle_data *idle; 384 long cpu = (long) hcpu; 385 386 idle = &per_cpu(s390_idle, cpu); 387 switch (action & ~CPU_TASKS_FROZEN) { 388 case CPU_DYING: 389 idle->nohz_delay = 0; 390 default: 391 break; 392 } 393 return NOTIFY_OK; 394 } 395 396 void __init vtime_init(void) 397 { 398 /* Enable cpu timer interrupts on the boot cpu. */ 399 init_cpu_vtimer(); 400 cpu_notifier(s390_nohz_notify, 0); 401 } 402