1 /* 2 * Virtual cpu timer based timer functions. 3 * 4 * Copyright IBM Corp. 2004, 2012 5 * Author(s): Jan Glauber <jan.glauber@de.ibm.com> 6 */ 7 8 #include <linux/kernel_stat.h> 9 #include <linux/export.h> 10 #include <linux/kernel.h> 11 #include <linux/timex.h> 12 #include <linux/types.h> 13 #include <linux/time.h> 14 15 #include <asm/cputime.h> 16 #include <asm/vtimer.h> 17 #include <asm/vtime.h> 18 19 static void virt_timer_expire(void); 20 21 static LIST_HEAD(virt_timer_list); 22 static DEFINE_SPINLOCK(virt_timer_lock); 23 static atomic64_t virt_timer_current; 24 static atomic64_t virt_timer_elapsed; 25 26 static inline u64 get_vtimer(void) 27 { 28 u64 timer; 29 30 asm volatile("stpt %0" : "=m" (timer)); 31 return timer; 32 } 33 34 static inline void set_vtimer(u64 expires) 35 { 36 u64 timer; 37 38 asm volatile( 39 " stpt %0\n" /* Store current cpu timer value */ 40 " spt %1" /* Set new value imm. afterwards */ 41 : "=m" (timer) : "m" (expires)); 42 S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer; 43 S390_lowcore.last_update_timer = expires; 44 } 45 46 static inline int virt_timer_forward(u64 elapsed) 47 { 48 BUG_ON(!irqs_disabled()); 49 50 if (list_empty(&virt_timer_list)) 51 return 0; 52 elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed); 53 return elapsed >= atomic64_read(&virt_timer_current); 54 } 55 56 /* 57 * Update process times based on virtual cpu times stored by entry.S 58 * to the lowcore fields user_timer, system_timer & steal_clock. 59 */ 60 static int do_account_vtime(struct task_struct *tsk, int hardirq_offset) 61 { 62 struct thread_info *ti = task_thread_info(tsk); 63 u64 timer, clock, user, system, steal; 64 65 timer = S390_lowcore.last_update_timer; 66 clock = S390_lowcore.last_update_clock; 67 asm volatile( 68 " stpt %0\n" /* Store current cpu timer value */ 69 #ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES 70 " stckf %1" /* Store current tod clock value */ 71 #else 72 " stck %1" /* Store current tod clock value */ 73 #endif 74 : "=m" (S390_lowcore.last_update_timer), 75 "=m" (S390_lowcore.last_update_clock)); 76 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; 77 S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock; 78 79 user = S390_lowcore.user_timer - ti->user_timer; 80 S390_lowcore.steal_timer -= user; 81 ti->user_timer = S390_lowcore.user_timer; 82 account_user_time(tsk, user, user); 83 84 system = S390_lowcore.system_timer - ti->system_timer; 85 S390_lowcore.steal_timer -= system; 86 ti->system_timer = S390_lowcore.system_timer; 87 account_system_time(tsk, hardirq_offset, system, system); 88 89 steal = S390_lowcore.steal_timer; 90 if ((s64) steal > 0) { 91 S390_lowcore.steal_timer = 0; 92 account_steal_time(steal); 93 } 94 95 return virt_timer_forward(user + system); 96 } 97 98 void vtime_task_switch(struct task_struct *prev) 99 { 100 struct thread_info *ti; 101 102 do_account_vtime(prev, 0); 103 ti = task_thread_info(prev); 104 ti->user_timer = S390_lowcore.user_timer; 105 ti->system_timer = S390_lowcore.system_timer; 106 ti = task_thread_info(current); 107 S390_lowcore.user_timer = ti->user_timer; 108 S390_lowcore.system_timer = ti->system_timer; 109 } 110 111 /* 112 * In s390, accounting pending user time also implies 113 * accounting system time in order to correctly compute 114 * the stolen time accounting. 115 */ 116 void vtime_account_user(struct task_struct *tsk) 117 { 118 if (do_account_vtime(tsk, HARDIRQ_OFFSET)) 119 virt_timer_expire(); 120 } 121 122 /* 123 * Update process times based on virtual cpu times stored by entry.S 124 * to the lowcore fields user_timer, system_timer & steal_clock. 125 */ 126 void vtime_account_irq_enter(struct task_struct *tsk) 127 { 128 struct thread_info *ti = task_thread_info(tsk); 129 u64 timer, system; 130 131 WARN_ON_ONCE(!irqs_disabled()); 132 133 timer = S390_lowcore.last_update_timer; 134 S390_lowcore.last_update_timer = get_vtimer(); 135 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; 136 137 system = S390_lowcore.system_timer - ti->system_timer; 138 S390_lowcore.steal_timer -= system; 139 ti->system_timer = S390_lowcore.system_timer; 140 account_system_time(tsk, 0, system, system); 141 142 virt_timer_forward(system); 143 } 144 EXPORT_SYMBOL_GPL(vtime_account_irq_enter); 145 146 void vtime_account_system(struct task_struct *tsk) 147 __attribute__((alias("vtime_account_irq_enter"))); 148 EXPORT_SYMBOL_GPL(vtime_account_system); 149 150 /* 151 * Sorted add to a list. List is linear searched until first bigger 152 * element is found. 153 */ 154 static void list_add_sorted(struct vtimer_list *timer, struct list_head *head) 155 { 156 struct vtimer_list *tmp; 157 158 list_for_each_entry(tmp, head, entry) { 159 if (tmp->expires > timer->expires) { 160 list_add_tail(&timer->entry, &tmp->entry); 161 return; 162 } 163 } 164 list_add_tail(&timer->entry, head); 165 } 166 167 /* 168 * Handler for expired virtual CPU timer. 169 */ 170 static void virt_timer_expire(void) 171 { 172 struct vtimer_list *timer, *tmp; 173 unsigned long elapsed; 174 LIST_HEAD(cb_list); 175 176 /* walk timer list, fire all expired timers */ 177 spin_lock(&virt_timer_lock); 178 elapsed = atomic64_read(&virt_timer_elapsed); 179 list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) { 180 if (timer->expires < elapsed) 181 /* move expired timer to the callback queue */ 182 list_move_tail(&timer->entry, &cb_list); 183 else 184 timer->expires -= elapsed; 185 } 186 if (!list_empty(&virt_timer_list)) { 187 timer = list_first_entry(&virt_timer_list, 188 struct vtimer_list, entry); 189 atomic64_set(&virt_timer_current, timer->expires); 190 } 191 atomic64_sub(elapsed, &virt_timer_elapsed); 192 spin_unlock(&virt_timer_lock); 193 194 /* Do callbacks and recharge periodic timers */ 195 list_for_each_entry_safe(timer, tmp, &cb_list, entry) { 196 list_del_init(&timer->entry); 197 timer->function(timer->data); 198 if (timer->interval) { 199 /* Recharge interval timer */ 200 timer->expires = timer->interval + 201 atomic64_read(&virt_timer_elapsed); 202 spin_lock(&virt_timer_lock); 203 list_add_sorted(timer, &virt_timer_list); 204 spin_unlock(&virt_timer_lock); 205 } 206 } 207 } 208 209 void init_virt_timer(struct vtimer_list *timer) 210 { 211 timer->function = NULL; 212 INIT_LIST_HEAD(&timer->entry); 213 } 214 EXPORT_SYMBOL(init_virt_timer); 215 216 static inline int vtimer_pending(struct vtimer_list *timer) 217 { 218 return !list_empty(&timer->entry); 219 } 220 221 static void internal_add_vtimer(struct vtimer_list *timer) 222 { 223 if (list_empty(&virt_timer_list)) { 224 /* First timer, just program it. */ 225 atomic64_set(&virt_timer_current, timer->expires); 226 atomic64_set(&virt_timer_elapsed, 0); 227 list_add(&timer->entry, &virt_timer_list); 228 } else { 229 /* Update timer against current base. */ 230 timer->expires += atomic64_read(&virt_timer_elapsed); 231 if (likely((s64) timer->expires < 232 (s64) atomic64_read(&virt_timer_current))) 233 /* The new timer expires before the current timer. */ 234 atomic64_set(&virt_timer_current, timer->expires); 235 /* Insert new timer into the list. */ 236 list_add_sorted(timer, &virt_timer_list); 237 } 238 } 239 240 static void __add_vtimer(struct vtimer_list *timer, int periodic) 241 { 242 unsigned long flags; 243 244 timer->interval = periodic ? timer->expires : 0; 245 spin_lock_irqsave(&virt_timer_lock, flags); 246 internal_add_vtimer(timer); 247 spin_unlock_irqrestore(&virt_timer_lock, flags); 248 } 249 250 /* 251 * add_virt_timer - add an oneshot virtual CPU timer 252 */ 253 void add_virt_timer(struct vtimer_list *timer) 254 { 255 __add_vtimer(timer, 0); 256 } 257 EXPORT_SYMBOL(add_virt_timer); 258 259 /* 260 * add_virt_timer_int - add an interval virtual CPU timer 261 */ 262 void add_virt_timer_periodic(struct vtimer_list *timer) 263 { 264 __add_vtimer(timer, 1); 265 } 266 EXPORT_SYMBOL(add_virt_timer_periodic); 267 268 static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic) 269 { 270 unsigned long flags; 271 int rc; 272 273 BUG_ON(!timer->function); 274 275 if (timer->expires == expires && vtimer_pending(timer)) 276 return 1; 277 spin_lock_irqsave(&virt_timer_lock, flags); 278 rc = vtimer_pending(timer); 279 if (rc) 280 list_del_init(&timer->entry); 281 timer->interval = periodic ? expires : 0; 282 timer->expires = expires; 283 internal_add_vtimer(timer); 284 spin_unlock_irqrestore(&virt_timer_lock, flags); 285 return rc; 286 } 287 288 /* 289 * returns whether it has modified a pending timer (1) or not (0) 290 */ 291 int mod_virt_timer(struct vtimer_list *timer, u64 expires) 292 { 293 return __mod_vtimer(timer, expires, 0); 294 } 295 EXPORT_SYMBOL(mod_virt_timer); 296 297 /* 298 * returns whether it has modified a pending timer (1) or not (0) 299 */ 300 int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires) 301 { 302 return __mod_vtimer(timer, expires, 1); 303 } 304 EXPORT_SYMBOL(mod_virt_timer_periodic); 305 306 /* 307 * Delete a virtual timer. 308 * 309 * returns whether the deleted timer was pending (1) or not (0) 310 */ 311 int del_virt_timer(struct vtimer_list *timer) 312 { 313 unsigned long flags; 314 315 if (!vtimer_pending(timer)) 316 return 0; 317 spin_lock_irqsave(&virt_timer_lock, flags); 318 list_del_init(&timer->entry); 319 spin_unlock_irqrestore(&virt_timer_lock, flags); 320 return 1; 321 } 322 EXPORT_SYMBOL(del_virt_timer); 323 324 /* 325 * Start the virtual CPU timer on the current CPU. 326 */ 327 void vtime_init(void) 328 { 329 /* set initial cpu timer */ 330 set_vtimer(VTIMER_MAX_SLICE); 331 } 332