1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Virtual Processor Dispatch Trace Log 4 * 5 * (C) Copyright IBM Corporation 2009 6 * 7 * Author: Jeremy Kerr <jk@ozlabs.org> 8 */ 9 10 #include <linux/slab.h> 11 #include <linux/spinlock.h> 12 #include <asm/smp.h> 13 #include <linux/uaccess.h> 14 #include <linux/debugfs.h> 15 #include <asm/firmware.h> 16 #include <asm/dtl.h> 17 #include <asm/lppaca.h> 18 #include <asm/plpar_wrappers.h> 19 #include <asm/machdep.h> 20 21 struct dtl { 22 struct dtl_entry *buf; 23 int cpu; 24 int buf_entries; 25 u64 last_idx; 26 spinlock_t lock; 27 }; 28 static DEFINE_PER_CPU(struct dtl, cpu_dtl); 29 30 static u8 dtl_event_mask = DTL_LOG_ALL; 31 32 33 /* 34 * Size of per-cpu log buffers. Firmware requires that the buffer does 35 * not cross a 4k boundary. 36 */ 37 static int dtl_buf_entries = N_DISPATCH_LOG; 38 39 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE 40 41 /* 42 * When CONFIG_VIRT_CPU_ACCOUNTING_NATIVE = y, the cpu accounting code controls 43 * reading from the dispatch trace log. If other code wants to consume 44 * DTL entries, it can set this pointer to a function that will get 45 * called once for each DTL entry that gets processed. 46 */ 47 static void (*dtl_consumer)(struct dtl_entry *entry, u64 index); 48 49 struct dtl_ring { 50 u64 write_index; 51 struct dtl_entry *write_ptr; 52 struct dtl_entry *buf; 53 struct dtl_entry *buf_end; 54 }; 55 56 static DEFINE_PER_CPU(struct dtl_ring, dtl_rings); 57 58 static atomic_t dtl_count; 59 60 /* 61 * Scan the dispatch trace log and count up the stolen time. 62 * Should be called with interrupts disabled. 63 */ 64 static notrace u64 scan_dispatch_log(u64 stop_tb) 65 { 66 u64 i = local_paca->dtl_ridx; 67 struct dtl_entry *dtl = local_paca->dtl_curr; 68 struct dtl_entry *dtl_end = local_paca->dispatch_log_end; 69 struct lppaca *vpa = local_paca->lppaca_ptr; 70 u64 tb_delta; 71 u64 stolen = 0; 72 u64 dtb; 73 74 if (!dtl) 75 return 0; 76 77 if (i == be64_to_cpu(vpa->dtl_idx)) 78 return 0; 79 while (i < be64_to_cpu(vpa->dtl_idx)) { 80 dtb = be64_to_cpu(dtl->timebase); 81 tb_delta = be32_to_cpu(dtl->enqueue_to_dispatch_time) + 82 be32_to_cpu(dtl->ready_to_enqueue_time); 83 barrier(); 84 if (i + N_DISPATCH_LOG < be64_to_cpu(vpa->dtl_idx)) { 85 /* buffer has overflowed */ 86 i = be64_to_cpu(vpa->dtl_idx) - N_DISPATCH_LOG; 87 dtl = local_paca->dispatch_log + (i % N_DISPATCH_LOG); 88 continue; 89 } 90 if (dtb > stop_tb) 91 break; 92 if (dtl_consumer) 93 dtl_consumer(dtl, i); 94 stolen += tb_delta; 95 ++i; 96 ++dtl; 97 if (dtl == dtl_end) 98 dtl = local_paca->dispatch_log; 99 } 100 local_paca->dtl_ridx = i; 101 local_paca->dtl_curr = dtl; 102 return stolen; 103 } 104 105 /* 106 * Accumulate stolen time by scanning the dispatch trace log. 107 * Called on entry from user mode. 108 */ 109 void notrace pseries_accumulate_stolen_time(void) 110 { 111 u64 sst, ust; 112 struct cpu_accounting_data *acct = &local_paca->accounting; 113 114 sst = scan_dispatch_log(acct->starttime_user); 115 ust = scan_dispatch_log(acct->starttime); 116 acct->stime -= sst; 117 acct->utime -= ust; 118 acct->steal_time += ust + sst; 119 } 120 121 u64 pseries_calculate_stolen_time(u64 stop_tb) 122 { 123 if (!firmware_has_feature(FW_FEATURE_SPLPAR)) 124 return 0; 125 126 if (get_paca()->dtl_ridx != be64_to_cpu(get_lppaca()->dtl_idx)) 127 return scan_dispatch_log(stop_tb); 128 129 return 0; 130 } 131 132 /* 133 * The cpu accounting code controls the DTL ring buffer, and we get 134 * given entries as they are processed. 135 */ 136 static void consume_dtle(struct dtl_entry *dtle, u64 index) 137 { 138 struct dtl_ring *dtlr = this_cpu_ptr(&dtl_rings); 139 struct dtl_entry *wp = dtlr->write_ptr; 140 struct lppaca *vpa = local_paca->lppaca_ptr; 141 142 if (!wp) 143 return; 144 145 *wp = *dtle; 146 barrier(); 147 148 /* check for hypervisor ring buffer overflow, ignore this entry if so */ 149 if (index + N_DISPATCH_LOG < be64_to_cpu(vpa->dtl_idx)) 150 return; 151 152 ++wp; 153 if (wp == dtlr->buf_end) 154 wp = dtlr->buf; 155 dtlr->write_ptr = wp; 156 157 /* incrementing write_index makes the new entry visible */ 158 smp_wmb(); 159 ++dtlr->write_index; 160 } 161 162 static int dtl_start(struct dtl *dtl) 163 { 164 struct dtl_ring *dtlr = &per_cpu(dtl_rings, dtl->cpu); 165 166 dtlr->buf = dtl->buf; 167 dtlr->buf_end = dtl->buf + dtl->buf_entries; 168 dtlr->write_index = 0; 169 170 /* setting write_ptr enables logging into our buffer */ 171 smp_wmb(); 172 dtlr->write_ptr = dtl->buf; 173 174 /* enable event logging */ 175 lppaca_of(dtl->cpu).dtl_enable_mask |= dtl_event_mask; 176 177 dtl_consumer = consume_dtle; 178 atomic_inc(&dtl_count); 179 return 0; 180 } 181 182 static void dtl_stop(struct dtl *dtl) 183 { 184 struct dtl_ring *dtlr = &per_cpu(dtl_rings, dtl->cpu); 185 186 dtlr->write_ptr = NULL; 187 smp_wmb(); 188 189 dtlr->buf = NULL; 190 191 /* restore dtl_enable_mask */ 192 lppaca_of(dtl->cpu).dtl_enable_mask = DTL_LOG_PREEMPT; 193 194 if (atomic_dec_and_test(&dtl_count)) 195 dtl_consumer = NULL; 196 } 197 198 static u64 dtl_current_index(struct dtl *dtl) 199 { 200 return per_cpu(dtl_rings, dtl->cpu).write_index; 201 } 202 203 #else /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ 204 205 static int dtl_start(struct dtl *dtl) 206 { 207 unsigned long addr; 208 int ret, hwcpu; 209 210 /* Register our dtl buffer with the hypervisor. The HV expects the 211 * buffer size to be passed in the second word of the buffer */ 212 ((u32 *)dtl->buf)[1] = cpu_to_be32(DISPATCH_LOG_BYTES); 213 214 hwcpu = get_hard_smp_processor_id(dtl->cpu); 215 addr = __pa(dtl->buf); 216 ret = register_dtl(hwcpu, addr); 217 if (ret) { 218 printk(KERN_WARNING "%s: DTL registration for cpu %d (hw %d) " 219 "failed with %d\n", __func__, dtl->cpu, hwcpu, ret); 220 return -EIO; 221 } 222 223 /* set our initial buffer indices */ 224 lppaca_of(dtl->cpu).dtl_idx = 0; 225 226 /* ensure that our updates to the lppaca fields have occurred before 227 * we actually enable the logging */ 228 smp_wmb(); 229 230 /* enable event logging */ 231 lppaca_of(dtl->cpu).dtl_enable_mask = dtl_event_mask; 232 233 return 0; 234 } 235 236 static void dtl_stop(struct dtl *dtl) 237 { 238 int hwcpu = get_hard_smp_processor_id(dtl->cpu); 239 240 lppaca_of(dtl->cpu).dtl_enable_mask = 0x0; 241 242 unregister_dtl(hwcpu); 243 } 244 245 static u64 dtl_current_index(struct dtl *dtl) 246 { 247 return be64_to_cpu(lppaca_of(dtl->cpu).dtl_idx); 248 } 249 #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ 250 251 static int dtl_enable(struct dtl *dtl) 252 { 253 long int n_entries; 254 long int rc; 255 struct dtl_entry *buf = NULL; 256 257 if (!dtl_cache) 258 return -ENOMEM; 259 260 /* only allow one reader */ 261 if (dtl->buf) 262 return -EBUSY; 263 264 /* ensure there are no other conflicting dtl users */ 265 if (!read_trylock(&dtl_access_lock)) 266 return -EBUSY; 267 268 n_entries = dtl_buf_entries; 269 buf = kmem_cache_alloc_node(dtl_cache, GFP_KERNEL, cpu_to_node(dtl->cpu)); 270 if (!buf) { 271 printk(KERN_WARNING "%s: buffer alloc failed for cpu %d\n", 272 __func__, dtl->cpu); 273 read_unlock(&dtl_access_lock); 274 return -ENOMEM; 275 } 276 277 spin_lock(&dtl->lock); 278 rc = -EBUSY; 279 if (!dtl->buf) { 280 /* store the original allocation size for use during read */ 281 dtl->buf_entries = n_entries; 282 dtl->buf = buf; 283 dtl->last_idx = 0; 284 rc = dtl_start(dtl); 285 if (rc) 286 dtl->buf = NULL; 287 } 288 spin_unlock(&dtl->lock); 289 290 if (rc) { 291 read_unlock(&dtl_access_lock); 292 kmem_cache_free(dtl_cache, buf); 293 } 294 295 return rc; 296 } 297 298 static void dtl_disable(struct dtl *dtl) 299 { 300 spin_lock(&dtl->lock); 301 dtl_stop(dtl); 302 kmem_cache_free(dtl_cache, dtl->buf); 303 dtl->buf = NULL; 304 dtl->buf_entries = 0; 305 spin_unlock(&dtl->lock); 306 read_unlock(&dtl_access_lock); 307 } 308 309 /* file interface */ 310 311 static int dtl_file_open(struct inode *inode, struct file *filp) 312 { 313 struct dtl *dtl = inode->i_private; 314 int rc; 315 316 rc = dtl_enable(dtl); 317 if (rc) 318 return rc; 319 320 filp->private_data = dtl; 321 return 0; 322 } 323 324 static int dtl_file_release(struct inode *inode, struct file *filp) 325 { 326 struct dtl *dtl = inode->i_private; 327 dtl_disable(dtl); 328 return 0; 329 } 330 331 static ssize_t dtl_file_read(struct file *filp, char __user *buf, size_t len, 332 loff_t *pos) 333 { 334 long int rc, n_read, n_req, read_size; 335 struct dtl *dtl; 336 u64 cur_idx, last_idx, i; 337 338 if ((len % sizeof(struct dtl_entry)) != 0) 339 return -EINVAL; 340 341 dtl = filp->private_data; 342 343 /* requested number of entries to read */ 344 n_req = len / sizeof(struct dtl_entry); 345 346 /* actual number of entries read */ 347 n_read = 0; 348 349 spin_lock(&dtl->lock); 350 351 cur_idx = dtl_current_index(dtl); 352 last_idx = dtl->last_idx; 353 354 if (last_idx + dtl->buf_entries <= cur_idx) 355 last_idx = cur_idx - dtl->buf_entries + 1; 356 357 if (last_idx + n_req > cur_idx) 358 n_req = cur_idx - last_idx; 359 360 if (n_req > 0) 361 dtl->last_idx = last_idx + n_req; 362 363 spin_unlock(&dtl->lock); 364 365 if (n_req <= 0) 366 return 0; 367 368 i = last_idx % dtl->buf_entries; 369 370 /* read the tail of the buffer if we've wrapped */ 371 if (i + n_req > dtl->buf_entries) { 372 read_size = dtl->buf_entries - i; 373 374 rc = copy_to_user(buf, &dtl->buf[i], 375 read_size * sizeof(struct dtl_entry)); 376 if (rc) 377 return -EFAULT; 378 379 i = 0; 380 n_req -= read_size; 381 n_read += read_size; 382 buf += read_size * sizeof(struct dtl_entry); 383 } 384 385 /* .. and now the head */ 386 rc = copy_to_user(buf, &dtl->buf[i], n_req * sizeof(struct dtl_entry)); 387 if (rc) 388 return -EFAULT; 389 390 n_read += n_req; 391 392 return n_read * sizeof(struct dtl_entry); 393 } 394 395 static const struct file_operations dtl_fops = { 396 .open = dtl_file_open, 397 .release = dtl_file_release, 398 .read = dtl_file_read, 399 .llseek = no_llseek, 400 }; 401 402 static struct dentry *dtl_dir; 403 404 static void dtl_setup_file(struct dtl *dtl) 405 { 406 char name[10]; 407 408 sprintf(name, "cpu-%d", dtl->cpu); 409 410 debugfs_create_file(name, 0400, dtl_dir, dtl, &dtl_fops); 411 } 412 413 static int dtl_init(void) 414 { 415 int i; 416 417 if (!firmware_has_feature(FW_FEATURE_SPLPAR)) 418 return -ENODEV; 419 420 /* set up common debugfs structure */ 421 422 dtl_dir = debugfs_create_dir("dtl", arch_debugfs_dir); 423 424 debugfs_create_x8("dtl_event_mask", 0600, dtl_dir, &dtl_event_mask); 425 debugfs_create_u32("dtl_buf_entries", 0400, dtl_dir, &dtl_buf_entries); 426 427 /* set up the per-cpu log structures */ 428 for_each_possible_cpu(i) { 429 struct dtl *dtl = &per_cpu(cpu_dtl, i); 430 spin_lock_init(&dtl->lock); 431 dtl->cpu = i; 432 433 dtl_setup_file(dtl); 434 } 435 436 return 0; 437 } 438 machine_arch_initcall(pseries, dtl_init); 439