1fe9959a2SEmilio G. Cota /* 2fe9959a2SEmilio G. Cota * qsp.c - QEMU Synchronization Profiler 3fe9959a2SEmilio G. Cota * 4fe9959a2SEmilio G. Cota * Copyright (C) 2018, Emilio G. Cota <cota@braap.org> 5fe9959a2SEmilio G. Cota * 6fe9959a2SEmilio G. Cota * License: GNU GPL, version 2 or later. 7fe9959a2SEmilio G. Cota * See the COPYING file in the top-level directory. 8fe9959a2SEmilio G. Cota * 9fe9959a2SEmilio G. Cota * QSP profiles the time spent in synchronization primitives, which can 10fe9959a2SEmilio G. Cota * help diagnose performance problems, e.g. scalability issues when 11fe9959a2SEmilio G. Cota * contention is high. 12fe9959a2SEmilio G. Cota * 13fe9959a2SEmilio G. Cota * The primitives currently supported are mutexes, recursive mutexes and 14fe9959a2SEmilio G. Cota * condition variables. Note that not all related functions are intercepted; 15fe9959a2SEmilio G. Cota * instead we profile only those functions that can have a performance impact, 16fe9959a2SEmilio G. Cota * either due to blocking (e.g. cond_wait, mutex_lock) or cache line 17fe9959a2SEmilio G. Cota * contention (e.g. mutex_lock, mutex_trylock). 18fe9959a2SEmilio G. Cota * 19fe9959a2SEmilio G. Cota * QSP's design focuses on speed and scalability. This is achieved 20fe9959a2SEmilio G. Cota * by having threads do their profiling entirely on thread-local data. 21fe9959a2SEmilio G. Cota * The appropriate thread-local data is found via a QHT, i.e. a concurrent hash 22fe9959a2SEmilio G. Cota * table. To aggregate data in order to generate a report, we iterate over 23fe9959a2SEmilio G. Cota * all entries in the hash table. Depending on the number of threads and 24fe9959a2SEmilio G. Cota * synchronization objects this might be expensive, but note that it is 25fe9959a2SEmilio G. Cota * very rarely called -- reports are generated only when requested by users. 26fe9959a2SEmilio G. Cota * 27fe9959a2SEmilio G. Cota * Reports are generated as a table where each row represents a call site. A 28fe9959a2SEmilio G. Cota * call site is the triplet formed by the __file__ and __LINE__ of the caller 29fe9959a2SEmilio G. Cota * as well as the address of the "object" (i.e. mutex, rec. mutex or condvar) 30*d557de4aSEmilio G. Cota * being operated on. Optionally, call sites that operate on different objects 31*d557de4aSEmilio G. Cota * of the same type can be coalesced, which can be particularly useful when 32*d557de4aSEmilio G. Cota * profiling dynamically-allocated objects. 33fe9959a2SEmilio G. Cota * 34fe9959a2SEmilio G. Cota * Alternative designs considered: 35fe9959a2SEmilio G. Cota * 36fe9959a2SEmilio G. Cota * - Use an off-the-shelf profiler such as mutrace. This is not a viable option 37fe9959a2SEmilio G. Cota * for us because QEMU has __malloc_hook set (by one of the libraries it 38fe9959a2SEmilio G. Cota * uses); leaving this hook unset is required to avoid deadlock in mutrace. 39fe9959a2SEmilio G. Cota * 40fe9959a2SEmilio G. Cota * - Use a glib HT for each thread, protecting each HT with its own lock. 41fe9959a2SEmilio G. Cota * This isn't simpler than the current design, and is 10% slower in the 42fe9959a2SEmilio G. Cota * atomic_add-bench microbenchmark (-m option). 43fe9959a2SEmilio G. Cota * 44fe9959a2SEmilio G. Cota * - For reports, just use a binary tree as we aggregate data, instead of having 45fe9959a2SEmilio G. Cota * an intermediate hash table. This would simplify the code only slightly, but 46fe9959a2SEmilio G. Cota * would perform badly if there were many threads and objects to track. 47fe9959a2SEmilio G. Cota * 48996e8d9aSEmilio G. Cota * - Wrap operations on qsp entries with RCU read-side critical sections, so 49996e8d9aSEmilio G. Cota * that qsp_reset() can delete entries. Unfortunately, the overhead of calling 50996e8d9aSEmilio G. Cota * rcu_read_lock/unlock slows down atomic_add-bench -m by 24%. Having 51996e8d9aSEmilio G. Cota * a snapshot that is updated on qsp_reset() avoids this overhead. 52996e8d9aSEmilio G. Cota * 53fe9959a2SEmilio G. Cota * Related Work: 54fe9959a2SEmilio G. Cota * - Lennart Poettering's mutrace: http://0pointer.de/blog/projects/mutrace.html 55fe9959a2SEmilio G. Cota * - Lozi, David, Thomas, Lawall and Muller. "Remote Core Locking: Migrating 56fe9959a2SEmilio G. Cota * Critical-Section Execution to Improve the Performance of Multithreaded 57fe9959a2SEmilio G. Cota * Applications", USENIX ATC'12. 58fe9959a2SEmilio G. Cota */ 59fe9959a2SEmilio G. Cota #include "qemu/osdep.h" 60fe9959a2SEmilio G. Cota #include "qemu/thread.h" 61fe9959a2SEmilio G. Cota #include "qemu/timer.h" 62fe9959a2SEmilio G. Cota #include "qemu/qht.h" 63996e8d9aSEmilio G. Cota #include "qemu/rcu.h" 64fe9959a2SEmilio G. Cota #include "exec/tb-hash-xx.h" 65fe9959a2SEmilio G. Cota 66fe9959a2SEmilio G. Cota enum QSPType { 67fe9959a2SEmilio G. Cota QSP_MUTEX, 68fe9959a2SEmilio G. Cota QSP_REC_MUTEX, 69fe9959a2SEmilio G. Cota QSP_CONDVAR, 70fe9959a2SEmilio G. Cota }; 71fe9959a2SEmilio G. Cota 72fe9959a2SEmilio G. Cota struct QSPCallSite { 73fe9959a2SEmilio G. Cota const void *obj; 74fe9959a2SEmilio G. Cota const char *file; /* i.e. __FILE__; shortened later */ 75fe9959a2SEmilio G. Cota int line; 76fe9959a2SEmilio G. Cota enum QSPType type; 77fe9959a2SEmilio G. Cota }; 78fe9959a2SEmilio G. Cota typedef struct QSPCallSite QSPCallSite; 79fe9959a2SEmilio G. Cota 80fe9959a2SEmilio G. Cota struct QSPEntry { 81fe9959a2SEmilio G. Cota void *thread_ptr; 82fe9959a2SEmilio G. Cota const QSPCallSite *callsite; 83fe9959a2SEmilio G. Cota uint64_t n_acqs; 84fe9959a2SEmilio G. Cota uint64_t ns; 85*d557de4aSEmilio G. Cota unsigned int n_objs; /* count of coalesced objs; only used for reporting */ 86fe9959a2SEmilio G. Cota #ifndef CONFIG_ATOMIC64 87fe9959a2SEmilio G. Cota /* 88fe9959a2SEmilio G. Cota * If we cannot update the counts atomically, then use a seqlock. 89fe9959a2SEmilio G. Cota * We don't need an associated lock because the updates are thread-local. 90fe9959a2SEmilio G. Cota */ 91fe9959a2SEmilio G. Cota QemuSeqLock sequence; 92fe9959a2SEmilio G. Cota #endif 93fe9959a2SEmilio G. Cota }; 94fe9959a2SEmilio G. Cota typedef struct QSPEntry QSPEntry; 95fe9959a2SEmilio G. Cota 96996e8d9aSEmilio G. Cota struct QSPSnapshot { 97996e8d9aSEmilio G. Cota struct rcu_head rcu; 98996e8d9aSEmilio G. Cota struct qht ht; 99996e8d9aSEmilio G. Cota }; 100996e8d9aSEmilio G. Cota typedef struct QSPSnapshot QSPSnapshot; 101996e8d9aSEmilio G. Cota 102fe9959a2SEmilio G. Cota /* initial sizing for hash tables */ 103fe9959a2SEmilio G. Cota #define QSP_INITIAL_SIZE 64 104fe9959a2SEmilio G. Cota 105fe9959a2SEmilio G. Cota /* If this file is moved, QSP_REL_PATH should be updated accordingly */ 106fe9959a2SEmilio G. Cota #define QSP_REL_PATH "util/qsp.c" 107fe9959a2SEmilio G. Cota 108fe9959a2SEmilio G. Cota /* this file's full path. Used to present all call sites with relative paths */ 109fe9959a2SEmilio G. Cota static size_t qsp_qemu_path_len; 110fe9959a2SEmilio G. Cota 111fe9959a2SEmilio G. Cota /* the address of qsp_thread gives us a unique 'thread ID' */ 112fe9959a2SEmilio G. Cota static __thread int qsp_thread; 113fe9959a2SEmilio G. Cota 114fe9959a2SEmilio G. Cota /* 115fe9959a2SEmilio G. Cota * Call sites are the same for all threads, so we track them in a separate hash 116fe9959a2SEmilio G. Cota * table to save memory. 117fe9959a2SEmilio G. Cota */ 118fe9959a2SEmilio G. Cota static struct qht qsp_callsite_ht; 119fe9959a2SEmilio G. Cota 120fe9959a2SEmilio G. Cota static struct qht qsp_ht; 121996e8d9aSEmilio G. Cota static QSPSnapshot *qsp_snapshot; 122fe9959a2SEmilio G. Cota static bool qsp_initialized, qsp_initializing; 123fe9959a2SEmilio G. Cota 124fe9959a2SEmilio G. Cota static const char * const qsp_typenames[] = { 125fe9959a2SEmilio G. Cota [QSP_MUTEX] = "mutex", 126fe9959a2SEmilio G. Cota [QSP_REC_MUTEX] = "rec_mutex", 127fe9959a2SEmilio G. Cota [QSP_CONDVAR] = "condvar", 128fe9959a2SEmilio G. Cota }; 129fe9959a2SEmilio G. Cota 130fe9959a2SEmilio G. Cota QemuMutexLockFunc qemu_mutex_lock_func = qemu_mutex_lock_impl; 131fe9959a2SEmilio G. Cota QemuMutexTrylockFunc qemu_mutex_trylock_func = qemu_mutex_trylock_impl; 132fe9959a2SEmilio G. Cota QemuRecMutexLockFunc qemu_rec_mutex_lock_func = qemu_rec_mutex_lock_impl; 133fe9959a2SEmilio G. Cota QemuRecMutexTrylockFunc qemu_rec_mutex_trylock_func = 134fe9959a2SEmilio G. Cota qemu_rec_mutex_trylock_impl; 135fe9959a2SEmilio G. Cota QemuCondWaitFunc qemu_cond_wait_func = qemu_cond_wait_impl; 136fe9959a2SEmilio G. Cota 137fe9959a2SEmilio G. Cota /* 138fe9959a2SEmilio G. Cota * It pays off to _not_ hash callsite->file; hashing a string is slow, and 139fe9959a2SEmilio G. Cota * without it we still get a pretty unique hash. 140fe9959a2SEmilio G. Cota */ 141fe9959a2SEmilio G. Cota static inline 142fe9959a2SEmilio G. Cota uint32_t do_qsp_callsite_hash(const QSPCallSite *callsite, uint64_t a) 143fe9959a2SEmilio G. Cota { 144fe9959a2SEmilio G. Cota uint64_t b = (uint64_t)(uintptr_t)callsite->obj; 145fe9959a2SEmilio G. Cota uint32_t e = callsite->line; 146fe9959a2SEmilio G. Cota uint32_t f = callsite->type; 147fe9959a2SEmilio G. Cota 148fe9959a2SEmilio G. Cota return tb_hash_func7(a, b, e, f, 0); 149fe9959a2SEmilio G. Cota } 150fe9959a2SEmilio G. Cota 151fe9959a2SEmilio G. Cota static inline 152fe9959a2SEmilio G. Cota uint32_t qsp_callsite_hash(const QSPCallSite *callsite) 153fe9959a2SEmilio G. Cota { 154fe9959a2SEmilio G. Cota return do_qsp_callsite_hash(callsite, 0); 155fe9959a2SEmilio G. Cota } 156fe9959a2SEmilio G. Cota 157fe9959a2SEmilio G. Cota static inline uint32_t do_qsp_entry_hash(const QSPEntry *entry, uint64_t a) 158fe9959a2SEmilio G. Cota { 159fe9959a2SEmilio G. Cota return do_qsp_callsite_hash(entry->callsite, a); 160fe9959a2SEmilio G. Cota } 161fe9959a2SEmilio G. Cota 162fe9959a2SEmilio G. Cota static uint32_t qsp_entry_hash(const QSPEntry *entry) 163fe9959a2SEmilio G. Cota { 164fe9959a2SEmilio G. Cota return do_qsp_entry_hash(entry, (uint64_t)(uintptr_t)entry->thread_ptr); 165fe9959a2SEmilio G. Cota } 166fe9959a2SEmilio G. Cota 167fe9959a2SEmilio G. Cota static uint32_t qsp_entry_no_thread_hash(const QSPEntry *entry) 168fe9959a2SEmilio G. Cota { 169fe9959a2SEmilio G. Cota return do_qsp_entry_hash(entry, 0); 170fe9959a2SEmilio G. Cota } 171fe9959a2SEmilio G. Cota 172*d557de4aSEmilio G. Cota /* without the objects we need to hash the file name to get a decent hash */ 173*d557de4aSEmilio G. Cota static uint32_t qsp_entry_no_thread_obj_hash(const QSPEntry *entry) 174*d557de4aSEmilio G. Cota { 175*d557de4aSEmilio G. Cota const QSPCallSite *callsite = entry->callsite; 176*d557de4aSEmilio G. Cota uint64_t a = g_str_hash(callsite->file); 177*d557de4aSEmilio G. Cota uint64_t b = callsite->line; 178*d557de4aSEmilio G. Cota uint32_t e = callsite->type; 179*d557de4aSEmilio G. Cota 180*d557de4aSEmilio G. Cota return tb_hash_func7(a, b, e, 0, 0); 181*d557de4aSEmilio G. Cota } 182*d557de4aSEmilio G. Cota 183fe9959a2SEmilio G. Cota static bool qsp_callsite_cmp(const void *ap, const void *bp) 184fe9959a2SEmilio G. Cota { 185fe9959a2SEmilio G. Cota const QSPCallSite *a = ap; 186fe9959a2SEmilio G. Cota const QSPCallSite *b = bp; 187fe9959a2SEmilio G. Cota 188fe9959a2SEmilio G. Cota return a == b || 189fe9959a2SEmilio G. Cota (a->obj == b->obj && 190fe9959a2SEmilio G. Cota a->line == b->line && 191fe9959a2SEmilio G. Cota a->type == b->type && 192fe9959a2SEmilio G. Cota (a->file == b->file || !strcmp(a->file, b->file))); 193fe9959a2SEmilio G. Cota } 194fe9959a2SEmilio G. Cota 195*d557de4aSEmilio G. Cota static bool qsp_callsite_no_obj_cmp(const void *ap, const void *bp) 196*d557de4aSEmilio G. Cota { 197*d557de4aSEmilio G. Cota const QSPCallSite *a = ap; 198*d557de4aSEmilio G. Cota const QSPCallSite *b = bp; 199*d557de4aSEmilio G. Cota 200*d557de4aSEmilio G. Cota return a == b || 201*d557de4aSEmilio G. Cota (a->line == b->line && 202*d557de4aSEmilio G. Cota a->type == b->type && 203*d557de4aSEmilio G. Cota (a->file == b->file || !strcmp(a->file, b->file))); 204*d557de4aSEmilio G. Cota } 205*d557de4aSEmilio G. Cota 206fe9959a2SEmilio G. Cota static bool qsp_entry_no_thread_cmp(const void *ap, const void *bp) 207fe9959a2SEmilio G. Cota { 208fe9959a2SEmilio G. Cota const QSPEntry *a = ap; 209fe9959a2SEmilio G. Cota const QSPEntry *b = bp; 210fe9959a2SEmilio G. Cota 211fe9959a2SEmilio G. Cota return qsp_callsite_cmp(a->callsite, b->callsite); 212fe9959a2SEmilio G. Cota } 213fe9959a2SEmilio G. Cota 214*d557de4aSEmilio G. Cota static bool qsp_entry_no_thread_obj_cmp(const void *ap, const void *bp) 215*d557de4aSEmilio G. Cota { 216*d557de4aSEmilio G. Cota const QSPEntry *a = ap; 217*d557de4aSEmilio G. Cota const QSPEntry *b = bp; 218*d557de4aSEmilio G. Cota 219*d557de4aSEmilio G. Cota return qsp_callsite_no_obj_cmp(a->callsite, b->callsite); 220*d557de4aSEmilio G. Cota } 221*d557de4aSEmilio G. Cota 222fe9959a2SEmilio G. Cota static bool qsp_entry_cmp(const void *ap, const void *bp) 223fe9959a2SEmilio G. Cota { 224fe9959a2SEmilio G. Cota const QSPEntry *a = ap; 225fe9959a2SEmilio G. Cota const QSPEntry *b = bp; 226fe9959a2SEmilio G. Cota 227fe9959a2SEmilio G. Cota return a->thread_ptr == b->thread_ptr && 228fe9959a2SEmilio G. Cota qsp_callsite_cmp(a->callsite, b->callsite); 229fe9959a2SEmilio G. Cota } 230fe9959a2SEmilio G. Cota 231fe9959a2SEmilio G. Cota /* 232fe9959a2SEmilio G. Cota * Normally we'd call this from a constructor function, but we want it to work 233fe9959a2SEmilio G. Cota * via libutil as well. 234fe9959a2SEmilio G. Cota */ 235fe9959a2SEmilio G. Cota static void qsp_do_init(void) 236fe9959a2SEmilio G. Cota { 237fe9959a2SEmilio G. Cota /* make sure this file's path in the tree is up to date with QSP_REL_PATH */ 238fe9959a2SEmilio G. Cota g_assert(strstr(__FILE__, QSP_REL_PATH)); 239fe9959a2SEmilio G. Cota qsp_qemu_path_len = strlen(__FILE__) - strlen(QSP_REL_PATH); 240fe9959a2SEmilio G. Cota 241fe9959a2SEmilio G. Cota qht_init(&qsp_ht, qsp_entry_cmp, QSP_INITIAL_SIZE, 242fe9959a2SEmilio G. Cota QHT_MODE_AUTO_RESIZE | QHT_MODE_RAW_MUTEXES); 243fe9959a2SEmilio G. Cota qht_init(&qsp_callsite_ht, qsp_callsite_cmp, QSP_INITIAL_SIZE, 244fe9959a2SEmilio G. Cota QHT_MODE_AUTO_RESIZE | QHT_MODE_RAW_MUTEXES); 245fe9959a2SEmilio G. Cota } 246fe9959a2SEmilio G. Cota 247fe9959a2SEmilio G. Cota static __attribute__((noinline)) void qsp_init__slowpath(void) 248fe9959a2SEmilio G. Cota { 249fe9959a2SEmilio G. Cota if (atomic_cmpxchg(&qsp_initializing, false, true) == false) { 250fe9959a2SEmilio G. Cota qsp_do_init(); 251fe9959a2SEmilio G. Cota atomic_set(&qsp_initialized, true); 252fe9959a2SEmilio G. Cota } else { 253fe9959a2SEmilio G. Cota while (!atomic_read(&qsp_initialized)) { 254fe9959a2SEmilio G. Cota cpu_relax(); 255fe9959a2SEmilio G. Cota } 256fe9959a2SEmilio G. Cota } 257fe9959a2SEmilio G. Cota } 258fe9959a2SEmilio G. Cota 259fe9959a2SEmilio G. Cota /* qsp_init() must be called from _all_ exported functions */ 260fe9959a2SEmilio G. Cota static inline void qsp_init(void) 261fe9959a2SEmilio G. Cota { 262fe9959a2SEmilio G. Cota if (likely(atomic_read(&qsp_initialized))) { 263fe9959a2SEmilio G. Cota return; 264fe9959a2SEmilio G. Cota } 265fe9959a2SEmilio G. Cota qsp_init__slowpath(); 266fe9959a2SEmilio G. Cota } 267fe9959a2SEmilio G. Cota 268fe9959a2SEmilio G. Cota static QSPCallSite *qsp_callsite_find(const QSPCallSite *orig) 269fe9959a2SEmilio G. Cota { 270fe9959a2SEmilio G. Cota QSPCallSite *callsite; 271fe9959a2SEmilio G. Cota uint32_t hash; 272fe9959a2SEmilio G. Cota 273fe9959a2SEmilio G. Cota hash = qsp_callsite_hash(orig); 274fe9959a2SEmilio G. Cota callsite = qht_lookup(&qsp_callsite_ht, orig, hash); 275fe9959a2SEmilio G. Cota if (callsite == NULL) { 276fe9959a2SEmilio G. Cota void *existing = NULL; 277fe9959a2SEmilio G. Cota 278fe9959a2SEmilio G. Cota callsite = g_new(QSPCallSite, 1); 279fe9959a2SEmilio G. Cota memcpy(callsite, orig, sizeof(*callsite)); 280fe9959a2SEmilio G. Cota qht_insert(&qsp_callsite_ht, callsite, hash, &existing); 281fe9959a2SEmilio G. Cota if (unlikely(existing)) { 282fe9959a2SEmilio G. Cota g_free(callsite); 283fe9959a2SEmilio G. Cota callsite = existing; 284fe9959a2SEmilio G. Cota } 285fe9959a2SEmilio G. Cota } 286fe9959a2SEmilio G. Cota return callsite; 287fe9959a2SEmilio G. Cota } 288fe9959a2SEmilio G. Cota 289fe9959a2SEmilio G. Cota static QSPEntry * 290fe9959a2SEmilio G. Cota qsp_entry_create(struct qht *ht, const QSPEntry *entry, uint32_t hash) 291fe9959a2SEmilio G. Cota { 292fe9959a2SEmilio G. Cota QSPEntry *e; 293fe9959a2SEmilio G. Cota void *existing = NULL; 294fe9959a2SEmilio G. Cota 295fe9959a2SEmilio G. Cota e = g_new0(QSPEntry, 1); 296fe9959a2SEmilio G. Cota e->thread_ptr = entry->thread_ptr; 297fe9959a2SEmilio G. Cota e->callsite = qsp_callsite_find(entry->callsite); 298fe9959a2SEmilio G. Cota 299fe9959a2SEmilio G. Cota qht_insert(ht, e, hash, &existing); 300fe9959a2SEmilio G. Cota if (unlikely(existing)) { 301fe9959a2SEmilio G. Cota g_free(e); 302fe9959a2SEmilio G. Cota e = existing; 303fe9959a2SEmilio G. Cota } 304fe9959a2SEmilio G. Cota return e; 305fe9959a2SEmilio G. Cota } 306fe9959a2SEmilio G. Cota 307fe9959a2SEmilio G. Cota static QSPEntry * 308fe9959a2SEmilio G. Cota qsp_entry_find(struct qht *ht, const QSPEntry *entry, uint32_t hash) 309fe9959a2SEmilio G. Cota { 310fe9959a2SEmilio G. Cota QSPEntry *e; 311fe9959a2SEmilio G. Cota 312fe9959a2SEmilio G. Cota e = qht_lookup(ht, entry, hash); 313fe9959a2SEmilio G. Cota if (e == NULL) { 314fe9959a2SEmilio G. Cota e = qsp_entry_create(ht, entry, hash); 315fe9959a2SEmilio G. Cota } 316fe9959a2SEmilio G. Cota return e; 317fe9959a2SEmilio G. Cota } 318fe9959a2SEmilio G. Cota 319fe9959a2SEmilio G. Cota /* 320fe9959a2SEmilio G. Cota * Note: Entries are never removed, so callers do not have to be in an RCU 321fe9959a2SEmilio G. Cota * read-side critical section. 322fe9959a2SEmilio G. Cota */ 323fe9959a2SEmilio G. Cota static QSPEntry *qsp_entry_get(const void *obj, const char *file, int line, 324fe9959a2SEmilio G. Cota enum QSPType type) 325fe9959a2SEmilio G. Cota { 326fe9959a2SEmilio G. Cota QSPCallSite callsite = { 327fe9959a2SEmilio G. Cota .obj = obj, 328fe9959a2SEmilio G. Cota .file = file, 329fe9959a2SEmilio G. Cota .line = line, 330fe9959a2SEmilio G. Cota .type = type, 331fe9959a2SEmilio G. Cota }; 332fe9959a2SEmilio G. Cota QSPEntry orig; 333fe9959a2SEmilio G. Cota uint32_t hash; 334fe9959a2SEmilio G. Cota 335fe9959a2SEmilio G. Cota qsp_init(); 336fe9959a2SEmilio G. Cota 337fe9959a2SEmilio G. Cota orig.thread_ptr = &qsp_thread; 338fe9959a2SEmilio G. Cota orig.callsite = &callsite; 339fe9959a2SEmilio G. Cota 340fe9959a2SEmilio G. Cota hash = qsp_entry_hash(&orig); 341fe9959a2SEmilio G. Cota return qsp_entry_find(&qsp_ht, &orig, hash); 342fe9959a2SEmilio G. Cota } 343fe9959a2SEmilio G. Cota 344fe9959a2SEmilio G. Cota /* 345fe9959a2SEmilio G. Cota * @from is in the global hash table; read it atomically if the host 346fe9959a2SEmilio G. Cota * supports it, otherwise use the seqlock. 347fe9959a2SEmilio G. Cota */ 348fe9959a2SEmilio G. Cota static void qsp_entry_aggregate(QSPEntry *to, const QSPEntry *from) 349fe9959a2SEmilio G. Cota { 350fe9959a2SEmilio G. Cota #ifdef CONFIG_ATOMIC64 351fe9959a2SEmilio G. Cota to->ns += atomic_read__nocheck(&from->ns); 352fe9959a2SEmilio G. Cota to->n_acqs += atomic_read__nocheck(&from->n_acqs); 353fe9959a2SEmilio G. Cota #else 354fe9959a2SEmilio G. Cota unsigned int version; 355fe9959a2SEmilio G. Cota uint64_t ns, n_acqs; 356fe9959a2SEmilio G. Cota 357fe9959a2SEmilio G. Cota do { 358fe9959a2SEmilio G. Cota version = seqlock_read_begin(&from->sequence); 359fe9959a2SEmilio G. Cota ns = atomic_read__nocheck(&from->ns); 360fe9959a2SEmilio G. Cota n_acqs = atomic_read__nocheck(&from->n_acqs); 361fe9959a2SEmilio G. Cota } while (seqlock_read_retry(&from->sequence, version)); 362fe9959a2SEmilio G. Cota 363fe9959a2SEmilio G. Cota to->ns += ns; 364fe9959a2SEmilio G. Cota to->n_acqs += n_acqs; 365fe9959a2SEmilio G. Cota #endif 366fe9959a2SEmilio G. Cota } 367fe9959a2SEmilio G. Cota 368fe9959a2SEmilio G. Cota /* 369fe9959a2SEmilio G. Cota * @e is in the global hash table; it is only written to by the current thread, 370fe9959a2SEmilio G. Cota * so we write to it atomically (as in "write once") to prevent torn reads. 371fe9959a2SEmilio G. Cota * If the host doesn't support u64 atomics, use the seqlock. 372fe9959a2SEmilio G. Cota */ 373fe9959a2SEmilio G. Cota static inline void do_qsp_entry_record(QSPEntry *e, int64_t delta, bool acq) 374fe9959a2SEmilio G. Cota { 375fe9959a2SEmilio G. Cota #ifndef CONFIG_ATOMIC64 376fe9959a2SEmilio G. Cota seqlock_write_begin(&e->sequence); 377fe9959a2SEmilio G. Cota #endif 378fe9959a2SEmilio G. Cota atomic_set__nocheck(&e->ns, e->ns + delta); 379fe9959a2SEmilio G. Cota if (acq) { 380fe9959a2SEmilio G. Cota atomic_set__nocheck(&e->n_acqs, e->n_acqs + 1); 381fe9959a2SEmilio G. Cota } 382fe9959a2SEmilio G. Cota #ifndef CONFIG_ATOMIC64 383fe9959a2SEmilio G. Cota seqlock_write_end(&e->sequence); 384fe9959a2SEmilio G. Cota #endif 385fe9959a2SEmilio G. Cota } 386fe9959a2SEmilio G. Cota 387fe9959a2SEmilio G. Cota static inline void qsp_entry_record(QSPEntry *e, int64_t delta) 388fe9959a2SEmilio G. Cota { 389fe9959a2SEmilio G. Cota do_qsp_entry_record(e, delta, true); 390fe9959a2SEmilio G. Cota } 391fe9959a2SEmilio G. Cota 392fe9959a2SEmilio G. Cota #define QSP_GEN_VOID(type_, qsp_t_, func_, impl_) \ 393fe9959a2SEmilio G. Cota static void func_(type_ *obj, const char *file, int line) \ 394fe9959a2SEmilio G. Cota { \ 395fe9959a2SEmilio G. Cota QSPEntry *e; \ 396fe9959a2SEmilio G. Cota int64_t t0, t1; \ 397fe9959a2SEmilio G. Cota \ 398fe9959a2SEmilio G. Cota t0 = get_clock(); \ 399fe9959a2SEmilio G. Cota impl_(obj, file, line); \ 400fe9959a2SEmilio G. Cota t1 = get_clock(); \ 401fe9959a2SEmilio G. Cota \ 402fe9959a2SEmilio G. Cota e = qsp_entry_get(obj, file, line, qsp_t_); \ 403fe9959a2SEmilio G. Cota qsp_entry_record(e, t1 - t0); \ 404fe9959a2SEmilio G. Cota } 405fe9959a2SEmilio G. Cota 406fe9959a2SEmilio G. Cota #define QSP_GEN_RET1(type_, qsp_t_, func_, impl_) \ 407fe9959a2SEmilio G. Cota static int func_(type_ *obj, const char *file, int line) \ 408fe9959a2SEmilio G. Cota { \ 409fe9959a2SEmilio G. Cota QSPEntry *e; \ 410fe9959a2SEmilio G. Cota int64_t t0, t1; \ 411fe9959a2SEmilio G. Cota int err; \ 412fe9959a2SEmilio G. Cota \ 413fe9959a2SEmilio G. Cota t0 = get_clock(); \ 414fe9959a2SEmilio G. Cota err = impl_(obj, file, line); \ 415fe9959a2SEmilio G. Cota t1 = get_clock(); \ 416fe9959a2SEmilio G. Cota \ 417fe9959a2SEmilio G. Cota e = qsp_entry_get(obj, file, line, qsp_t_); \ 418fe9959a2SEmilio G. Cota do_qsp_entry_record(e, t1 - t0, !err); \ 419fe9959a2SEmilio G. Cota return err; \ 420fe9959a2SEmilio G. Cota } 421fe9959a2SEmilio G. Cota 422fe9959a2SEmilio G. Cota QSP_GEN_VOID(QemuMutex, QSP_MUTEX, qsp_mutex_lock, qemu_mutex_lock_impl) 423fe9959a2SEmilio G. Cota QSP_GEN_RET1(QemuMutex, QSP_MUTEX, qsp_mutex_trylock, qemu_mutex_trylock_impl) 424fe9959a2SEmilio G. Cota 425fe9959a2SEmilio G. Cota QSP_GEN_VOID(QemuRecMutex, QSP_REC_MUTEX, qsp_rec_mutex_lock, 426fe9959a2SEmilio G. Cota qemu_rec_mutex_lock_impl) 427fe9959a2SEmilio G. Cota QSP_GEN_RET1(QemuRecMutex, QSP_REC_MUTEX, qsp_rec_mutex_trylock, 428fe9959a2SEmilio G. Cota qemu_rec_mutex_trylock_impl) 429fe9959a2SEmilio G. Cota 430fe9959a2SEmilio G. Cota #undef QSP_GEN_RET1 431fe9959a2SEmilio G. Cota #undef QSP_GEN_VOID 432fe9959a2SEmilio G. Cota 433fe9959a2SEmilio G. Cota static void 434fe9959a2SEmilio G. Cota qsp_cond_wait(QemuCond *cond, QemuMutex *mutex, const char *file, int line) 435fe9959a2SEmilio G. Cota { 436fe9959a2SEmilio G. Cota QSPEntry *e; 437fe9959a2SEmilio G. Cota int64_t t0, t1; 438fe9959a2SEmilio G. Cota 439fe9959a2SEmilio G. Cota t0 = get_clock(); 440fe9959a2SEmilio G. Cota qemu_cond_wait_impl(cond, mutex, file, line); 441fe9959a2SEmilio G. Cota t1 = get_clock(); 442fe9959a2SEmilio G. Cota 443fe9959a2SEmilio G. Cota e = qsp_entry_get(cond, file, line, QSP_CONDVAR); 444fe9959a2SEmilio G. Cota qsp_entry_record(e, t1 - t0); 445fe9959a2SEmilio G. Cota } 446fe9959a2SEmilio G. Cota 447fe9959a2SEmilio G. Cota bool qsp_is_enabled(void) 448fe9959a2SEmilio G. Cota { 449fe9959a2SEmilio G. Cota return atomic_read(&qemu_mutex_lock_func) == qsp_mutex_lock; 450fe9959a2SEmilio G. Cota } 451fe9959a2SEmilio G. Cota 452fe9959a2SEmilio G. Cota void qsp_enable(void) 453fe9959a2SEmilio G. Cota { 454fe9959a2SEmilio G. Cota atomic_set(&qemu_mutex_lock_func, qsp_mutex_lock); 455fe9959a2SEmilio G. Cota atomic_set(&qemu_mutex_trylock_func, qsp_mutex_trylock); 456fe9959a2SEmilio G. Cota atomic_set(&qemu_rec_mutex_lock_func, qsp_rec_mutex_lock); 457fe9959a2SEmilio G. Cota atomic_set(&qemu_rec_mutex_trylock_func, qsp_rec_mutex_trylock); 458fe9959a2SEmilio G. Cota atomic_set(&qemu_cond_wait_func, qsp_cond_wait); 459fe9959a2SEmilio G. Cota } 460fe9959a2SEmilio G. Cota 461fe9959a2SEmilio G. Cota void qsp_disable(void) 462fe9959a2SEmilio G. Cota { 463fe9959a2SEmilio G. Cota atomic_set(&qemu_mutex_lock_func, qemu_mutex_lock_impl); 464fe9959a2SEmilio G. Cota atomic_set(&qemu_mutex_trylock_func, qemu_mutex_trylock_impl); 465fe9959a2SEmilio G. Cota atomic_set(&qemu_rec_mutex_lock_func, qemu_rec_mutex_lock_impl); 466fe9959a2SEmilio G. Cota atomic_set(&qemu_rec_mutex_trylock_func, qemu_rec_mutex_trylock_impl); 467fe9959a2SEmilio G. Cota atomic_set(&qemu_cond_wait_func, qemu_cond_wait_impl); 468fe9959a2SEmilio G. Cota } 469fe9959a2SEmilio G. Cota 470fe9959a2SEmilio G. Cota static gint qsp_tree_cmp(gconstpointer ap, gconstpointer bp, gpointer up) 471fe9959a2SEmilio G. Cota { 472fe9959a2SEmilio G. Cota const QSPEntry *a = ap; 473fe9959a2SEmilio G. Cota const QSPEntry *b = bp; 4740a22777cSEmilio G. Cota enum QSPSortBy sort_by = *(enum QSPSortBy *)up; 475fe9959a2SEmilio G. Cota const QSPCallSite *ca; 476fe9959a2SEmilio G. Cota const QSPCallSite *cb; 477fe9959a2SEmilio G. Cota 4780a22777cSEmilio G. Cota switch (sort_by) { 4790a22777cSEmilio G. Cota case QSP_SORT_BY_TOTAL_WAIT_TIME: 480fe9959a2SEmilio G. Cota if (a->ns > b->ns) { 481fe9959a2SEmilio G. Cota return -1; 482fe9959a2SEmilio G. Cota } else if (a->ns < b->ns) { 483fe9959a2SEmilio G. Cota return 1; 484fe9959a2SEmilio G. Cota } 4850a22777cSEmilio G. Cota break; 4860a22777cSEmilio G. Cota case QSP_SORT_BY_AVG_WAIT_TIME: 4870a22777cSEmilio G. Cota { 4880a22777cSEmilio G. Cota double avg_a = a->n_acqs ? a->ns / a->n_acqs : 0; 4890a22777cSEmilio G. Cota double avg_b = b->n_acqs ? b->ns / b->n_acqs : 0; 4900a22777cSEmilio G. Cota 4910a22777cSEmilio G. Cota if (avg_a > avg_b) { 4920a22777cSEmilio G. Cota return -1; 4930a22777cSEmilio G. Cota } else if (avg_a < avg_b) { 4940a22777cSEmilio G. Cota return 1; 4950a22777cSEmilio G. Cota } 4960a22777cSEmilio G. Cota break; 4970a22777cSEmilio G. Cota } 4980a22777cSEmilio G. Cota default: 4990a22777cSEmilio G. Cota g_assert_not_reached(); 5000a22777cSEmilio G. Cota } 5010a22777cSEmilio G. Cota 502fe9959a2SEmilio G. Cota ca = a->callsite; 503fe9959a2SEmilio G. Cota cb = b->callsite; 504fe9959a2SEmilio G. Cota /* Break the tie with the object's address */ 505fe9959a2SEmilio G. Cota if (ca->obj < cb->obj) { 506fe9959a2SEmilio G. Cota return -1; 507fe9959a2SEmilio G. Cota } else if (ca->obj > cb->obj) { 508fe9959a2SEmilio G. Cota return 1; 509fe9959a2SEmilio G. Cota } else { 510fe9959a2SEmilio G. Cota int cmp; 511fe9959a2SEmilio G. Cota 512fe9959a2SEmilio G. Cota /* same obj. Break the tie with the callsite's file */ 513fe9959a2SEmilio G. Cota cmp = strcmp(ca->file, cb->file); 514fe9959a2SEmilio G. Cota if (cmp) { 515fe9959a2SEmilio G. Cota return cmp; 516fe9959a2SEmilio G. Cota } 517fe9959a2SEmilio G. Cota /* same callsite file. Break the tie with the callsite's line */ 518fe9959a2SEmilio G. Cota g_assert(ca->line != cb->line); 519fe9959a2SEmilio G. Cota if (ca->line < cb->line) { 520fe9959a2SEmilio G. Cota return -1; 521fe9959a2SEmilio G. Cota } else if (ca->line > cb->line) { 522fe9959a2SEmilio G. Cota return 1; 523fe9959a2SEmilio G. Cota } else { 524fe9959a2SEmilio G. Cota /* break the tie with the callsite's type */ 525fe9959a2SEmilio G. Cota return cb->type - ca->type; 526fe9959a2SEmilio G. Cota } 527fe9959a2SEmilio G. Cota } 528fe9959a2SEmilio G. Cota } 529fe9959a2SEmilio G. Cota 530fe9959a2SEmilio G. Cota static void qsp_sort(struct qht *ht, void *p, uint32_t h, void *userp) 531fe9959a2SEmilio G. Cota { 532fe9959a2SEmilio G. Cota QSPEntry *e = p; 533fe9959a2SEmilio G. Cota GTree *tree = userp; 534fe9959a2SEmilio G. Cota 535fe9959a2SEmilio G. Cota g_tree_insert(tree, e, NULL); 536fe9959a2SEmilio G. Cota } 537fe9959a2SEmilio G. Cota 538fe9959a2SEmilio G. Cota static void qsp_aggregate(struct qht *global_ht, void *p, uint32_t h, void *up) 539fe9959a2SEmilio G. Cota { 540fe9959a2SEmilio G. Cota struct qht *ht = up; 541fe9959a2SEmilio G. Cota const QSPEntry *e = p; 542fe9959a2SEmilio G. Cota QSPEntry *agg; 543fe9959a2SEmilio G. Cota uint32_t hash; 544fe9959a2SEmilio G. Cota 545fe9959a2SEmilio G. Cota hash = qsp_entry_no_thread_hash(e); 546fe9959a2SEmilio G. Cota agg = qsp_entry_find(ht, e, hash); 547fe9959a2SEmilio G. Cota qsp_entry_aggregate(agg, e); 548fe9959a2SEmilio G. Cota } 549fe9959a2SEmilio G. Cota 550996e8d9aSEmilio G. Cota static void qsp_iter_diff(struct qht *orig, void *p, uint32_t hash, void *htp) 551996e8d9aSEmilio G. Cota { 552996e8d9aSEmilio G. Cota struct qht *ht = htp; 553996e8d9aSEmilio G. Cota QSPEntry *old = p; 554996e8d9aSEmilio G. Cota QSPEntry *new; 555996e8d9aSEmilio G. Cota 556996e8d9aSEmilio G. Cota new = qht_lookup(ht, old, hash); 557996e8d9aSEmilio G. Cota /* entries are never deleted, so we must have this one */ 558996e8d9aSEmilio G. Cota g_assert(new != NULL); 559996e8d9aSEmilio G. Cota /* our reading of the stats happened after the snapshot was taken */ 560996e8d9aSEmilio G. Cota g_assert(new->n_acqs >= old->n_acqs); 561996e8d9aSEmilio G. Cota g_assert(new->ns >= old->ns); 562996e8d9aSEmilio G. Cota 563996e8d9aSEmilio G. Cota new->n_acqs -= old->n_acqs; 564996e8d9aSEmilio G. Cota new->ns -= old->ns; 565996e8d9aSEmilio G. Cota 566996e8d9aSEmilio G. Cota /* No point in reporting an empty entry */ 567996e8d9aSEmilio G. Cota if (new->n_acqs == 0 && new->ns == 0) { 568996e8d9aSEmilio G. Cota bool removed = qht_remove(ht, new, hash); 569996e8d9aSEmilio G. Cota 570996e8d9aSEmilio G. Cota g_assert(removed); 571996e8d9aSEmilio G. Cota g_free(new); 572996e8d9aSEmilio G. Cota } 573996e8d9aSEmilio G. Cota } 574996e8d9aSEmilio G. Cota 575996e8d9aSEmilio G. Cota static void qsp_diff(struct qht *orig, struct qht *new) 576996e8d9aSEmilio G. Cota { 577996e8d9aSEmilio G. Cota qht_iter(orig, qsp_iter_diff, new); 578996e8d9aSEmilio G. Cota } 579996e8d9aSEmilio G. Cota 580*d557de4aSEmilio G. Cota static void 581*d557de4aSEmilio G. Cota qsp_iter_callsite_coalesce(struct qht *orig, void *p, uint32_t h, void *htp) 582*d557de4aSEmilio G. Cota { 583*d557de4aSEmilio G. Cota struct qht *ht = htp; 584*d557de4aSEmilio G. Cota QSPEntry *old = p; 585*d557de4aSEmilio G. Cota QSPEntry *e; 586*d557de4aSEmilio G. Cota uint32_t hash; 587*d557de4aSEmilio G. Cota 588*d557de4aSEmilio G. Cota hash = qsp_entry_no_thread_obj_hash(old); 589*d557de4aSEmilio G. Cota e = qht_lookup(ht, old, hash); 590*d557de4aSEmilio G. Cota if (e == NULL) { 591*d557de4aSEmilio G. Cota e = qsp_entry_create(ht, old, hash); 592*d557de4aSEmilio G. Cota e->n_objs = 1; 593*d557de4aSEmilio G. Cota } else if (e->callsite->obj != old->callsite->obj) { 594*d557de4aSEmilio G. Cota e->n_objs++; 595*d557de4aSEmilio G. Cota } 596*d557de4aSEmilio G. Cota e->ns += old->ns; 597*d557de4aSEmilio G. Cota e->n_acqs += old->n_acqs; 598*d557de4aSEmilio G. Cota } 599*d557de4aSEmilio G. Cota 600996e8d9aSEmilio G. Cota static void qsp_ht_delete(struct qht *ht, void *p, uint32_t h, void *htp) 601996e8d9aSEmilio G. Cota { 602996e8d9aSEmilio G. Cota g_free(p); 603996e8d9aSEmilio G. Cota } 604996e8d9aSEmilio G. Cota 605*d557de4aSEmilio G. Cota static void qsp_mktree(GTree *tree, bool callsite_coalesce) 606fe9959a2SEmilio G. Cota { 607996e8d9aSEmilio G. Cota QSPSnapshot *snap; 608*d557de4aSEmilio G. Cota struct qht ht, coalesce_ht; 609*d557de4aSEmilio G. Cota struct qht *htp; 610fe9959a2SEmilio G. Cota 611996e8d9aSEmilio G. Cota /* 612996e8d9aSEmilio G. Cota * First, see if there's a prior snapshot, so that we read the global hash 613996e8d9aSEmilio G. Cota * table _after_ the snapshot has been created, which guarantees that 614996e8d9aSEmilio G. Cota * the entries we'll read will be a superset of the snapshot's entries. 615996e8d9aSEmilio G. Cota * 616996e8d9aSEmilio G. Cota * We must remain in an RCU read-side critical section until we're done 617996e8d9aSEmilio G. Cota * with the snapshot. 618996e8d9aSEmilio G. Cota */ 619996e8d9aSEmilio G. Cota rcu_read_lock(); 620996e8d9aSEmilio G. Cota snap = atomic_rcu_read(&qsp_snapshot); 621996e8d9aSEmilio G. Cota 622fe9959a2SEmilio G. Cota /* Aggregate all results from the global hash table into a local one */ 623fe9959a2SEmilio G. Cota qht_init(&ht, qsp_entry_no_thread_cmp, QSP_INITIAL_SIZE, 624fe9959a2SEmilio G. Cota QHT_MODE_AUTO_RESIZE | QHT_MODE_RAW_MUTEXES); 625fe9959a2SEmilio G. Cota qht_iter(&qsp_ht, qsp_aggregate, &ht); 626fe9959a2SEmilio G. Cota 627996e8d9aSEmilio G. Cota /* compute the difference wrt the snapshot, if any */ 628996e8d9aSEmilio G. Cota if (snap) { 629996e8d9aSEmilio G. Cota qsp_diff(&snap->ht, &ht); 630996e8d9aSEmilio G. Cota } 631996e8d9aSEmilio G. Cota /* done with the snapshot; RCU can reclaim it */ 632996e8d9aSEmilio G. Cota rcu_read_unlock(); 633996e8d9aSEmilio G. Cota 634*d557de4aSEmilio G. Cota htp = &ht; 635*d557de4aSEmilio G. Cota if (callsite_coalesce) { 636*d557de4aSEmilio G. Cota qht_init(&coalesce_ht, qsp_entry_no_thread_obj_cmp, QSP_INITIAL_SIZE, 637*d557de4aSEmilio G. Cota QHT_MODE_AUTO_RESIZE | QHT_MODE_RAW_MUTEXES); 638*d557de4aSEmilio G. Cota qht_iter(&ht, qsp_iter_callsite_coalesce, &coalesce_ht); 639*d557de4aSEmilio G. Cota 640*d557de4aSEmilio G. Cota /* free the previous hash table, and point htp to coalesce_ht */ 641*d557de4aSEmilio G. Cota qht_iter(&ht, qsp_ht_delete, NULL); 642*d557de4aSEmilio G. Cota qht_destroy(&ht); 643*d557de4aSEmilio G. Cota htp = &coalesce_ht; 644*d557de4aSEmilio G. Cota } 645*d557de4aSEmilio G. Cota 646fe9959a2SEmilio G. Cota /* sort the hash table elements by using a tree */ 647*d557de4aSEmilio G. Cota qht_iter(htp, qsp_sort, tree); 648fe9959a2SEmilio G. Cota 649fe9959a2SEmilio G. Cota /* free the hash table, but keep the elements (those are in the tree now) */ 650*d557de4aSEmilio G. Cota qht_destroy(htp); 651fe9959a2SEmilio G. Cota } 652fe9959a2SEmilio G. Cota 653fe9959a2SEmilio G. Cota /* free string with g_free */ 654fe9959a2SEmilio G. Cota static char *qsp_at(const QSPCallSite *callsite) 655fe9959a2SEmilio G. Cota { 656fe9959a2SEmilio G. Cota GString *s = g_string_new(NULL); 657fe9959a2SEmilio G. Cota const char *shortened; 658fe9959a2SEmilio G. Cota 659fe9959a2SEmilio G. Cota /* remove the absolute path to qemu */ 660fe9959a2SEmilio G. Cota if (unlikely(strlen(callsite->file) < qsp_qemu_path_len)) { 661fe9959a2SEmilio G. Cota shortened = callsite->file; 662fe9959a2SEmilio G. Cota } else { 663fe9959a2SEmilio G. Cota shortened = callsite->file + qsp_qemu_path_len; 664fe9959a2SEmilio G. Cota } 665fe9959a2SEmilio G. Cota g_string_append_printf(s, "%s:%u", shortened, callsite->line); 666fe9959a2SEmilio G. Cota return g_string_free(s, FALSE); 667fe9959a2SEmilio G. Cota } 668fe9959a2SEmilio G. Cota 669fe9959a2SEmilio G. Cota struct QSPReportEntry { 670fe9959a2SEmilio G. Cota const void *obj; 671fe9959a2SEmilio G. Cota char *callsite_at; 672fe9959a2SEmilio G. Cota const char *typename; 673fe9959a2SEmilio G. Cota double time_s; 674fe9959a2SEmilio G. Cota double ns_avg; 675fe9959a2SEmilio G. Cota uint64_t n_acqs; 676*d557de4aSEmilio G. Cota unsigned int n_objs; 677fe9959a2SEmilio G. Cota }; 678fe9959a2SEmilio G. Cota typedef struct QSPReportEntry QSPReportEntry; 679fe9959a2SEmilio G. Cota 680fe9959a2SEmilio G. Cota struct QSPReport { 681fe9959a2SEmilio G. Cota QSPReportEntry *entries; 682fe9959a2SEmilio G. Cota size_t n_entries; 683fe9959a2SEmilio G. Cota size_t max_n_entries; 684fe9959a2SEmilio G. Cota }; 685fe9959a2SEmilio G. Cota typedef struct QSPReport QSPReport; 686fe9959a2SEmilio G. Cota 687fe9959a2SEmilio G. Cota static gboolean qsp_tree_report(gpointer key, gpointer value, gpointer udata) 688fe9959a2SEmilio G. Cota { 689fe9959a2SEmilio G. Cota const QSPEntry *e = key; 690fe9959a2SEmilio G. Cota QSPReport *report = udata; 691fe9959a2SEmilio G. Cota QSPReportEntry *entry; 692fe9959a2SEmilio G. Cota 693fe9959a2SEmilio G. Cota if (report->n_entries == report->max_n_entries) { 694fe9959a2SEmilio G. Cota return TRUE; 695fe9959a2SEmilio G. Cota } 696fe9959a2SEmilio G. Cota entry = &report->entries[report->n_entries]; 697fe9959a2SEmilio G. Cota report->n_entries++; 698fe9959a2SEmilio G. Cota 699fe9959a2SEmilio G. Cota entry->obj = e->callsite->obj; 700*d557de4aSEmilio G. Cota entry->n_objs = e->n_objs; 701fe9959a2SEmilio G. Cota entry->callsite_at = qsp_at(e->callsite); 702fe9959a2SEmilio G. Cota entry->typename = qsp_typenames[e->callsite->type]; 703fe9959a2SEmilio G. Cota entry->time_s = e->ns * 1e-9; 704fe9959a2SEmilio G. Cota entry->n_acqs = e->n_acqs; 705fe9959a2SEmilio G. Cota entry->ns_avg = e->n_acqs ? e->ns / e->n_acqs : 0; 706fe9959a2SEmilio G. Cota return FALSE; 707fe9959a2SEmilio G. Cota } 708fe9959a2SEmilio G. Cota 709fe9959a2SEmilio G. Cota static void 710fe9959a2SEmilio G. Cota pr_report(const QSPReport *rep, FILE *f, fprintf_function pr) 711fe9959a2SEmilio G. Cota { 712fe9959a2SEmilio G. Cota char *dashes; 713fe9959a2SEmilio G. Cota size_t max_len = 0; 714fe9959a2SEmilio G. Cota int callsite_len = 0; 715fe9959a2SEmilio G. Cota int callsite_rspace; 716fe9959a2SEmilio G. Cota int n_dashes; 717fe9959a2SEmilio G. Cota size_t i; 718fe9959a2SEmilio G. Cota 719fe9959a2SEmilio G. Cota /* find out the maximum length of all 'callsite' fields */ 720fe9959a2SEmilio G. Cota for (i = 0; i < rep->n_entries; i++) { 721fe9959a2SEmilio G. Cota const QSPReportEntry *e = &rep->entries[i]; 722fe9959a2SEmilio G. Cota size_t len = strlen(e->callsite_at); 723fe9959a2SEmilio G. Cota 724fe9959a2SEmilio G. Cota if (len > max_len) { 725fe9959a2SEmilio G. Cota max_len = len; 726fe9959a2SEmilio G. Cota } 727fe9959a2SEmilio G. Cota } 728fe9959a2SEmilio G. Cota 729fe9959a2SEmilio G. Cota callsite_len = MAX(max_len, strlen("Call site")); 730fe9959a2SEmilio G. Cota /* white space to leave to the right of "Call site" */ 731fe9959a2SEmilio G. Cota callsite_rspace = callsite_len - strlen("Call site"); 732fe9959a2SEmilio G. Cota 733fe9959a2SEmilio G. Cota pr(f, "Type Object Call site%*s Wait Time (s) " 734fe9959a2SEmilio G. Cota " Count Average (us)\n", callsite_rspace, ""); 735fe9959a2SEmilio G. Cota 736fe9959a2SEmilio G. Cota /* build a horizontal rule with dashes */ 737fe9959a2SEmilio G. Cota n_dashes = 79 + callsite_rspace; 738fe9959a2SEmilio G. Cota dashes = g_malloc(n_dashes + 1); 739fe9959a2SEmilio G. Cota memset(dashes, '-', n_dashes); 740fe9959a2SEmilio G. Cota dashes[n_dashes] = '\0'; 741fe9959a2SEmilio G. Cota pr(f, "%s\n", dashes); 742fe9959a2SEmilio G. Cota 743fe9959a2SEmilio G. Cota for (i = 0; i < rep->n_entries; i++) { 744fe9959a2SEmilio G. Cota const QSPReportEntry *e = &rep->entries[i]; 745*d557de4aSEmilio G. Cota GString *s = g_string_new(NULL); 746fe9959a2SEmilio G. Cota 747*d557de4aSEmilio G. Cota g_string_append_printf(s, "%-9s ", e->typename); 748*d557de4aSEmilio G. Cota if (e->n_objs > 1) { 749*d557de4aSEmilio G. Cota g_string_append_printf(s, "[%12u]", e->n_objs); 750*d557de4aSEmilio G. Cota } else { 751*d557de4aSEmilio G. Cota g_string_append_printf(s, "%14p", e->obj); 752*d557de4aSEmilio G. Cota } 753*d557de4aSEmilio G. Cota g_string_append_printf(s, " %s%*s %13.5f %12" PRIu64 " %12.2f\n", 754*d557de4aSEmilio G. Cota e->callsite_at, 755*d557de4aSEmilio G. Cota callsite_len - (int)strlen(e->callsite_at), "", 756*d557de4aSEmilio G. Cota e->time_s, e->n_acqs, e->ns_avg * 1e-3); 757*d557de4aSEmilio G. Cota pr(f, "%s", s->str); 758*d557de4aSEmilio G. Cota g_string_free(s, TRUE); 759fe9959a2SEmilio G. Cota } 760fe9959a2SEmilio G. Cota 761fe9959a2SEmilio G. Cota pr(f, "%s\n", dashes); 762fe9959a2SEmilio G. Cota g_free(dashes); 763fe9959a2SEmilio G. Cota } 764fe9959a2SEmilio G. Cota 765fe9959a2SEmilio G. Cota static void report_destroy(QSPReport *rep) 766fe9959a2SEmilio G. Cota { 767fe9959a2SEmilio G. Cota size_t i; 768fe9959a2SEmilio G. Cota 769fe9959a2SEmilio G. Cota for (i = 0; i < rep->n_entries; i++) { 770fe9959a2SEmilio G. Cota QSPReportEntry *e = &rep->entries[i]; 771fe9959a2SEmilio G. Cota 772fe9959a2SEmilio G. Cota g_free(e->callsite_at); 773fe9959a2SEmilio G. Cota } 774fe9959a2SEmilio G. Cota g_free(rep->entries); 775fe9959a2SEmilio G. Cota } 776fe9959a2SEmilio G. Cota 7770a22777cSEmilio G. Cota void qsp_report(FILE *f, fprintf_function cpu_fprintf, size_t max, 778*d557de4aSEmilio G. Cota enum QSPSortBy sort_by, bool callsite_coalesce) 779fe9959a2SEmilio G. Cota { 7800a22777cSEmilio G. Cota GTree *tree = g_tree_new_full(qsp_tree_cmp, &sort_by, g_free, NULL); 781fe9959a2SEmilio G. Cota QSPReport rep; 782fe9959a2SEmilio G. Cota 783fe9959a2SEmilio G. Cota qsp_init(); 784fe9959a2SEmilio G. Cota 785fe9959a2SEmilio G. Cota rep.entries = g_new0(QSPReportEntry, max); 786fe9959a2SEmilio G. Cota rep.n_entries = 0; 787fe9959a2SEmilio G. Cota rep.max_n_entries = max; 788fe9959a2SEmilio G. Cota 789*d557de4aSEmilio G. Cota qsp_mktree(tree, callsite_coalesce); 790fe9959a2SEmilio G. Cota g_tree_foreach(tree, qsp_tree_report, &rep); 791fe9959a2SEmilio G. Cota g_tree_destroy(tree); 792fe9959a2SEmilio G. Cota 793fe9959a2SEmilio G. Cota pr_report(&rep, f, cpu_fprintf); 794fe9959a2SEmilio G. Cota report_destroy(&rep); 795fe9959a2SEmilio G. Cota } 796996e8d9aSEmilio G. Cota 797996e8d9aSEmilio G. Cota static void qsp_snapshot_destroy(QSPSnapshot *snap) 798996e8d9aSEmilio G. Cota { 799996e8d9aSEmilio G. Cota qht_iter(&snap->ht, qsp_ht_delete, NULL); 800996e8d9aSEmilio G. Cota qht_destroy(&snap->ht); 801996e8d9aSEmilio G. Cota g_free(snap); 802996e8d9aSEmilio G. Cota } 803996e8d9aSEmilio G. Cota 804996e8d9aSEmilio G. Cota void qsp_reset(void) 805996e8d9aSEmilio G. Cota { 806996e8d9aSEmilio G. Cota QSPSnapshot *new = g_new(QSPSnapshot, 1); 807996e8d9aSEmilio G. Cota QSPSnapshot *old; 808996e8d9aSEmilio G. Cota 809996e8d9aSEmilio G. Cota qsp_init(); 810996e8d9aSEmilio G. Cota 811996e8d9aSEmilio G. Cota qht_init(&new->ht, qsp_entry_cmp, QSP_INITIAL_SIZE, 812996e8d9aSEmilio G. Cota QHT_MODE_AUTO_RESIZE | QHT_MODE_RAW_MUTEXES); 813996e8d9aSEmilio G. Cota 814996e8d9aSEmilio G. Cota /* take a snapshot of the current state */ 815996e8d9aSEmilio G. Cota qht_iter(&qsp_ht, qsp_aggregate, &new->ht); 816996e8d9aSEmilio G. Cota 817996e8d9aSEmilio G. Cota /* replace the previous snapshot, if any */ 818996e8d9aSEmilio G. Cota old = atomic_xchg(&qsp_snapshot, new); 819996e8d9aSEmilio G. Cota if (old) { 820996e8d9aSEmilio G. Cota call_rcu(old, qsp_snapshot_destroy, rcu); 821996e8d9aSEmilio G. Cota } 822996e8d9aSEmilio G. Cota } 823