1*fe9959a2SEmilio G. Cota /* 2*fe9959a2SEmilio G. Cota * qsp.c - QEMU Synchronization Profiler 3*fe9959a2SEmilio G. Cota * 4*fe9959a2SEmilio G. Cota * Copyright (C) 2018, Emilio G. Cota <cota@braap.org> 5*fe9959a2SEmilio G. Cota * 6*fe9959a2SEmilio G. Cota * License: GNU GPL, version 2 or later. 7*fe9959a2SEmilio G. Cota * See the COPYING file in the top-level directory. 8*fe9959a2SEmilio G. Cota * 9*fe9959a2SEmilio G. Cota * QSP profiles the time spent in synchronization primitives, which can 10*fe9959a2SEmilio G. Cota * help diagnose performance problems, e.g. scalability issues when 11*fe9959a2SEmilio G. Cota * contention is high. 12*fe9959a2SEmilio G. Cota * 13*fe9959a2SEmilio G. Cota * The primitives currently supported are mutexes, recursive mutexes and 14*fe9959a2SEmilio G. Cota * condition variables. Note that not all related functions are intercepted; 15*fe9959a2SEmilio G. Cota * instead we profile only those functions that can have a performance impact, 16*fe9959a2SEmilio G. Cota * either due to blocking (e.g. cond_wait, mutex_lock) or cache line 17*fe9959a2SEmilio G. Cota * contention (e.g. mutex_lock, mutex_trylock). 18*fe9959a2SEmilio G. Cota * 19*fe9959a2SEmilio G. Cota * QSP's design focuses on speed and scalability. This is achieved 20*fe9959a2SEmilio G. Cota * by having threads do their profiling entirely on thread-local data. 21*fe9959a2SEmilio G. Cota * The appropriate thread-local data is found via a QHT, i.e. a concurrent hash 22*fe9959a2SEmilio G. Cota * table. To aggregate data in order to generate a report, we iterate over 23*fe9959a2SEmilio G. Cota * all entries in the hash table. Depending on the number of threads and 24*fe9959a2SEmilio G. Cota * synchronization objects this might be expensive, but note that it is 25*fe9959a2SEmilio G. Cota * very rarely called -- reports are generated only when requested by users. 26*fe9959a2SEmilio G. Cota * 27*fe9959a2SEmilio G. Cota * Reports are generated as a table where each row represents a call site. A 28*fe9959a2SEmilio G. Cota * call site is the triplet formed by the __file__ and __LINE__ of the caller 29*fe9959a2SEmilio G. Cota * as well as the address of the "object" (i.e. mutex, rec. mutex or condvar) 30*fe9959a2SEmilio G. Cota * being operated on. Focusing on call sites instead of just on objects might 31*fe9959a2SEmilio G. Cota * seem puzzling. However, it is a sensible choice since otherwise dealing with 32*fe9959a2SEmilio G. Cota * dynamically-allocated objects becomes difficult (e.g. what to do when an 33*fe9959a2SEmilio G. Cota * object is destroyed, or reused?). Furthermore, the call site info is of most 34*fe9959a2SEmilio G. Cota * importance, since it is callers, and not objects, what cause wait time. 35*fe9959a2SEmilio G. Cota * 36*fe9959a2SEmilio G. Cota * Alternative designs considered: 37*fe9959a2SEmilio G. Cota * 38*fe9959a2SEmilio G. Cota * - Use an off-the-shelf profiler such as mutrace. This is not a viable option 39*fe9959a2SEmilio G. Cota * for us because QEMU has __malloc_hook set (by one of the libraries it 40*fe9959a2SEmilio G. Cota * uses); leaving this hook unset is required to avoid deadlock in mutrace. 41*fe9959a2SEmilio G. Cota * 42*fe9959a2SEmilio G. Cota * - Use a glib HT for each thread, protecting each HT with its own lock. 43*fe9959a2SEmilio G. Cota * This isn't simpler than the current design, and is 10% slower in the 44*fe9959a2SEmilio G. Cota * atomic_add-bench microbenchmark (-m option). 45*fe9959a2SEmilio G. Cota * 46*fe9959a2SEmilio G. Cota * - For reports, just use a binary tree as we aggregate data, instead of having 47*fe9959a2SEmilio G. Cota * an intermediate hash table. This would simplify the code only slightly, but 48*fe9959a2SEmilio G. Cota * would perform badly if there were many threads and objects to track. 49*fe9959a2SEmilio G. Cota * 50*fe9959a2SEmilio G. Cota * Related Work: 51*fe9959a2SEmilio G. Cota * - Lennart Poettering's mutrace: http://0pointer.de/blog/projects/mutrace.html 52*fe9959a2SEmilio G. Cota * - Lozi, David, Thomas, Lawall and Muller. "Remote Core Locking: Migrating 53*fe9959a2SEmilio G. Cota * Critical-Section Execution to Improve the Performance of Multithreaded 54*fe9959a2SEmilio G. Cota * Applications", USENIX ATC'12. 55*fe9959a2SEmilio G. Cota */ 56*fe9959a2SEmilio G. Cota #include "qemu/osdep.h" 57*fe9959a2SEmilio G. Cota #include "qemu/thread.h" 58*fe9959a2SEmilio G. Cota #include "qemu/timer.h" 59*fe9959a2SEmilio G. Cota #include "qemu/qht.h" 60*fe9959a2SEmilio G. Cota #include "exec/tb-hash-xx.h" 61*fe9959a2SEmilio G. Cota 62*fe9959a2SEmilio G. Cota enum QSPType { 63*fe9959a2SEmilio G. Cota QSP_MUTEX, 64*fe9959a2SEmilio G. Cota QSP_REC_MUTEX, 65*fe9959a2SEmilio G. Cota QSP_CONDVAR, 66*fe9959a2SEmilio G. Cota }; 67*fe9959a2SEmilio G. Cota 68*fe9959a2SEmilio G. Cota struct QSPCallSite { 69*fe9959a2SEmilio G. Cota const void *obj; 70*fe9959a2SEmilio G. Cota const char *file; /* i.e. __FILE__; shortened later */ 71*fe9959a2SEmilio G. Cota int line; 72*fe9959a2SEmilio G. Cota enum QSPType type; 73*fe9959a2SEmilio G. Cota }; 74*fe9959a2SEmilio G. Cota typedef struct QSPCallSite QSPCallSite; 75*fe9959a2SEmilio G. Cota 76*fe9959a2SEmilio G. Cota struct QSPEntry { 77*fe9959a2SEmilio G. Cota void *thread_ptr; 78*fe9959a2SEmilio G. Cota const QSPCallSite *callsite; 79*fe9959a2SEmilio G. Cota uint64_t n_acqs; 80*fe9959a2SEmilio G. Cota uint64_t ns; 81*fe9959a2SEmilio G. Cota #ifndef CONFIG_ATOMIC64 82*fe9959a2SEmilio G. Cota /* 83*fe9959a2SEmilio G. Cota * If we cannot update the counts atomically, then use a seqlock. 84*fe9959a2SEmilio G. Cota * We don't need an associated lock because the updates are thread-local. 85*fe9959a2SEmilio G. Cota */ 86*fe9959a2SEmilio G. Cota QemuSeqLock sequence; 87*fe9959a2SEmilio G. Cota #endif 88*fe9959a2SEmilio G. Cota }; 89*fe9959a2SEmilio G. Cota typedef struct QSPEntry QSPEntry; 90*fe9959a2SEmilio G. Cota 91*fe9959a2SEmilio G. Cota /* initial sizing for hash tables */ 92*fe9959a2SEmilio G. Cota #define QSP_INITIAL_SIZE 64 93*fe9959a2SEmilio G. Cota 94*fe9959a2SEmilio G. Cota /* If this file is moved, QSP_REL_PATH should be updated accordingly */ 95*fe9959a2SEmilio G. Cota #define QSP_REL_PATH "util/qsp.c" 96*fe9959a2SEmilio G. Cota 97*fe9959a2SEmilio G. Cota /* this file's full path. Used to present all call sites with relative paths */ 98*fe9959a2SEmilio G. Cota static size_t qsp_qemu_path_len; 99*fe9959a2SEmilio G. Cota 100*fe9959a2SEmilio G. Cota /* the address of qsp_thread gives us a unique 'thread ID' */ 101*fe9959a2SEmilio G. Cota static __thread int qsp_thread; 102*fe9959a2SEmilio G. Cota 103*fe9959a2SEmilio G. Cota /* 104*fe9959a2SEmilio G. Cota * Call sites are the same for all threads, so we track them in a separate hash 105*fe9959a2SEmilio G. Cota * table to save memory. 106*fe9959a2SEmilio G. Cota */ 107*fe9959a2SEmilio G. Cota static struct qht qsp_callsite_ht; 108*fe9959a2SEmilio G. Cota 109*fe9959a2SEmilio G. Cota static struct qht qsp_ht; 110*fe9959a2SEmilio G. Cota static bool qsp_initialized, qsp_initializing; 111*fe9959a2SEmilio G. Cota 112*fe9959a2SEmilio G. Cota static const char * const qsp_typenames[] = { 113*fe9959a2SEmilio G. Cota [QSP_MUTEX] = "mutex", 114*fe9959a2SEmilio G. Cota [QSP_REC_MUTEX] = "rec_mutex", 115*fe9959a2SEmilio G. Cota [QSP_CONDVAR] = "condvar", 116*fe9959a2SEmilio G. Cota }; 117*fe9959a2SEmilio G. Cota 118*fe9959a2SEmilio G. Cota QemuMutexLockFunc qemu_mutex_lock_func = qemu_mutex_lock_impl; 119*fe9959a2SEmilio G. Cota QemuMutexTrylockFunc qemu_mutex_trylock_func = qemu_mutex_trylock_impl; 120*fe9959a2SEmilio G. Cota QemuRecMutexLockFunc qemu_rec_mutex_lock_func = qemu_rec_mutex_lock_impl; 121*fe9959a2SEmilio G. Cota QemuRecMutexTrylockFunc qemu_rec_mutex_trylock_func = 122*fe9959a2SEmilio G. Cota qemu_rec_mutex_trylock_impl; 123*fe9959a2SEmilio G. Cota QemuCondWaitFunc qemu_cond_wait_func = qemu_cond_wait_impl; 124*fe9959a2SEmilio G. Cota 125*fe9959a2SEmilio G. Cota /* 126*fe9959a2SEmilio G. Cota * It pays off to _not_ hash callsite->file; hashing a string is slow, and 127*fe9959a2SEmilio G. Cota * without it we still get a pretty unique hash. 128*fe9959a2SEmilio G. Cota */ 129*fe9959a2SEmilio G. Cota static inline 130*fe9959a2SEmilio G. Cota uint32_t do_qsp_callsite_hash(const QSPCallSite *callsite, uint64_t a) 131*fe9959a2SEmilio G. Cota { 132*fe9959a2SEmilio G. Cota uint64_t b = (uint64_t)(uintptr_t)callsite->obj; 133*fe9959a2SEmilio G. Cota uint32_t e = callsite->line; 134*fe9959a2SEmilio G. Cota uint32_t f = callsite->type; 135*fe9959a2SEmilio G. Cota 136*fe9959a2SEmilio G. Cota return tb_hash_func7(a, b, e, f, 0); 137*fe9959a2SEmilio G. Cota } 138*fe9959a2SEmilio G. Cota 139*fe9959a2SEmilio G. Cota static inline 140*fe9959a2SEmilio G. Cota uint32_t qsp_callsite_hash(const QSPCallSite *callsite) 141*fe9959a2SEmilio G. Cota { 142*fe9959a2SEmilio G. Cota return do_qsp_callsite_hash(callsite, 0); 143*fe9959a2SEmilio G. Cota } 144*fe9959a2SEmilio G. Cota 145*fe9959a2SEmilio G. Cota static inline uint32_t do_qsp_entry_hash(const QSPEntry *entry, uint64_t a) 146*fe9959a2SEmilio G. Cota { 147*fe9959a2SEmilio G. Cota return do_qsp_callsite_hash(entry->callsite, a); 148*fe9959a2SEmilio G. Cota } 149*fe9959a2SEmilio G. Cota 150*fe9959a2SEmilio G. Cota static uint32_t qsp_entry_hash(const QSPEntry *entry) 151*fe9959a2SEmilio G. Cota { 152*fe9959a2SEmilio G. Cota return do_qsp_entry_hash(entry, (uint64_t)(uintptr_t)entry->thread_ptr); 153*fe9959a2SEmilio G. Cota } 154*fe9959a2SEmilio G. Cota 155*fe9959a2SEmilio G. Cota static uint32_t qsp_entry_no_thread_hash(const QSPEntry *entry) 156*fe9959a2SEmilio G. Cota { 157*fe9959a2SEmilio G. Cota return do_qsp_entry_hash(entry, 0); 158*fe9959a2SEmilio G. Cota } 159*fe9959a2SEmilio G. Cota 160*fe9959a2SEmilio G. Cota static bool qsp_callsite_cmp(const void *ap, const void *bp) 161*fe9959a2SEmilio G. Cota { 162*fe9959a2SEmilio G. Cota const QSPCallSite *a = ap; 163*fe9959a2SEmilio G. Cota const QSPCallSite *b = bp; 164*fe9959a2SEmilio G. Cota 165*fe9959a2SEmilio G. Cota return a == b || 166*fe9959a2SEmilio G. Cota (a->obj == b->obj && 167*fe9959a2SEmilio G. Cota a->line == b->line && 168*fe9959a2SEmilio G. Cota a->type == b->type && 169*fe9959a2SEmilio G. Cota (a->file == b->file || !strcmp(a->file, b->file))); 170*fe9959a2SEmilio G. Cota } 171*fe9959a2SEmilio G. Cota 172*fe9959a2SEmilio G. Cota static bool qsp_entry_no_thread_cmp(const void *ap, const void *bp) 173*fe9959a2SEmilio G. Cota { 174*fe9959a2SEmilio G. Cota const QSPEntry *a = ap; 175*fe9959a2SEmilio G. Cota const QSPEntry *b = bp; 176*fe9959a2SEmilio G. Cota 177*fe9959a2SEmilio G. Cota return qsp_callsite_cmp(a->callsite, b->callsite); 178*fe9959a2SEmilio G. Cota } 179*fe9959a2SEmilio G. Cota 180*fe9959a2SEmilio G. Cota static bool qsp_entry_cmp(const void *ap, const void *bp) 181*fe9959a2SEmilio G. Cota { 182*fe9959a2SEmilio G. Cota const QSPEntry *a = ap; 183*fe9959a2SEmilio G. Cota const QSPEntry *b = bp; 184*fe9959a2SEmilio G. Cota 185*fe9959a2SEmilio G. Cota return a->thread_ptr == b->thread_ptr && 186*fe9959a2SEmilio G. Cota qsp_callsite_cmp(a->callsite, b->callsite); 187*fe9959a2SEmilio G. Cota } 188*fe9959a2SEmilio G. Cota 189*fe9959a2SEmilio G. Cota /* 190*fe9959a2SEmilio G. Cota * Normally we'd call this from a constructor function, but we want it to work 191*fe9959a2SEmilio G. Cota * via libutil as well. 192*fe9959a2SEmilio G. Cota */ 193*fe9959a2SEmilio G. Cota static void qsp_do_init(void) 194*fe9959a2SEmilio G. Cota { 195*fe9959a2SEmilio G. Cota /* make sure this file's path in the tree is up to date with QSP_REL_PATH */ 196*fe9959a2SEmilio G. Cota g_assert(strstr(__FILE__, QSP_REL_PATH)); 197*fe9959a2SEmilio G. Cota qsp_qemu_path_len = strlen(__FILE__) - strlen(QSP_REL_PATH); 198*fe9959a2SEmilio G. Cota 199*fe9959a2SEmilio G. Cota qht_init(&qsp_ht, qsp_entry_cmp, QSP_INITIAL_SIZE, 200*fe9959a2SEmilio G. Cota QHT_MODE_AUTO_RESIZE | QHT_MODE_RAW_MUTEXES); 201*fe9959a2SEmilio G. Cota qht_init(&qsp_callsite_ht, qsp_callsite_cmp, QSP_INITIAL_SIZE, 202*fe9959a2SEmilio G. Cota QHT_MODE_AUTO_RESIZE | QHT_MODE_RAW_MUTEXES); 203*fe9959a2SEmilio G. Cota } 204*fe9959a2SEmilio G. Cota 205*fe9959a2SEmilio G. Cota static __attribute__((noinline)) void qsp_init__slowpath(void) 206*fe9959a2SEmilio G. Cota { 207*fe9959a2SEmilio G. Cota if (atomic_cmpxchg(&qsp_initializing, false, true) == false) { 208*fe9959a2SEmilio G. Cota qsp_do_init(); 209*fe9959a2SEmilio G. Cota atomic_set(&qsp_initialized, true); 210*fe9959a2SEmilio G. Cota } else { 211*fe9959a2SEmilio G. Cota while (!atomic_read(&qsp_initialized)) { 212*fe9959a2SEmilio G. Cota cpu_relax(); 213*fe9959a2SEmilio G. Cota } 214*fe9959a2SEmilio G. Cota } 215*fe9959a2SEmilio G. Cota } 216*fe9959a2SEmilio G. Cota 217*fe9959a2SEmilio G. Cota /* qsp_init() must be called from _all_ exported functions */ 218*fe9959a2SEmilio G. Cota static inline void qsp_init(void) 219*fe9959a2SEmilio G. Cota { 220*fe9959a2SEmilio G. Cota if (likely(atomic_read(&qsp_initialized))) { 221*fe9959a2SEmilio G. Cota return; 222*fe9959a2SEmilio G. Cota } 223*fe9959a2SEmilio G. Cota qsp_init__slowpath(); 224*fe9959a2SEmilio G. Cota } 225*fe9959a2SEmilio G. Cota 226*fe9959a2SEmilio G. Cota static QSPCallSite *qsp_callsite_find(const QSPCallSite *orig) 227*fe9959a2SEmilio G. Cota { 228*fe9959a2SEmilio G. Cota QSPCallSite *callsite; 229*fe9959a2SEmilio G. Cota uint32_t hash; 230*fe9959a2SEmilio G. Cota 231*fe9959a2SEmilio G. Cota hash = qsp_callsite_hash(orig); 232*fe9959a2SEmilio G. Cota callsite = qht_lookup(&qsp_callsite_ht, orig, hash); 233*fe9959a2SEmilio G. Cota if (callsite == NULL) { 234*fe9959a2SEmilio G. Cota void *existing = NULL; 235*fe9959a2SEmilio G. Cota 236*fe9959a2SEmilio G. Cota callsite = g_new(QSPCallSite, 1); 237*fe9959a2SEmilio G. Cota memcpy(callsite, orig, sizeof(*callsite)); 238*fe9959a2SEmilio G. Cota qht_insert(&qsp_callsite_ht, callsite, hash, &existing); 239*fe9959a2SEmilio G. Cota if (unlikely(existing)) { 240*fe9959a2SEmilio G. Cota g_free(callsite); 241*fe9959a2SEmilio G. Cota callsite = existing; 242*fe9959a2SEmilio G. Cota } 243*fe9959a2SEmilio G. Cota } 244*fe9959a2SEmilio G. Cota return callsite; 245*fe9959a2SEmilio G. Cota } 246*fe9959a2SEmilio G. Cota 247*fe9959a2SEmilio G. Cota static QSPEntry * 248*fe9959a2SEmilio G. Cota qsp_entry_create(struct qht *ht, const QSPEntry *entry, uint32_t hash) 249*fe9959a2SEmilio G. Cota { 250*fe9959a2SEmilio G. Cota QSPEntry *e; 251*fe9959a2SEmilio G. Cota void *existing = NULL; 252*fe9959a2SEmilio G. Cota 253*fe9959a2SEmilio G. Cota e = g_new0(QSPEntry, 1); 254*fe9959a2SEmilio G. Cota e->thread_ptr = entry->thread_ptr; 255*fe9959a2SEmilio G. Cota e->callsite = qsp_callsite_find(entry->callsite); 256*fe9959a2SEmilio G. Cota 257*fe9959a2SEmilio G. Cota qht_insert(ht, e, hash, &existing); 258*fe9959a2SEmilio G. Cota if (unlikely(existing)) { 259*fe9959a2SEmilio G. Cota g_free(e); 260*fe9959a2SEmilio G. Cota e = existing; 261*fe9959a2SEmilio G. Cota } 262*fe9959a2SEmilio G. Cota return e; 263*fe9959a2SEmilio G. Cota } 264*fe9959a2SEmilio G. Cota 265*fe9959a2SEmilio G. Cota static QSPEntry * 266*fe9959a2SEmilio G. Cota qsp_entry_find(struct qht *ht, const QSPEntry *entry, uint32_t hash) 267*fe9959a2SEmilio G. Cota { 268*fe9959a2SEmilio G. Cota QSPEntry *e; 269*fe9959a2SEmilio G. Cota 270*fe9959a2SEmilio G. Cota e = qht_lookup(ht, entry, hash); 271*fe9959a2SEmilio G. Cota if (e == NULL) { 272*fe9959a2SEmilio G. Cota e = qsp_entry_create(ht, entry, hash); 273*fe9959a2SEmilio G. Cota } 274*fe9959a2SEmilio G. Cota return e; 275*fe9959a2SEmilio G. Cota } 276*fe9959a2SEmilio G. Cota 277*fe9959a2SEmilio G. Cota /* 278*fe9959a2SEmilio G. Cota * Note: Entries are never removed, so callers do not have to be in an RCU 279*fe9959a2SEmilio G. Cota * read-side critical section. 280*fe9959a2SEmilio G. Cota */ 281*fe9959a2SEmilio G. Cota static QSPEntry *qsp_entry_get(const void *obj, const char *file, int line, 282*fe9959a2SEmilio G. Cota enum QSPType type) 283*fe9959a2SEmilio G. Cota { 284*fe9959a2SEmilio G. Cota QSPCallSite callsite = { 285*fe9959a2SEmilio G. Cota .obj = obj, 286*fe9959a2SEmilio G. Cota .file = file, 287*fe9959a2SEmilio G. Cota .line = line, 288*fe9959a2SEmilio G. Cota .type = type, 289*fe9959a2SEmilio G. Cota }; 290*fe9959a2SEmilio G. Cota QSPEntry orig; 291*fe9959a2SEmilio G. Cota uint32_t hash; 292*fe9959a2SEmilio G. Cota 293*fe9959a2SEmilio G. Cota qsp_init(); 294*fe9959a2SEmilio G. Cota 295*fe9959a2SEmilio G. Cota orig.thread_ptr = &qsp_thread; 296*fe9959a2SEmilio G. Cota orig.callsite = &callsite; 297*fe9959a2SEmilio G. Cota 298*fe9959a2SEmilio G. Cota hash = qsp_entry_hash(&orig); 299*fe9959a2SEmilio G. Cota return qsp_entry_find(&qsp_ht, &orig, hash); 300*fe9959a2SEmilio G. Cota } 301*fe9959a2SEmilio G. Cota 302*fe9959a2SEmilio G. Cota /* 303*fe9959a2SEmilio G. Cota * @from is in the global hash table; read it atomically if the host 304*fe9959a2SEmilio G. Cota * supports it, otherwise use the seqlock. 305*fe9959a2SEmilio G. Cota */ 306*fe9959a2SEmilio G. Cota static void qsp_entry_aggregate(QSPEntry *to, const QSPEntry *from) 307*fe9959a2SEmilio G. Cota { 308*fe9959a2SEmilio G. Cota #ifdef CONFIG_ATOMIC64 309*fe9959a2SEmilio G. Cota to->ns += atomic_read__nocheck(&from->ns); 310*fe9959a2SEmilio G. Cota to->n_acqs += atomic_read__nocheck(&from->n_acqs); 311*fe9959a2SEmilio G. Cota #else 312*fe9959a2SEmilio G. Cota unsigned int version; 313*fe9959a2SEmilio G. Cota uint64_t ns, n_acqs; 314*fe9959a2SEmilio G. Cota 315*fe9959a2SEmilio G. Cota do { 316*fe9959a2SEmilio G. Cota version = seqlock_read_begin(&from->sequence); 317*fe9959a2SEmilio G. Cota ns = atomic_read__nocheck(&from->ns); 318*fe9959a2SEmilio G. Cota n_acqs = atomic_read__nocheck(&from->n_acqs); 319*fe9959a2SEmilio G. Cota } while (seqlock_read_retry(&from->sequence, version)); 320*fe9959a2SEmilio G. Cota 321*fe9959a2SEmilio G. Cota to->ns += ns; 322*fe9959a2SEmilio G. Cota to->n_acqs += n_acqs; 323*fe9959a2SEmilio G. Cota #endif 324*fe9959a2SEmilio G. Cota } 325*fe9959a2SEmilio G. Cota 326*fe9959a2SEmilio G. Cota /* 327*fe9959a2SEmilio G. Cota * @e is in the global hash table; it is only written to by the current thread, 328*fe9959a2SEmilio G. Cota * so we write to it atomically (as in "write once") to prevent torn reads. 329*fe9959a2SEmilio G. Cota * If the host doesn't support u64 atomics, use the seqlock. 330*fe9959a2SEmilio G. Cota */ 331*fe9959a2SEmilio G. Cota static inline void do_qsp_entry_record(QSPEntry *e, int64_t delta, bool acq) 332*fe9959a2SEmilio G. Cota { 333*fe9959a2SEmilio G. Cota #ifndef CONFIG_ATOMIC64 334*fe9959a2SEmilio G. Cota seqlock_write_begin(&e->sequence); 335*fe9959a2SEmilio G. Cota #endif 336*fe9959a2SEmilio G. Cota atomic_set__nocheck(&e->ns, e->ns + delta); 337*fe9959a2SEmilio G. Cota if (acq) { 338*fe9959a2SEmilio G. Cota atomic_set__nocheck(&e->n_acqs, e->n_acqs + 1); 339*fe9959a2SEmilio G. Cota } 340*fe9959a2SEmilio G. Cota #ifndef CONFIG_ATOMIC64 341*fe9959a2SEmilio G. Cota seqlock_write_end(&e->sequence); 342*fe9959a2SEmilio G. Cota #endif 343*fe9959a2SEmilio G. Cota } 344*fe9959a2SEmilio G. Cota 345*fe9959a2SEmilio G. Cota static inline void qsp_entry_record(QSPEntry *e, int64_t delta) 346*fe9959a2SEmilio G. Cota { 347*fe9959a2SEmilio G. Cota do_qsp_entry_record(e, delta, true); 348*fe9959a2SEmilio G. Cota } 349*fe9959a2SEmilio G. Cota 350*fe9959a2SEmilio G. Cota #define QSP_GEN_VOID(type_, qsp_t_, func_, impl_) \ 351*fe9959a2SEmilio G. Cota static void func_(type_ *obj, const char *file, int line) \ 352*fe9959a2SEmilio G. Cota { \ 353*fe9959a2SEmilio G. Cota QSPEntry *e; \ 354*fe9959a2SEmilio G. Cota int64_t t0, t1; \ 355*fe9959a2SEmilio G. Cota \ 356*fe9959a2SEmilio G. Cota t0 = get_clock(); \ 357*fe9959a2SEmilio G. Cota impl_(obj, file, line); \ 358*fe9959a2SEmilio G. Cota t1 = get_clock(); \ 359*fe9959a2SEmilio G. Cota \ 360*fe9959a2SEmilio G. Cota e = qsp_entry_get(obj, file, line, qsp_t_); \ 361*fe9959a2SEmilio G. Cota qsp_entry_record(e, t1 - t0); \ 362*fe9959a2SEmilio G. Cota } 363*fe9959a2SEmilio G. Cota 364*fe9959a2SEmilio G. Cota #define QSP_GEN_RET1(type_, qsp_t_, func_, impl_) \ 365*fe9959a2SEmilio G. Cota static int func_(type_ *obj, const char *file, int line) \ 366*fe9959a2SEmilio G. Cota { \ 367*fe9959a2SEmilio G. Cota QSPEntry *e; \ 368*fe9959a2SEmilio G. Cota int64_t t0, t1; \ 369*fe9959a2SEmilio G. Cota int err; \ 370*fe9959a2SEmilio G. Cota \ 371*fe9959a2SEmilio G. Cota t0 = get_clock(); \ 372*fe9959a2SEmilio G. Cota err = impl_(obj, file, line); \ 373*fe9959a2SEmilio G. Cota t1 = get_clock(); \ 374*fe9959a2SEmilio G. Cota \ 375*fe9959a2SEmilio G. Cota e = qsp_entry_get(obj, file, line, qsp_t_); \ 376*fe9959a2SEmilio G. Cota do_qsp_entry_record(e, t1 - t0, !err); \ 377*fe9959a2SEmilio G. Cota return err; \ 378*fe9959a2SEmilio G. Cota } 379*fe9959a2SEmilio G. Cota 380*fe9959a2SEmilio G. Cota QSP_GEN_VOID(QemuMutex, QSP_MUTEX, qsp_mutex_lock, qemu_mutex_lock_impl) 381*fe9959a2SEmilio G. Cota QSP_GEN_RET1(QemuMutex, QSP_MUTEX, qsp_mutex_trylock, qemu_mutex_trylock_impl) 382*fe9959a2SEmilio G. Cota 383*fe9959a2SEmilio G. Cota QSP_GEN_VOID(QemuRecMutex, QSP_REC_MUTEX, qsp_rec_mutex_lock, 384*fe9959a2SEmilio G. Cota qemu_rec_mutex_lock_impl) 385*fe9959a2SEmilio G. Cota QSP_GEN_RET1(QemuRecMutex, QSP_REC_MUTEX, qsp_rec_mutex_trylock, 386*fe9959a2SEmilio G. Cota qemu_rec_mutex_trylock_impl) 387*fe9959a2SEmilio G. Cota 388*fe9959a2SEmilio G. Cota #undef QSP_GEN_RET1 389*fe9959a2SEmilio G. Cota #undef QSP_GEN_VOID 390*fe9959a2SEmilio G. Cota 391*fe9959a2SEmilio G. Cota static void 392*fe9959a2SEmilio G. Cota qsp_cond_wait(QemuCond *cond, QemuMutex *mutex, const char *file, int line) 393*fe9959a2SEmilio G. Cota { 394*fe9959a2SEmilio G. Cota QSPEntry *e; 395*fe9959a2SEmilio G. Cota int64_t t0, t1; 396*fe9959a2SEmilio G. Cota 397*fe9959a2SEmilio G. Cota t0 = get_clock(); 398*fe9959a2SEmilio G. Cota qemu_cond_wait_impl(cond, mutex, file, line); 399*fe9959a2SEmilio G. Cota t1 = get_clock(); 400*fe9959a2SEmilio G. Cota 401*fe9959a2SEmilio G. Cota e = qsp_entry_get(cond, file, line, QSP_CONDVAR); 402*fe9959a2SEmilio G. Cota qsp_entry_record(e, t1 - t0); 403*fe9959a2SEmilio G. Cota } 404*fe9959a2SEmilio G. Cota 405*fe9959a2SEmilio G. Cota bool qsp_is_enabled(void) 406*fe9959a2SEmilio G. Cota { 407*fe9959a2SEmilio G. Cota return atomic_read(&qemu_mutex_lock_func) == qsp_mutex_lock; 408*fe9959a2SEmilio G. Cota } 409*fe9959a2SEmilio G. Cota 410*fe9959a2SEmilio G. Cota void qsp_enable(void) 411*fe9959a2SEmilio G. Cota { 412*fe9959a2SEmilio G. Cota atomic_set(&qemu_mutex_lock_func, qsp_mutex_lock); 413*fe9959a2SEmilio G. Cota atomic_set(&qemu_mutex_trylock_func, qsp_mutex_trylock); 414*fe9959a2SEmilio G. Cota atomic_set(&qemu_rec_mutex_lock_func, qsp_rec_mutex_lock); 415*fe9959a2SEmilio G. Cota atomic_set(&qemu_rec_mutex_trylock_func, qsp_rec_mutex_trylock); 416*fe9959a2SEmilio G. Cota atomic_set(&qemu_cond_wait_func, qsp_cond_wait); 417*fe9959a2SEmilio G. Cota } 418*fe9959a2SEmilio G. Cota 419*fe9959a2SEmilio G. Cota void qsp_disable(void) 420*fe9959a2SEmilio G. Cota { 421*fe9959a2SEmilio G. Cota atomic_set(&qemu_mutex_lock_func, qemu_mutex_lock_impl); 422*fe9959a2SEmilio G. Cota atomic_set(&qemu_mutex_trylock_func, qemu_mutex_trylock_impl); 423*fe9959a2SEmilio G. Cota atomic_set(&qemu_rec_mutex_lock_func, qemu_rec_mutex_lock_impl); 424*fe9959a2SEmilio G. Cota atomic_set(&qemu_rec_mutex_trylock_func, qemu_rec_mutex_trylock_impl); 425*fe9959a2SEmilio G. Cota atomic_set(&qemu_cond_wait_func, qemu_cond_wait_impl); 426*fe9959a2SEmilio G. Cota } 427*fe9959a2SEmilio G. Cota 428*fe9959a2SEmilio G. Cota static gint qsp_tree_cmp(gconstpointer ap, gconstpointer bp, gpointer up) 429*fe9959a2SEmilio G. Cota { 430*fe9959a2SEmilio G. Cota const QSPEntry *a = ap; 431*fe9959a2SEmilio G. Cota const QSPEntry *b = bp; 432*fe9959a2SEmilio G. Cota const QSPCallSite *ca; 433*fe9959a2SEmilio G. Cota const QSPCallSite *cb; 434*fe9959a2SEmilio G. Cota 435*fe9959a2SEmilio G. Cota if (a->ns > b->ns) { 436*fe9959a2SEmilio G. Cota return -1; 437*fe9959a2SEmilio G. Cota } else if (a->ns < b->ns) { 438*fe9959a2SEmilio G. Cota return 1; 439*fe9959a2SEmilio G. Cota } 440*fe9959a2SEmilio G. Cota ca = a->callsite; 441*fe9959a2SEmilio G. Cota cb = b->callsite; 442*fe9959a2SEmilio G. Cota /* Break the tie with the object's address */ 443*fe9959a2SEmilio G. Cota if (ca->obj < cb->obj) { 444*fe9959a2SEmilio G. Cota return -1; 445*fe9959a2SEmilio G. Cota } else if (ca->obj > cb->obj) { 446*fe9959a2SEmilio G. Cota return 1; 447*fe9959a2SEmilio G. Cota } else { 448*fe9959a2SEmilio G. Cota int cmp; 449*fe9959a2SEmilio G. Cota 450*fe9959a2SEmilio G. Cota /* same obj. Break the tie with the callsite's file */ 451*fe9959a2SEmilio G. Cota cmp = strcmp(ca->file, cb->file); 452*fe9959a2SEmilio G. Cota if (cmp) { 453*fe9959a2SEmilio G. Cota return cmp; 454*fe9959a2SEmilio G. Cota } 455*fe9959a2SEmilio G. Cota /* same callsite file. Break the tie with the callsite's line */ 456*fe9959a2SEmilio G. Cota g_assert(ca->line != cb->line); 457*fe9959a2SEmilio G. Cota if (ca->line < cb->line) { 458*fe9959a2SEmilio G. Cota return -1; 459*fe9959a2SEmilio G. Cota } else if (ca->line > cb->line) { 460*fe9959a2SEmilio G. Cota return 1; 461*fe9959a2SEmilio G. Cota } else { 462*fe9959a2SEmilio G. Cota /* break the tie with the callsite's type */ 463*fe9959a2SEmilio G. Cota return cb->type - ca->type; 464*fe9959a2SEmilio G. Cota } 465*fe9959a2SEmilio G. Cota } 466*fe9959a2SEmilio G. Cota } 467*fe9959a2SEmilio G. Cota 468*fe9959a2SEmilio G. Cota static void qsp_sort(struct qht *ht, void *p, uint32_t h, void *userp) 469*fe9959a2SEmilio G. Cota { 470*fe9959a2SEmilio G. Cota QSPEntry *e = p; 471*fe9959a2SEmilio G. Cota GTree *tree = userp; 472*fe9959a2SEmilio G. Cota 473*fe9959a2SEmilio G. Cota g_tree_insert(tree, e, NULL); 474*fe9959a2SEmilio G. Cota } 475*fe9959a2SEmilio G. Cota 476*fe9959a2SEmilio G. Cota static void qsp_aggregate(struct qht *global_ht, void *p, uint32_t h, void *up) 477*fe9959a2SEmilio G. Cota { 478*fe9959a2SEmilio G. Cota struct qht *ht = up; 479*fe9959a2SEmilio G. Cota const QSPEntry *e = p; 480*fe9959a2SEmilio G. Cota QSPEntry *agg; 481*fe9959a2SEmilio G. Cota uint32_t hash; 482*fe9959a2SEmilio G. Cota 483*fe9959a2SEmilio G. Cota hash = qsp_entry_no_thread_hash(e); 484*fe9959a2SEmilio G. Cota agg = qsp_entry_find(ht, e, hash); 485*fe9959a2SEmilio G. Cota qsp_entry_aggregate(agg, e); 486*fe9959a2SEmilio G. Cota } 487*fe9959a2SEmilio G. Cota 488*fe9959a2SEmilio G. Cota static void qsp_mktree(GTree *tree) 489*fe9959a2SEmilio G. Cota { 490*fe9959a2SEmilio G. Cota struct qht ht; 491*fe9959a2SEmilio G. Cota 492*fe9959a2SEmilio G. Cota /* Aggregate all results from the global hash table into a local one */ 493*fe9959a2SEmilio G. Cota qht_init(&ht, qsp_entry_no_thread_cmp, QSP_INITIAL_SIZE, 494*fe9959a2SEmilio G. Cota QHT_MODE_AUTO_RESIZE | QHT_MODE_RAW_MUTEXES); 495*fe9959a2SEmilio G. Cota qht_iter(&qsp_ht, qsp_aggregate, &ht); 496*fe9959a2SEmilio G. Cota 497*fe9959a2SEmilio G. Cota /* sort the hash table elements by using a tree */ 498*fe9959a2SEmilio G. Cota qht_iter(&ht, qsp_sort, tree); 499*fe9959a2SEmilio G. Cota 500*fe9959a2SEmilio G. Cota /* free the hash table, but keep the elements (those are in the tree now) */ 501*fe9959a2SEmilio G. Cota qht_destroy(&ht); 502*fe9959a2SEmilio G. Cota } 503*fe9959a2SEmilio G. Cota 504*fe9959a2SEmilio G. Cota /* free string with g_free */ 505*fe9959a2SEmilio G. Cota static char *qsp_at(const QSPCallSite *callsite) 506*fe9959a2SEmilio G. Cota { 507*fe9959a2SEmilio G. Cota GString *s = g_string_new(NULL); 508*fe9959a2SEmilio G. Cota const char *shortened; 509*fe9959a2SEmilio G. Cota 510*fe9959a2SEmilio G. Cota /* remove the absolute path to qemu */ 511*fe9959a2SEmilio G. Cota if (unlikely(strlen(callsite->file) < qsp_qemu_path_len)) { 512*fe9959a2SEmilio G. Cota shortened = callsite->file; 513*fe9959a2SEmilio G. Cota } else { 514*fe9959a2SEmilio G. Cota shortened = callsite->file + qsp_qemu_path_len; 515*fe9959a2SEmilio G. Cota } 516*fe9959a2SEmilio G. Cota g_string_append_printf(s, "%s:%u", shortened, callsite->line); 517*fe9959a2SEmilio G. Cota return g_string_free(s, FALSE); 518*fe9959a2SEmilio G. Cota } 519*fe9959a2SEmilio G. Cota 520*fe9959a2SEmilio G. Cota struct QSPReportEntry { 521*fe9959a2SEmilio G. Cota const void *obj; 522*fe9959a2SEmilio G. Cota char *callsite_at; 523*fe9959a2SEmilio G. Cota const char *typename; 524*fe9959a2SEmilio G. Cota double time_s; 525*fe9959a2SEmilio G. Cota double ns_avg; 526*fe9959a2SEmilio G. Cota uint64_t n_acqs; 527*fe9959a2SEmilio G. Cota }; 528*fe9959a2SEmilio G. Cota typedef struct QSPReportEntry QSPReportEntry; 529*fe9959a2SEmilio G. Cota 530*fe9959a2SEmilio G. Cota struct QSPReport { 531*fe9959a2SEmilio G. Cota QSPReportEntry *entries; 532*fe9959a2SEmilio G. Cota size_t n_entries; 533*fe9959a2SEmilio G. Cota size_t max_n_entries; 534*fe9959a2SEmilio G. Cota }; 535*fe9959a2SEmilio G. Cota typedef struct QSPReport QSPReport; 536*fe9959a2SEmilio G. Cota 537*fe9959a2SEmilio G. Cota static gboolean qsp_tree_report(gpointer key, gpointer value, gpointer udata) 538*fe9959a2SEmilio G. Cota { 539*fe9959a2SEmilio G. Cota const QSPEntry *e = key; 540*fe9959a2SEmilio G. Cota QSPReport *report = udata; 541*fe9959a2SEmilio G. Cota QSPReportEntry *entry; 542*fe9959a2SEmilio G. Cota 543*fe9959a2SEmilio G. Cota if (report->n_entries == report->max_n_entries) { 544*fe9959a2SEmilio G. Cota return TRUE; 545*fe9959a2SEmilio G. Cota } 546*fe9959a2SEmilio G. Cota entry = &report->entries[report->n_entries]; 547*fe9959a2SEmilio G. Cota report->n_entries++; 548*fe9959a2SEmilio G. Cota 549*fe9959a2SEmilio G. Cota entry->obj = e->callsite->obj; 550*fe9959a2SEmilio G. Cota entry->callsite_at = qsp_at(e->callsite); 551*fe9959a2SEmilio G. Cota entry->typename = qsp_typenames[e->callsite->type]; 552*fe9959a2SEmilio G. Cota entry->time_s = e->ns * 1e-9; 553*fe9959a2SEmilio G. Cota entry->n_acqs = e->n_acqs; 554*fe9959a2SEmilio G. Cota entry->ns_avg = e->n_acqs ? e->ns / e->n_acqs : 0; 555*fe9959a2SEmilio G. Cota return FALSE; 556*fe9959a2SEmilio G. Cota } 557*fe9959a2SEmilio G. Cota 558*fe9959a2SEmilio G. Cota static void 559*fe9959a2SEmilio G. Cota pr_report(const QSPReport *rep, FILE *f, fprintf_function pr) 560*fe9959a2SEmilio G. Cota { 561*fe9959a2SEmilio G. Cota char *dashes; 562*fe9959a2SEmilio G. Cota size_t max_len = 0; 563*fe9959a2SEmilio G. Cota int callsite_len = 0; 564*fe9959a2SEmilio G. Cota int callsite_rspace; 565*fe9959a2SEmilio G. Cota int n_dashes; 566*fe9959a2SEmilio G. Cota size_t i; 567*fe9959a2SEmilio G. Cota 568*fe9959a2SEmilio G. Cota /* find out the maximum length of all 'callsite' fields */ 569*fe9959a2SEmilio G. Cota for (i = 0; i < rep->n_entries; i++) { 570*fe9959a2SEmilio G. Cota const QSPReportEntry *e = &rep->entries[i]; 571*fe9959a2SEmilio G. Cota size_t len = strlen(e->callsite_at); 572*fe9959a2SEmilio G. Cota 573*fe9959a2SEmilio G. Cota if (len > max_len) { 574*fe9959a2SEmilio G. Cota max_len = len; 575*fe9959a2SEmilio G. Cota } 576*fe9959a2SEmilio G. Cota } 577*fe9959a2SEmilio G. Cota 578*fe9959a2SEmilio G. Cota callsite_len = MAX(max_len, strlen("Call site")); 579*fe9959a2SEmilio G. Cota /* white space to leave to the right of "Call site" */ 580*fe9959a2SEmilio G. Cota callsite_rspace = callsite_len - strlen("Call site"); 581*fe9959a2SEmilio G. Cota 582*fe9959a2SEmilio G. Cota pr(f, "Type Object Call site%*s Wait Time (s) " 583*fe9959a2SEmilio G. Cota " Count Average (us)\n", callsite_rspace, ""); 584*fe9959a2SEmilio G. Cota 585*fe9959a2SEmilio G. Cota /* build a horizontal rule with dashes */ 586*fe9959a2SEmilio G. Cota n_dashes = 79 + callsite_rspace; 587*fe9959a2SEmilio G. Cota dashes = g_malloc(n_dashes + 1); 588*fe9959a2SEmilio G. Cota memset(dashes, '-', n_dashes); 589*fe9959a2SEmilio G. Cota dashes[n_dashes] = '\0'; 590*fe9959a2SEmilio G. Cota pr(f, "%s\n", dashes); 591*fe9959a2SEmilio G. Cota 592*fe9959a2SEmilio G. Cota for (i = 0; i < rep->n_entries; i++) { 593*fe9959a2SEmilio G. Cota const QSPReportEntry *e = &rep->entries[i]; 594*fe9959a2SEmilio G. Cota 595*fe9959a2SEmilio G. Cota pr(f, "%-9s %14p %s%*s %13.5f %12" PRIu64 " %12.2f\n", e->typename, 596*fe9959a2SEmilio G. Cota e->obj, e->callsite_at, callsite_len - (int)strlen(e->callsite_at), 597*fe9959a2SEmilio G. Cota "", e->time_s, e->n_acqs, e->ns_avg * 1e-3); 598*fe9959a2SEmilio G. Cota } 599*fe9959a2SEmilio G. Cota 600*fe9959a2SEmilio G. Cota pr(f, "%s\n", dashes); 601*fe9959a2SEmilio G. Cota g_free(dashes); 602*fe9959a2SEmilio G. Cota } 603*fe9959a2SEmilio G. Cota 604*fe9959a2SEmilio G. Cota static void report_destroy(QSPReport *rep) 605*fe9959a2SEmilio G. Cota { 606*fe9959a2SEmilio G. Cota size_t i; 607*fe9959a2SEmilio G. Cota 608*fe9959a2SEmilio G. Cota for (i = 0; i < rep->n_entries; i++) { 609*fe9959a2SEmilio G. Cota QSPReportEntry *e = &rep->entries[i]; 610*fe9959a2SEmilio G. Cota 611*fe9959a2SEmilio G. Cota g_free(e->callsite_at); 612*fe9959a2SEmilio G. Cota } 613*fe9959a2SEmilio G. Cota g_free(rep->entries); 614*fe9959a2SEmilio G. Cota } 615*fe9959a2SEmilio G. Cota 616*fe9959a2SEmilio G. Cota void qsp_report(FILE *f, fprintf_function cpu_fprintf, size_t max) 617*fe9959a2SEmilio G. Cota { 618*fe9959a2SEmilio G. Cota GTree *tree = g_tree_new_full(qsp_tree_cmp, NULL, g_free, NULL); 619*fe9959a2SEmilio G. Cota QSPReport rep; 620*fe9959a2SEmilio G. Cota 621*fe9959a2SEmilio G. Cota qsp_init(); 622*fe9959a2SEmilio G. Cota 623*fe9959a2SEmilio G. Cota rep.entries = g_new0(QSPReportEntry, max); 624*fe9959a2SEmilio G. Cota rep.n_entries = 0; 625*fe9959a2SEmilio G. Cota rep.max_n_entries = max; 626*fe9959a2SEmilio G. Cota 627*fe9959a2SEmilio G. Cota qsp_mktree(tree); 628*fe9959a2SEmilio G. Cota g_tree_foreach(tree, qsp_tree_report, &rep); 629*fe9959a2SEmilio G. Cota g_tree_destroy(tree); 630*fe9959a2SEmilio G. Cota 631*fe9959a2SEmilio G. Cota pr_report(&rep, f, cpu_fprintf); 632*fe9959a2SEmilio G. Cota report_destroy(&rep); 633*fe9959a2SEmilio G. Cota } 634