1 /* 2 * This program is free software; you can redistribute it and/or modify 3 * it under the terms of the GNU General Public License as published by 4 * the Free Software Foundation; either version 2 of the License, or 5 * (at your option) any later version. 6 * 7 * This program is distributed in the hope that it will be useful, 8 * but WITHOUT ANY WARRANTY; without even the implied warranty of 9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 * GNU General Public License for more details. 11 * 12 * You should have received a copy of the GNU General Public License 13 * along with this program; if not, write to the Free Software 14 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 15 * 16 * Copyright (C) IBM Corporation, 2005 17 * Jeff Muizelaar, 2006, 2007 18 * Pekka Paalanen, 2008 <pq@iki.fi> 19 * 20 * Derived from the read-mod example from relay-examples by Tom Zanussi. 21 */ 22 #define DEBUG 1 23 24 #include <linux/module.h> 25 #include <linux/debugfs.h> 26 #include <linux/uaccess.h> 27 #include <linux/io.h> 28 #include <linux/version.h> 29 #include <linux/kallsyms.h> 30 #include <asm/pgtable.h> 31 #include <linux/mmiotrace.h> 32 #include <asm/e820.h> /* for ISA_START_ADDRESS */ 33 #include <asm/atomic.h> 34 #include <linux/percpu.h> 35 #include <linux/cpu.h> 36 37 #include "pf_in.h" 38 39 #define NAME "mmiotrace: " 40 41 struct trap_reason { 42 unsigned long addr; 43 unsigned long ip; 44 enum reason_type type; 45 int active_traces; 46 }; 47 48 struct remap_trace { 49 struct list_head list; 50 struct kmmio_probe probe; 51 resource_size_t phys; 52 unsigned long id; 53 }; 54 55 /* Accessed per-cpu. */ 56 static DEFINE_PER_CPU(struct trap_reason, pf_reason); 57 static DEFINE_PER_CPU(struct mmiotrace_rw, cpu_trace); 58 59 static DEFINE_MUTEX(mmiotrace_mutex); 60 static DEFINE_SPINLOCK(trace_lock); 61 static atomic_t mmiotrace_enabled; 62 static LIST_HEAD(trace_list); /* struct remap_trace */ 63 64 /* 65 * Locking in this file: 66 * - mmiotrace_mutex enforces enable/disable_mmiotrace() critical sections. 67 * - mmiotrace_enabled may be modified only when holding mmiotrace_mutex 68 * and trace_lock. 69 * - Routines depending on is_enabled() must take trace_lock. 70 * - trace_list users must hold trace_lock. 71 * - is_enabled() guarantees that mmio_trace_{rw,mapping} are allowed. 72 * - pre/post callbacks assume the effect of is_enabled() being true. 73 */ 74 75 /* module parameters */ 76 static unsigned long filter_offset; 77 static int nommiotrace; 78 static int trace_pc; 79 80 module_param(filter_offset, ulong, 0); 81 module_param(nommiotrace, bool, 0); 82 module_param(trace_pc, bool, 0); 83 84 MODULE_PARM_DESC(filter_offset, "Start address of traced mappings."); 85 MODULE_PARM_DESC(nommiotrace, "Disable actual MMIO tracing."); 86 MODULE_PARM_DESC(trace_pc, "Record address of faulting instructions."); 87 88 static bool is_enabled(void) 89 { 90 return atomic_read(&mmiotrace_enabled); 91 } 92 93 static void print_pte(unsigned long address) 94 { 95 unsigned int level; 96 pte_t *pte = lookup_address(address, &level); 97 98 if (!pte) { 99 pr_err(NAME "Error in %s: no pte for page 0x%08lx\n", 100 __func__, address); 101 return; 102 } 103 104 if (level == PG_LEVEL_2M) { 105 pr_emerg(NAME "4MB pages are not currently supported: " 106 "0x%08lx\n", address); 107 BUG(); 108 } 109 pr_info(NAME "pte for 0x%lx: 0x%llx 0x%llx\n", address, 110 (unsigned long long)pte_val(*pte), 111 (unsigned long long)pte_val(*pte) & _PAGE_PRESENT); 112 } 113 114 /* 115 * For some reason the pre/post pairs have been called in an 116 * unmatched order. Report and die. 117 */ 118 static void die_kmmio_nesting_error(struct pt_regs *regs, unsigned long addr) 119 { 120 const struct trap_reason *my_reason = &get_cpu_var(pf_reason); 121 pr_emerg(NAME "unexpected fault for address: 0x%08lx, " 122 "last fault for address: 0x%08lx\n", 123 addr, my_reason->addr); 124 print_pte(addr); 125 print_symbol(KERN_EMERG "faulting IP is at %s\n", regs->ip); 126 print_symbol(KERN_EMERG "last faulting IP was at %s\n", my_reason->ip); 127 #ifdef __i386__ 128 pr_emerg("eax: %08lx ebx: %08lx ecx: %08lx edx: %08lx\n", 129 regs->ax, regs->bx, regs->cx, regs->dx); 130 pr_emerg("esi: %08lx edi: %08lx ebp: %08lx esp: %08lx\n", 131 regs->si, regs->di, regs->bp, regs->sp); 132 #else 133 pr_emerg("rax: %016lx rcx: %016lx rdx: %016lx\n", 134 regs->ax, regs->cx, regs->dx); 135 pr_emerg("rsi: %016lx rdi: %016lx rbp: %016lx rsp: %016lx\n", 136 regs->si, regs->di, regs->bp, regs->sp); 137 #endif 138 put_cpu_var(pf_reason); 139 BUG(); 140 } 141 142 static void pre(struct kmmio_probe *p, struct pt_regs *regs, 143 unsigned long addr) 144 { 145 struct trap_reason *my_reason = &get_cpu_var(pf_reason); 146 struct mmiotrace_rw *my_trace = &get_cpu_var(cpu_trace); 147 const unsigned long instptr = instruction_pointer(regs); 148 const enum reason_type type = get_ins_type(instptr); 149 struct remap_trace *trace = p->private; 150 151 /* it doesn't make sense to have more than one active trace per cpu */ 152 if (my_reason->active_traces) 153 die_kmmio_nesting_error(regs, addr); 154 else 155 my_reason->active_traces++; 156 157 my_reason->type = type; 158 my_reason->addr = addr; 159 my_reason->ip = instptr; 160 161 my_trace->phys = addr - trace->probe.addr + trace->phys; 162 my_trace->map_id = trace->id; 163 164 /* 165 * Only record the program counter when requested. 166 * It may taint clean-room reverse engineering. 167 */ 168 if (trace_pc) 169 my_trace->pc = instptr; 170 else 171 my_trace->pc = 0; 172 173 /* 174 * XXX: the timestamp recorded will be *after* the tracing has been 175 * done, not at the time we hit the instruction. SMP implications 176 * on event ordering? 177 */ 178 179 switch (type) { 180 case REG_READ: 181 my_trace->opcode = MMIO_READ; 182 my_trace->width = get_ins_mem_width(instptr); 183 break; 184 case REG_WRITE: 185 my_trace->opcode = MMIO_WRITE; 186 my_trace->width = get_ins_mem_width(instptr); 187 my_trace->value = get_ins_reg_val(instptr, regs); 188 break; 189 case IMM_WRITE: 190 my_trace->opcode = MMIO_WRITE; 191 my_trace->width = get_ins_mem_width(instptr); 192 my_trace->value = get_ins_imm_val(instptr); 193 break; 194 default: 195 { 196 unsigned char *ip = (unsigned char *)instptr; 197 my_trace->opcode = MMIO_UNKNOWN_OP; 198 my_trace->width = 0; 199 my_trace->value = (*ip) << 16 | *(ip + 1) << 8 | 200 *(ip + 2); 201 } 202 } 203 put_cpu_var(cpu_trace); 204 put_cpu_var(pf_reason); 205 } 206 207 static void post(struct kmmio_probe *p, unsigned long condition, 208 struct pt_regs *regs) 209 { 210 struct trap_reason *my_reason = &get_cpu_var(pf_reason); 211 struct mmiotrace_rw *my_trace = &get_cpu_var(cpu_trace); 212 213 /* this should always return the active_trace count to 0 */ 214 my_reason->active_traces--; 215 if (my_reason->active_traces) { 216 pr_emerg(NAME "unexpected post handler"); 217 BUG(); 218 } 219 220 switch (my_reason->type) { 221 case REG_READ: 222 my_trace->value = get_ins_reg_val(my_reason->ip, regs); 223 break; 224 default: 225 break; 226 } 227 228 mmio_trace_rw(my_trace); 229 put_cpu_var(cpu_trace); 230 put_cpu_var(pf_reason); 231 } 232 233 static void ioremap_trace_core(resource_size_t offset, unsigned long size, 234 void __iomem *addr) 235 { 236 static atomic_t next_id; 237 struct remap_trace *trace = kmalloc(sizeof(*trace), GFP_KERNEL); 238 /* These are page-unaligned. */ 239 struct mmiotrace_map map = { 240 .phys = offset, 241 .virt = (unsigned long)addr, 242 .len = size, 243 .opcode = MMIO_PROBE 244 }; 245 246 if (!trace) { 247 pr_err(NAME "kmalloc failed in ioremap\n"); 248 return; 249 } 250 251 *trace = (struct remap_trace) { 252 .probe = { 253 .addr = (unsigned long)addr, 254 .len = size, 255 .pre_handler = pre, 256 .post_handler = post, 257 .private = trace 258 }, 259 .phys = offset, 260 .id = atomic_inc_return(&next_id) 261 }; 262 map.map_id = trace->id; 263 264 spin_lock_irq(&trace_lock); 265 if (!is_enabled()) { 266 kfree(trace); 267 goto not_enabled; 268 } 269 270 mmio_trace_mapping(&map); 271 list_add_tail(&trace->list, &trace_list); 272 if (!nommiotrace) 273 register_kmmio_probe(&trace->probe); 274 275 not_enabled: 276 spin_unlock_irq(&trace_lock); 277 } 278 279 void mmiotrace_ioremap(resource_size_t offset, unsigned long size, 280 void __iomem *addr) 281 { 282 if (!is_enabled()) /* recheck and proper locking in *_core() */ 283 return; 284 285 pr_debug(NAME "ioremap_*(0x%llx, 0x%lx) = %p\n", 286 (unsigned long long)offset, size, addr); 287 if ((filter_offset) && (offset != filter_offset)) 288 return; 289 ioremap_trace_core(offset, size, addr); 290 } 291 292 static void iounmap_trace_core(volatile void __iomem *addr) 293 { 294 struct mmiotrace_map map = { 295 .phys = 0, 296 .virt = (unsigned long)addr, 297 .len = 0, 298 .opcode = MMIO_UNPROBE 299 }; 300 struct remap_trace *trace; 301 struct remap_trace *tmp; 302 struct remap_trace *found_trace = NULL; 303 304 pr_debug(NAME "Unmapping %p.\n", addr); 305 306 spin_lock_irq(&trace_lock); 307 if (!is_enabled()) 308 goto not_enabled; 309 310 list_for_each_entry_safe(trace, tmp, &trace_list, list) { 311 if ((unsigned long)addr == trace->probe.addr) { 312 if (!nommiotrace) 313 unregister_kmmio_probe(&trace->probe); 314 list_del(&trace->list); 315 found_trace = trace; 316 break; 317 } 318 } 319 map.map_id = (found_trace) ? found_trace->id : -1; 320 mmio_trace_mapping(&map); 321 322 not_enabled: 323 spin_unlock_irq(&trace_lock); 324 if (found_trace) { 325 synchronize_rcu(); /* unregister_kmmio_probe() requirement */ 326 kfree(found_trace); 327 } 328 } 329 330 void mmiotrace_iounmap(volatile void __iomem *addr) 331 { 332 might_sleep(); 333 if (is_enabled()) /* recheck and proper locking in *_core() */ 334 iounmap_trace_core(addr); 335 } 336 337 int mmiotrace_printk(const char *fmt, ...) 338 { 339 int ret = 0; 340 va_list args; 341 unsigned long flags; 342 va_start(args, fmt); 343 344 spin_lock_irqsave(&trace_lock, flags); 345 if (is_enabled()) 346 ret = mmio_trace_printk(fmt, args); 347 spin_unlock_irqrestore(&trace_lock, flags); 348 349 va_end(args); 350 return ret; 351 } 352 EXPORT_SYMBOL(mmiotrace_printk); 353 354 static void clear_trace_list(void) 355 { 356 struct remap_trace *trace; 357 struct remap_trace *tmp; 358 359 /* 360 * No locking required, because the caller ensures we are in a 361 * critical section via mutex, and is_enabled() is false, 362 * i.e. nothing can traverse or modify this list. 363 * Caller also ensures is_enabled() cannot change. 364 */ 365 list_for_each_entry(trace, &trace_list, list) { 366 pr_notice(NAME "purging non-iounmapped " 367 "trace @0x%08lx, size 0x%lx.\n", 368 trace->probe.addr, trace->probe.len); 369 if (!nommiotrace) 370 unregister_kmmio_probe(&trace->probe); 371 } 372 synchronize_rcu(); /* unregister_kmmio_probe() requirement */ 373 374 list_for_each_entry_safe(trace, tmp, &trace_list, list) { 375 list_del(&trace->list); 376 kfree(trace); 377 } 378 } 379 380 #ifdef CONFIG_HOTPLUG_CPU 381 static cpumask_t downed_cpus; 382 383 static void enter_uniprocessor(void) 384 { 385 int cpu; 386 int err; 387 388 get_online_cpus(); 389 downed_cpus = cpu_online_map; 390 cpu_clear(first_cpu(cpu_online_map), downed_cpus); 391 if (num_online_cpus() > 1) 392 pr_notice(NAME "Disabling non-boot CPUs...\n"); 393 put_online_cpus(); 394 395 for_each_cpu_mask(cpu, downed_cpus) { 396 err = cpu_down(cpu); 397 if (!err) 398 pr_info(NAME "CPU%d is down.\n", cpu); 399 else 400 pr_err(NAME "Error taking CPU%d down: %d\n", cpu, err); 401 } 402 if (num_online_cpus() > 1) 403 pr_warning(NAME "multiple CPUs still online, " 404 "may miss events.\n"); 405 } 406 407 /* __ref because leave_uniprocessor calls cpu_up which is __cpuinit, 408 but this whole function is ifdefed CONFIG_HOTPLUG_CPU */ 409 static void __ref leave_uniprocessor(void) 410 { 411 int cpu; 412 int err; 413 414 if (cpus_weight(downed_cpus) == 0) 415 return; 416 pr_notice(NAME "Re-enabling CPUs...\n"); 417 for_each_cpu_mask(cpu, downed_cpus) { 418 err = cpu_up(cpu); 419 if (!err) 420 pr_info(NAME "enabled CPU%d.\n", cpu); 421 else 422 pr_err(NAME "cannot re-enable CPU%d: %d\n", cpu, err); 423 } 424 } 425 426 #else /* !CONFIG_HOTPLUG_CPU */ 427 static void enter_uniprocessor(void) 428 { 429 if (num_online_cpus() > 1) 430 pr_warning(NAME "multiple CPUs are online, may miss events. " 431 "Suggest booting with maxcpus=1 kernel argument.\n"); 432 } 433 434 static void leave_uniprocessor(void) 435 { 436 } 437 #endif 438 439 void enable_mmiotrace(void) 440 { 441 mutex_lock(&mmiotrace_mutex); 442 if (is_enabled()) 443 goto out; 444 445 if (nommiotrace) 446 pr_info(NAME "MMIO tracing disabled.\n"); 447 enter_uniprocessor(); 448 spin_lock_irq(&trace_lock); 449 atomic_inc(&mmiotrace_enabled); 450 spin_unlock_irq(&trace_lock); 451 pr_info(NAME "enabled.\n"); 452 out: 453 mutex_unlock(&mmiotrace_mutex); 454 } 455 456 void disable_mmiotrace(void) 457 { 458 mutex_lock(&mmiotrace_mutex); 459 if (!is_enabled()) 460 goto out; 461 462 spin_lock_irq(&trace_lock); 463 atomic_dec(&mmiotrace_enabled); 464 BUG_ON(is_enabled()); 465 spin_unlock_irq(&trace_lock); 466 467 clear_trace_list(); /* guarantees: no more kmmio callbacks */ 468 leave_uniprocessor(); 469 pr_info(NAME "disabled.\n"); 470 out: 471 mutex_unlock(&mmiotrace_mutex); 472 } 473