1 /* 2 * Copyright (C) 1995 Linus Torvalds 3 * 4 * Pentium III FXSR, SSE support 5 * Gareth Hughes <gareth@valinux.com>, May 2000 6 * 7 * X86-64 port 8 * Andi Kleen. 9 * 10 * CPU hotplug support - ashok.raj@intel.com 11 */ 12 13 /* 14 * This file handles the architecture-dependent parts of process handling.. 15 */ 16 17 #include <linux/cpu.h> 18 #include <linux/errno.h> 19 #include <linux/sched.h> 20 #include <linux/fs.h> 21 #include <linux/kernel.h> 22 #include <linux/mm.h> 23 #include <linux/elfcore.h> 24 #include <linux/smp.h> 25 #include <linux/slab.h> 26 #include <linux/user.h> 27 #include <linux/interrupt.h> 28 #include <linux/delay.h> 29 #include <linux/module.h> 30 #include <linux/ptrace.h> 31 #include <linux/notifier.h> 32 #include <linux/kprobes.h> 33 #include <linux/kdebug.h> 34 #include <linux/prctl.h> 35 #include <linux/uaccess.h> 36 #include <linux/io.h> 37 #include <linux/ftrace.h> 38 39 #include <asm/pgtable.h> 40 #include <asm/processor.h> 41 #include <asm/i387.h> 42 #include <asm/fpu-internal.h> 43 #include <asm/mmu_context.h> 44 #include <asm/prctl.h> 45 #include <asm/desc.h> 46 #include <asm/proto.h> 47 #include <asm/ia32.h> 48 #include <asm/idle.h> 49 #include <asm/syscalls.h> 50 #include <asm/debugreg.h> 51 #include <asm/switch_to.h> 52 53 asmlinkage extern void ret_from_fork(void); 54 55 DEFINE_PER_CPU(unsigned long, old_rsp); 56 57 /* Prints also some state that isn't saved in the pt_regs */ 58 void __show_regs(struct pt_regs *regs, int all) 59 { 60 unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs; 61 unsigned long d0, d1, d2, d3, d6, d7; 62 unsigned int fsindex, gsindex; 63 unsigned int ds, cs, es; 64 65 show_regs_common(); 66 printk(KERN_DEFAULT "RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->ip); 67 printk_address(regs->ip, 1); 68 printk(KERN_DEFAULT "RSP: %04lx:%016lx EFLAGS: %08lx\n", regs->ss, 69 regs->sp, regs->flags); 70 printk(KERN_DEFAULT "RAX: %016lx RBX: %016lx RCX: %016lx\n", 71 regs->ax, regs->bx, regs->cx); 72 printk(KERN_DEFAULT "RDX: %016lx RSI: %016lx RDI: %016lx\n", 73 regs->dx, regs->si, regs->di); 74 printk(KERN_DEFAULT "RBP: %016lx R08: %016lx R09: %016lx\n", 75 regs->bp, regs->r8, regs->r9); 76 printk(KERN_DEFAULT "R10: %016lx R11: %016lx R12: %016lx\n", 77 regs->r10, regs->r11, regs->r12); 78 printk(KERN_DEFAULT "R13: %016lx R14: %016lx R15: %016lx\n", 79 regs->r13, regs->r14, regs->r15); 80 81 asm("movl %%ds,%0" : "=r" (ds)); 82 asm("movl %%cs,%0" : "=r" (cs)); 83 asm("movl %%es,%0" : "=r" (es)); 84 asm("movl %%fs,%0" : "=r" (fsindex)); 85 asm("movl %%gs,%0" : "=r" (gsindex)); 86 87 rdmsrl(MSR_FS_BASE, fs); 88 rdmsrl(MSR_GS_BASE, gs); 89 rdmsrl(MSR_KERNEL_GS_BASE, shadowgs); 90 91 if (!all) 92 return; 93 94 cr0 = read_cr0(); 95 cr2 = read_cr2(); 96 cr3 = read_cr3(); 97 cr4 = read_cr4(); 98 99 printk(KERN_DEFAULT "FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n", 100 fs, fsindex, gs, gsindex, shadowgs); 101 printk(KERN_DEFAULT "CS: %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds, 102 es, cr0); 103 printk(KERN_DEFAULT "CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3, 104 cr4); 105 106 get_debugreg(d0, 0); 107 get_debugreg(d1, 1); 108 get_debugreg(d2, 2); 109 printk(KERN_DEFAULT "DR0: %016lx DR1: %016lx DR2: %016lx\n", d0, d1, d2); 110 get_debugreg(d3, 3); 111 get_debugreg(d6, 6); 112 get_debugreg(d7, 7); 113 printk(KERN_DEFAULT "DR3: %016lx DR6: %016lx DR7: %016lx\n", d3, d6, d7); 114 } 115 116 void release_thread(struct task_struct *dead_task) 117 { 118 if (dead_task->mm) { 119 if (dead_task->mm->context.size) { 120 pr_warn("WARNING: dead process %8s still has LDT? <%p/%d>\n", 121 dead_task->comm, 122 dead_task->mm->context.ldt, 123 dead_task->mm->context.size); 124 BUG(); 125 } 126 } 127 } 128 129 static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr) 130 { 131 struct user_desc ud = { 132 .base_addr = addr, 133 .limit = 0xfffff, 134 .seg_32bit = 1, 135 .limit_in_pages = 1, 136 .useable = 1, 137 }; 138 struct desc_struct *desc = t->thread.tls_array; 139 desc += tls; 140 fill_ldt(desc, &ud); 141 } 142 143 static inline u32 read_32bit_tls(struct task_struct *t, int tls) 144 { 145 return get_desc_base(&t->thread.tls_array[tls]); 146 } 147 148 int copy_thread(unsigned long clone_flags, unsigned long sp, 149 unsigned long unused, 150 struct task_struct *p, struct pt_regs *regs) 151 { 152 int err; 153 struct pt_regs *childregs; 154 struct task_struct *me = current; 155 156 childregs = ((struct pt_regs *) 157 (THREAD_SIZE + task_stack_page(p))) - 1; 158 *childregs = *regs; 159 160 childregs->ax = 0; 161 if (user_mode(regs)) 162 childregs->sp = sp; 163 else 164 childregs->sp = (unsigned long)childregs; 165 166 p->thread.sp = (unsigned long) childregs; 167 p->thread.sp0 = (unsigned long) (childregs+1); 168 p->thread.usersp = me->thread.usersp; 169 170 set_tsk_thread_flag(p, TIF_FORK); 171 172 p->fpu_counter = 0; 173 p->thread.io_bitmap_ptr = NULL; 174 175 savesegment(gs, p->thread.gsindex); 176 p->thread.gs = p->thread.gsindex ? 0 : me->thread.gs; 177 savesegment(fs, p->thread.fsindex); 178 p->thread.fs = p->thread.fsindex ? 0 : me->thread.fs; 179 savesegment(es, p->thread.es); 180 savesegment(ds, p->thread.ds); 181 182 err = -ENOMEM; 183 memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps)); 184 185 if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) { 186 p->thread.io_bitmap_ptr = kmemdup(me->thread.io_bitmap_ptr, 187 IO_BITMAP_BYTES, GFP_KERNEL); 188 if (!p->thread.io_bitmap_ptr) { 189 p->thread.io_bitmap_max = 0; 190 return -ENOMEM; 191 } 192 set_tsk_thread_flag(p, TIF_IO_BITMAP); 193 } 194 195 /* 196 * Set a new TLS for the child thread? 197 */ 198 if (clone_flags & CLONE_SETTLS) { 199 #ifdef CONFIG_IA32_EMULATION 200 if (test_thread_flag(TIF_IA32)) 201 err = do_set_thread_area(p, -1, 202 (struct user_desc __user *)childregs->si, 0); 203 else 204 #endif 205 err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8); 206 if (err) 207 goto out; 208 } 209 err = 0; 210 out: 211 if (err && p->thread.io_bitmap_ptr) { 212 kfree(p->thread.io_bitmap_ptr); 213 p->thread.io_bitmap_max = 0; 214 } 215 216 return err; 217 } 218 219 static void 220 start_thread_common(struct pt_regs *regs, unsigned long new_ip, 221 unsigned long new_sp, 222 unsigned int _cs, unsigned int _ss, unsigned int _ds) 223 { 224 loadsegment(fs, 0); 225 loadsegment(es, _ds); 226 loadsegment(ds, _ds); 227 load_gs_index(0); 228 current->thread.usersp = new_sp; 229 regs->ip = new_ip; 230 regs->sp = new_sp; 231 this_cpu_write(old_rsp, new_sp); 232 regs->cs = _cs; 233 regs->ss = _ss; 234 regs->flags = X86_EFLAGS_IF; 235 /* 236 * Free the old FP and other extended state 237 */ 238 free_thread_xstate(current); 239 } 240 241 void 242 start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp) 243 { 244 start_thread_common(regs, new_ip, new_sp, 245 __USER_CS, __USER_DS, 0); 246 } 247 248 #ifdef CONFIG_IA32_EMULATION 249 void start_thread_ia32(struct pt_regs *regs, u32 new_ip, u32 new_sp) 250 { 251 start_thread_common(regs, new_ip, new_sp, 252 test_thread_flag(TIF_X32) 253 ? __USER_CS : __USER32_CS, 254 __USER_DS, __USER_DS); 255 } 256 #endif 257 258 /* 259 * switch_to(x,y) should switch tasks from x to y. 260 * 261 * This could still be optimized: 262 * - fold all the options into a flag word and test it with a single test. 263 * - could test fs/gs bitsliced 264 * 265 * Kprobes not supported here. Set the probe on schedule instead. 266 * Function graph tracer not supported too. 267 */ 268 __notrace_funcgraph struct task_struct * 269 __switch_to(struct task_struct *prev_p, struct task_struct *next_p) 270 { 271 struct thread_struct *prev = &prev_p->thread; 272 struct thread_struct *next = &next_p->thread; 273 int cpu = smp_processor_id(); 274 struct tss_struct *tss = &per_cpu(init_tss, cpu); 275 unsigned fsindex, gsindex; 276 fpu_switch_t fpu; 277 278 fpu = switch_fpu_prepare(prev_p, next_p, cpu); 279 280 /* 281 * Reload esp0, LDT and the page table pointer: 282 */ 283 load_sp0(tss, next); 284 285 /* 286 * Switch DS and ES. 287 * This won't pick up thread selector changes, but I guess that is ok. 288 */ 289 savesegment(es, prev->es); 290 if (unlikely(next->es | prev->es)) 291 loadsegment(es, next->es); 292 293 savesegment(ds, prev->ds); 294 if (unlikely(next->ds | prev->ds)) 295 loadsegment(ds, next->ds); 296 297 298 /* We must save %fs and %gs before load_TLS() because 299 * %fs and %gs may be cleared by load_TLS(). 300 * 301 * (e.g. xen_load_tls()) 302 */ 303 savesegment(fs, fsindex); 304 savesegment(gs, gsindex); 305 306 load_TLS(next, cpu); 307 308 /* 309 * Leave lazy mode, flushing any hypercalls made here. 310 * This must be done before restoring TLS segments so 311 * the GDT and LDT are properly updated, and must be 312 * done before math_state_restore, so the TS bit is up 313 * to date. 314 */ 315 arch_end_context_switch(next_p); 316 317 /* 318 * Switch FS and GS. 319 * 320 * Segment register != 0 always requires a reload. Also 321 * reload when it has changed. When prev process used 64bit 322 * base always reload to avoid an information leak. 323 */ 324 if (unlikely(fsindex | next->fsindex | prev->fs)) { 325 loadsegment(fs, next->fsindex); 326 /* 327 * Check if the user used a selector != 0; if yes 328 * clear 64bit base, since overloaded base is always 329 * mapped to the Null selector 330 */ 331 if (fsindex) 332 prev->fs = 0; 333 } 334 /* when next process has a 64bit base use it */ 335 if (next->fs) 336 wrmsrl(MSR_FS_BASE, next->fs); 337 prev->fsindex = fsindex; 338 339 if (unlikely(gsindex | next->gsindex | prev->gs)) { 340 load_gs_index(next->gsindex); 341 if (gsindex) 342 prev->gs = 0; 343 } 344 if (next->gs) 345 wrmsrl(MSR_KERNEL_GS_BASE, next->gs); 346 prev->gsindex = gsindex; 347 348 switch_fpu_finish(next_p, fpu); 349 350 /* 351 * Switch the PDA and FPU contexts. 352 */ 353 prev->usersp = this_cpu_read(old_rsp); 354 this_cpu_write(old_rsp, next->usersp); 355 this_cpu_write(current_task, next_p); 356 357 this_cpu_write(kernel_stack, 358 (unsigned long)task_stack_page(next_p) + 359 THREAD_SIZE - KERNEL_STACK_OFFSET); 360 361 /* 362 * Now maybe reload the debug registers and handle I/O bitmaps 363 */ 364 if (unlikely(task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT || 365 task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV)) 366 __switch_to_xtra(prev_p, next_p, tss); 367 368 return prev_p; 369 } 370 371 void set_personality_64bit(void) 372 { 373 /* inherit personality from parent */ 374 375 /* Make sure to be in 64bit mode */ 376 clear_thread_flag(TIF_IA32); 377 clear_thread_flag(TIF_ADDR32); 378 clear_thread_flag(TIF_X32); 379 380 /* Ensure the corresponding mm is not marked. */ 381 if (current->mm) 382 current->mm->context.ia32_compat = 0; 383 384 /* TBD: overwrites user setup. Should have two bits. 385 But 64bit processes have always behaved this way, 386 so it's not too bad. The main problem is just that 387 32bit childs are affected again. */ 388 current->personality &= ~READ_IMPLIES_EXEC; 389 } 390 391 void set_personality_ia32(bool x32) 392 { 393 /* inherit personality from parent */ 394 395 /* Make sure to be in 32bit mode */ 396 set_thread_flag(TIF_ADDR32); 397 398 /* Mark the associated mm as containing 32-bit tasks. */ 399 if (current->mm) 400 current->mm->context.ia32_compat = 1; 401 402 if (x32) { 403 clear_thread_flag(TIF_IA32); 404 set_thread_flag(TIF_X32); 405 current->personality &= ~READ_IMPLIES_EXEC; 406 /* is_compat_task() uses the presence of the x32 407 syscall bit flag to determine compat status */ 408 current_thread_info()->status &= ~TS_COMPAT; 409 } else { 410 set_thread_flag(TIF_IA32); 411 clear_thread_flag(TIF_X32); 412 current->personality |= force_personality32; 413 /* Prepare the first "return" to user space */ 414 current_thread_info()->status |= TS_COMPAT; 415 } 416 } 417 EXPORT_SYMBOL_GPL(set_personality_ia32); 418 419 unsigned long get_wchan(struct task_struct *p) 420 { 421 unsigned long stack; 422 u64 fp, ip; 423 int count = 0; 424 425 if (!p || p == current || p->state == TASK_RUNNING) 426 return 0; 427 stack = (unsigned long)task_stack_page(p); 428 if (p->thread.sp < stack || p->thread.sp >= stack+THREAD_SIZE) 429 return 0; 430 fp = *(u64 *)(p->thread.sp); 431 do { 432 if (fp < (unsigned long)stack || 433 fp >= (unsigned long)stack+THREAD_SIZE) 434 return 0; 435 ip = *(u64 *)(fp+8); 436 if (!in_sched_functions(ip)) 437 return ip; 438 fp = *(u64 *)fp; 439 } while (count++ < 16); 440 return 0; 441 } 442 443 long do_arch_prctl(struct task_struct *task, int code, unsigned long addr) 444 { 445 int ret = 0; 446 int doit = task == current; 447 int cpu; 448 449 switch (code) { 450 case ARCH_SET_GS: 451 if (addr >= TASK_SIZE_OF(task)) 452 return -EPERM; 453 cpu = get_cpu(); 454 /* handle small bases via the GDT because that's faster to 455 switch. */ 456 if (addr <= 0xffffffff) { 457 set_32bit_tls(task, GS_TLS, addr); 458 if (doit) { 459 load_TLS(&task->thread, cpu); 460 load_gs_index(GS_TLS_SEL); 461 } 462 task->thread.gsindex = GS_TLS_SEL; 463 task->thread.gs = 0; 464 } else { 465 task->thread.gsindex = 0; 466 task->thread.gs = addr; 467 if (doit) { 468 load_gs_index(0); 469 ret = wrmsrl_safe(MSR_KERNEL_GS_BASE, addr); 470 } 471 } 472 put_cpu(); 473 break; 474 case ARCH_SET_FS: 475 /* Not strictly needed for fs, but do it for symmetry 476 with gs */ 477 if (addr >= TASK_SIZE_OF(task)) 478 return -EPERM; 479 cpu = get_cpu(); 480 /* handle small bases via the GDT because that's faster to 481 switch. */ 482 if (addr <= 0xffffffff) { 483 set_32bit_tls(task, FS_TLS, addr); 484 if (doit) { 485 load_TLS(&task->thread, cpu); 486 loadsegment(fs, FS_TLS_SEL); 487 } 488 task->thread.fsindex = FS_TLS_SEL; 489 task->thread.fs = 0; 490 } else { 491 task->thread.fsindex = 0; 492 task->thread.fs = addr; 493 if (doit) { 494 /* set the selector to 0 to not confuse 495 __switch_to */ 496 loadsegment(fs, 0); 497 ret = wrmsrl_safe(MSR_FS_BASE, addr); 498 } 499 } 500 put_cpu(); 501 break; 502 case ARCH_GET_FS: { 503 unsigned long base; 504 if (task->thread.fsindex == FS_TLS_SEL) 505 base = read_32bit_tls(task, FS_TLS); 506 else if (doit) 507 rdmsrl(MSR_FS_BASE, base); 508 else 509 base = task->thread.fs; 510 ret = put_user(base, (unsigned long __user *)addr); 511 break; 512 } 513 case ARCH_GET_GS: { 514 unsigned long base; 515 unsigned gsindex; 516 if (task->thread.gsindex == GS_TLS_SEL) 517 base = read_32bit_tls(task, GS_TLS); 518 else if (doit) { 519 savesegment(gs, gsindex); 520 if (gsindex) 521 rdmsrl(MSR_KERNEL_GS_BASE, base); 522 else 523 base = task->thread.gs; 524 } else 525 base = task->thread.gs; 526 ret = put_user(base, (unsigned long __user *)addr); 527 break; 528 } 529 530 default: 531 ret = -EINVAL; 532 break; 533 } 534 535 return ret; 536 } 537 538 long sys_arch_prctl(int code, unsigned long addr) 539 { 540 return do_arch_prctl(current, code, addr); 541 } 542 543 unsigned long KSTK_ESP(struct task_struct *task) 544 { 545 return (test_tsk_thread_flag(task, TIF_IA32)) ? 546 (task_pt_regs(task)->sp) : ((task)->thread.usersp); 547 } 548