paravirt.c (abc745f85c1193d2a052addf0031d59b4436c246) paravirt.c (5c83511bdb9832c86be20fb86b783356e2f58062)
1/* Paravirtualization interfaces
2 Copyright (C) 2006 Rusty Russell IBM Corporation
3
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2 of the License, or
7 (at your option) any later version.
8

--- 114 unchanged lines hidden (view full) ---

123DEFINE_STATIC_KEY_TRUE(virt_spin_lock_key);
124
125void __init native_pv_lock_init(void)
126{
127 if (!static_cpu_has(X86_FEATURE_HYPERVISOR))
128 static_branch_disable(&virt_spin_lock_key);
129}
130
1/* Paravirtualization interfaces
2 Copyright (C) 2006 Rusty Russell IBM Corporation
3
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2 of the License, or
7 (at your option) any later version.
8

--- 114 unchanged lines hidden (view full) ---

123DEFINE_STATIC_KEY_TRUE(virt_spin_lock_key);
124
125void __init native_pv_lock_init(void)
126{
127 if (!static_cpu_has(X86_FEATURE_HYPERVISOR))
128 static_branch_disable(&virt_spin_lock_key);
129}
130
131/*
132 * Neat trick to map patch type back to the call within the
133 * corresponding structure.
134 */
135static void *get_call_destination(u8 type)
136{
137 struct paravirt_patch_template tmpl = {
138 .pv_init_ops = pv_init_ops,
139 .pv_time_ops = pv_time_ops,
140 .pv_cpu_ops = pv_cpu_ops,
141 .pv_irq_ops = pv_irq_ops,
142 .pv_mmu_ops = pv_mmu_ops,
143#ifdef CONFIG_PARAVIRT_SPINLOCKS
144 .pv_lock_ops = pv_lock_ops,
145#endif
146 };
147 return *((void **)&tmpl + type);
148}
149
150unsigned paravirt_patch_default(u8 type, void *insnbuf,
151 unsigned long addr, unsigned len)
152{
131unsigned paravirt_patch_default(u8 type, void *insnbuf,
132 unsigned long addr, unsigned len)
133{
153 void *opfunc = get_call_destination(type);
134 /*
135 * Neat trick to map patch type back to the call within the
136 * corresponding structure.
137 */
138 void *opfunc = *((void **)&pv_ops + type);
154 unsigned ret;
155
156 if (opfunc == NULL)
157 /* If there's no function, patch it with a ud2a (BUG) */
158 ret = paravirt_patch_insns(insnbuf, len, ud2a, ud2a+sizeof(ud2a));
159 else if (opfunc == _paravirt_nop)
160 ret = 0;
161
162 /* identity functions just return their single argument */
163 else if (opfunc == _paravirt_ident_32)
164 ret = paravirt_patch_ident_32(insnbuf, len);
165 else if (opfunc == _paravirt_ident_64)
166 ret = paravirt_patch_ident_64(insnbuf, len);
167
139 unsigned ret;
140
141 if (opfunc == NULL)
142 /* If there's no function, patch it with a ud2a (BUG) */
143 ret = paravirt_patch_insns(insnbuf, len, ud2a, ud2a+sizeof(ud2a));
144 else if (opfunc == _paravirt_nop)
145 ret = 0;
146
147 /* identity functions just return their single argument */
148 else if (opfunc == _paravirt_ident_32)
149 ret = paravirt_patch_ident_32(insnbuf, len);
150 else if (opfunc == _paravirt_ident_64)
151 ret = paravirt_patch_ident_64(insnbuf, len);
152
168 else if (type == PARAVIRT_PATCH(pv_cpu_ops.iret) ||
169 type == PARAVIRT_PATCH(pv_cpu_ops.usergs_sysret64))
153 else if (type == PARAVIRT_PATCH(cpu.iret) ||
154 type == PARAVIRT_PATCH(cpu.usergs_sysret64))
170 /* If operation requires a jmp, then jmp */
171 ret = paravirt_patch_jmp(insnbuf, opfunc, addr, len);
172 else
173 /* Otherwise call the function. */
174 ret = paravirt_patch_call(insnbuf, opfunc, addr, len);
175
176 return ret;
177}

--- 133 unchanged lines hidden (view full) ---

311 .kernel_rpl = 0,
312 .shared_kernel_pmd = 1, /* Only used when CONFIG_X86_PAE is set */
313
314#ifdef CONFIG_X86_64
315 .extra_user_64bit_cs = __USER_CS,
316#endif
317};
318
155 /* If operation requires a jmp, then jmp */
156 ret = paravirt_patch_jmp(insnbuf, opfunc, addr, len);
157 else
158 /* Otherwise call the function. */
159 ret = paravirt_patch_call(insnbuf, opfunc, addr, len);
160
161 return ret;
162}

--- 133 unchanged lines hidden (view full) ---

296 .kernel_rpl = 0,
297 .shared_kernel_pmd = 1, /* Only used when CONFIG_X86_PAE is set */
298
299#ifdef CONFIG_X86_64
300 .extra_user_64bit_cs = __USER_CS,
301#endif
302};
303
319struct pv_init_ops pv_init_ops = {
320 .patch = native_patch,
321};
304#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE)
305/* 32-bit pagetable entries */
306#define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_32)
307#else
308/* 64-bit pagetable entries */
309#define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_64)
310#endif
322
311
323struct pv_time_ops pv_time_ops = {
324 .sched_clock = native_sched_clock,
325 .steal_clock = native_steal_clock,
326};
312struct paravirt_patch_template pv_ops = {
313 /* Init ops. */
314 .init.patch = native_patch,
327
315
328__visible struct pv_irq_ops pv_irq_ops = {
329 .save_fl = __PV_IS_CALLEE_SAVE(native_save_fl),
330 .restore_fl = __PV_IS_CALLEE_SAVE(native_restore_fl),
331 .irq_disable = __PV_IS_CALLEE_SAVE(native_irq_disable),
332 .irq_enable = __PV_IS_CALLEE_SAVE(native_irq_enable),
333 .safe_halt = native_safe_halt,
334 .halt = native_halt,
335};
316 /* Time ops. */
317 .time.sched_clock = native_sched_clock,
318 .time.steal_clock = native_steal_clock,
336
319
337__visible struct pv_cpu_ops pv_cpu_ops = {
338 .cpuid = native_cpuid,
339 .get_debugreg = native_get_debugreg,
340 .set_debugreg = native_set_debugreg,
341 .read_cr0 = native_read_cr0,
342 .write_cr0 = native_write_cr0,
343 .write_cr4 = native_write_cr4,
320 /* Cpu ops. */
321 .cpu.cpuid = native_cpuid,
322 .cpu.get_debugreg = native_get_debugreg,
323 .cpu.set_debugreg = native_set_debugreg,
324 .cpu.read_cr0 = native_read_cr0,
325 .cpu.write_cr0 = native_write_cr0,
326 .cpu.write_cr4 = native_write_cr4,
344#ifdef CONFIG_X86_64
327#ifdef CONFIG_X86_64
345 .read_cr8 = native_read_cr8,
346 .write_cr8 = native_write_cr8,
328 .cpu.read_cr8 = native_read_cr8,
329 .cpu.write_cr8 = native_write_cr8,
347#endif
330#endif
348 .wbinvd = native_wbinvd,
349 .read_msr = native_read_msr,
350 .write_msr = native_write_msr,
351 .read_msr_safe = native_read_msr_safe,
352 .write_msr_safe = native_write_msr_safe,
353 .read_pmc = native_read_pmc,
354 .load_tr_desc = native_load_tr_desc,
355 .set_ldt = native_set_ldt,
356 .load_gdt = native_load_gdt,
357 .load_idt = native_load_idt,
358 .store_tr = native_store_tr,
359 .load_tls = native_load_tls,
331 .cpu.wbinvd = native_wbinvd,
332 .cpu.read_msr = native_read_msr,
333 .cpu.write_msr = native_write_msr,
334 .cpu.read_msr_safe = native_read_msr_safe,
335 .cpu.write_msr_safe = native_write_msr_safe,
336 .cpu.read_pmc = native_read_pmc,
337 .cpu.load_tr_desc = native_load_tr_desc,
338 .cpu.set_ldt = native_set_ldt,
339 .cpu.load_gdt = native_load_gdt,
340 .cpu.load_idt = native_load_idt,
341 .cpu.store_tr = native_store_tr,
342 .cpu.load_tls = native_load_tls,
360#ifdef CONFIG_X86_64
343#ifdef CONFIG_X86_64
361 .load_gs_index = native_load_gs_index,
344 .cpu.load_gs_index = native_load_gs_index,
362#endif
345#endif
363 .write_ldt_entry = native_write_ldt_entry,
364 .write_gdt_entry = native_write_gdt_entry,
365 .write_idt_entry = native_write_idt_entry,
346 .cpu.write_ldt_entry = native_write_ldt_entry,
347 .cpu.write_gdt_entry = native_write_gdt_entry,
348 .cpu.write_idt_entry = native_write_idt_entry,
366
349
367 .alloc_ldt = paravirt_nop,
368 .free_ldt = paravirt_nop,
350 .cpu.alloc_ldt = paravirt_nop,
351 .cpu.free_ldt = paravirt_nop,
369
352
370 .load_sp0 = native_load_sp0,
353 .cpu.load_sp0 = native_load_sp0,
371
372#ifdef CONFIG_X86_64
354
355#ifdef CONFIG_X86_64
373 .usergs_sysret64 = native_usergs_sysret64,
356 .cpu.usergs_sysret64 = native_usergs_sysret64,
374#endif
357#endif
375 .iret = native_iret,
376 .swapgs = native_swapgs,
358 .cpu.iret = native_iret,
359 .cpu.swapgs = native_swapgs,
377
360
378 .set_iopl_mask = native_set_iopl_mask,
379 .io_delay = native_io_delay,
361 .cpu.set_iopl_mask = native_set_iopl_mask,
362 .cpu.io_delay = native_io_delay,
380
363
381 .start_context_switch = paravirt_nop,
382 .end_context_switch = paravirt_nop,
383};
364 .cpu.start_context_switch = paravirt_nop,
365 .cpu.end_context_switch = paravirt_nop,
384
366
385/* At this point, native_get/set_debugreg has real function entries */
386NOKPROBE_SYMBOL(native_get_debugreg);
387NOKPROBE_SYMBOL(native_set_debugreg);
388NOKPROBE_SYMBOL(native_load_idt);
367 /* Irq ops. */
368 .irq.save_fl = __PV_IS_CALLEE_SAVE(native_save_fl),
369 .irq.restore_fl = __PV_IS_CALLEE_SAVE(native_restore_fl),
370 .irq.irq_disable = __PV_IS_CALLEE_SAVE(native_irq_disable),
371 .irq.irq_enable = __PV_IS_CALLEE_SAVE(native_irq_enable),
372 .irq.safe_halt = native_safe_halt,
373 .irq.halt = native_halt,
389
374
390#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE)
391/* 32-bit pagetable entries */
392#define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_32)
393#else
394/* 64-bit pagetable entries */
395#define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_64)
396#endif
375 /* Mmu ops. */
376 .mmu.read_cr2 = native_read_cr2,
377 .mmu.write_cr2 = native_write_cr2,
378 .mmu.read_cr3 = __native_read_cr3,
379 .mmu.write_cr3 = native_write_cr3,
397
380
398struct pv_mmu_ops pv_mmu_ops __ro_after_init = {
381 .mmu.flush_tlb_user = native_flush_tlb,
382 .mmu.flush_tlb_kernel = native_flush_tlb_global,
383 .mmu.flush_tlb_one_user = native_flush_tlb_one_user,
384 .mmu.flush_tlb_others = native_flush_tlb_others,
385 .mmu.tlb_remove_table =
386 (void (*)(struct mmu_gather *, void *))tlb_remove_page,
399
387
400 .read_cr2 = native_read_cr2,
401 .write_cr2 = native_write_cr2,
402 .read_cr3 = __native_read_cr3,
403 .write_cr3 = native_write_cr3,
388 .mmu.pgd_alloc = __paravirt_pgd_alloc,
389 .mmu.pgd_free = paravirt_nop,
404
390
405 .flush_tlb_user = native_flush_tlb,
406 .flush_tlb_kernel = native_flush_tlb_global,
407 .flush_tlb_one_user = native_flush_tlb_one_user,
408 .flush_tlb_others = native_flush_tlb_others,
409 .tlb_remove_table = (void (*)(struct mmu_gather *, void *))tlb_remove_page,
391 .mmu.alloc_pte = paravirt_nop,
392 .mmu.alloc_pmd = paravirt_nop,
393 .mmu.alloc_pud = paravirt_nop,
394 .mmu.alloc_p4d = paravirt_nop,
395 .mmu.release_pte = paravirt_nop,
396 .mmu.release_pmd = paravirt_nop,
397 .mmu.release_pud = paravirt_nop,
398 .mmu.release_p4d = paravirt_nop,
410
399
411 .pgd_alloc = __paravirt_pgd_alloc,
412 .pgd_free = paravirt_nop,
400 .mmu.set_pte = native_set_pte,
401 .mmu.set_pte_at = native_set_pte_at,
402 .mmu.set_pmd = native_set_pmd,
413
403
414 .alloc_pte = paravirt_nop,
415 .alloc_pmd = paravirt_nop,
416 .alloc_pud = paravirt_nop,
417 .alloc_p4d = paravirt_nop,
418 .release_pte = paravirt_nop,
419 .release_pmd = paravirt_nop,
420 .release_pud = paravirt_nop,
421 .release_p4d = paravirt_nop,
404 .mmu.ptep_modify_prot_start = __ptep_modify_prot_start,
405 .mmu.ptep_modify_prot_commit = __ptep_modify_prot_commit,
422
406
423 .set_pte = native_set_pte,
424 .set_pte_at = native_set_pte_at,
425 .set_pmd = native_set_pmd,
426
427 .ptep_modify_prot_start = __ptep_modify_prot_start,
428 .ptep_modify_prot_commit = __ptep_modify_prot_commit,
429
430#if CONFIG_PGTABLE_LEVELS >= 3
431#ifdef CONFIG_X86_PAE
407#if CONFIG_PGTABLE_LEVELS >= 3
408#ifdef CONFIG_X86_PAE
432 .set_pte_atomic = native_set_pte_atomic,
433 .pte_clear = native_pte_clear,
434 .pmd_clear = native_pmd_clear,
409 .mmu.set_pte_atomic = native_set_pte_atomic,
410 .mmu.pte_clear = native_pte_clear,
411 .mmu.pmd_clear = native_pmd_clear,
435#endif
412#endif
436 .set_pud = native_set_pud,
413 .mmu.set_pud = native_set_pud,
437
414
438 .pmd_val = PTE_IDENT,
439 .make_pmd = PTE_IDENT,
415 .mmu.pmd_val = PTE_IDENT,
416 .mmu.make_pmd = PTE_IDENT,
440
441#if CONFIG_PGTABLE_LEVELS >= 4
417
418#if CONFIG_PGTABLE_LEVELS >= 4
442 .pud_val = PTE_IDENT,
443 .make_pud = PTE_IDENT,
419 .mmu.pud_val = PTE_IDENT,
420 .mmu.make_pud = PTE_IDENT,
444
421
445 .set_p4d = native_set_p4d,
422 .mmu.set_p4d = native_set_p4d,
446
447#if CONFIG_PGTABLE_LEVELS >= 5
423
424#if CONFIG_PGTABLE_LEVELS >= 5
448 .p4d_val = PTE_IDENT,
449 .make_p4d = PTE_IDENT,
425 .mmu.p4d_val = PTE_IDENT,
426 .mmu.make_p4d = PTE_IDENT,
450
427
451 .set_pgd = native_set_pgd,
428 .mmu.set_pgd = native_set_pgd,
452#endif /* CONFIG_PGTABLE_LEVELS >= 5 */
453#endif /* CONFIG_PGTABLE_LEVELS >= 4 */
454#endif /* CONFIG_PGTABLE_LEVELS >= 3 */
455
429#endif /* CONFIG_PGTABLE_LEVELS >= 5 */
430#endif /* CONFIG_PGTABLE_LEVELS >= 4 */
431#endif /* CONFIG_PGTABLE_LEVELS >= 3 */
432
456 .pte_val = PTE_IDENT,
457 .pgd_val = PTE_IDENT,
433 .mmu.pte_val = PTE_IDENT,
434 .mmu.pgd_val = PTE_IDENT,
458
435
459 .make_pte = PTE_IDENT,
460 .make_pgd = PTE_IDENT,
436 .mmu.make_pte = PTE_IDENT,
437 .mmu.make_pgd = PTE_IDENT,
461
438
462 .dup_mmap = paravirt_nop,
463 .exit_mmap = paravirt_nop,
464 .activate_mm = paravirt_nop,
439 .mmu.dup_mmap = paravirt_nop,
440 .mmu.exit_mmap = paravirt_nop,
441 .mmu.activate_mm = paravirt_nop,
465
442
466 .lazy_mode = {
467 .enter = paravirt_nop,
468 .leave = paravirt_nop,
469 .flush = paravirt_nop,
443 .mmu.lazy_mode = {
444 .enter = paravirt_nop,
445 .leave = paravirt_nop,
446 .flush = paravirt_nop,
470 },
471
447 },
448
472 .set_fixmap = native_set_fixmap,
449 .mmu.set_fixmap = native_set_fixmap,
450
451#if defined(CONFIG_PARAVIRT_SPINLOCKS)
452 /* Lock ops. */
453#ifdef CONFIG_SMP
454 .lock.queued_spin_lock_slowpath = native_queued_spin_lock_slowpath,
455 .lock.queued_spin_unlock =
456 PV_CALLEE_SAVE(__native_queued_spin_unlock),
457 .lock.wait = paravirt_nop,
458 .lock.kick = paravirt_nop,
459 .lock.vcpu_is_preempted =
460 PV_CALLEE_SAVE(__native_vcpu_is_preempted),
461#endif /* SMP */
462#endif
473};
474
463};
464
475EXPORT_SYMBOL_GPL(pv_time_ops);
476EXPORT_SYMBOL (pv_cpu_ops);
477EXPORT_SYMBOL (pv_mmu_ops);
465/* At this point, native_get/set_debugreg has real function entries */
466NOKPROBE_SYMBOL(native_get_debugreg);
467NOKPROBE_SYMBOL(native_set_debugreg);
468NOKPROBE_SYMBOL(native_load_idt);
469
470EXPORT_SYMBOL_GPL(pv_ops);
478EXPORT_SYMBOL_GPL(pv_info);
471EXPORT_SYMBOL_GPL(pv_info);
479EXPORT_SYMBOL (pv_irq_ops);