paravirt.c revision 365d46dc9be9b3c833990a06f3994b1987eda578
1/* Paravirtualization interfaces 2 Copyright (C) 2006 Rusty Russell IBM Corporation 3 4 This program is free software; you can redistribute it and/or modify 5 it under the terms of the GNU General Public License as published by 6 the Free Software Foundation; either version 2 of the License, or 7 (at your option) any later version. 8 9 This program is distributed in the hope that it will be useful, 10 but WITHOUT ANY WARRANTY; without even the implied warranty of 11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 GNU General Public License for more details. 13 14 You should have received a copy of the GNU General Public License 15 along with this program; if not, write to the Free Software 16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 17 18 2007 - x86_64 support added by Glauber de Oliveira Costa, Red Hat Inc 19*/ 20 21#include <linux/errno.h> 22#include <linux/module.h> 23#include <linux/efi.h> 24#include <linux/bcd.h> 25#include <linux/highmem.h> 26 27#include <asm/bug.h> 28#include <asm/paravirt.h> 29#include <asm/desc.h> 30#include <asm/setup.h> 31#include <asm/arch_hooks.h> 32#include <asm/pgtable.h> 33#include <asm/time.h> 34#include <asm/pgalloc.h> 35#include <asm/irq.h> 36#include <asm/delay.h> 37#include <asm/fixmap.h> 38#include <asm/apic.h> 39#include <asm/tlbflush.h> 40#include <asm/timer.h> 41 42/* nop stub */ 43void _paravirt_nop(void) 44{ 45} 46 47static void __init default_banner(void) 48{ 49 printk(KERN_INFO "Booting paravirtualized kernel on %s\n", 50 pv_info.name); 51} 52 53char *memory_setup(void) 54{ 55 return pv_init_ops.memory_setup(); 56} 57 58/* Simple instruction patching code. */ 59#define DEF_NATIVE(ops, name, code) \ 60 extern const char start_##ops##_##name[], end_##ops##_##name[]; \ 61 asm("start_" #ops "_" #name ": " code "; end_" #ops "_" #name ":") 62 63/* Undefined instruction for dealing with missing ops pointers. */ 64static const unsigned char ud2a[] = { 0x0f, 0x0b }; 65 66unsigned paravirt_patch_nop(void) 67{ 68 return 0; 69} 70 71unsigned paravirt_patch_ignore(unsigned len) 72{ 73 return len; 74} 75 76struct branch { 77 unsigned char opcode; 78 u32 delta; 79} __attribute__((packed)); 80 81unsigned paravirt_patch_call(void *insnbuf, 82 const void *target, u16 tgt_clobbers, 83 unsigned long addr, u16 site_clobbers, 84 unsigned len) 85{ 86 struct branch *b = insnbuf; 87 unsigned long delta = (unsigned long)target - (addr+5); 88 89 if (tgt_clobbers & ~site_clobbers) 90 return len; /* target would clobber too much for this site */ 91 if (len < 5) 92 return len; /* call too long for patch site */ 93 94 b->opcode = 0xe8; /* call */ 95 b->delta = delta; 96 BUILD_BUG_ON(sizeof(*b) != 5); 97 98 return 5; 99} 100 101unsigned paravirt_patch_jmp(void *insnbuf, const void *target, 102 unsigned long addr, unsigned len) 103{ 104 struct branch *b = insnbuf; 105 unsigned long delta = (unsigned long)target - (addr+5); 106 107 if (len < 5) 108 return len; /* call too long for patch site */ 109 110 b->opcode = 0xe9; /* jmp */ 111 b->delta = delta; 112 113 return 5; 114} 115 116/* Neat trick to map patch type back to the call within the 117 * corresponding structure. */ 118static void *get_call_destination(u8 type) 119{ 120 struct paravirt_patch_template tmpl = { 121 .pv_init_ops = pv_init_ops, 122 .pv_time_ops = pv_time_ops, 123 .pv_cpu_ops = pv_cpu_ops, 124 .pv_irq_ops = pv_irq_ops, 125 .pv_apic_ops = pv_apic_ops, 126 .pv_mmu_ops = pv_mmu_ops, 127 .pv_lock_ops = pv_lock_ops, 128 }; 129 return *((void **)&tmpl + type); 130} 131 132unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf, 133 unsigned long addr, unsigned len) 134{ 135 void *opfunc = get_call_destination(type); 136 unsigned ret; 137 138 if (opfunc == NULL) 139 /* If there's no function, patch it with a ud2a (BUG) */ 140 ret = paravirt_patch_insns(insnbuf, len, ud2a, ud2a+sizeof(ud2a)); 141 else if (opfunc == paravirt_nop) 142 /* If the operation is a nop, then nop the callsite */ 143 ret = paravirt_patch_nop(); 144 else if (type == PARAVIRT_PATCH(pv_cpu_ops.iret) || 145 type == PARAVIRT_PATCH(pv_cpu_ops.irq_enable_sysexit) || 146 type == PARAVIRT_PATCH(pv_cpu_ops.usergs_sysret32) || 147 type == PARAVIRT_PATCH(pv_cpu_ops.usergs_sysret64)) 148 /* If operation requires a jmp, then jmp */ 149 ret = paravirt_patch_jmp(insnbuf, opfunc, addr, len); 150 else 151 /* Otherwise call the function; assume target could 152 clobber any caller-save reg */ 153 ret = paravirt_patch_call(insnbuf, opfunc, CLBR_ANY, 154 addr, clobbers, len); 155 156 return ret; 157} 158 159unsigned paravirt_patch_insns(void *insnbuf, unsigned len, 160 const char *start, const char *end) 161{ 162 unsigned insn_len = end - start; 163 164 if (insn_len > len || start == NULL) 165 insn_len = len; 166 else 167 memcpy(insnbuf, start, insn_len); 168 169 return insn_len; 170} 171 172void init_IRQ(void) 173{ 174 pv_irq_ops.init_IRQ(); 175} 176 177static void native_flush_tlb(void) 178{ 179 __native_flush_tlb(); 180} 181 182/* 183 * Global pages have to be flushed a bit differently. Not a real 184 * performance problem because this does not happen often. 185 */ 186static void native_flush_tlb_global(void) 187{ 188 __native_flush_tlb_global(); 189} 190 191static void native_flush_tlb_single(unsigned long addr) 192{ 193 __native_flush_tlb_single(addr); 194} 195 196/* These are in entry.S */ 197extern void native_iret(void); 198extern void native_irq_enable_sysexit(void); 199extern void native_usergs_sysret32(void); 200extern void native_usergs_sysret64(void); 201 202static int __init print_banner(void) 203{ 204 pv_init_ops.banner(); 205 return 0; 206} 207core_initcall(print_banner); 208 209static struct resource reserve_ioports = { 210 .start = 0, 211 .end = IO_SPACE_LIMIT, 212 .name = "paravirt-ioport", 213 .flags = IORESOURCE_IO | IORESOURCE_BUSY, 214}; 215 216/* 217 * Reserve the whole legacy IO space to prevent any legacy drivers 218 * from wasting time probing for their hardware. This is a fairly 219 * brute-force approach to disabling all non-virtual drivers. 220 * 221 * Note that this must be called very early to have any effect. 222 */ 223int paravirt_disable_iospace(void) 224{ 225 return request_resource(&ioport_resource, &reserve_ioports); 226} 227 228static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAZY_NONE; 229 230static inline void enter_lazy(enum paravirt_lazy_mode mode) 231{ 232 BUG_ON(__get_cpu_var(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE); 233 BUG_ON(preemptible()); 234 235 __get_cpu_var(paravirt_lazy_mode) = mode; 236} 237 238void paravirt_leave_lazy(enum paravirt_lazy_mode mode) 239{ 240 BUG_ON(__get_cpu_var(paravirt_lazy_mode) != mode); 241 BUG_ON(preemptible()); 242 243 __get_cpu_var(paravirt_lazy_mode) = PARAVIRT_LAZY_NONE; 244} 245 246void paravirt_enter_lazy_mmu(void) 247{ 248 enter_lazy(PARAVIRT_LAZY_MMU); 249} 250 251void paravirt_leave_lazy_mmu(void) 252{ 253 paravirt_leave_lazy(PARAVIRT_LAZY_MMU); 254} 255 256void paravirt_enter_lazy_cpu(void) 257{ 258 enter_lazy(PARAVIRT_LAZY_CPU); 259} 260 261void paravirt_leave_lazy_cpu(void) 262{ 263 paravirt_leave_lazy(PARAVIRT_LAZY_CPU); 264} 265 266enum paravirt_lazy_mode paravirt_get_lazy_mode(void) 267{ 268 return __get_cpu_var(paravirt_lazy_mode); 269} 270 271struct pv_info pv_info = { 272 .name = "bare hardware", 273 .paravirt_enabled = 0, 274 .kernel_rpl = 0, 275 .shared_kernel_pmd = 1, /* Only used when CONFIG_X86_PAE is set */ 276}; 277 278struct pv_init_ops pv_init_ops = { 279 .patch = native_patch, 280 .banner = default_banner, 281 .arch_setup = paravirt_nop, 282 .memory_setup = machine_specific_memory_setup, 283}; 284 285struct pv_time_ops pv_time_ops = { 286 .time_init = hpet_time_init, 287 .get_wallclock = native_get_wallclock, 288 .set_wallclock = native_set_wallclock, 289 .sched_clock = native_sched_clock, 290 .get_tsc_khz = native_calibrate_tsc, 291}; 292 293struct pv_irq_ops pv_irq_ops = { 294 .init_IRQ = native_init_IRQ, 295 .save_fl = native_save_fl, 296 .restore_fl = native_restore_fl, 297 .irq_disable = native_irq_disable, 298 .irq_enable = native_irq_enable, 299 .safe_halt = native_safe_halt, 300 .halt = native_halt, 301#ifdef CONFIG_X86_64 302 .adjust_exception_frame = paravirt_nop, 303#endif 304}; 305 306struct pv_cpu_ops pv_cpu_ops = { 307 .cpuid = native_cpuid, 308 .get_debugreg = native_get_debugreg, 309 .set_debugreg = native_set_debugreg, 310 .clts = native_clts, 311 .read_cr0 = native_read_cr0, 312 .write_cr0 = native_write_cr0, 313 .read_cr4 = native_read_cr4, 314 .read_cr4_safe = native_read_cr4_safe, 315 .write_cr4 = native_write_cr4, 316#ifdef CONFIG_X86_64 317 .read_cr8 = native_read_cr8, 318 .write_cr8 = native_write_cr8, 319#endif 320 .wbinvd = native_wbinvd, 321 .read_msr = native_read_msr_safe, 322 .read_msr_amd = native_read_msr_amd_safe, 323 .write_msr = native_write_msr_safe, 324 .read_tsc = native_read_tsc, 325 .read_pmc = native_read_pmc, 326 .read_tscp = native_read_tscp, 327 .load_tr_desc = native_load_tr_desc, 328 .set_ldt = native_set_ldt, 329 .load_gdt = native_load_gdt, 330 .load_idt = native_load_idt, 331 .store_gdt = native_store_gdt, 332 .store_idt = native_store_idt, 333 .store_tr = native_store_tr, 334 .load_tls = native_load_tls, 335#ifdef CONFIG_X86_64 336 .load_gs_index = native_load_gs_index, 337#endif 338 .write_ldt_entry = native_write_ldt_entry, 339 .write_gdt_entry = native_write_gdt_entry, 340 .write_idt_entry = native_write_idt_entry, 341 342 .alloc_ldt = paravirt_nop, 343 .free_ldt = paravirt_nop, 344 345 .load_sp0 = native_load_sp0, 346 347#if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION) 348 .irq_enable_sysexit = native_irq_enable_sysexit, 349#endif 350#ifdef CONFIG_X86_64 351#ifdef CONFIG_IA32_EMULATION 352 .usergs_sysret32 = native_usergs_sysret32, 353#endif 354 .usergs_sysret64 = native_usergs_sysret64, 355#endif 356 .iret = native_iret, 357 .swapgs = native_swapgs, 358 359 .set_iopl_mask = native_set_iopl_mask, 360 .io_delay = native_io_delay, 361 362 .lazy_mode = { 363 .enter = paravirt_nop, 364 .leave = paravirt_nop, 365 }, 366}; 367 368struct pv_apic_ops pv_apic_ops = { 369#ifdef CONFIG_X86_LOCAL_APIC 370 .setup_boot_clock = setup_boot_APIC_clock, 371 .setup_secondary_clock = setup_secondary_APIC_clock, 372 .startup_ipi_hook = paravirt_nop, 373#endif 374}; 375 376struct pv_mmu_ops pv_mmu_ops = { 377#ifndef CONFIG_X86_64 378 .pagetable_setup_start = native_pagetable_setup_start, 379 .pagetable_setup_done = native_pagetable_setup_done, 380#else 381 .pagetable_setup_start = paravirt_nop, 382 .pagetable_setup_done = paravirt_nop, 383#endif 384 385 .read_cr2 = native_read_cr2, 386 .write_cr2 = native_write_cr2, 387 .read_cr3 = native_read_cr3, 388 .write_cr3 = native_write_cr3, 389 390 .flush_tlb_user = native_flush_tlb, 391 .flush_tlb_kernel = native_flush_tlb_global, 392 .flush_tlb_single = native_flush_tlb_single, 393 .flush_tlb_others = native_flush_tlb_others, 394 395 .pgd_alloc = __paravirt_pgd_alloc, 396 .pgd_free = paravirt_nop, 397 398 .alloc_pte = paravirt_nop, 399 .alloc_pmd = paravirt_nop, 400 .alloc_pmd_clone = paravirt_nop, 401 .alloc_pud = paravirt_nop, 402 .release_pte = paravirt_nop, 403 .release_pmd = paravirt_nop, 404 .release_pud = paravirt_nop, 405 406 .set_pte = native_set_pte, 407 .set_pte_at = native_set_pte_at, 408 .set_pmd = native_set_pmd, 409 .pte_update = paravirt_nop, 410 .pte_update_defer = paravirt_nop, 411 412 .ptep_modify_prot_start = __ptep_modify_prot_start, 413 .ptep_modify_prot_commit = __ptep_modify_prot_commit, 414 415#ifdef CONFIG_HIGHPTE 416 .kmap_atomic_pte = kmap_atomic, 417#endif 418 419#if PAGETABLE_LEVELS >= 3 420#ifdef CONFIG_X86_PAE 421 .set_pte_atomic = native_set_pte_atomic, 422 .set_pte_present = native_set_pte_present, 423 .pte_clear = native_pte_clear, 424 .pmd_clear = native_pmd_clear, 425#endif 426 .set_pud = native_set_pud, 427 .pmd_val = native_pmd_val, 428 .make_pmd = native_make_pmd, 429 430#if PAGETABLE_LEVELS == 4 431 .pud_val = native_pud_val, 432 .make_pud = native_make_pud, 433 .set_pgd = native_set_pgd, 434#endif 435#endif /* PAGETABLE_LEVELS >= 3 */ 436 437 .pte_val = native_pte_val, 438 .pte_flags = native_pte_flags, 439 .pgd_val = native_pgd_val, 440 441 .make_pte = native_make_pte, 442 .make_pgd = native_make_pgd, 443 444 .dup_mmap = paravirt_nop, 445 .exit_mmap = paravirt_nop, 446 .activate_mm = paravirt_nop, 447 448 .lazy_mode = { 449 .enter = paravirt_nop, 450 .leave = paravirt_nop, 451 }, 452 453 .set_fixmap = native_set_fixmap, 454}; 455 456EXPORT_SYMBOL_GPL(pv_time_ops); 457EXPORT_SYMBOL (pv_cpu_ops); 458EXPORT_SYMBOL (pv_mmu_ops); 459EXPORT_SYMBOL_GPL(pv_apic_ops); 460EXPORT_SYMBOL_GPL(pv_info); 461EXPORT_SYMBOL (pv_irq_ops); 462