1/* 2 * Xen SMP support 3 * 4 * This file implements the Xen versions of smp_ops. SMP under Xen is 5 * very straightforward. Bringing a CPU up is simply a matter of 6 * loading its initial context and setting it running. 7 * 8 * IPIs are handled through the Xen event mechanism. 9 * 10 * Because virtual CPUs can be scheduled onto any real CPU, there's no 11 * useful topology information for the kernel to make use of. As a 12 * result, all CPUs are treated as if they're single-core and 13 * single-threaded. 14 */ 15#include <linux/sched.h> 16#include <linux/err.h> 17#include <linux/slab.h> 18#include <linux/smp.h> 19 20#include <asm/paravirt.h> 21#include <asm/desc.h> 22#include <asm/pgtable.h> 23#include <asm/cpu.h> 24 25#include <xen/interface/xen.h> 26#include <xen/interface/vcpu.h> 27 28#include <asm/xen/interface.h> 29#include <asm/xen/hypercall.h> 30 31#include <xen/xen.h> 32#include <xen/page.h> 33#include <xen/events.h> 34 35#include <xen/hvc-console.h> 36#include "xen-ops.h" 37#include "mmu.h" 38 39cpumask_var_t xen_cpu_initialized_map; 40 41static DEFINE_PER_CPU(int, xen_resched_irq); 42static DEFINE_PER_CPU(int, xen_callfunc_irq); 43static DEFINE_PER_CPU(int, xen_callfuncsingle_irq); 44static DEFINE_PER_CPU(int, xen_debug_irq) = -1; 45 46static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id); 47static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id); 48 49/* 50 * Reschedule call back. 51 */ 52static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id) 53{ 54 inc_irq_stat(irq_resched_count); 55 scheduler_ipi(); 56 57 return IRQ_HANDLED; 58} 59 60static void __cpuinit cpu_bringup(void) 61{ 62 int cpu; 63 64 cpu_init(); 65 touch_softlockup_watchdog(); 66 preempt_disable(); 67 68 xen_enable_sysenter(); 69 xen_enable_syscall(); 70 71 cpu = smp_processor_id(); 72 smp_store_cpu_info(cpu); 73 cpu_data(cpu).x86_max_cores = 1; 74 set_cpu_sibling_map(cpu); 75 76 xen_setup_cpu_clockevents(); 77 78 notify_cpu_starting(cpu); 79 80 ipi_call_lock(); 81 set_cpu_online(cpu, true); 82 ipi_call_unlock(); 83 84 this_cpu_write(cpu_state, CPU_ONLINE); 85 86 wmb(); 87 88 /* We can take interrupts now: we're officially "up". */ 89 local_irq_enable(); 90 91 wmb(); /* make sure everything is out */ 92} 93 94static void __cpuinit cpu_bringup_and_idle(void) 95{ 96 cpu_bringup(); 97 cpu_idle(); 98} 99 100static int xen_smp_intr_init(unsigned int cpu) 101{ 102 int rc; 103 const char *resched_name, *callfunc_name, *debug_name; 104 105 resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu); 106 rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR, 107 cpu, 108 xen_reschedule_interrupt, 109 IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING, 110 resched_name, 111 NULL); 112 if (rc < 0) 113 goto fail; 114 per_cpu(xen_resched_irq, cpu) = rc; 115 116 callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu); 117 rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR, 118 cpu, 119 xen_call_function_interrupt, 120 IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING, 121 callfunc_name, 122 NULL); 123 if (rc < 0) 124 goto fail; 125 per_cpu(xen_callfunc_irq, cpu) = rc; 126 127 debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu); 128 rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu, xen_debug_interrupt, 129 IRQF_DISABLED | IRQF_PERCPU | IRQF_NOBALANCING, 130 debug_name, NULL); 131 if (rc < 0) 132 goto fail; 133 per_cpu(xen_debug_irq, cpu) = rc; 134 135 callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu); 136 rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR, 137 cpu, 138 xen_call_function_single_interrupt, 139 IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING, 140 callfunc_name, 141 NULL); 142 if (rc < 0) 143 goto fail; 144 per_cpu(xen_callfuncsingle_irq, cpu) = rc; 145 146 return 0; 147 148 fail: 149 if (per_cpu(xen_resched_irq, cpu) >= 0) 150 unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu), NULL); 151 if (per_cpu(xen_callfunc_irq, cpu) >= 0) 152 unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu), NULL); 153 if (per_cpu(xen_debug_irq, cpu) >= 0) 154 unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu), NULL); 155 if (per_cpu(xen_callfuncsingle_irq, cpu) >= 0) 156 unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu), 157 NULL); 158 159 return rc; 160} 161 162static void __init xen_fill_possible_map(void) 163{ 164 int i, rc; 165 166 if (xen_initial_domain()) 167 return; 168 169 for (i = 0; i < nr_cpu_ids; i++) { 170 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL); 171 if (rc >= 0) { 172 num_processors++; 173 set_cpu_possible(i, true); 174 } 175 } 176} 177 178static void __init xen_filter_cpu_maps(void) 179{ 180 int i, rc; 181 unsigned int subtract = 0; 182 183 if (!xen_initial_domain()) 184 return; 185 186 num_processors = 0; 187 disabled_cpus = 0; 188 for (i = 0; i < nr_cpu_ids; i++) { 189 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL); 190 if (rc >= 0) { 191 num_processors++; 192 set_cpu_possible(i, true); 193 } else { 194 set_cpu_possible(i, false); 195 set_cpu_present(i, false); 196 subtract++; 197 } 198 } 199#ifdef CONFIG_HOTPLUG_CPU 200 /* This is akin to using 'nr_cpus' on the Linux command line. 201 * Which is OK as when we use 'dom0_max_vcpus=X' we can only 202 * have up to X, while nr_cpu_ids is greater than X. This 203 * normally is not a problem, except when CPU hotplugging 204 * is involved and then there might be more than X CPUs 205 * in the guest - which will not work as there is no 206 * hypercall to expand the max number of VCPUs an already 207 * running guest has. So cap it up to X. */ 208 if (subtract) 209 nr_cpu_ids = nr_cpu_ids - subtract; 210#endif 211 212} 213 214static void __init xen_smp_prepare_boot_cpu(void) 215{ 216 BUG_ON(smp_processor_id() != 0); 217 native_smp_prepare_boot_cpu(); 218 219 /* We've switched to the "real" per-cpu gdt, so make sure the 220 old memory can be recycled */ 221 make_lowmem_page_readwrite(xen_initial_gdt); 222 223 xen_filter_cpu_maps(); 224 xen_setup_vcpu_info_placement(); 225} 226 227static void __init xen_smp_prepare_cpus(unsigned int max_cpus) 228{ 229 unsigned cpu; 230 unsigned int i; 231 232 if (skip_ioapic_setup) { 233 char *m = (max_cpus == 0) ? 234 "The nosmp parameter is incompatible with Xen; " \ 235 "use Xen dom0_max_vcpus=1 parameter" : 236 "The noapic parameter is incompatible with Xen"; 237 238 xen_raw_printk(m); 239 panic(m); 240 } 241 xen_init_lock_cpu(0); 242 243 smp_store_cpu_info(0); 244 cpu_data(0).x86_max_cores = 1; 245 246 for_each_possible_cpu(i) { 247 zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL); 248 zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL); 249 zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL); 250 } 251 set_cpu_sibling_map(0); 252 253 if (xen_smp_intr_init(0)) 254 BUG(); 255 256 if (!alloc_cpumask_var(&xen_cpu_initialized_map, GFP_KERNEL)) 257 panic("could not allocate xen_cpu_initialized_map\n"); 258 259 cpumask_copy(xen_cpu_initialized_map, cpumask_of(0)); 260 261 /* Restrict the possible_map according to max_cpus. */ 262 while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) { 263 for (cpu = nr_cpu_ids - 1; !cpu_possible(cpu); cpu--) 264 continue; 265 set_cpu_possible(cpu, false); 266 } 267 268 for_each_possible_cpu (cpu) { 269 struct task_struct *idle; 270 271 if (cpu == 0) 272 continue; 273 274 idle = fork_idle(cpu); 275 if (IS_ERR(idle)) 276 panic("failed fork for CPU %d", cpu); 277 278 set_cpu_present(cpu, true); 279 } 280} 281 282static int __cpuinit 283cpu_initialize_context(unsigned int cpu, struct task_struct *idle) 284{ 285 struct vcpu_guest_context *ctxt; 286 struct desc_struct *gdt; 287 unsigned long gdt_mfn; 288 289 if (cpumask_test_and_set_cpu(cpu, xen_cpu_initialized_map)) 290 return 0; 291 292 ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL); 293 if (ctxt == NULL) 294 return -ENOMEM; 295 296 gdt = get_cpu_gdt_table(cpu); 297 298 ctxt->flags = VGCF_IN_KERNEL; 299 ctxt->user_regs.ds = __USER_DS; 300 ctxt->user_regs.es = __USER_DS; 301 ctxt->user_regs.ss = __KERNEL_DS; 302#ifdef CONFIG_X86_32 303 ctxt->user_regs.fs = __KERNEL_PERCPU; 304 ctxt->user_regs.gs = __KERNEL_STACK_CANARY; 305#else 306 ctxt->gs_base_kernel = per_cpu_offset(cpu); 307#endif 308 ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle; 309 ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */ 310 311 memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt)); 312 313 xen_copy_trap_info(ctxt->trap_ctxt); 314 315 ctxt->ldt_ents = 0; 316 317 BUG_ON((unsigned long)gdt & ~PAGE_MASK); 318 319 gdt_mfn = arbitrary_virt_to_mfn(gdt); 320 make_lowmem_page_readonly(gdt); 321 make_lowmem_page_readonly(mfn_to_virt(gdt_mfn)); 322 323 ctxt->gdt_frames[0] = gdt_mfn; 324 ctxt->gdt_ents = GDT_ENTRIES; 325 326 ctxt->user_regs.cs = __KERNEL_CS; 327 ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs); 328 329 ctxt->kernel_ss = __KERNEL_DS; 330 ctxt->kernel_sp = idle->thread.sp0; 331 332#ifdef CONFIG_X86_32 333 ctxt->event_callback_cs = __KERNEL_CS; 334 ctxt->failsafe_callback_cs = __KERNEL_CS; 335#endif 336 ctxt->event_callback_eip = (unsigned long)xen_hypervisor_callback; 337 ctxt->failsafe_callback_eip = (unsigned long)xen_failsafe_callback; 338 339 per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir); 340 ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_mfn(swapper_pg_dir)); 341 342 if (HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, ctxt)) 343 BUG(); 344 345 kfree(ctxt); 346 return 0; 347} 348 349static int __cpuinit xen_cpu_up(unsigned int cpu) 350{ 351 struct task_struct *idle = idle_task(cpu); 352 int rc; 353 354 per_cpu(current_task, cpu) = idle; 355#ifdef CONFIG_X86_32 356 irq_ctx_init(cpu); 357#else 358 clear_tsk_thread_flag(idle, TIF_FORK); 359 per_cpu(kernel_stack, cpu) = 360 (unsigned long)task_stack_page(idle) - 361 KERNEL_STACK_OFFSET + THREAD_SIZE; 362#endif 363 xen_setup_runstate_info(cpu); 364 xen_setup_timer(cpu); 365 xen_init_lock_cpu(cpu); 366 367 per_cpu(cpu_state, cpu) = CPU_UP_PREPARE; 368 369 /* make sure interrupts start blocked */ 370 per_cpu(xen_vcpu, cpu)->evtchn_upcall_mask = 1; 371 372 rc = cpu_initialize_context(cpu, idle); 373 if (rc) 374 return rc; 375 376 if (num_online_cpus() == 1) 377 alternatives_smp_switch(1); 378 379 rc = xen_smp_intr_init(cpu); 380 if (rc) 381 return rc; 382 383 rc = HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL); 384 BUG_ON(rc); 385 386 while(per_cpu(cpu_state, cpu) != CPU_ONLINE) { 387 HYPERVISOR_sched_op(SCHEDOP_yield, NULL); 388 barrier(); 389 } 390 391 return 0; 392} 393 394static void xen_smp_cpus_done(unsigned int max_cpus) 395{ 396} 397 398#ifdef CONFIG_HOTPLUG_CPU 399static int xen_cpu_disable(void) 400{ 401 unsigned int cpu = smp_processor_id(); 402 if (cpu == 0) 403 return -EBUSY; 404 405 cpu_disable_common(); 406 407 load_cr3(swapper_pg_dir); 408 return 0; 409} 410 411static void xen_cpu_die(unsigned int cpu) 412{ 413 while (HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) { 414 current->state = TASK_UNINTERRUPTIBLE; 415 schedule_timeout(HZ/10); 416 } 417 unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu), NULL); 418 unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu), NULL); 419 unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu), NULL); 420 unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu), NULL); 421 xen_uninit_lock_cpu(cpu); 422 xen_teardown_timer(cpu); 423 424 if (num_online_cpus() == 1) 425 alternatives_smp_switch(0); 426} 427 428static void __cpuinit xen_play_dead(void) /* used only with HOTPLUG_CPU */ 429{ 430 play_dead_common(); 431 HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL); 432 cpu_bringup(); 433 /* 434 * Balance out the preempt calls - as we are running in cpu_idle 435 * loop which has been called at bootup from cpu_bringup_and_idle. 436 * The cpucpu_bringup_and_idle called cpu_bringup which made a 437 * preempt_disable() So this preempt_enable will balance it out. 438 */ 439 preempt_enable(); 440} 441 442#else /* !CONFIG_HOTPLUG_CPU */ 443static int xen_cpu_disable(void) 444{ 445 return -ENOSYS; 446} 447 448static void xen_cpu_die(unsigned int cpu) 449{ 450 BUG(); 451} 452 453static void xen_play_dead(void) 454{ 455 BUG(); 456} 457 458#endif 459static void stop_self(void *v) 460{ 461 int cpu = smp_processor_id(); 462 463 /* make sure we're not pinning something down */ 464 load_cr3(swapper_pg_dir); 465 /* should set up a minimal gdt */ 466 467 set_cpu_online(cpu, false); 468 469 HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL); 470 BUG(); 471} 472 473static void xen_stop_other_cpus(int wait) 474{ 475 smp_call_function(stop_self, NULL, wait); 476} 477 478static void xen_smp_send_reschedule(int cpu) 479{ 480 xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR); 481} 482 483static void xen_send_IPI_mask(const struct cpumask *mask, 484 enum ipi_vector vector) 485{ 486 unsigned cpu; 487 488 for_each_cpu_and(cpu, mask, cpu_online_mask) 489 xen_send_IPI_one(cpu, vector); 490} 491 492static void xen_smp_send_call_function_ipi(const struct cpumask *mask) 493{ 494 int cpu; 495 496 xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR); 497 498 /* Make sure other vcpus get a chance to run if they need to. */ 499 for_each_cpu(cpu, mask) { 500 if (xen_vcpu_stolen(cpu)) { 501 HYPERVISOR_sched_op(SCHEDOP_yield, NULL); 502 break; 503 } 504 } 505} 506 507static void xen_smp_send_call_function_single_ipi(int cpu) 508{ 509 xen_send_IPI_mask(cpumask_of(cpu), 510 XEN_CALL_FUNCTION_SINGLE_VECTOR); 511} 512 513static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id) 514{ 515 irq_enter(); 516 generic_smp_call_function_interrupt(); 517 inc_irq_stat(irq_call_count); 518 irq_exit(); 519 520 return IRQ_HANDLED; 521} 522 523static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id) 524{ 525 irq_enter(); 526 generic_smp_call_function_single_interrupt(); 527 inc_irq_stat(irq_call_count); 528 irq_exit(); 529 530 return IRQ_HANDLED; 531} 532 533static const struct smp_ops xen_smp_ops __initconst = { 534 .smp_prepare_boot_cpu = xen_smp_prepare_boot_cpu, 535 .smp_prepare_cpus = xen_smp_prepare_cpus, 536 .smp_cpus_done = xen_smp_cpus_done, 537 538 .cpu_up = xen_cpu_up, 539 .cpu_die = xen_cpu_die, 540 .cpu_disable = xen_cpu_disable, 541 .play_dead = xen_play_dead, 542 543 .stop_other_cpus = xen_stop_other_cpus, 544 .smp_send_reschedule = xen_smp_send_reschedule, 545 546 .send_call_func_ipi = xen_smp_send_call_function_ipi, 547 .send_call_func_single_ipi = xen_smp_send_call_function_single_ipi, 548}; 549 550void __init xen_smp_init(void) 551{ 552 smp_ops = xen_smp_ops; 553 xen_fill_possible_map(); 554 xen_init_spinlocks(); 555} 556 557static void __init xen_hvm_smp_prepare_cpus(unsigned int max_cpus) 558{ 559 native_smp_prepare_cpus(max_cpus); 560 WARN_ON(xen_smp_intr_init(0)); 561 562 xen_init_lock_cpu(0); 563} 564 565static int __cpuinit xen_hvm_cpu_up(unsigned int cpu) 566{ 567 int rc; 568 rc = native_cpu_up(cpu); 569 WARN_ON (xen_smp_intr_init(cpu)); 570 return rc; 571} 572 573static void xen_hvm_cpu_die(unsigned int cpu) 574{ 575 unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu), NULL); 576 unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu), NULL); 577 unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu), NULL); 578 unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu), NULL); 579 native_cpu_die(cpu); 580} 581 582void __init xen_hvm_smp_init(void) 583{ 584 if (!xen_have_vector_callback) 585 return; 586 smp_ops.smp_prepare_cpus = xen_hvm_smp_prepare_cpus; 587 smp_ops.smp_send_reschedule = xen_smp_send_reschedule; 588 smp_ops.cpu_up = xen_hvm_cpu_up; 589 smp_ops.cpu_die = xen_hvm_cpu_die; 590 smp_ops.send_call_func_ipi = xen_smp_send_call_function_ipi; 591 smp_ops.send_call_func_single_ipi = xen_smp_send_call_function_single_ipi; 592} 593