kvm.c revision 5202397df819d3c5a3f201bd4af6b86542115fb6 
1/*
2 * KVM paravirt_ops implementation
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
17 *
18 * Copyright (C) 2007, Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
19 * Copyright IBM Corporation, 2007
20 *   Authors: Anthony Liguori <aliguori@us.ibm.com>
21 */
22
23#include <linux/module.h>
24#include <linux/kernel.h>
25#include <linux/kvm_para.h>
26#include <linux/cpu.h>
27#include <linux/mm.h>
28#include <linux/highmem.h>
29#include <linux/hardirq.h>
30#include <linux/notifier.h>
31#include <linux/reboot.h>
32#include <linux/hash.h>
33#include <linux/sched.h>
34#include <linux/slab.h>
35#include <linux/kprobes.h>
36#include <asm/timer.h>
37#include <asm/cpu.h>
38#include <asm/traps.h>
39#include <asm/desc.h>
40#include <asm/tlbflush.h>
41
42static int kvmapf = 1;
43
44static int parse_no_kvmapf(char *arg)
45{
46        kvmapf = 0;
47        return 0;
48}
49
50early_param("no-kvmapf", parse_no_kvmapf);
51
52static int steal_acc = 1;
53static int parse_no_stealacc(char *arg)
54{
55        steal_acc = 0;
56        return 0;
57}
58
59early_param("no-steal-acc", parse_no_stealacc);
60
61static DEFINE_PER_CPU(struct kvm_vcpu_pv_apf_data, apf_reason) __aligned(64);
62static DEFINE_PER_CPU(struct kvm_steal_time, steal_time) __aligned(64);
63static int has_steal_clock = 0;
64
65/*
66 * No need for any "IO delay" on KVM
67 */
68static void kvm_io_delay(void)
69{
70}
71
72#define KVM_TASK_SLEEP_HASHBITS 8
73#define KVM_TASK_SLEEP_HASHSIZE (1<<KVM_TASK_SLEEP_HASHBITS)
74
75struct kvm_task_sleep_node {
76	struct hlist_node link;
77	wait_queue_head_t wq;
78	u32 token;
79	int cpu;
80	bool halted;
81	struct mm_struct *mm;
82};
83
84static struct kvm_task_sleep_head {
85	spinlock_t lock;
86	struct hlist_head list;
87} async_pf_sleepers[KVM_TASK_SLEEP_HASHSIZE];
88
89static struct kvm_task_sleep_node *_find_apf_task(struct kvm_task_sleep_head *b,
90						  u32 token)
91{
92	struct hlist_node *p;
93
94	hlist_for_each(p, &b->list) {
95		struct kvm_task_sleep_node *n =
96			hlist_entry(p, typeof(*n), link);
97		if (n->token == token)
98			return n;
99	}
100
101	return NULL;
102}
103
104void kvm_async_pf_task_wait(u32 token)
105{
106	u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS);
107	struct kvm_task_sleep_head *b = &async_pf_sleepers[key];
108	struct kvm_task_sleep_node n, *e;
109	DEFINE_WAIT(wait);
110	int cpu, idle;
111
112	cpu = get_cpu();
113	idle = idle_cpu(cpu);
114	put_cpu();
115
116	spin_lock(&b->lock);
117	e = _find_apf_task(b, token);
118	if (e) {
119		/* dummy entry exist -> wake up was delivered ahead of PF */
120		hlist_del(&e->link);
121		kfree(e);
122		spin_unlock(&b->lock);
123		return;
124	}
125
126	n.token = token;
127	n.cpu = smp_processor_id();
128	n.mm = current->active_mm;
129	n.halted = idle || preempt_count() > 1;
130	atomic_inc(&n.mm->mm_count);
131	init_waitqueue_head(&n.wq);
132	hlist_add_head(&n.link, &b->list);
133	spin_unlock(&b->lock);
134
135	for (;;) {
136		if (!n.halted)
137			prepare_to_wait(&n.wq, &wait, TASK_UNINTERRUPTIBLE);
138		if (hlist_unhashed(&n.link))
139			break;
140
141		if (!n.halted) {
142			local_irq_enable();
143			schedule();
144			local_irq_disable();
145		} else {
146			/*
147			 * We cannot reschedule. So halt.
148			 */
149			native_safe_halt();
150			local_irq_disable();
151		}
152	}
153	if (!n.halted)
154		finish_wait(&n.wq, &wait);
155
156	return;
157}
158EXPORT_SYMBOL_GPL(kvm_async_pf_task_wait);
159
160static void apf_task_wake_one(struct kvm_task_sleep_node *n)
161{
162	hlist_del_init(&n->link);
163	if (!n->mm)
164		return;
165	mmdrop(n->mm);
166	if (n->halted)
167		smp_send_reschedule(n->cpu);
168	else if (waitqueue_active(&n->wq))
169		wake_up(&n->wq);
170}
171
172static void apf_task_wake_all(void)
173{
174	int i;
175
176	for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++) {
177		struct hlist_node *p, *next;
178		struct kvm_task_sleep_head *b = &async_pf_sleepers[i];
179		spin_lock(&b->lock);
180		hlist_for_each_safe(p, next, &b->list) {
181			struct kvm_task_sleep_node *n =
182				hlist_entry(p, typeof(*n), link);
183			if (n->cpu == smp_processor_id())
184				apf_task_wake_one(n);
185		}
186		spin_unlock(&b->lock);
187	}
188}
189
190void kvm_async_pf_task_wake(u32 token)
191{
192	u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS);
193	struct kvm_task_sleep_head *b = &async_pf_sleepers[key];
194	struct kvm_task_sleep_node *n;
195
196	if (token == ~0) {
197		apf_task_wake_all();
198		return;
199	}
200
201again:
202	spin_lock(&b->lock);
203	n = _find_apf_task(b, token);
204	if (!n) {
205		/*
206		 * async PF was not yet handled.
207		 * Add dummy entry for the token.
208		 */
209		n = kmalloc(sizeof(*n), GFP_ATOMIC);
210		if (!n) {
211			/*
212			 * Allocation failed! Busy wait while other cpu
213			 * handles async PF.
214			 */
215			spin_unlock(&b->lock);
216			cpu_relax();
217			goto again;
218		}
219		n->token = token;
220		n->cpu = smp_processor_id();
221		n->mm = NULL;
222		init_waitqueue_head(&n->wq);
223		hlist_add_head(&n->link, &b->list);
224	} else
225		apf_task_wake_one(n);
226	spin_unlock(&b->lock);
227	return;
228}
229EXPORT_SYMBOL_GPL(kvm_async_pf_task_wake);
230
231u32 kvm_read_and_reset_pf_reason(void)
232{
233	u32 reason = 0;
234
235	if (__get_cpu_var(apf_reason).enabled) {
236		reason = __get_cpu_var(apf_reason).reason;
237		__get_cpu_var(apf_reason).reason = 0;
238	}
239
240	return reason;
241}
242EXPORT_SYMBOL_GPL(kvm_read_and_reset_pf_reason);
243
244dotraplinkage void __kprobes
245do_async_page_fault(struct pt_regs *regs, unsigned long error_code)
246{
247	switch (kvm_read_and_reset_pf_reason()) {
248	default:
249		do_page_fault(regs, error_code);
250		break;
251	case KVM_PV_REASON_PAGE_NOT_PRESENT:
252		/* page is swapped out by the host. */
253		kvm_async_pf_task_wait((u32)read_cr2());
254		break;
255	case KVM_PV_REASON_PAGE_READY:
256		kvm_async_pf_task_wake((u32)read_cr2());
257		break;
258	}
259}
260
261static void __init paravirt_ops_setup(void)
262{
263	pv_info.name = "KVM";
264	pv_info.paravirt_enabled = 1;
265
266	if (kvm_para_has_feature(KVM_FEATURE_NOP_IO_DELAY))
267		pv_cpu_ops.io_delay = kvm_io_delay;
268
269#ifdef CONFIG_X86_IO_APIC
270	no_timer_check = 1;
271#endif
272}
273
274static void kvm_register_steal_time(void)
275{
276	int cpu = smp_processor_id();
277	struct kvm_steal_time *st = &per_cpu(steal_time, cpu);
278
279	if (!has_steal_clock)
280		return;
281
282	memset(st, 0, sizeof(*st));
283
284	wrmsrl(MSR_KVM_STEAL_TIME, (__pa(st) | KVM_MSR_ENABLED));
285	printk(KERN_INFO "kvm-stealtime: cpu %d, msr %lx\n",
286		cpu, __pa(st));
287}
288
289void __cpuinit kvm_guest_cpu_init(void)
290{
291	if (!kvm_para_available())
292		return;
293
294	if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF) && kvmapf) {
295		u64 pa = __pa(&__get_cpu_var(apf_reason));
296
297#ifdef CONFIG_PREEMPT
298		pa |= KVM_ASYNC_PF_SEND_ALWAYS;
299#endif
300		wrmsrl(MSR_KVM_ASYNC_PF_EN, pa | KVM_ASYNC_PF_ENABLED);
301		__get_cpu_var(apf_reason).enabled = 1;
302		printk(KERN_INFO"KVM setup async PF for cpu %d\n",
303		       smp_processor_id());
304	}
305
306	if (has_steal_clock)
307		kvm_register_steal_time();
308}
309
310static void kvm_pv_disable_apf(void *unused)
311{
312	if (!__get_cpu_var(apf_reason).enabled)
313		return;
314
315	wrmsrl(MSR_KVM_ASYNC_PF_EN, 0);
316	__get_cpu_var(apf_reason).enabled = 0;
317
318	printk(KERN_INFO"Unregister pv shared memory for cpu %d\n",
319	       smp_processor_id());
320}
321
322static int kvm_pv_reboot_notify(struct notifier_block *nb,
323				unsigned long code, void *unused)
324{
325	if (code == SYS_RESTART)
326		on_each_cpu(kvm_pv_disable_apf, NULL, 1);
327	return NOTIFY_DONE;
328}
329
330static struct notifier_block kvm_pv_reboot_nb = {
331	.notifier_call = kvm_pv_reboot_notify,
332};
333
334static u64 kvm_steal_clock(int cpu)
335{
336	u64 steal;
337	struct kvm_steal_time *src;
338	int version;
339
340	src = &per_cpu(steal_time, cpu);
341	do {
342		version = src->version;
343		rmb();
344		steal = src->steal;
345		rmb();
346	} while ((version & 1) || (version != src->version));
347
348	return steal;
349}
350
351void kvm_disable_steal_time(void)
352{
353	if (!has_steal_clock)
354		return;
355
356	wrmsr(MSR_KVM_STEAL_TIME, 0, 0);
357}
358
359#ifdef CONFIG_SMP
360static void __init kvm_smp_prepare_boot_cpu(void)
361{
362#ifdef CONFIG_KVM_CLOCK
363	WARN_ON(kvm_register_clock("primary cpu clock"));
364#endif
365	kvm_guest_cpu_init();
366	native_smp_prepare_boot_cpu();
367}
368
369static void __cpuinit kvm_guest_cpu_online(void *dummy)
370{
371	kvm_guest_cpu_init();
372}
373
374static void kvm_guest_cpu_offline(void *dummy)
375{
376	kvm_disable_steal_time();
377	kvm_pv_disable_apf(NULL);
378	apf_task_wake_all();
379}
380
381static int __cpuinit kvm_cpu_notify(struct notifier_block *self,
382				    unsigned long action, void *hcpu)
383{
384	int cpu = (unsigned long)hcpu;
385	switch (action) {
386	case CPU_ONLINE:
387	case CPU_DOWN_FAILED:
388	case CPU_ONLINE_FROZEN:
389		smp_call_function_single(cpu, kvm_guest_cpu_online, NULL, 0);
390		break;
391	case CPU_DOWN_PREPARE:
392	case CPU_DOWN_PREPARE_FROZEN:
393		smp_call_function_single(cpu, kvm_guest_cpu_offline, NULL, 1);
394		break;
395	default:
396		break;
397	}
398	return NOTIFY_OK;
399}
400
401static struct notifier_block __cpuinitdata kvm_cpu_notifier = {
402        .notifier_call  = kvm_cpu_notify,
403};
404#endif
405
406static void __init kvm_apf_trap_init(void)
407{
408	set_intr_gate(14, &async_page_fault);
409}
410
411void __init kvm_guest_init(void)
412{
413	int i;
414
415	if (!kvm_para_available())
416		return;
417
418	paravirt_ops_setup();
419	register_reboot_notifier(&kvm_pv_reboot_nb);
420	for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++)
421		spin_lock_init(&async_pf_sleepers[i].lock);
422	if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF))
423		x86_init.irqs.trap_init = kvm_apf_trap_init;
424
425	if (kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) {
426		has_steal_clock = 1;
427		pv_time_ops.steal_clock = kvm_steal_clock;
428	}
429
430#ifdef CONFIG_SMP
431	smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu;
432	register_cpu_notifier(&kvm_cpu_notifier);
433#else
434	kvm_guest_cpu_init();
435#endif
436}
437
438static __init int activate_jump_labels(void)
439{
440	if (has_steal_clock) {
441		jump_label_inc(&paravirt_steal_enabled);
442		if (steal_acc)
443			jump_label_inc(&paravirt_steal_rq_enabled);
444	}
445
446	return 0;
447}
448arch_initcall(activate_jump_labels);
449