1/* SMP TLB support routines. 2 * 3 * Copyright (C) 2006-2008 Panasonic Corporation 4 * All Rights Reserved. 5 * 6 * This program is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU General Public License 8 * version 2 as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 */ 15#include <linux/interrupt.h> 16#include <linux/spinlock.h> 17#include <linux/init.h> 18#include <linux/jiffies.h> 19#include <linux/cpumask.h> 20#include <linux/err.h> 21#include <linux/kernel.h> 22#include <linux/delay.h> 23#include <linux/sched.h> 24#include <linux/profile.h> 25#include <linux/smp.h> 26#include <asm/tlbflush.h> 27#include <asm/bitops.h> 28#include <asm/processor.h> 29#include <asm/bug.h> 30#include <asm/exceptions.h> 31#include <asm/hardirq.h> 32#include <asm/fpu.h> 33#include <asm/mmu_context.h> 34#include <asm/thread_info.h> 35#include <asm/cpu-regs.h> 36#include <asm/intctl-regs.h> 37 38/* 39 * For flush TLB 40 */ 41#define FLUSH_ALL 0xffffffff 42 43static cpumask_t flush_cpumask; 44static struct mm_struct *flush_mm; 45static unsigned long flush_va; 46static DEFINE_SPINLOCK(tlbstate_lock); 47 48DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate) = { 49 &init_mm, 0 50}; 51 52static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, 53 unsigned long va); 54static void do_flush_tlb_all(void *info); 55 56/** 57 * smp_flush_tlb - Callback to invalidate the TLB. 58 * @unused: Callback context (ignored). 59 */ 60void smp_flush_tlb(void *unused) 61{ 62 unsigned long cpu_id; 63 64 cpu_id = get_cpu(); 65 66 if (!cpumask_test_cpu(cpu_id, &flush_cpumask)) 67 /* This was a BUG() but until someone can quote me the line 68 * from the intel manual that guarantees an IPI to multiple 69 * CPUs is retried _only_ on the erroring CPUs its staying as a 70 * return 71 * 72 * BUG(); 73 */ 74 goto out; 75 76 if (flush_va == FLUSH_ALL) 77 local_flush_tlb(); 78 else 79 local_flush_tlb_page(flush_mm, flush_va); 80 81 smp_mb__before_clear_bit(); 82 cpumask_clear_cpu(cpu_id, &flush_cpumask); 83 smp_mb__after_clear_bit(); 84out: 85 put_cpu(); 86} 87 88/** 89 * flush_tlb_others - Tell the specified CPUs to invalidate their TLBs 90 * @cpumask: The list of CPUs to target. 91 * @mm: The VM context to flush from (if va!=FLUSH_ALL). 92 * @va: Virtual address to flush or FLUSH_ALL to flush everything. 93 */ 94static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, 95 unsigned long va) 96{ 97 cpumask_t tmp; 98 99 /* A couple of sanity checks (to be removed): 100 * - mask must not be empty 101 * - current CPU must not be in mask 102 * - we do not send IPIs to as-yet unbooted CPUs. 103 */ 104 BUG_ON(!mm); 105 BUG_ON(cpumask_empty(&cpumask)); 106 BUG_ON(cpumask_test_cpu(smp_processor_id(), &cpumask)); 107 108 cpumask_and(&tmp, &cpumask, cpu_online_mask); 109 BUG_ON(!cpumask_equal(&cpumask, &tmp)); 110 111 /* I'm not happy about this global shared spinlock in the MM hot path, 112 * but we'll see how contended it is. 113 * 114 * Temporarily this turns IRQs off, so that lockups are detected by the 115 * NMI watchdog. 116 */ 117 spin_lock(&tlbstate_lock); 118 119 flush_mm = mm; 120 flush_va = va; 121#if NR_CPUS <= BITS_PER_LONG 122 atomic_set_mask(cpumask.bits[0], &flush_cpumask.bits[0]); 123#else 124#error Not supported. 125#endif 126 127 /* FIXME: if NR_CPUS>=3, change send_IPI_mask */ 128 smp_call_function(smp_flush_tlb, NULL, 1); 129 130 while (!cpumask_empty(&flush_cpumask)) 131 /* Lockup detection does not belong here */ 132 smp_mb(); 133 134 flush_mm = NULL; 135 flush_va = 0; 136 spin_unlock(&tlbstate_lock); 137} 138 139/** 140 * flush_tlb_mm - Invalidate TLB of specified VM context 141 * @mm: The VM context to invalidate. 142 */ 143void flush_tlb_mm(struct mm_struct *mm) 144{ 145 cpumask_t cpu_mask; 146 147 preempt_disable(); 148 cpumask_copy(&cpu_mask, mm_cpumask(mm)); 149 cpumask_clear_cpu(smp_processor_id(), &cpu_mask); 150 151 local_flush_tlb(); 152 if (!cpumask_empty(&cpu_mask)) 153 flush_tlb_others(cpu_mask, mm, FLUSH_ALL); 154 155 preempt_enable(); 156} 157 158/** 159 * flush_tlb_current_task - Invalidate TLB of current task 160 */ 161void flush_tlb_current_task(void) 162{ 163 struct mm_struct *mm = current->mm; 164 cpumask_t cpu_mask; 165 166 preempt_disable(); 167 cpumask_copy(&cpu_mask, mm_cpumask(mm)); 168 cpumask_clear_cpu(smp_processor_id(), &cpu_mask); 169 170 local_flush_tlb(); 171 if (!cpumask_empty(&cpu_mask)) 172 flush_tlb_others(cpu_mask, mm, FLUSH_ALL); 173 174 preempt_enable(); 175} 176 177/** 178 * flush_tlb_page - Invalidate TLB of page 179 * @vma: The VM context to invalidate the page for. 180 * @va: The virtual address of the page to invalidate. 181 */ 182void flush_tlb_page(struct vm_area_struct *vma, unsigned long va) 183{ 184 struct mm_struct *mm = vma->vm_mm; 185 cpumask_t cpu_mask; 186 187 preempt_disable(); 188 cpumask_copy(&cpu_mask, mm_cpumask(mm)); 189 cpumask_clear_cpu(smp_processor_id(), &cpu_mask); 190 191 local_flush_tlb_page(mm, va); 192 if (!cpumask_empty(&cpu_mask)) 193 flush_tlb_others(cpu_mask, mm, va); 194 195 preempt_enable(); 196} 197 198/** 199 * do_flush_tlb_all - Callback to completely invalidate a TLB 200 * @unused: Callback context (ignored). 201 */ 202static void do_flush_tlb_all(void *unused) 203{ 204 local_flush_tlb_all(); 205} 206 207/** 208 * flush_tlb_all - Completely invalidate TLBs on all CPUs 209 */ 210void flush_tlb_all(void) 211{ 212 on_each_cpu(do_flush_tlb_all, 0, 1); 213} 214