tlb-r4k.c revision a5e696e5d0f1377ff6beb10d2f40edb6a3d1de18
1/* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) 7 * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org 8 * Carsten Langgaard, carstenl@mips.com 9 * Copyright (C) 2002 MIPS Technologies, Inc. All rights reserved. 10 */ 11#include <linux/init.h> 12#include <linux/sched.h> 13#include <linux/mm.h> 14 15#include <asm/cpu.h> 16#include <asm/bootinfo.h> 17#include <asm/mmu_context.h> 18#include <asm/pgtable.h> 19#include <asm/system.h> 20 21extern void build_tlb_refill_handler(void); 22 23/* 24 * Make sure all entries differ. If they're not different 25 * MIPS32 will take revenge ... 26 */ 27#define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1))) 28 29/* Atomicity and interruptability */ 30#ifdef CONFIG_MIPS_MT_SMTC 31 32#include <asm/smtc.h> 33#include <asm/mipsmtregs.h> 34 35#define ENTER_CRITICAL(flags) \ 36 { \ 37 unsigned int mvpflags; \ 38 local_irq_save(flags);\ 39 mvpflags = dvpe() 40#define EXIT_CRITICAL(flags) \ 41 evpe(mvpflags); \ 42 local_irq_restore(flags); \ 43 } 44#else 45 46#define ENTER_CRITICAL(flags) local_irq_save(flags) 47#define EXIT_CRITICAL(flags) local_irq_restore(flags) 48 49#endif /* CONFIG_MIPS_MT_SMTC */ 50 51#if defined(CONFIG_CPU_LOONGSON2) 52/* 53 * LOONGSON2 has a 4 entry itlb which is a subset of dtlb, 54 * unfortrunately, itlb is not totally transparent to software. 55 */ 56#define FLUSH_ITLB write_c0_diag(4); 57 58#define FLUSH_ITLB_VM(vma) { if ((vma)->vm_flags & VM_EXEC) write_c0_diag(4); } 59 60#else 61 62#define FLUSH_ITLB 63#define FLUSH_ITLB_VM(vma) 64 65#endif 66 67void local_flush_tlb_all(void) 68{ 69 unsigned long flags; 70 unsigned long old_ctx; 71 int entry; 72 73 ENTER_CRITICAL(flags); 74 /* Save old context and create impossible VPN2 value */ 75 old_ctx = read_c0_entryhi(); 76 write_c0_entrylo0(0); 77 write_c0_entrylo1(0); 78 79 entry = read_c0_wired(); 80 81 /* Blast 'em all away. */ 82 while (entry < current_cpu_data.tlbsize) { 83 /* Make sure all entries differ. */ 84 write_c0_entryhi(UNIQUE_ENTRYHI(entry)); 85 write_c0_index(entry); 86 mtc0_tlbw_hazard(); 87 tlb_write_indexed(); 88 entry++; 89 } 90 tlbw_use_hazard(); 91 write_c0_entryhi(old_ctx); 92 FLUSH_ITLB; 93 EXIT_CRITICAL(flags); 94} 95 96/* All entries common to a mm share an asid. To effectively flush 97 these entries, we just bump the asid. */ 98void local_flush_tlb_mm(struct mm_struct *mm) 99{ 100 int cpu; 101 102 preempt_disable(); 103 104 cpu = smp_processor_id(); 105 106 if (cpu_context(cpu, mm) != 0) { 107 drop_mmu_context(mm, cpu); 108 } 109 110 preempt_enable(); 111} 112 113void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 114 unsigned long end) 115{ 116 struct mm_struct *mm = vma->vm_mm; 117 int cpu = smp_processor_id(); 118 119 if (cpu_context(cpu, mm) != 0) { 120 unsigned long size, flags; 121 122 ENTER_CRITICAL(flags); 123 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 124 size = (size + 1) >> 1; 125 if (size <= current_cpu_data.tlbsize/2) { 126 int oldpid = read_c0_entryhi(); 127 int newpid = cpu_asid(cpu, mm); 128 129 start &= (PAGE_MASK << 1); 130 end += ((PAGE_SIZE << 1) - 1); 131 end &= (PAGE_MASK << 1); 132 while (start < end) { 133 int idx; 134 135 write_c0_entryhi(start | newpid); 136 start += (PAGE_SIZE << 1); 137 mtc0_tlbw_hazard(); 138 tlb_probe(); 139 tlb_probe_hazard(); 140 idx = read_c0_index(); 141 write_c0_entrylo0(0); 142 write_c0_entrylo1(0); 143 if (idx < 0) 144 continue; 145 /* Make sure all entries differ. */ 146 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 147 mtc0_tlbw_hazard(); 148 tlb_write_indexed(); 149 } 150 tlbw_use_hazard(); 151 write_c0_entryhi(oldpid); 152 } else { 153 drop_mmu_context(mm, cpu); 154 } 155 FLUSH_ITLB; 156 EXIT_CRITICAL(flags); 157 } 158} 159 160void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) 161{ 162 unsigned long size, flags; 163 164 ENTER_CRITICAL(flags); 165 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 166 size = (size + 1) >> 1; 167 if (size <= current_cpu_data.tlbsize / 2) { 168 int pid = read_c0_entryhi(); 169 170 start &= (PAGE_MASK << 1); 171 end += ((PAGE_SIZE << 1) - 1); 172 end &= (PAGE_MASK << 1); 173 174 while (start < end) { 175 int idx; 176 177 write_c0_entryhi(start); 178 start += (PAGE_SIZE << 1); 179 mtc0_tlbw_hazard(); 180 tlb_probe(); 181 tlb_probe_hazard(); 182 idx = read_c0_index(); 183 write_c0_entrylo0(0); 184 write_c0_entrylo1(0); 185 if (idx < 0) 186 continue; 187 /* Make sure all entries differ. */ 188 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 189 mtc0_tlbw_hazard(); 190 tlb_write_indexed(); 191 } 192 tlbw_use_hazard(); 193 write_c0_entryhi(pid); 194 } else { 195 local_flush_tlb_all(); 196 } 197 FLUSH_ITLB; 198 EXIT_CRITICAL(flags); 199} 200 201void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) 202{ 203 int cpu = smp_processor_id(); 204 205 if (cpu_context(cpu, vma->vm_mm) != 0) { 206 unsigned long flags; 207 int oldpid, newpid, idx; 208 209 newpid = cpu_asid(cpu, vma->vm_mm); 210 page &= (PAGE_MASK << 1); 211 ENTER_CRITICAL(flags); 212 oldpid = read_c0_entryhi(); 213 write_c0_entryhi(page | newpid); 214 mtc0_tlbw_hazard(); 215 tlb_probe(); 216 tlb_probe_hazard(); 217 idx = read_c0_index(); 218 write_c0_entrylo0(0); 219 write_c0_entrylo1(0); 220 if (idx < 0) 221 goto finish; 222 /* Make sure all entries differ. */ 223 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 224 mtc0_tlbw_hazard(); 225 tlb_write_indexed(); 226 tlbw_use_hazard(); 227 228 finish: 229 write_c0_entryhi(oldpid); 230 FLUSH_ITLB_VM(vma); 231 EXIT_CRITICAL(flags); 232 } 233} 234 235/* 236 * This one is only used for pages with the global bit set so we don't care 237 * much about the ASID. 238 */ 239void local_flush_tlb_one(unsigned long page) 240{ 241 unsigned long flags; 242 int oldpid, idx; 243 244 ENTER_CRITICAL(flags); 245 oldpid = read_c0_entryhi(); 246 page &= (PAGE_MASK << 1); 247 write_c0_entryhi(page); 248 mtc0_tlbw_hazard(); 249 tlb_probe(); 250 tlb_probe_hazard(); 251 idx = read_c0_index(); 252 write_c0_entrylo0(0); 253 write_c0_entrylo1(0); 254 if (idx >= 0) { 255 /* Make sure all entries differ. */ 256 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 257 mtc0_tlbw_hazard(); 258 tlb_write_indexed(); 259 tlbw_use_hazard(); 260 } 261 write_c0_entryhi(oldpid); 262 FLUSH_ITLB; 263 EXIT_CRITICAL(flags); 264} 265 266/* 267 * We will need multiple versions of update_mmu_cache(), one that just 268 * updates the TLB with the new pte(s), and another which also checks 269 * for the R4k "end of page" hardware bug and does the needy. 270 */ 271void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte) 272{ 273 unsigned long flags; 274 pgd_t *pgdp; 275 pud_t *pudp; 276 pmd_t *pmdp; 277 pte_t *ptep; 278 int idx, pid; 279 280 /* 281 * Handle debugger faulting in for debugee. 282 */ 283 if (current->active_mm != vma->vm_mm) 284 return; 285 286 ENTER_CRITICAL(flags); 287 288 pid = read_c0_entryhi() & ASID_MASK; 289 address &= (PAGE_MASK << 1); 290 write_c0_entryhi(address | pid); 291 pgdp = pgd_offset(vma->vm_mm, address); 292 mtc0_tlbw_hazard(); 293 tlb_probe(); 294 tlb_probe_hazard(); 295 pudp = pud_offset(pgdp, address); 296 pmdp = pmd_offset(pudp, address); 297 idx = read_c0_index(); 298 ptep = pte_offset_map(pmdp, address); 299 300#if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32) 301 write_c0_entrylo0(ptep->pte_high); 302 ptep++; 303 write_c0_entrylo1(ptep->pte_high); 304#else 305 write_c0_entrylo0(pte_val(*ptep++) >> 6); 306 write_c0_entrylo1(pte_val(*ptep) >> 6); 307#endif 308 mtc0_tlbw_hazard(); 309 if (idx < 0) 310 tlb_write_random(); 311 else 312 tlb_write_indexed(); 313 tlbw_use_hazard(); 314 FLUSH_ITLB_VM(vma); 315 EXIT_CRITICAL(flags); 316} 317 318#if 0 319static void r4k_update_mmu_cache_hwbug(struct vm_area_struct * vma, 320 unsigned long address, pte_t pte) 321{ 322 unsigned long flags; 323 unsigned int asid; 324 pgd_t *pgdp; 325 pmd_t *pmdp; 326 pte_t *ptep; 327 int idx; 328 329 ENTER_CRITICAL(flags); 330 address &= (PAGE_MASK << 1); 331 asid = read_c0_entryhi() & ASID_MASK; 332 write_c0_entryhi(address | asid); 333 pgdp = pgd_offset(vma->vm_mm, address); 334 mtc0_tlbw_hazard(); 335 tlb_probe(); 336 tlb_probe_hazard(); 337 pmdp = pmd_offset(pgdp, address); 338 idx = read_c0_index(); 339 ptep = pte_offset_map(pmdp, address); 340 write_c0_entrylo0(pte_val(*ptep++) >> 6); 341 write_c0_entrylo1(pte_val(*ptep) >> 6); 342 mtc0_tlbw_hazard(); 343 if (idx < 0) 344 tlb_write_random(); 345 else 346 tlb_write_indexed(); 347 tlbw_use_hazard(); 348 EXIT_CRITICAL(flags); 349} 350#endif 351 352void __init add_wired_entry(unsigned long entrylo0, unsigned long entrylo1, 353 unsigned long entryhi, unsigned long pagemask) 354{ 355 unsigned long flags; 356 unsigned long wired; 357 unsigned long old_pagemask; 358 unsigned long old_ctx; 359 360 ENTER_CRITICAL(flags); 361 /* Save old context and create impossible VPN2 value */ 362 old_ctx = read_c0_entryhi(); 363 old_pagemask = read_c0_pagemask(); 364 wired = read_c0_wired(); 365 write_c0_wired(wired + 1); 366 write_c0_index(wired); 367 tlbw_use_hazard(); /* What is the hazard here? */ 368 write_c0_pagemask(pagemask); 369 write_c0_entryhi(entryhi); 370 write_c0_entrylo0(entrylo0); 371 write_c0_entrylo1(entrylo1); 372 mtc0_tlbw_hazard(); 373 tlb_write_indexed(); 374 tlbw_use_hazard(); 375 376 write_c0_entryhi(old_ctx); 377 tlbw_use_hazard(); /* What is the hazard here? */ 378 write_c0_pagemask(old_pagemask); 379 local_flush_tlb_all(); 380 EXIT_CRITICAL(flags); 381} 382 383/* 384 * Used for loading TLB entries before trap_init() has started, when we 385 * don't actually want to add a wired entry which remains throughout the 386 * lifetime of the system 387 */ 388 389static int temp_tlb_entry __cpuinitdata; 390 391__init int add_temporary_entry(unsigned long entrylo0, unsigned long entrylo1, 392 unsigned long entryhi, unsigned long pagemask) 393{ 394 int ret = 0; 395 unsigned long flags; 396 unsigned long wired; 397 unsigned long old_pagemask; 398 unsigned long old_ctx; 399 400 ENTER_CRITICAL(flags); 401 /* Save old context and create impossible VPN2 value */ 402 old_ctx = read_c0_entryhi(); 403 old_pagemask = read_c0_pagemask(); 404 wired = read_c0_wired(); 405 if (--temp_tlb_entry < wired) { 406 printk(KERN_WARNING 407 "No TLB space left for add_temporary_entry\n"); 408 ret = -ENOSPC; 409 goto out; 410 } 411 412 write_c0_index(temp_tlb_entry); 413 write_c0_pagemask(pagemask); 414 write_c0_entryhi(entryhi); 415 write_c0_entrylo0(entrylo0); 416 write_c0_entrylo1(entrylo1); 417 mtc0_tlbw_hazard(); 418 tlb_write_indexed(); 419 tlbw_use_hazard(); 420 421 write_c0_entryhi(old_ctx); 422 write_c0_pagemask(old_pagemask); 423out: 424 EXIT_CRITICAL(flags); 425 return ret; 426} 427 428static void __cpuinit probe_tlb(unsigned long config) 429{ 430 struct cpuinfo_mips *c = ¤t_cpu_data; 431 unsigned int reg; 432 433 /* 434 * If this isn't a MIPS32 / MIPS64 compliant CPU. Config 1 register 435 * is not supported, we assume R4k style. Cpu probing already figured 436 * out the number of tlb entries. 437 */ 438 if ((c->processor_id & 0xff0000) == PRID_COMP_LEGACY) 439 return; 440#ifdef CONFIG_MIPS_MT_SMTC 441 /* 442 * If TLB is shared in SMTC system, total size already 443 * has been calculated and written into cpu_data tlbsize 444 */ 445 if((smtc_status & SMTC_TLB_SHARED) == SMTC_TLB_SHARED) 446 return; 447#endif /* CONFIG_MIPS_MT_SMTC */ 448 449 reg = read_c0_config1(); 450 if (!((config >> 7) & 3)) 451 panic("No TLB present"); 452 453 c->tlbsize = ((reg >> 25) & 0x3f) + 1; 454} 455 456static int __cpuinitdata ntlb = 0; 457static int __init set_ntlb(char *str) 458{ 459 get_option(&str, &ntlb); 460 return 1; 461} 462 463__setup("ntlb=", set_ntlb); 464 465void __cpuinit tlb_init(void) 466{ 467 unsigned int config = read_c0_config(); 468 469 /* 470 * You should never change this register: 471 * - On R4600 1.7 the tlbp never hits for pages smaller than 472 * the value in the c0_pagemask register. 473 * - The entire mm handling assumes the c0_pagemask register to 474 * be set to fixed-size pages. 475 */ 476 probe_tlb(config); 477 write_c0_pagemask(PM_DEFAULT_MASK); 478 write_c0_wired(0); 479 if (current_cpu_type() == CPU_R10000 || 480 current_cpu_type() == CPU_R12000 || 481 current_cpu_type() == CPU_R14000) 482 write_c0_framemask(0); 483 temp_tlb_entry = current_cpu_data.tlbsize - 1; 484 485 /* From this point on the ARC firmware is dead. */ 486 local_flush_tlb_all(); 487 488 /* Did I tell you that ARC SUCKS? */ 489 490 if (ntlb) { 491 if (ntlb > 1 && ntlb <= current_cpu_data.tlbsize) { 492 int wired = current_cpu_data.tlbsize - ntlb; 493 write_c0_wired(wired); 494 write_c0_index(wired-1); 495 printk("Restricting TLB to %d entries\n", ntlb); 496 } else 497 printk("Ignoring invalid argument ntlb=%d\n", ntlb); 498 } 499 500 build_tlb_refill_handler(); 501} 502