1/* 2 * Copyright 2010 Tilera Corporation. All Rights Reserved. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation, version 2. 7 * 8 * This program is distributed in the hope that it will be useful, but 9 * WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 11 * NON INFRINGEMENT. See the GNU General Public License for 12 * more details. 13 * 14 * From i386 code copyright (C) 1995 Linus Torvalds 15 */ 16 17#include <linux/signal.h> 18#include <linux/sched.h> 19#include <linux/kernel.h> 20#include <linux/errno.h> 21#include <linux/string.h> 22#include <linux/types.h> 23#include <linux/ptrace.h> 24#include <linux/mman.h> 25#include <linux/mm.h> 26#include <linux/smp.h> 27#include <linux/interrupt.h> 28#include <linux/init.h> 29#include <linux/tty.h> 30#include <linux/vt_kern.h> /* For unblank_screen() */ 31#include <linux/highmem.h> 32#include <linux/module.h> 33#include <linux/kprobes.h> 34#include <linux/hugetlb.h> 35#include <linux/syscalls.h> 36#include <linux/uaccess.h> 37#include <linux/kdebug.h> 38 39#include <asm/pgalloc.h> 40#include <asm/sections.h> 41#include <asm/traps.h> 42#include <asm/syscalls.h> 43 44#include <arch/interrupts.h> 45 46static noinline void force_sig_info_fault(const char *type, int si_signo, 47 int si_code, unsigned long address, 48 int fault_num, 49 struct task_struct *tsk, 50 struct pt_regs *regs) 51{ 52 siginfo_t info; 53 54 if (unlikely(tsk->pid < 2)) { 55 panic("Signal %d (code %d) at %#lx sent to %s!", 56 si_signo, si_code & 0xffff, address, 57 is_idle_task(tsk) ? "the idle task" : "init"); 58 } 59 60 info.si_signo = si_signo; 61 info.si_errno = 0; 62 info.si_code = si_code; 63 info.si_addr = (void __user *)address; 64 info.si_trapno = fault_num; 65 trace_unhandled_signal(type, regs, address, si_signo); 66 force_sig_info(si_signo, &info, tsk); 67} 68 69#ifndef __tilegx__ 70/* 71 * Synthesize the fault a PL0 process would get by doing a word-load of 72 * an unaligned address or a high kernel address. 73 */ 74SYSCALL_DEFINE1(cmpxchg_badaddr, unsigned long, address) 75{ 76 struct pt_regs *regs = current_pt_regs(); 77 78 if (address >= PAGE_OFFSET) 79 force_sig_info_fault("atomic segfault", SIGSEGV, SEGV_MAPERR, 80 address, INT_DTLB_MISS, current, regs); 81 else 82 force_sig_info_fault("atomic alignment fault", SIGBUS, 83 BUS_ADRALN, address, 84 INT_UNALIGN_DATA, current, regs); 85 86 /* 87 * Adjust pc to point at the actual instruction, which is unusual 88 * for syscalls normally, but is appropriate when we are claiming 89 * that a syscall swint1 caused a page fault or bus error. 90 */ 91 regs->pc -= 8; 92 93 /* 94 * Mark this as a caller-save interrupt, like a normal page fault, 95 * so that when we go through the signal handler path we will 96 * properly restore r0, r1, and r2 for the signal handler arguments. 97 */ 98 regs->flags |= PT_FLAGS_CALLER_SAVES; 99 100 return 0; 101} 102#endif 103 104static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address) 105{ 106 unsigned index = pgd_index(address); 107 pgd_t *pgd_k; 108 pud_t *pud, *pud_k; 109 pmd_t *pmd, *pmd_k; 110 111 pgd += index; 112 pgd_k = init_mm.pgd + index; 113 114 if (!pgd_present(*pgd_k)) 115 return NULL; 116 117 pud = pud_offset(pgd, address); 118 pud_k = pud_offset(pgd_k, address); 119 if (!pud_present(*pud_k)) 120 return NULL; 121 122 pmd = pmd_offset(pud, address); 123 pmd_k = pmd_offset(pud_k, address); 124 if (!pmd_present(*pmd_k)) 125 return NULL; 126 if (!pmd_present(*pmd)) 127 set_pmd(pmd, *pmd_k); 128 else 129 BUG_ON(pmd_ptfn(*pmd) != pmd_ptfn(*pmd_k)); 130 return pmd_k; 131} 132 133/* 134 * Handle a fault on the vmalloc area. 135 */ 136static inline int vmalloc_fault(pgd_t *pgd, unsigned long address) 137{ 138 pmd_t *pmd_k; 139 pte_t *pte_k; 140 141 /* Make sure we are in vmalloc area */ 142 if (!(address >= VMALLOC_START && address < VMALLOC_END)) 143 return -1; 144 145 /* 146 * Synchronize this task's top level page-table 147 * with the 'reference' page table. 148 */ 149 pmd_k = vmalloc_sync_one(pgd, address); 150 if (!pmd_k) 151 return -1; 152 pte_k = pte_offset_kernel(pmd_k, address); 153 if (!pte_present(*pte_k)) 154 return -1; 155 return 0; 156} 157 158/* Wait until this PTE has completed migration. */ 159static void wait_for_migration(pte_t *pte) 160{ 161 if (pte_migrating(*pte)) { 162 /* 163 * Wait until the migrater fixes up this pte. 164 * We scale the loop count by the clock rate so we'll wait for 165 * a few seconds here. 166 */ 167 int retries = 0; 168 int bound = get_clock_rate(); 169 while (pte_migrating(*pte)) { 170 barrier(); 171 if (++retries > bound) 172 panic("Hit migrating PTE (%#llx) and" 173 " page PFN %#lx still migrating", 174 pte->val, pte_pfn(*pte)); 175 } 176 } 177} 178 179/* 180 * It's not generally safe to use "current" to get the page table pointer, 181 * since we might be running an oprofile interrupt in the middle of a 182 * task switch. 183 */ 184static pgd_t *get_current_pgd(void) 185{ 186 HV_Context ctx = hv_inquire_context(); 187 unsigned long pgd_pfn = ctx.page_table >> PAGE_SHIFT; 188 struct page *pgd_page = pfn_to_page(pgd_pfn); 189 BUG_ON(PageHighMem(pgd_page)); 190 return (pgd_t *) __va(ctx.page_table); 191} 192 193/* 194 * We can receive a page fault from a migrating PTE at any time. 195 * Handle it by just waiting until the fault resolves. 196 * 197 * It's also possible to get a migrating kernel PTE that resolves 198 * itself during the downcall from hypervisor to Linux. We just check 199 * here to see if the PTE seems valid, and if so we retry it. 200 * 201 * NOTE! We MUST NOT take any locks for this case. We may be in an 202 * interrupt or a critical region, and must do as little as possible. 203 * Similarly, we can't use atomic ops here, since we may be handling a 204 * fault caused by an atomic op access. 205 * 206 * If we find a migrating PTE while we're in an NMI context, and we're 207 * at a PC that has a registered exception handler, we don't wait, 208 * since this thread may (e.g.) have been interrupted while migrating 209 * its own stack, which would then cause us to self-deadlock. 210 */ 211static int handle_migrating_pte(pgd_t *pgd, int fault_num, 212 unsigned long address, unsigned long pc, 213 int is_kernel_mode, int write) 214{ 215 pud_t *pud; 216 pmd_t *pmd; 217 pte_t *pte; 218 pte_t pteval; 219 220 if (pgd_addr_invalid(address)) 221 return 0; 222 223 pgd += pgd_index(address); 224 pud = pud_offset(pgd, address); 225 if (!pud || !pud_present(*pud)) 226 return 0; 227 pmd = pmd_offset(pud, address); 228 if (!pmd || !pmd_present(*pmd)) 229 return 0; 230 pte = pmd_huge_page(*pmd) ? ((pte_t *)pmd) : 231 pte_offset_kernel(pmd, address); 232 pteval = *pte; 233 if (pte_migrating(pteval)) { 234 if (in_nmi() && search_exception_tables(pc)) 235 return 0; 236 wait_for_migration(pte); 237 return 1; 238 } 239 240 if (!is_kernel_mode || !pte_present(pteval)) 241 return 0; 242 if (fault_num == INT_ITLB_MISS) { 243 if (pte_exec(pteval)) 244 return 1; 245 } else if (write) { 246 if (pte_write(pteval)) 247 return 1; 248 } else { 249 if (pte_read(pteval)) 250 return 1; 251 } 252 253 return 0; 254} 255 256/* 257 * This routine is responsible for faulting in user pages. 258 * It passes the work off to one of the appropriate routines. 259 * It returns true if the fault was successfully handled. 260 */ 261static int handle_page_fault(struct pt_regs *regs, 262 int fault_num, 263 int is_page_fault, 264 unsigned long address, 265 int write) 266{ 267 struct task_struct *tsk; 268 struct mm_struct *mm; 269 struct vm_area_struct *vma; 270 unsigned long stack_offset; 271 int fault; 272 int si_code; 273 int is_kernel_mode; 274 pgd_t *pgd; 275 unsigned int flags; 276 277 /* on TILE, protection faults are always writes */ 278 if (!is_page_fault) 279 write = 1; 280 281 flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; 282 283 is_kernel_mode = !user_mode(regs); 284 285 tsk = validate_current(); 286 287 /* 288 * Check to see if we might be overwriting the stack, and bail 289 * out if so. The page fault code is a relatively likely 290 * place to get trapped in an infinite regress, and once we 291 * overwrite the whole stack, it becomes very hard to recover. 292 */ 293 stack_offset = stack_pointer & (THREAD_SIZE-1); 294 if (stack_offset < THREAD_SIZE / 8) { 295 pr_alert("Potential stack overrun: sp %#lx\n", 296 stack_pointer); 297 show_regs(regs); 298 pr_alert("Killing current process %d/%s\n", 299 tsk->pid, tsk->comm); 300 do_group_exit(SIGKILL); 301 } 302 303 /* 304 * Early on, we need to check for migrating PTE entries; 305 * see homecache.c. If we find a migrating PTE, we wait until 306 * the backing page claims to be done migrating, then we proceed. 307 * For kernel PTEs, we rewrite the PTE and return and retry. 308 * Otherwise, we treat the fault like a normal "no PTE" fault, 309 * rather than trying to patch up the existing PTE. 310 */ 311 pgd = get_current_pgd(); 312 if (handle_migrating_pte(pgd, fault_num, address, regs->pc, 313 is_kernel_mode, write)) 314 return 1; 315 316 si_code = SEGV_MAPERR; 317 318 /* 319 * We fault-in kernel-space virtual memory on-demand. The 320 * 'reference' page table is init_mm.pgd. 321 * 322 * NOTE! We MUST NOT take any locks for this case. We may 323 * be in an interrupt or a critical region, and should 324 * only copy the information from the master page table, 325 * nothing more. 326 * 327 * This verifies that the fault happens in kernel space 328 * and that the fault was not a protection fault. 329 */ 330 if (unlikely(address >= TASK_SIZE && 331 !is_arch_mappable_range(address, 0))) { 332 if (is_kernel_mode && is_page_fault && 333 vmalloc_fault(pgd, address) >= 0) 334 return 1; 335 /* 336 * Don't take the mm semaphore here. If we fixup a prefetch 337 * fault we could otherwise deadlock. 338 */ 339 mm = NULL; /* happy compiler */ 340 vma = NULL; 341 goto bad_area_nosemaphore; 342 } 343 344 /* 345 * If we're trying to touch user-space addresses, we must 346 * be either at PL0, or else with interrupts enabled in the 347 * kernel, so either way we can re-enable interrupts here 348 * unless we are doing atomic access to user space with 349 * interrupts disabled. 350 */ 351 if (!(regs->flags & PT_FLAGS_DISABLE_IRQ)) 352 local_irq_enable(); 353 354 mm = tsk->mm; 355 356 /* 357 * If we're in an interrupt, have no user context or are running in an 358 * atomic region then we must not take the fault. 359 */ 360 if (in_atomic() || !mm) { 361 vma = NULL; /* happy compiler */ 362 goto bad_area_nosemaphore; 363 } 364 365 if (!is_kernel_mode) 366 flags |= FAULT_FLAG_USER; 367 368 /* 369 * When running in the kernel we expect faults to occur only to 370 * addresses in user space. All other faults represent errors in the 371 * kernel and should generate an OOPS. Unfortunately, in the case of an 372 * erroneous fault occurring in a code path which already holds mmap_sem 373 * we will deadlock attempting to validate the fault against the 374 * address space. Luckily the kernel only validly references user 375 * space from well defined areas of code, which are listed in the 376 * exceptions table. 377 * 378 * As the vast majority of faults will be valid we will only perform 379 * the source reference check when there is a possibility of a deadlock. 380 * Attempt to lock the address space, if we cannot we then validate the 381 * source. If this is invalid we can skip the address space check, 382 * thus avoiding the deadlock. 383 */ 384 if (!down_read_trylock(&mm->mmap_sem)) { 385 if (is_kernel_mode && 386 !search_exception_tables(regs->pc)) { 387 vma = NULL; /* happy compiler */ 388 goto bad_area_nosemaphore; 389 } 390 391retry: 392 down_read(&mm->mmap_sem); 393 } 394 395 vma = find_vma(mm, address); 396 if (!vma) 397 goto bad_area; 398 if (vma->vm_start <= address) 399 goto good_area; 400 if (!(vma->vm_flags & VM_GROWSDOWN)) 401 goto bad_area; 402 if (regs->sp < PAGE_OFFSET) { 403 /* 404 * accessing the stack below sp is always a bug. 405 */ 406 if (address < regs->sp) 407 goto bad_area; 408 } 409 if (expand_stack(vma, address)) 410 goto bad_area; 411 412/* 413 * Ok, we have a good vm_area for this memory access, so 414 * we can handle it.. 415 */ 416good_area: 417 si_code = SEGV_ACCERR; 418 if (fault_num == INT_ITLB_MISS) { 419 if (!(vma->vm_flags & VM_EXEC)) 420 goto bad_area; 421 } else if (write) { 422#ifdef TEST_VERIFY_AREA 423 if (!is_page_fault && regs->cs == KERNEL_CS) 424 pr_err("WP fault at "REGFMT"\n", regs->eip); 425#endif 426 if (!(vma->vm_flags & VM_WRITE)) 427 goto bad_area; 428 flags |= FAULT_FLAG_WRITE; 429 } else { 430 if (!is_page_fault || !(vma->vm_flags & VM_READ)) 431 goto bad_area; 432 } 433 434 /* 435 * If for any reason at all we couldn't handle the fault, 436 * make sure we exit gracefully rather than endlessly redo 437 * the fault. 438 */ 439 fault = handle_mm_fault(mm, vma, address, flags); 440 441 if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) 442 return 0; 443 444 if (unlikely(fault & VM_FAULT_ERROR)) { 445 if (fault & VM_FAULT_OOM) 446 goto out_of_memory; 447 else if (fault & VM_FAULT_SIGBUS) 448 goto do_sigbus; 449 BUG(); 450 } 451 if (flags & FAULT_FLAG_ALLOW_RETRY) { 452 if (fault & VM_FAULT_MAJOR) 453 tsk->maj_flt++; 454 else 455 tsk->min_flt++; 456 if (fault & VM_FAULT_RETRY) { 457 flags &= ~FAULT_FLAG_ALLOW_RETRY; 458 flags |= FAULT_FLAG_TRIED; 459 460 /* 461 * No need to up_read(&mm->mmap_sem) as we would 462 * have already released it in __lock_page_or_retry 463 * in mm/filemap.c. 464 */ 465 goto retry; 466 } 467 } 468 469#if CHIP_HAS_TILE_DMA() 470 /* If this was a DMA TLB fault, restart the DMA engine. */ 471 switch (fault_num) { 472 case INT_DMATLB_MISS: 473 case INT_DMATLB_MISS_DWNCL: 474 case INT_DMATLB_ACCESS: 475 case INT_DMATLB_ACCESS_DWNCL: 476 __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__REQUEST_MASK); 477 break; 478 } 479#endif 480 481 up_read(&mm->mmap_sem); 482 return 1; 483 484/* 485 * Something tried to access memory that isn't in our memory map.. 486 * Fix it, but check if it's kernel or user first.. 487 */ 488bad_area: 489 up_read(&mm->mmap_sem); 490 491bad_area_nosemaphore: 492 /* User mode accesses just cause a SIGSEGV */ 493 if (!is_kernel_mode) { 494 /* 495 * It's possible to have interrupts off here. 496 */ 497 local_irq_enable(); 498 499 force_sig_info_fault("segfault", SIGSEGV, si_code, address, 500 fault_num, tsk, regs); 501 return 0; 502 } 503 504no_context: 505 /* Are we prepared to handle this kernel fault? */ 506 if (fixup_exception(regs)) 507 return 0; 508 509/* 510 * Oops. The kernel tried to access some bad page. We'll have to 511 * terminate things with extreme prejudice. 512 */ 513 514 bust_spinlocks(1); 515 516 /* FIXME: no lookup_address() yet */ 517#ifdef SUPPORT_LOOKUP_ADDRESS 518 if (fault_num == INT_ITLB_MISS) { 519 pte_t *pte = lookup_address(address); 520 521 if (pte && pte_present(*pte) && !pte_exec_kernel(*pte)) 522 pr_crit("kernel tried to execute" 523 " non-executable page - exploit attempt?" 524 " (uid: %d)\n", current->uid); 525 } 526#endif 527 if (address < PAGE_SIZE) 528 pr_alert("Unable to handle kernel NULL pointer dereference\n"); 529 else 530 pr_alert("Unable to handle kernel paging request\n"); 531 pr_alert(" at virtual address "REGFMT", pc "REGFMT"\n", 532 address, regs->pc); 533 534 show_regs(regs); 535 536 if (unlikely(tsk->pid < 2)) { 537 panic("Kernel page fault running %s!", 538 is_idle_task(tsk) ? "the idle task" : "init"); 539 } 540 541 /* 542 * More FIXME: we should probably copy the i386 here and 543 * implement a generic die() routine. Not today. 544 */ 545#ifdef SUPPORT_DIE 546 die("Oops", regs); 547#endif 548 bust_spinlocks(1); 549 550 do_group_exit(SIGKILL); 551 552/* 553 * We ran out of memory, or some other thing happened to us that made 554 * us unable to handle the page fault gracefully. 555 */ 556out_of_memory: 557 up_read(&mm->mmap_sem); 558 if (is_kernel_mode) 559 goto no_context; 560 pagefault_out_of_memory(); 561 return 0; 562 563do_sigbus: 564 up_read(&mm->mmap_sem); 565 566 /* Kernel mode? Handle exceptions or die */ 567 if (is_kernel_mode) 568 goto no_context; 569 570 force_sig_info_fault("bus error", SIGBUS, BUS_ADRERR, address, 571 fault_num, tsk, regs); 572 return 0; 573} 574 575#ifndef __tilegx__ 576 577/* We must release ICS before panicking or we won't get anywhere. */ 578#define ics_panic(fmt, ...) do { \ 579 __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0); \ 580 panic(fmt, __VA_ARGS__); \ 581} while (0) 582 583/* 584 * When we take an ITLB or DTLB fault or access violation in the 585 * supervisor while the critical section bit is set, the hypervisor is 586 * reluctant to write new values into the EX_CONTEXT_K_x registers, 587 * since that might indicate we have not yet squirreled the SPR 588 * contents away and can thus safely take a recursive interrupt. 589 * Accordingly, the hypervisor passes us the PC via SYSTEM_SAVE_K_2. 590 * 591 * Note that this routine is called before homecache_tlb_defer_enter(), 592 * which means that we can properly unlock any atomics that might 593 * be used there (good), but also means we must be very sensitive 594 * to not touch any data structures that might be located in memory 595 * that could migrate, as we could be entering the kernel on a dataplane 596 * cpu that has been deferring kernel TLB updates. This means, for 597 * example, that we can't migrate init_mm or its pgd. 598 */ 599struct intvec_state do_page_fault_ics(struct pt_regs *regs, int fault_num, 600 unsigned long address, 601 unsigned long info) 602{ 603 unsigned long pc = info & ~1; 604 int write = info & 1; 605 pgd_t *pgd = get_current_pgd(); 606 607 /* Retval is 1 at first since we will handle the fault fully. */ 608 struct intvec_state state = { 609 do_page_fault, fault_num, address, write, 1 610 }; 611 612 /* Validate that we are plausibly in the right routine. */ 613 if ((pc & 0x7) != 0 || pc < PAGE_OFFSET || 614 (fault_num != INT_DTLB_MISS && 615 fault_num != INT_DTLB_ACCESS)) { 616 unsigned long old_pc = regs->pc; 617 regs->pc = pc; 618 ics_panic("Bad ICS page fault args:" 619 " old PC %#lx, fault %d/%d at %#lx\n", 620 old_pc, fault_num, write, address); 621 } 622 623 /* We might be faulting on a vmalloc page, so check that first. */ 624 if (fault_num != INT_DTLB_ACCESS && vmalloc_fault(pgd, address) >= 0) 625 return state; 626 627 /* 628 * If we faulted with ICS set in sys_cmpxchg, we are providing 629 * a user syscall service that should generate a signal on 630 * fault. We didn't set up a kernel stack on initial entry to 631 * sys_cmpxchg, but instead had one set up by the fault, which 632 * (because sys_cmpxchg never releases ICS) came to us via the 633 * SYSTEM_SAVE_K_2 mechanism, and thus EX_CONTEXT_K_[01] are 634 * still referencing the original user code. We release the 635 * atomic lock and rewrite pt_regs so that it appears that we 636 * came from user-space directly, and after we finish the 637 * fault we'll go back to user space and re-issue the swint. 638 * This way the backtrace information is correct if we need to 639 * emit a stack dump at any point while handling this. 640 * 641 * Must match register use in sys_cmpxchg(). 642 */ 643 if (pc >= (unsigned long) sys_cmpxchg && 644 pc < (unsigned long) __sys_cmpxchg_end) { 645#ifdef CONFIG_SMP 646 /* Don't unlock before we could have locked. */ 647 if (pc >= (unsigned long)__sys_cmpxchg_grab_lock) { 648 int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]); 649 __atomic_fault_unlock(lock_ptr); 650 } 651#endif 652 regs->sp = regs->regs[27]; 653 } 654 655 /* 656 * We can also fault in the atomic assembly, in which 657 * case we use the exception table to do the first-level fixup. 658 * We may re-fixup again in the real fault handler if it 659 * turns out the faulting address is just bad, and not, 660 * for example, migrating. 661 */ 662 else if (pc >= (unsigned long) __start_atomic_asm_code && 663 pc < (unsigned long) __end_atomic_asm_code) { 664 const struct exception_table_entry *fixup; 665#ifdef CONFIG_SMP 666 /* Unlock the atomic lock. */ 667 int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]); 668 __atomic_fault_unlock(lock_ptr); 669#endif 670 fixup = search_exception_tables(pc); 671 if (!fixup) 672 ics_panic("ICS atomic fault not in table:" 673 " PC %#lx, fault %d", pc, fault_num); 674 regs->pc = fixup->fixup; 675 regs->ex1 = PL_ICS_EX1(KERNEL_PL, 0); 676 } 677 678 /* 679 * Now that we have released the atomic lock (if necessary), 680 * it's safe to spin if the PTE that caused the fault was migrating. 681 */ 682 if (fault_num == INT_DTLB_ACCESS) 683 write = 1; 684 if (handle_migrating_pte(pgd, fault_num, address, pc, 1, write)) 685 return state; 686 687 /* Return zero so that we continue on with normal fault handling. */ 688 state.retval = 0; 689 return state; 690} 691 692#endif /* !__tilegx__ */ 693 694/* 695 * This routine handles page faults. It determines the address, and the 696 * problem, and then passes it handle_page_fault() for normal DTLB and 697 * ITLB issues, and for DMA or SN processor faults when we are in user 698 * space. For the latter, if we're in kernel mode, we just save the 699 * interrupt away appropriately and return immediately. We can't do 700 * page faults for user code while in kernel mode. 701 */ 702void do_page_fault(struct pt_regs *regs, int fault_num, 703 unsigned long address, unsigned long write) 704{ 705 int is_page_fault; 706 707#ifdef CONFIG_KPROBES 708 /* 709 * This is to notify the fault handler of the kprobes. The 710 * exception code is redundant as it is also carried in REGS, 711 * but we pass it anyhow. 712 */ 713 if (notify_die(DIE_PAGE_FAULT, "page fault", regs, -1, 714 regs->faultnum, SIGSEGV) == NOTIFY_STOP) 715 return; 716#endif 717 718#ifdef __tilegx__ 719 /* 720 * We don't need early do_page_fault_ics() support, since unlike 721 * Pro we don't need to worry about unlocking the atomic locks. 722 * There is only one current case in GX where we touch any memory 723 * under ICS other than our own kernel stack, and we handle that 724 * here. (If we crash due to trying to touch our own stack, 725 * we're in too much trouble for C code to help out anyway.) 726 */ 727 if (write & ~1) { 728 unsigned long pc = write & ~1; 729 if (pc >= (unsigned long) __start_unalign_asm_code && 730 pc < (unsigned long) __end_unalign_asm_code) { 731 struct thread_info *ti = current_thread_info(); 732 /* 733 * Our EX_CONTEXT is still what it was from the 734 * initial unalign exception, but now we've faulted 735 * on the JIT page. We would like to complete the 736 * page fault however is appropriate, and then retry 737 * the instruction that caused the unalign exception. 738 * Our state has been "corrupted" by setting the low 739 * bit in "sp", and stashing r0..r3 in the 740 * thread_info area, so we revert all of that, then 741 * continue as if this were a normal page fault. 742 */ 743 regs->sp &= ~1UL; 744 regs->regs[0] = ti->unalign_jit_tmp[0]; 745 regs->regs[1] = ti->unalign_jit_tmp[1]; 746 regs->regs[2] = ti->unalign_jit_tmp[2]; 747 regs->regs[3] = ti->unalign_jit_tmp[3]; 748 write &= 1; 749 } else { 750 pr_alert("%s/%d: ICS set at page fault at %#lx: %#lx\n", 751 current->comm, current->pid, pc, address); 752 show_regs(regs); 753 do_group_exit(SIGKILL); 754 return; 755 } 756 } 757#else 758 /* This case should have been handled by do_page_fault_ics(). */ 759 BUG_ON(write & ~1); 760#endif 761 762#if CHIP_HAS_TILE_DMA() 763 /* 764 * If it's a DMA fault, suspend the transfer while we're 765 * handling the miss; we'll restart after it's handled. If we 766 * don't suspend, it's possible that this process could swap 767 * out and back in, and restart the engine since the DMA is 768 * still 'running'. 769 */ 770 if (fault_num == INT_DMATLB_MISS || 771 fault_num == INT_DMATLB_ACCESS || 772 fault_num == INT_DMATLB_MISS_DWNCL || 773 fault_num == INT_DMATLB_ACCESS_DWNCL) { 774 __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__SUSPEND_MASK); 775 while (__insn_mfspr(SPR_DMA_USER_STATUS) & 776 SPR_DMA_STATUS__BUSY_MASK) 777 ; 778 } 779#endif 780 781 /* Validate fault num and decide if this is a first-time page fault. */ 782 switch (fault_num) { 783 case INT_ITLB_MISS: 784 case INT_DTLB_MISS: 785#if CHIP_HAS_TILE_DMA() 786 case INT_DMATLB_MISS: 787 case INT_DMATLB_MISS_DWNCL: 788#endif 789 is_page_fault = 1; 790 break; 791 792 case INT_DTLB_ACCESS: 793#if CHIP_HAS_TILE_DMA() 794 case INT_DMATLB_ACCESS: 795 case INT_DMATLB_ACCESS_DWNCL: 796#endif 797 is_page_fault = 0; 798 break; 799 800 default: 801 panic("Bad fault number %d in do_page_fault", fault_num); 802 } 803 804#if CHIP_HAS_TILE_DMA() 805 if (!user_mode(regs)) { 806 struct async_tlb *async; 807 switch (fault_num) { 808#if CHIP_HAS_TILE_DMA() 809 case INT_DMATLB_MISS: 810 case INT_DMATLB_ACCESS: 811 case INT_DMATLB_MISS_DWNCL: 812 case INT_DMATLB_ACCESS_DWNCL: 813 async = ¤t->thread.dma_async_tlb; 814 break; 815#endif 816 default: 817 async = NULL; 818 } 819 if (async) { 820 821 /* 822 * No vmalloc check required, so we can allow 823 * interrupts immediately at this point. 824 */ 825 local_irq_enable(); 826 827 set_thread_flag(TIF_ASYNC_TLB); 828 if (async->fault_num != 0) { 829 panic("Second async fault %d;" 830 " old fault was %d (%#lx/%ld)", 831 fault_num, async->fault_num, 832 address, write); 833 } 834 BUG_ON(fault_num == 0); 835 async->fault_num = fault_num; 836 async->is_fault = is_page_fault; 837 async->is_write = write; 838 async->address = address; 839 return; 840 } 841 } 842#endif 843 844 handle_page_fault(regs, fault_num, is_page_fault, address, write); 845} 846 847 848#if CHIP_HAS_TILE_DMA() 849/* 850 * This routine effectively re-issues asynchronous page faults 851 * when we are returning to user space. 852 */ 853void do_async_page_fault(struct pt_regs *regs) 854{ 855 struct async_tlb *async = ¤t->thread.dma_async_tlb; 856 857 /* 858 * Clear thread flag early. If we re-interrupt while processing 859 * code here, we will reset it and recall this routine before 860 * returning to user space. 861 */ 862 clear_thread_flag(TIF_ASYNC_TLB); 863 864 if (async->fault_num) { 865 /* 866 * Clear async->fault_num before calling the page-fault 867 * handler so that if we re-interrupt before returning 868 * from the function we have somewhere to put the 869 * information from the new interrupt. 870 */ 871 int fault_num = async->fault_num; 872 async->fault_num = 0; 873 handle_page_fault(regs, fault_num, async->is_fault, 874 async->address, async->is_write); 875 } 876} 877#endif /* CHIP_HAS_TILE_DMA() */ 878 879 880void vmalloc_sync_all(void) 881{ 882#ifdef __tilegx__ 883 /* Currently all L1 kernel pmd's are static and shared. */ 884 BUILD_BUG_ON(pgd_index(VMALLOC_END - PAGE_SIZE) != 885 pgd_index(VMALLOC_START)); 886#else 887 /* 888 * Note that races in the updates of insync and start aren't 889 * problematic: insync can only get set bits added, and updates to 890 * start are only improving performance (without affecting correctness 891 * if undone). 892 */ 893 static DECLARE_BITMAP(insync, PTRS_PER_PGD); 894 static unsigned long start = PAGE_OFFSET; 895 unsigned long address; 896 897 BUILD_BUG_ON(PAGE_OFFSET & ~PGDIR_MASK); 898 for (address = start; address >= PAGE_OFFSET; address += PGDIR_SIZE) { 899 if (!test_bit(pgd_index(address), insync)) { 900 unsigned long flags; 901 struct list_head *pos; 902 903 spin_lock_irqsave(&pgd_lock, flags); 904 list_for_each(pos, &pgd_list) 905 if (!vmalloc_sync_one(list_to_pgd(pos), 906 address)) { 907 /* Must be at first entry in list. */ 908 BUG_ON(pos != pgd_list.next); 909 break; 910 } 911 spin_unlock_irqrestore(&pgd_lock, flags); 912 if (pos != pgd_list.next) 913 set_bit(pgd_index(address), insync); 914 } 915 if (address == start && test_bit(pgd_index(address), insync)) 916 start = address + PGDIR_SIZE; 917 } 918#endif 919} 920