11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifndef _S390_TLBFLUSH_H 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define _S390_TLBFLUSH_H 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mm.h> 553492b1de46a7576170e865062ffcfc93bb5650bGerald Schaefer#include <linux/sched.h> 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/processor.h> 7c1821c2e9711adc3cd298a16b7237c92a2cee78dGerald Schaefer#include <asm/pgalloc.h> 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 10ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky * Flush all tlb entries on the local cpu. 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 12ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefskystatic inline void __tlb_flush_local(void) 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 14ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky asm volatile("ptlb" : : : "memory"); 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 17374b8f45f1d5cb17f45ba1d7c74ce8cc9e2f1407Martin Schwidefsky#ifdef CONFIG_SMP 18ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky/* 19ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky * Flush all tlb entries on all cpus. 20ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky */ 21a806170e29c5468b1d641a22518243bdf1b8d58bHeiko Carstensvoid smp_ptlb_all(void); 22a806170e29c5468b1d641a22518243bdf1b8d58bHeiko Carstens 23ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefskystatic inline void __tlb_flush_global(void) 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2594c12cc7d196bab34aaa98d38521549fa1e5ef76Martin Schwidefsky register unsigned long reg2 asm("2"); 2694c12cc7d196bab34aaa98d38521549fa1e5ef76Martin Schwidefsky register unsigned long reg3 asm("3"); 2794c12cc7d196bab34aaa98d38521549fa1e5ef76Martin Schwidefsky register unsigned long reg4 asm("4"); 2894c12cc7d196bab34aaa98d38521549fa1e5ef76Martin Schwidefsky long dummy; 2994c12cc7d196bab34aaa98d38521549fa1e5ef76Martin Schwidefsky 301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifndef __s390x__ 311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!MACHINE_HAS_CSP) { 321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds smp_ptlb_all(); 331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return; 341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif /* __s390x__ */ 3694c12cc7d196bab34aaa98d38521549fa1e5ef76Martin Schwidefsky 3794c12cc7d196bab34aaa98d38521549fa1e5ef76Martin Schwidefsky dummy = 0; 3894c12cc7d196bab34aaa98d38521549fa1e5ef76Martin Schwidefsky reg2 = reg3 = 0; 3994c12cc7d196bab34aaa98d38521549fa1e5ef76Martin Schwidefsky reg4 = ((unsigned long) &dummy) + 1; 4094c12cc7d196bab34aaa98d38521549fa1e5ef76Martin Schwidefsky asm volatile( 4194c12cc7d196bab34aaa98d38521549fa1e5ef76Martin Schwidefsky " csp %0,%2" 4294c12cc7d196bab34aaa98d38521549fa1e5ef76Martin Schwidefsky : : "d" (reg2), "d" (reg3), "d" (reg4), "m" (dummy) : "cc" ); 431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 45374b8f45f1d5cb17f45ba1d7c74ce8cc9e2f1407Martin Schwidefskystatic inline void __tlb_flush_full(struct mm_struct *mm) 46374b8f45f1d5cb17f45ba1d7c74ce8cc9e2f1407Martin Schwidefsky{ 47374b8f45f1d5cb17f45ba1d7c74ce8cc9e2f1407Martin Schwidefsky cpumask_t local_cpumask; 48374b8f45f1d5cb17f45ba1d7c74ce8cc9e2f1407Martin Schwidefsky 49374b8f45f1d5cb17f45ba1d7c74ce8cc9e2f1407Martin Schwidefsky preempt_disable(); 50374b8f45f1d5cb17f45ba1d7c74ce8cc9e2f1407Martin Schwidefsky /* 51374b8f45f1d5cb17f45ba1d7c74ce8cc9e2f1407Martin Schwidefsky * If the process only ran on the local cpu, do a local flush. 52374b8f45f1d5cb17f45ba1d7c74ce8cc9e2f1407Martin Schwidefsky */ 530f1959f50646612b247d624bdbf8b0c8816f2a93KOSAKI Motohiro cpumask_copy(&local_cpumask, cpumask_of(smp_processor_id())); 54005f8eee6f3c8173e492d7bd4d51bda990eb468bRusty Russell if (cpumask_equal(mm_cpumask(mm), &local_cpumask)) 55374b8f45f1d5cb17f45ba1d7c74ce8cc9e2f1407Martin Schwidefsky __tlb_flush_local(); 56374b8f45f1d5cb17f45ba1d7c74ce8cc9e2f1407Martin Schwidefsky else 57374b8f45f1d5cb17f45ba1d7c74ce8cc9e2f1407Martin Schwidefsky __tlb_flush_global(); 58374b8f45f1d5cb17f45ba1d7c74ce8cc9e2f1407Martin Schwidefsky preempt_enable(); 59374b8f45f1d5cb17f45ba1d7c74ce8cc9e2f1407Martin Schwidefsky} 60374b8f45f1d5cb17f45ba1d7c74ce8cc9e2f1407Martin Schwidefsky#else 61374b8f45f1d5cb17f45ba1d7c74ce8cc9e2f1407Martin Schwidefsky#define __tlb_flush_full(mm) __tlb_flush_local() 62e1c4d0142d32f97706b752d94c9e20c3f21901f8Jan Glauber#define __tlb_flush_global() __tlb_flush_local() 63374b8f45f1d5cb17f45ba1d7c74ce8cc9e2f1407Martin Schwidefsky#endif 64374b8f45f1d5cb17f45ba1d7c74ce8cc9e2f1407Martin Schwidefsky 651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 66ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky * Flush all tlb entries of a page table on all cpus. 671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 686f457e1a149eb39ee58d51913e8023fc27c52806Martin Schwidefskystatic inline void __tlb_flush_idte(unsigned long asce) 69ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky{ 70ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky asm volatile( 71ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky " .insn rrf,0xb98e0000,0,%0,%1,0" 726f457e1a149eb39ee58d51913e8023fc27c52806Martin Schwidefsky : : "a" (2048), "a" (asce) : "cc" ); 73ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky} 741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 75ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefskystatic inline void __tlb_flush_mm(struct mm_struct * mm) 761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 77005f8eee6f3c8173e492d7bd4d51bda990eb468bRusty Russell if (unlikely(cpumask_empty(mm_cpumask(mm)))) 781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return; 79ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky /* 80ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky * If the machine has IDTE we prefer to do a per mm flush 81ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky * on all cpus instead of doing a local flush if the mm 82ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky * only ran on the local cpu. 83ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky */ 84e5992f2e6c3829cd43dbc4438ee13dcd6506f7f3Martin Schwidefsky if (MACHINE_HAS_IDTE && list_empty(&mm->context.gmap_list)) 85146e4b3c8b92071b18f0b2e6f47165bad4f9e825Martin Schwidefsky __tlb_flush_idte((unsigned long) mm->pgd | 86146e4b3c8b92071b18f0b2e6f47165bad4f9e825Martin Schwidefsky mm->context.asce_bits); 87043d07084b5347a26eab0a07aa13a4a929ad9e71Martin Schwidefsky else 88043d07084b5347a26eab0a07aa13a4a929ad9e71Martin Schwidefsky __tlb_flush_full(mm); 891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 91ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefskystatic inline void __tlb_flush_mm_cond(struct mm_struct * mm) 921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 93050eef364ad700590a605a0749f825cab4834b1eMartin Schwidefsky spin_lock(&mm->page_table_lock); 94050eef364ad700590a605a0749f825cab4834b1eMartin Schwidefsky if (mm->context.flush_mm) { 95ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky __tlb_flush_mm(mm); 96050eef364ad700590a605a0749f825cab4834b1eMartin Schwidefsky mm->context.flush_mm = 0; 97050eef364ad700590a605a0749f825cab4834b1eMartin Schwidefsky } 98050eef364ad700590a605a0749f825cab4834b1eMartin Schwidefsky spin_unlock(&mm->page_table_lock); 991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 101ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky/* 102ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky * TLB flushing: 103ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky * flush_tlb() - flushes the current mm struct TLBs 104ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky * flush_tlb_all() - flushes all processes TLBs 105ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky * flush_tlb_mm(mm) - flushes the specified mm context TLB's 106ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky * flush_tlb_page(vma, vmaddr) - flushes one page 107ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky * flush_tlb_range(vma, start, end) - flushes a range of pages 108ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky * flush_tlb_kernel_range(start, end) - flushes a range of kernel pages 109ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky */ 1101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 111ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky/* 112ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky * flush_tlb_mm goes together with ptep_set_wrprotect for the 113ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky * copy_page_range operation and flush_tlb_range is related to 114ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky * ptep_get_and_clear for change_protection. ptep_set_wrprotect and 115ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky * ptep_get_and_clear do not flush the TLBs directly if the mm has 116ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky * only one user. At the end of the update the flush_tlb_mm and 117ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky * flush_tlb_range functions need to do the flush. 118ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky */ 119ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky#define flush_tlb() do { } while (0) 120ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky#define flush_tlb_all() do { } while (0) 121ba8a9229ab9e80278c28ad68b15053f65b2b0a7cMartin Schwidefsky#define flush_tlb_page(vma, addr) do { } while (0) 1228ffd74a0924e4e04f6455eb2d2187a9564678d01Martin Schwidefsky 1238ffd74a0924e4e04f6455eb2d2187a9564678d01Martin Schwidefskystatic inline void flush_tlb_mm(struct mm_struct *mm) 1248ffd74a0924e4e04f6455eb2d2187a9564678d01Martin Schwidefsky{ 1258ffd74a0924e4e04f6455eb2d2187a9564678d01Martin Schwidefsky __tlb_flush_mm_cond(mm); 1268ffd74a0924e4e04f6455eb2d2187a9564678d01Martin Schwidefsky} 1278ffd74a0924e4e04f6455eb2d2187a9564678d01Martin Schwidefsky 1288ffd74a0924e4e04f6455eb2d2187a9564678d01Martin Schwidefskystatic inline void flush_tlb_range(struct vm_area_struct *vma, 1298ffd74a0924e4e04f6455eb2d2187a9564678d01Martin Schwidefsky unsigned long start, unsigned long end) 1308ffd74a0924e4e04f6455eb2d2187a9564678d01Martin Schwidefsky{ 1318ffd74a0924e4e04f6455eb2d2187a9564678d01Martin Schwidefsky __tlb_flush_mm_cond(vma->vm_mm); 1328ffd74a0924e4e04f6455eb2d2187a9564678d01Martin Schwidefsky} 1338ffd74a0924e4e04f6455eb2d2187a9564678d01Martin Schwidefsky 1348ffd74a0924e4e04f6455eb2d2187a9564678d01Martin Schwidefskystatic inline void flush_tlb_kernel_range(unsigned long start, 1358ffd74a0924e4e04f6455eb2d2187a9564678d01Martin Schwidefsky unsigned long end) 1368ffd74a0924e4e04f6455eb2d2187a9564678d01Martin Schwidefsky{ 1378ffd74a0924e4e04f6455eb2d2187a9564678d01Martin Schwidefsky __tlb_flush_mm(&init_mm); 1388ffd74a0924e4e04f6455eb2d2187a9564678d01Martin Schwidefsky} 1391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif /* _S390_TLBFLUSH_H */ 141