1b99fbc10df4fc616908aa0504cba125acf9b4e48Ralf Baechle#include <linux/compiler.h> 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/module.h> 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/highmem.h> 452ab320ac560af3333191a473e56615fb48fff95Yoichi Yuasa#include <linux/sched.h> 5631330f5847b3f8a7ea67d689e9f7c56833ccaa6Ralf Baechle#include <linux/smp.h> 6bb86bf28aec6d0a207ae09f38a43e94133d4d6dbRalf Baechle#include <asm/fixmap.h> 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/tlbflush.h> 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9bb86bf28aec6d0a207ae09f38a43e94133d4d6dbRalf Baechlestatic pte_t *kmap_pte; 10bb86bf28aec6d0a207ae09f38a43e94133d4d6dbRalf Baechle 11bb86bf28aec6d0a207ae09f38a43e94133d4d6dbRalf Baechleunsigned long highstart_pfn, highend_pfn; 12bb86bf28aec6d0a207ae09f38a43e94133d4d6dbRalf Baechle 133e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstravoid *kmap(struct page *page) 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *addr; 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds might_sleep(); 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!PageHighMem(page)) 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return page_address(page); 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds addr = kmap_high(page); 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flush_tlb_one((unsigned long)addr); 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return addr; 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 253e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter ZijlstraEXPORT_SYMBOL(kmap); 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 273e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstravoid kunmap(struct page *page) 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 29b72b7092f8f5f0729cc9f0868997351f21dbc5cdRalf Baechle BUG_ON(in_interrupt()); 301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!PageHighMem(page)) 311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return; 321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kunmap_high(page); 331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 343e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter ZijlstraEXPORT_SYMBOL(kunmap); 351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * kmap_atomic/kunmap_atomic is significantly faster than kmap/kunmap because 381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * no global lock is needed and because the kmap code must perform a global TLB 391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * invalidation when the kmap pool wraps. 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * However when holding an atomic kmap is is not legal to sleep, so atomic 421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * kmaps are appropriate for short, tight code paths only. 431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 45a24401bcf4a67c8fe17e649e74eeb09b08b79ef5Cong Wangvoid *kmap_atomic(struct page *page) 461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long vaddr; 483e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstra int idx, type; 491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */ 51a866374aecc90c7d90619727ccd851ac096b2fc7Peter Zijlstra pagefault_disable(); 521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!PageHighMem(page)) 531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return page_address(page); 541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 553e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstra type = kmap_atomic_idx_push(); 561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds idx = type + KM_TYPE_NR*smp_processor_id(); 571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); 581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef CONFIG_DEBUG_HIGHMEM 59b72b7092f8f5f0729cc9f0868997351f21dbc5cdRalf Baechle BUG_ON(!pte_none(*(kmap_pte - idx))); 601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 61bb86bf28aec6d0a207ae09f38a43e94133d4d6dbRalf Baechle set_pte(kmap_pte-idx, mk_pte(page, PAGE_KERNEL)); 621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds local_flush_tlb_one((unsigned long)vaddr); 631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return (void*) vaddr; 651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 66a24401bcf4a67c8fe17e649e74eeb09b08b79ef5Cong WangEXPORT_SYMBOL(kmap_atomic); 671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 683e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstravoid __kunmap_atomic(void *kvaddr) 691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK; 71b99fbc10df4fc616908aa0504cba125acf9b4e48Ralf Baechle int type __maybe_unused; 721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (vaddr < FIXADDR_START) { // FIXME 74a866374aecc90c7d90619727ccd851ac096b2fc7Peter Zijlstra pagefault_enable(); 751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return; 761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7820273941f2129aa5a432796d98a276ed73d60782Peter Zijlstra type = kmap_atomic_idx(); 793e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstra#ifdef CONFIG_DEBUG_HIGHMEM 803e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstra { 813e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstra int idx = type + KM_TYPE_NR * smp_processor_id(); 821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 833e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstra BUG_ON(vaddr != __fix_to_virt(FIX_KMAP_BEGIN + idx)); 841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 853e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstra /* 863e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstra * force other mappings to Oops if they'll try to access 873e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstra * this pte without first remap it 883e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstra */ 893e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstra pte_clear(&init_mm, vaddr, kmap_pte-idx); 903e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstra local_flush_tlb_one(vaddr); 913e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstra } 923e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstra#endif 9320273941f2129aa5a432796d98a276ed73d60782Peter Zijlstra kmap_atomic_idx_pop(); 94a866374aecc90c7d90619727ccd851ac096b2fc7Peter Zijlstra pagefault_enable(); 951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 963e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter ZijlstraEXPORT_SYMBOL(__kunmap_atomic); 971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9860080265a13ea43f0ebdcd25671dcab05ed01308Ralf Baechle/* 9960080265a13ea43f0ebdcd25671dcab05ed01308Ralf Baechle * This is the same as kmap_atomic() but can map memory that doesn't 10060080265a13ea43f0ebdcd25671dcab05ed01308Ralf Baechle * have a struct page associated with it. 10160080265a13ea43f0ebdcd25671dcab05ed01308Ralf Baechle */ 1023e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstravoid *kmap_atomic_pfn(unsigned long pfn) 10360080265a13ea43f0ebdcd25671dcab05ed01308Ralf Baechle{ 10460080265a13ea43f0ebdcd25671dcab05ed01308Ralf Baechle unsigned long vaddr; 1053e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstra int idx, type; 10660080265a13ea43f0ebdcd25671dcab05ed01308Ralf Baechle 107a866374aecc90c7d90619727ccd851ac096b2fc7Peter Zijlstra pagefault_disable(); 10860080265a13ea43f0ebdcd25671dcab05ed01308Ralf Baechle 1093e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstra type = kmap_atomic_idx_push(); 11060080265a13ea43f0ebdcd25671dcab05ed01308Ralf Baechle idx = type + KM_TYPE_NR*smp_processor_id(); 11160080265a13ea43f0ebdcd25671dcab05ed01308Ralf Baechle vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); 112bb86bf28aec6d0a207ae09f38a43e94133d4d6dbRalf Baechle set_pte(kmap_pte-idx, pfn_pte(pfn, PAGE_KERNEL)); 11360080265a13ea43f0ebdcd25671dcab05ed01308Ralf Baechle flush_tlb_one(vaddr); 11460080265a13ea43f0ebdcd25671dcab05ed01308Ralf Baechle 11560080265a13ea43f0ebdcd25671dcab05ed01308Ralf Baechle return (void*) vaddr; 11660080265a13ea43f0ebdcd25671dcab05ed01308Ralf Baechle} 11760080265a13ea43f0ebdcd25671dcab05ed01308Ralf Baechle 1183e4d3af501cccdc8a8cca41bdbe57d54ad7e7e73Peter Zijlstrastruct page *kmap_atomic_to_page(void *ptr) 1191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long idx, vaddr = (unsigned long)ptr; 1211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pte_t *pte; 1221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (vaddr < FIXADDR_START) 1241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return virt_to_page(ptr); 1251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds idx = virt_to_fix(vaddr); 1271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pte = kmap_pte - (idx - FIX_KMAP_BEGIN); 1281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return pte_page(*pte); 1291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 131bb86bf28aec6d0a207ae09f38a43e94133d4d6dbRalf Baechlevoid __init kmap_init(void) 132bb86bf28aec6d0a207ae09f38a43e94133d4d6dbRalf Baechle{ 133bb86bf28aec6d0a207ae09f38a43e94133d4d6dbRalf Baechle unsigned long kmap_vstart; 134bb86bf28aec6d0a207ae09f38a43e94133d4d6dbRalf Baechle 135bb86bf28aec6d0a207ae09f38a43e94133d4d6dbRalf Baechle /* cache the first kmap pte */ 136bb86bf28aec6d0a207ae09f38a43e94133d4d6dbRalf Baechle kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN); 137bb86bf28aec6d0a207ae09f38a43e94133d4d6dbRalf Baechle kmap_pte = kmap_get_fixmap_pte(kmap_vstart); 138bb86bf28aec6d0a207ae09f38a43e94133d4d6dbRalf Baechle} 139