11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* arch/sparc64/mm/tlb.c 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (C) 2004 David S. Miller <davem@redhat.com> 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/kernel.h> 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/percpu.h> 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mm.h> 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/swap.h> 10c9f2946fbec88d4baa3a6d47eb3a8e6b08b05cd9David S. Miller#include <linux/preempt.h> 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/pgtable.h> 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/pgalloc.h> 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/tlbflush.h> 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/cacheflush.h> 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/mmu_context.h> 171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/tlb.h> 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* Heavily inspired by the ppc64 code. */ 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2190f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstrastatic DEFINE_PER_CPU(struct tlb_batch, tlb_batch); 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid flush_tlb_pending(void) 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2590f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra struct tlb_batch *tb = &get_cpu_var(tlb_batch); 26f36391d2790d04993f48da6a45810033a2cdf847David S. Miller struct mm_struct *mm = tb->mm; 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 28f36391d2790d04993f48da6a45810033a2cdf847David S. Miller if (!tb->tlb_nr) 29f36391d2790d04993f48da6a45810033a2cdf847David S. Miller goto out; 3074bf4312fff083ab25c3f357cc653ada7995e5f6David S. Miller 31f36391d2790d04993f48da6a45810033a2cdf847David S. Miller flush_tsb_user(tb); 32f36391d2790d04993f48da6a45810033a2cdf847David S. Miller 33f36391d2790d04993f48da6a45810033a2cdf847David S. Miller if (CTX_VALID(mm->context)) { 34f36391d2790d04993f48da6a45810033a2cdf847David S. Miller if (tb->tlb_nr == 1) { 35f36391d2790d04993f48da6a45810033a2cdf847David S. Miller global_flush_tlb_page(mm, tb->vaddrs[0]); 36f36391d2790d04993f48da6a45810033a2cdf847David S. Miller } else { 371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef CONFIG_SMP 3890f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra smp_flush_tlb_pending(tb->mm, tb->tlb_nr, 3990f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra &tb->vaddrs[0]); 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#else 4190f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra __flush_tlb_pending(CTX_HWBITS(tb->mm->context), 4290f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra tb->tlb_nr, &tb->vaddrs[0]); 431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 46c9f2946fbec88d4baa3a6d47eb3a8e6b08b05cd9David S. Miller 47f36391d2790d04993f48da6a45810033a2cdf847David S. Miller tb->tlb_nr = 0; 48f36391d2790d04993f48da6a45810033a2cdf847David S. Miller 49f36391d2790d04993f48da6a45810033a2cdf847David S. Millerout: 5090f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra put_cpu_var(tlb_batch); 511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 53f36391d2790d04993f48da6a45810033a2cdf847David S. Millervoid arch_enter_lazy_mmu_mode(void) 54f36391d2790d04993f48da6a45810033a2cdf847David S. Miller{ 55494fc42170bf0747ac28e12ef13a7d388d5ff2c7Christoph Lameter struct tlb_batch *tb = this_cpu_ptr(&tlb_batch); 56f36391d2790d04993f48da6a45810033a2cdf847David S. Miller 57f36391d2790d04993f48da6a45810033a2cdf847David S. Miller tb->active = 1; 58f36391d2790d04993f48da6a45810033a2cdf847David S. Miller} 59f36391d2790d04993f48da6a45810033a2cdf847David S. Miller 60f36391d2790d04993f48da6a45810033a2cdf847David S. Millervoid arch_leave_lazy_mmu_mode(void) 61f36391d2790d04993f48da6a45810033a2cdf847David S. Miller{ 62494fc42170bf0747ac28e12ef13a7d388d5ff2c7Christoph Lameter struct tlb_batch *tb = this_cpu_ptr(&tlb_batch); 63f36391d2790d04993f48da6a45810033a2cdf847David S. Miller 64f36391d2790d04993f48da6a45810033a2cdf847David S. Miller if (tb->tlb_nr) 65f36391d2790d04993f48da6a45810033a2cdf847David S. Miller flush_tlb_pending(); 66f36391d2790d04993f48da6a45810033a2cdf847David S. Miller tb->active = 0; 67f36391d2790d04993f48da6a45810033a2cdf847David S. Miller} 68f36391d2790d04993f48da6a45810033a2cdf847David S. Miller 699e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Millerstatic void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr, 709e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller bool exec) 711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 7290f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra struct tlb_batch *tb = &get_cpu_var(tlb_batch); 731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long nr; 741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds vaddr &= PAGE_MASK; 769e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller if (exec) 771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds vaddr |= 0x1UL; 781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 799e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller nr = tb->tlb_nr; 809e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller 819e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller if (unlikely(nr != 0 && mm != tb->mm)) { 829e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller flush_tlb_pending(); 839e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller nr = 0; 849e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller } 859e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller 86f36391d2790d04993f48da6a45810033a2cdf847David S. Miller if (!tb->active) { 87f36391d2790d04993f48da6a45810033a2cdf847David S. Miller flush_tsb_user_page(mm, vaddr); 8823a01138efe216f8084cfaa74b0b90dd4b097441Dave Kleikamp global_flush_tlb_page(mm, vaddr); 89f0af97070acbad5d6a361f485828223a4faaa0eeDavid S. Miller goto out; 90f36391d2790d04993f48da6a45810033a2cdf847David S. Miller } 91f36391d2790d04993f48da6a45810033a2cdf847David S. Miller 929e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller if (nr == 0) 939e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller tb->mm = mm; 949e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller 959e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller tb->vaddrs[nr] = vaddr; 969e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller tb->tlb_nr = ++nr; 979e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller if (nr >= TLB_BATCH_NR) 989e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller flush_tlb_pending(); 999e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller 100f0af97070acbad5d6a361f485828223a4faaa0eeDavid S. Millerout: 1019e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller put_cpu_var(tlb_batch); 1029e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller} 1039e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller 1049e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Millervoid tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, 1059e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller pte_t *ptep, pte_t orig, int fullmm) 1069e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller{ 1077a591cfe4efef8a232e4938d44ae6693b319f6d7David S. Miller if (tlb_type != hypervisor && 1087a591cfe4efef8a232e4938d44ae6693b319f6d7David S. Miller pte_dirty(orig)) { 1091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long paddr, pfn = pte_pfn(orig); 1101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct address_space *mapping; 1111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct page *page; 1121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!pfn_valid(pfn)) 1141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto no_cache_flush; 1151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds page = pfn_to_page(pfn); 1171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (PageReserved(page)) 1181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto no_cache_flush; 1191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* A real file page? */ 1211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mapping = page_mapping(page); 1221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!mapping) 1231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto no_cache_flush; 1241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds paddr = (unsigned long) page_address(page); 1261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if ((paddr ^ vaddr) & (1 << 13)) 1271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flush_dcache_page_all(mm, page); 1281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsno_cache_flush: 1319e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller if (!fullmm) 1329e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller tlb_batch_add_one(mm, vaddr, pte_exec(orig)); 1339e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller} 1349e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller 1359e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller#ifdef CONFIG_TRANSPARENT_HUGEPAGE 1369e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Millerstatic void tlb_batch_pmd_scan(struct mm_struct *mm, unsigned long vaddr, 1375b1e94fa439a3227beefad58c28c17f68287a8e9David S. Miller pmd_t pmd) 1389e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller{ 1399e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller unsigned long end; 1409e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller pte_t *pte; 1419e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller 1429e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller pte = pte_offset_map(&pmd, vaddr); 1439e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller end = vaddr + HPAGE_SIZE; 1449e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller while (vaddr < end) { 1455b1e94fa439a3227beefad58c28c17f68287a8e9David S. Miller if (pte_val(*pte) & _PAGE_VALID) { 1465b1e94fa439a3227beefad58c28c17f68287a8e9David S. Miller bool exec = pte_exec(*pte); 1475b1e94fa439a3227beefad58c28c17f68287a8e9David S. Miller 1489e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller tlb_batch_add_one(mm, vaddr, exec); 1495b1e94fa439a3227beefad58c28c17f68287a8e9David S. Miller } 1509e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller pte++; 1519e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller vaddr += PAGE_SIZE; 1529e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller } 1539e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller pte_unmap(pte); 1549e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller} 1551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1569e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Millervoid set_pmd_at(struct mm_struct *mm, unsigned long addr, 1579e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller pmd_t *pmdp, pmd_t pmd) 1589e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller{ 1599e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller pmd_t orig = *pmdp; 1609e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller 1619e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller *pmdp = pmd; 1629e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller 1639e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller if (mm == &init_mm) 1641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return; 1659e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller 166a7b9403f0e6d5f99139dca18be885819c8d380a1David S. Miller if ((pmd_val(pmd) ^ pmd_val(orig)) & _PAGE_PMD_HUGE) { 167a7b9403f0e6d5f99139dca18be885819c8d380a1David S. Miller if (pmd_val(pmd) & _PAGE_PMD_HUGE) 1689e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller mm->context.huge_pte_count++; 1699e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller else 1709e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller mm->context.huge_pte_count--; 1710fbebed682ff2788dee58e8d7f7dda46e33aa10bDavid S. Miller 1720fbebed682ff2788dee58e8d7f7dda46e33aa10bDavid S. Miller /* Do not try to allocate the TSB hash table if we 1730fbebed682ff2788dee58e8d7f7dda46e33aa10bDavid S. Miller * don't have one already. We have various locks held 1740fbebed682ff2788dee58e8d7f7dda46e33aa10bDavid S. Miller * and thus we'll end up doing a GFP_KERNEL allocation 1750fbebed682ff2788dee58e8d7f7dda46e33aa10bDavid S. Miller * in an atomic context. 1760fbebed682ff2788dee58e8d7f7dda46e33aa10bDavid S. Miller * 1770fbebed682ff2788dee58e8d7f7dda46e33aa10bDavid S. Miller * Instead, we let the first TLB miss on a hugepage 1780fbebed682ff2788dee58e8d7f7dda46e33aa10bDavid S. Miller * take care of this. 1790fbebed682ff2788dee58e8d7f7dda46e33aa10bDavid S. Miller */ 18090f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra } 1811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1829e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller if (!pmd_none(orig)) { 1839e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller addr &= HPAGE_MASK; 184a7b9403f0e6d5f99139dca18be885819c8d380a1David S. Miller if (pmd_trans_huge(orig)) { 1855b1e94fa439a3227beefad58c28c17f68287a8e9David S. Miller pte_t orig_pte = __pte(pmd_val(orig)); 1865b1e94fa439a3227beefad58c28c17f68287a8e9David S. Miller bool exec = pte_exec(orig_pte); 1875b1e94fa439a3227beefad58c28c17f68287a8e9David S. Miller 1889e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller tlb_batch_add_one(mm, addr, exec); 18937b3a8ff3e086cd5c369e77d2383b691b2874cd6David S. Miller tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec); 19037b3a8ff3e086cd5c369e77d2383b691b2874cd6David S. Miller } else { 1915b1e94fa439a3227beefad58c28c17f68287a8e9David S. Miller tlb_batch_pmd_scan(mm, addr, orig); 19237b3a8ff3e086cd5c369e77d2383b691b2874cd6David S. Miller } 1931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1949e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller} 1951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 19651e5ef1bb7ab0e5fa7de4e802da5ab22fe35f0bfDavid S. Millervoid pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, 19751e5ef1bb7ab0e5fa7de4e802da5ab22fe35f0bfDavid S. Miller pmd_t *pmdp) 19851e5ef1bb7ab0e5fa7de4e802da5ab22fe35f0bfDavid S. Miller{ 19951e5ef1bb7ab0e5fa7de4e802da5ab22fe35f0bfDavid S. Miller pmd_t entry = *pmdp; 20051e5ef1bb7ab0e5fa7de4e802da5ab22fe35f0bfDavid S. Miller 20151e5ef1bb7ab0e5fa7de4e802da5ab22fe35f0bfDavid S. Miller pmd_val(entry) &= ~_PAGE_VALID; 20251e5ef1bb7ab0e5fa7de4e802da5ab22fe35f0bfDavid S. Miller 20351e5ef1bb7ab0e5fa7de4e802da5ab22fe35f0bfDavid S. Miller set_pmd_at(vma->vm_mm, address, pmdp, entry); 20451e5ef1bb7ab0e5fa7de4e802da5ab22fe35f0bfDavid S. Miller flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE); 20551e5ef1bb7ab0e5fa7de4e802da5ab22fe35f0bfDavid S. Miller} 20651e5ef1bb7ab0e5fa7de4e802da5ab22fe35f0bfDavid S. Miller 2076b0b50b0617fad5f2af3b928596a25f7de8dbf50Aneesh Kumar K.Vvoid pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, 2086b0b50b0617fad5f2af3b928596a25f7de8dbf50Aneesh Kumar K.V pgtable_t pgtable) 2099e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller{ 2109e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller struct list_head *lh = (struct list_head *) pgtable; 2111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2129e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller assert_spin_locked(&mm->page_table_lock); 21390f08e399d054d017c0e2c5089a0f44a76418271Peter Zijlstra 2149e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller /* FIFO */ 215c389a250ab4cfa4a3775d9f2c45271618af6d5b2Kirill A. Shutemov if (!pmd_huge_pte(mm, pmdp)) 2169e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller INIT_LIST_HEAD(lh); 2179e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller else 218c389a250ab4cfa4a3775d9f2c45271618af6d5b2Kirill A. Shutemov list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); 219c389a250ab4cfa4a3775d9f2c45271618af6d5b2Kirill A. Shutemov pmd_huge_pte(mm, pmdp) = pgtable; 2209e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller} 2219e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller 2226b0b50b0617fad5f2af3b928596a25f7de8dbf50Aneesh Kumar K.Vpgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) 2239e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller{ 2249e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller struct list_head *lh; 2259e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller pgtable_t pgtable; 2269e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller 2279e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller assert_spin_locked(&mm->page_table_lock); 2289e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller 2299e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller /* FIFO */ 230c389a250ab4cfa4a3775d9f2c45271618af6d5b2Kirill A. Shutemov pgtable = pmd_huge_pte(mm, pmdp); 2319e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller lh = (struct list_head *) pgtable; 2329e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller if (list_empty(lh)) 233c389a250ab4cfa4a3775d9f2c45271618af6d5b2Kirill A. Shutemov pmd_huge_pte(mm, pmdp) = NULL; 2349e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller else { 235c389a250ab4cfa4a3775d9f2c45271618af6d5b2Kirill A. Shutemov pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next; 2369e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller list_del(lh); 2379e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller } 2389e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller pte_val(pgtable[0]) = 0; 2399e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller pte_val(pgtable[1]) = 0; 2409e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller 2419e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller return pgtable; 2421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2439e695d2ecc8451cc2c1603d60b5c8e7f5581923aDavid Miller#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 244