1#ifndef _LINUX_HUGE_MM_H 2#define _LINUX_HUGE_MM_H 3 4extern int do_huge_pmd_anonymous_page(struct mm_struct *mm, 5 struct vm_area_struct *vma, 6 unsigned long address, pmd_t *pmd, 7 unsigned int flags); 8extern int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, 9 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, 10 struct vm_area_struct *vma); 11extern int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, 12 unsigned long address, pmd_t *pmd, 13 pmd_t orig_pmd); 14extern pgtable_t get_pmd_huge_pte(struct mm_struct *mm); 15extern struct page *follow_trans_huge_pmd(struct mm_struct *mm, 16 unsigned long addr, 17 pmd_t *pmd, 18 unsigned int flags); 19extern int zap_huge_pmd(struct mmu_gather *tlb, 20 struct vm_area_struct *vma, 21 pmd_t *pmd, unsigned long addr); 22extern int mincore_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 23 unsigned long addr, unsigned long end, 24 unsigned char *vec); 25extern int move_huge_pmd(struct vm_area_struct *vma, 26 struct vm_area_struct *new_vma, 27 unsigned long old_addr, 28 unsigned long new_addr, unsigned long old_end, 29 pmd_t *old_pmd, pmd_t *new_pmd); 30extern int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 31 unsigned long addr, pgprot_t newprot); 32 33enum transparent_hugepage_flag { 34 TRANSPARENT_HUGEPAGE_FLAG, 35 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 36 TRANSPARENT_HUGEPAGE_DEFRAG_FLAG, 37 TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, 38 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG, 39#ifdef CONFIG_DEBUG_VM 40 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG, 41#endif 42}; 43 44enum page_check_address_pmd_flag { 45 PAGE_CHECK_ADDRESS_PMD_FLAG, 46 PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG, 47 PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG, 48}; 49extern pmd_t *page_check_address_pmd(struct page *page, 50 struct mm_struct *mm, 51 unsigned long address, 52 enum page_check_address_pmd_flag flag); 53 54#define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT) 55#define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER) 56 57#ifdef CONFIG_TRANSPARENT_HUGEPAGE 58#define HPAGE_PMD_SHIFT HPAGE_SHIFT 59#define HPAGE_PMD_MASK HPAGE_MASK 60#define HPAGE_PMD_SIZE HPAGE_SIZE 61 62#define transparent_hugepage_enabled(__vma) \ 63 ((transparent_hugepage_flags & \ 64 (1<<TRANSPARENT_HUGEPAGE_FLAG) || \ 65 (transparent_hugepage_flags & \ 66 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG) && \ 67 ((__vma)->vm_flags & VM_HUGEPAGE))) && \ 68 !((__vma)->vm_flags & VM_NOHUGEPAGE) && \ 69 !is_vma_temporary_stack(__vma)) 70#define transparent_hugepage_defrag(__vma) \ 71 ((transparent_hugepage_flags & \ 72 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)) || \ 73 (transparent_hugepage_flags & \ 74 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG) && \ 75 (__vma)->vm_flags & VM_HUGEPAGE)) 76#ifdef CONFIG_DEBUG_VM 77#define transparent_hugepage_debug_cow() \ 78 (transparent_hugepage_flags & \ 79 (1<<TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG)) 80#else /* CONFIG_DEBUG_VM */ 81#define transparent_hugepage_debug_cow() 0 82#endif /* CONFIG_DEBUG_VM */ 83 84extern unsigned long transparent_hugepage_flags; 85extern int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, 86 pmd_t *dst_pmd, pmd_t *src_pmd, 87 struct vm_area_struct *vma, 88 unsigned long addr, unsigned long end); 89extern int handle_pte_fault(struct mm_struct *mm, 90 struct vm_area_struct *vma, unsigned long address, 91 pte_t *pte, pmd_t *pmd, unsigned int flags); 92extern int split_huge_page(struct page *page); 93extern void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd); 94#define split_huge_page_pmd(__mm, __pmd) \ 95 do { \ 96 pmd_t *____pmd = (__pmd); \ 97 if (unlikely(pmd_trans_huge(*____pmd))) \ 98 __split_huge_page_pmd(__mm, ____pmd); \ 99 } while (0) 100#define wait_split_huge_page(__anon_vma, __pmd) \ 101 do { \ 102 pmd_t *____pmd = (__pmd); \ 103 anon_vma_lock(__anon_vma); \ 104 anon_vma_unlock(__anon_vma); \ 105 BUG_ON(pmd_trans_splitting(*____pmd) || \ 106 pmd_trans_huge(*____pmd)); \ 107 } while (0) 108#if HPAGE_PMD_ORDER > MAX_ORDER 109#error "hugepages can't be allocated by the buddy allocator" 110#endif 111extern int hugepage_madvise(struct vm_area_struct *vma, 112 unsigned long *vm_flags, int advice); 113extern void __vma_adjust_trans_huge(struct vm_area_struct *vma, 114 unsigned long start, 115 unsigned long end, 116 long adjust_next); 117extern int __pmd_trans_huge_lock(pmd_t *pmd, 118 struct vm_area_struct *vma); 119/* mmap_sem must be held on entry */ 120static inline int pmd_trans_huge_lock(pmd_t *pmd, 121 struct vm_area_struct *vma) 122{ 123 VM_BUG_ON(!rwsem_is_locked(&vma->vm_mm->mmap_sem)); 124 if (pmd_trans_huge(*pmd)) 125 return __pmd_trans_huge_lock(pmd, vma); 126 else 127 return 0; 128} 129static inline void vma_adjust_trans_huge(struct vm_area_struct *vma, 130 unsigned long start, 131 unsigned long end, 132 long adjust_next) 133{ 134 if (!vma->anon_vma || vma->vm_ops) 135 return; 136 __vma_adjust_trans_huge(vma, start, end, adjust_next); 137} 138static inline int hpage_nr_pages(struct page *page) 139{ 140 if (unlikely(PageTransHuge(page))) 141 return HPAGE_PMD_NR; 142 return 1; 143} 144static inline struct page *compound_trans_head(struct page *page) 145{ 146 if (PageTail(page)) { 147 struct page *head; 148 head = page->first_page; 149 smp_rmb(); 150 /* 151 * head may be a dangling pointer. 152 * __split_huge_page_refcount clears PageTail before 153 * overwriting first_page, so if PageTail is still 154 * there it means the head pointer isn't dangling. 155 */ 156 if (PageTail(page)) 157 return head; 158 } 159 return page; 160} 161#else /* CONFIG_TRANSPARENT_HUGEPAGE */ 162#define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; }) 163#define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; }) 164#define HPAGE_PMD_SIZE ({ BUILD_BUG(); 0; }) 165 166#define hpage_nr_pages(x) 1 167 168#define transparent_hugepage_enabled(__vma) 0 169 170#define transparent_hugepage_flags 0UL 171static inline int split_huge_page(struct page *page) 172{ 173 return 0; 174} 175#define split_huge_page_pmd(__mm, __pmd) \ 176 do { } while (0) 177#define wait_split_huge_page(__anon_vma, __pmd) \ 178 do { } while (0) 179#define compound_trans_head(page) compound_head(page) 180static inline int hugepage_madvise(struct vm_area_struct *vma, 181 unsigned long *vm_flags, int advice) 182{ 183 BUG(); 184 return 0; 185} 186static inline void vma_adjust_trans_huge(struct vm_area_struct *vma, 187 unsigned long start, 188 unsigned long end, 189 long adjust_next) 190{ 191} 192static inline int pmd_trans_huge_lock(pmd_t *pmd, 193 struct vm_area_struct *vma) 194{ 195 return 0; 196} 197#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 198 199#endif /* _LINUX_HUGE_MM_H */ 200