1#ifndef _LINUX_MM_H 2#define _LINUX_MM_H 3 4#include <linux/sched.h> 5#include <linux/errno.h> 6#include <linux/capability.h> 7 8#ifdef __KERNEL__ 9 10#include <linux/gfp.h> 11#include <linux/list.h> 12#include <linux/mmzone.h> 13#include <linux/rbtree.h> 14#include <linux/prio_tree.h> 15#include <linux/fs.h> 16#include <linux/mutex.h> 17#include <linux/debug_locks.h> 18 19struct mempolicy; 20struct anon_vma; 21 22#ifndef CONFIG_DISCONTIGMEM /* Don't use mapnrs, do it properly */ 23extern unsigned long max_mapnr; 24#endif 25 26extern unsigned long num_physpages; 27extern void * high_memory; 28extern unsigned long vmalloc_earlyreserve; 29extern int page_cluster; 30 31#ifdef CONFIG_SYSCTL 32extern int sysctl_legacy_va_layout; 33#else 34#define sysctl_legacy_va_layout 0 35#endif 36 37#include <asm/page.h> 38#include <asm/pgtable.h> 39#include <asm/processor.h> 40 41#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n)) 42 43/* 44 * Linux kernel virtual memory manager primitives. 45 * The idea being to have a "virtual" mm in the same way 46 * we have a virtual fs - giving a cleaner interface to the 47 * mm details, and allowing different kinds of memory mappings 48 * (from shared memory to executable loading to arbitrary 49 * mmap() functions). 50 */ 51 52/* 53 * This struct defines a memory VMM memory area. There is one of these 54 * per VM-area/task. A VM area is any part of the process virtual memory 55 * space that has a special rule for the page-fault handlers (ie a shared 56 * library, the executable area etc). 57 */ 58struct vm_area_struct { 59 struct mm_struct * vm_mm; /* The address space we belong to. */ 60 unsigned long vm_start; /* Our start address within vm_mm. */ 61 unsigned long vm_end; /* The first byte after our end address 62 within vm_mm. */ 63 64 /* linked list of VM areas per task, sorted by address */ 65 struct vm_area_struct *vm_next; 66 67 pgprot_t vm_page_prot; /* Access permissions of this VMA. */ 68 unsigned long vm_flags; /* Flags, listed below. */ 69 70 struct rb_node vm_rb; 71 72 /* 73 * For areas with an address space and backing store, 74 * linkage into the address_space->i_mmap prio tree, or 75 * linkage to the list of like vmas hanging off its node, or 76 * linkage of vma in the address_space->i_mmap_nonlinear list. 77 */ 78 union { 79 struct { 80 struct list_head list; 81 void *parent; /* aligns with prio_tree_node parent */ 82 struct vm_area_struct *head; 83 } vm_set; 84 85 struct raw_prio_tree_node prio_tree_node; 86 } shared; 87 88 /* 89 * A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma 90 * list, after a COW of one of the file pages. A MAP_SHARED vma 91 * can only be in the i_mmap tree. An anonymous MAP_PRIVATE, stack 92 * or brk vma (with NULL file) can only be in an anon_vma list. 93 */ 94 struct list_head anon_vma_node; /* Serialized by anon_vma->lock */ 95 struct anon_vma *anon_vma; /* Serialized by page_table_lock */ 96 97 /* Function pointers to deal with this struct. */ 98 struct vm_operations_struct * vm_ops; 99 100 /* Information about our backing store: */ 101 unsigned long vm_pgoff; /* Offset (within vm_file) in PAGE_SIZE 102 units, *not* PAGE_CACHE_SIZE */ 103 struct file * vm_file; /* File we map to (can be NULL). */ 104 void * vm_private_data; /* was vm_pte (shared mem) */ 105 unsigned long vm_truncate_count;/* truncate_count or restart_addr */ 106 107#ifndef CONFIG_MMU 108 atomic_t vm_usage; /* refcount (VMAs shared if !MMU) */ 109#endif 110#ifdef CONFIG_NUMA 111 struct mempolicy *vm_policy; /* NUMA policy for the VMA */ 112#endif 113}; 114 115/* 116 * This struct defines the per-mm list of VMAs for uClinux. If CONFIG_MMU is 117 * disabled, then there's a single shared list of VMAs maintained by the 118 * system, and mm's subscribe to these individually 119 */ 120struct vm_list_struct { 121 struct vm_list_struct *next; 122 struct vm_area_struct *vma; 123}; 124 125#ifndef CONFIG_MMU 126extern struct rb_root nommu_vma_tree; 127extern struct rw_semaphore nommu_vma_sem; 128 129extern unsigned int kobjsize(const void *objp); 130#endif 131 132/* 133 * vm_flags.. 134 */ 135#define VM_READ 0x00000001 /* currently active flags */ 136#define VM_WRITE 0x00000002 137#define VM_EXEC 0x00000004 138#define VM_SHARED 0x00000008 139 140/* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */ 141#define VM_MAYREAD 0x00000010 /* limits for mprotect() etc */ 142#define VM_MAYWRITE 0x00000020 143#define VM_MAYEXEC 0x00000040 144#define VM_MAYSHARE 0x00000080 145 146#define VM_GROWSDOWN 0x00000100 /* general info on the segment */ 147#define VM_GROWSUP 0x00000200 148#define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */ 149#define VM_DENYWRITE 0x00000800 /* ETXTBSY on write attempts.. */ 150 151#define VM_EXECUTABLE 0x00001000 152#define VM_LOCKED 0x00002000 153#define VM_IO 0x00004000 /* Memory mapped I/O or similar */ 154 155 /* Used by sys_madvise() */ 156#define VM_SEQ_READ 0x00008000 /* App will access data sequentially */ 157#define VM_RAND_READ 0x00010000 /* App will not benefit from clustered reads */ 158 159#define VM_DONTCOPY 0x00020000 /* Do not copy this vma on fork */ 160#define VM_DONTEXPAND 0x00040000 /* Cannot expand with mremap() */ 161#define VM_RESERVED 0x00080000 /* Count as reserved_vm like IO */ 162#define VM_ACCOUNT 0x00100000 /* Is a VM accounted object */ 163#define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */ 164#define VM_NONLINEAR 0x00800000 /* Is non-linear (remap_file_pages) */ 165#define VM_MAPPED_COPY 0x01000000 /* T if mapped copy of data (nommu mmap) */ 166#define VM_INSERTPAGE 0x02000000 /* The vma has had "vm_insert_page()" done on it */ 167 168#ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */ 169#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS 170#endif 171 172#ifdef CONFIG_STACK_GROWSUP 173#define VM_STACK_FLAGS (VM_GROWSUP | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT) 174#else 175#define VM_STACK_FLAGS (VM_GROWSDOWN | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT) 176#endif 177 178#define VM_READHINTMASK (VM_SEQ_READ | VM_RAND_READ) 179#define VM_ClearReadHint(v) (v)->vm_flags &= ~VM_READHINTMASK 180#define VM_NormalReadHint(v) (!((v)->vm_flags & VM_READHINTMASK)) 181#define VM_SequentialReadHint(v) ((v)->vm_flags & VM_SEQ_READ) 182#define VM_RandomReadHint(v) ((v)->vm_flags & VM_RAND_READ) 183 184/* 185 * mapping from the currently active vm_flags protection bits (the 186 * low four bits) to a page protection mask.. 187 */ 188extern pgprot_t protection_map[16]; 189 190 191/* 192 * These are the virtual MM functions - opening of an area, closing and 193 * unmapping it (needed to keep files on disk up-to-date etc), pointer 194 * to the functions called when a no-page or a wp-page exception occurs. 195 */ 196struct vm_operations_struct { 197 void (*open)(struct vm_area_struct * area); 198 void (*close)(struct vm_area_struct * area); 199 struct page * (*nopage)(struct vm_area_struct * area, unsigned long address, int *type); 200 int (*populate)(struct vm_area_struct * area, unsigned long address, unsigned long len, pgprot_t prot, unsigned long pgoff, int nonblock); 201 202 /* notification that a previously read-only page is about to become 203 * writable, if an error is returned it will cause a SIGBUS */ 204 int (*page_mkwrite)(struct vm_area_struct *vma, struct page *page); 205#ifdef CONFIG_NUMA 206 int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new); 207 struct mempolicy *(*get_policy)(struct vm_area_struct *vma, 208 unsigned long addr); 209 int (*migrate)(struct vm_area_struct *vma, const nodemask_t *from, 210 const nodemask_t *to, unsigned long flags); 211#endif 212}; 213 214struct mmu_gather; 215struct inode; 216 217/* 218 * Each physical page in the system has a struct page associated with 219 * it to keep track of whatever it is we are using the page for at the 220 * moment. Note that we have no way to track which tasks are using 221 * a page. 222 */ 223struct page { 224 unsigned long flags; /* Atomic flags, some possibly 225 * updated asynchronously */ 226 atomic_t _count; /* Usage count, see below. */ 227 atomic_t _mapcount; /* Count of ptes mapped in mms, 228 * to show when page is mapped 229 * & limit reverse map searches. 230 */ 231 union { 232 struct { 233 unsigned long private; /* Mapping-private opaque data: 234 * usually used for buffer_heads 235 * if PagePrivate set; used for 236 * swp_entry_t if PageSwapCache; 237 * indicates order in the buddy 238 * system if PG_buddy is set. 239 */ 240 struct address_space *mapping; /* If low bit clear, points to 241 * inode address_space, or NULL. 242 * If page mapped as anonymous 243 * memory, low bit is set, and 244 * it points to anon_vma object: 245 * see PAGE_MAPPING_ANON below. 246 */ 247 }; 248#if NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS 249 spinlock_t ptl; 250#endif 251 }; 252 pgoff_t index; /* Our offset within mapping. */ 253 struct list_head lru; /* Pageout list, eg. active_list 254 * protected by zone->lru_lock ! 255 */ 256 /* 257 * On machines where all RAM is mapped into kernel address space, 258 * we can simply calculate the virtual address. On machines with 259 * highmem some memory is mapped into kernel virtual memory 260 * dynamically, so we need a place to store that address. 261 * Note that this field could be 16 bits on x86 ... ;) 262 * 263 * Architectures with slow multiplication can define 264 * WANT_PAGE_VIRTUAL in asm/page.h 265 */ 266#if defined(WANT_PAGE_VIRTUAL) 267 void *virtual; /* Kernel virtual address (NULL if 268 not kmapped, ie. highmem) */ 269#endif /* WANT_PAGE_VIRTUAL */ 270}; 271 272#define page_private(page) ((page)->private) 273#define set_page_private(page, v) ((page)->private = (v)) 274 275/* 276 * FIXME: take this include out, include page-flags.h in 277 * files which need it (119 of them) 278 */ 279#include <linux/page-flags.h> 280 281/* 282 * Methods to modify the page usage count. 283 * 284 * What counts for a page usage: 285 * - cache mapping (page->mapping) 286 * - private data (page->private) 287 * - page mapped in a task's page tables, each mapping 288 * is counted separately 289 * 290 * Also, many kernel routines increase the page count before a critical 291 * routine so they can be sure the page doesn't go away from under them. 292 */ 293 294/* 295 * Drop a ref, return true if the logical refcount fell to zero (the page has 296 * no users) 297 */ 298static inline int put_page_testzero(struct page *page) 299{ 300 BUG_ON(atomic_read(&page->_count) == 0); 301 return atomic_dec_and_test(&page->_count); 302} 303 304/* 305 * Try to grab a ref unless the page has a refcount of zero, return false if 306 * that is the case. 307 */ 308static inline int get_page_unless_zero(struct page *page) 309{ 310 return atomic_inc_not_zero(&page->_count); 311} 312 313extern void FASTCALL(__page_cache_release(struct page *)); 314 315static inline int page_count(struct page *page) 316{ 317 if (unlikely(PageCompound(page))) 318 page = (struct page *)page_private(page); 319 return atomic_read(&page->_count); 320} 321 322static inline void get_page(struct page *page) 323{ 324 if (unlikely(PageCompound(page))) 325 page = (struct page *)page_private(page); 326 atomic_inc(&page->_count); 327} 328 329/* 330 * Setup the page count before being freed into the page allocator for 331 * the first time (boot or memory hotplug) 332 */ 333static inline void init_page_count(struct page *page) 334{ 335 atomic_set(&page->_count, 1); 336} 337 338void put_page(struct page *page); 339void put_pages_list(struct list_head *pages); 340 341void split_page(struct page *page, unsigned int order); 342 343/* 344 * Multiple processes may "see" the same page. E.g. for untouched 345 * mappings of /dev/null, all processes see the same page full of 346 * zeroes, and text pages of executables and shared libraries have 347 * only one copy in memory, at most, normally. 348 * 349 * For the non-reserved pages, page_count(page) denotes a reference count. 350 * page_count() == 0 means the page is free. page->lru is then used for 351 * freelist management in the buddy allocator. 352 * page_count() == 1 means the page is used for exactly one purpose 353 * (e.g. a private data page of one process). 354 * 355 * A page may be used for kmalloc() or anyone else who does a 356 * __get_free_page(). In this case the page_count() is at least 1, and 357 * all other fields are unused but should be 0 or NULL. The 358 * management of this page is the responsibility of the one who uses 359 * it. 360 * 361 * The other pages (we may call them "process pages") are completely 362 * managed by the Linux memory manager: I/O, buffers, swapping etc. 363 * The following discussion applies only to them. 364 * 365 * A page may belong to an inode's memory mapping. In this case, 366 * page->mapping is the pointer to the inode, and page->index is the 367 * file offset of the page, in units of PAGE_CACHE_SIZE. 368 * 369 * A page contains an opaque `private' member, which belongs to the 370 * page's address_space. Usually, this is the address of a circular 371 * list of the page's disk buffers. 372 * 373 * For pages belonging to inodes, the page_count() is the number of 374 * attaches, plus 1 if `private' contains something, plus one for 375 * the page cache itself. 376 * 377 * Instead of keeping dirty/clean pages in per address-space lists, we instead 378 * now tag pages as dirty/under writeback in the radix tree. 379 * 380 * There is also a per-mapping radix tree mapping index to the page 381 * in memory if present. The tree is rooted at mapping->root. 382 * 383 * All process pages can do I/O: 384 * - inode pages may need to be read from disk, 385 * - inode pages which have been modified and are MAP_SHARED may need 386 * to be written to disk, 387 * - private pages which have been modified may need to be swapped out 388 * to swap space and (later) to be read back into memory. 389 */ 390 391/* 392 * The zone field is never updated after free_area_init_core() 393 * sets it, so none of the operations on it need to be atomic. 394 */ 395 396 397/* 398 * page->flags layout: 399 * 400 * There are three possibilities for how page->flags get 401 * laid out. The first is for the normal case, without 402 * sparsemem. The second is for sparsemem when there is 403 * plenty of space for node and section. The last is when 404 * we have run out of space and have to fall back to an 405 * alternate (slower) way of determining the node. 406 * 407 * No sparsemem: | NODE | ZONE | ... | FLAGS | 408 * with space for node: | SECTION | NODE | ZONE | ... | FLAGS | 409 * no space for node: | SECTION | ZONE | ... | FLAGS | 410 */ 411#ifdef CONFIG_SPARSEMEM 412#define SECTIONS_WIDTH SECTIONS_SHIFT 413#else 414#define SECTIONS_WIDTH 0 415#endif 416 417#define ZONES_WIDTH ZONES_SHIFT 418 419#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT <= FLAGS_RESERVED 420#define NODES_WIDTH NODES_SHIFT 421#else 422#define NODES_WIDTH 0 423#endif 424 425/* Page flags: | [SECTION] | [NODE] | ZONE | ... | FLAGS | */ 426#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH) 427#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH) 428#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH) 429 430/* 431 * We are going to use the flags for the page to node mapping if its in 432 * there. This includes the case where there is no node, so it is implicit. 433 */ 434#define FLAGS_HAS_NODE (NODES_WIDTH > 0 || NODES_SHIFT == 0) 435 436#ifndef PFN_SECTION_SHIFT 437#define PFN_SECTION_SHIFT 0 438#endif 439 440/* 441 * Define the bit shifts to access each section. For non-existant 442 * sections we define the shift as 0; that plus a 0 mask ensures 443 * the compiler will optimise away reference to them. 444 */ 445#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0)) 446#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0)) 447#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0)) 448 449/* NODE:ZONE or SECTION:ZONE is used to lookup the zone from a page. */ 450#if FLAGS_HAS_NODE 451#define ZONETABLE_SHIFT (NODES_SHIFT + ZONES_SHIFT) 452#else 453#define ZONETABLE_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT) 454#endif 455#define ZONETABLE_PGSHIFT ZONES_PGSHIFT 456 457#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > FLAGS_RESERVED 458#error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > FLAGS_RESERVED 459#endif 460 461#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1) 462#define NODES_MASK ((1UL << NODES_WIDTH) - 1) 463#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1) 464#define ZONETABLE_MASK ((1UL << ZONETABLE_SHIFT) - 1) 465 466static inline unsigned long page_zonenum(struct page *page) 467{ 468 return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK; 469} 470 471struct zone; 472extern struct zone *zone_table[]; 473 474static inline int page_zone_id(struct page *page) 475{ 476 return (page->flags >> ZONETABLE_PGSHIFT) & ZONETABLE_MASK; 477} 478static inline struct zone *page_zone(struct page *page) 479{ 480 return zone_table[page_zone_id(page)]; 481} 482 483static inline unsigned long page_to_nid(struct page *page) 484{ 485 if (FLAGS_HAS_NODE) 486 return (page->flags >> NODES_PGSHIFT) & NODES_MASK; 487 else 488 return page_zone(page)->zone_pgdat->node_id; 489} 490static inline unsigned long page_to_section(struct page *page) 491{ 492 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK; 493} 494 495static inline void set_page_zone(struct page *page, unsigned long zone) 496{ 497 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT); 498 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT; 499} 500static inline void set_page_node(struct page *page, unsigned long node) 501{ 502 page->flags &= ~(NODES_MASK << NODES_PGSHIFT); 503 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT; 504} 505static inline void set_page_section(struct page *page, unsigned long section) 506{ 507 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT); 508 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT; 509} 510 511static inline void set_page_links(struct page *page, unsigned long zone, 512 unsigned long node, unsigned long pfn) 513{ 514 set_page_zone(page, zone); 515 set_page_node(page, node); 516 set_page_section(page, pfn_to_section_nr(pfn)); 517} 518 519/* 520 * Some inline functions in vmstat.h depend on page_zone() 521 */ 522#include <linux/vmstat.h> 523 524#ifndef CONFIG_DISCONTIGMEM 525/* The array of struct pages - for discontigmem use pgdat->lmem_map */ 526extern struct page *mem_map; 527#endif 528 529static __always_inline void *lowmem_page_address(struct page *page) 530{ 531 return __va(page_to_pfn(page) << PAGE_SHIFT); 532} 533 534#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL) 535#define HASHED_PAGE_VIRTUAL 536#endif 537 538#if defined(WANT_PAGE_VIRTUAL) 539#define page_address(page) ((page)->virtual) 540#define set_page_address(page, address) \ 541 do { \ 542 (page)->virtual = (address); \ 543 } while(0) 544#define page_address_init() do { } while(0) 545#endif 546 547#if defined(HASHED_PAGE_VIRTUAL) 548void *page_address(struct page *page); 549void set_page_address(struct page *page, void *virtual); 550void page_address_init(void); 551#endif 552 553#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL) 554#define page_address(page) lowmem_page_address(page) 555#define set_page_address(page, address) do { } while(0) 556#define page_address_init() do { } while(0) 557#endif 558 559/* 560 * On an anonymous page mapped into a user virtual memory area, 561 * page->mapping points to its anon_vma, not to a struct address_space; 562 * with the PAGE_MAPPING_ANON bit set to distinguish it. 563 * 564 * Please note that, confusingly, "page_mapping" refers to the inode 565 * address_space which maps the page from disk; whereas "page_mapped" 566 * refers to user virtual address space into which the page is mapped. 567 */ 568#define PAGE_MAPPING_ANON 1 569 570extern struct address_space swapper_space; 571static inline struct address_space *page_mapping(struct page *page) 572{ 573 struct address_space *mapping = page->mapping; 574 575 if (unlikely(PageSwapCache(page))) 576 mapping = &swapper_space; 577 else if (unlikely((unsigned long)mapping & PAGE_MAPPING_ANON)) 578 mapping = NULL; 579 return mapping; 580} 581 582static inline int PageAnon(struct page *page) 583{ 584 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0; 585} 586 587/* 588 * Return the pagecache index of the passed page. Regular pagecache pages 589 * use ->index whereas swapcache pages use ->private 590 */ 591static inline pgoff_t page_index(struct page *page) 592{ 593 if (unlikely(PageSwapCache(page))) 594 return page_private(page); 595 return page->index; 596} 597 598/* 599 * The atomic page->_mapcount, like _count, starts from -1: 600 * so that transitions both from it and to it can be tracked, 601 * using atomic_inc_and_test and atomic_add_negative(-1). 602 */ 603static inline void reset_page_mapcount(struct page *page) 604{ 605 atomic_set(&(page)->_mapcount, -1); 606} 607 608static inline int page_mapcount(struct page *page) 609{ 610 return atomic_read(&(page)->_mapcount) + 1; 611} 612 613/* 614 * Return true if this page is mapped into pagetables. 615 */ 616static inline int page_mapped(struct page *page) 617{ 618 return atomic_read(&(page)->_mapcount) >= 0; 619} 620 621/* 622 * Error return values for the *_nopage functions 623 */ 624#define NOPAGE_SIGBUS (NULL) 625#define NOPAGE_OOM ((struct page *) (-1)) 626 627/* 628 * Different kinds of faults, as returned by handle_mm_fault(). 629 * Used to decide whether a process gets delivered SIGBUS or 630 * just gets major/minor fault counters bumped up. 631 */ 632#define VM_FAULT_OOM 0x00 633#define VM_FAULT_SIGBUS 0x01 634#define VM_FAULT_MINOR 0x02 635#define VM_FAULT_MAJOR 0x03 636 637/* 638 * Special case for get_user_pages. 639 * Must be in a distinct bit from the above VM_FAULT_ flags. 640 */ 641#define VM_FAULT_WRITE 0x10 642 643#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK) 644 645extern void show_free_areas(void); 646 647#ifdef CONFIG_SHMEM 648struct page *shmem_nopage(struct vm_area_struct *vma, 649 unsigned long address, int *type); 650int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *new); 651struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, 652 unsigned long addr); 653int shmem_lock(struct file *file, int lock, struct user_struct *user); 654#else 655#define shmem_nopage filemap_nopage 656 657static inline int shmem_lock(struct file *file, int lock, 658 struct user_struct *user) 659{ 660 return 0; 661} 662 663static inline int shmem_set_policy(struct vm_area_struct *vma, 664 struct mempolicy *new) 665{ 666 return 0; 667} 668 669static inline struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, 670 unsigned long addr) 671{ 672 return NULL; 673} 674#endif 675struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags); 676extern int shmem_mmap(struct file *file, struct vm_area_struct *vma); 677 678int shmem_zero_setup(struct vm_area_struct *); 679 680#ifndef CONFIG_MMU 681extern unsigned long shmem_get_unmapped_area(struct file *file, 682 unsigned long addr, 683 unsigned long len, 684 unsigned long pgoff, 685 unsigned long flags); 686#endif 687 688static inline int can_do_mlock(void) 689{ 690 if (capable(CAP_IPC_LOCK)) 691 return 1; 692 if (current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur != 0) 693 return 1; 694 return 0; 695} 696extern int user_shm_lock(size_t, struct user_struct *); 697extern void user_shm_unlock(size_t, struct user_struct *); 698 699/* 700 * Parameter block passed down to zap_pte_range in exceptional cases. 701 */ 702struct zap_details { 703 struct vm_area_struct *nonlinear_vma; /* Check page->index if set */ 704 struct address_space *check_mapping; /* Check page->mapping if set */ 705 pgoff_t first_index; /* Lowest page->index to unmap */ 706 pgoff_t last_index; /* Highest page->index to unmap */ 707 spinlock_t *i_mmap_lock; /* For unmap_mapping_range: */ 708 unsigned long truncate_count; /* Compare vm_truncate_count */ 709}; 710 711struct page *vm_normal_page(struct vm_area_struct *, unsigned long, pte_t); 712unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address, 713 unsigned long size, struct zap_details *); 714unsigned long unmap_vmas(struct mmu_gather **tlb, 715 struct vm_area_struct *start_vma, unsigned long start_addr, 716 unsigned long end_addr, unsigned long *nr_accounted, 717 struct zap_details *); 718void free_pgd_range(struct mmu_gather **tlb, unsigned long addr, 719 unsigned long end, unsigned long floor, unsigned long ceiling); 720void free_pgtables(struct mmu_gather **tlb, struct vm_area_struct *start_vma, 721 unsigned long floor, unsigned long ceiling); 722int copy_page_range(struct mm_struct *dst, struct mm_struct *src, 723 struct vm_area_struct *vma); 724int zeromap_page_range(struct vm_area_struct *vma, unsigned long from, 725 unsigned long size, pgprot_t prot); 726void unmap_mapping_range(struct address_space *mapping, 727 loff_t const holebegin, loff_t const holelen, int even_cows); 728 729static inline void unmap_shared_mapping_range(struct address_space *mapping, 730 loff_t const holebegin, loff_t const holelen) 731{ 732 unmap_mapping_range(mapping, holebegin, holelen, 0); 733} 734 735extern int vmtruncate(struct inode * inode, loff_t offset); 736extern int vmtruncate_range(struct inode * inode, loff_t offset, loff_t end); 737extern int install_page(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, struct page *page, pgprot_t prot); 738extern int install_file_pte(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, unsigned long pgoff, pgprot_t prot); 739 740#ifdef CONFIG_MMU 741extern int __handle_mm_fault(struct mm_struct *mm,struct vm_area_struct *vma, 742 unsigned long address, int write_access); 743 744static inline int handle_mm_fault(struct mm_struct *mm, 745 struct vm_area_struct *vma, unsigned long address, 746 int write_access) 747{ 748 return __handle_mm_fault(mm, vma, address, write_access) & 749 (~VM_FAULT_WRITE); 750} 751#else 752static inline int handle_mm_fault(struct mm_struct *mm, 753 struct vm_area_struct *vma, unsigned long address, 754 int write_access) 755{ 756 /* should never happen if there's no MMU */ 757 BUG(); 758 return VM_FAULT_SIGBUS; 759} 760#endif 761 762extern int make_pages_present(unsigned long addr, unsigned long end); 763extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write); 764void install_arg_page(struct vm_area_struct *, struct page *, unsigned long); 765 766int get_user_pages(struct task_struct *tsk, struct mm_struct *mm, unsigned long start, 767 int len, int write, int force, struct page **pages, struct vm_area_struct **vmas); 768void print_bad_pte(struct vm_area_struct *, pte_t, unsigned long); 769 770int __set_page_dirty_buffers(struct page *page); 771int __set_page_dirty_nobuffers(struct page *page); 772int redirty_page_for_writepage(struct writeback_control *wbc, 773 struct page *page); 774int FASTCALL(set_page_dirty(struct page *page)); 775int set_page_dirty_lock(struct page *page); 776int clear_page_dirty_for_io(struct page *page); 777 778extern unsigned long do_mremap(unsigned long addr, 779 unsigned long old_len, unsigned long new_len, 780 unsigned long flags, unsigned long new_addr); 781 782/* 783 * Prototype to add a shrinker callback for ageable caches. 784 * 785 * These functions are passed a count `nr_to_scan' and a gfpmask. They should 786 * scan `nr_to_scan' objects, attempting to free them. 787 * 788 * The callback must return the number of objects which remain in the cache. 789 * 790 * The callback will be passed nr_to_scan == 0 when the VM is querying the 791 * cache size, so a fastpath for that case is appropriate. 792 */ 793typedef int (*shrinker_t)(int nr_to_scan, gfp_t gfp_mask); 794 795/* 796 * Add an aging callback. The int is the number of 'seeks' it takes 797 * to recreate one of the objects that these functions age. 798 */ 799 800#define DEFAULT_SEEKS 2 801struct shrinker; 802extern struct shrinker *set_shrinker(int, shrinker_t); 803extern void remove_shrinker(struct shrinker *shrinker); 804 805extern pte_t *FASTCALL(get_locked_pte(struct mm_struct *mm, unsigned long addr, spinlock_t **ptl)); 806 807int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address); 808int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address); 809int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address); 810int __pte_alloc_kernel(pmd_t *pmd, unsigned long address); 811 812/* 813 * The following ifdef needed to get the 4level-fixup.h header to work. 814 * Remove it when 4level-fixup.h has been removed. 815 */ 816#if defined(CONFIG_MMU) && !defined(__ARCH_HAS_4LEVEL_HACK) 817static inline pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address) 818{ 819 return (unlikely(pgd_none(*pgd)) && __pud_alloc(mm, pgd, address))? 820 NULL: pud_offset(pgd, address); 821} 822 823static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) 824{ 825 return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))? 826 NULL: pmd_offset(pud, address); 827} 828#endif /* CONFIG_MMU && !__ARCH_HAS_4LEVEL_HACK */ 829 830#if NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS 831/* 832 * We tuck a spinlock to guard each pagetable page into its struct page, 833 * at page->private, with BUILD_BUG_ON to make sure that this will not 834 * overflow into the next struct page (as it might with DEBUG_SPINLOCK). 835 * When freeing, reset page->mapping so free_pages_check won't complain. 836 */ 837#define __pte_lockptr(page) &((page)->ptl) 838#define pte_lock_init(_page) do { \ 839 spin_lock_init(__pte_lockptr(_page)); \ 840} while (0) 841#define pte_lock_deinit(page) ((page)->mapping = NULL) 842#define pte_lockptr(mm, pmd) ({(void)(mm); __pte_lockptr(pmd_page(*(pmd)));}) 843#else 844/* 845 * We use mm->page_table_lock to guard all pagetable pages of the mm. 846 */ 847#define pte_lock_init(page) do {} while (0) 848#define pte_lock_deinit(page) do {} while (0) 849#define pte_lockptr(mm, pmd) ({(void)(pmd); &(mm)->page_table_lock;}) 850#endif /* NR_CPUS < CONFIG_SPLIT_PTLOCK_CPUS */ 851 852#define pte_offset_map_lock(mm, pmd, address, ptlp) \ 853({ \ 854 spinlock_t *__ptl = pte_lockptr(mm, pmd); \ 855 pte_t *__pte = pte_offset_map(pmd, address); \ 856 *(ptlp) = __ptl; \ 857 spin_lock(__ptl); \ 858 __pte; \ 859}) 860 861#define pte_unmap_unlock(pte, ptl) do { \ 862 spin_unlock(ptl); \ 863 pte_unmap(pte); \ 864} while (0) 865 866#define pte_alloc_map(mm, pmd, address) \ 867 ((unlikely(!pmd_present(*(pmd))) && __pte_alloc(mm, pmd, address))? \ 868 NULL: pte_offset_map(pmd, address)) 869 870#define pte_alloc_map_lock(mm, pmd, address, ptlp) \ 871 ((unlikely(!pmd_present(*(pmd))) && __pte_alloc(mm, pmd, address))? \ 872 NULL: pte_offset_map_lock(mm, pmd, address, ptlp)) 873 874#define pte_alloc_kernel(pmd, address) \ 875 ((unlikely(!pmd_present(*(pmd))) && __pte_alloc_kernel(pmd, address))? \ 876 NULL: pte_offset_kernel(pmd, address)) 877 878extern void free_area_init(unsigned long * zones_size); 879extern void free_area_init_node(int nid, pg_data_t *pgdat, 880 unsigned long * zones_size, unsigned long zone_start_pfn, 881 unsigned long *zholes_size); 882extern void memmap_init_zone(unsigned long, int, unsigned long, unsigned long); 883extern void setup_per_zone_pages_min(void); 884extern void mem_init(void); 885extern void show_mem(void); 886extern void si_meminfo(struct sysinfo * val); 887extern void si_meminfo_node(struct sysinfo *val, int nid); 888 889#ifdef CONFIG_NUMA 890extern void setup_per_cpu_pageset(void); 891#else 892static inline void setup_per_cpu_pageset(void) {} 893#endif 894 895/* prio_tree.c */ 896void vma_prio_tree_add(struct vm_area_struct *, struct vm_area_struct *old); 897void vma_prio_tree_insert(struct vm_area_struct *, struct prio_tree_root *); 898void vma_prio_tree_remove(struct vm_area_struct *, struct prio_tree_root *); 899struct vm_area_struct *vma_prio_tree_next(struct vm_area_struct *vma, 900 struct prio_tree_iter *iter); 901 902#define vma_prio_tree_foreach(vma, iter, root, begin, end) \ 903 for (prio_tree_iter_init(iter, root, begin, end), vma = NULL; \ 904 (vma = vma_prio_tree_next(vma, iter)); ) 905 906static inline void vma_nonlinear_insert(struct vm_area_struct *vma, 907 struct list_head *list) 908{ 909 vma->shared.vm_set.parent = NULL; 910 list_add_tail(&vma->shared.vm_set.list, list); 911} 912 913/* mmap.c */ 914extern int __vm_enough_memory(long pages, int cap_sys_admin); 915extern void vma_adjust(struct vm_area_struct *vma, unsigned long start, 916 unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert); 917extern struct vm_area_struct *vma_merge(struct mm_struct *, 918 struct vm_area_struct *prev, unsigned long addr, unsigned long end, 919 unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t, 920 struct mempolicy *); 921extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *); 922extern int split_vma(struct mm_struct *, 923 struct vm_area_struct *, unsigned long addr, int new_below); 924extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *); 925extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *, 926 struct rb_node **, struct rb_node *); 927extern void unlink_file_vma(struct vm_area_struct *); 928extern struct vm_area_struct *copy_vma(struct vm_area_struct **, 929 unsigned long addr, unsigned long len, pgoff_t pgoff); 930extern void exit_mmap(struct mm_struct *); 931extern int may_expand_vm(struct mm_struct *mm, unsigned long npages); 932 933extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); 934 935extern unsigned long do_mmap_pgoff(struct file *file, unsigned long addr, 936 unsigned long len, unsigned long prot, 937 unsigned long flag, unsigned long pgoff); 938 939static inline unsigned long do_mmap(struct file *file, unsigned long addr, 940 unsigned long len, unsigned long prot, 941 unsigned long flag, unsigned long offset) 942{ 943 unsigned long ret = -EINVAL; 944 if ((offset + PAGE_ALIGN(len)) < offset) 945 goto out; 946 if (!(offset & ~PAGE_MASK)) 947 ret = do_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT); 948out: 949 return ret; 950} 951 952extern int do_munmap(struct mm_struct *, unsigned long, size_t); 953 954extern unsigned long do_brk(unsigned long, unsigned long); 955 956/* filemap.c */ 957extern unsigned long page_unuse(struct page *); 958extern void truncate_inode_pages(struct address_space *, loff_t); 959extern void truncate_inode_pages_range(struct address_space *, 960 loff_t lstart, loff_t lend); 961 962/* generic vm_area_ops exported for stackable file systems */ 963extern struct page *filemap_nopage(struct vm_area_struct *, unsigned long, int *); 964extern int filemap_populate(struct vm_area_struct *, unsigned long, 965 unsigned long, pgprot_t, unsigned long, int); 966 967/* mm/page-writeback.c */ 968int write_one_page(struct page *page, int wait); 969 970/* readahead.c */ 971#define VM_MAX_READAHEAD 128 /* kbytes */ 972#define VM_MIN_READAHEAD 16 /* kbytes (includes current page) */ 973#define VM_MAX_CACHE_HIT 256 /* max pages in a row in cache before 974 * turning readahead off */ 975 976int do_page_cache_readahead(struct address_space *mapping, struct file *filp, 977 pgoff_t offset, unsigned long nr_to_read); 978int force_page_cache_readahead(struct address_space *mapping, struct file *filp, 979 pgoff_t offset, unsigned long nr_to_read); 980unsigned long page_cache_readahead(struct address_space *mapping, 981 struct file_ra_state *ra, 982 struct file *filp, 983 pgoff_t offset, 984 unsigned long size); 985void handle_ra_miss(struct address_space *mapping, 986 struct file_ra_state *ra, pgoff_t offset); 987unsigned long max_sane_readahead(unsigned long nr); 988 989/* Do stack extension */ 990extern int expand_stack(struct vm_area_struct *vma, unsigned long address); 991#ifdef CONFIG_IA64 992extern int expand_upwards(struct vm_area_struct *vma, unsigned long address); 993#endif 994 995/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */ 996extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr); 997extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr, 998 struct vm_area_struct **pprev); 999 1000/* Look up the first VMA which intersects the interval start_addr..end_addr-1, 1001 NULL if none. Assume start_addr < end_addr. */ 1002static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr) 1003{ 1004 struct vm_area_struct * vma = find_vma(mm,start_addr); 1005 1006 if (vma && end_addr <= vma->vm_start) 1007 vma = NULL; 1008 return vma; 1009} 1010 1011static inline unsigned long vma_pages(struct vm_area_struct *vma) 1012{ 1013 return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; 1014} 1015 1016struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr); 1017struct page *vmalloc_to_page(void *addr); 1018unsigned long vmalloc_to_pfn(void *addr); 1019int remap_pfn_range(struct vm_area_struct *, unsigned long addr, 1020 unsigned long pfn, unsigned long size, pgprot_t); 1021int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *); 1022 1023struct page *follow_page(struct vm_area_struct *, unsigned long address, 1024 unsigned int foll_flags); 1025#define FOLL_WRITE 0x01 /* check pte is writable */ 1026#define FOLL_TOUCH 0x02 /* mark page accessed */ 1027#define FOLL_GET 0x04 /* do get_page on page */ 1028#define FOLL_ANON 0x08 /* give ZERO_PAGE if no pgtable */ 1029 1030#ifdef CONFIG_PROC_FS 1031void vm_stat_account(struct mm_struct *, unsigned long, struct file *, long); 1032#else 1033static inline void vm_stat_account(struct mm_struct *mm, 1034 unsigned long flags, struct file *file, long pages) 1035{ 1036} 1037#endif /* CONFIG_PROC_FS */ 1038 1039#ifndef CONFIG_DEBUG_PAGEALLOC 1040static inline void 1041kernel_map_pages(struct page *page, int numpages, int enable) 1042{ 1043 if (!PageHighMem(page) && !enable) 1044 debug_check_no_locks_freed(page_address(page), 1045 numpages * PAGE_SIZE); 1046} 1047#endif 1048 1049extern struct vm_area_struct *get_gate_vma(struct task_struct *tsk); 1050#ifdef __HAVE_ARCH_GATE_AREA 1051int in_gate_area_no_task(unsigned long addr); 1052int in_gate_area(struct task_struct *task, unsigned long addr); 1053#else 1054int in_gate_area_no_task(unsigned long addr); 1055#define in_gate_area(task, addr) ({(void)task; in_gate_area_no_task(addr);}) 1056#endif /* __HAVE_ARCH_GATE_AREA */ 1057 1058/* /proc/<pid>/oom_adj set to -17 protects from the oom-killer */ 1059#define OOM_DISABLE -17 1060 1061int drop_caches_sysctl_handler(struct ctl_table *, int, struct file *, 1062 void __user *, size_t *, loff_t *); 1063unsigned long shrink_slab(unsigned long scanned, gfp_t gfp_mask, 1064 unsigned long lru_pages); 1065void drop_pagecache(void); 1066void drop_slab(void); 1067 1068#ifndef CONFIG_MMU 1069#define randomize_va_space 0 1070#else 1071extern int randomize_va_space; 1072#endif 1073 1074const char *arch_vma_name(struct vm_area_struct *vma); 1075 1076#endif /* __KERNEL__ */ 1077#endif /* _LINUX_MM_H */ 1078