1#ifndef _LINUX_MMZONE_H 2#define _LINUX_MMZONE_H 3 4#ifndef __ASSEMBLY__ 5#ifndef __GENERATING_BOUNDS_H 6 7#include <linux/spinlock.h> 8#include <linux/list.h> 9#include <linux/wait.h> 10#include <linux/bitops.h> 11#include <linux/cache.h> 12#include <linux/threads.h> 13#include <linux/numa.h> 14#include <linux/init.h> 15#include <linux/seqlock.h> 16#include <linux/nodemask.h> 17#include <linux/pageblock-flags.h> 18#include <generated/bounds.h> 19#include <linux/atomic.h> 20#include <asm/page.h> 21 22/* Free memory management - zoned buddy allocator. */ 23#ifndef CONFIG_FORCE_MAX_ZONEORDER 24#define MAX_ORDER 11 25#else 26#define MAX_ORDER CONFIG_FORCE_MAX_ZONEORDER 27#endif 28#define MAX_ORDER_NR_PAGES (1 << (MAX_ORDER - 1)) 29 30/* 31 * PAGE_ALLOC_COSTLY_ORDER is the order at which allocations are deemed 32 * costly to service. That is between allocation orders which should 33 * coelesce naturally under reasonable reclaim pressure and those which 34 * will not. 35 */ 36#define PAGE_ALLOC_COSTLY_ORDER 3 37 38#define MIGRATE_UNMOVABLE 0 39#define MIGRATE_RECLAIMABLE 1 40#define MIGRATE_MOVABLE 2 41#define MIGRATE_PCPTYPES 3 /* the number of types on the pcp lists */ 42#define MIGRATE_RESERVE 3 43#define MIGRATE_ISOLATE 4 /* can't allocate from here */ 44#define MIGRATE_TYPES 5 45 46#define for_each_migratetype_order(order, type) \ 47 for (order = 0; order < MAX_ORDER; order++) \ 48 for (type = 0; type < MIGRATE_TYPES; type++) 49 50extern int page_group_by_mobility_disabled; 51 52static inline int get_pageblock_migratetype(struct page *page) 53{ 54 return get_pageblock_flags_group(page, PB_migrate, PB_migrate_end); 55} 56 57struct free_area { 58 struct list_head free_list[MIGRATE_TYPES]; 59 unsigned long nr_free; 60}; 61 62struct pglist_data; 63 64/* 65 * zone->lock and zone->lru_lock are two of the hottest locks in the kernel. 66 * So add a wild amount of padding here to ensure that they fall into separate 67 * cachelines. There are very few zone structures in the machine, so space 68 * consumption is not a concern here. 69 */ 70#if defined(CONFIG_SMP) 71struct zone_padding { 72 char x[0]; 73} ____cacheline_internodealigned_in_smp; 74#define ZONE_PADDING(name) struct zone_padding name; 75#else 76#define ZONE_PADDING(name) 77#endif 78 79enum zone_stat_item { 80 /* First 128 byte cacheline (assuming 64 bit words) */ 81 NR_FREE_PAGES, 82 NR_LRU_BASE, 83 NR_INACTIVE_ANON = NR_LRU_BASE, /* must match order of LRU_[IN]ACTIVE */ 84 NR_ACTIVE_ANON, /* " " " " " */ 85 NR_INACTIVE_FILE, /* " " " " " */ 86 NR_ACTIVE_FILE, /* " " " " " */ 87 NR_UNEVICTABLE, /* " " " " " */ 88 NR_MLOCK, /* mlock()ed pages found and moved off LRU */ 89 NR_ANON_PAGES, /* Mapped anonymous pages */ 90 NR_FILE_MAPPED, /* pagecache pages mapped into pagetables. 91 only modified from process context */ 92 NR_FILE_PAGES, 93 NR_FILE_DIRTY, 94 NR_WRITEBACK, 95 NR_SLAB_RECLAIMABLE, 96 NR_SLAB_UNRECLAIMABLE, 97 NR_PAGETABLE, /* used for pagetables */ 98 NR_KERNEL_STACK, 99 /* Second 128 byte cacheline */ 100 NR_UNSTABLE_NFS, /* NFS unstable pages */ 101 NR_BOUNCE, 102 NR_VMSCAN_WRITE, 103 NR_VMSCAN_IMMEDIATE, /* Prioritise for reclaim when writeback ends */ 104 NR_WRITEBACK_TEMP, /* Writeback using temporary buffers */ 105 NR_ISOLATED_ANON, /* Temporary isolated pages from anon lru */ 106 NR_ISOLATED_FILE, /* Temporary isolated pages from file lru */ 107 NR_SHMEM, /* shmem pages (included tmpfs/GEM pages) */ 108 NR_DIRTIED, /* page dirtyings since bootup */ 109 NR_WRITTEN, /* page writings since bootup */ 110#ifdef CONFIG_NUMA 111 NUMA_HIT, /* allocated in intended node */ 112 NUMA_MISS, /* allocated in non intended node */ 113 NUMA_FOREIGN, /* was intended here, hit elsewhere */ 114 NUMA_INTERLEAVE_HIT, /* interleaver preferred this zone */ 115 NUMA_LOCAL, /* allocation from local node */ 116 NUMA_OTHER, /* allocation from other node */ 117#endif 118 NR_ANON_TRANSPARENT_HUGEPAGES, 119 NR_VM_ZONE_STAT_ITEMS }; 120 121/* 122 * We do arithmetic on the LRU lists in various places in the code, 123 * so it is important to keep the active lists LRU_ACTIVE higher in 124 * the array than the corresponding inactive lists, and to keep 125 * the *_FILE lists LRU_FILE higher than the corresponding _ANON lists. 126 * 127 * This has to be kept in sync with the statistics in zone_stat_item 128 * above and the descriptions in vmstat_text in mm/vmstat.c 129 */ 130#define LRU_BASE 0 131#define LRU_ACTIVE 1 132#define LRU_FILE 2 133 134enum lru_list { 135 LRU_INACTIVE_ANON = LRU_BASE, 136 LRU_ACTIVE_ANON = LRU_BASE + LRU_ACTIVE, 137 LRU_INACTIVE_FILE = LRU_BASE + LRU_FILE, 138 LRU_ACTIVE_FILE = LRU_BASE + LRU_FILE + LRU_ACTIVE, 139 LRU_UNEVICTABLE, 140 NR_LRU_LISTS 141}; 142 143#define for_each_lru(lru) for (lru = 0; lru < NR_LRU_LISTS; lru++) 144 145#define for_each_evictable_lru(lru) for (lru = 0; lru <= LRU_ACTIVE_FILE; lru++) 146 147static inline int is_file_lru(enum lru_list lru) 148{ 149 return (lru == LRU_INACTIVE_FILE || lru == LRU_ACTIVE_FILE); 150} 151 152static inline int is_active_lru(enum lru_list lru) 153{ 154 return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE); 155} 156 157static inline int is_unevictable_lru(enum lru_list lru) 158{ 159 return (lru == LRU_UNEVICTABLE); 160} 161 162struct lruvec { 163 struct list_head lists[NR_LRU_LISTS]; 164}; 165 166/* Mask used at gathering information at once (see memcontrol.c) */ 167#define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE)) 168#define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON)) 169#define LRU_ALL_EVICTABLE (LRU_ALL_FILE | LRU_ALL_ANON) 170#define LRU_ALL ((1 << NR_LRU_LISTS) - 1) 171 172/* Isolate inactive pages */ 173#define ISOLATE_INACTIVE ((__force isolate_mode_t)0x1) 174/* Isolate active pages */ 175#define ISOLATE_ACTIVE ((__force isolate_mode_t)0x2) 176/* Isolate clean file */ 177#define ISOLATE_CLEAN ((__force isolate_mode_t)0x4) 178/* Isolate unmapped file */ 179#define ISOLATE_UNMAPPED ((__force isolate_mode_t)0x8) 180/* Isolate for asynchronous migration */ 181#define ISOLATE_ASYNC_MIGRATE ((__force isolate_mode_t)0x10) 182 183/* LRU Isolation modes. */ 184typedef unsigned __bitwise__ isolate_mode_t; 185 186enum zone_watermarks { 187 WMARK_MIN, 188 WMARK_LOW, 189 WMARK_HIGH, 190 NR_WMARK 191}; 192 193#define min_wmark_pages(z) (z->watermark[WMARK_MIN]) 194#define low_wmark_pages(z) (z->watermark[WMARK_LOW]) 195#define high_wmark_pages(z) (z->watermark[WMARK_HIGH]) 196 197struct per_cpu_pages { 198 int count; /* number of pages in the list */ 199 int high; /* high watermark, emptying needed */ 200 int batch; /* chunk size for buddy add/remove */ 201 202 /* Lists of pages, one per migrate type stored on the pcp-lists */ 203 struct list_head lists[MIGRATE_PCPTYPES]; 204}; 205 206struct per_cpu_pageset { 207 struct per_cpu_pages pcp; 208#ifdef CONFIG_NUMA 209 s8 expire; 210#endif 211#ifdef CONFIG_SMP 212 s8 stat_threshold; 213 s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS]; 214#endif 215}; 216 217#endif /* !__GENERATING_BOUNDS.H */ 218 219enum zone_type { 220#ifdef CONFIG_ZONE_DMA 221 /* 222 * ZONE_DMA is used when there are devices that are not able 223 * to do DMA to all of addressable memory (ZONE_NORMAL). Then we 224 * carve out the portion of memory that is needed for these devices. 225 * The range is arch specific. 226 * 227 * Some examples 228 * 229 * Architecture Limit 230 * --------------------------- 231 * parisc, ia64, sparc <4G 232 * s390 <2G 233 * arm Various 234 * alpha Unlimited or 0-16MB. 235 * 236 * i386, x86_64 and multiple other arches 237 * <16M. 238 */ 239 ZONE_DMA, 240#endif 241#ifdef CONFIG_ZONE_DMA32 242 /* 243 * x86_64 needs two ZONE_DMAs because it supports devices that are 244 * only able to do DMA to the lower 16M but also 32 bit devices that 245 * can only do DMA areas below 4G. 246 */ 247 ZONE_DMA32, 248#endif 249 /* 250 * Normal addressable memory is in ZONE_NORMAL. DMA operations can be 251 * performed on pages in ZONE_NORMAL if the DMA devices support 252 * transfers to all addressable memory. 253 */ 254 ZONE_NORMAL, 255#ifdef CONFIG_HIGHMEM 256 /* 257 * A memory area that is only addressable by the kernel through 258 * mapping portions into its own address space. This is for example 259 * used by i386 to allow the kernel to address the memory beyond 260 * 900MB. The kernel will set up special mappings (page 261 * table entries on i386) for each page that the kernel needs to 262 * access. 263 */ 264 ZONE_HIGHMEM, 265#endif 266 ZONE_MOVABLE, 267 __MAX_NR_ZONES 268}; 269 270#ifndef __GENERATING_BOUNDS_H 271 272/* 273 * When a memory allocation must conform to specific limitations (such 274 * as being suitable for DMA) the caller will pass in hints to the 275 * allocator in the gfp_mask, in the zone modifier bits. These bits 276 * are used to select a priority ordered list of memory zones which 277 * match the requested limits. See gfp_zone() in include/linux/gfp.h 278 */ 279 280#if MAX_NR_ZONES < 2 281#define ZONES_SHIFT 0 282#elif MAX_NR_ZONES <= 2 283#define ZONES_SHIFT 1 284#elif MAX_NR_ZONES <= 4 285#define ZONES_SHIFT 2 286#else 287#error ZONES_SHIFT -- too many zones configured adjust calculation 288#endif 289 290struct zone_reclaim_stat { 291 /* 292 * The pageout code in vmscan.c keeps track of how many of the 293 * mem/swap backed and file backed pages are refeferenced. 294 * The higher the rotated/scanned ratio, the more valuable 295 * that cache is. 296 * 297 * The anon LRU stats live in [0], file LRU stats in [1] 298 */ 299 unsigned long recent_rotated[2]; 300 unsigned long recent_scanned[2]; 301}; 302 303struct zone { 304 /* Fields commonly accessed by the page allocator */ 305 306 /* zone watermarks, access with *_wmark_pages(zone) macros */ 307 unsigned long watermark[NR_WMARK]; 308 309 /* 310 * When free pages are below this point, additional steps are taken 311 * when reading the number of free pages to avoid per-cpu counter 312 * drift allowing watermarks to be breached 313 */ 314 unsigned long percpu_drift_mark; 315 316 /* 317 * We don't know if the memory that we're going to allocate will be freeable 318 * or/and it will be released eventually, so to avoid totally wasting several 319 * GB of ram we must reserve some of the lower zone memory (otherwise we risk 320 * to run OOM on the lower zones despite there's tons of freeable ram 321 * on the higher zones). This array is recalculated at runtime if the 322 * sysctl_lowmem_reserve_ratio sysctl changes. 323 */ 324 unsigned long lowmem_reserve[MAX_NR_ZONES]; 325 326 /* 327 * This is a per-zone reserve of pages that should not be 328 * considered dirtyable memory. 329 */ 330 unsigned long dirty_balance_reserve; 331 332#ifdef CONFIG_NUMA 333 int node; 334 /* 335 * zone reclaim becomes active if more unmapped pages exist. 336 */ 337 unsigned long min_unmapped_pages; 338 unsigned long min_slab_pages; 339#endif 340 struct per_cpu_pageset __percpu *pageset; 341 /* 342 * free areas of different sizes 343 */ 344 spinlock_t lock; 345 int all_unreclaimable; /* All pages pinned */ 346#ifdef CONFIG_MEMORY_HOTPLUG 347 /* see spanned/present_pages for more description */ 348 seqlock_t span_seqlock; 349#endif 350 struct free_area free_area[MAX_ORDER]; 351 352#ifndef CONFIG_SPARSEMEM 353 /* 354 * Flags for a pageblock_nr_pages block. See pageblock-flags.h. 355 * In SPARSEMEM, this map is stored in struct mem_section 356 */ 357 unsigned long *pageblock_flags; 358#endif /* CONFIG_SPARSEMEM */ 359 360#ifdef CONFIG_COMPACTION 361 /* 362 * On compaction failure, 1<<compact_defer_shift compactions 363 * are skipped before trying again. The number attempted since 364 * last failure is tracked with compact_considered. 365 */ 366 unsigned int compact_considered; 367 unsigned int compact_defer_shift; 368 int compact_order_failed; 369#endif 370 371 ZONE_PADDING(_pad1_) 372 373 /* Fields commonly accessed by the page reclaim scanner */ 374 spinlock_t lru_lock; 375 struct lruvec lruvec; 376 377 struct zone_reclaim_stat reclaim_stat; 378 379 unsigned long pages_scanned; /* since last reclaim */ 380 unsigned long flags; /* zone flags, see below */ 381 382 /* Zone statistics */ 383 atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; 384 385 /* 386 * The target ratio of ACTIVE_ANON to INACTIVE_ANON pages on 387 * this zone's LRU. Maintained by the pageout code. 388 */ 389 unsigned int inactive_ratio; 390 391 392 ZONE_PADDING(_pad2_) 393 /* Rarely used or read-mostly fields */ 394 395 /* 396 * wait_table -- the array holding the hash table 397 * wait_table_hash_nr_entries -- the size of the hash table array 398 * wait_table_bits -- wait_table_size == (1 << wait_table_bits) 399 * 400 * The purpose of all these is to keep track of the people 401 * waiting for a page to become available and make them 402 * runnable again when possible. The trouble is that this 403 * consumes a lot of space, especially when so few things 404 * wait on pages at a given time. So instead of using 405 * per-page waitqueues, we use a waitqueue hash table. 406 * 407 * The bucket discipline is to sleep on the same queue when 408 * colliding and wake all in that wait queue when removing. 409 * When something wakes, it must check to be sure its page is 410 * truly available, a la thundering herd. The cost of a 411 * collision is great, but given the expected load of the 412 * table, they should be so rare as to be outweighed by the 413 * benefits from the saved space. 414 * 415 * __wait_on_page_locked() and unlock_page() in mm/filemap.c, are the 416 * primary users of these fields, and in mm/page_alloc.c 417 * free_area_init_core() performs the initialization of them. 418 */ 419 wait_queue_head_t * wait_table; 420 unsigned long wait_table_hash_nr_entries; 421 unsigned long wait_table_bits; 422 423 /* 424 * Discontig memory support fields. 425 */ 426 struct pglist_data *zone_pgdat; 427 /* zone_start_pfn == zone_start_paddr >> PAGE_SHIFT */ 428 unsigned long zone_start_pfn; 429 430 /* 431 * zone_start_pfn, spanned_pages and present_pages are all 432 * protected by span_seqlock. It is a seqlock because it has 433 * to be read outside of zone->lock, and it is done in the main 434 * allocator path. But, it is written quite infrequently. 435 * 436 * The lock is declared along with zone->lock because it is 437 * frequently read in proximity to zone->lock. It's good to 438 * give them a chance of being in the same cacheline. 439 */ 440 unsigned long spanned_pages; /* total size, including holes */ 441 unsigned long present_pages; /* amount of memory (excluding holes) */ 442 443 /* 444 * rarely used fields: 445 */ 446 const char *name; 447} ____cacheline_internodealigned_in_smp; 448 449typedef enum { 450 ZONE_RECLAIM_LOCKED, /* prevents concurrent reclaim */ 451 ZONE_OOM_LOCKED, /* zone is in OOM killer zonelist */ 452 ZONE_CONGESTED, /* zone has many dirty pages backed by 453 * a congested BDI 454 */ 455} zone_flags_t; 456 457static inline void zone_set_flag(struct zone *zone, zone_flags_t flag) 458{ 459 set_bit(flag, &zone->flags); 460} 461 462static inline int zone_test_and_set_flag(struct zone *zone, zone_flags_t flag) 463{ 464 return test_and_set_bit(flag, &zone->flags); 465} 466 467static inline void zone_clear_flag(struct zone *zone, zone_flags_t flag) 468{ 469 clear_bit(flag, &zone->flags); 470} 471 472static inline int zone_is_reclaim_congested(const struct zone *zone) 473{ 474 return test_bit(ZONE_CONGESTED, &zone->flags); 475} 476 477static inline int zone_is_reclaim_locked(const struct zone *zone) 478{ 479 return test_bit(ZONE_RECLAIM_LOCKED, &zone->flags); 480} 481 482static inline int zone_is_oom_locked(const struct zone *zone) 483{ 484 return test_bit(ZONE_OOM_LOCKED, &zone->flags); 485} 486 487/* 488 * The "priority" of VM scanning is how much of the queues we will scan in one 489 * go. A value of 12 for DEF_PRIORITY implies that we will scan 1/4096th of the 490 * queues ("queue_length >> 12") during an aging round. 491 */ 492#define DEF_PRIORITY 12 493 494/* Maximum number of zones on a zonelist */ 495#define MAX_ZONES_PER_ZONELIST (MAX_NUMNODES * MAX_NR_ZONES) 496 497#ifdef CONFIG_NUMA 498 499/* 500 * The NUMA zonelists are doubled because we need zonelists that restrict the 501 * allocations to a single node for GFP_THISNODE. 502 * 503 * [0] : Zonelist with fallback 504 * [1] : No fallback (GFP_THISNODE) 505 */ 506#define MAX_ZONELISTS 2 507 508 509/* 510 * We cache key information from each zonelist for smaller cache 511 * footprint when scanning for free pages in get_page_from_freelist(). 512 * 513 * 1) The BITMAP fullzones tracks which zones in a zonelist have come 514 * up short of free memory since the last time (last_fullzone_zap) 515 * we zero'd fullzones. 516 * 2) The array z_to_n[] maps each zone in the zonelist to its node 517 * id, so that we can efficiently evaluate whether that node is 518 * set in the current tasks mems_allowed. 519 * 520 * Both fullzones and z_to_n[] are one-to-one with the zonelist, 521 * indexed by a zones offset in the zonelist zones[] array. 522 * 523 * The get_page_from_freelist() routine does two scans. During the 524 * first scan, we skip zones whose corresponding bit in 'fullzones' 525 * is set or whose corresponding node in current->mems_allowed (which 526 * comes from cpusets) is not set. During the second scan, we bypass 527 * this zonelist_cache, to ensure we look methodically at each zone. 528 * 529 * Once per second, we zero out (zap) fullzones, forcing us to 530 * reconsider nodes that might have regained more free memory. 531 * The field last_full_zap is the time we last zapped fullzones. 532 * 533 * This mechanism reduces the amount of time we waste repeatedly 534 * reexaming zones for free memory when they just came up low on 535 * memory momentarilly ago. 536 * 537 * The zonelist_cache struct members logically belong in struct 538 * zonelist. However, the mempolicy zonelists constructed for 539 * MPOL_BIND are intentionally variable length (and usually much 540 * shorter). A general purpose mechanism for handling structs with 541 * multiple variable length members is more mechanism than we want 542 * here. We resort to some special case hackery instead. 543 * 544 * The MPOL_BIND zonelists don't need this zonelist_cache (in good 545 * part because they are shorter), so we put the fixed length stuff 546 * at the front of the zonelist struct, ending in a variable length 547 * zones[], as is needed by MPOL_BIND. 548 * 549 * Then we put the optional zonelist cache on the end of the zonelist 550 * struct. This optional stuff is found by a 'zlcache_ptr' pointer in 551 * the fixed length portion at the front of the struct. This pointer 552 * both enables us to find the zonelist cache, and in the case of 553 * MPOL_BIND zonelists, (which will just set the zlcache_ptr to NULL) 554 * to know that the zonelist cache is not there. 555 * 556 * The end result is that struct zonelists come in two flavors: 557 * 1) The full, fixed length version, shown below, and 558 * 2) The custom zonelists for MPOL_BIND. 559 * The custom MPOL_BIND zonelists have a NULL zlcache_ptr and no zlcache. 560 * 561 * Even though there may be multiple CPU cores on a node modifying 562 * fullzones or last_full_zap in the same zonelist_cache at the same 563 * time, we don't lock it. This is just hint data - if it is wrong now 564 * and then, the allocator will still function, perhaps a bit slower. 565 */ 566 567 568struct zonelist_cache { 569 unsigned short z_to_n[MAX_ZONES_PER_ZONELIST]; /* zone->nid */ 570 DECLARE_BITMAP(fullzones, MAX_ZONES_PER_ZONELIST); /* zone full? */ 571 unsigned long last_full_zap; /* when last zap'd (jiffies) */ 572}; 573#else 574#define MAX_ZONELISTS 1 575struct zonelist_cache; 576#endif 577 578/* 579 * This struct contains information about a zone in a zonelist. It is stored 580 * here to avoid dereferences into large structures and lookups of tables 581 */ 582struct zoneref { 583 struct zone *zone; /* Pointer to actual zone */ 584 int zone_idx; /* zone_idx(zoneref->zone) */ 585}; 586 587/* 588 * One allocation request operates on a zonelist. A zonelist 589 * is a list of zones, the first one is the 'goal' of the 590 * allocation, the other zones are fallback zones, in decreasing 591 * priority. 592 * 593 * If zlcache_ptr is not NULL, then it is just the address of zlcache, 594 * as explained above. If zlcache_ptr is NULL, there is no zlcache. 595 * * 596 * To speed the reading of the zonelist, the zonerefs contain the zone index 597 * of the entry being read. Helper functions to access information given 598 * a struct zoneref are 599 * 600 * zonelist_zone() - Return the struct zone * for an entry in _zonerefs 601 * zonelist_zone_idx() - Return the index of the zone for an entry 602 * zonelist_node_idx() - Return the index of the node for an entry 603 */ 604struct zonelist { 605 struct zonelist_cache *zlcache_ptr; // NULL or &zlcache 606 struct zoneref _zonerefs[MAX_ZONES_PER_ZONELIST + 1]; 607#ifdef CONFIG_NUMA 608 struct zonelist_cache zlcache; // optional ... 609#endif 610}; 611 612#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 613struct node_active_region { 614 unsigned long start_pfn; 615 unsigned long end_pfn; 616 int nid; 617}; 618#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 619 620#ifndef CONFIG_DISCONTIGMEM 621/* The array of struct pages - for discontigmem use pgdat->lmem_map */ 622extern struct page *mem_map; 623#endif 624 625/* 626 * The pg_data_t structure is used in machines with CONFIG_DISCONTIGMEM 627 * (mostly NUMA machines?) to denote a higher-level memory zone than the 628 * zone denotes. 629 * 630 * On NUMA machines, each NUMA node would have a pg_data_t to describe 631 * it's memory layout. 632 * 633 * Memory statistics and page replacement data structures are maintained on a 634 * per-zone basis. 635 */ 636struct bootmem_data; 637typedef struct pglist_data { 638 struct zone node_zones[MAX_NR_ZONES]; 639 struct zonelist node_zonelists[MAX_ZONELISTS]; 640 int nr_zones; 641#ifdef CONFIG_FLAT_NODE_MEM_MAP /* means !SPARSEMEM */ 642 struct page *node_mem_map; 643#ifdef CONFIG_CGROUP_MEM_RES_CTLR 644 struct page_cgroup *node_page_cgroup; 645#endif 646#endif 647#ifndef CONFIG_NO_BOOTMEM 648 struct bootmem_data *bdata; 649#endif 650#ifdef CONFIG_MEMORY_HOTPLUG 651 /* 652 * Must be held any time you expect node_start_pfn, node_present_pages 653 * or node_spanned_pages stay constant. Holding this will also 654 * guarantee that any pfn_valid() stays that way. 655 * 656 * Nests above zone->lock and zone->size_seqlock. 657 */ 658 spinlock_t node_size_lock; 659#endif 660 unsigned long node_start_pfn; 661 unsigned long node_present_pages; /* total number of physical pages */ 662 unsigned long node_spanned_pages; /* total size of physical page 663 range, including holes */ 664 int node_id; 665 wait_queue_head_t kswapd_wait; 666 struct task_struct *kswapd; /* Protected by lock_memory_hotplug() */ 667 int kswapd_max_order; 668 enum zone_type classzone_idx; 669} pg_data_t; 670 671#define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages) 672#define node_spanned_pages(nid) (NODE_DATA(nid)->node_spanned_pages) 673#ifdef CONFIG_FLAT_NODE_MEM_MAP 674#define pgdat_page_nr(pgdat, pagenr) ((pgdat)->node_mem_map + (pagenr)) 675#else 676#define pgdat_page_nr(pgdat, pagenr) pfn_to_page((pgdat)->node_start_pfn + (pagenr)) 677#endif 678#define nid_page_nr(nid, pagenr) pgdat_page_nr(NODE_DATA(nid),(pagenr)) 679 680#define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn) 681 682#define node_end_pfn(nid) ({\ 683 pg_data_t *__pgdat = NODE_DATA(nid);\ 684 __pgdat->node_start_pfn + __pgdat->node_spanned_pages;\ 685}) 686 687#include <linux/memory_hotplug.h> 688 689extern struct mutex zonelists_mutex; 690void build_all_zonelists(void *data); 691void wakeup_kswapd(struct zone *zone, int order, enum zone_type classzone_idx); 692bool zone_watermark_ok(struct zone *z, int order, unsigned long mark, 693 int classzone_idx, int alloc_flags); 694bool zone_watermark_ok_safe(struct zone *z, int order, unsigned long mark, 695 int classzone_idx, int alloc_flags); 696enum memmap_context { 697 MEMMAP_EARLY, 698 MEMMAP_HOTPLUG, 699}; 700extern int init_currently_empty_zone(struct zone *zone, unsigned long start_pfn, 701 unsigned long size, 702 enum memmap_context context); 703 704#ifdef CONFIG_HAVE_MEMORY_PRESENT 705void memory_present(int nid, unsigned long start, unsigned long end); 706#else 707static inline void memory_present(int nid, unsigned long start, unsigned long end) {} 708#endif 709 710#ifdef CONFIG_HAVE_MEMORYLESS_NODES 711int local_memory_node(int node_id); 712#else 713static inline int local_memory_node(int node_id) { return node_id; }; 714#endif 715 716#ifdef CONFIG_NEED_NODE_MEMMAP_SIZE 717unsigned long __init node_memmap_size_bytes(int, unsigned long, unsigned long); 718#endif 719 720/* 721 * zone_idx() returns 0 for the ZONE_DMA zone, 1 for the ZONE_NORMAL zone, etc. 722 */ 723#define zone_idx(zone) ((zone) - (zone)->zone_pgdat->node_zones) 724 725static inline int populated_zone(struct zone *zone) 726{ 727 return (!!zone->present_pages); 728} 729 730extern int movable_zone; 731 732static inline int zone_movable_is_highmem(void) 733{ 734#if defined(CONFIG_HIGHMEM) && defined(CONFIG_HAVE_MEMBLOCK_NODE) 735 return movable_zone == ZONE_HIGHMEM; 736#else 737 return 0; 738#endif 739} 740 741static inline int is_highmem_idx(enum zone_type idx) 742{ 743#ifdef CONFIG_HIGHMEM 744 return (idx == ZONE_HIGHMEM || 745 (idx == ZONE_MOVABLE && zone_movable_is_highmem())); 746#else 747 return 0; 748#endif 749} 750 751static inline int is_normal_idx(enum zone_type idx) 752{ 753 return (idx == ZONE_NORMAL); 754} 755 756/** 757 * is_highmem - helper function to quickly check if a struct zone is a 758 * highmem zone or not. This is an attempt to keep references 759 * to ZONE_{DMA/NORMAL/HIGHMEM/etc} in general code to a minimum. 760 * @zone - pointer to struct zone variable 761 */ 762static inline int is_highmem(struct zone *zone) 763{ 764#ifdef CONFIG_HIGHMEM 765 int zone_off = (char *)zone - (char *)zone->zone_pgdat->node_zones; 766 return zone_off == ZONE_HIGHMEM * sizeof(*zone) || 767 (zone_off == ZONE_MOVABLE * sizeof(*zone) && 768 zone_movable_is_highmem()); 769#else 770 return 0; 771#endif 772} 773 774static inline int is_normal(struct zone *zone) 775{ 776 return zone == zone->zone_pgdat->node_zones + ZONE_NORMAL; 777} 778 779static inline int is_dma32(struct zone *zone) 780{ 781#ifdef CONFIG_ZONE_DMA32 782 return zone == zone->zone_pgdat->node_zones + ZONE_DMA32; 783#else 784 return 0; 785#endif 786} 787 788static inline int is_dma(struct zone *zone) 789{ 790#ifdef CONFIG_ZONE_DMA 791 return zone == zone->zone_pgdat->node_zones + ZONE_DMA; 792#else 793 return 0; 794#endif 795} 796 797/* These two functions are used to setup the per zone pages min values */ 798struct ctl_table; 799int min_free_kbytes_sysctl_handler(struct ctl_table *, int, 800 void __user *, size_t *, loff_t *); 801extern int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1]; 802int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *, int, 803 void __user *, size_t *, loff_t *); 804int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *, int, 805 void __user *, size_t *, loff_t *); 806int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *, int, 807 void __user *, size_t *, loff_t *); 808int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *, int, 809 void __user *, size_t *, loff_t *); 810 811extern int numa_zonelist_order_handler(struct ctl_table *, int, 812 void __user *, size_t *, loff_t *); 813extern char numa_zonelist_order[]; 814#define NUMA_ZONELIST_ORDER_LEN 16 /* string buffer size */ 815 816#ifndef CONFIG_NEED_MULTIPLE_NODES 817 818extern struct pglist_data contig_page_data; 819#define NODE_DATA(nid) (&contig_page_data) 820#define NODE_MEM_MAP(nid) mem_map 821 822#else /* CONFIG_NEED_MULTIPLE_NODES */ 823 824#include <asm/mmzone.h> 825 826#endif /* !CONFIG_NEED_MULTIPLE_NODES */ 827 828extern struct pglist_data *first_online_pgdat(void); 829extern struct pglist_data *next_online_pgdat(struct pglist_data *pgdat); 830extern struct zone *next_zone(struct zone *zone); 831 832/** 833 * for_each_online_pgdat - helper macro to iterate over all online nodes 834 * @pgdat - pointer to a pg_data_t variable 835 */ 836#define for_each_online_pgdat(pgdat) \ 837 for (pgdat = first_online_pgdat(); \ 838 pgdat; \ 839 pgdat = next_online_pgdat(pgdat)) 840/** 841 * for_each_zone - helper macro to iterate over all memory zones 842 * @zone - pointer to struct zone variable 843 * 844 * The user only needs to declare the zone variable, for_each_zone 845 * fills it in. 846 */ 847#define for_each_zone(zone) \ 848 for (zone = (first_online_pgdat())->node_zones; \ 849 zone; \ 850 zone = next_zone(zone)) 851 852#define for_each_populated_zone(zone) \ 853 for (zone = (first_online_pgdat())->node_zones; \ 854 zone; \ 855 zone = next_zone(zone)) \ 856 if (!populated_zone(zone)) \ 857 ; /* do nothing */ \ 858 else 859 860static inline struct zone *zonelist_zone(struct zoneref *zoneref) 861{ 862 return zoneref->zone; 863} 864 865static inline int zonelist_zone_idx(struct zoneref *zoneref) 866{ 867 return zoneref->zone_idx; 868} 869 870static inline int zonelist_node_idx(struct zoneref *zoneref) 871{ 872#ifdef CONFIG_NUMA 873 /* zone_to_nid not available in this context */ 874 return zoneref->zone->node; 875#else 876 return 0; 877#endif /* CONFIG_NUMA */ 878} 879 880/** 881 * next_zones_zonelist - Returns the next zone at or below highest_zoneidx within the allowed nodemask using a cursor within a zonelist as a starting point 882 * @z - The cursor used as a starting point for the search 883 * @highest_zoneidx - The zone index of the highest zone to return 884 * @nodes - An optional nodemask to filter the zonelist with 885 * @zone - The first suitable zone found is returned via this parameter 886 * 887 * This function returns the next zone at or below a given zone index that is 888 * within the allowed nodemask using a cursor as the starting point for the 889 * search. The zoneref returned is a cursor that represents the current zone 890 * being examined. It should be advanced by one before calling 891 * next_zones_zonelist again. 892 */ 893struct zoneref *next_zones_zonelist(struct zoneref *z, 894 enum zone_type highest_zoneidx, 895 nodemask_t *nodes, 896 struct zone **zone); 897 898/** 899 * first_zones_zonelist - Returns the first zone at or below highest_zoneidx within the allowed nodemask in a zonelist 900 * @zonelist - The zonelist to search for a suitable zone 901 * @highest_zoneidx - The zone index of the highest zone to return 902 * @nodes - An optional nodemask to filter the zonelist with 903 * @zone - The first suitable zone found is returned via this parameter 904 * 905 * This function returns the first zone at or below a given zone index that is 906 * within the allowed nodemask. The zoneref returned is a cursor that can be 907 * used to iterate the zonelist with next_zones_zonelist by advancing it by 908 * one before calling. 909 */ 910static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist, 911 enum zone_type highest_zoneidx, 912 nodemask_t *nodes, 913 struct zone **zone) 914{ 915 return next_zones_zonelist(zonelist->_zonerefs, highest_zoneidx, nodes, 916 zone); 917} 918 919/** 920 * for_each_zone_zonelist_nodemask - helper macro to iterate over valid zones in a zonelist at or below a given zone index and within a nodemask 921 * @zone - The current zone in the iterator 922 * @z - The current pointer within zonelist->zones being iterated 923 * @zlist - The zonelist being iterated 924 * @highidx - The zone index of the highest zone to return 925 * @nodemask - Nodemask allowed by the allocator 926 * 927 * This iterator iterates though all zones at or below a given zone index and 928 * within a given nodemask 929 */ 930#define for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, nodemask) \ 931 for (z = first_zones_zonelist(zlist, highidx, nodemask, &zone); \ 932 zone; \ 933 z = next_zones_zonelist(++z, highidx, nodemask, &zone)) \ 934 935/** 936 * for_each_zone_zonelist - helper macro to iterate over valid zones in a zonelist at or below a given zone index 937 * @zone - The current zone in the iterator 938 * @z - The current pointer within zonelist->zones being iterated 939 * @zlist - The zonelist being iterated 940 * @highidx - The zone index of the highest zone to return 941 * 942 * This iterator iterates though all zones at or below a given zone index. 943 */ 944#define for_each_zone_zonelist(zone, z, zlist, highidx) \ 945 for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, NULL) 946 947#ifdef CONFIG_SPARSEMEM 948#include <asm/sparsemem.h> 949#endif 950 951#if !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) && \ 952 !defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) 953static inline unsigned long early_pfn_to_nid(unsigned long pfn) 954{ 955 return 0; 956} 957#endif 958 959#ifdef CONFIG_FLATMEM 960#define pfn_to_nid(pfn) (0) 961#endif 962 963#ifdef CONFIG_SPARSEMEM 964 965/* 966 * SECTION_SHIFT #bits space required to store a section # 967 * 968 * PA_SECTION_SHIFT physical address to/from section number 969 * PFN_SECTION_SHIFT pfn to/from section number 970 */ 971#define SECTIONS_SHIFT (MAX_PHYSMEM_BITS - SECTION_SIZE_BITS) 972 973#define PA_SECTION_SHIFT (SECTION_SIZE_BITS) 974#define PFN_SECTION_SHIFT (SECTION_SIZE_BITS - PAGE_SHIFT) 975 976#define NR_MEM_SECTIONS (1UL << SECTIONS_SHIFT) 977 978#define PAGES_PER_SECTION (1UL << PFN_SECTION_SHIFT) 979#define PAGE_SECTION_MASK (~(PAGES_PER_SECTION-1)) 980 981#define SECTION_BLOCKFLAGS_BITS \ 982 ((1UL << (PFN_SECTION_SHIFT - pageblock_order)) * NR_PAGEBLOCK_BITS) 983 984#if (MAX_ORDER - 1 + PAGE_SHIFT) > SECTION_SIZE_BITS 985#error Allocator MAX_ORDER exceeds SECTION_SIZE 986#endif 987 988#define pfn_to_section_nr(pfn) ((pfn) >> PFN_SECTION_SHIFT) 989#define section_nr_to_pfn(sec) ((sec) << PFN_SECTION_SHIFT) 990 991#define SECTION_ALIGN_UP(pfn) (((pfn) + PAGES_PER_SECTION - 1) & PAGE_SECTION_MASK) 992#define SECTION_ALIGN_DOWN(pfn) ((pfn) & PAGE_SECTION_MASK) 993 994struct page; 995struct page_cgroup; 996struct mem_section { 997 /* 998 * This is, logically, a pointer to an array of struct 999 * pages. However, it is stored with some other magic. 1000 * (see sparse.c::sparse_init_one_section()) 1001 * 1002 * Additionally during early boot we encode node id of 1003 * the location of the section here to guide allocation. 1004 * (see sparse.c::memory_present()) 1005 * 1006 * Making it a UL at least makes someone do a cast 1007 * before using it wrong. 1008 */ 1009 unsigned long section_mem_map; 1010 1011 /* See declaration of similar field in struct zone */ 1012 unsigned long *pageblock_flags; 1013#ifdef CONFIG_CGROUP_MEM_RES_CTLR 1014 /* 1015 * If !SPARSEMEM, pgdat doesn't have page_cgroup pointer. We use 1016 * section. (see memcontrol.h/page_cgroup.h about this.) 1017 */ 1018 struct page_cgroup *page_cgroup; 1019 unsigned long pad; 1020#endif 1021}; 1022 1023#ifdef CONFIG_SPARSEMEM_EXTREME 1024#define SECTIONS_PER_ROOT (PAGE_SIZE / sizeof (struct mem_section)) 1025#else 1026#define SECTIONS_PER_ROOT 1 1027#endif 1028 1029#define SECTION_NR_TO_ROOT(sec) ((sec) / SECTIONS_PER_ROOT) 1030#define NR_SECTION_ROOTS DIV_ROUND_UP(NR_MEM_SECTIONS, SECTIONS_PER_ROOT) 1031#define SECTION_ROOT_MASK (SECTIONS_PER_ROOT - 1) 1032 1033#ifdef CONFIG_SPARSEMEM_EXTREME 1034extern struct mem_section *mem_section[NR_SECTION_ROOTS]; 1035#else 1036extern struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT]; 1037#endif 1038 1039static inline struct mem_section *__nr_to_section(unsigned long nr) 1040{ 1041 if (!mem_section[SECTION_NR_TO_ROOT(nr)]) 1042 return NULL; 1043 return &mem_section[SECTION_NR_TO_ROOT(nr)][nr & SECTION_ROOT_MASK]; 1044} 1045extern int __section_nr(struct mem_section* ms); 1046extern unsigned long usemap_size(void); 1047 1048/* 1049 * We use the lower bits of the mem_map pointer to store 1050 * a little bit of information. There should be at least 1051 * 3 bits here due to 32-bit alignment. 1052 */ 1053#define SECTION_MARKED_PRESENT (1UL<<0) 1054#define SECTION_HAS_MEM_MAP (1UL<<1) 1055#define SECTION_MAP_LAST_BIT (1UL<<2) 1056#define SECTION_MAP_MASK (~(SECTION_MAP_LAST_BIT-1)) 1057#define SECTION_NID_SHIFT 2 1058 1059static inline struct page *__section_mem_map_addr(struct mem_section *section) 1060{ 1061 unsigned long map = section->section_mem_map; 1062 map &= SECTION_MAP_MASK; 1063 return (struct page *)map; 1064} 1065 1066static inline int present_section(struct mem_section *section) 1067{ 1068 return (section && (section->section_mem_map & SECTION_MARKED_PRESENT)); 1069} 1070 1071static inline int present_section_nr(unsigned long nr) 1072{ 1073 return present_section(__nr_to_section(nr)); 1074} 1075 1076static inline int valid_section(struct mem_section *section) 1077{ 1078 return (section && (section->section_mem_map & SECTION_HAS_MEM_MAP)); 1079} 1080 1081static inline int valid_section_nr(unsigned long nr) 1082{ 1083 return valid_section(__nr_to_section(nr)); 1084} 1085 1086static inline struct mem_section *__pfn_to_section(unsigned long pfn) 1087{ 1088 return __nr_to_section(pfn_to_section_nr(pfn)); 1089} 1090 1091#ifndef CONFIG_HAVE_ARCH_PFN_VALID 1092static inline int pfn_valid(unsigned long pfn) 1093{ 1094 if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS) 1095 return 0; 1096 return valid_section(__nr_to_section(pfn_to_section_nr(pfn))); 1097} 1098#endif 1099 1100static inline int pfn_present(unsigned long pfn) 1101{ 1102 if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS) 1103 return 0; 1104 return present_section(__nr_to_section(pfn_to_section_nr(pfn))); 1105} 1106 1107/* 1108 * These are _only_ used during initialisation, therefore they 1109 * can use __initdata ... They could have names to indicate 1110 * this restriction. 1111 */ 1112#ifdef CONFIG_NUMA 1113#define pfn_to_nid(pfn) \ 1114({ \ 1115 unsigned long __pfn_to_nid_pfn = (pfn); \ 1116 page_to_nid(pfn_to_page(__pfn_to_nid_pfn)); \ 1117}) 1118#else 1119#define pfn_to_nid(pfn) (0) 1120#endif 1121 1122#define early_pfn_valid(pfn) pfn_valid(pfn) 1123void sparse_init(void); 1124#else 1125#define sparse_init() do {} while (0) 1126#define sparse_index_init(_sec, _nid) do {} while (0) 1127#endif /* CONFIG_SPARSEMEM */ 1128 1129#ifdef CONFIG_NODES_SPAN_OTHER_NODES 1130bool early_pfn_in_nid(unsigned long pfn, int nid); 1131#else 1132#define early_pfn_in_nid(pfn, nid) (1) 1133#endif 1134 1135#ifndef early_pfn_valid 1136#define early_pfn_valid(pfn) (1) 1137#endif 1138 1139void memory_present(int nid, unsigned long start, unsigned long end); 1140unsigned long __init node_memmap_size_bytes(int, unsigned long, unsigned long); 1141 1142/* 1143 * If it is possible to have holes within a MAX_ORDER_NR_PAGES, then we 1144 * need to check pfn validility within that MAX_ORDER_NR_PAGES block. 1145 * pfn_valid_within() should be used in this case; we optimise this away 1146 * when we have no holes within a MAX_ORDER_NR_PAGES block. 1147 */ 1148#ifdef CONFIG_HOLES_IN_ZONE 1149#define pfn_valid_within(pfn) pfn_valid(pfn) 1150#else 1151#define pfn_valid_within(pfn) (1) 1152#endif 1153 1154#ifdef CONFIG_ARCH_HAS_HOLES_MEMORYMODEL 1155/* 1156 * pfn_valid() is meant to be able to tell if a given PFN has valid memmap 1157 * associated with it or not. In FLATMEM, it is expected that holes always 1158 * have valid memmap as long as there is valid PFNs either side of the hole. 1159 * In SPARSEMEM, it is assumed that a valid section has a memmap for the 1160 * entire section. 1161 * 1162 * However, an ARM, and maybe other embedded architectures in the future 1163 * free memmap backing holes to save memory on the assumption the memmap is 1164 * never used. The page_zone linkages are then broken even though pfn_valid() 1165 * returns true. A walker of the full memmap must then do this additional 1166 * check to ensure the memmap they are looking at is sane by making sure 1167 * the zone and PFN linkages are still valid. This is expensive, but walkers 1168 * of the full memmap are extremely rare. 1169 */ 1170int memmap_valid_within(unsigned long pfn, 1171 struct page *page, struct zone *zone); 1172#else 1173static inline int memmap_valid_within(unsigned long pfn, 1174 struct page *page, struct zone *zone) 1175{ 1176 return 1; 1177} 1178#endif /* CONFIG_ARCH_HAS_HOLES_MEMORYMODEL */ 1179 1180#endif /* !__GENERATING_BOUNDS.H */ 1181#endif /* !__ASSEMBLY__ */ 1182#endif /* _LINUX_MMZONE_H */ 1183