segment.c revision 187b5b8b3dfcfc73126f2743c89cc47df3bf07be
1/* 2 * fs/f2fs/segment.c 3 * 4 * Copyright (c) 2012 Samsung Electronics Co., Ltd. 5 * http://www.samsung.com/ 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11#include <linux/fs.h> 12#include <linux/f2fs_fs.h> 13#include <linux/bio.h> 14#include <linux/blkdev.h> 15#include <linux/prefetch.h> 16#include <linux/vmalloc.h> 17#include <linux/swap.h> 18 19#include "f2fs.h" 20#include "segment.h" 21#include "node.h" 22#include <trace/events/f2fs.h> 23 24#define __reverse_ffz(x) __reverse_ffs(~(x)) 25 26static struct kmem_cache *discard_entry_slab; 27 28/* 29 * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since 30 * MSB and LSB are reversed in a byte by f2fs_set_bit. 31 */ 32static inline unsigned long __reverse_ffs(unsigned long word) 33{ 34 int num = 0; 35 36#if BITS_PER_LONG == 64 37 if ((word & 0xffffffff) == 0) { 38 num += 32; 39 word >>= 32; 40 } 41#endif 42 if ((word & 0xffff) == 0) { 43 num += 16; 44 word >>= 16; 45 } 46 if ((word & 0xff) == 0) { 47 num += 8; 48 word >>= 8; 49 } 50 if ((word & 0xf0) == 0) 51 num += 4; 52 else 53 word >>= 4; 54 if ((word & 0xc) == 0) 55 num += 2; 56 else 57 word >>= 2; 58 if ((word & 0x2) == 0) 59 num += 1; 60 return num; 61} 62 63/* 64 * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c becasue 65 * f2fs_set_bit makes MSB and LSB reversed in a byte. 66 * Example: 67 * LSB <--> MSB 68 * f2fs_set_bit(0, bitmap) => 0000 0001 69 * f2fs_set_bit(7, bitmap) => 1000 0000 70 */ 71static unsigned long __find_rev_next_bit(const unsigned long *addr, 72 unsigned long size, unsigned long offset) 73{ 74 const unsigned long *p = addr + BIT_WORD(offset); 75 unsigned long result = offset & ~(BITS_PER_LONG - 1); 76 unsigned long tmp; 77 unsigned long mask, submask; 78 unsigned long quot, rest; 79 80 if (offset >= size) 81 return size; 82 83 size -= result; 84 offset %= BITS_PER_LONG; 85 if (!offset) 86 goto aligned; 87 88 tmp = *(p++); 89 quot = (offset >> 3) << 3; 90 rest = offset & 0x7; 91 mask = ~0UL << quot; 92 submask = (unsigned char)(0xff << rest) >> rest; 93 submask <<= quot; 94 mask &= submask; 95 tmp &= mask; 96 if (size < BITS_PER_LONG) 97 goto found_first; 98 if (tmp) 99 goto found_middle; 100 101 size -= BITS_PER_LONG; 102 result += BITS_PER_LONG; 103aligned: 104 while (size & ~(BITS_PER_LONG-1)) { 105 tmp = *(p++); 106 if (tmp) 107 goto found_middle; 108 result += BITS_PER_LONG; 109 size -= BITS_PER_LONG; 110 } 111 if (!size) 112 return result; 113 tmp = *p; 114found_first: 115 tmp &= (~0UL >> (BITS_PER_LONG - size)); 116 if (tmp == 0UL) /* Are any bits set? */ 117 return result + size; /* Nope. */ 118found_middle: 119 return result + __reverse_ffs(tmp); 120} 121 122static unsigned long __find_rev_next_zero_bit(const unsigned long *addr, 123 unsigned long size, unsigned long offset) 124{ 125 const unsigned long *p = addr + BIT_WORD(offset); 126 unsigned long result = offset & ~(BITS_PER_LONG - 1); 127 unsigned long tmp; 128 unsigned long mask, submask; 129 unsigned long quot, rest; 130 131 if (offset >= size) 132 return size; 133 134 size -= result; 135 offset %= BITS_PER_LONG; 136 if (!offset) 137 goto aligned; 138 139 tmp = *(p++); 140 quot = (offset >> 3) << 3; 141 rest = offset & 0x7; 142 mask = ~(~0UL << quot); 143 submask = (unsigned char)~((unsigned char)(0xff << rest) >> rest); 144 submask <<= quot; 145 mask += submask; 146 tmp |= mask; 147 if (size < BITS_PER_LONG) 148 goto found_first; 149 if (~tmp) 150 goto found_middle; 151 152 size -= BITS_PER_LONG; 153 result += BITS_PER_LONG; 154aligned: 155 while (size & ~(BITS_PER_LONG - 1)) { 156 tmp = *(p++); 157 if (~tmp) 158 goto found_middle; 159 result += BITS_PER_LONG; 160 size -= BITS_PER_LONG; 161 } 162 if (!size) 163 return result; 164 tmp = *p; 165 166found_first: 167 tmp |= ~0UL << size; 168 if (tmp == ~0UL) /* Are any bits zero? */ 169 return result + size; /* Nope. */ 170found_middle: 171 return result + __reverse_ffz(tmp); 172} 173 174/* 175 * This function balances dirty node and dentry pages. 176 * In addition, it controls garbage collection. 177 */ 178void f2fs_balance_fs(struct f2fs_sb_info *sbi) 179{ 180 /* 181 * We should do GC or end up with checkpoint, if there are so many dirty 182 * dir/node pages without enough free segments. 183 */ 184 if (has_not_enough_free_secs(sbi, 0)) { 185 mutex_lock(&sbi->gc_mutex); 186 f2fs_gc(sbi); 187 } 188} 189 190void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi) 191{ 192 /* check the # of cached NAT entries and prefree segments */ 193 if (try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK) || 194 excess_prefree_segs(sbi)) 195 f2fs_sync_fs(sbi->sb, true); 196} 197 198static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, 199 enum dirty_type dirty_type) 200{ 201 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 202 203 /* need not be added */ 204 if (IS_CURSEG(sbi, segno)) 205 return; 206 207 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type])) 208 dirty_i->nr_dirty[dirty_type]++; 209 210 if (dirty_type == DIRTY) { 211 struct seg_entry *sentry = get_seg_entry(sbi, segno); 212 enum dirty_type t = sentry->type; 213 214 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t])) 215 dirty_i->nr_dirty[t]++; 216 } 217} 218 219static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, 220 enum dirty_type dirty_type) 221{ 222 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 223 224 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type])) 225 dirty_i->nr_dirty[dirty_type]--; 226 227 if (dirty_type == DIRTY) { 228 struct seg_entry *sentry = get_seg_entry(sbi, segno); 229 enum dirty_type t = sentry->type; 230 231 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t])) 232 dirty_i->nr_dirty[t]--; 233 234 if (get_valid_blocks(sbi, segno, sbi->segs_per_sec) == 0) 235 clear_bit(GET_SECNO(sbi, segno), 236 dirty_i->victim_secmap); 237 } 238} 239 240/* 241 * Should not occur error such as -ENOMEM. 242 * Adding dirty entry into seglist is not critical operation. 243 * If a given segment is one of current working segments, it won't be added. 244 */ 245static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno) 246{ 247 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 248 unsigned short valid_blocks; 249 250 if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno)) 251 return; 252 253 mutex_lock(&dirty_i->seglist_lock); 254 255 valid_blocks = get_valid_blocks(sbi, segno, 0); 256 257 if (valid_blocks == 0) { 258 __locate_dirty_segment(sbi, segno, PRE); 259 __remove_dirty_segment(sbi, segno, DIRTY); 260 } else if (valid_blocks < sbi->blocks_per_seg) { 261 __locate_dirty_segment(sbi, segno, DIRTY); 262 } else { 263 /* Recovery routine with SSR needs this */ 264 __remove_dirty_segment(sbi, segno, DIRTY); 265 } 266 267 mutex_unlock(&dirty_i->seglist_lock); 268} 269 270static void f2fs_issue_discard(struct f2fs_sb_info *sbi, 271 block_t blkstart, block_t blklen) 272{ 273 sector_t start = SECTOR_FROM_BLOCK(sbi, blkstart); 274 sector_t len = SECTOR_FROM_BLOCK(sbi, blklen); 275 blkdev_issue_discard(sbi->sb->s_bdev, start, len, GFP_NOFS, 0); 276 trace_f2fs_issue_discard(sbi->sb, blkstart, blklen); 277} 278 279static void add_discard_addrs(struct f2fs_sb_info *sbi, 280 unsigned int segno, struct seg_entry *se) 281{ 282 struct list_head *head = &SM_I(sbi)->discard_list; 283 struct discard_entry *new; 284 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); 285 int max_blocks = sbi->blocks_per_seg; 286 unsigned long *cur_map = (unsigned long *)se->cur_valid_map; 287 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; 288 unsigned long dmap[entries]; 289 unsigned int start = 0, end = -1; 290 int i; 291 292 if (!test_opt(sbi, DISCARD)) 293 return; 294 295 /* zero block will be discarded through the prefree list */ 296 if (!se->valid_blocks || se->valid_blocks == max_blocks) 297 return; 298 299 /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */ 300 for (i = 0; i < entries; i++) 301 dmap[i] = (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i]; 302 303 while (SM_I(sbi)->nr_discards <= SM_I(sbi)->max_discards) { 304 start = __find_rev_next_bit(dmap, max_blocks, end + 1); 305 if (start >= max_blocks) 306 break; 307 308 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1); 309 310 new = f2fs_kmem_cache_alloc(discard_entry_slab, GFP_NOFS); 311 INIT_LIST_HEAD(&new->list); 312 new->blkaddr = START_BLOCK(sbi, segno) + start; 313 new->len = end - start; 314 315 list_add_tail(&new->list, head); 316 SM_I(sbi)->nr_discards += end - start; 317 } 318} 319 320/* 321 * Should call clear_prefree_segments after checkpoint is done. 322 */ 323static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi) 324{ 325 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 326 unsigned int segno = -1; 327 unsigned int total_segs = TOTAL_SEGS(sbi); 328 329 mutex_lock(&dirty_i->seglist_lock); 330 while (1) { 331 segno = find_next_bit(dirty_i->dirty_segmap[PRE], total_segs, 332 segno + 1); 333 if (segno >= total_segs) 334 break; 335 __set_test_and_free(sbi, segno); 336 } 337 mutex_unlock(&dirty_i->seglist_lock); 338} 339 340void clear_prefree_segments(struct f2fs_sb_info *sbi) 341{ 342 struct list_head *head = &(SM_I(sbi)->discard_list); 343 struct list_head *this, *next; 344 struct discard_entry *entry; 345 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 346 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE]; 347 unsigned int total_segs = TOTAL_SEGS(sbi); 348 unsigned int start = 0, end = -1; 349 350 mutex_lock(&dirty_i->seglist_lock); 351 352 while (1) { 353 int i; 354 start = find_next_bit(prefree_map, total_segs, end + 1); 355 if (start >= total_segs) 356 break; 357 end = find_next_zero_bit(prefree_map, total_segs, start + 1); 358 359 for (i = start; i < end; i++) 360 clear_bit(i, prefree_map); 361 362 dirty_i->nr_dirty[PRE] -= end - start; 363 364 if (!test_opt(sbi, DISCARD)) 365 continue; 366 367 f2fs_issue_discard(sbi, START_BLOCK(sbi, start), 368 (end - start) << sbi->log_blocks_per_seg); 369 } 370 mutex_unlock(&dirty_i->seglist_lock); 371 372 /* send small discards */ 373 list_for_each_safe(this, next, head) { 374 entry = list_entry(this, struct discard_entry, list); 375 f2fs_issue_discard(sbi, entry->blkaddr, entry->len); 376 list_del(&entry->list); 377 SM_I(sbi)->nr_discards -= entry->len; 378 kmem_cache_free(discard_entry_slab, entry); 379 } 380} 381 382static void __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno) 383{ 384 struct sit_info *sit_i = SIT_I(sbi); 385 if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) 386 sit_i->dirty_sentries++; 387} 388 389static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type, 390 unsigned int segno, int modified) 391{ 392 struct seg_entry *se = get_seg_entry(sbi, segno); 393 se->type = type; 394 if (modified) 395 __mark_sit_entry_dirty(sbi, segno); 396} 397 398static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del) 399{ 400 struct seg_entry *se; 401 unsigned int segno, offset; 402 long int new_vblocks; 403 404 segno = GET_SEGNO(sbi, blkaddr); 405 406 se = get_seg_entry(sbi, segno); 407 new_vblocks = se->valid_blocks + del; 408 offset = GET_SEGOFF_FROM_SEG0(sbi, blkaddr) & (sbi->blocks_per_seg - 1); 409 410 f2fs_bug_on((new_vblocks >> (sizeof(unsigned short) << 3) || 411 (new_vblocks > sbi->blocks_per_seg))); 412 413 se->valid_blocks = new_vblocks; 414 se->mtime = get_mtime(sbi); 415 SIT_I(sbi)->max_mtime = se->mtime; 416 417 /* Update valid block bitmap */ 418 if (del > 0) { 419 if (f2fs_set_bit(offset, se->cur_valid_map)) 420 BUG(); 421 } else { 422 if (!f2fs_clear_bit(offset, se->cur_valid_map)) 423 BUG(); 424 } 425 if (!f2fs_test_bit(offset, se->ckpt_valid_map)) 426 se->ckpt_valid_blocks += del; 427 428 __mark_sit_entry_dirty(sbi, segno); 429 430 /* update total number of valid blocks to be written in ckpt area */ 431 SIT_I(sbi)->written_valid_blocks += del; 432 433 if (sbi->segs_per_sec > 1) 434 get_sec_entry(sbi, segno)->valid_blocks += del; 435} 436 437static void refresh_sit_entry(struct f2fs_sb_info *sbi, 438 block_t old_blkaddr, block_t new_blkaddr) 439{ 440 update_sit_entry(sbi, new_blkaddr, 1); 441 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) 442 update_sit_entry(sbi, old_blkaddr, -1); 443} 444 445void invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr) 446{ 447 unsigned int segno = GET_SEGNO(sbi, addr); 448 struct sit_info *sit_i = SIT_I(sbi); 449 450 f2fs_bug_on(addr == NULL_ADDR); 451 if (addr == NEW_ADDR) 452 return; 453 454 /* add it into sit main buffer */ 455 mutex_lock(&sit_i->sentry_lock); 456 457 update_sit_entry(sbi, addr, -1); 458 459 /* add it into dirty seglist */ 460 locate_dirty_segment(sbi, segno); 461 462 mutex_unlock(&sit_i->sentry_lock); 463} 464 465/* 466 * This function should be resided under the curseg_mutex lock 467 */ 468static void __add_sum_entry(struct f2fs_sb_info *sbi, int type, 469 struct f2fs_summary *sum) 470{ 471 struct curseg_info *curseg = CURSEG_I(sbi, type); 472 void *addr = curseg->sum_blk; 473 addr += curseg->next_blkoff * sizeof(struct f2fs_summary); 474 memcpy(addr, sum, sizeof(struct f2fs_summary)); 475} 476 477/* 478 * Calculate the number of current summary pages for writing 479 */ 480int npages_for_summary_flush(struct f2fs_sb_info *sbi) 481{ 482 int valid_sum_count = 0; 483 int i, sum_in_page; 484 485 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 486 if (sbi->ckpt->alloc_type[i] == SSR) 487 valid_sum_count += sbi->blocks_per_seg; 488 else 489 valid_sum_count += curseg_blkoff(sbi, i); 490 } 491 492 sum_in_page = (PAGE_CACHE_SIZE - 2 * SUM_JOURNAL_SIZE - 493 SUM_FOOTER_SIZE) / SUMMARY_SIZE; 494 if (valid_sum_count <= sum_in_page) 495 return 1; 496 else if ((valid_sum_count - sum_in_page) <= 497 (PAGE_CACHE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE) 498 return 2; 499 return 3; 500} 501 502/* 503 * Caller should put this summary page 504 */ 505struct page *get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno) 506{ 507 return get_meta_page(sbi, GET_SUM_BLOCK(sbi, segno)); 508} 509 510static void write_sum_page(struct f2fs_sb_info *sbi, 511 struct f2fs_summary_block *sum_blk, block_t blk_addr) 512{ 513 struct page *page = grab_meta_page(sbi, blk_addr); 514 void *kaddr = page_address(page); 515 memcpy(kaddr, sum_blk, PAGE_CACHE_SIZE); 516 set_page_dirty(page); 517 f2fs_put_page(page, 1); 518} 519 520static int is_next_segment_free(struct f2fs_sb_info *sbi, int type) 521{ 522 struct curseg_info *curseg = CURSEG_I(sbi, type); 523 unsigned int segno = curseg->segno + 1; 524 struct free_segmap_info *free_i = FREE_I(sbi); 525 526 if (segno < TOTAL_SEGS(sbi) && segno % sbi->segs_per_sec) 527 return !test_bit(segno, free_i->free_segmap); 528 return 0; 529} 530 531/* 532 * Find a new segment from the free segments bitmap to right order 533 * This function should be returned with success, otherwise BUG 534 */ 535static void get_new_segment(struct f2fs_sb_info *sbi, 536 unsigned int *newseg, bool new_sec, int dir) 537{ 538 struct free_segmap_info *free_i = FREE_I(sbi); 539 unsigned int segno, secno, zoneno; 540 unsigned int total_zones = TOTAL_SECS(sbi) / sbi->secs_per_zone; 541 unsigned int hint = *newseg / sbi->segs_per_sec; 542 unsigned int old_zoneno = GET_ZONENO_FROM_SEGNO(sbi, *newseg); 543 unsigned int left_start = hint; 544 bool init = true; 545 int go_left = 0; 546 int i; 547 548 write_lock(&free_i->segmap_lock); 549 550 if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) { 551 segno = find_next_zero_bit(free_i->free_segmap, 552 TOTAL_SEGS(sbi), *newseg + 1); 553 if (segno - *newseg < sbi->segs_per_sec - 554 (*newseg % sbi->segs_per_sec)) 555 goto got_it; 556 } 557find_other_zone: 558 secno = find_next_zero_bit(free_i->free_secmap, TOTAL_SECS(sbi), hint); 559 if (secno >= TOTAL_SECS(sbi)) { 560 if (dir == ALLOC_RIGHT) { 561 secno = find_next_zero_bit(free_i->free_secmap, 562 TOTAL_SECS(sbi), 0); 563 f2fs_bug_on(secno >= TOTAL_SECS(sbi)); 564 } else { 565 go_left = 1; 566 left_start = hint - 1; 567 } 568 } 569 if (go_left == 0) 570 goto skip_left; 571 572 while (test_bit(left_start, free_i->free_secmap)) { 573 if (left_start > 0) { 574 left_start--; 575 continue; 576 } 577 left_start = find_next_zero_bit(free_i->free_secmap, 578 TOTAL_SECS(sbi), 0); 579 f2fs_bug_on(left_start >= TOTAL_SECS(sbi)); 580 break; 581 } 582 secno = left_start; 583skip_left: 584 hint = secno; 585 segno = secno * sbi->segs_per_sec; 586 zoneno = secno / sbi->secs_per_zone; 587 588 /* give up on finding another zone */ 589 if (!init) 590 goto got_it; 591 if (sbi->secs_per_zone == 1) 592 goto got_it; 593 if (zoneno == old_zoneno) 594 goto got_it; 595 if (dir == ALLOC_LEFT) { 596 if (!go_left && zoneno + 1 >= total_zones) 597 goto got_it; 598 if (go_left && zoneno == 0) 599 goto got_it; 600 } 601 for (i = 0; i < NR_CURSEG_TYPE; i++) 602 if (CURSEG_I(sbi, i)->zone == zoneno) 603 break; 604 605 if (i < NR_CURSEG_TYPE) { 606 /* zone is in user, try another */ 607 if (go_left) 608 hint = zoneno * sbi->secs_per_zone - 1; 609 else if (zoneno + 1 >= total_zones) 610 hint = 0; 611 else 612 hint = (zoneno + 1) * sbi->secs_per_zone; 613 init = false; 614 goto find_other_zone; 615 } 616got_it: 617 /* set it as dirty segment in free segmap */ 618 f2fs_bug_on(test_bit(segno, free_i->free_segmap)); 619 __set_inuse(sbi, segno); 620 *newseg = segno; 621 write_unlock(&free_i->segmap_lock); 622} 623 624static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified) 625{ 626 struct curseg_info *curseg = CURSEG_I(sbi, type); 627 struct summary_footer *sum_footer; 628 629 curseg->segno = curseg->next_segno; 630 curseg->zone = GET_ZONENO_FROM_SEGNO(sbi, curseg->segno); 631 curseg->next_blkoff = 0; 632 curseg->next_segno = NULL_SEGNO; 633 634 sum_footer = &(curseg->sum_blk->footer); 635 memset(sum_footer, 0, sizeof(struct summary_footer)); 636 if (IS_DATASEG(type)) 637 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA); 638 if (IS_NODESEG(type)) 639 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE); 640 __set_sit_entry_type(sbi, type, curseg->segno, modified); 641} 642 643/* 644 * Allocate a current working segment. 645 * This function always allocates a free segment in LFS manner. 646 */ 647static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) 648{ 649 struct curseg_info *curseg = CURSEG_I(sbi, type); 650 unsigned int segno = curseg->segno; 651 int dir = ALLOC_LEFT; 652 653 write_sum_page(sbi, curseg->sum_blk, 654 GET_SUM_BLOCK(sbi, segno)); 655 if (type == CURSEG_WARM_DATA || type == CURSEG_COLD_DATA) 656 dir = ALLOC_RIGHT; 657 658 if (test_opt(sbi, NOHEAP)) 659 dir = ALLOC_RIGHT; 660 661 get_new_segment(sbi, &segno, new_sec, dir); 662 curseg->next_segno = segno; 663 reset_curseg(sbi, type, 1); 664 curseg->alloc_type = LFS; 665} 666 667static void __next_free_blkoff(struct f2fs_sb_info *sbi, 668 struct curseg_info *seg, block_t start) 669{ 670 struct seg_entry *se = get_seg_entry(sbi, seg->segno); 671 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); 672 unsigned long target_map[entries]; 673 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; 674 unsigned long *cur_map = (unsigned long *)se->cur_valid_map; 675 int i, pos; 676 677 for (i = 0; i < entries; i++) 678 target_map[i] = ckpt_map[i] | cur_map[i]; 679 680 pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start); 681 682 seg->next_blkoff = pos; 683} 684 685/* 686 * If a segment is written by LFS manner, next block offset is just obtained 687 * by increasing the current block offset. However, if a segment is written by 688 * SSR manner, next block offset obtained by calling __next_free_blkoff 689 */ 690static void __refresh_next_blkoff(struct f2fs_sb_info *sbi, 691 struct curseg_info *seg) 692{ 693 if (seg->alloc_type == SSR) 694 __next_free_blkoff(sbi, seg, seg->next_blkoff + 1); 695 else 696 seg->next_blkoff++; 697} 698 699/* 700 * This function always allocates a used segment (from dirty seglist) by SSR 701 * manner, so it should recover the existing segment information of valid blocks 702 */ 703static void change_curseg(struct f2fs_sb_info *sbi, int type, bool reuse) 704{ 705 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 706 struct curseg_info *curseg = CURSEG_I(sbi, type); 707 unsigned int new_segno = curseg->next_segno; 708 struct f2fs_summary_block *sum_node; 709 struct page *sum_page; 710 711 write_sum_page(sbi, curseg->sum_blk, 712 GET_SUM_BLOCK(sbi, curseg->segno)); 713 __set_test_and_inuse(sbi, new_segno); 714 715 mutex_lock(&dirty_i->seglist_lock); 716 __remove_dirty_segment(sbi, new_segno, PRE); 717 __remove_dirty_segment(sbi, new_segno, DIRTY); 718 mutex_unlock(&dirty_i->seglist_lock); 719 720 reset_curseg(sbi, type, 1); 721 curseg->alloc_type = SSR; 722 __next_free_blkoff(sbi, curseg, 0); 723 724 if (reuse) { 725 sum_page = get_sum_page(sbi, new_segno); 726 sum_node = (struct f2fs_summary_block *)page_address(sum_page); 727 memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE); 728 f2fs_put_page(sum_page, 1); 729 } 730} 731 732static int get_ssr_segment(struct f2fs_sb_info *sbi, int type) 733{ 734 struct curseg_info *curseg = CURSEG_I(sbi, type); 735 const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops; 736 737 if (IS_NODESEG(type) || !has_not_enough_free_secs(sbi, 0)) 738 return v_ops->get_victim(sbi, 739 &(curseg)->next_segno, BG_GC, type, SSR); 740 741 /* For data segments, let's do SSR more intensively */ 742 for (; type >= CURSEG_HOT_DATA; type--) 743 if (v_ops->get_victim(sbi, &(curseg)->next_segno, 744 BG_GC, type, SSR)) 745 return 1; 746 return 0; 747} 748 749/* 750 * flush out current segment and replace it with new segment 751 * This function should be returned with success, otherwise BUG 752 */ 753static void allocate_segment_by_default(struct f2fs_sb_info *sbi, 754 int type, bool force) 755{ 756 struct curseg_info *curseg = CURSEG_I(sbi, type); 757 758 if (force) 759 new_curseg(sbi, type, true); 760 else if (type == CURSEG_WARM_NODE) 761 new_curseg(sbi, type, false); 762 else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type)) 763 new_curseg(sbi, type, false); 764 else if (need_SSR(sbi) && get_ssr_segment(sbi, type)) 765 change_curseg(sbi, type, true); 766 else 767 new_curseg(sbi, type, false); 768 769 stat_inc_seg_type(sbi, curseg); 770} 771 772void allocate_new_segments(struct f2fs_sb_info *sbi) 773{ 774 struct curseg_info *curseg; 775 unsigned int old_curseg; 776 int i; 777 778 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 779 curseg = CURSEG_I(sbi, i); 780 old_curseg = curseg->segno; 781 SIT_I(sbi)->s_ops->allocate_segment(sbi, i, true); 782 locate_dirty_segment(sbi, old_curseg); 783 } 784} 785 786static const struct segment_allocation default_salloc_ops = { 787 .allocate_segment = allocate_segment_by_default, 788}; 789 790static void f2fs_end_io_write(struct bio *bio, int err) 791{ 792 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 793 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1; 794 struct f2fs_sb_info *sbi = F2FS_SB(bvec->bv_page->mapping->host->i_sb); 795 796 do { 797 struct page *page = bvec->bv_page; 798 799 if (--bvec >= bio->bi_io_vec) 800 prefetchw(&bvec->bv_page->flags); 801 if (!uptodate) { 802 SetPageError(page); 803 if (page->mapping) 804 set_bit(AS_EIO, &page->mapping->flags); 805 806 set_ckpt_flags(sbi->ckpt, CP_ERROR_FLAG); 807 sbi->sb->s_flags |= MS_RDONLY; 808 } 809 end_page_writeback(page); 810 dec_page_count(sbi, F2FS_WRITEBACK); 811 } while (bvec >= bio->bi_io_vec); 812 813 if (bio->bi_private) 814 complete(bio->bi_private); 815 816 if (!get_pages(sbi, F2FS_WRITEBACK) && 817 !list_empty(&sbi->cp_wait.task_list)) 818 wake_up(&sbi->cp_wait); 819 820 bio_put(bio); 821} 822 823struct bio *f2fs_bio_alloc(struct block_device *bdev, int npages) 824{ 825 struct bio *bio; 826 827 /* No failure on bio allocation */ 828 bio = bio_alloc(GFP_NOIO, npages); 829 bio->bi_bdev = bdev; 830 bio->bi_private = NULL; 831 832 return bio; 833} 834 835static void do_submit_bio(struct f2fs_sb_info *sbi, 836 enum page_type type, bool sync) 837{ 838 int rw = sync ? WRITE_SYNC : WRITE; 839 enum page_type btype = PAGE_TYPE_OF_BIO(type); 840 struct f2fs_bio_info *io = &sbi->write_io[btype]; 841 842 if (!io->bio) 843 return; 844 845 if (type >= META_FLUSH) 846 rw = WRITE_FLUSH_FUA; 847 848 if (btype == META) 849 rw |= REQ_META; 850 851 trace_f2fs_submit_write_bio(sbi->sb, rw, btype, io->bio); 852 853 /* 854 * META_FLUSH is only from the checkpoint procedure, and we should wait 855 * this metadata bio for FS consistency. 856 */ 857 if (type == META_FLUSH) { 858 DECLARE_COMPLETION_ONSTACK(wait); 859 io->bio->bi_private = &wait; 860 submit_bio(rw, io->bio); 861 wait_for_completion(&wait); 862 } else { 863 submit_bio(rw, io->bio); 864 } 865 io->bio = NULL; 866} 867 868void f2fs_submit_bio(struct f2fs_sb_info *sbi, enum page_type type, bool sync) 869{ 870 struct f2fs_bio_info *io = &sbi->write_io[PAGE_TYPE_OF_BIO(type)]; 871 872 if (!io->bio) 873 return; 874 875 mutex_lock(&io->io_mutex); 876 do_submit_bio(sbi, type, sync); 877 mutex_unlock(&io->io_mutex); 878} 879 880static void submit_write_page(struct f2fs_sb_info *sbi, struct page *page, 881 block_t blk_addr, enum page_type type) 882{ 883 struct block_device *bdev = sbi->sb->s_bdev; 884 struct f2fs_bio_info *io = &sbi->write_io[type]; 885 int bio_blocks; 886 887 verify_block_addr(sbi, blk_addr); 888 889 mutex_lock(&io->io_mutex); 890 891 inc_page_count(sbi, F2FS_WRITEBACK); 892 893 if (io->bio && io->last_block_in_bio != blk_addr - 1) 894 do_submit_bio(sbi, type, false); 895alloc_new: 896 if (io->bio == NULL) { 897 bio_blocks = MAX_BIO_BLOCKS(max_hw_blocks(sbi)); 898 io->bio = f2fs_bio_alloc(bdev, bio_blocks); 899 io->bio->bi_sector = SECTOR_FROM_BLOCK(sbi, blk_addr); 900 io->bio->bi_end_io = f2fs_end_io_write; 901 /* 902 * The end_io will be assigned at the sumbission phase. 903 * Until then, let bio_add_page() merge consecutive IOs as much 904 * as possible. 905 */ 906 } 907 908 if (bio_add_page(io->bio, page, PAGE_CACHE_SIZE, 0) < 909 PAGE_CACHE_SIZE) { 910 do_submit_bio(sbi, type, false); 911 goto alloc_new; 912 } 913 914 io->last_block_in_bio = blk_addr; 915 916 mutex_unlock(&io->io_mutex); 917 trace_f2fs_submit_write_page(page, WRITE, type, blk_addr); 918} 919 920void f2fs_wait_on_page_writeback(struct page *page, 921 enum page_type type, bool sync) 922{ 923 struct f2fs_sb_info *sbi = F2FS_SB(page->mapping->host->i_sb); 924 if (PageWriteback(page)) { 925 f2fs_submit_bio(sbi, type, sync); 926 wait_on_page_writeback(page); 927 } 928} 929 930static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type) 931{ 932 struct curseg_info *curseg = CURSEG_I(sbi, type); 933 if (curseg->next_blkoff < sbi->blocks_per_seg) 934 return true; 935 return false; 936} 937 938static int __get_segment_type_2(struct page *page, enum page_type p_type) 939{ 940 if (p_type == DATA) 941 return CURSEG_HOT_DATA; 942 else 943 return CURSEG_HOT_NODE; 944} 945 946static int __get_segment_type_4(struct page *page, enum page_type p_type) 947{ 948 if (p_type == DATA) { 949 struct inode *inode = page->mapping->host; 950 951 if (S_ISDIR(inode->i_mode)) 952 return CURSEG_HOT_DATA; 953 else 954 return CURSEG_COLD_DATA; 955 } else { 956 if (IS_DNODE(page) && !is_cold_node(page)) 957 return CURSEG_HOT_NODE; 958 else 959 return CURSEG_COLD_NODE; 960 } 961} 962 963static int __get_segment_type_6(struct page *page, enum page_type p_type) 964{ 965 if (p_type == DATA) { 966 struct inode *inode = page->mapping->host; 967 968 if (S_ISDIR(inode->i_mode)) 969 return CURSEG_HOT_DATA; 970 else if (is_cold_data(page) || file_is_cold(inode)) 971 return CURSEG_COLD_DATA; 972 else 973 return CURSEG_WARM_DATA; 974 } else { 975 if (IS_DNODE(page)) 976 return is_cold_node(page) ? CURSEG_WARM_NODE : 977 CURSEG_HOT_NODE; 978 else 979 return CURSEG_COLD_NODE; 980 } 981} 982 983static int __get_segment_type(struct page *page, enum page_type p_type) 984{ 985 struct f2fs_sb_info *sbi = F2FS_SB(page->mapping->host->i_sb); 986 switch (sbi->active_logs) { 987 case 2: 988 return __get_segment_type_2(page, p_type); 989 case 4: 990 return __get_segment_type_4(page, p_type); 991 } 992 /* NR_CURSEG_TYPE(6) logs by default */ 993 f2fs_bug_on(sbi->active_logs != NR_CURSEG_TYPE); 994 return __get_segment_type_6(page, p_type); 995} 996 997static void do_write_page(struct f2fs_sb_info *sbi, struct page *page, 998 block_t old_blkaddr, block_t *new_blkaddr, 999 struct f2fs_summary *sum, enum page_type p_type) 1000{ 1001 struct sit_info *sit_i = SIT_I(sbi); 1002 struct curseg_info *curseg; 1003 unsigned int old_cursegno; 1004 int type; 1005 1006 type = __get_segment_type(page, p_type); 1007 curseg = CURSEG_I(sbi, type); 1008 1009 mutex_lock(&curseg->curseg_mutex); 1010 1011 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); 1012 old_cursegno = curseg->segno; 1013 1014 /* 1015 * __add_sum_entry should be resided under the curseg_mutex 1016 * because, this function updates a summary entry in the 1017 * current summary block. 1018 */ 1019 __add_sum_entry(sbi, type, sum); 1020 1021 mutex_lock(&sit_i->sentry_lock); 1022 __refresh_next_blkoff(sbi, curseg); 1023 1024 stat_inc_block_count(sbi, curseg); 1025 1026 /* 1027 * SIT information should be updated before segment allocation, 1028 * since SSR needs latest valid block information. 1029 */ 1030 refresh_sit_entry(sbi, old_blkaddr, *new_blkaddr); 1031 1032 if (!__has_curseg_space(sbi, type)) 1033 sit_i->s_ops->allocate_segment(sbi, type, false); 1034 1035 locate_dirty_segment(sbi, old_cursegno); 1036 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); 1037 mutex_unlock(&sit_i->sentry_lock); 1038 1039 if (p_type == NODE) 1040 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg)); 1041 1042 /* writeout dirty page into bdev */ 1043 submit_write_page(sbi, page, *new_blkaddr, p_type); 1044 1045 mutex_unlock(&curseg->curseg_mutex); 1046} 1047 1048void write_meta_page(struct f2fs_sb_info *sbi, struct page *page) 1049{ 1050 set_page_writeback(page); 1051 submit_write_page(sbi, page, page->index, META); 1052} 1053 1054void write_node_page(struct f2fs_sb_info *sbi, struct page *page, 1055 unsigned int nid, block_t old_blkaddr, block_t *new_blkaddr) 1056{ 1057 struct f2fs_summary sum; 1058 set_summary(&sum, nid, 0, 0); 1059 do_write_page(sbi, page, old_blkaddr, new_blkaddr, &sum, NODE); 1060} 1061 1062void write_data_page(struct inode *inode, struct page *page, 1063 struct dnode_of_data *dn, block_t old_blkaddr, 1064 block_t *new_blkaddr) 1065{ 1066 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb); 1067 struct f2fs_summary sum; 1068 struct node_info ni; 1069 1070 f2fs_bug_on(old_blkaddr == NULL_ADDR); 1071 get_node_info(sbi, dn->nid, &ni); 1072 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); 1073 1074 do_write_page(sbi, page, old_blkaddr, 1075 new_blkaddr, &sum, DATA); 1076} 1077 1078void rewrite_data_page(struct f2fs_sb_info *sbi, struct page *page, 1079 block_t old_blk_addr) 1080{ 1081 submit_write_page(sbi, page, old_blk_addr, DATA); 1082} 1083 1084void recover_data_page(struct f2fs_sb_info *sbi, 1085 struct page *page, struct f2fs_summary *sum, 1086 block_t old_blkaddr, block_t new_blkaddr) 1087{ 1088 struct sit_info *sit_i = SIT_I(sbi); 1089 struct curseg_info *curseg; 1090 unsigned int segno, old_cursegno; 1091 struct seg_entry *se; 1092 int type; 1093 1094 segno = GET_SEGNO(sbi, new_blkaddr); 1095 se = get_seg_entry(sbi, segno); 1096 type = se->type; 1097 1098 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) { 1099 if (old_blkaddr == NULL_ADDR) 1100 type = CURSEG_COLD_DATA; 1101 else 1102 type = CURSEG_WARM_DATA; 1103 } 1104 curseg = CURSEG_I(sbi, type); 1105 1106 mutex_lock(&curseg->curseg_mutex); 1107 mutex_lock(&sit_i->sentry_lock); 1108 1109 old_cursegno = curseg->segno; 1110 1111 /* change the current segment */ 1112 if (segno != curseg->segno) { 1113 curseg->next_segno = segno; 1114 change_curseg(sbi, type, true); 1115 } 1116 1117 curseg->next_blkoff = GET_SEGOFF_FROM_SEG0(sbi, new_blkaddr) & 1118 (sbi->blocks_per_seg - 1); 1119 __add_sum_entry(sbi, type, sum); 1120 1121 refresh_sit_entry(sbi, old_blkaddr, new_blkaddr); 1122 1123 locate_dirty_segment(sbi, old_cursegno); 1124 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); 1125 1126 mutex_unlock(&sit_i->sentry_lock); 1127 mutex_unlock(&curseg->curseg_mutex); 1128} 1129 1130void rewrite_node_page(struct f2fs_sb_info *sbi, 1131 struct page *page, struct f2fs_summary *sum, 1132 block_t old_blkaddr, block_t new_blkaddr) 1133{ 1134 struct sit_info *sit_i = SIT_I(sbi); 1135 int type = CURSEG_WARM_NODE; 1136 struct curseg_info *curseg; 1137 unsigned int segno, old_cursegno; 1138 block_t next_blkaddr = next_blkaddr_of_node(page); 1139 unsigned int next_segno = GET_SEGNO(sbi, next_blkaddr); 1140 1141 curseg = CURSEG_I(sbi, type); 1142 1143 mutex_lock(&curseg->curseg_mutex); 1144 mutex_lock(&sit_i->sentry_lock); 1145 1146 segno = GET_SEGNO(sbi, new_blkaddr); 1147 old_cursegno = curseg->segno; 1148 1149 /* change the current segment */ 1150 if (segno != curseg->segno) { 1151 curseg->next_segno = segno; 1152 change_curseg(sbi, type, true); 1153 } 1154 curseg->next_blkoff = GET_SEGOFF_FROM_SEG0(sbi, new_blkaddr) & 1155 (sbi->blocks_per_seg - 1); 1156 __add_sum_entry(sbi, type, sum); 1157 1158 /* change the current log to the next block addr in advance */ 1159 if (next_segno != segno) { 1160 curseg->next_segno = next_segno; 1161 change_curseg(sbi, type, true); 1162 } 1163 curseg->next_blkoff = GET_SEGOFF_FROM_SEG0(sbi, next_blkaddr) & 1164 (sbi->blocks_per_seg - 1); 1165 1166 /* rewrite node page */ 1167 set_page_writeback(page); 1168 submit_write_page(sbi, page, new_blkaddr, NODE); 1169 f2fs_submit_bio(sbi, NODE, true); 1170 refresh_sit_entry(sbi, old_blkaddr, new_blkaddr); 1171 1172 locate_dirty_segment(sbi, old_cursegno); 1173 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); 1174 1175 mutex_unlock(&sit_i->sentry_lock); 1176 mutex_unlock(&curseg->curseg_mutex); 1177} 1178 1179static int read_compacted_summaries(struct f2fs_sb_info *sbi) 1180{ 1181 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 1182 struct curseg_info *seg_i; 1183 unsigned char *kaddr; 1184 struct page *page; 1185 block_t start; 1186 int i, j, offset; 1187 1188 start = start_sum_block(sbi); 1189 1190 page = get_meta_page(sbi, start++); 1191 kaddr = (unsigned char *)page_address(page); 1192 1193 /* Step 1: restore nat cache */ 1194 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); 1195 memcpy(&seg_i->sum_blk->n_nats, kaddr, SUM_JOURNAL_SIZE); 1196 1197 /* Step 2: restore sit cache */ 1198 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); 1199 memcpy(&seg_i->sum_blk->n_sits, kaddr + SUM_JOURNAL_SIZE, 1200 SUM_JOURNAL_SIZE); 1201 offset = 2 * SUM_JOURNAL_SIZE; 1202 1203 /* Step 3: restore summary entries */ 1204 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 1205 unsigned short blk_off; 1206 unsigned int segno; 1207 1208 seg_i = CURSEG_I(sbi, i); 1209 segno = le32_to_cpu(ckpt->cur_data_segno[i]); 1210 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]); 1211 seg_i->next_segno = segno; 1212 reset_curseg(sbi, i, 0); 1213 seg_i->alloc_type = ckpt->alloc_type[i]; 1214 seg_i->next_blkoff = blk_off; 1215 1216 if (seg_i->alloc_type == SSR) 1217 blk_off = sbi->blocks_per_seg; 1218 1219 for (j = 0; j < blk_off; j++) { 1220 struct f2fs_summary *s; 1221 s = (struct f2fs_summary *)(kaddr + offset); 1222 seg_i->sum_blk->entries[j] = *s; 1223 offset += SUMMARY_SIZE; 1224 if (offset + SUMMARY_SIZE <= PAGE_CACHE_SIZE - 1225 SUM_FOOTER_SIZE) 1226 continue; 1227 1228 f2fs_put_page(page, 1); 1229 page = NULL; 1230 1231 page = get_meta_page(sbi, start++); 1232 kaddr = (unsigned char *)page_address(page); 1233 offset = 0; 1234 } 1235 } 1236 f2fs_put_page(page, 1); 1237 return 0; 1238} 1239 1240static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) 1241{ 1242 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 1243 struct f2fs_summary_block *sum; 1244 struct curseg_info *curseg; 1245 struct page *new; 1246 unsigned short blk_off; 1247 unsigned int segno = 0; 1248 block_t blk_addr = 0; 1249 1250 /* get segment number and block addr */ 1251 if (IS_DATASEG(type)) { 1252 segno = le32_to_cpu(ckpt->cur_data_segno[type]); 1253 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type - 1254 CURSEG_HOT_DATA]); 1255 if (is_set_ckpt_flags(ckpt, CP_UMOUNT_FLAG)) 1256 blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type); 1257 else 1258 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type); 1259 } else { 1260 segno = le32_to_cpu(ckpt->cur_node_segno[type - 1261 CURSEG_HOT_NODE]); 1262 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type - 1263 CURSEG_HOT_NODE]); 1264 if (is_set_ckpt_flags(ckpt, CP_UMOUNT_FLAG)) 1265 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE, 1266 type - CURSEG_HOT_NODE); 1267 else 1268 blk_addr = GET_SUM_BLOCK(sbi, segno); 1269 } 1270 1271 new = get_meta_page(sbi, blk_addr); 1272 sum = (struct f2fs_summary_block *)page_address(new); 1273 1274 if (IS_NODESEG(type)) { 1275 if (is_set_ckpt_flags(ckpt, CP_UMOUNT_FLAG)) { 1276 struct f2fs_summary *ns = &sum->entries[0]; 1277 int i; 1278 for (i = 0; i < sbi->blocks_per_seg; i++, ns++) { 1279 ns->version = 0; 1280 ns->ofs_in_node = 0; 1281 } 1282 } else { 1283 if (restore_node_summary(sbi, segno, sum)) { 1284 f2fs_put_page(new, 1); 1285 return -EINVAL; 1286 } 1287 } 1288 } 1289 1290 /* set uncompleted segment to curseg */ 1291 curseg = CURSEG_I(sbi, type); 1292 mutex_lock(&curseg->curseg_mutex); 1293 memcpy(curseg->sum_blk, sum, PAGE_CACHE_SIZE); 1294 curseg->next_segno = segno; 1295 reset_curseg(sbi, type, 0); 1296 curseg->alloc_type = ckpt->alloc_type[type]; 1297 curseg->next_blkoff = blk_off; 1298 mutex_unlock(&curseg->curseg_mutex); 1299 f2fs_put_page(new, 1); 1300 return 0; 1301} 1302 1303static int restore_curseg_summaries(struct f2fs_sb_info *sbi) 1304{ 1305 int type = CURSEG_HOT_DATA; 1306 1307 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG)) { 1308 /* restore for compacted data summary */ 1309 if (read_compacted_summaries(sbi)) 1310 return -EINVAL; 1311 type = CURSEG_HOT_NODE; 1312 } 1313 1314 for (; type <= CURSEG_COLD_NODE; type++) 1315 if (read_normal_summaries(sbi, type)) 1316 return -EINVAL; 1317 return 0; 1318} 1319 1320static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) 1321{ 1322 struct page *page; 1323 unsigned char *kaddr; 1324 struct f2fs_summary *summary; 1325 struct curseg_info *seg_i; 1326 int written_size = 0; 1327 int i, j; 1328 1329 page = grab_meta_page(sbi, blkaddr++); 1330 kaddr = (unsigned char *)page_address(page); 1331 1332 /* Step 1: write nat cache */ 1333 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); 1334 memcpy(kaddr, &seg_i->sum_blk->n_nats, SUM_JOURNAL_SIZE); 1335 written_size += SUM_JOURNAL_SIZE; 1336 1337 /* Step 2: write sit cache */ 1338 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); 1339 memcpy(kaddr + written_size, &seg_i->sum_blk->n_sits, 1340 SUM_JOURNAL_SIZE); 1341 written_size += SUM_JOURNAL_SIZE; 1342 1343 /* Step 3: write summary entries */ 1344 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 1345 unsigned short blkoff; 1346 seg_i = CURSEG_I(sbi, i); 1347 if (sbi->ckpt->alloc_type[i] == SSR) 1348 blkoff = sbi->blocks_per_seg; 1349 else 1350 blkoff = curseg_blkoff(sbi, i); 1351 1352 for (j = 0; j < blkoff; j++) { 1353 if (!page) { 1354 page = grab_meta_page(sbi, blkaddr++); 1355 kaddr = (unsigned char *)page_address(page); 1356 written_size = 0; 1357 } 1358 summary = (struct f2fs_summary *)(kaddr + written_size); 1359 *summary = seg_i->sum_blk->entries[j]; 1360 written_size += SUMMARY_SIZE; 1361 1362 if (written_size + SUMMARY_SIZE <= PAGE_CACHE_SIZE - 1363 SUM_FOOTER_SIZE) 1364 continue; 1365 1366 set_page_dirty(page); 1367 f2fs_put_page(page, 1); 1368 page = NULL; 1369 } 1370 } 1371 if (page) { 1372 set_page_dirty(page); 1373 f2fs_put_page(page, 1); 1374 } 1375} 1376 1377static void write_normal_summaries(struct f2fs_sb_info *sbi, 1378 block_t blkaddr, int type) 1379{ 1380 int i, end; 1381 if (IS_DATASEG(type)) 1382 end = type + NR_CURSEG_DATA_TYPE; 1383 else 1384 end = type + NR_CURSEG_NODE_TYPE; 1385 1386 for (i = type; i < end; i++) { 1387 struct curseg_info *sum = CURSEG_I(sbi, i); 1388 mutex_lock(&sum->curseg_mutex); 1389 write_sum_page(sbi, sum->sum_blk, blkaddr + (i - type)); 1390 mutex_unlock(&sum->curseg_mutex); 1391 } 1392} 1393 1394void write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk) 1395{ 1396 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG)) 1397 write_compacted_summaries(sbi, start_blk); 1398 else 1399 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA); 1400} 1401 1402void write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk) 1403{ 1404 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_UMOUNT_FLAG)) 1405 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE); 1406} 1407 1408int lookup_journal_in_cursum(struct f2fs_summary_block *sum, int type, 1409 unsigned int val, int alloc) 1410{ 1411 int i; 1412 1413 if (type == NAT_JOURNAL) { 1414 for (i = 0; i < nats_in_cursum(sum); i++) { 1415 if (le32_to_cpu(nid_in_journal(sum, i)) == val) 1416 return i; 1417 } 1418 if (alloc && nats_in_cursum(sum) < NAT_JOURNAL_ENTRIES) 1419 return update_nats_in_cursum(sum, 1); 1420 } else if (type == SIT_JOURNAL) { 1421 for (i = 0; i < sits_in_cursum(sum); i++) 1422 if (le32_to_cpu(segno_in_journal(sum, i)) == val) 1423 return i; 1424 if (alloc && sits_in_cursum(sum) < SIT_JOURNAL_ENTRIES) 1425 return update_sits_in_cursum(sum, 1); 1426 } 1427 return -1; 1428} 1429 1430static struct page *get_current_sit_page(struct f2fs_sb_info *sbi, 1431 unsigned int segno) 1432{ 1433 struct sit_info *sit_i = SIT_I(sbi); 1434 unsigned int offset = SIT_BLOCK_OFFSET(sit_i, segno); 1435 block_t blk_addr = sit_i->sit_base_addr + offset; 1436 1437 check_seg_range(sbi, segno); 1438 1439 /* calculate sit block address */ 1440 if (f2fs_test_bit(offset, sit_i->sit_bitmap)) 1441 blk_addr += sit_i->sit_blocks; 1442 1443 return get_meta_page(sbi, blk_addr); 1444} 1445 1446static struct page *get_next_sit_page(struct f2fs_sb_info *sbi, 1447 unsigned int start) 1448{ 1449 struct sit_info *sit_i = SIT_I(sbi); 1450 struct page *src_page, *dst_page; 1451 pgoff_t src_off, dst_off; 1452 void *src_addr, *dst_addr; 1453 1454 src_off = current_sit_addr(sbi, start); 1455 dst_off = next_sit_addr(sbi, src_off); 1456 1457 /* get current sit block page without lock */ 1458 src_page = get_meta_page(sbi, src_off); 1459 dst_page = grab_meta_page(sbi, dst_off); 1460 f2fs_bug_on(PageDirty(src_page)); 1461 1462 src_addr = page_address(src_page); 1463 dst_addr = page_address(dst_page); 1464 memcpy(dst_addr, src_addr, PAGE_CACHE_SIZE); 1465 1466 set_page_dirty(dst_page); 1467 f2fs_put_page(src_page, 1); 1468 1469 set_to_next_sit(sit_i, start); 1470 1471 return dst_page; 1472} 1473 1474static bool flush_sits_in_journal(struct f2fs_sb_info *sbi) 1475{ 1476 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 1477 struct f2fs_summary_block *sum = curseg->sum_blk; 1478 int i; 1479 1480 /* 1481 * If the journal area in the current summary is full of sit entries, 1482 * all the sit entries will be flushed. Otherwise the sit entries 1483 * are not able to replace with newly hot sit entries. 1484 */ 1485 if (sits_in_cursum(sum) >= SIT_JOURNAL_ENTRIES) { 1486 for (i = sits_in_cursum(sum) - 1; i >= 0; i--) { 1487 unsigned int segno; 1488 segno = le32_to_cpu(segno_in_journal(sum, i)); 1489 __mark_sit_entry_dirty(sbi, segno); 1490 } 1491 update_sits_in_cursum(sum, -sits_in_cursum(sum)); 1492 return true; 1493 } 1494 return false; 1495} 1496 1497/* 1498 * CP calls this function, which flushes SIT entries including sit_journal, 1499 * and moves prefree segs to free segs. 1500 */ 1501void flush_sit_entries(struct f2fs_sb_info *sbi) 1502{ 1503 struct sit_info *sit_i = SIT_I(sbi); 1504 unsigned long *bitmap = sit_i->dirty_sentries_bitmap; 1505 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 1506 struct f2fs_summary_block *sum = curseg->sum_blk; 1507 unsigned long nsegs = TOTAL_SEGS(sbi); 1508 struct page *page = NULL; 1509 struct f2fs_sit_block *raw_sit = NULL; 1510 unsigned int start = 0, end = 0; 1511 unsigned int segno = -1; 1512 bool flushed; 1513 1514 mutex_lock(&curseg->curseg_mutex); 1515 mutex_lock(&sit_i->sentry_lock); 1516 1517 /* 1518 * "flushed" indicates whether sit entries in journal are flushed 1519 * to the SIT area or not. 1520 */ 1521 flushed = flush_sits_in_journal(sbi); 1522 1523 while ((segno = find_next_bit(bitmap, nsegs, segno + 1)) < nsegs) { 1524 struct seg_entry *se = get_seg_entry(sbi, segno); 1525 int sit_offset, offset; 1526 1527 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno); 1528 1529 /* add discard candidates */ 1530 if (SM_I(sbi)->nr_discards < SM_I(sbi)->max_discards) 1531 add_discard_addrs(sbi, segno, se); 1532 1533 if (flushed) 1534 goto to_sit_page; 1535 1536 offset = lookup_journal_in_cursum(sum, SIT_JOURNAL, segno, 1); 1537 if (offset >= 0) { 1538 segno_in_journal(sum, offset) = cpu_to_le32(segno); 1539 seg_info_to_raw_sit(se, &sit_in_journal(sum, offset)); 1540 goto flush_done; 1541 } 1542to_sit_page: 1543 if (!page || (start > segno) || (segno > end)) { 1544 if (page) { 1545 f2fs_put_page(page, 1); 1546 page = NULL; 1547 } 1548 1549 start = START_SEGNO(sit_i, segno); 1550 end = start + SIT_ENTRY_PER_BLOCK - 1; 1551 1552 /* read sit block that will be updated */ 1553 page = get_next_sit_page(sbi, start); 1554 raw_sit = page_address(page); 1555 } 1556 1557 /* udpate entry in SIT block */ 1558 seg_info_to_raw_sit(se, &raw_sit->entries[sit_offset]); 1559flush_done: 1560 __clear_bit(segno, bitmap); 1561 sit_i->dirty_sentries--; 1562 } 1563 mutex_unlock(&sit_i->sentry_lock); 1564 mutex_unlock(&curseg->curseg_mutex); 1565 1566 /* writeout last modified SIT block */ 1567 f2fs_put_page(page, 1); 1568 1569 set_prefree_as_free_segments(sbi); 1570} 1571 1572static int build_sit_info(struct f2fs_sb_info *sbi) 1573{ 1574 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); 1575 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 1576 struct sit_info *sit_i; 1577 unsigned int sit_segs, start; 1578 char *src_bitmap, *dst_bitmap; 1579 unsigned int bitmap_size; 1580 1581 /* allocate memory for SIT information */ 1582 sit_i = kzalloc(sizeof(struct sit_info), GFP_KERNEL); 1583 if (!sit_i) 1584 return -ENOMEM; 1585 1586 SM_I(sbi)->sit_info = sit_i; 1587 1588 sit_i->sentries = vzalloc(TOTAL_SEGS(sbi) * sizeof(struct seg_entry)); 1589 if (!sit_i->sentries) 1590 return -ENOMEM; 1591 1592 bitmap_size = f2fs_bitmap_size(TOTAL_SEGS(sbi)); 1593 sit_i->dirty_sentries_bitmap = kzalloc(bitmap_size, GFP_KERNEL); 1594 if (!sit_i->dirty_sentries_bitmap) 1595 return -ENOMEM; 1596 1597 for (start = 0; start < TOTAL_SEGS(sbi); start++) { 1598 sit_i->sentries[start].cur_valid_map 1599 = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); 1600 sit_i->sentries[start].ckpt_valid_map 1601 = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); 1602 if (!sit_i->sentries[start].cur_valid_map 1603 || !sit_i->sentries[start].ckpt_valid_map) 1604 return -ENOMEM; 1605 } 1606 1607 if (sbi->segs_per_sec > 1) { 1608 sit_i->sec_entries = vzalloc(TOTAL_SECS(sbi) * 1609 sizeof(struct sec_entry)); 1610 if (!sit_i->sec_entries) 1611 return -ENOMEM; 1612 } 1613 1614 /* get information related with SIT */ 1615 sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1; 1616 1617 /* setup SIT bitmap from ckeckpoint pack */ 1618 bitmap_size = __bitmap_size(sbi, SIT_BITMAP); 1619 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP); 1620 1621 dst_bitmap = kmemdup(src_bitmap, bitmap_size, GFP_KERNEL); 1622 if (!dst_bitmap) 1623 return -ENOMEM; 1624 1625 /* init SIT information */ 1626 sit_i->s_ops = &default_salloc_ops; 1627 1628 sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr); 1629 sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg; 1630 sit_i->written_valid_blocks = le64_to_cpu(ckpt->valid_block_count); 1631 sit_i->sit_bitmap = dst_bitmap; 1632 sit_i->bitmap_size = bitmap_size; 1633 sit_i->dirty_sentries = 0; 1634 sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK; 1635 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time); 1636 sit_i->mounted_time = CURRENT_TIME_SEC.tv_sec; 1637 mutex_init(&sit_i->sentry_lock); 1638 return 0; 1639} 1640 1641static int build_free_segmap(struct f2fs_sb_info *sbi) 1642{ 1643 struct f2fs_sm_info *sm_info = SM_I(sbi); 1644 struct free_segmap_info *free_i; 1645 unsigned int bitmap_size, sec_bitmap_size; 1646 1647 /* allocate memory for free segmap information */ 1648 free_i = kzalloc(sizeof(struct free_segmap_info), GFP_KERNEL); 1649 if (!free_i) 1650 return -ENOMEM; 1651 1652 SM_I(sbi)->free_info = free_i; 1653 1654 bitmap_size = f2fs_bitmap_size(TOTAL_SEGS(sbi)); 1655 free_i->free_segmap = kmalloc(bitmap_size, GFP_KERNEL); 1656 if (!free_i->free_segmap) 1657 return -ENOMEM; 1658 1659 sec_bitmap_size = f2fs_bitmap_size(TOTAL_SECS(sbi)); 1660 free_i->free_secmap = kmalloc(sec_bitmap_size, GFP_KERNEL); 1661 if (!free_i->free_secmap) 1662 return -ENOMEM; 1663 1664 /* set all segments as dirty temporarily */ 1665 memset(free_i->free_segmap, 0xff, bitmap_size); 1666 memset(free_i->free_secmap, 0xff, sec_bitmap_size); 1667 1668 /* init free segmap information */ 1669 free_i->start_segno = 1670 (unsigned int) GET_SEGNO_FROM_SEG0(sbi, sm_info->main_blkaddr); 1671 free_i->free_segments = 0; 1672 free_i->free_sections = 0; 1673 rwlock_init(&free_i->segmap_lock); 1674 return 0; 1675} 1676 1677static int build_curseg(struct f2fs_sb_info *sbi) 1678{ 1679 struct curseg_info *array; 1680 int i; 1681 1682 array = kzalloc(sizeof(*array) * NR_CURSEG_TYPE, GFP_KERNEL); 1683 if (!array) 1684 return -ENOMEM; 1685 1686 SM_I(sbi)->curseg_array = array; 1687 1688 for (i = 0; i < NR_CURSEG_TYPE; i++) { 1689 mutex_init(&array[i].curseg_mutex); 1690 array[i].sum_blk = kzalloc(PAGE_CACHE_SIZE, GFP_KERNEL); 1691 if (!array[i].sum_blk) 1692 return -ENOMEM; 1693 array[i].segno = NULL_SEGNO; 1694 array[i].next_blkoff = 0; 1695 } 1696 return restore_curseg_summaries(sbi); 1697} 1698 1699static int ra_sit_pages(struct f2fs_sb_info *sbi, int start, int nrpages) 1700{ 1701 struct address_space *mapping = sbi->meta_inode->i_mapping; 1702 struct page *page; 1703 block_t blk_addr, prev_blk_addr = 0; 1704 int sit_blk_cnt = SIT_BLK_CNT(sbi); 1705 int blkno = start; 1706 1707 for (; blkno < start + nrpages && blkno < sit_blk_cnt; blkno++) { 1708 1709 blk_addr = current_sit_addr(sbi, blkno * SIT_ENTRY_PER_BLOCK); 1710 1711 if (blkno != start && prev_blk_addr + 1 != blk_addr) 1712 break; 1713 prev_blk_addr = blk_addr; 1714repeat: 1715 page = grab_cache_page(mapping, blk_addr); 1716 if (!page) { 1717 cond_resched(); 1718 goto repeat; 1719 } 1720 if (PageUptodate(page)) { 1721 mark_page_accessed(page); 1722 f2fs_put_page(page, 1); 1723 continue; 1724 } 1725 1726 submit_read_page(sbi, page, blk_addr, READ_SYNC | REQ_META); 1727 1728 mark_page_accessed(page); 1729 f2fs_put_page(page, 0); 1730 } 1731 1732 f2fs_submit_read_bio(sbi, READ_SYNC | REQ_META); 1733 return blkno - start; 1734} 1735 1736static void build_sit_entries(struct f2fs_sb_info *sbi) 1737{ 1738 struct sit_info *sit_i = SIT_I(sbi); 1739 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 1740 struct f2fs_summary_block *sum = curseg->sum_blk; 1741 int sit_blk_cnt = SIT_BLK_CNT(sbi); 1742 unsigned int i, start, end; 1743 unsigned int readed, start_blk = 0; 1744 int nrpages = MAX_BIO_BLOCKS(max_hw_blocks(sbi)); 1745 1746 do { 1747 readed = ra_sit_pages(sbi, start_blk, nrpages); 1748 1749 start = start_blk * sit_i->sents_per_block; 1750 end = (start_blk + readed) * sit_i->sents_per_block; 1751 1752 for (; start < end && start < TOTAL_SEGS(sbi); start++) { 1753 struct seg_entry *se = &sit_i->sentries[start]; 1754 struct f2fs_sit_block *sit_blk; 1755 struct f2fs_sit_entry sit; 1756 struct page *page; 1757 1758 mutex_lock(&curseg->curseg_mutex); 1759 for (i = 0; i < sits_in_cursum(sum); i++) { 1760 if (le32_to_cpu(segno_in_journal(sum, i)) == start) { 1761 sit = sit_in_journal(sum, i); 1762 mutex_unlock(&curseg->curseg_mutex); 1763 goto got_it; 1764 } 1765 } 1766 mutex_unlock(&curseg->curseg_mutex); 1767 1768 page = get_current_sit_page(sbi, start); 1769 sit_blk = (struct f2fs_sit_block *)page_address(page); 1770 sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)]; 1771 f2fs_put_page(page, 1); 1772got_it: 1773 check_block_count(sbi, start, &sit); 1774 seg_info_from_raw_sit(se, &sit); 1775 if (sbi->segs_per_sec > 1) { 1776 struct sec_entry *e = get_sec_entry(sbi, start); 1777 e->valid_blocks += se->valid_blocks; 1778 } 1779 } 1780 start_blk += readed; 1781 } while (start_blk < sit_blk_cnt); 1782} 1783 1784static void init_free_segmap(struct f2fs_sb_info *sbi) 1785{ 1786 unsigned int start; 1787 int type; 1788 1789 for (start = 0; start < TOTAL_SEGS(sbi); start++) { 1790 struct seg_entry *sentry = get_seg_entry(sbi, start); 1791 if (!sentry->valid_blocks) 1792 __set_free(sbi, start); 1793 } 1794 1795 /* set use the current segments */ 1796 for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) { 1797 struct curseg_info *curseg_t = CURSEG_I(sbi, type); 1798 __set_test_and_inuse(sbi, curseg_t->segno); 1799 } 1800} 1801 1802static void init_dirty_segmap(struct f2fs_sb_info *sbi) 1803{ 1804 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1805 struct free_segmap_info *free_i = FREE_I(sbi); 1806 unsigned int segno = 0, offset = 0, total_segs = TOTAL_SEGS(sbi); 1807 unsigned short valid_blocks; 1808 1809 while (1) { 1810 /* find dirty segment based on free segmap */ 1811 segno = find_next_inuse(free_i, total_segs, offset); 1812 if (segno >= total_segs) 1813 break; 1814 offset = segno + 1; 1815 valid_blocks = get_valid_blocks(sbi, segno, 0); 1816 if (valid_blocks >= sbi->blocks_per_seg || !valid_blocks) 1817 continue; 1818 mutex_lock(&dirty_i->seglist_lock); 1819 __locate_dirty_segment(sbi, segno, DIRTY); 1820 mutex_unlock(&dirty_i->seglist_lock); 1821 } 1822} 1823 1824static int init_victim_secmap(struct f2fs_sb_info *sbi) 1825{ 1826 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1827 unsigned int bitmap_size = f2fs_bitmap_size(TOTAL_SECS(sbi)); 1828 1829 dirty_i->victim_secmap = kzalloc(bitmap_size, GFP_KERNEL); 1830 if (!dirty_i->victim_secmap) 1831 return -ENOMEM; 1832 return 0; 1833} 1834 1835static int build_dirty_segmap(struct f2fs_sb_info *sbi) 1836{ 1837 struct dirty_seglist_info *dirty_i; 1838 unsigned int bitmap_size, i; 1839 1840 /* allocate memory for dirty segments list information */ 1841 dirty_i = kzalloc(sizeof(struct dirty_seglist_info), GFP_KERNEL); 1842 if (!dirty_i) 1843 return -ENOMEM; 1844 1845 SM_I(sbi)->dirty_info = dirty_i; 1846 mutex_init(&dirty_i->seglist_lock); 1847 1848 bitmap_size = f2fs_bitmap_size(TOTAL_SEGS(sbi)); 1849 1850 for (i = 0; i < NR_DIRTY_TYPE; i++) { 1851 dirty_i->dirty_segmap[i] = kzalloc(bitmap_size, GFP_KERNEL); 1852 if (!dirty_i->dirty_segmap[i]) 1853 return -ENOMEM; 1854 } 1855 1856 init_dirty_segmap(sbi); 1857 return init_victim_secmap(sbi); 1858} 1859 1860/* 1861 * Update min, max modified time for cost-benefit GC algorithm 1862 */ 1863static void init_min_max_mtime(struct f2fs_sb_info *sbi) 1864{ 1865 struct sit_info *sit_i = SIT_I(sbi); 1866 unsigned int segno; 1867 1868 mutex_lock(&sit_i->sentry_lock); 1869 1870 sit_i->min_mtime = LLONG_MAX; 1871 1872 for (segno = 0; segno < TOTAL_SEGS(sbi); segno += sbi->segs_per_sec) { 1873 unsigned int i; 1874 unsigned long long mtime = 0; 1875 1876 for (i = 0; i < sbi->segs_per_sec; i++) 1877 mtime += get_seg_entry(sbi, segno + i)->mtime; 1878 1879 mtime = div_u64(mtime, sbi->segs_per_sec); 1880 1881 if (sit_i->min_mtime > mtime) 1882 sit_i->min_mtime = mtime; 1883 } 1884 sit_i->max_mtime = get_mtime(sbi); 1885 mutex_unlock(&sit_i->sentry_lock); 1886} 1887 1888int build_segment_manager(struct f2fs_sb_info *sbi) 1889{ 1890 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); 1891 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 1892 struct f2fs_sm_info *sm_info; 1893 int err; 1894 1895 sm_info = kzalloc(sizeof(struct f2fs_sm_info), GFP_KERNEL); 1896 if (!sm_info) 1897 return -ENOMEM; 1898 1899 /* init sm info */ 1900 sbi->sm_info = sm_info; 1901 INIT_LIST_HEAD(&sm_info->wblist_head); 1902 spin_lock_init(&sm_info->wblist_lock); 1903 sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr); 1904 sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr); 1905 sm_info->segment_count = le32_to_cpu(raw_super->segment_count); 1906 sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count); 1907 sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count); 1908 sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main); 1909 sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr); 1910 sm_info->rec_prefree_segments = DEF_RECLAIM_PREFREE_SEGMENTS; 1911 1912 INIT_LIST_HEAD(&sm_info->discard_list); 1913 sm_info->nr_discards = 0; 1914 sm_info->max_discards = 0; 1915 1916 err = build_sit_info(sbi); 1917 if (err) 1918 return err; 1919 err = build_free_segmap(sbi); 1920 if (err) 1921 return err; 1922 err = build_curseg(sbi); 1923 if (err) 1924 return err; 1925 1926 /* reinit free segmap based on SIT */ 1927 build_sit_entries(sbi); 1928 1929 init_free_segmap(sbi); 1930 err = build_dirty_segmap(sbi); 1931 if (err) 1932 return err; 1933 1934 init_min_max_mtime(sbi); 1935 return 0; 1936} 1937 1938static void discard_dirty_segmap(struct f2fs_sb_info *sbi, 1939 enum dirty_type dirty_type) 1940{ 1941 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1942 1943 mutex_lock(&dirty_i->seglist_lock); 1944 kfree(dirty_i->dirty_segmap[dirty_type]); 1945 dirty_i->nr_dirty[dirty_type] = 0; 1946 mutex_unlock(&dirty_i->seglist_lock); 1947} 1948 1949static void destroy_victim_secmap(struct f2fs_sb_info *sbi) 1950{ 1951 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1952 kfree(dirty_i->victim_secmap); 1953} 1954 1955static void destroy_dirty_segmap(struct f2fs_sb_info *sbi) 1956{ 1957 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1958 int i; 1959 1960 if (!dirty_i) 1961 return; 1962 1963 /* discard pre-free/dirty segments list */ 1964 for (i = 0; i < NR_DIRTY_TYPE; i++) 1965 discard_dirty_segmap(sbi, i); 1966 1967 destroy_victim_secmap(sbi); 1968 SM_I(sbi)->dirty_info = NULL; 1969 kfree(dirty_i); 1970} 1971 1972static void destroy_curseg(struct f2fs_sb_info *sbi) 1973{ 1974 struct curseg_info *array = SM_I(sbi)->curseg_array; 1975 int i; 1976 1977 if (!array) 1978 return; 1979 SM_I(sbi)->curseg_array = NULL; 1980 for (i = 0; i < NR_CURSEG_TYPE; i++) 1981 kfree(array[i].sum_blk); 1982 kfree(array); 1983} 1984 1985static void destroy_free_segmap(struct f2fs_sb_info *sbi) 1986{ 1987 struct free_segmap_info *free_i = SM_I(sbi)->free_info; 1988 if (!free_i) 1989 return; 1990 SM_I(sbi)->free_info = NULL; 1991 kfree(free_i->free_segmap); 1992 kfree(free_i->free_secmap); 1993 kfree(free_i); 1994} 1995 1996static void destroy_sit_info(struct f2fs_sb_info *sbi) 1997{ 1998 struct sit_info *sit_i = SIT_I(sbi); 1999 unsigned int start; 2000 2001 if (!sit_i) 2002 return; 2003 2004 if (sit_i->sentries) { 2005 for (start = 0; start < TOTAL_SEGS(sbi); start++) { 2006 kfree(sit_i->sentries[start].cur_valid_map); 2007 kfree(sit_i->sentries[start].ckpt_valid_map); 2008 } 2009 } 2010 vfree(sit_i->sentries); 2011 vfree(sit_i->sec_entries); 2012 kfree(sit_i->dirty_sentries_bitmap); 2013 2014 SM_I(sbi)->sit_info = NULL; 2015 kfree(sit_i->sit_bitmap); 2016 kfree(sit_i); 2017} 2018 2019void destroy_segment_manager(struct f2fs_sb_info *sbi) 2020{ 2021 struct f2fs_sm_info *sm_info = SM_I(sbi); 2022 if (!sm_info) 2023 return; 2024 destroy_dirty_segmap(sbi); 2025 destroy_curseg(sbi); 2026 destroy_free_segmap(sbi); 2027 destroy_sit_info(sbi); 2028 sbi->sm_info = NULL; 2029 kfree(sm_info); 2030} 2031 2032int __init create_segment_manager_caches(void) 2033{ 2034 discard_entry_slab = f2fs_kmem_cache_create("discard_entry", 2035 sizeof(struct discard_entry), NULL); 2036 if (!discard_entry_slab) 2037 return -ENOMEM; 2038 return 0; 2039} 2040 2041void destroy_segment_manager_caches(void) 2042{ 2043 kmem_cache_destroy(discard_entry_slab); 2044} 2045