segment.c revision 184a5cd2ce281f1207d72adb9ae18e416ca371db
1/* 2 * fs/f2fs/segment.c 3 * 4 * Copyright (c) 2012 Samsung Electronics Co., Ltd. 5 * http://www.samsung.com/ 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11#include <linux/fs.h> 12#include <linux/f2fs_fs.h> 13#include <linux/bio.h> 14#include <linux/blkdev.h> 15#include <linux/prefetch.h> 16#include <linux/kthread.h> 17#include <linux/vmalloc.h> 18#include <linux/swap.h> 19 20#include "f2fs.h" 21#include "segment.h" 22#include "node.h" 23#include <trace/events/f2fs.h> 24 25#define __reverse_ffz(x) __reverse_ffs(~(x)) 26 27static struct kmem_cache *discard_entry_slab; 28static struct kmem_cache *sit_entry_set_slab; 29 30/* 31 * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since 32 * MSB and LSB are reversed in a byte by f2fs_set_bit. 33 */ 34static inline unsigned long __reverse_ffs(unsigned long word) 35{ 36 int num = 0; 37 38#if BITS_PER_LONG == 64 39 if ((word & 0xffffffff) == 0) { 40 num += 32; 41 word >>= 32; 42 } 43#endif 44 if ((word & 0xffff) == 0) { 45 num += 16; 46 word >>= 16; 47 } 48 if ((word & 0xff) == 0) { 49 num += 8; 50 word >>= 8; 51 } 52 if ((word & 0xf0) == 0) 53 num += 4; 54 else 55 word >>= 4; 56 if ((word & 0xc) == 0) 57 num += 2; 58 else 59 word >>= 2; 60 if ((word & 0x2) == 0) 61 num += 1; 62 return num; 63} 64 65/* 66 * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because 67 * f2fs_set_bit makes MSB and LSB reversed in a byte. 68 * Example: 69 * LSB <--> MSB 70 * f2fs_set_bit(0, bitmap) => 0000 0001 71 * f2fs_set_bit(7, bitmap) => 1000 0000 72 */ 73static unsigned long __find_rev_next_bit(const unsigned long *addr, 74 unsigned long size, unsigned long offset) 75{ 76 const unsigned long *p = addr + BIT_WORD(offset); 77 unsigned long result = offset & ~(BITS_PER_LONG - 1); 78 unsigned long tmp; 79 unsigned long mask, submask; 80 unsigned long quot, rest; 81 82 if (offset >= size) 83 return size; 84 85 size -= result; 86 offset %= BITS_PER_LONG; 87 if (!offset) 88 goto aligned; 89 90 tmp = *(p++); 91 quot = (offset >> 3) << 3; 92 rest = offset & 0x7; 93 mask = ~0UL << quot; 94 submask = (unsigned char)(0xff << rest) >> rest; 95 submask <<= quot; 96 mask &= submask; 97 tmp &= mask; 98 if (size < BITS_PER_LONG) 99 goto found_first; 100 if (tmp) 101 goto found_middle; 102 103 size -= BITS_PER_LONG; 104 result += BITS_PER_LONG; 105aligned: 106 while (size & ~(BITS_PER_LONG-1)) { 107 tmp = *(p++); 108 if (tmp) 109 goto found_middle; 110 result += BITS_PER_LONG; 111 size -= BITS_PER_LONG; 112 } 113 if (!size) 114 return result; 115 tmp = *p; 116found_first: 117 tmp &= (~0UL >> (BITS_PER_LONG - size)); 118 if (tmp == 0UL) /* Are any bits set? */ 119 return result + size; /* Nope. */ 120found_middle: 121 return result + __reverse_ffs(tmp); 122} 123 124static unsigned long __find_rev_next_zero_bit(const unsigned long *addr, 125 unsigned long size, unsigned long offset) 126{ 127 const unsigned long *p = addr + BIT_WORD(offset); 128 unsigned long result = offset & ~(BITS_PER_LONG - 1); 129 unsigned long tmp; 130 unsigned long mask, submask; 131 unsigned long quot, rest; 132 133 if (offset >= size) 134 return size; 135 136 size -= result; 137 offset %= BITS_PER_LONG; 138 if (!offset) 139 goto aligned; 140 141 tmp = *(p++); 142 quot = (offset >> 3) << 3; 143 rest = offset & 0x7; 144 mask = ~(~0UL << quot); 145 submask = (unsigned char)~((unsigned char)(0xff << rest) >> rest); 146 submask <<= quot; 147 mask += submask; 148 tmp |= mask; 149 if (size < BITS_PER_LONG) 150 goto found_first; 151 if (~tmp) 152 goto found_middle; 153 154 size -= BITS_PER_LONG; 155 result += BITS_PER_LONG; 156aligned: 157 while (size & ~(BITS_PER_LONG - 1)) { 158 tmp = *(p++); 159 if (~tmp) 160 goto found_middle; 161 result += BITS_PER_LONG; 162 size -= BITS_PER_LONG; 163 } 164 if (!size) 165 return result; 166 tmp = *p; 167 168found_first: 169 tmp |= ~0UL << size; 170 if (tmp == ~0UL) /* Are any bits zero? */ 171 return result + size; /* Nope. */ 172found_middle: 173 return result + __reverse_ffz(tmp); 174} 175 176/* 177 * This function balances dirty node and dentry pages. 178 * In addition, it controls garbage collection. 179 */ 180void f2fs_balance_fs(struct f2fs_sb_info *sbi) 181{ 182 /* 183 * We should do GC or end up with checkpoint, if there are so many dirty 184 * dir/node pages without enough free segments. 185 */ 186 if (has_not_enough_free_secs(sbi, 0)) { 187 mutex_lock(&sbi->gc_mutex); 188 f2fs_gc(sbi); 189 } 190} 191 192void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi) 193{ 194 /* check the # of cached NAT entries and prefree segments */ 195 if (try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK) || 196 excess_prefree_segs(sbi)) 197 f2fs_sync_fs(sbi->sb, true); 198} 199 200static int issue_flush_thread(void *data) 201{ 202 struct f2fs_sb_info *sbi = data; 203 struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info; 204 wait_queue_head_t *q = &fcc->flush_wait_queue; 205repeat: 206 if (kthread_should_stop()) 207 return 0; 208 209 spin_lock(&fcc->issue_lock); 210 if (fcc->issue_list) { 211 fcc->dispatch_list = fcc->issue_list; 212 fcc->issue_list = fcc->issue_tail = NULL; 213 } 214 spin_unlock(&fcc->issue_lock); 215 216 if (fcc->dispatch_list) { 217 struct bio *bio = bio_alloc(GFP_NOIO, 0); 218 struct flush_cmd *cmd, *next; 219 int ret; 220 221 bio->bi_bdev = sbi->sb->s_bdev; 222 ret = submit_bio_wait(WRITE_FLUSH, bio); 223 224 for (cmd = fcc->dispatch_list; cmd; cmd = next) { 225 cmd->ret = ret; 226 next = cmd->next; 227 complete(&cmd->wait); 228 } 229 bio_put(bio); 230 fcc->dispatch_list = NULL; 231 } 232 233 wait_event_interruptible(*q, 234 kthread_should_stop() || fcc->issue_list); 235 goto repeat; 236} 237 238int f2fs_issue_flush(struct f2fs_sb_info *sbi) 239{ 240 struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info; 241 struct flush_cmd cmd; 242 243 trace_f2fs_issue_flush(sbi->sb, test_opt(sbi, NOBARRIER), 244 test_opt(sbi, FLUSH_MERGE)); 245 246 if (test_opt(sbi, NOBARRIER)) 247 return 0; 248 249 if (!test_opt(sbi, FLUSH_MERGE)) 250 return blkdev_issue_flush(sbi->sb->s_bdev, GFP_KERNEL, NULL); 251 252 init_completion(&cmd.wait); 253 cmd.next = NULL; 254 255 spin_lock(&fcc->issue_lock); 256 if (fcc->issue_list) 257 fcc->issue_tail->next = &cmd; 258 else 259 fcc->issue_list = &cmd; 260 fcc->issue_tail = &cmd; 261 spin_unlock(&fcc->issue_lock); 262 263 if (!fcc->dispatch_list) 264 wake_up(&fcc->flush_wait_queue); 265 266 wait_for_completion(&cmd.wait); 267 268 return cmd.ret; 269} 270 271int create_flush_cmd_control(struct f2fs_sb_info *sbi) 272{ 273 dev_t dev = sbi->sb->s_bdev->bd_dev; 274 struct flush_cmd_control *fcc; 275 int err = 0; 276 277 fcc = kzalloc(sizeof(struct flush_cmd_control), GFP_KERNEL); 278 if (!fcc) 279 return -ENOMEM; 280 spin_lock_init(&fcc->issue_lock); 281 init_waitqueue_head(&fcc->flush_wait_queue); 282 SM_I(sbi)->cmd_control_info = fcc; 283 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi, 284 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev)); 285 if (IS_ERR(fcc->f2fs_issue_flush)) { 286 err = PTR_ERR(fcc->f2fs_issue_flush); 287 kfree(fcc); 288 SM_I(sbi)->cmd_control_info = NULL; 289 return err; 290 } 291 292 return err; 293} 294 295void destroy_flush_cmd_control(struct f2fs_sb_info *sbi) 296{ 297 struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info; 298 299 if (fcc && fcc->f2fs_issue_flush) 300 kthread_stop(fcc->f2fs_issue_flush); 301 kfree(fcc); 302 SM_I(sbi)->cmd_control_info = NULL; 303} 304 305static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, 306 enum dirty_type dirty_type) 307{ 308 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 309 310 /* need not be added */ 311 if (IS_CURSEG(sbi, segno)) 312 return; 313 314 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type])) 315 dirty_i->nr_dirty[dirty_type]++; 316 317 if (dirty_type == DIRTY) { 318 struct seg_entry *sentry = get_seg_entry(sbi, segno); 319 enum dirty_type t = sentry->type; 320 321 if (unlikely(t >= DIRTY)) { 322 f2fs_bug_on(sbi, 1); 323 return; 324 } 325 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t])) 326 dirty_i->nr_dirty[t]++; 327 } 328} 329 330static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, 331 enum dirty_type dirty_type) 332{ 333 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 334 335 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type])) 336 dirty_i->nr_dirty[dirty_type]--; 337 338 if (dirty_type == DIRTY) { 339 struct seg_entry *sentry = get_seg_entry(sbi, segno); 340 enum dirty_type t = sentry->type; 341 342 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t])) 343 dirty_i->nr_dirty[t]--; 344 345 if (get_valid_blocks(sbi, segno, sbi->segs_per_sec) == 0) 346 clear_bit(GET_SECNO(sbi, segno), 347 dirty_i->victim_secmap); 348 } 349} 350 351/* 352 * Should not occur error such as -ENOMEM. 353 * Adding dirty entry into seglist is not critical operation. 354 * If a given segment is one of current working segments, it won't be added. 355 */ 356static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno) 357{ 358 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 359 unsigned short valid_blocks; 360 361 if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno)) 362 return; 363 364 mutex_lock(&dirty_i->seglist_lock); 365 366 valid_blocks = get_valid_blocks(sbi, segno, 0); 367 368 if (valid_blocks == 0) { 369 __locate_dirty_segment(sbi, segno, PRE); 370 __remove_dirty_segment(sbi, segno, DIRTY); 371 } else if (valid_blocks < sbi->blocks_per_seg) { 372 __locate_dirty_segment(sbi, segno, DIRTY); 373 } else { 374 /* Recovery routine with SSR needs this */ 375 __remove_dirty_segment(sbi, segno, DIRTY); 376 } 377 378 mutex_unlock(&dirty_i->seglist_lock); 379} 380 381static int f2fs_issue_discard(struct f2fs_sb_info *sbi, 382 block_t blkstart, block_t blklen) 383{ 384 sector_t start = SECTOR_FROM_BLOCK(sbi, blkstart); 385 sector_t len = SECTOR_FROM_BLOCK(sbi, blklen); 386 trace_f2fs_issue_discard(sbi->sb, blkstart, blklen); 387 return blkdev_issue_discard(sbi->sb->s_bdev, start, len, GFP_NOFS, 0); 388} 389 390void discard_next_dnode(struct f2fs_sb_info *sbi, block_t blkaddr) 391{ 392 if (f2fs_issue_discard(sbi, blkaddr, 1)) { 393 struct page *page = grab_meta_page(sbi, blkaddr); 394 /* zero-filled page */ 395 set_page_dirty(page); 396 f2fs_put_page(page, 1); 397 } 398} 399 400static void add_discard_addrs(struct f2fs_sb_info *sbi, 401 unsigned int segno, struct seg_entry *se) 402{ 403 struct list_head *head = &SM_I(sbi)->discard_list; 404 struct discard_entry *new; 405 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); 406 int max_blocks = sbi->blocks_per_seg; 407 unsigned long *cur_map = (unsigned long *)se->cur_valid_map; 408 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; 409 unsigned long dmap[entries]; 410 unsigned int start = 0, end = -1; 411 int i; 412 413 if (!test_opt(sbi, DISCARD)) 414 return; 415 416 /* zero block will be discarded through the prefree list */ 417 if (!se->valid_blocks || se->valid_blocks == max_blocks) 418 return; 419 420 /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */ 421 for (i = 0; i < entries; i++) 422 dmap[i] = (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i]; 423 424 while (SM_I(sbi)->nr_discards <= SM_I(sbi)->max_discards) { 425 start = __find_rev_next_bit(dmap, max_blocks, end + 1); 426 if (start >= max_blocks) 427 break; 428 429 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1); 430 431 new = f2fs_kmem_cache_alloc(discard_entry_slab, GFP_NOFS); 432 INIT_LIST_HEAD(&new->list); 433 new->blkaddr = START_BLOCK(sbi, segno) + start; 434 new->len = end - start; 435 436 list_add_tail(&new->list, head); 437 SM_I(sbi)->nr_discards += end - start; 438 } 439} 440 441/* 442 * Should call clear_prefree_segments after checkpoint is done. 443 */ 444static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi) 445{ 446 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 447 unsigned int segno; 448 unsigned int total_segs = TOTAL_SEGS(sbi); 449 450 mutex_lock(&dirty_i->seglist_lock); 451 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], total_segs) 452 __set_test_and_free(sbi, segno); 453 mutex_unlock(&dirty_i->seglist_lock); 454} 455 456void clear_prefree_segments(struct f2fs_sb_info *sbi) 457{ 458 struct list_head *head = &(SM_I(sbi)->discard_list); 459 struct discard_entry *entry, *this; 460 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 461 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE]; 462 unsigned int total_segs = TOTAL_SEGS(sbi); 463 unsigned int start = 0, end = -1; 464 465 mutex_lock(&dirty_i->seglist_lock); 466 467 while (1) { 468 int i; 469 start = find_next_bit(prefree_map, total_segs, end + 1); 470 if (start >= total_segs) 471 break; 472 end = find_next_zero_bit(prefree_map, total_segs, start + 1); 473 474 for (i = start; i < end; i++) 475 clear_bit(i, prefree_map); 476 477 dirty_i->nr_dirty[PRE] -= end - start; 478 479 if (!test_opt(sbi, DISCARD)) 480 continue; 481 482 f2fs_issue_discard(sbi, START_BLOCK(sbi, start), 483 (end - start) << sbi->log_blocks_per_seg); 484 } 485 mutex_unlock(&dirty_i->seglist_lock); 486 487 /* send small discards */ 488 list_for_each_entry_safe(entry, this, head, list) { 489 f2fs_issue_discard(sbi, entry->blkaddr, entry->len); 490 list_del(&entry->list); 491 SM_I(sbi)->nr_discards -= entry->len; 492 kmem_cache_free(discard_entry_slab, entry); 493 } 494} 495 496static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno) 497{ 498 struct sit_info *sit_i = SIT_I(sbi); 499 500 if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) { 501 sit_i->dirty_sentries++; 502 return false; 503 } 504 505 return true; 506} 507 508static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type, 509 unsigned int segno, int modified) 510{ 511 struct seg_entry *se = get_seg_entry(sbi, segno); 512 se->type = type; 513 if (modified) 514 __mark_sit_entry_dirty(sbi, segno); 515} 516 517static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del) 518{ 519 struct seg_entry *se; 520 unsigned int segno, offset; 521 long int new_vblocks; 522 523 segno = GET_SEGNO(sbi, blkaddr); 524 525 se = get_seg_entry(sbi, segno); 526 new_vblocks = se->valid_blocks + del; 527 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); 528 529 f2fs_bug_on(sbi, (new_vblocks >> (sizeof(unsigned short) << 3) || 530 (new_vblocks > sbi->blocks_per_seg))); 531 532 se->valid_blocks = new_vblocks; 533 se->mtime = get_mtime(sbi); 534 SIT_I(sbi)->max_mtime = se->mtime; 535 536 /* Update valid block bitmap */ 537 if (del > 0) { 538 if (f2fs_set_bit(offset, se->cur_valid_map)) 539 f2fs_bug_on(sbi, 1); 540 } else { 541 if (!f2fs_clear_bit(offset, se->cur_valid_map)) 542 f2fs_bug_on(sbi, 1); 543 } 544 if (!f2fs_test_bit(offset, se->ckpt_valid_map)) 545 se->ckpt_valid_blocks += del; 546 547 __mark_sit_entry_dirty(sbi, segno); 548 549 /* update total number of valid blocks to be written in ckpt area */ 550 SIT_I(sbi)->written_valid_blocks += del; 551 552 if (sbi->segs_per_sec > 1) 553 get_sec_entry(sbi, segno)->valid_blocks += del; 554} 555 556void refresh_sit_entry(struct f2fs_sb_info *sbi, block_t old, block_t new) 557{ 558 update_sit_entry(sbi, new, 1); 559 if (GET_SEGNO(sbi, old) != NULL_SEGNO) 560 update_sit_entry(sbi, old, -1); 561 562 locate_dirty_segment(sbi, GET_SEGNO(sbi, old)); 563 locate_dirty_segment(sbi, GET_SEGNO(sbi, new)); 564} 565 566void invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr) 567{ 568 unsigned int segno = GET_SEGNO(sbi, addr); 569 struct sit_info *sit_i = SIT_I(sbi); 570 571 f2fs_bug_on(sbi, addr == NULL_ADDR); 572 if (addr == NEW_ADDR) 573 return; 574 575 /* add it into sit main buffer */ 576 mutex_lock(&sit_i->sentry_lock); 577 578 update_sit_entry(sbi, addr, -1); 579 580 /* add it into dirty seglist */ 581 locate_dirty_segment(sbi, segno); 582 583 mutex_unlock(&sit_i->sentry_lock); 584} 585 586/* 587 * This function should be resided under the curseg_mutex lock 588 */ 589static void __add_sum_entry(struct f2fs_sb_info *sbi, int type, 590 struct f2fs_summary *sum) 591{ 592 struct curseg_info *curseg = CURSEG_I(sbi, type); 593 void *addr = curseg->sum_blk; 594 addr += curseg->next_blkoff * sizeof(struct f2fs_summary); 595 memcpy(addr, sum, sizeof(struct f2fs_summary)); 596} 597 598/* 599 * Calculate the number of current summary pages for writing 600 */ 601int npages_for_summary_flush(struct f2fs_sb_info *sbi) 602{ 603 int valid_sum_count = 0; 604 int i, sum_in_page; 605 606 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 607 if (sbi->ckpt->alloc_type[i] == SSR) 608 valid_sum_count += sbi->blocks_per_seg; 609 else 610 valid_sum_count += curseg_blkoff(sbi, i); 611 } 612 613 sum_in_page = (PAGE_CACHE_SIZE - 2 * SUM_JOURNAL_SIZE - 614 SUM_FOOTER_SIZE) / SUMMARY_SIZE; 615 if (valid_sum_count <= sum_in_page) 616 return 1; 617 else if ((valid_sum_count - sum_in_page) <= 618 (PAGE_CACHE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE) 619 return 2; 620 return 3; 621} 622 623/* 624 * Caller should put this summary page 625 */ 626struct page *get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno) 627{ 628 return get_meta_page(sbi, GET_SUM_BLOCK(sbi, segno)); 629} 630 631static void write_sum_page(struct f2fs_sb_info *sbi, 632 struct f2fs_summary_block *sum_blk, block_t blk_addr) 633{ 634 struct page *page = grab_meta_page(sbi, blk_addr); 635 void *kaddr = page_address(page); 636 memcpy(kaddr, sum_blk, PAGE_CACHE_SIZE); 637 set_page_dirty(page); 638 f2fs_put_page(page, 1); 639} 640 641static int is_next_segment_free(struct f2fs_sb_info *sbi, int type) 642{ 643 struct curseg_info *curseg = CURSEG_I(sbi, type); 644 unsigned int segno = curseg->segno + 1; 645 struct free_segmap_info *free_i = FREE_I(sbi); 646 647 if (segno < TOTAL_SEGS(sbi) && segno % sbi->segs_per_sec) 648 return !test_bit(segno, free_i->free_segmap); 649 return 0; 650} 651 652/* 653 * Find a new segment from the free segments bitmap to right order 654 * This function should be returned with success, otherwise BUG 655 */ 656static void get_new_segment(struct f2fs_sb_info *sbi, 657 unsigned int *newseg, bool new_sec, int dir) 658{ 659 struct free_segmap_info *free_i = FREE_I(sbi); 660 unsigned int segno, secno, zoneno; 661 unsigned int total_zones = TOTAL_SECS(sbi) / sbi->secs_per_zone; 662 unsigned int hint = *newseg / sbi->segs_per_sec; 663 unsigned int old_zoneno = GET_ZONENO_FROM_SEGNO(sbi, *newseg); 664 unsigned int left_start = hint; 665 bool init = true; 666 int go_left = 0; 667 int i; 668 669 write_lock(&free_i->segmap_lock); 670 671 if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) { 672 segno = find_next_zero_bit(free_i->free_segmap, 673 TOTAL_SEGS(sbi), *newseg + 1); 674 if (segno - *newseg < sbi->segs_per_sec - 675 (*newseg % sbi->segs_per_sec)) 676 goto got_it; 677 } 678find_other_zone: 679 secno = find_next_zero_bit(free_i->free_secmap, TOTAL_SECS(sbi), hint); 680 if (secno >= TOTAL_SECS(sbi)) { 681 if (dir == ALLOC_RIGHT) { 682 secno = find_next_zero_bit(free_i->free_secmap, 683 TOTAL_SECS(sbi), 0); 684 f2fs_bug_on(sbi, secno >= TOTAL_SECS(sbi)); 685 } else { 686 go_left = 1; 687 left_start = hint - 1; 688 } 689 } 690 if (go_left == 0) 691 goto skip_left; 692 693 while (test_bit(left_start, free_i->free_secmap)) { 694 if (left_start > 0) { 695 left_start--; 696 continue; 697 } 698 left_start = find_next_zero_bit(free_i->free_secmap, 699 TOTAL_SECS(sbi), 0); 700 f2fs_bug_on(sbi, left_start >= TOTAL_SECS(sbi)); 701 break; 702 } 703 secno = left_start; 704skip_left: 705 hint = secno; 706 segno = secno * sbi->segs_per_sec; 707 zoneno = secno / sbi->secs_per_zone; 708 709 /* give up on finding another zone */ 710 if (!init) 711 goto got_it; 712 if (sbi->secs_per_zone == 1) 713 goto got_it; 714 if (zoneno == old_zoneno) 715 goto got_it; 716 if (dir == ALLOC_LEFT) { 717 if (!go_left && zoneno + 1 >= total_zones) 718 goto got_it; 719 if (go_left && zoneno == 0) 720 goto got_it; 721 } 722 for (i = 0; i < NR_CURSEG_TYPE; i++) 723 if (CURSEG_I(sbi, i)->zone == zoneno) 724 break; 725 726 if (i < NR_CURSEG_TYPE) { 727 /* zone is in user, try another */ 728 if (go_left) 729 hint = zoneno * sbi->secs_per_zone - 1; 730 else if (zoneno + 1 >= total_zones) 731 hint = 0; 732 else 733 hint = (zoneno + 1) * sbi->secs_per_zone; 734 init = false; 735 goto find_other_zone; 736 } 737got_it: 738 /* set it as dirty segment in free segmap */ 739 f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap)); 740 __set_inuse(sbi, segno); 741 *newseg = segno; 742 write_unlock(&free_i->segmap_lock); 743} 744 745static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified) 746{ 747 struct curseg_info *curseg = CURSEG_I(sbi, type); 748 struct summary_footer *sum_footer; 749 750 curseg->segno = curseg->next_segno; 751 curseg->zone = GET_ZONENO_FROM_SEGNO(sbi, curseg->segno); 752 curseg->next_blkoff = 0; 753 curseg->next_segno = NULL_SEGNO; 754 755 sum_footer = &(curseg->sum_blk->footer); 756 memset(sum_footer, 0, sizeof(struct summary_footer)); 757 if (IS_DATASEG(type)) 758 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA); 759 if (IS_NODESEG(type)) 760 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE); 761 __set_sit_entry_type(sbi, type, curseg->segno, modified); 762} 763 764/* 765 * Allocate a current working segment. 766 * This function always allocates a free segment in LFS manner. 767 */ 768static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) 769{ 770 struct curseg_info *curseg = CURSEG_I(sbi, type); 771 unsigned int segno = curseg->segno; 772 int dir = ALLOC_LEFT; 773 774 write_sum_page(sbi, curseg->sum_blk, 775 GET_SUM_BLOCK(sbi, segno)); 776 if (type == CURSEG_WARM_DATA || type == CURSEG_COLD_DATA) 777 dir = ALLOC_RIGHT; 778 779 if (test_opt(sbi, NOHEAP)) 780 dir = ALLOC_RIGHT; 781 782 get_new_segment(sbi, &segno, new_sec, dir); 783 curseg->next_segno = segno; 784 reset_curseg(sbi, type, 1); 785 curseg->alloc_type = LFS; 786} 787 788static void __next_free_blkoff(struct f2fs_sb_info *sbi, 789 struct curseg_info *seg, block_t start) 790{ 791 struct seg_entry *se = get_seg_entry(sbi, seg->segno); 792 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); 793 unsigned long target_map[entries]; 794 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; 795 unsigned long *cur_map = (unsigned long *)se->cur_valid_map; 796 int i, pos; 797 798 for (i = 0; i < entries; i++) 799 target_map[i] = ckpt_map[i] | cur_map[i]; 800 801 pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start); 802 803 seg->next_blkoff = pos; 804} 805 806/* 807 * If a segment is written by LFS manner, next block offset is just obtained 808 * by increasing the current block offset. However, if a segment is written by 809 * SSR manner, next block offset obtained by calling __next_free_blkoff 810 */ 811static void __refresh_next_blkoff(struct f2fs_sb_info *sbi, 812 struct curseg_info *seg) 813{ 814 if (seg->alloc_type == SSR) 815 __next_free_blkoff(sbi, seg, seg->next_blkoff + 1); 816 else 817 seg->next_blkoff++; 818} 819 820/* 821 * This function always allocates a used segment(from dirty seglist) by SSR 822 * manner, so it should recover the existing segment information of valid blocks 823 */ 824static void change_curseg(struct f2fs_sb_info *sbi, int type, bool reuse) 825{ 826 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 827 struct curseg_info *curseg = CURSEG_I(sbi, type); 828 unsigned int new_segno = curseg->next_segno; 829 struct f2fs_summary_block *sum_node; 830 struct page *sum_page; 831 832 write_sum_page(sbi, curseg->sum_blk, 833 GET_SUM_BLOCK(sbi, curseg->segno)); 834 __set_test_and_inuse(sbi, new_segno); 835 836 mutex_lock(&dirty_i->seglist_lock); 837 __remove_dirty_segment(sbi, new_segno, PRE); 838 __remove_dirty_segment(sbi, new_segno, DIRTY); 839 mutex_unlock(&dirty_i->seglist_lock); 840 841 reset_curseg(sbi, type, 1); 842 curseg->alloc_type = SSR; 843 __next_free_blkoff(sbi, curseg, 0); 844 845 if (reuse) { 846 sum_page = get_sum_page(sbi, new_segno); 847 sum_node = (struct f2fs_summary_block *)page_address(sum_page); 848 memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE); 849 f2fs_put_page(sum_page, 1); 850 } 851} 852 853static int get_ssr_segment(struct f2fs_sb_info *sbi, int type) 854{ 855 struct curseg_info *curseg = CURSEG_I(sbi, type); 856 const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops; 857 858 if (IS_NODESEG(type) || !has_not_enough_free_secs(sbi, 0)) 859 return v_ops->get_victim(sbi, 860 &(curseg)->next_segno, BG_GC, type, SSR); 861 862 /* For data segments, let's do SSR more intensively */ 863 for (; type >= CURSEG_HOT_DATA; type--) 864 if (v_ops->get_victim(sbi, &(curseg)->next_segno, 865 BG_GC, type, SSR)) 866 return 1; 867 return 0; 868} 869 870/* 871 * flush out current segment and replace it with new segment 872 * This function should be returned with success, otherwise BUG 873 */ 874static void allocate_segment_by_default(struct f2fs_sb_info *sbi, 875 int type, bool force) 876{ 877 struct curseg_info *curseg = CURSEG_I(sbi, type); 878 879 if (force) 880 new_curseg(sbi, type, true); 881 else if (type == CURSEG_WARM_NODE) 882 new_curseg(sbi, type, false); 883 else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type)) 884 new_curseg(sbi, type, false); 885 else if (need_SSR(sbi) && get_ssr_segment(sbi, type)) 886 change_curseg(sbi, type, true); 887 else 888 new_curseg(sbi, type, false); 889 890 stat_inc_seg_type(sbi, curseg); 891} 892 893void allocate_new_segments(struct f2fs_sb_info *sbi) 894{ 895 struct curseg_info *curseg; 896 unsigned int old_curseg; 897 int i; 898 899 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 900 curseg = CURSEG_I(sbi, i); 901 old_curseg = curseg->segno; 902 SIT_I(sbi)->s_ops->allocate_segment(sbi, i, true); 903 locate_dirty_segment(sbi, old_curseg); 904 } 905} 906 907static const struct segment_allocation default_salloc_ops = { 908 .allocate_segment = allocate_segment_by_default, 909}; 910 911static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type) 912{ 913 struct curseg_info *curseg = CURSEG_I(sbi, type); 914 if (curseg->next_blkoff < sbi->blocks_per_seg) 915 return true; 916 return false; 917} 918 919static int __get_segment_type_2(struct page *page, enum page_type p_type) 920{ 921 if (p_type == DATA) 922 return CURSEG_HOT_DATA; 923 else 924 return CURSEG_HOT_NODE; 925} 926 927static int __get_segment_type_4(struct page *page, enum page_type p_type) 928{ 929 if (p_type == DATA) { 930 struct inode *inode = page->mapping->host; 931 932 if (S_ISDIR(inode->i_mode)) 933 return CURSEG_HOT_DATA; 934 else 935 return CURSEG_COLD_DATA; 936 } else { 937 if (IS_DNODE(page) && !is_cold_node(page)) 938 return CURSEG_HOT_NODE; 939 else 940 return CURSEG_COLD_NODE; 941 } 942} 943 944static int __get_segment_type_6(struct page *page, enum page_type p_type) 945{ 946 if (p_type == DATA) { 947 struct inode *inode = page->mapping->host; 948 949 if (S_ISDIR(inode->i_mode)) 950 return CURSEG_HOT_DATA; 951 else if (is_cold_data(page) || file_is_cold(inode)) 952 return CURSEG_COLD_DATA; 953 else 954 return CURSEG_WARM_DATA; 955 } else { 956 if (IS_DNODE(page)) 957 return is_cold_node(page) ? CURSEG_WARM_NODE : 958 CURSEG_HOT_NODE; 959 else 960 return CURSEG_COLD_NODE; 961 } 962} 963 964static int __get_segment_type(struct page *page, enum page_type p_type) 965{ 966 switch (F2FS_P_SB(page)->active_logs) { 967 case 2: 968 return __get_segment_type_2(page, p_type); 969 case 4: 970 return __get_segment_type_4(page, p_type); 971 } 972 /* NR_CURSEG_TYPE(6) logs by default */ 973 f2fs_bug_on(F2FS_P_SB(page), 974 F2FS_P_SB(page)->active_logs != NR_CURSEG_TYPE); 975 return __get_segment_type_6(page, p_type); 976} 977 978void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page, 979 block_t old_blkaddr, block_t *new_blkaddr, 980 struct f2fs_summary *sum, int type) 981{ 982 struct sit_info *sit_i = SIT_I(sbi); 983 struct curseg_info *curseg; 984 985 curseg = CURSEG_I(sbi, type); 986 987 mutex_lock(&curseg->curseg_mutex); 988 989 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); 990 991 /* 992 * __add_sum_entry should be resided under the curseg_mutex 993 * because, this function updates a summary entry in the 994 * current summary block. 995 */ 996 __add_sum_entry(sbi, type, sum); 997 998 mutex_lock(&sit_i->sentry_lock); 999 __refresh_next_blkoff(sbi, curseg); 1000 1001 stat_inc_block_count(sbi, curseg); 1002 1003 if (!__has_curseg_space(sbi, type)) 1004 sit_i->s_ops->allocate_segment(sbi, type, false); 1005 /* 1006 * SIT information should be updated before segment allocation, 1007 * since SSR needs latest valid block information. 1008 */ 1009 refresh_sit_entry(sbi, old_blkaddr, *new_blkaddr); 1010 1011 mutex_unlock(&sit_i->sentry_lock); 1012 1013 if (page && IS_NODESEG(type)) 1014 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg)); 1015 1016 mutex_unlock(&curseg->curseg_mutex); 1017} 1018 1019static void do_write_page(struct f2fs_sb_info *sbi, struct page *page, 1020 block_t old_blkaddr, block_t *new_blkaddr, 1021 struct f2fs_summary *sum, struct f2fs_io_info *fio) 1022{ 1023 int type = __get_segment_type(page, fio->type); 1024 1025 allocate_data_block(sbi, page, old_blkaddr, new_blkaddr, sum, type); 1026 1027 /* writeout dirty page into bdev */ 1028 f2fs_submit_page_mbio(sbi, page, *new_blkaddr, fio); 1029} 1030 1031void write_meta_page(struct f2fs_sb_info *sbi, struct page *page) 1032{ 1033 struct f2fs_io_info fio = { 1034 .type = META, 1035 .rw = WRITE_SYNC | REQ_META | REQ_PRIO 1036 }; 1037 1038 set_page_writeback(page); 1039 f2fs_submit_page_mbio(sbi, page, page->index, &fio); 1040} 1041 1042void write_node_page(struct f2fs_sb_info *sbi, struct page *page, 1043 struct f2fs_io_info *fio, 1044 unsigned int nid, block_t old_blkaddr, block_t *new_blkaddr) 1045{ 1046 struct f2fs_summary sum; 1047 set_summary(&sum, nid, 0, 0); 1048 do_write_page(sbi, page, old_blkaddr, new_blkaddr, &sum, fio); 1049} 1050 1051void write_data_page(struct page *page, struct dnode_of_data *dn, 1052 block_t *new_blkaddr, struct f2fs_io_info *fio) 1053{ 1054 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 1055 struct f2fs_summary sum; 1056 struct node_info ni; 1057 1058 f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR); 1059 get_node_info(sbi, dn->nid, &ni); 1060 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); 1061 1062 do_write_page(sbi, page, dn->data_blkaddr, new_blkaddr, &sum, fio); 1063} 1064 1065void rewrite_data_page(struct page *page, block_t old_blkaddr, 1066 struct f2fs_io_info *fio) 1067{ 1068 f2fs_submit_page_mbio(F2FS_P_SB(page), page, old_blkaddr, fio); 1069} 1070 1071void recover_data_page(struct f2fs_sb_info *sbi, 1072 struct page *page, struct f2fs_summary *sum, 1073 block_t old_blkaddr, block_t new_blkaddr) 1074{ 1075 struct sit_info *sit_i = SIT_I(sbi); 1076 struct curseg_info *curseg; 1077 unsigned int segno, old_cursegno; 1078 struct seg_entry *se; 1079 int type; 1080 1081 segno = GET_SEGNO(sbi, new_blkaddr); 1082 se = get_seg_entry(sbi, segno); 1083 type = se->type; 1084 1085 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) { 1086 if (old_blkaddr == NULL_ADDR) 1087 type = CURSEG_COLD_DATA; 1088 else 1089 type = CURSEG_WARM_DATA; 1090 } 1091 curseg = CURSEG_I(sbi, type); 1092 1093 mutex_lock(&curseg->curseg_mutex); 1094 mutex_lock(&sit_i->sentry_lock); 1095 1096 old_cursegno = curseg->segno; 1097 1098 /* change the current segment */ 1099 if (segno != curseg->segno) { 1100 curseg->next_segno = segno; 1101 change_curseg(sbi, type, true); 1102 } 1103 1104 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr); 1105 __add_sum_entry(sbi, type, sum); 1106 1107 refresh_sit_entry(sbi, old_blkaddr, new_blkaddr); 1108 locate_dirty_segment(sbi, old_cursegno); 1109 1110 mutex_unlock(&sit_i->sentry_lock); 1111 mutex_unlock(&curseg->curseg_mutex); 1112} 1113 1114static inline bool is_merged_page(struct f2fs_sb_info *sbi, 1115 struct page *page, enum page_type type) 1116{ 1117 enum page_type btype = PAGE_TYPE_OF_BIO(type); 1118 struct f2fs_bio_info *io = &sbi->write_io[btype]; 1119 struct bio_vec *bvec; 1120 int i; 1121 1122 down_read(&io->io_rwsem); 1123 if (!io->bio) 1124 goto out; 1125 1126 bio_for_each_segment_all(bvec, io->bio, i) { 1127 if (page == bvec->bv_page) { 1128 up_read(&io->io_rwsem); 1129 return true; 1130 } 1131 } 1132 1133out: 1134 up_read(&io->io_rwsem); 1135 return false; 1136} 1137 1138void f2fs_wait_on_page_writeback(struct page *page, 1139 enum page_type type) 1140{ 1141 if (PageWriteback(page)) { 1142 struct f2fs_sb_info *sbi = F2FS_P_SB(page); 1143 1144 if (is_merged_page(sbi, page, type)) 1145 f2fs_submit_merged_bio(sbi, type, WRITE); 1146 wait_on_page_writeback(page); 1147 } 1148} 1149 1150static int read_compacted_summaries(struct f2fs_sb_info *sbi) 1151{ 1152 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 1153 struct curseg_info *seg_i; 1154 unsigned char *kaddr; 1155 struct page *page; 1156 block_t start; 1157 int i, j, offset; 1158 1159 start = start_sum_block(sbi); 1160 1161 page = get_meta_page(sbi, start++); 1162 kaddr = (unsigned char *)page_address(page); 1163 1164 /* Step 1: restore nat cache */ 1165 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); 1166 memcpy(&seg_i->sum_blk->n_nats, kaddr, SUM_JOURNAL_SIZE); 1167 1168 /* Step 2: restore sit cache */ 1169 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); 1170 memcpy(&seg_i->sum_blk->n_sits, kaddr + SUM_JOURNAL_SIZE, 1171 SUM_JOURNAL_SIZE); 1172 offset = 2 * SUM_JOURNAL_SIZE; 1173 1174 /* Step 3: restore summary entries */ 1175 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 1176 unsigned short blk_off; 1177 unsigned int segno; 1178 1179 seg_i = CURSEG_I(sbi, i); 1180 segno = le32_to_cpu(ckpt->cur_data_segno[i]); 1181 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]); 1182 seg_i->next_segno = segno; 1183 reset_curseg(sbi, i, 0); 1184 seg_i->alloc_type = ckpt->alloc_type[i]; 1185 seg_i->next_blkoff = blk_off; 1186 1187 if (seg_i->alloc_type == SSR) 1188 blk_off = sbi->blocks_per_seg; 1189 1190 for (j = 0; j < blk_off; j++) { 1191 struct f2fs_summary *s; 1192 s = (struct f2fs_summary *)(kaddr + offset); 1193 seg_i->sum_blk->entries[j] = *s; 1194 offset += SUMMARY_SIZE; 1195 if (offset + SUMMARY_SIZE <= PAGE_CACHE_SIZE - 1196 SUM_FOOTER_SIZE) 1197 continue; 1198 1199 f2fs_put_page(page, 1); 1200 page = NULL; 1201 1202 page = get_meta_page(sbi, start++); 1203 kaddr = (unsigned char *)page_address(page); 1204 offset = 0; 1205 } 1206 } 1207 f2fs_put_page(page, 1); 1208 return 0; 1209} 1210 1211static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) 1212{ 1213 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 1214 struct f2fs_summary_block *sum; 1215 struct curseg_info *curseg; 1216 struct page *new; 1217 unsigned short blk_off; 1218 unsigned int segno = 0; 1219 block_t blk_addr = 0; 1220 1221 /* get segment number and block addr */ 1222 if (IS_DATASEG(type)) { 1223 segno = le32_to_cpu(ckpt->cur_data_segno[type]); 1224 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type - 1225 CURSEG_HOT_DATA]); 1226 if (is_set_ckpt_flags(ckpt, CP_UMOUNT_FLAG)) 1227 blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type); 1228 else 1229 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type); 1230 } else { 1231 segno = le32_to_cpu(ckpt->cur_node_segno[type - 1232 CURSEG_HOT_NODE]); 1233 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type - 1234 CURSEG_HOT_NODE]); 1235 if (is_set_ckpt_flags(ckpt, CP_UMOUNT_FLAG)) 1236 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE, 1237 type - CURSEG_HOT_NODE); 1238 else 1239 blk_addr = GET_SUM_BLOCK(sbi, segno); 1240 } 1241 1242 new = get_meta_page(sbi, blk_addr); 1243 sum = (struct f2fs_summary_block *)page_address(new); 1244 1245 if (IS_NODESEG(type)) { 1246 if (is_set_ckpt_flags(ckpt, CP_UMOUNT_FLAG)) { 1247 struct f2fs_summary *ns = &sum->entries[0]; 1248 int i; 1249 for (i = 0; i < sbi->blocks_per_seg; i++, ns++) { 1250 ns->version = 0; 1251 ns->ofs_in_node = 0; 1252 } 1253 } else { 1254 int err; 1255 1256 err = restore_node_summary(sbi, segno, sum); 1257 if (err) { 1258 f2fs_put_page(new, 1); 1259 return err; 1260 } 1261 } 1262 } 1263 1264 /* set uncompleted segment to curseg */ 1265 curseg = CURSEG_I(sbi, type); 1266 mutex_lock(&curseg->curseg_mutex); 1267 memcpy(curseg->sum_blk, sum, PAGE_CACHE_SIZE); 1268 curseg->next_segno = segno; 1269 reset_curseg(sbi, type, 0); 1270 curseg->alloc_type = ckpt->alloc_type[type]; 1271 curseg->next_blkoff = blk_off; 1272 mutex_unlock(&curseg->curseg_mutex); 1273 f2fs_put_page(new, 1); 1274 return 0; 1275} 1276 1277static int restore_curseg_summaries(struct f2fs_sb_info *sbi) 1278{ 1279 int type = CURSEG_HOT_DATA; 1280 int err; 1281 1282 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG)) { 1283 /* restore for compacted data summary */ 1284 if (read_compacted_summaries(sbi)) 1285 return -EINVAL; 1286 type = CURSEG_HOT_NODE; 1287 } 1288 1289 for (; type <= CURSEG_COLD_NODE; type++) { 1290 err = read_normal_summaries(sbi, type); 1291 if (err) 1292 return err; 1293 } 1294 1295 return 0; 1296} 1297 1298static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) 1299{ 1300 struct page *page; 1301 unsigned char *kaddr; 1302 struct f2fs_summary *summary; 1303 struct curseg_info *seg_i; 1304 int written_size = 0; 1305 int i, j; 1306 1307 page = grab_meta_page(sbi, blkaddr++); 1308 kaddr = (unsigned char *)page_address(page); 1309 1310 /* Step 1: write nat cache */ 1311 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); 1312 memcpy(kaddr, &seg_i->sum_blk->n_nats, SUM_JOURNAL_SIZE); 1313 written_size += SUM_JOURNAL_SIZE; 1314 1315 /* Step 2: write sit cache */ 1316 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); 1317 memcpy(kaddr + written_size, &seg_i->sum_blk->n_sits, 1318 SUM_JOURNAL_SIZE); 1319 written_size += SUM_JOURNAL_SIZE; 1320 1321 /* Step 3: write summary entries */ 1322 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 1323 unsigned short blkoff; 1324 seg_i = CURSEG_I(sbi, i); 1325 if (sbi->ckpt->alloc_type[i] == SSR) 1326 blkoff = sbi->blocks_per_seg; 1327 else 1328 blkoff = curseg_blkoff(sbi, i); 1329 1330 for (j = 0; j < blkoff; j++) { 1331 if (!page) { 1332 page = grab_meta_page(sbi, blkaddr++); 1333 kaddr = (unsigned char *)page_address(page); 1334 written_size = 0; 1335 } 1336 summary = (struct f2fs_summary *)(kaddr + written_size); 1337 *summary = seg_i->sum_blk->entries[j]; 1338 written_size += SUMMARY_SIZE; 1339 1340 if (written_size + SUMMARY_SIZE <= PAGE_CACHE_SIZE - 1341 SUM_FOOTER_SIZE) 1342 continue; 1343 1344 set_page_dirty(page); 1345 f2fs_put_page(page, 1); 1346 page = NULL; 1347 } 1348 } 1349 if (page) { 1350 set_page_dirty(page); 1351 f2fs_put_page(page, 1); 1352 } 1353} 1354 1355static void write_normal_summaries(struct f2fs_sb_info *sbi, 1356 block_t blkaddr, int type) 1357{ 1358 int i, end; 1359 if (IS_DATASEG(type)) 1360 end = type + NR_CURSEG_DATA_TYPE; 1361 else 1362 end = type + NR_CURSEG_NODE_TYPE; 1363 1364 for (i = type; i < end; i++) { 1365 struct curseg_info *sum = CURSEG_I(sbi, i); 1366 mutex_lock(&sum->curseg_mutex); 1367 write_sum_page(sbi, sum->sum_blk, blkaddr + (i - type)); 1368 mutex_unlock(&sum->curseg_mutex); 1369 } 1370} 1371 1372void write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk) 1373{ 1374 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG)) 1375 write_compacted_summaries(sbi, start_blk); 1376 else 1377 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA); 1378} 1379 1380void write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk) 1381{ 1382 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_UMOUNT_FLAG)) 1383 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE); 1384} 1385 1386int lookup_journal_in_cursum(struct f2fs_summary_block *sum, int type, 1387 unsigned int val, int alloc) 1388{ 1389 int i; 1390 1391 if (type == NAT_JOURNAL) { 1392 for (i = 0; i < nats_in_cursum(sum); i++) { 1393 if (le32_to_cpu(nid_in_journal(sum, i)) == val) 1394 return i; 1395 } 1396 if (alloc && nats_in_cursum(sum) < NAT_JOURNAL_ENTRIES) 1397 return update_nats_in_cursum(sum, 1); 1398 } else if (type == SIT_JOURNAL) { 1399 for (i = 0; i < sits_in_cursum(sum); i++) 1400 if (le32_to_cpu(segno_in_journal(sum, i)) == val) 1401 return i; 1402 if (alloc && sits_in_cursum(sum) < SIT_JOURNAL_ENTRIES) 1403 return update_sits_in_cursum(sum, 1); 1404 } 1405 return -1; 1406} 1407 1408static struct page *get_current_sit_page(struct f2fs_sb_info *sbi, 1409 unsigned int segno) 1410{ 1411 struct sit_info *sit_i = SIT_I(sbi); 1412 unsigned int offset = SIT_BLOCK_OFFSET(segno); 1413 block_t blk_addr = sit_i->sit_base_addr + offset; 1414 1415 check_seg_range(sbi, segno); 1416 1417 /* calculate sit block address */ 1418 if (f2fs_test_bit(offset, sit_i->sit_bitmap)) 1419 blk_addr += sit_i->sit_blocks; 1420 1421 return get_meta_page(sbi, blk_addr); 1422} 1423 1424static struct page *get_next_sit_page(struct f2fs_sb_info *sbi, 1425 unsigned int start) 1426{ 1427 struct sit_info *sit_i = SIT_I(sbi); 1428 struct page *src_page, *dst_page; 1429 pgoff_t src_off, dst_off; 1430 void *src_addr, *dst_addr; 1431 1432 src_off = current_sit_addr(sbi, start); 1433 dst_off = next_sit_addr(sbi, src_off); 1434 1435 /* get current sit block page without lock */ 1436 src_page = get_meta_page(sbi, src_off); 1437 dst_page = grab_meta_page(sbi, dst_off); 1438 f2fs_bug_on(sbi, PageDirty(src_page)); 1439 1440 src_addr = page_address(src_page); 1441 dst_addr = page_address(dst_page); 1442 memcpy(dst_addr, src_addr, PAGE_CACHE_SIZE); 1443 1444 set_page_dirty(dst_page); 1445 f2fs_put_page(src_page, 1); 1446 1447 set_to_next_sit(sit_i, start); 1448 1449 return dst_page; 1450} 1451 1452static struct sit_entry_set *grab_sit_entry_set(void) 1453{ 1454 struct sit_entry_set *ses = 1455 f2fs_kmem_cache_alloc(sit_entry_set_slab, GFP_ATOMIC); 1456 1457 ses->entry_cnt = 0; 1458 INIT_LIST_HEAD(&ses->set_list); 1459 return ses; 1460} 1461 1462static void release_sit_entry_set(struct sit_entry_set *ses) 1463{ 1464 list_del(&ses->set_list); 1465 kmem_cache_free(sit_entry_set_slab, ses); 1466} 1467 1468static void adjust_sit_entry_set(struct sit_entry_set *ses, 1469 struct list_head *head) 1470{ 1471 struct sit_entry_set *next = ses; 1472 1473 if (list_is_last(&ses->set_list, head)) 1474 return; 1475 1476 list_for_each_entry_continue(next, head, set_list) 1477 if (ses->entry_cnt <= next->entry_cnt) 1478 break; 1479 1480 list_move_tail(&ses->set_list, &next->set_list); 1481} 1482 1483static void add_sit_entry(unsigned int segno, struct list_head *head) 1484{ 1485 struct sit_entry_set *ses; 1486 unsigned int start_segno = START_SEGNO(segno); 1487 1488 list_for_each_entry(ses, head, set_list) { 1489 if (ses->start_segno == start_segno) { 1490 ses->entry_cnt++; 1491 adjust_sit_entry_set(ses, head); 1492 return; 1493 } 1494 } 1495 1496 ses = grab_sit_entry_set(); 1497 1498 ses->start_segno = start_segno; 1499 ses->entry_cnt++; 1500 list_add(&ses->set_list, head); 1501} 1502 1503static void add_sits_in_set(struct f2fs_sb_info *sbi) 1504{ 1505 struct f2fs_sm_info *sm_info = SM_I(sbi); 1506 struct list_head *set_list = &sm_info->sit_entry_set; 1507 unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap; 1508 unsigned long nsegs = TOTAL_SEGS(sbi); 1509 unsigned int segno; 1510 1511 for_each_set_bit(segno, bitmap, nsegs) 1512 add_sit_entry(segno, set_list); 1513} 1514 1515static void remove_sits_in_journal(struct f2fs_sb_info *sbi) 1516{ 1517 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 1518 struct f2fs_summary_block *sum = curseg->sum_blk; 1519 int i; 1520 1521 for (i = sits_in_cursum(sum) - 1; i >= 0; i--) { 1522 unsigned int segno; 1523 bool dirtied; 1524 1525 segno = le32_to_cpu(segno_in_journal(sum, i)); 1526 dirtied = __mark_sit_entry_dirty(sbi, segno); 1527 1528 if (!dirtied) 1529 add_sit_entry(segno, &SM_I(sbi)->sit_entry_set); 1530 } 1531 update_sits_in_cursum(sum, -sits_in_cursum(sum)); 1532} 1533 1534/* 1535 * CP calls this function, which flushes SIT entries including sit_journal, 1536 * and moves prefree segs to free segs. 1537 */ 1538void flush_sit_entries(struct f2fs_sb_info *sbi) 1539{ 1540 struct sit_info *sit_i = SIT_I(sbi); 1541 unsigned long *bitmap = sit_i->dirty_sentries_bitmap; 1542 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 1543 struct f2fs_summary_block *sum = curseg->sum_blk; 1544 struct sit_entry_set *ses, *tmp; 1545 struct list_head *head = &SM_I(sbi)->sit_entry_set; 1546 unsigned long nsegs = TOTAL_SEGS(sbi); 1547 bool to_journal = true; 1548 1549 mutex_lock(&curseg->curseg_mutex); 1550 mutex_lock(&sit_i->sentry_lock); 1551 1552 /* 1553 * add and account sit entries of dirty bitmap in sit entry 1554 * set temporarily 1555 */ 1556 add_sits_in_set(sbi); 1557 1558 /* 1559 * if there are no enough space in journal to store dirty sit 1560 * entries, remove all entries from journal and add and account 1561 * them in sit entry set. 1562 */ 1563 if (!__has_cursum_space(sum, sit_i->dirty_sentries, SIT_JOURNAL)) 1564 remove_sits_in_journal(sbi); 1565 1566 if (!sit_i->dirty_sentries) 1567 goto out; 1568 1569 /* 1570 * there are two steps to flush sit entries: 1571 * #1, flush sit entries to journal in current cold data summary block. 1572 * #2, flush sit entries to sit page. 1573 */ 1574 list_for_each_entry_safe(ses, tmp, head, set_list) { 1575 struct page *page; 1576 struct f2fs_sit_block *raw_sit = NULL; 1577 unsigned int start_segno = ses->start_segno; 1578 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK, 1579 nsegs); 1580 unsigned int segno = start_segno; 1581 1582 if (to_journal && 1583 !__has_cursum_space(sum, ses->entry_cnt, SIT_JOURNAL)) 1584 to_journal = false; 1585 1586 if (!to_journal) { 1587 page = get_next_sit_page(sbi, start_segno); 1588 raw_sit = page_address(page); 1589 } 1590 1591 /* flush dirty sit entries in region of current sit set */ 1592 for_each_set_bit_from(segno, bitmap, end) { 1593 int offset, sit_offset; 1594 struct seg_entry *se = get_seg_entry(sbi, segno); 1595 1596 /* add discard candidates */ 1597 if (SM_I(sbi)->nr_discards < SM_I(sbi)->max_discards) 1598 add_discard_addrs(sbi, segno, se); 1599 1600 if (to_journal) { 1601 offset = lookup_journal_in_cursum(sum, 1602 SIT_JOURNAL, segno, 1); 1603 f2fs_bug_on(sbi, offset < 0); 1604 segno_in_journal(sum, offset) = 1605 cpu_to_le32(segno); 1606 seg_info_to_raw_sit(se, 1607 &sit_in_journal(sum, offset)); 1608 } else { 1609 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno); 1610 seg_info_to_raw_sit(se, 1611 &raw_sit->entries[sit_offset]); 1612 } 1613 1614 __clear_bit(segno, bitmap); 1615 sit_i->dirty_sentries--; 1616 ses->entry_cnt--; 1617 } 1618 1619 if (!to_journal) 1620 f2fs_put_page(page, 1); 1621 1622 f2fs_bug_on(sbi, ses->entry_cnt); 1623 release_sit_entry_set(ses); 1624 } 1625 1626 f2fs_bug_on(sbi, !list_empty(head)); 1627 f2fs_bug_on(sbi, sit_i->dirty_sentries); 1628 1629out: 1630 mutex_unlock(&sit_i->sentry_lock); 1631 mutex_unlock(&curseg->curseg_mutex); 1632 1633 set_prefree_as_free_segments(sbi); 1634} 1635 1636static int build_sit_info(struct f2fs_sb_info *sbi) 1637{ 1638 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); 1639 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 1640 struct sit_info *sit_i; 1641 unsigned int sit_segs, start; 1642 char *src_bitmap, *dst_bitmap; 1643 unsigned int bitmap_size; 1644 1645 /* allocate memory for SIT information */ 1646 sit_i = kzalloc(sizeof(struct sit_info), GFP_KERNEL); 1647 if (!sit_i) 1648 return -ENOMEM; 1649 1650 SM_I(sbi)->sit_info = sit_i; 1651 1652 sit_i->sentries = vzalloc(TOTAL_SEGS(sbi) * sizeof(struct seg_entry)); 1653 if (!sit_i->sentries) 1654 return -ENOMEM; 1655 1656 bitmap_size = f2fs_bitmap_size(TOTAL_SEGS(sbi)); 1657 sit_i->dirty_sentries_bitmap = kzalloc(bitmap_size, GFP_KERNEL); 1658 if (!sit_i->dirty_sentries_bitmap) 1659 return -ENOMEM; 1660 1661 for (start = 0; start < TOTAL_SEGS(sbi); start++) { 1662 sit_i->sentries[start].cur_valid_map 1663 = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); 1664 sit_i->sentries[start].ckpt_valid_map 1665 = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); 1666 if (!sit_i->sentries[start].cur_valid_map 1667 || !sit_i->sentries[start].ckpt_valid_map) 1668 return -ENOMEM; 1669 } 1670 1671 if (sbi->segs_per_sec > 1) { 1672 sit_i->sec_entries = vzalloc(TOTAL_SECS(sbi) * 1673 sizeof(struct sec_entry)); 1674 if (!sit_i->sec_entries) 1675 return -ENOMEM; 1676 } 1677 1678 /* get information related with SIT */ 1679 sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1; 1680 1681 /* setup SIT bitmap from ckeckpoint pack */ 1682 bitmap_size = __bitmap_size(sbi, SIT_BITMAP); 1683 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP); 1684 1685 dst_bitmap = kmemdup(src_bitmap, bitmap_size, GFP_KERNEL); 1686 if (!dst_bitmap) 1687 return -ENOMEM; 1688 1689 /* init SIT information */ 1690 sit_i->s_ops = &default_salloc_ops; 1691 1692 sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr); 1693 sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg; 1694 sit_i->written_valid_blocks = le64_to_cpu(ckpt->valid_block_count); 1695 sit_i->sit_bitmap = dst_bitmap; 1696 sit_i->bitmap_size = bitmap_size; 1697 sit_i->dirty_sentries = 0; 1698 sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK; 1699 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time); 1700 sit_i->mounted_time = CURRENT_TIME_SEC.tv_sec; 1701 mutex_init(&sit_i->sentry_lock); 1702 return 0; 1703} 1704 1705static int build_free_segmap(struct f2fs_sb_info *sbi) 1706{ 1707 struct f2fs_sm_info *sm_info = SM_I(sbi); 1708 struct free_segmap_info *free_i; 1709 unsigned int bitmap_size, sec_bitmap_size; 1710 1711 /* allocate memory for free segmap information */ 1712 free_i = kzalloc(sizeof(struct free_segmap_info), GFP_KERNEL); 1713 if (!free_i) 1714 return -ENOMEM; 1715 1716 SM_I(sbi)->free_info = free_i; 1717 1718 bitmap_size = f2fs_bitmap_size(TOTAL_SEGS(sbi)); 1719 free_i->free_segmap = kmalloc(bitmap_size, GFP_KERNEL); 1720 if (!free_i->free_segmap) 1721 return -ENOMEM; 1722 1723 sec_bitmap_size = f2fs_bitmap_size(TOTAL_SECS(sbi)); 1724 free_i->free_secmap = kmalloc(sec_bitmap_size, GFP_KERNEL); 1725 if (!free_i->free_secmap) 1726 return -ENOMEM; 1727 1728 /* set all segments as dirty temporarily */ 1729 memset(free_i->free_segmap, 0xff, bitmap_size); 1730 memset(free_i->free_secmap, 0xff, sec_bitmap_size); 1731 1732 /* init free segmap information */ 1733 free_i->start_segno = 1734 (unsigned int) GET_SEGNO_FROM_SEG0(sbi, sm_info->main_blkaddr); 1735 free_i->free_segments = 0; 1736 free_i->free_sections = 0; 1737 rwlock_init(&free_i->segmap_lock); 1738 return 0; 1739} 1740 1741static int build_curseg(struct f2fs_sb_info *sbi) 1742{ 1743 struct curseg_info *array; 1744 int i; 1745 1746 array = kcalloc(NR_CURSEG_TYPE, sizeof(*array), GFP_KERNEL); 1747 if (!array) 1748 return -ENOMEM; 1749 1750 SM_I(sbi)->curseg_array = array; 1751 1752 for (i = 0; i < NR_CURSEG_TYPE; i++) { 1753 mutex_init(&array[i].curseg_mutex); 1754 array[i].sum_blk = kzalloc(PAGE_CACHE_SIZE, GFP_KERNEL); 1755 if (!array[i].sum_blk) 1756 return -ENOMEM; 1757 array[i].segno = NULL_SEGNO; 1758 array[i].next_blkoff = 0; 1759 } 1760 return restore_curseg_summaries(sbi); 1761} 1762 1763static void build_sit_entries(struct f2fs_sb_info *sbi) 1764{ 1765 struct sit_info *sit_i = SIT_I(sbi); 1766 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 1767 struct f2fs_summary_block *sum = curseg->sum_blk; 1768 int sit_blk_cnt = SIT_BLK_CNT(sbi); 1769 unsigned int i, start, end; 1770 unsigned int readed, start_blk = 0; 1771 int nrpages = MAX_BIO_BLOCKS(max_hw_blocks(sbi)); 1772 1773 do { 1774 readed = ra_meta_pages(sbi, start_blk, nrpages, META_SIT); 1775 1776 start = start_blk * sit_i->sents_per_block; 1777 end = (start_blk + readed) * sit_i->sents_per_block; 1778 1779 for (; start < end && start < TOTAL_SEGS(sbi); start++) { 1780 struct seg_entry *se = &sit_i->sentries[start]; 1781 struct f2fs_sit_block *sit_blk; 1782 struct f2fs_sit_entry sit; 1783 struct page *page; 1784 1785 mutex_lock(&curseg->curseg_mutex); 1786 for (i = 0; i < sits_in_cursum(sum); i++) { 1787 if (le32_to_cpu(segno_in_journal(sum, i)) 1788 == start) { 1789 sit = sit_in_journal(sum, i); 1790 mutex_unlock(&curseg->curseg_mutex); 1791 goto got_it; 1792 } 1793 } 1794 mutex_unlock(&curseg->curseg_mutex); 1795 1796 page = get_current_sit_page(sbi, start); 1797 sit_blk = (struct f2fs_sit_block *)page_address(page); 1798 sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)]; 1799 f2fs_put_page(page, 1); 1800got_it: 1801 check_block_count(sbi, start, &sit); 1802 seg_info_from_raw_sit(se, &sit); 1803 if (sbi->segs_per_sec > 1) { 1804 struct sec_entry *e = get_sec_entry(sbi, start); 1805 e->valid_blocks += se->valid_blocks; 1806 } 1807 } 1808 start_blk += readed; 1809 } while (start_blk < sit_blk_cnt); 1810} 1811 1812static void init_free_segmap(struct f2fs_sb_info *sbi) 1813{ 1814 unsigned int start; 1815 int type; 1816 1817 for (start = 0; start < TOTAL_SEGS(sbi); start++) { 1818 struct seg_entry *sentry = get_seg_entry(sbi, start); 1819 if (!sentry->valid_blocks) 1820 __set_free(sbi, start); 1821 } 1822 1823 /* set use the current segments */ 1824 for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) { 1825 struct curseg_info *curseg_t = CURSEG_I(sbi, type); 1826 __set_test_and_inuse(sbi, curseg_t->segno); 1827 } 1828} 1829 1830static void init_dirty_segmap(struct f2fs_sb_info *sbi) 1831{ 1832 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1833 struct free_segmap_info *free_i = FREE_I(sbi); 1834 unsigned int segno = 0, offset = 0, total_segs = TOTAL_SEGS(sbi); 1835 unsigned short valid_blocks; 1836 1837 while (1) { 1838 /* find dirty segment based on free segmap */ 1839 segno = find_next_inuse(free_i, total_segs, offset); 1840 if (segno >= total_segs) 1841 break; 1842 offset = segno + 1; 1843 valid_blocks = get_valid_blocks(sbi, segno, 0); 1844 if (valid_blocks == sbi->blocks_per_seg || !valid_blocks) 1845 continue; 1846 if (valid_blocks > sbi->blocks_per_seg) { 1847 f2fs_bug_on(sbi, 1); 1848 continue; 1849 } 1850 mutex_lock(&dirty_i->seglist_lock); 1851 __locate_dirty_segment(sbi, segno, DIRTY); 1852 mutex_unlock(&dirty_i->seglist_lock); 1853 } 1854} 1855 1856static int init_victim_secmap(struct f2fs_sb_info *sbi) 1857{ 1858 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1859 unsigned int bitmap_size = f2fs_bitmap_size(TOTAL_SECS(sbi)); 1860 1861 dirty_i->victim_secmap = kzalloc(bitmap_size, GFP_KERNEL); 1862 if (!dirty_i->victim_secmap) 1863 return -ENOMEM; 1864 return 0; 1865} 1866 1867static int build_dirty_segmap(struct f2fs_sb_info *sbi) 1868{ 1869 struct dirty_seglist_info *dirty_i; 1870 unsigned int bitmap_size, i; 1871 1872 /* allocate memory for dirty segments list information */ 1873 dirty_i = kzalloc(sizeof(struct dirty_seglist_info), GFP_KERNEL); 1874 if (!dirty_i) 1875 return -ENOMEM; 1876 1877 SM_I(sbi)->dirty_info = dirty_i; 1878 mutex_init(&dirty_i->seglist_lock); 1879 1880 bitmap_size = f2fs_bitmap_size(TOTAL_SEGS(sbi)); 1881 1882 for (i = 0; i < NR_DIRTY_TYPE; i++) { 1883 dirty_i->dirty_segmap[i] = kzalloc(bitmap_size, GFP_KERNEL); 1884 if (!dirty_i->dirty_segmap[i]) 1885 return -ENOMEM; 1886 } 1887 1888 init_dirty_segmap(sbi); 1889 return init_victim_secmap(sbi); 1890} 1891 1892/* 1893 * Update min, max modified time for cost-benefit GC algorithm 1894 */ 1895static void init_min_max_mtime(struct f2fs_sb_info *sbi) 1896{ 1897 struct sit_info *sit_i = SIT_I(sbi); 1898 unsigned int segno; 1899 1900 mutex_lock(&sit_i->sentry_lock); 1901 1902 sit_i->min_mtime = LLONG_MAX; 1903 1904 for (segno = 0; segno < TOTAL_SEGS(sbi); segno += sbi->segs_per_sec) { 1905 unsigned int i; 1906 unsigned long long mtime = 0; 1907 1908 for (i = 0; i < sbi->segs_per_sec; i++) 1909 mtime += get_seg_entry(sbi, segno + i)->mtime; 1910 1911 mtime = div_u64(mtime, sbi->segs_per_sec); 1912 1913 if (sit_i->min_mtime > mtime) 1914 sit_i->min_mtime = mtime; 1915 } 1916 sit_i->max_mtime = get_mtime(sbi); 1917 mutex_unlock(&sit_i->sentry_lock); 1918} 1919 1920int build_segment_manager(struct f2fs_sb_info *sbi) 1921{ 1922 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); 1923 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 1924 struct f2fs_sm_info *sm_info; 1925 int err; 1926 1927 sm_info = kzalloc(sizeof(struct f2fs_sm_info), GFP_KERNEL); 1928 if (!sm_info) 1929 return -ENOMEM; 1930 1931 /* init sm info */ 1932 sbi->sm_info = sm_info; 1933 sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr); 1934 sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr); 1935 sm_info->segment_count = le32_to_cpu(raw_super->segment_count); 1936 sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count); 1937 sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count); 1938 sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main); 1939 sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr); 1940 sm_info->rec_prefree_segments = sm_info->main_segments * 1941 DEF_RECLAIM_PREFREE_SEGMENTS / 100; 1942 sm_info->ipu_policy = F2FS_IPU_DISABLE; 1943 sm_info->min_ipu_util = DEF_MIN_IPU_UTIL; 1944 1945 INIT_LIST_HEAD(&sm_info->discard_list); 1946 sm_info->nr_discards = 0; 1947 sm_info->max_discards = 0; 1948 1949 INIT_LIST_HEAD(&sm_info->sit_entry_set); 1950 1951 if (test_opt(sbi, FLUSH_MERGE) && !f2fs_readonly(sbi->sb)) { 1952 err = create_flush_cmd_control(sbi); 1953 if (err) 1954 return err; 1955 } 1956 1957 err = build_sit_info(sbi); 1958 if (err) 1959 return err; 1960 err = build_free_segmap(sbi); 1961 if (err) 1962 return err; 1963 err = build_curseg(sbi); 1964 if (err) 1965 return err; 1966 1967 /* reinit free segmap based on SIT */ 1968 build_sit_entries(sbi); 1969 1970 init_free_segmap(sbi); 1971 err = build_dirty_segmap(sbi); 1972 if (err) 1973 return err; 1974 1975 init_min_max_mtime(sbi); 1976 return 0; 1977} 1978 1979static void discard_dirty_segmap(struct f2fs_sb_info *sbi, 1980 enum dirty_type dirty_type) 1981{ 1982 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1983 1984 mutex_lock(&dirty_i->seglist_lock); 1985 kfree(dirty_i->dirty_segmap[dirty_type]); 1986 dirty_i->nr_dirty[dirty_type] = 0; 1987 mutex_unlock(&dirty_i->seglist_lock); 1988} 1989 1990static void destroy_victim_secmap(struct f2fs_sb_info *sbi) 1991{ 1992 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1993 kfree(dirty_i->victim_secmap); 1994} 1995 1996static void destroy_dirty_segmap(struct f2fs_sb_info *sbi) 1997{ 1998 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1999 int i; 2000 2001 if (!dirty_i) 2002 return; 2003 2004 /* discard pre-free/dirty segments list */ 2005 for (i = 0; i < NR_DIRTY_TYPE; i++) 2006 discard_dirty_segmap(sbi, i); 2007 2008 destroy_victim_secmap(sbi); 2009 SM_I(sbi)->dirty_info = NULL; 2010 kfree(dirty_i); 2011} 2012 2013static void destroy_curseg(struct f2fs_sb_info *sbi) 2014{ 2015 struct curseg_info *array = SM_I(sbi)->curseg_array; 2016 int i; 2017 2018 if (!array) 2019 return; 2020 SM_I(sbi)->curseg_array = NULL; 2021 for (i = 0; i < NR_CURSEG_TYPE; i++) 2022 kfree(array[i].sum_blk); 2023 kfree(array); 2024} 2025 2026static void destroy_free_segmap(struct f2fs_sb_info *sbi) 2027{ 2028 struct free_segmap_info *free_i = SM_I(sbi)->free_info; 2029 if (!free_i) 2030 return; 2031 SM_I(sbi)->free_info = NULL; 2032 kfree(free_i->free_segmap); 2033 kfree(free_i->free_secmap); 2034 kfree(free_i); 2035} 2036 2037static void destroy_sit_info(struct f2fs_sb_info *sbi) 2038{ 2039 struct sit_info *sit_i = SIT_I(sbi); 2040 unsigned int start; 2041 2042 if (!sit_i) 2043 return; 2044 2045 if (sit_i->sentries) { 2046 for (start = 0; start < TOTAL_SEGS(sbi); start++) { 2047 kfree(sit_i->sentries[start].cur_valid_map); 2048 kfree(sit_i->sentries[start].ckpt_valid_map); 2049 } 2050 } 2051 vfree(sit_i->sentries); 2052 vfree(sit_i->sec_entries); 2053 kfree(sit_i->dirty_sentries_bitmap); 2054 2055 SM_I(sbi)->sit_info = NULL; 2056 kfree(sit_i->sit_bitmap); 2057 kfree(sit_i); 2058} 2059 2060void destroy_segment_manager(struct f2fs_sb_info *sbi) 2061{ 2062 struct f2fs_sm_info *sm_info = SM_I(sbi); 2063 2064 if (!sm_info) 2065 return; 2066 destroy_flush_cmd_control(sbi); 2067 destroy_dirty_segmap(sbi); 2068 destroy_curseg(sbi); 2069 destroy_free_segmap(sbi); 2070 destroy_sit_info(sbi); 2071 sbi->sm_info = NULL; 2072 kfree(sm_info); 2073} 2074 2075int __init create_segment_manager_caches(void) 2076{ 2077 discard_entry_slab = f2fs_kmem_cache_create("discard_entry", 2078 sizeof(struct discard_entry)); 2079 if (!discard_entry_slab) 2080 goto fail; 2081 2082 sit_entry_set_slab = f2fs_kmem_cache_create("sit_entry_set", 2083 sizeof(struct nat_entry_set)); 2084 if (!sit_entry_set_slab) 2085 goto destory_discard_entry; 2086 return 0; 2087 2088destory_discard_entry: 2089 kmem_cache_destroy(discard_entry_slab); 2090fail: 2091 return -ENOMEM; 2092} 2093 2094void destroy_segment_manager_caches(void) 2095{ 2096 kmem_cache_destroy(sit_entry_set_slab); 2097 kmem_cache_destroy(discard_entry_slab); 2098} 2099