segment.c revision 4f024f3797c43cb4b73cd2c50cec728842d0e49e
1/* 2 * fs/f2fs/segment.c 3 * 4 * Copyright (c) 2012 Samsung Electronics Co., Ltd. 5 * http://www.samsung.com/ 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11#include <linux/fs.h> 12#include <linux/f2fs_fs.h> 13#include <linux/bio.h> 14#include <linux/blkdev.h> 15#include <linux/prefetch.h> 16#include <linux/vmalloc.h> 17 18#include "f2fs.h" 19#include "segment.h" 20#include "node.h" 21#include <trace/events/f2fs.h> 22 23/* 24 * This function balances dirty node and dentry pages. 25 * In addition, it controls garbage collection. 26 */ 27void f2fs_balance_fs(struct f2fs_sb_info *sbi) 28{ 29 /* 30 * We should do GC or end up with checkpoint, if there are so many dirty 31 * dir/node pages without enough free segments. 32 */ 33 if (has_not_enough_free_secs(sbi, 0)) { 34 mutex_lock(&sbi->gc_mutex); 35 f2fs_gc(sbi); 36 } 37} 38 39void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi) 40{ 41 /* check the # of cached NAT entries and prefree segments */ 42 if (try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK) || 43 excess_prefree_segs(sbi)) 44 f2fs_sync_fs(sbi->sb, true); 45} 46 47static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, 48 enum dirty_type dirty_type) 49{ 50 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 51 52 /* need not be added */ 53 if (IS_CURSEG(sbi, segno)) 54 return; 55 56 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type])) 57 dirty_i->nr_dirty[dirty_type]++; 58 59 if (dirty_type == DIRTY) { 60 struct seg_entry *sentry = get_seg_entry(sbi, segno); 61 enum dirty_type t = sentry->type; 62 63 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t])) 64 dirty_i->nr_dirty[t]++; 65 } 66} 67 68static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, 69 enum dirty_type dirty_type) 70{ 71 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 72 73 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type])) 74 dirty_i->nr_dirty[dirty_type]--; 75 76 if (dirty_type == DIRTY) { 77 struct seg_entry *sentry = get_seg_entry(sbi, segno); 78 enum dirty_type t = sentry->type; 79 80 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t])) 81 dirty_i->nr_dirty[t]--; 82 83 if (get_valid_blocks(sbi, segno, sbi->segs_per_sec) == 0) 84 clear_bit(GET_SECNO(sbi, segno), 85 dirty_i->victim_secmap); 86 } 87} 88 89/* 90 * Should not occur error such as -ENOMEM. 91 * Adding dirty entry into seglist is not critical operation. 92 * If a given segment is one of current working segments, it won't be added. 93 */ 94static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno) 95{ 96 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 97 unsigned short valid_blocks; 98 99 if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno)) 100 return; 101 102 mutex_lock(&dirty_i->seglist_lock); 103 104 valid_blocks = get_valid_blocks(sbi, segno, 0); 105 106 if (valid_blocks == 0) { 107 __locate_dirty_segment(sbi, segno, PRE); 108 __remove_dirty_segment(sbi, segno, DIRTY); 109 } else if (valid_blocks < sbi->blocks_per_seg) { 110 __locate_dirty_segment(sbi, segno, DIRTY); 111 } else { 112 /* Recovery routine with SSR needs this */ 113 __remove_dirty_segment(sbi, segno, DIRTY); 114 } 115 116 mutex_unlock(&dirty_i->seglist_lock); 117} 118 119/* 120 * Should call clear_prefree_segments after checkpoint is done. 121 */ 122static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi) 123{ 124 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 125 unsigned int segno = -1; 126 unsigned int total_segs = TOTAL_SEGS(sbi); 127 128 mutex_lock(&dirty_i->seglist_lock); 129 while (1) { 130 segno = find_next_bit(dirty_i->dirty_segmap[PRE], total_segs, 131 segno + 1); 132 if (segno >= total_segs) 133 break; 134 __set_test_and_free(sbi, segno); 135 } 136 mutex_unlock(&dirty_i->seglist_lock); 137} 138 139void clear_prefree_segments(struct f2fs_sb_info *sbi) 140{ 141 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 142 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE]; 143 unsigned int total_segs = TOTAL_SEGS(sbi); 144 unsigned int start = 0, end = -1; 145 146 mutex_lock(&dirty_i->seglist_lock); 147 148 while (1) { 149 int i; 150 start = find_next_bit(prefree_map, total_segs, end + 1); 151 if (start >= total_segs) 152 break; 153 end = find_next_zero_bit(prefree_map, total_segs, start + 1); 154 155 for (i = start; i < end; i++) 156 clear_bit(i, prefree_map); 157 158 dirty_i->nr_dirty[PRE] -= end - start; 159 160 if (!test_opt(sbi, DISCARD)) 161 continue; 162 163 blkdev_issue_discard(sbi->sb->s_bdev, 164 START_BLOCK(sbi, start) << 165 sbi->log_sectors_per_block, 166 (1 << (sbi->log_sectors_per_block + 167 sbi->log_blocks_per_seg)) * (end - start), 168 GFP_NOFS, 0); 169 } 170 mutex_unlock(&dirty_i->seglist_lock); 171} 172 173static void __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno) 174{ 175 struct sit_info *sit_i = SIT_I(sbi); 176 if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) 177 sit_i->dirty_sentries++; 178} 179 180static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type, 181 unsigned int segno, int modified) 182{ 183 struct seg_entry *se = get_seg_entry(sbi, segno); 184 se->type = type; 185 if (modified) 186 __mark_sit_entry_dirty(sbi, segno); 187} 188 189static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del) 190{ 191 struct seg_entry *se; 192 unsigned int segno, offset; 193 long int new_vblocks; 194 195 segno = GET_SEGNO(sbi, blkaddr); 196 197 se = get_seg_entry(sbi, segno); 198 new_vblocks = se->valid_blocks + del; 199 offset = GET_SEGOFF_FROM_SEG0(sbi, blkaddr) & (sbi->blocks_per_seg - 1); 200 201 f2fs_bug_on((new_vblocks >> (sizeof(unsigned short) << 3) || 202 (new_vblocks > sbi->blocks_per_seg))); 203 204 se->valid_blocks = new_vblocks; 205 se->mtime = get_mtime(sbi); 206 SIT_I(sbi)->max_mtime = se->mtime; 207 208 /* Update valid block bitmap */ 209 if (del > 0) { 210 if (f2fs_set_bit(offset, se->cur_valid_map)) 211 BUG(); 212 } else { 213 if (!f2fs_clear_bit(offset, se->cur_valid_map)) 214 BUG(); 215 } 216 if (!f2fs_test_bit(offset, se->ckpt_valid_map)) 217 se->ckpt_valid_blocks += del; 218 219 __mark_sit_entry_dirty(sbi, segno); 220 221 /* update total number of valid blocks to be written in ckpt area */ 222 SIT_I(sbi)->written_valid_blocks += del; 223 224 if (sbi->segs_per_sec > 1) 225 get_sec_entry(sbi, segno)->valid_blocks += del; 226} 227 228static void refresh_sit_entry(struct f2fs_sb_info *sbi, 229 block_t old_blkaddr, block_t new_blkaddr) 230{ 231 update_sit_entry(sbi, new_blkaddr, 1); 232 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) 233 update_sit_entry(sbi, old_blkaddr, -1); 234} 235 236void invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr) 237{ 238 unsigned int segno = GET_SEGNO(sbi, addr); 239 struct sit_info *sit_i = SIT_I(sbi); 240 241 f2fs_bug_on(addr == NULL_ADDR); 242 if (addr == NEW_ADDR) 243 return; 244 245 /* add it into sit main buffer */ 246 mutex_lock(&sit_i->sentry_lock); 247 248 update_sit_entry(sbi, addr, -1); 249 250 /* add it into dirty seglist */ 251 locate_dirty_segment(sbi, segno); 252 253 mutex_unlock(&sit_i->sentry_lock); 254} 255 256/* 257 * This function should be resided under the curseg_mutex lock 258 */ 259static void __add_sum_entry(struct f2fs_sb_info *sbi, int type, 260 struct f2fs_summary *sum) 261{ 262 struct curseg_info *curseg = CURSEG_I(sbi, type); 263 void *addr = curseg->sum_blk; 264 addr += curseg->next_blkoff * sizeof(struct f2fs_summary); 265 memcpy(addr, sum, sizeof(struct f2fs_summary)); 266} 267 268/* 269 * Calculate the number of current summary pages for writing 270 */ 271int npages_for_summary_flush(struct f2fs_sb_info *sbi) 272{ 273 int valid_sum_count = 0; 274 int i, sum_in_page; 275 276 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 277 if (sbi->ckpt->alloc_type[i] == SSR) 278 valid_sum_count += sbi->blocks_per_seg; 279 else 280 valid_sum_count += curseg_blkoff(sbi, i); 281 } 282 283 sum_in_page = (PAGE_CACHE_SIZE - 2 * SUM_JOURNAL_SIZE - 284 SUM_FOOTER_SIZE) / SUMMARY_SIZE; 285 if (valid_sum_count <= sum_in_page) 286 return 1; 287 else if ((valid_sum_count - sum_in_page) <= 288 (PAGE_CACHE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE) 289 return 2; 290 return 3; 291} 292 293/* 294 * Caller should put this summary page 295 */ 296struct page *get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno) 297{ 298 return get_meta_page(sbi, GET_SUM_BLOCK(sbi, segno)); 299} 300 301static void write_sum_page(struct f2fs_sb_info *sbi, 302 struct f2fs_summary_block *sum_blk, block_t blk_addr) 303{ 304 struct page *page = grab_meta_page(sbi, blk_addr); 305 void *kaddr = page_address(page); 306 memcpy(kaddr, sum_blk, PAGE_CACHE_SIZE); 307 set_page_dirty(page); 308 f2fs_put_page(page, 1); 309} 310 311static int is_next_segment_free(struct f2fs_sb_info *sbi, int type) 312{ 313 struct curseg_info *curseg = CURSEG_I(sbi, type); 314 unsigned int segno = curseg->segno + 1; 315 struct free_segmap_info *free_i = FREE_I(sbi); 316 317 if (segno < TOTAL_SEGS(sbi) && segno % sbi->segs_per_sec) 318 return !test_bit(segno, free_i->free_segmap); 319 return 0; 320} 321 322/* 323 * Find a new segment from the free segments bitmap to right order 324 * This function should be returned with success, otherwise BUG 325 */ 326static void get_new_segment(struct f2fs_sb_info *sbi, 327 unsigned int *newseg, bool new_sec, int dir) 328{ 329 struct free_segmap_info *free_i = FREE_I(sbi); 330 unsigned int segno, secno, zoneno; 331 unsigned int total_zones = TOTAL_SECS(sbi) / sbi->secs_per_zone; 332 unsigned int hint = *newseg / sbi->segs_per_sec; 333 unsigned int old_zoneno = GET_ZONENO_FROM_SEGNO(sbi, *newseg); 334 unsigned int left_start = hint; 335 bool init = true; 336 int go_left = 0; 337 int i; 338 339 write_lock(&free_i->segmap_lock); 340 341 if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) { 342 segno = find_next_zero_bit(free_i->free_segmap, 343 TOTAL_SEGS(sbi), *newseg + 1); 344 if (segno - *newseg < sbi->segs_per_sec - 345 (*newseg % sbi->segs_per_sec)) 346 goto got_it; 347 } 348find_other_zone: 349 secno = find_next_zero_bit(free_i->free_secmap, TOTAL_SECS(sbi), hint); 350 if (secno >= TOTAL_SECS(sbi)) { 351 if (dir == ALLOC_RIGHT) { 352 secno = find_next_zero_bit(free_i->free_secmap, 353 TOTAL_SECS(sbi), 0); 354 f2fs_bug_on(secno >= TOTAL_SECS(sbi)); 355 } else { 356 go_left = 1; 357 left_start = hint - 1; 358 } 359 } 360 if (go_left == 0) 361 goto skip_left; 362 363 while (test_bit(left_start, free_i->free_secmap)) { 364 if (left_start > 0) { 365 left_start--; 366 continue; 367 } 368 left_start = find_next_zero_bit(free_i->free_secmap, 369 TOTAL_SECS(sbi), 0); 370 f2fs_bug_on(left_start >= TOTAL_SECS(sbi)); 371 break; 372 } 373 secno = left_start; 374skip_left: 375 hint = secno; 376 segno = secno * sbi->segs_per_sec; 377 zoneno = secno / sbi->secs_per_zone; 378 379 /* give up on finding another zone */ 380 if (!init) 381 goto got_it; 382 if (sbi->secs_per_zone == 1) 383 goto got_it; 384 if (zoneno == old_zoneno) 385 goto got_it; 386 if (dir == ALLOC_LEFT) { 387 if (!go_left && zoneno + 1 >= total_zones) 388 goto got_it; 389 if (go_left && zoneno == 0) 390 goto got_it; 391 } 392 for (i = 0; i < NR_CURSEG_TYPE; i++) 393 if (CURSEG_I(sbi, i)->zone == zoneno) 394 break; 395 396 if (i < NR_CURSEG_TYPE) { 397 /* zone is in user, try another */ 398 if (go_left) 399 hint = zoneno * sbi->secs_per_zone - 1; 400 else if (zoneno + 1 >= total_zones) 401 hint = 0; 402 else 403 hint = (zoneno + 1) * sbi->secs_per_zone; 404 init = false; 405 goto find_other_zone; 406 } 407got_it: 408 /* set it as dirty segment in free segmap */ 409 f2fs_bug_on(test_bit(segno, free_i->free_segmap)); 410 __set_inuse(sbi, segno); 411 *newseg = segno; 412 write_unlock(&free_i->segmap_lock); 413} 414 415static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified) 416{ 417 struct curseg_info *curseg = CURSEG_I(sbi, type); 418 struct summary_footer *sum_footer; 419 420 curseg->segno = curseg->next_segno; 421 curseg->zone = GET_ZONENO_FROM_SEGNO(sbi, curseg->segno); 422 curseg->next_blkoff = 0; 423 curseg->next_segno = NULL_SEGNO; 424 425 sum_footer = &(curseg->sum_blk->footer); 426 memset(sum_footer, 0, sizeof(struct summary_footer)); 427 if (IS_DATASEG(type)) 428 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA); 429 if (IS_NODESEG(type)) 430 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE); 431 __set_sit_entry_type(sbi, type, curseg->segno, modified); 432} 433 434/* 435 * Allocate a current working segment. 436 * This function always allocates a free segment in LFS manner. 437 */ 438static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) 439{ 440 struct curseg_info *curseg = CURSEG_I(sbi, type); 441 unsigned int segno = curseg->segno; 442 int dir = ALLOC_LEFT; 443 444 write_sum_page(sbi, curseg->sum_blk, 445 GET_SUM_BLOCK(sbi, segno)); 446 if (type == CURSEG_WARM_DATA || type == CURSEG_COLD_DATA) 447 dir = ALLOC_RIGHT; 448 449 if (test_opt(sbi, NOHEAP)) 450 dir = ALLOC_RIGHT; 451 452 get_new_segment(sbi, &segno, new_sec, dir); 453 curseg->next_segno = segno; 454 reset_curseg(sbi, type, 1); 455 curseg->alloc_type = LFS; 456} 457 458static void __next_free_blkoff(struct f2fs_sb_info *sbi, 459 struct curseg_info *seg, block_t start) 460{ 461 struct seg_entry *se = get_seg_entry(sbi, seg->segno); 462 block_t ofs; 463 for (ofs = start; ofs < sbi->blocks_per_seg; ofs++) { 464 if (!f2fs_test_bit(ofs, se->ckpt_valid_map) 465 && !f2fs_test_bit(ofs, se->cur_valid_map)) 466 break; 467 } 468 seg->next_blkoff = ofs; 469} 470 471/* 472 * If a segment is written by LFS manner, next block offset is just obtained 473 * by increasing the current block offset. However, if a segment is written by 474 * SSR manner, next block offset obtained by calling __next_free_blkoff 475 */ 476static void __refresh_next_blkoff(struct f2fs_sb_info *sbi, 477 struct curseg_info *seg) 478{ 479 if (seg->alloc_type == SSR) 480 __next_free_blkoff(sbi, seg, seg->next_blkoff + 1); 481 else 482 seg->next_blkoff++; 483} 484 485/* 486 * This function always allocates a used segment (from dirty seglist) by SSR 487 * manner, so it should recover the existing segment information of valid blocks 488 */ 489static void change_curseg(struct f2fs_sb_info *sbi, int type, bool reuse) 490{ 491 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 492 struct curseg_info *curseg = CURSEG_I(sbi, type); 493 unsigned int new_segno = curseg->next_segno; 494 struct f2fs_summary_block *sum_node; 495 struct page *sum_page; 496 497 write_sum_page(sbi, curseg->sum_blk, 498 GET_SUM_BLOCK(sbi, curseg->segno)); 499 __set_test_and_inuse(sbi, new_segno); 500 501 mutex_lock(&dirty_i->seglist_lock); 502 __remove_dirty_segment(sbi, new_segno, PRE); 503 __remove_dirty_segment(sbi, new_segno, DIRTY); 504 mutex_unlock(&dirty_i->seglist_lock); 505 506 reset_curseg(sbi, type, 1); 507 curseg->alloc_type = SSR; 508 __next_free_blkoff(sbi, curseg, 0); 509 510 if (reuse) { 511 sum_page = get_sum_page(sbi, new_segno); 512 sum_node = (struct f2fs_summary_block *)page_address(sum_page); 513 memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE); 514 f2fs_put_page(sum_page, 1); 515 } 516} 517 518static int get_ssr_segment(struct f2fs_sb_info *sbi, int type) 519{ 520 struct curseg_info *curseg = CURSEG_I(sbi, type); 521 const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops; 522 523 if (IS_NODESEG(type) || !has_not_enough_free_secs(sbi, 0)) 524 return v_ops->get_victim(sbi, 525 &(curseg)->next_segno, BG_GC, type, SSR); 526 527 /* For data segments, let's do SSR more intensively */ 528 for (; type >= CURSEG_HOT_DATA; type--) 529 if (v_ops->get_victim(sbi, &(curseg)->next_segno, 530 BG_GC, type, SSR)) 531 return 1; 532 return 0; 533} 534 535/* 536 * flush out current segment and replace it with new segment 537 * This function should be returned with success, otherwise BUG 538 */ 539static void allocate_segment_by_default(struct f2fs_sb_info *sbi, 540 int type, bool force) 541{ 542 struct curseg_info *curseg = CURSEG_I(sbi, type); 543 544 if (force) 545 new_curseg(sbi, type, true); 546 else if (type == CURSEG_WARM_NODE) 547 new_curseg(sbi, type, false); 548 else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type)) 549 new_curseg(sbi, type, false); 550 else if (need_SSR(sbi) && get_ssr_segment(sbi, type)) 551 change_curseg(sbi, type, true); 552 else 553 new_curseg(sbi, type, false); 554 555 stat_inc_seg_type(sbi, curseg); 556} 557 558void allocate_new_segments(struct f2fs_sb_info *sbi) 559{ 560 struct curseg_info *curseg; 561 unsigned int old_curseg; 562 int i; 563 564 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 565 curseg = CURSEG_I(sbi, i); 566 old_curseg = curseg->segno; 567 SIT_I(sbi)->s_ops->allocate_segment(sbi, i, true); 568 locate_dirty_segment(sbi, old_curseg); 569 } 570} 571 572static const struct segment_allocation default_salloc_ops = { 573 .allocate_segment = allocate_segment_by_default, 574}; 575 576static void f2fs_end_io_write(struct bio *bio, int err) 577{ 578 struct bio_private *p = bio->bi_private; 579 struct bio_vec *bvec; 580 int i; 581 582 bio_for_each_segment_all(bvec, bio, i) { 583 struct page *page = bvec->bv_page; 584 585 if (err) { 586 SetPageError(page); 587 if (page->mapping) 588 set_bit(AS_EIO, &page->mapping->flags); 589 set_ckpt_flags(p->sbi->ckpt, CP_ERROR_FLAG); 590 p->sbi->sb->s_flags |= MS_RDONLY; 591 } 592 end_page_writeback(page); 593 dec_page_count(p->sbi, F2FS_WRITEBACK); 594 } 595 596 if (p->is_sync) 597 complete(p->wait); 598 599 if (!get_pages(p->sbi, F2FS_WRITEBACK) && 600 !list_empty(&p->sbi->cp_wait.task_list)) 601 wake_up(&p->sbi->cp_wait); 602 603 kfree(p); 604 bio_put(bio); 605} 606 607struct bio *f2fs_bio_alloc(struct block_device *bdev, int npages) 608{ 609 struct bio *bio; 610 611 /* No failure on bio allocation */ 612 bio = bio_alloc(GFP_NOIO, npages); 613 bio->bi_bdev = bdev; 614 bio->bi_private = NULL; 615 616 return bio; 617} 618 619static void do_submit_bio(struct f2fs_sb_info *sbi, 620 enum page_type type, bool sync) 621{ 622 int rw = sync ? WRITE_SYNC : WRITE; 623 enum page_type btype = type > META ? META : type; 624 625 if (type >= META_FLUSH) 626 rw = WRITE_FLUSH_FUA; 627 628 if (btype == META) 629 rw |= REQ_META; 630 631 if (sbi->bio[btype]) { 632 struct bio_private *p = sbi->bio[btype]->bi_private; 633 p->sbi = sbi; 634 sbi->bio[btype]->bi_end_io = f2fs_end_io_write; 635 636 trace_f2fs_do_submit_bio(sbi->sb, btype, sync, sbi->bio[btype]); 637 638 if (type == META_FLUSH) { 639 DECLARE_COMPLETION_ONSTACK(wait); 640 p->is_sync = true; 641 p->wait = &wait; 642 submit_bio(rw, sbi->bio[btype]); 643 wait_for_completion(&wait); 644 } else { 645 p->is_sync = false; 646 submit_bio(rw, sbi->bio[btype]); 647 } 648 sbi->bio[btype] = NULL; 649 } 650} 651 652void f2fs_submit_bio(struct f2fs_sb_info *sbi, enum page_type type, bool sync) 653{ 654 down_write(&sbi->bio_sem); 655 do_submit_bio(sbi, type, sync); 656 up_write(&sbi->bio_sem); 657} 658 659static void submit_write_page(struct f2fs_sb_info *sbi, struct page *page, 660 block_t blk_addr, enum page_type type) 661{ 662 struct block_device *bdev = sbi->sb->s_bdev; 663 int bio_blocks; 664 665 verify_block_addr(sbi, blk_addr); 666 667 down_write(&sbi->bio_sem); 668 669 inc_page_count(sbi, F2FS_WRITEBACK); 670 671 if (sbi->bio[type] && sbi->last_block_in_bio[type] != blk_addr - 1) 672 do_submit_bio(sbi, type, false); 673alloc_new: 674 if (sbi->bio[type] == NULL) { 675 struct bio_private *priv; 676retry: 677 priv = kmalloc(sizeof(struct bio_private), GFP_NOFS); 678 if (!priv) { 679 cond_resched(); 680 goto retry; 681 } 682 683 bio_blocks = MAX_BIO_BLOCKS(max_hw_blocks(sbi)); 684 sbi->bio[type] = f2fs_bio_alloc(bdev, bio_blocks); 685 sbi->bio[type]->bi_iter.bi_sector = SECTOR_FROM_BLOCK(sbi, blk_addr); 686 sbi->bio[type]->bi_private = priv; 687 /* 688 * The end_io will be assigned at the sumbission phase. 689 * Until then, let bio_add_page() merge consecutive IOs as much 690 * as possible. 691 */ 692 } 693 694 if (bio_add_page(sbi->bio[type], page, PAGE_CACHE_SIZE, 0) < 695 PAGE_CACHE_SIZE) { 696 do_submit_bio(sbi, type, false); 697 goto alloc_new; 698 } 699 700 sbi->last_block_in_bio[type] = blk_addr; 701 702 up_write(&sbi->bio_sem); 703 trace_f2fs_submit_write_page(page, blk_addr, type); 704} 705 706void f2fs_wait_on_page_writeback(struct page *page, 707 enum page_type type, bool sync) 708{ 709 struct f2fs_sb_info *sbi = F2FS_SB(page->mapping->host->i_sb); 710 if (PageWriteback(page)) { 711 f2fs_submit_bio(sbi, type, sync); 712 wait_on_page_writeback(page); 713 } 714} 715 716static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type) 717{ 718 struct curseg_info *curseg = CURSEG_I(sbi, type); 719 if (curseg->next_blkoff < sbi->blocks_per_seg) 720 return true; 721 return false; 722} 723 724static int __get_segment_type_2(struct page *page, enum page_type p_type) 725{ 726 if (p_type == DATA) 727 return CURSEG_HOT_DATA; 728 else 729 return CURSEG_HOT_NODE; 730} 731 732static int __get_segment_type_4(struct page *page, enum page_type p_type) 733{ 734 if (p_type == DATA) { 735 struct inode *inode = page->mapping->host; 736 737 if (S_ISDIR(inode->i_mode)) 738 return CURSEG_HOT_DATA; 739 else 740 return CURSEG_COLD_DATA; 741 } else { 742 if (IS_DNODE(page) && !is_cold_node(page)) 743 return CURSEG_HOT_NODE; 744 else 745 return CURSEG_COLD_NODE; 746 } 747} 748 749static int __get_segment_type_6(struct page *page, enum page_type p_type) 750{ 751 if (p_type == DATA) { 752 struct inode *inode = page->mapping->host; 753 754 if (S_ISDIR(inode->i_mode)) 755 return CURSEG_HOT_DATA; 756 else if (is_cold_data(page) || file_is_cold(inode)) 757 return CURSEG_COLD_DATA; 758 else 759 return CURSEG_WARM_DATA; 760 } else { 761 if (IS_DNODE(page)) 762 return is_cold_node(page) ? CURSEG_WARM_NODE : 763 CURSEG_HOT_NODE; 764 else 765 return CURSEG_COLD_NODE; 766 } 767} 768 769static int __get_segment_type(struct page *page, enum page_type p_type) 770{ 771 struct f2fs_sb_info *sbi = F2FS_SB(page->mapping->host->i_sb); 772 switch (sbi->active_logs) { 773 case 2: 774 return __get_segment_type_2(page, p_type); 775 case 4: 776 return __get_segment_type_4(page, p_type); 777 } 778 /* NR_CURSEG_TYPE(6) logs by default */ 779 f2fs_bug_on(sbi->active_logs != NR_CURSEG_TYPE); 780 return __get_segment_type_6(page, p_type); 781} 782 783static void do_write_page(struct f2fs_sb_info *sbi, struct page *page, 784 block_t old_blkaddr, block_t *new_blkaddr, 785 struct f2fs_summary *sum, enum page_type p_type) 786{ 787 struct sit_info *sit_i = SIT_I(sbi); 788 struct curseg_info *curseg; 789 unsigned int old_cursegno; 790 int type; 791 792 type = __get_segment_type(page, p_type); 793 curseg = CURSEG_I(sbi, type); 794 795 mutex_lock(&curseg->curseg_mutex); 796 797 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); 798 old_cursegno = curseg->segno; 799 800 /* 801 * __add_sum_entry should be resided under the curseg_mutex 802 * because, this function updates a summary entry in the 803 * current summary block. 804 */ 805 __add_sum_entry(sbi, type, sum); 806 807 mutex_lock(&sit_i->sentry_lock); 808 __refresh_next_blkoff(sbi, curseg); 809 810 stat_inc_block_count(sbi, curseg); 811 812 /* 813 * SIT information should be updated before segment allocation, 814 * since SSR needs latest valid block information. 815 */ 816 refresh_sit_entry(sbi, old_blkaddr, *new_blkaddr); 817 818 if (!__has_curseg_space(sbi, type)) 819 sit_i->s_ops->allocate_segment(sbi, type, false); 820 821 locate_dirty_segment(sbi, old_cursegno); 822 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); 823 mutex_unlock(&sit_i->sentry_lock); 824 825 if (p_type == NODE) 826 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg)); 827 828 /* writeout dirty page into bdev */ 829 submit_write_page(sbi, page, *new_blkaddr, p_type); 830 831 mutex_unlock(&curseg->curseg_mutex); 832} 833 834void write_meta_page(struct f2fs_sb_info *sbi, struct page *page) 835{ 836 set_page_writeback(page); 837 submit_write_page(sbi, page, page->index, META); 838} 839 840void write_node_page(struct f2fs_sb_info *sbi, struct page *page, 841 unsigned int nid, block_t old_blkaddr, block_t *new_blkaddr) 842{ 843 struct f2fs_summary sum; 844 set_summary(&sum, nid, 0, 0); 845 do_write_page(sbi, page, old_blkaddr, new_blkaddr, &sum, NODE); 846} 847 848void write_data_page(struct inode *inode, struct page *page, 849 struct dnode_of_data *dn, block_t old_blkaddr, 850 block_t *new_blkaddr) 851{ 852 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb); 853 struct f2fs_summary sum; 854 struct node_info ni; 855 856 f2fs_bug_on(old_blkaddr == NULL_ADDR); 857 get_node_info(sbi, dn->nid, &ni); 858 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); 859 860 do_write_page(sbi, page, old_blkaddr, 861 new_blkaddr, &sum, DATA); 862} 863 864void rewrite_data_page(struct f2fs_sb_info *sbi, struct page *page, 865 block_t old_blk_addr) 866{ 867 submit_write_page(sbi, page, old_blk_addr, DATA); 868} 869 870void recover_data_page(struct f2fs_sb_info *sbi, 871 struct page *page, struct f2fs_summary *sum, 872 block_t old_blkaddr, block_t new_blkaddr) 873{ 874 struct sit_info *sit_i = SIT_I(sbi); 875 struct curseg_info *curseg; 876 unsigned int segno, old_cursegno; 877 struct seg_entry *se; 878 int type; 879 880 segno = GET_SEGNO(sbi, new_blkaddr); 881 se = get_seg_entry(sbi, segno); 882 type = se->type; 883 884 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) { 885 if (old_blkaddr == NULL_ADDR) 886 type = CURSEG_COLD_DATA; 887 else 888 type = CURSEG_WARM_DATA; 889 } 890 curseg = CURSEG_I(sbi, type); 891 892 mutex_lock(&curseg->curseg_mutex); 893 mutex_lock(&sit_i->sentry_lock); 894 895 old_cursegno = curseg->segno; 896 897 /* change the current segment */ 898 if (segno != curseg->segno) { 899 curseg->next_segno = segno; 900 change_curseg(sbi, type, true); 901 } 902 903 curseg->next_blkoff = GET_SEGOFF_FROM_SEG0(sbi, new_blkaddr) & 904 (sbi->blocks_per_seg - 1); 905 __add_sum_entry(sbi, type, sum); 906 907 refresh_sit_entry(sbi, old_blkaddr, new_blkaddr); 908 909 locate_dirty_segment(sbi, old_cursegno); 910 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); 911 912 mutex_unlock(&sit_i->sentry_lock); 913 mutex_unlock(&curseg->curseg_mutex); 914} 915 916void rewrite_node_page(struct f2fs_sb_info *sbi, 917 struct page *page, struct f2fs_summary *sum, 918 block_t old_blkaddr, block_t new_blkaddr) 919{ 920 struct sit_info *sit_i = SIT_I(sbi); 921 int type = CURSEG_WARM_NODE; 922 struct curseg_info *curseg; 923 unsigned int segno, old_cursegno; 924 block_t next_blkaddr = next_blkaddr_of_node(page); 925 unsigned int next_segno = GET_SEGNO(sbi, next_blkaddr); 926 927 curseg = CURSEG_I(sbi, type); 928 929 mutex_lock(&curseg->curseg_mutex); 930 mutex_lock(&sit_i->sentry_lock); 931 932 segno = GET_SEGNO(sbi, new_blkaddr); 933 old_cursegno = curseg->segno; 934 935 /* change the current segment */ 936 if (segno != curseg->segno) { 937 curseg->next_segno = segno; 938 change_curseg(sbi, type, true); 939 } 940 curseg->next_blkoff = GET_SEGOFF_FROM_SEG0(sbi, new_blkaddr) & 941 (sbi->blocks_per_seg - 1); 942 __add_sum_entry(sbi, type, sum); 943 944 /* change the current log to the next block addr in advance */ 945 if (next_segno != segno) { 946 curseg->next_segno = next_segno; 947 change_curseg(sbi, type, true); 948 } 949 curseg->next_blkoff = GET_SEGOFF_FROM_SEG0(sbi, next_blkaddr) & 950 (sbi->blocks_per_seg - 1); 951 952 /* rewrite node page */ 953 set_page_writeback(page); 954 submit_write_page(sbi, page, new_blkaddr, NODE); 955 f2fs_submit_bio(sbi, NODE, true); 956 refresh_sit_entry(sbi, old_blkaddr, new_blkaddr); 957 958 locate_dirty_segment(sbi, old_cursegno); 959 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); 960 961 mutex_unlock(&sit_i->sentry_lock); 962 mutex_unlock(&curseg->curseg_mutex); 963} 964 965static int read_compacted_summaries(struct f2fs_sb_info *sbi) 966{ 967 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 968 struct curseg_info *seg_i; 969 unsigned char *kaddr; 970 struct page *page; 971 block_t start; 972 int i, j, offset; 973 974 start = start_sum_block(sbi); 975 976 page = get_meta_page(sbi, start++); 977 kaddr = (unsigned char *)page_address(page); 978 979 /* Step 1: restore nat cache */ 980 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); 981 memcpy(&seg_i->sum_blk->n_nats, kaddr, SUM_JOURNAL_SIZE); 982 983 /* Step 2: restore sit cache */ 984 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); 985 memcpy(&seg_i->sum_blk->n_sits, kaddr + SUM_JOURNAL_SIZE, 986 SUM_JOURNAL_SIZE); 987 offset = 2 * SUM_JOURNAL_SIZE; 988 989 /* Step 3: restore summary entries */ 990 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 991 unsigned short blk_off; 992 unsigned int segno; 993 994 seg_i = CURSEG_I(sbi, i); 995 segno = le32_to_cpu(ckpt->cur_data_segno[i]); 996 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]); 997 seg_i->next_segno = segno; 998 reset_curseg(sbi, i, 0); 999 seg_i->alloc_type = ckpt->alloc_type[i]; 1000 seg_i->next_blkoff = blk_off; 1001 1002 if (seg_i->alloc_type == SSR) 1003 blk_off = sbi->blocks_per_seg; 1004 1005 for (j = 0; j < blk_off; j++) { 1006 struct f2fs_summary *s; 1007 s = (struct f2fs_summary *)(kaddr + offset); 1008 seg_i->sum_blk->entries[j] = *s; 1009 offset += SUMMARY_SIZE; 1010 if (offset + SUMMARY_SIZE <= PAGE_CACHE_SIZE - 1011 SUM_FOOTER_SIZE) 1012 continue; 1013 1014 f2fs_put_page(page, 1); 1015 page = NULL; 1016 1017 page = get_meta_page(sbi, start++); 1018 kaddr = (unsigned char *)page_address(page); 1019 offset = 0; 1020 } 1021 } 1022 f2fs_put_page(page, 1); 1023 return 0; 1024} 1025 1026static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) 1027{ 1028 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 1029 struct f2fs_summary_block *sum; 1030 struct curseg_info *curseg; 1031 struct page *new; 1032 unsigned short blk_off; 1033 unsigned int segno = 0; 1034 block_t blk_addr = 0; 1035 1036 /* get segment number and block addr */ 1037 if (IS_DATASEG(type)) { 1038 segno = le32_to_cpu(ckpt->cur_data_segno[type]); 1039 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type - 1040 CURSEG_HOT_DATA]); 1041 if (is_set_ckpt_flags(ckpt, CP_UMOUNT_FLAG)) 1042 blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type); 1043 else 1044 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type); 1045 } else { 1046 segno = le32_to_cpu(ckpt->cur_node_segno[type - 1047 CURSEG_HOT_NODE]); 1048 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type - 1049 CURSEG_HOT_NODE]); 1050 if (is_set_ckpt_flags(ckpt, CP_UMOUNT_FLAG)) 1051 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE, 1052 type - CURSEG_HOT_NODE); 1053 else 1054 blk_addr = GET_SUM_BLOCK(sbi, segno); 1055 } 1056 1057 new = get_meta_page(sbi, blk_addr); 1058 sum = (struct f2fs_summary_block *)page_address(new); 1059 1060 if (IS_NODESEG(type)) { 1061 if (is_set_ckpt_flags(ckpt, CP_UMOUNT_FLAG)) { 1062 struct f2fs_summary *ns = &sum->entries[0]; 1063 int i; 1064 for (i = 0; i < sbi->blocks_per_seg; i++, ns++) { 1065 ns->version = 0; 1066 ns->ofs_in_node = 0; 1067 } 1068 } else { 1069 if (restore_node_summary(sbi, segno, sum)) { 1070 f2fs_put_page(new, 1); 1071 return -EINVAL; 1072 } 1073 } 1074 } 1075 1076 /* set uncompleted segment to curseg */ 1077 curseg = CURSEG_I(sbi, type); 1078 mutex_lock(&curseg->curseg_mutex); 1079 memcpy(curseg->sum_blk, sum, PAGE_CACHE_SIZE); 1080 curseg->next_segno = segno; 1081 reset_curseg(sbi, type, 0); 1082 curseg->alloc_type = ckpt->alloc_type[type]; 1083 curseg->next_blkoff = blk_off; 1084 mutex_unlock(&curseg->curseg_mutex); 1085 f2fs_put_page(new, 1); 1086 return 0; 1087} 1088 1089static int restore_curseg_summaries(struct f2fs_sb_info *sbi) 1090{ 1091 int type = CURSEG_HOT_DATA; 1092 1093 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG)) { 1094 /* restore for compacted data summary */ 1095 if (read_compacted_summaries(sbi)) 1096 return -EINVAL; 1097 type = CURSEG_HOT_NODE; 1098 } 1099 1100 for (; type <= CURSEG_COLD_NODE; type++) 1101 if (read_normal_summaries(sbi, type)) 1102 return -EINVAL; 1103 return 0; 1104} 1105 1106static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) 1107{ 1108 struct page *page; 1109 unsigned char *kaddr; 1110 struct f2fs_summary *summary; 1111 struct curseg_info *seg_i; 1112 int written_size = 0; 1113 int i, j; 1114 1115 page = grab_meta_page(sbi, blkaddr++); 1116 kaddr = (unsigned char *)page_address(page); 1117 1118 /* Step 1: write nat cache */ 1119 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); 1120 memcpy(kaddr, &seg_i->sum_blk->n_nats, SUM_JOURNAL_SIZE); 1121 written_size += SUM_JOURNAL_SIZE; 1122 1123 /* Step 2: write sit cache */ 1124 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); 1125 memcpy(kaddr + written_size, &seg_i->sum_blk->n_sits, 1126 SUM_JOURNAL_SIZE); 1127 written_size += SUM_JOURNAL_SIZE; 1128 1129 /* Step 3: write summary entries */ 1130 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { 1131 unsigned short blkoff; 1132 seg_i = CURSEG_I(sbi, i); 1133 if (sbi->ckpt->alloc_type[i] == SSR) 1134 blkoff = sbi->blocks_per_seg; 1135 else 1136 blkoff = curseg_blkoff(sbi, i); 1137 1138 for (j = 0; j < blkoff; j++) { 1139 if (!page) { 1140 page = grab_meta_page(sbi, blkaddr++); 1141 kaddr = (unsigned char *)page_address(page); 1142 written_size = 0; 1143 } 1144 summary = (struct f2fs_summary *)(kaddr + written_size); 1145 *summary = seg_i->sum_blk->entries[j]; 1146 written_size += SUMMARY_SIZE; 1147 1148 if (written_size + SUMMARY_SIZE <= PAGE_CACHE_SIZE - 1149 SUM_FOOTER_SIZE) 1150 continue; 1151 1152 set_page_dirty(page); 1153 f2fs_put_page(page, 1); 1154 page = NULL; 1155 } 1156 } 1157 if (page) { 1158 set_page_dirty(page); 1159 f2fs_put_page(page, 1); 1160 } 1161} 1162 1163static void write_normal_summaries(struct f2fs_sb_info *sbi, 1164 block_t blkaddr, int type) 1165{ 1166 int i, end; 1167 if (IS_DATASEG(type)) 1168 end = type + NR_CURSEG_DATA_TYPE; 1169 else 1170 end = type + NR_CURSEG_NODE_TYPE; 1171 1172 for (i = type; i < end; i++) { 1173 struct curseg_info *sum = CURSEG_I(sbi, i); 1174 mutex_lock(&sum->curseg_mutex); 1175 write_sum_page(sbi, sum->sum_blk, blkaddr + (i - type)); 1176 mutex_unlock(&sum->curseg_mutex); 1177 } 1178} 1179 1180void write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk) 1181{ 1182 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG)) 1183 write_compacted_summaries(sbi, start_blk); 1184 else 1185 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA); 1186} 1187 1188void write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk) 1189{ 1190 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_UMOUNT_FLAG)) 1191 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE); 1192} 1193 1194int lookup_journal_in_cursum(struct f2fs_summary_block *sum, int type, 1195 unsigned int val, int alloc) 1196{ 1197 int i; 1198 1199 if (type == NAT_JOURNAL) { 1200 for (i = 0; i < nats_in_cursum(sum); i++) { 1201 if (le32_to_cpu(nid_in_journal(sum, i)) == val) 1202 return i; 1203 } 1204 if (alloc && nats_in_cursum(sum) < NAT_JOURNAL_ENTRIES) 1205 return update_nats_in_cursum(sum, 1); 1206 } else if (type == SIT_JOURNAL) { 1207 for (i = 0; i < sits_in_cursum(sum); i++) 1208 if (le32_to_cpu(segno_in_journal(sum, i)) == val) 1209 return i; 1210 if (alloc && sits_in_cursum(sum) < SIT_JOURNAL_ENTRIES) 1211 return update_sits_in_cursum(sum, 1); 1212 } 1213 return -1; 1214} 1215 1216static struct page *get_current_sit_page(struct f2fs_sb_info *sbi, 1217 unsigned int segno) 1218{ 1219 struct sit_info *sit_i = SIT_I(sbi); 1220 unsigned int offset = SIT_BLOCK_OFFSET(sit_i, segno); 1221 block_t blk_addr = sit_i->sit_base_addr + offset; 1222 1223 check_seg_range(sbi, segno); 1224 1225 /* calculate sit block address */ 1226 if (f2fs_test_bit(offset, sit_i->sit_bitmap)) 1227 blk_addr += sit_i->sit_blocks; 1228 1229 return get_meta_page(sbi, blk_addr); 1230} 1231 1232static struct page *get_next_sit_page(struct f2fs_sb_info *sbi, 1233 unsigned int start) 1234{ 1235 struct sit_info *sit_i = SIT_I(sbi); 1236 struct page *src_page, *dst_page; 1237 pgoff_t src_off, dst_off; 1238 void *src_addr, *dst_addr; 1239 1240 src_off = current_sit_addr(sbi, start); 1241 dst_off = next_sit_addr(sbi, src_off); 1242 1243 /* get current sit block page without lock */ 1244 src_page = get_meta_page(sbi, src_off); 1245 dst_page = grab_meta_page(sbi, dst_off); 1246 f2fs_bug_on(PageDirty(src_page)); 1247 1248 src_addr = page_address(src_page); 1249 dst_addr = page_address(dst_page); 1250 memcpy(dst_addr, src_addr, PAGE_CACHE_SIZE); 1251 1252 set_page_dirty(dst_page); 1253 f2fs_put_page(src_page, 1); 1254 1255 set_to_next_sit(sit_i, start); 1256 1257 return dst_page; 1258} 1259 1260static bool flush_sits_in_journal(struct f2fs_sb_info *sbi) 1261{ 1262 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 1263 struct f2fs_summary_block *sum = curseg->sum_blk; 1264 int i; 1265 1266 /* 1267 * If the journal area in the current summary is full of sit entries, 1268 * all the sit entries will be flushed. Otherwise the sit entries 1269 * are not able to replace with newly hot sit entries. 1270 */ 1271 if (sits_in_cursum(sum) >= SIT_JOURNAL_ENTRIES) { 1272 for (i = sits_in_cursum(sum) - 1; i >= 0; i--) { 1273 unsigned int segno; 1274 segno = le32_to_cpu(segno_in_journal(sum, i)); 1275 __mark_sit_entry_dirty(sbi, segno); 1276 } 1277 update_sits_in_cursum(sum, -sits_in_cursum(sum)); 1278 return true; 1279 } 1280 return false; 1281} 1282 1283/* 1284 * CP calls this function, which flushes SIT entries including sit_journal, 1285 * and moves prefree segs to free segs. 1286 */ 1287void flush_sit_entries(struct f2fs_sb_info *sbi) 1288{ 1289 struct sit_info *sit_i = SIT_I(sbi); 1290 unsigned long *bitmap = sit_i->dirty_sentries_bitmap; 1291 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 1292 struct f2fs_summary_block *sum = curseg->sum_blk; 1293 unsigned long nsegs = TOTAL_SEGS(sbi); 1294 struct page *page = NULL; 1295 struct f2fs_sit_block *raw_sit = NULL; 1296 unsigned int start = 0, end = 0; 1297 unsigned int segno = -1; 1298 bool flushed; 1299 1300 mutex_lock(&curseg->curseg_mutex); 1301 mutex_lock(&sit_i->sentry_lock); 1302 1303 /* 1304 * "flushed" indicates whether sit entries in journal are flushed 1305 * to the SIT area or not. 1306 */ 1307 flushed = flush_sits_in_journal(sbi); 1308 1309 while ((segno = find_next_bit(bitmap, nsegs, segno + 1)) < nsegs) { 1310 struct seg_entry *se = get_seg_entry(sbi, segno); 1311 int sit_offset, offset; 1312 1313 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno); 1314 1315 if (flushed) 1316 goto to_sit_page; 1317 1318 offset = lookup_journal_in_cursum(sum, SIT_JOURNAL, segno, 1); 1319 if (offset >= 0) { 1320 segno_in_journal(sum, offset) = cpu_to_le32(segno); 1321 seg_info_to_raw_sit(se, &sit_in_journal(sum, offset)); 1322 goto flush_done; 1323 } 1324to_sit_page: 1325 if (!page || (start > segno) || (segno > end)) { 1326 if (page) { 1327 f2fs_put_page(page, 1); 1328 page = NULL; 1329 } 1330 1331 start = START_SEGNO(sit_i, segno); 1332 end = start + SIT_ENTRY_PER_BLOCK - 1; 1333 1334 /* read sit block that will be updated */ 1335 page = get_next_sit_page(sbi, start); 1336 raw_sit = page_address(page); 1337 } 1338 1339 /* udpate entry in SIT block */ 1340 seg_info_to_raw_sit(se, &raw_sit->entries[sit_offset]); 1341flush_done: 1342 __clear_bit(segno, bitmap); 1343 sit_i->dirty_sentries--; 1344 } 1345 mutex_unlock(&sit_i->sentry_lock); 1346 mutex_unlock(&curseg->curseg_mutex); 1347 1348 /* writeout last modified SIT block */ 1349 f2fs_put_page(page, 1); 1350 1351 set_prefree_as_free_segments(sbi); 1352} 1353 1354static int build_sit_info(struct f2fs_sb_info *sbi) 1355{ 1356 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); 1357 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 1358 struct sit_info *sit_i; 1359 unsigned int sit_segs, start; 1360 char *src_bitmap, *dst_bitmap; 1361 unsigned int bitmap_size; 1362 1363 /* allocate memory for SIT information */ 1364 sit_i = kzalloc(sizeof(struct sit_info), GFP_KERNEL); 1365 if (!sit_i) 1366 return -ENOMEM; 1367 1368 SM_I(sbi)->sit_info = sit_i; 1369 1370 sit_i->sentries = vzalloc(TOTAL_SEGS(sbi) * sizeof(struct seg_entry)); 1371 if (!sit_i->sentries) 1372 return -ENOMEM; 1373 1374 bitmap_size = f2fs_bitmap_size(TOTAL_SEGS(sbi)); 1375 sit_i->dirty_sentries_bitmap = kzalloc(bitmap_size, GFP_KERNEL); 1376 if (!sit_i->dirty_sentries_bitmap) 1377 return -ENOMEM; 1378 1379 for (start = 0; start < TOTAL_SEGS(sbi); start++) { 1380 sit_i->sentries[start].cur_valid_map 1381 = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); 1382 sit_i->sentries[start].ckpt_valid_map 1383 = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); 1384 if (!sit_i->sentries[start].cur_valid_map 1385 || !sit_i->sentries[start].ckpt_valid_map) 1386 return -ENOMEM; 1387 } 1388 1389 if (sbi->segs_per_sec > 1) { 1390 sit_i->sec_entries = vzalloc(TOTAL_SECS(sbi) * 1391 sizeof(struct sec_entry)); 1392 if (!sit_i->sec_entries) 1393 return -ENOMEM; 1394 } 1395 1396 /* get information related with SIT */ 1397 sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1; 1398 1399 /* setup SIT bitmap from ckeckpoint pack */ 1400 bitmap_size = __bitmap_size(sbi, SIT_BITMAP); 1401 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP); 1402 1403 dst_bitmap = kmemdup(src_bitmap, bitmap_size, GFP_KERNEL); 1404 if (!dst_bitmap) 1405 return -ENOMEM; 1406 1407 /* init SIT information */ 1408 sit_i->s_ops = &default_salloc_ops; 1409 1410 sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr); 1411 sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg; 1412 sit_i->written_valid_blocks = le64_to_cpu(ckpt->valid_block_count); 1413 sit_i->sit_bitmap = dst_bitmap; 1414 sit_i->bitmap_size = bitmap_size; 1415 sit_i->dirty_sentries = 0; 1416 sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK; 1417 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time); 1418 sit_i->mounted_time = CURRENT_TIME_SEC.tv_sec; 1419 mutex_init(&sit_i->sentry_lock); 1420 return 0; 1421} 1422 1423static int build_free_segmap(struct f2fs_sb_info *sbi) 1424{ 1425 struct f2fs_sm_info *sm_info = SM_I(sbi); 1426 struct free_segmap_info *free_i; 1427 unsigned int bitmap_size, sec_bitmap_size; 1428 1429 /* allocate memory for free segmap information */ 1430 free_i = kzalloc(sizeof(struct free_segmap_info), GFP_KERNEL); 1431 if (!free_i) 1432 return -ENOMEM; 1433 1434 SM_I(sbi)->free_info = free_i; 1435 1436 bitmap_size = f2fs_bitmap_size(TOTAL_SEGS(sbi)); 1437 free_i->free_segmap = kmalloc(bitmap_size, GFP_KERNEL); 1438 if (!free_i->free_segmap) 1439 return -ENOMEM; 1440 1441 sec_bitmap_size = f2fs_bitmap_size(TOTAL_SECS(sbi)); 1442 free_i->free_secmap = kmalloc(sec_bitmap_size, GFP_KERNEL); 1443 if (!free_i->free_secmap) 1444 return -ENOMEM; 1445 1446 /* set all segments as dirty temporarily */ 1447 memset(free_i->free_segmap, 0xff, bitmap_size); 1448 memset(free_i->free_secmap, 0xff, sec_bitmap_size); 1449 1450 /* init free segmap information */ 1451 free_i->start_segno = 1452 (unsigned int) GET_SEGNO_FROM_SEG0(sbi, sm_info->main_blkaddr); 1453 free_i->free_segments = 0; 1454 free_i->free_sections = 0; 1455 rwlock_init(&free_i->segmap_lock); 1456 return 0; 1457} 1458 1459static int build_curseg(struct f2fs_sb_info *sbi) 1460{ 1461 struct curseg_info *array; 1462 int i; 1463 1464 array = kzalloc(sizeof(*array) * NR_CURSEG_TYPE, GFP_KERNEL); 1465 if (!array) 1466 return -ENOMEM; 1467 1468 SM_I(sbi)->curseg_array = array; 1469 1470 for (i = 0; i < NR_CURSEG_TYPE; i++) { 1471 mutex_init(&array[i].curseg_mutex); 1472 array[i].sum_blk = kzalloc(PAGE_CACHE_SIZE, GFP_KERNEL); 1473 if (!array[i].sum_blk) 1474 return -ENOMEM; 1475 array[i].segno = NULL_SEGNO; 1476 array[i].next_blkoff = 0; 1477 } 1478 return restore_curseg_summaries(sbi); 1479} 1480 1481static void build_sit_entries(struct f2fs_sb_info *sbi) 1482{ 1483 struct sit_info *sit_i = SIT_I(sbi); 1484 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); 1485 struct f2fs_summary_block *sum = curseg->sum_blk; 1486 unsigned int start; 1487 1488 for (start = 0; start < TOTAL_SEGS(sbi); start++) { 1489 struct seg_entry *se = &sit_i->sentries[start]; 1490 struct f2fs_sit_block *sit_blk; 1491 struct f2fs_sit_entry sit; 1492 struct page *page; 1493 int i; 1494 1495 mutex_lock(&curseg->curseg_mutex); 1496 for (i = 0; i < sits_in_cursum(sum); i++) { 1497 if (le32_to_cpu(segno_in_journal(sum, i)) == start) { 1498 sit = sit_in_journal(sum, i); 1499 mutex_unlock(&curseg->curseg_mutex); 1500 goto got_it; 1501 } 1502 } 1503 mutex_unlock(&curseg->curseg_mutex); 1504 page = get_current_sit_page(sbi, start); 1505 sit_blk = (struct f2fs_sit_block *)page_address(page); 1506 sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)]; 1507 f2fs_put_page(page, 1); 1508got_it: 1509 check_block_count(sbi, start, &sit); 1510 seg_info_from_raw_sit(se, &sit); 1511 if (sbi->segs_per_sec > 1) { 1512 struct sec_entry *e = get_sec_entry(sbi, start); 1513 e->valid_blocks += se->valid_blocks; 1514 } 1515 } 1516} 1517 1518static void init_free_segmap(struct f2fs_sb_info *sbi) 1519{ 1520 unsigned int start; 1521 int type; 1522 1523 for (start = 0; start < TOTAL_SEGS(sbi); start++) { 1524 struct seg_entry *sentry = get_seg_entry(sbi, start); 1525 if (!sentry->valid_blocks) 1526 __set_free(sbi, start); 1527 } 1528 1529 /* set use the current segments */ 1530 for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) { 1531 struct curseg_info *curseg_t = CURSEG_I(sbi, type); 1532 __set_test_and_inuse(sbi, curseg_t->segno); 1533 } 1534} 1535 1536static void init_dirty_segmap(struct f2fs_sb_info *sbi) 1537{ 1538 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1539 struct free_segmap_info *free_i = FREE_I(sbi); 1540 unsigned int segno = 0, offset = 0, total_segs = TOTAL_SEGS(sbi); 1541 unsigned short valid_blocks; 1542 1543 while (1) { 1544 /* find dirty segment based on free segmap */ 1545 segno = find_next_inuse(free_i, total_segs, offset); 1546 if (segno >= total_segs) 1547 break; 1548 offset = segno + 1; 1549 valid_blocks = get_valid_blocks(sbi, segno, 0); 1550 if (valid_blocks >= sbi->blocks_per_seg || !valid_blocks) 1551 continue; 1552 mutex_lock(&dirty_i->seglist_lock); 1553 __locate_dirty_segment(sbi, segno, DIRTY); 1554 mutex_unlock(&dirty_i->seglist_lock); 1555 } 1556} 1557 1558static int init_victim_secmap(struct f2fs_sb_info *sbi) 1559{ 1560 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1561 unsigned int bitmap_size = f2fs_bitmap_size(TOTAL_SECS(sbi)); 1562 1563 dirty_i->victim_secmap = kzalloc(bitmap_size, GFP_KERNEL); 1564 if (!dirty_i->victim_secmap) 1565 return -ENOMEM; 1566 return 0; 1567} 1568 1569static int build_dirty_segmap(struct f2fs_sb_info *sbi) 1570{ 1571 struct dirty_seglist_info *dirty_i; 1572 unsigned int bitmap_size, i; 1573 1574 /* allocate memory for dirty segments list information */ 1575 dirty_i = kzalloc(sizeof(struct dirty_seglist_info), GFP_KERNEL); 1576 if (!dirty_i) 1577 return -ENOMEM; 1578 1579 SM_I(sbi)->dirty_info = dirty_i; 1580 mutex_init(&dirty_i->seglist_lock); 1581 1582 bitmap_size = f2fs_bitmap_size(TOTAL_SEGS(sbi)); 1583 1584 for (i = 0; i < NR_DIRTY_TYPE; i++) { 1585 dirty_i->dirty_segmap[i] = kzalloc(bitmap_size, GFP_KERNEL); 1586 if (!dirty_i->dirty_segmap[i]) 1587 return -ENOMEM; 1588 } 1589 1590 init_dirty_segmap(sbi); 1591 return init_victim_secmap(sbi); 1592} 1593 1594/* 1595 * Update min, max modified time for cost-benefit GC algorithm 1596 */ 1597static void init_min_max_mtime(struct f2fs_sb_info *sbi) 1598{ 1599 struct sit_info *sit_i = SIT_I(sbi); 1600 unsigned int segno; 1601 1602 mutex_lock(&sit_i->sentry_lock); 1603 1604 sit_i->min_mtime = LLONG_MAX; 1605 1606 for (segno = 0; segno < TOTAL_SEGS(sbi); segno += sbi->segs_per_sec) { 1607 unsigned int i; 1608 unsigned long long mtime = 0; 1609 1610 for (i = 0; i < sbi->segs_per_sec; i++) 1611 mtime += get_seg_entry(sbi, segno + i)->mtime; 1612 1613 mtime = div_u64(mtime, sbi->segs_per_sec); 1614 1615 if (sit_i->min_mtime > mtime) 1616 sit_i->min_mtime = mtime; 1617 } 1618 sit_i->max_mtime = get_mtime(sbi); 1619 mutex_unlock(&sit_i->sentry_lock); 1620} 1621 1622int build_segment_manager(struct f2fs_sb_info *sbi) 1623{ 1624 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); 1625 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 1626 struct f2fs_sm_info *sm_info; 1627 int err; 1628 1629 sm_info = kzalloc(sizeof(struct f2fs_sm_info), GFP_KERNEL); 1630 if (!sm_info) 1631 return -ENOMEM; 1632 1633 /* init sm info */ 1634 sbi->sm_info = sm_info; 1635 INIT_LIST_HEAD(&sm_info->wblist_head); 1636 spin_lock_init(&sm_info->wblist_lock); 1637 sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr); 1638 sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr); 1639 sm_info->segment_count = le32_to_cpu(raw_super->segment_count); 1640 sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count); 1641 sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count); 1642 sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main); 1643 sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr); 1644 sm_info->rec_prefree_segments = DEF_RECLAIM_PREFREE_SEGMENTS; 1645 1646 err = build_sit_info(sbi); 1647 if (err) 1648 return err; 1649 err = build_free_segmap(sbi); 1650 if (err) 1651 return err; 1652 err = build_curseg(sbi); 1653 if (err) 1654 return err; 1655 1656 /* reinit free segmap based on SIT */ 1657 build_sit_entries(sbi); 1658 1659 init_free_segmap(sbi); 1660 err = build_dirty_segmap(sbi); 1661 if (err) 1662 return err; 1663 1664 init_min_max_mtime(sbi); 1665 return 0; 1666} 1667 1668static void discard_dirty_segmap(struct f2fs_sb_info *sbi, 1669 enum dirty_type dirty_type) 1670{ 1671 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1672 1673 mutex_lock(&dirty_i->seglist_lock); 1674 kfree(dirty_i->dirty_segmap[dirty_type]); 1675 dirty_i->nr_dirty[dirty_type] = 0; 1676 mutex_unlock(&dirty_i->seglist_lock); 1677} 1678 1679static void destroy_victim_secmap(struct f2fs_sb_info *sbi) 1680{ 1681 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1682 kfree(dirty_i->victim_secmap); 1683} 1684 1685static void destroy_dirty_segmap(struct f2fs_sb_info *sbi) 1686{ 1687 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); 1688 int i; 1689 1690 if (!dirty_i) 1691 return; 1692 1693 /* discard pre-free/dirty segments list */ 1694 for (i = 0; i < NR_DIRTY_TYPE; i++) 1695 discard_dirty_segmap(sbi, i); 1696 1697 destroy_victim_secmap(sbi); 1698 SM_I(sbi)->dirty_info = NULL; 1699 kfree(dirty_i); 1700} 1701 1702static void destroy_curseg(struct f2fs_sb_info *sbi) 1703{ 1704 struct curseg_info *array = SM_I(sbi)->curseg_array; 1705 int i; 1706 1707 if (!array) 1708 return; 1709 SM_I(sbi)->curseg_array = NULL; 1710 for (i = 0; i < NR_CURSEG_TYPE; i++) 1711 kfree(array[i].sum_blk); 1712 kfree(array); 1713} 1714 1715static void destroy_free_segmap(struct f2fs_sb_info *sbi) 1716{ 1717 struct free_segmap_info *free_i = SM_I(sbi)->free_info; 1718 if (!free_i) 1719 return; 1720 SM_I(sbi)->free_info = NULL; 1721 kfree(free_i->free_segmap); 1722 kfree(free_i->free_secmap); 1723 kfree(free_i); 1724} 1725 1726static void destroy_sit_info(struct f2fs_sb_info *sbi) 1727{ 1728 struct sit_info *sit_i = SIT_I(sbi); 1729 unsigned int start; 1730 1731 if (!sit_i) 1732 return; 1733 1734 if (sit_i->sentries) { 1735 for (start = 0; start < TOTAL_SEGS(sbi); start++) { 1736 kfree(sit_i->sentries[start].cur_valid_map); 1737 kfree(sit_i->sentries[start].ckpt_valid_map); 1738 } 1739 } 1740 vfree(sit_i->sentries); 1741 vfree(sit_i->sec_entries); 1742 kfree(sit_i->dirty_sentries_bitmap); 1743 1744 SM_I(sbi)->sit_info = NULL; 1745 kfree(sit_i->sit_bitmap); 1746 kfree(sit_i); 1747} 1748 1749void destroy_segment_manager(struct f2fs_sb_info *sbi) 1750{ 1751 struct f2fs_sm_info *sm_info = SM_I(sbi); 1752 if (!sm_info) 1753 return; 1754 destroy_dirty_segmap(sbi); 1755 destroy_curseg(sbi); 1756 destroy_free_segmap(sbi); 1757 destroy_sit_info(sbi); 1758 sbi->sm_info = NULL; 1759 kfree(sm_info); 1760} 1761