filesetup.c revision 4d4e80f2b4260f2c8b37a8612ce655502a799f7a
1#include <unistd.h> 2#include <fcntl.h> 3#include <string.h> 4#include <assert.h> 5#include <dirent.h> 6#include <sys/stat.h> 7#include <sys/mman.h> 8#include <sys/types.h> 9 10#include "fio.h" 11#include "smalloc.h" 12#include "filehash.h" 13 14static int root_warn; 15 16static int extend_file(struct thread_data *td, struct fio_file *f) 17{ 18 int r, new_layout = 0, unlink_file = 0, flags; 19 unsigned long long left; 20 unsigned int bs; 21 char *b; 22 23 if (read_only) { 24 log_err("fio: refusing extend of file due to read-only\n"); 25 return 0; 26 } 27 28 /* 29 * check if we need to lay the file out complete again. fio 30 * does that for operations involving reads, or for writes 31 * where overwrite is set 32 */ 33 if (td_read(td) || (td_write(td) && td->o.overwrite)) 34 new_layout = 1; 35 if (td_write(td) && !td->o.overwrite) 36 unlink_file = 1; 37 38 if (unlink_file || new_layout) { 39 if ((unlink(f->file_name) < 0) && (errno != ENOENT)) { 40 td_verror(td, errno, "unlink"); 41 return 1; 42 } 43 } 44 45 flags = O_WRONLY | O_CREAT; 46 if (new_layout) 47 flags |= O_TRUNC; 48 49 dprint(FD_FILE, "open file %s, flags %x\n", f->file_name, flags); 50 f->fd = open(f->file_name, flags, 0644); 51 if (f->fd < 0) { 52 td_verror(td, errno, "open"); 53 return 1; 54 } 55 56 if (!new_layout) 57 goto done; 58 59 dprint(FD_FILE, "truncate file %s, size %llu\n", f->file_name, 60 f->real_file_size); 61 if (ftruncate(f->fd, f->real_file_size) == -1) { 62 td_verror(td, errno, "ftruncate"); 63 goto err; 64 } 65 66 dprint(FD_FILE, "fallocate file %s, size %llu\n", f->file_name, 67 f->real_file_size); 68 if (posix_fallocate(f->fd, 0, f->real_file_size) < 0) { 69 td_verror(td, errno, "posix_fallocate"); 70 goto err; 71 } 72 73 b = malloc(td->o.max_bs[DDIR_WRITE]); 74 memset(b, 0, td->o.max_bs[DDIR_WRITE]); 75 76 left = f->real_file_size; 77 while (left && !td->terminate) { 78 bs = td->o.max_bs[DDIR_WRITE]; 79 if (bs > left) 80 bs = left; 81 82 r = write(f->fd, b, bs); 83 84 if (r == (int) bs) { 85 left -= bs; 86 continue; 87 } else { 88 if (r < 0) 89 td_verror(td, errno, "write"); 90 else 91 td_verror(td, EIO, "write"); 92 93 break; 94 } 95 } 96 97 if (td->terminate) 98 unlink(f->file_name); 99 else if (td->o.create_fsync) 100 fsync(f->fd); 101 102 free(b); 103done: 104 close(f->fd); 105 f->fd = -1; 106 return 0; 107err: 108 close(f->fd); 109 f->fd = -1; 110 return 1; 111} 112 113static unsigned long long get_rand_file_size(struct thread_data *td) 114{ 115 unsigned long long ret; 116 long r; 117 118 r = os_random_long(&td->file_size_state); 119 ret = td->o.file_size_low + (unsigned long long) ((double) (td->o.file_size_high - td->o.file_size_low) * (r / (RAND_MAX + 1.0))); 120 ret -= (ret % td->o.rw_min_bs); 121 return ret; 122} 123 124static int file_size(struct thread_data *td, struct fio_file *f) 125{ 126 struct stat st; 127 128 if (fstat(f->fd, &st) == -1) { 129 td_verror(td, errno, "fstat"); 130 return 1; 131 } 132 133 f->real_file_size = st.st_size; 134 return 0; 135} 136 137static int bdev_size(struct thread_data *td, struct fio_file *f) 138{ 139 unsigned long long bytes; 140 int r; 141 142 r = blockdev_size(f->fd, &bytes); 143 if (r) { 144 td_verror(td, r, "blockdev_size"); 145 return 1; 146 } 147 148 f->real_file_size = bytes; 149 return 0; 150} 151 152static int get_file_size(struct thread_data *td, struct fio_file *f) 153{ 154 int ret = 0; 155 156 if (f->flags & FIO_SIZE_KNOWN) 157 return 0; 158 159 if (f->filetype == FIO_TYPE_FILE) 160 ret = file_size(td, f); 161 else if (f->filetype == FIO_TYPE_BD) 162 ret = bdev_size(td, f); 163 else 164 f->real_file_size = -1; 165 166 if (ret) 167 return ret; 168 169 if (f->file_offset > f->real_file_size) { 170 log_err("%s: offset extends end (%Lu > %Lu)\n", td->o.name, f->file_offset, f->real_file_size); 171 return 1; 172 } 173 174 f->flags |= FIO_SIZE_KNOWN; 175 return 0; 176} 177 178int file_invalidate_cache(struct thread_data *td, struct fio_file *f) 179{ 180 int ret = 0; 181 182 dprint(FD_IO, "invalidate cache (%d)\n", td->o.odirect); 183 184 if (td->o.odirect) 185 return 0; 186 187 /* 188 * FIXME: add blockdev flushing too 189 */ 190 if (f->mmap) 191 ret = madvise(f->mmap, f->io_size, MADV_DONTNEED); 192 else if (f->filetype == FIO_TYPE_FILE) 193 ret = fadvise(f->fd, f->file_offset, f->io_size, POSIX_FADV_DONTNEED); 194 else if (f->filetype == FIO_TYPE_BD) { 195 ret = blockdev_invalidate_cache(f->fd); 196 if (ret < 0 && errno == EACCES && geteuid()) { 197 if (!root_warn) { 198 log_err("fio: only root may flush block devices. Cache flush bypassed!\n"); 199 root_warn = 1; 200 } 201 ret = 0; 202 } 203 } else if (f->filetype == FIO_TYPE_CHAR || f->filetype == FIO_TYPE_PIPE) 204 ret = 0; 205 206 if (ret < 0) { 207 td_verror(td, errno, "invalidate_cache"); 208 return 1; 209 } 210 211 return ret; 212} 213 214int generic_close_file(struct thread_data fio_unused *td, struct fio_file *f) 215{ 216 int ret = 0; 217 218 dprint(FD_FILE, "fd close %s\n", f->file_name); 219 220 remove_file_hash(f); 221 222 if (close(f->fd) < 0) 223 ret = errno; 224 225 f->fd = -1; 226 return ret; 227} 228 229static int file_lookup_open(struct fio_file *f, int flags) 230{ 231 struct fio_file *__f; 232 int from_hash; 233 234 __f = lookup_file_hash(f->file_name); 235 if (__f) { 236 /* 237 * racy, need the __f->lock locked 238 */ 239 f->lock = __f->lock; 240 f->lock_owner = __f->lock_owner; 241 f->lock_batch = __f->lock_batch; 242 f->lock_ddir = __f->lock_ddir; 243 f->fd = dup(__f->fd); 244 f->references++; 245 from_hash = 1; 246 } else { 247 f->fd = open(f->file_name, flags, 0600); 248 from_hash = 0; 249 } 250 251 return from_hash; 252} 253 254int generic_open_file(struct thread_data *td, struct fio_file *f) 255{ 256 int is_std = 0; 257 int flags = 0; 258 int from_hash = 0; 259 260 dprint(FD_FILE, "fd open %s\n", f->file_name); 261 262 if (!strcmp(f->file_name, "-")) { 263 if (td_rw(td)) { 264 log_err("fio: can't read/write to stdin/out\n"); 265 return 1; 266 } 267 is_std = 1; 268 269 /* 270 * move output logging to stderr, if we are writing to stdout 271 */ 272 if (td_write(td)) 273 f_out = stderr; 274 } 275 276 if (td->o.odirect) 277 flags |= OS_O_DIRECT; 278 if (td->o.sync_io) 279 flags |= O_SYNC; 280 if (f->filetype != FIO_TYPE_FILE) 281 flags |= O_NOATIME; 282 283open_again: 284 if (td_write(td)) { 285 assert(!read_only); 286 287 flags |= O_RDWR; 288 289 if (f->filetype == FIO_TYPE_FILE) 290 flags |= O_CREAT; 291 292 if (is_std) 293 f->fd = dup(STDOUT_FILENO); 294 else 295 from_hash = file_lookup_open(f, flags); 296 } else { 297 if (f->filetype == FIO_TYPE_CHAR && !read_only) 298 flags |= O_RDWR; 299 else 300 flags |= O_RDONLY; 301 302 if (is_std) 303 f->fd = dup(STDIN_FILENO); 304 else 305 from_hash = file_lookup_open(f, flags); 306 } 307 308 if (f->fd == -1) { 309 char buf[FIO_VERROR_SIZE]; 310 int __e = errno; 311 312 if (errno == EPERM && (flags & O_NOATIME)) { 313 flags &= ~O_NOATIME; 314 goto open_again; 315 } 316 317 snprintf(buf, sizeof(buf) - 1, "open(%s)", f->file_name); 318 319 td_verror(td, __e, buf); 320 } 321 322 if (get_file_size(td, f)) 323 goto err; 324 325 if (!from_hash && f->fd != -1) { 326 if (add_file_hash(f)) { 327 int ret; 328 329 /* 330 * OK to ignore, we haven't done anything with it 331 */ 332 ret = generic_close_file(td, f); 333 goto open_again; 334 } 335 } 336 337 return 0; 338err: 339 close(f->fd); 340 return 1; 341} 342 343int open_files(struct thread_data *td) 344{ 345 struct fio_file *f; 346 unsigned int i; 347 int err = 0; 348 349 dprint(FD_FILE, "open files\n"); 350 351 for_each_file(td, f, i) { 352 err = td_io_open_file(td, f); 353 if (err) { 354 if (td->error == EMFILE) { 355 log_err("fio: limited open files to: %d\n", td->nr_open_files); 356 td->o.open_files = td->nr_open_files; 357 err = 0; 358 clear_error(td); 359 } 360 break; 361 } 362 363 if (td->o.open_files == td->nr_open_files) 364 break; 365 } 366 367 if (!err) 368 return 0; 369 370 for_each_file(td, f, i) 371 td_io_close_file(td, f); 372 373 return err; 374} 375 376/* 377 * open/close all files, so that ->real_file_size gets set 378 */ 379static int get_file_sizes(struct thread_data *td) 380{ 381 struct fio_file *f; 382 unsigned int i; 383 int err = 0; 384 385 for_each_file(td, f, i) { 386 if (td->io_ops->open_file(td, f)) { 387 if (td->error != ENOENT) { 388 log_err("%s\n", td->verror); 389 err = 1; 390 } 391 clear_error(td); 392 } else { 393 if (td->io_ops->close_file) 394 td->io_ops->close_file(td, f); 395 } 396 397 if (f->real_file_size == -1ULL && td->o.size) 398 f->real_file_size = td->o.size / td->o.nr_files; 399 } 400 401 return err; 402} 403 404/* 405 * Open the files and setup files sizes, creating files if necessary. 406 */ 407int setup_files(struct thread_data *td) 408{ 409 unsigned long long total_size, extend_size; 410 struct fio_file *f; 411 unsigned int i; 412 int err = 0, need_extend; 413 414 dprint(FD_FILE, "setup files\n"); 415 416 /* 417 * if ioengine defines a setup() method, it's responsible for 418 * opening the files and setting f->real_file_size to indicate 419 * the valid range for that file. 420 */ 421 if (td->io_ops->setup) 422 err = td->io_ops->setup(td); 423 else 424 err = get_file_sizes(td); 425 426 if (err) 427 return err; 428 429 /* 430 * check sizes. if the files/devices do not exist and the size 431 * isn't passed to fio, abort. 432 */ 433 total_size = 0; 434 for_each_file(td, f, i) { 435 if (f->real_file_size == -1ULL) 436 total_size = -1ULL; 437 else 438 total_size += f->real_file_size; 439 } 440 441 /* 442 * device/file sizes are zero and no size given, punt 443 */ 444 if ((!total_size || total_size == -1ULL) && !td->o.size && 445 !(td->io_ops->flags & FIO_NOIO) && !td->o.fill_device) { 446 log_err("%s: you need to specify size=\n", td->o.name); 447 td_verror(td, EINVAL, "total_file_size"); 448 return 1; 449 } 450 451 /* 452 * now file sizes are known, so we can set ->io_size. if size= is 453 * not given, ->io_size is just equal to ->real_file_size. if size 454 * is given, ->io_size is size / nr_files. 455 */ 456 extend_size = total_size = 0; 457 need_extend = 0; 458 for_each_file(td, f, i) { 459 f->file_offset = td->o.start_offset; 460 461 if (!td->o.file_size_low) { 462 /* 463 * no file size range given, file size is equal to 464 * total size divided by number of files. if that is 465 * zero, set it to the real file size. 466 */ 467 f->io_size = td->o.size / td->o.nr_files; 468 if (!f->io_size) 469 f->io_size = f->real_file_size - f->file_offset; 470 } else if (f->real_file_size < td->o.file_size_low || 471 f->real_file_size > td->o.file_size_high) { 472 if (f->file_offset > td->o.file_size_low) 473 goto err_offset; 474 /* 475 * file size given. if it's fixed, use that. if it's a 476 * range, generate a random size in-between. 477 */ 478 if (td->o.file_size_low == td->o.file_size_high) 479 f->io_size = td->o.file_size_low - f->file_offset; 480 else 481 f->io_size = get_rand_file_size(td) - f->file_offset; 482 } else 483 f->io_size = f->real_file_size - f->file_offset; 484 485 if (f->io_size == -1ULL) 486 total_size = -1ULL; 487 else 488 total_size += f->io_size; 489 490 if (f->filetype == FIO_TYPE_FILE && 491 (f->io_size + f->file_offset) > f->real_file_size && 492 !(td->io_ops->flags & FIO_DISKLESSIO)) { 493 need_extend++; 494 extend_size += (f->io_size + f->file_offset); 495 f->flags |= FIO_FILE_EXTEND; 496 } 497 } 498 499 if (!td->o.size || td->o.size > total_size) 500 td->o.size = total_size; 501 502 /* 503 * See if we need to extend some files 504 */ 505 if (need_extend) { 506 temp_stall_ts = 1; 507 log_info("%s: Laying out IO file(s) (%u file(s) / %LuMiB)\n", 508 td->o.name, need_extend, extend_size >> 20); 509 510 for_each_file(td, f, i) { 511 if (!(f->flags & FIO_FILE_EXTEND)) 512 continue; 513 514 assert(f->filetype == FIO_TYPE_FILE); 515 f->flags &= ~FIO_FILE_EXTEND; 516 f->real_file_size = (f->io_size + f->file_offset); 517 err = extend_file(td, f); 518 if (err) 519 break; 520 } 521 temp_stall_ts = 0; 522 } 523 524 if (err) 525 return err; 526 527 if (!td->o.zone_size) 528 td->o.zone_size = td->o.size; 529 530 /* 531 * iolog already set the total io size, if we read back 532 * stored entries. 533 */ 534 if (!td->o.read_iolog_file) 535 td->total_io_size = td->o.size * td->o.loops; 536 return 0; 537err_offset: 538 log_err("%s: you need to specify valid offset=\n", td->o.name); 539 return 1; 540} 541 542int init_random_map(struct thread_data *td) 543{ 544 unsigned long long blocks, num_maps; 545 struct fio_file *f; 546 unsigned int i; 547 548 if (td->o.norandommap || !td_random(td)) 549 return 0; 550 551 for_each_file(td, f, i) { 552 blocks = (f->real_file_size + td->o.rw_min_bs - 1) / (unsigned long long) td->o.rw_min_bs; 553 num_maps = (blocks + BLOCKS_PER_MAP-1)/ (unsigned long long) BLOCKS_PER_MAP; 554 f->file_map = smalloc(num_maps * sizeof(long)); 555 if (!f->file_map) { 556 log_err("fio: failed allocating random map. If running a large number of jobs, try the 'norandommap' option\n"); 557 return 1; 558 } 559 f->num_maps = num_maps; 560 } 561 562 return 0; 563} 564 565void close_files(struct thread_data *td) 566{ 567 struct fio_file *f; 568 unsigned int i; 569 570 for_each_file(td, f, i) 571 td_io_close_file(td, f); 572} 573 574void close_and_free_files(struct thread_data *td) 575{ 576 struct fio_file *f; 577 unsigned int i; 578 579 dprint(FD_FILE, "close files\n"); 580 581 for_each_file(td, f, i) { 582 if (td->o.unlink && f->filetype == FIO_TYPE_FILE) 583 unlink(f->file_name); 584 585 td_io_close_file(td, f); 586 587 sfree(f->file_name); 588 f->file_name = NULL; 589 590 if (f->file_map) { 591 sfree(f->file_map); 592 f->file_map = NULL; 593 } 594 sfree(f); 595 } 596 597 td->o.filename = NULL; 598 free(td->files); 599 td->files = NULL; 600 td->o.nr_files = 0; 601} 602 603static void get_file_type(struct fio_file *f) 604{ 605 struct stat sb; 606 607 if (!strcmp(f->file_name, "-")) 608 f->filetype = FIO_TYPE_PIPE; 609 else 610 f->filetype = FIO_TYPE_FILE; 611 612 if (!lstat(f->file_name, &sb)) { 613 if (S_ISBLK(sb.st_mode)) 614 f->filetype = FIO_TYPE_BD; 615 else if (S_ISCHR(sb.st_mode)) 616 f->filetype = FIO_TYPE_CHAR; 617 else if (S_ISFIFO(sb.st_mode)) 618 f->filetype = FIO_TYPE_PIPE; 619 } 620} 621 622int add_file(struct thread_data *td, const char *fname) 623{ 624 int cur_files = td->files_index; 625 char file_name[PATH_MAX]; 626 struct fio_file *f; 627 int len = 0; 628 629 dprint(FD_FILE, "add file %s\n", fname); 630 631 f = smalloc(sizeof(*f)); 632 f->fd = -1; 633 634 td->files = realloc(td->files, (cur_files + 1) * sizeof(f)); 635 636 td->files[cur_files] = f; 637 638 /* 639 * init function, io engine may not be loaded yet 640 */ 641 if (td->io_ops && (td->io_ops->flags & FIO_DISKLESSIO)) 642 f->real_file_size = -1ULL; 643 644 if (td->o.directory) 645 len = sprintf(file_name, "%s/", td->o.directory); 646 647 sprintf(file_name + len, "%s", fname); 648 f->file_name = smalloc_strdup(file_name); 649 650 get_file_type(f); 651 652 switch (td->o.file_lock_mode) { 653 case FILE_LOCK_NONE: 654 break; 655 case FILE_LOCK_READWRITE: 656 f->lock = fio_mutex_rw_init(); 657 break; 658 case FILE_LOCK_EXCLUSIVE: 659 f->lock = fio_mutex_init(1); 660 break; 661 default: 662 log_err("fio: unknown lock mode: %d\n", td->o.file_lock_mode); 663 assert(0); 664 } 665 666 td->files_index++; 667 if (f->filetype == FIO_TYPE_FILE) 668 td->nr_normal_files++; 669 670 return cur_files; 671} 672 673void get_file(struct fio_file *f) 674{ 675 dprint(FD_FILE, "get file %s, ref=%d\n", f->file_name, f->references); 676 assert(f->flags & FIO_FILE_OPEN); 677 f->references++; 678} 679 680int put_file(struct thread_data *td, struct fio_file *f) 681{ 682 int ret = 0; 683 684 dprint(FD_FILE, "put file %s, ref=%d\n", f->file_name, f->references); 685 686 if (!(f->flags & FIO_FILE_OPEN)) 687 return 0; 688 689 assert(f->references); 690 if (--f->references) 691 return 0; 692 693 if (should_fsync(td) && td->o.fsync_on_close) 694 fsync(f->fd); 695 696 if (td->io_ops->close_file) 697 ret = td->io_ops->close_file(td, f); 698 699 td->nr_open_files--; 700 f->flags &= ~FIO_FILE_OPEN; 701 return ret; 702} 703 704void lock_file(struct thread_data *td, struct fio_file *f, enum fio_ddir ddir) 705{ 706 if (!f->lock || td->o.file_lock_mode == FILE_LOCK_NONE) 707 return; 708 709 if (f->lock_owner == td && f->lock_batch--) 710 return; 711 712 if (td->o.file_lock_mode == FILE_LOCK_READWRITE) { 713 if (ddir == DDIR_READ) 714 fio_mutex_down_read(f->lock); 715 else 716 fio_mutex_down_write(f->lock); 717 } else if (td->o.file_lock_mode == FILE_LOCK_EXCLUSIVE) 718 fio_mutex_down(f->lock); 719 720 f->lock_owner = td; 721 f->lock_batch = td->o.lockfile_batch; 722 f->lock_ddir = ddir; 723} 724 725void unlock_file(struct thread_data *td, struct fio_file *f) 726{ 727 if (!f->lock || td->o.file_lock_mode == FILE_LOCK_NONE) 728 return; 729 if (f->lock_batch) 730 return; 731 732 if (td->o.file_lock_mode == FILE_LOCK_READWRITE) { 733 const int is_read = f->lock_ddir == DDIR_READ; 734 int val = fio_mutex_getval(f->lock); 735 736 if ((is_read && val == 1) || (!is_read && val == -1)) 737 f->lock_owner = NULL; 738 739 if (is_read) 740 fio_mutex_up_read(f->lock); 741 else 742 fio_mutex_up_write(f->lock); 743 } else if (td->o.file_lock_mode == FILE_LOCK_EXCLUSIVE) { 744 int val = fio_mutex_getval(f->lock); 745 746 if (val == 0) 747 f->lock_owner = NULL; 748 749 fio_mutex_up(f->lock); 750 } 751} 752 753void unlock_file_all(struct thread_data *td, struct fio_file *f) 754{ 755 if (f->lock_owner != td) 756 return; 757 758 f->lock_batch = 0; 759 unlock_file(td, f); 760} 761 762static int recurse_dir(struct thread_data *td, const char *dirname) 763{ 764 struct dirent *dir; 765 int ret = 0; 766 DIR *D; 767 768 D = opendir(dirname); 769 if (!D) { 770 char buf[FIO_VERROR_SIZE]; 771 772 snprintf(buf, FIO_VERROR_SIZE - 1, "opendir(%s)", dirname); 773 td_verror(td, errno, buf); 774 return 1; 775 } 776 777 while ((dir = readdir(D)) != NULL) { 778 char full_path[PATH_MAX]; 779 struct stat sb; 780 781 if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, "..")) 782 continue; 783 784 sprintf(full_path, "%s/%s", dirname, dir->d_name); 785 786 if (lstat(full_path, &sb) == -1) { 787 if (errno != ENOENT) { 788 td_verror(td, errno, "stat"); 789 return 1; 790 } 791 } 792 793 if (S_ISREG(sb.st_mode)) { 794 add_file(td, full_path); 795 td->o.nr_files++; 796 continue; 797 } 798 if (!S_ISDIR(sb.st_mode)) 799 continue; 800 801 if ((ret = recurse_dir(td, full_path)) != 0) 802 break; 803 } 804 805 closedir(D); 806 return ret; 807} 808 809int add_dir_files(struct thread_data *td, const char *path) 810{ 811 int ret = recurse_dir(td, path); 812 813 if (!ret) 814 log_info("fio: opendir added %d files\n", td->o.nr_files); 815 816 return ret; 817} 818 819void dup_files(struct thread_data *td, struct thread_data *org) 820{ 821 struct fio_file *f; 822 unsigned int i; 823 size_t bytes; 824 825 if (!org->files) 826 return; 827 828 bytes = org->files_index * sizeof(f); 829 td->files = malloc(bytes); 830 memcpy(td->files, org->files, bytes); 831 832 for_each_file(td, f, i) { 833 struct fio_file *__f; 834 835 __f = smalloc(sizeof(*__f)); 836 837 if (f->file_name) 838 __f->file_name = smalloc_strdup(f->file_name); 839 840 td->files[i] = __f; 841 } 842} 843 844/* 845 * Returns the index that matches the filename, or -1 if not there 846 */ 847int get_fileno(struct thread_data *td, const char *fname) 848{ 849 struct fio_file *f; 850 unsigned int i; 851 852 for_each_file(td, f, i) 853 if (!strcmp(f->file_name, fname)) 854 return i; 855 856 return -1; 857} 858 859/* 860 * For log usage, where we add/open/close files automatically 861 */ 862void free_release_files(struct thread_data *td) 863{ 864 close_files(td); 865 td->files_index = 0; 866 td->nr_normal_files = 0; 867} 868