19de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher/* 29de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher * Create a squashfs filesystem. This is a highly compressed read only 39de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher * filesystem. 49de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher * 59de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher * Copyright (c) 2014 69de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher * Phillip Lougher <phillip@squashfs.org.uk> 79de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher * 89de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher * This program is free software; you can redistribute it and/or 99de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher * modify it under the terms of the GNU General Public License 109de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher * as published by the Free Software Foundation; either version 2, 119de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher * or (at your option) any later version. 129de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher * 139de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher * This program is distributed in the hope that it will be useful, 149de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher * but WITHOUT ANY WARRANTY; without even the implied warranty of 159de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 169de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher * GNU General Public License for more details. 179de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher * 189de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher * You should have received a copy of the GNU General Public License 199de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher * along with this program; if not, write to the Free Software 209de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 219de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher * 229de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher * process_fragments.c 239de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher */ 249de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher 259de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher#include <pthread.h> 269de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher#include <sys/ioctl.h> 279de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher#include <unistd.h> 289de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher#include <signal.h> 299de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher#include <sys/time.h> 308bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher#include <string.h> 319de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher#include <stdio.h> 329de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher#include <math.h> 339de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher#include <stdarg.h> 349de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher#include <errno.h> 359de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher#include <stdlib.h> 369de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher#include <dirent.h> 379de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher#include <sys/types.h> 389de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher#include <sys/stat.h> 398bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher#include <fcntl.h> 409de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher 419de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher#include "caches-queues-lists.h" 429de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher#include "squashfs_fs.h" 439de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher#include "mksquashfs.h" 449de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher#include "error.h" 459de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher#include "progressbar.h" 469de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher#include "info.h" 478bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher#include "compressor.h" 488bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher#include "process_fragments.h" 499de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher 509de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher#define FALSE 0 519de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher#define TRUE 1 529de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher 539de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougherextern struct queue *to_process_frag; 549de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougherextern struct seq_queue *to_main; 559de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougherextern int sparse_files; 56c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher 57c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher/* 58c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher * Compute 16 bit BSD checksum over the data, and check for sparseness 59c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher */ 608bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougherstatic int checksum_sparse(struct file_buffer *file_buffer) 61c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher{ 62c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher unsigned char *b = (unsigned char *) file_buffer->data; 63c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher unsigned short chksum = 0; 64c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher int bytes = file_buffer->size, sparse = TRUE, value; 65c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher 66c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher while(bytes --) { 67c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher chksum = (chksum & 1) ? (chksum >> 1) | 0x8000 : chksum >> 1; 68c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher value = *b++; 69c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher if(value) { 70c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher sparse = FALSE; 71c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher chksum += value; 72c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher } 73c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher } 74c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher 75c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher file_buffer->checksum = chksum; 76c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher return sparse; 77c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher} 78c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher 799de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher 808bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougherstatic int read_filesystem(int fd, long long byte, int bytes, void *buff) 818bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher{ 828bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher off_t off = byte; 838bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 848bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher TRACE("read_filesystem: reading from position 0x%llx, bytes %d\n", 858bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher byte, bytes); 868bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 878bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher if(lseek(fd, off, SEEK_SET) == -1) { 888bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher ERROR("read_filesystem: Lseek on destination failed because %s, " 898bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher "offset=0x%llx\n", strerror(errno), off); 908bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher return 0; 918bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher } else if(read_bytes(fd, buff, bytes) < bytes) { 928bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher ERROR("Read on destination failed\n"); 938bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher return 0; 948bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher } 958bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 968bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher return 1; 978bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher} 988bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 998bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 1008bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougherstatic struct file_buffer *get_fragment(struct fragment *fragment, 1018bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher char *data_buffer, int fd) 1028bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher{ 1038bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher struct squashfs_fragment_entry *disk_fragment; 1048bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher struct file_buffer *buffer, *compressed_buffer; 1058bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher long long start_block; 1068bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher int res, size, index = fragment->index; 1078bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher char locked; 1088bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 1098bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher /* 1108bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * Lookup fragment block in cache. 1118bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * If the fragment block doesn't exist, then get the compressed version 1128bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * from the writer cache or off disk, and decompress it. 1138bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * 1148bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * This routine has two things which complicate the code: 1158bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * 1168bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * 1. Multiple threads can simultaneously lookup/create the 1178bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * same buffer. This means a buffer needs to be "locked" 1188bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * when it is being filled in, to prevent other threads from 1198bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * using it when it is not ready. This is because we now do 1208bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * fragment duplicate checking in parallel. 1218bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * 2. We have two caches which need to be checked for the 1228bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * presence of fragment blocks: the normal fragment cache 1238bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * and a "reserve" cache. The reserve cache is used to 1248bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * prevent an unnecessary pipeline stall when the fragment cache 1258bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * is full of fragments waiting to be compressed. 1268bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher */ 1278bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex); 1288bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher pthread_mutex_lock(&dup_mutex); 1298bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 1308bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougheragain: 1318bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher buffer = cache_lookup_nowait(fragment_buffer, index, &locked); 1328bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher if(buffer) { 1338bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher pthread_mutex_unlock(&dup_mutex); 1348bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher if(locked) 1358bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher /* got a buffer being filled in. Wait for it */ 1368bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher cache_wait_unlock(buffer); 1378bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher goto finished; 1388bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher } 1398bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 1408bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher /* not in fragment cache, is it in the reserve cache? */ 1418bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher buffer = cache_lookup_nowait(reserve_cache, index, &locked); 1428bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher if(buffer) { 1438bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher pthread_mutex_unlock(&dup_mutex); 1448bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher if(locked) 1458bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher /* got a buffer being filled in. Wait for it */ 1468bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher cache_wait_unlock(buffer); 1478bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher goto finished; 1488bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher } 1498bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 1508bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher /* in neither cache, try to get it from the fragment cache */ 1518bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher buffer = cache_get_nowait(fragment_buffer, index); 1528bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher if(!buffer) { 1538bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher /* 1548bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * no room, get it from the reserve cache, this is 1558bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * dimensioned so it will always have space (no more than 1568bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * processors + 1 can have an outstanding reserve buffer) 1578bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher */ 1588bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher buffer = cache_get_nowait(reserve_cache, index); 1598bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher if(!buffer) { 1608bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher /* failsafe */ 1618bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher ERROR("no space in reserve cache\n"); 1628bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher goto again; 1638bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher } 1648bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher } 1658bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 1668bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher pthread_mutex_unlock(&dup_mutex); 1678bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 168943acada7a60a048b4be53a4df1e94e8b10e08a6Phillip Lougher compressed_buffer = cache_lookup(fwriter_buffer, index); 1698bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 1708bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex); 1718bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher pthread_mutex_lock(&fragment_mutex); 1728bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher disk_fragment = &fragment_table[index]; 1738bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher size = SQUASHFS_COMPRESSED_SIZE_BLOCK(disk_fragment->size); 1748bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher start_block = disk_fragment->start_block; 1758bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher pthread_cleanup_pop(1); 1768bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 1778bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher if(SQUASHFS_COMPRESSED_BLOCK(disk_fragment->size)) { 1788bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher int error; 1798bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher char *data; 1808bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 1818bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher if(compressed_buffer) 1828bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher data = compressed_buffer->data; 1838bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher else { 1848bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher res = read_filesystem(fd, start_block, size, data_buffer); 1858bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher if(res == 0) { 1868bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher ERROR("Failed to read fragment from output" 1878bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher " filesystem\n"); 1888bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher BAD_ERROR("Output filesystem corrupted?\n"); 1898bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher } 1908bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher data = data_buffer; 1918bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher } 1928bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 1938bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher res = compressor_uncompress(comp, buffer->data, data, size, 1948bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher block_size, &error); 1958bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher if(res == -1) 1968bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher BAD_ERROR("%s uncompress failed with error code %d\n", 1978bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher comp->name, error); 1988bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher } else if(compressed_buffer) 1998bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher memcpy(buffer->data, compressed_buffer->data, size); 2008bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher else { 2018bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher res = read_filesystem(fd, start_block, size, buffer->data); 2028bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher if(res == 0) { 2038bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher ERROR("Failed to read fragment from output " 2048bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher "filesystem\n"); 2058bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher BAD_ERROR("Output filesystem corrupted?\n"); 2068bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher } 2078bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher } 2088bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 2098bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher cache_unlock(buffer); 2108bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher cache_block_put(compressed_buffer); 2118bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 2128bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougherfinished: 2138bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher pthread_cleanup_pop(0); 2148bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 2158bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher return buffer; 2168bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher} 2178bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 2188bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 2198bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougherstruct file_buffer *get_fragment_cksum(struct file_info *file, 2208bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher char *data_buffer, int fd, unsigned short *checksum) 2218bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher{ 2228bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher struct file_buffer *frag_buffer; 2238bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher struct append_file *append; 2248bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher int index = file->fragment->index; 2258bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 2268bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher frag_buffer = get_fragment(file->fragment, data_buffer, fd); 2278bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 2288bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex); 2298bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 2308bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher for(append = file_mapping[index]; append; append = append->next) { 2318bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher int offset = append->file->fragment->offset; 2328bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher int size = append->file->fragment->size; 2338bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher char *data = frag_buffer->data + offset; 2348bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher unsigned short cksum = get_checksum_mem(data, size); 2358bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 2368bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher if(file == append->file) 2378bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher *checksum = cksum; 2388bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 2398bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher pthread_mutex_lock(&dup_mutex); 2408bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher append->file->fragment_checksum = cksum; 2418bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher append->file->have_frag_checksum = TRUE; 2428bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher pthread_mutex_unlock(&dup_mutex); 2438bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher } 2448bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 2458bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher pthread_cleanup_pop(0); 2468bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 2478bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher return frag_buffer; 2488bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher} 2498bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 2508bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 2518bb17b0275fa35318ad35c8fd477023004f940aaPhillip Loughervoid *frag_thrd(void *destination_file) 2529de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher{ 2539de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher sigset_t sigmask, old_mask; 2548bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher char *data_buffer; 2558bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher int fd; 2569de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher 2579de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher sigemptyset(&sigmask); 2589de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher sigaddset(&sigmask, SIGINT); 2599de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher sigaddset(&sigmask, SIGTERM); 2609de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher sigaddset(&sigmask, SIGUSR1); 2619de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher pthread_sigmask(SIG_BLOCK, &sigmask, &old_mask); 2629de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher 2638bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher fd = open(destination_file, O_RDONLY); 2648bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher if(fd == -1) 2658bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher BAD_ERROR("frag_thrd: can't open destination for reading\n"); 2668bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 2678bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher data_buffer = malloc(SQUASHFS_FILE_MAX_SIZE); 2688bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher if(data_buffer == NULL) 2698bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher MEM_ERROR(); 2708bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 2718bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher pthread_cleanup_push((void *) pthread_mutex_unlock, &dup_mutex); 2728bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 2739de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher while(1) { 2749de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher struct file_buffer *file_buffer = queue_get(to_process_frag); 2758bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher struct file_buffer *buffer; 276c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher int sparse = checksum_sparse(file_buffer); 2778bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher struct file_info *dupl_ptr; 2788bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher long long file_size; 2798bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher unsigned short checksum; 2808bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher char flag; 2818bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher int res; 2829de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher 283c6424dc4d55a711ff862409fa07b9c143049aba7Phillip Lougher if(sparse_files && sparse) { 2849de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher file_buffer->c_byte = 0; 2859de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher file_buffer->fragment = FALSE; 2869de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher } else 2879de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher file_buffer->c_byte = file_buffer->size; 2889de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher 2898bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher /* 2908bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * Specutively pull into the fragment cache any fragment blocks 2918bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * which contain fragments which *this* fragment may be 2928bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * be a duplicate. 2938bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * 2948bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * By ensuring the fragment block is in cache ahead of time 2958bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * should eliminate the parallelisation stall when the 2968bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * main thread needs to read the fragment block to do a 2978bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * duplicate check on it. 2988bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * 2998bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * If this is a fragment belonging to a larger file 3008bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * (with additional blocks) then ignore it. Here we're 3018bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * interested in the "low hanging fruit" of files which 3028bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * consist of only a fragment 3038bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher */ 3048bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher if(file_buffer->file_size != file_buffer->size) { 3058bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher seq_queue_put(to_main, file_buffer); 3068bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher continue; 3078bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher } 3088bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 3098bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher file_size = file_buffer->file_size; 3108bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 3118bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher pthread_mutex_lock(&dup_mutex); 3128bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher dupl_ptr = dupl[DUP_HASH(file_size)]; 3138bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher pthread_mutex_unlock(&dup_mutex); 3148bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 3158bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher file_buffer->dupl_start = dupl_ptr; 3168bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher file_buffer->duplicate = FALSE; 3178bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 3188bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher for(; dupl_ptr; dupl_ptr = dupl_ptr->next) { 3198bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher if(file_size != dupl_ptr->file_size || 3208bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher file_size != dupl_ptr->fragment->size) 3218bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher continue; 3228bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 3238bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher pthread_mutex_lock(&dup_mutex); 3248bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher flag = dupl_ptr->have_frag_checksum; 3258bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher checksum = dupl_ptr->fragment_checksum; 3268bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher pthread_mutex_unlock(&dup_mutex); 3278bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 3288bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher /* 3298bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * If we have the checksum and it matches then 3308bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * read in the fragment block. 3318bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * 3328bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * If we *don't* have the checksum, then we are 3338bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * appending, and the fragment block is on the 3348bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * "old" filesystem. Read it in and checksum 3358bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher * the entire fragment buffer 3368bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher */ 3378bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher if(!flag) { 3388bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher buffer = get_fragment_cksum(dupl_ptr, 3398bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher data_buffer, fd, &checksum); 3408bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher if(checksum != file_buffer->checksum) { 3418bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher cache_block_put(buffer); 3428bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher continue; 3438bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher } 3448bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher } else if(checksum == file_buffer->checksum) 3458bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher buffer = get_fragment(dupl_ptr->fragment, 3468bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher data_buffer, fd); 3478bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher else 3488bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher continue; 3498bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 3508bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher res = memcmp(file_buffer->data, buffer->data + 3518bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher dupl_ptr->fragment->offset, file_size); 3528bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher cache_block_put(buffer); 3538bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher if(res == 0) { 3548bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher struct file_buffer *dup = malloc(sizeof(*dup)); 3558bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher if(dup == NULL) 3568bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher MEM_ERROR(); 3578bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher memcpy(dup, file_buffer, sizeof(*dup)); 3588bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher cache_block_put(file_buffer); 3598bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher dup->dupl_start = dupl_ptr; 3608bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher dup->duplicate = TRUE; 3618bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher file_buffer = dup; 3628bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher break; 3638bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher } 3648bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher } 3658bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 3669de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher seq_queue_put(to_main, file_buffer); 3679de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher } 3688bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher 3698bb17b0275fa35318ad35c8fd477023004f940aaPhillip Lougher pthread_cleanup_pop(0); 3709de84acd2fcc5992704a433c0de402fcd8a5d143Phillip Lougher} 371