BackupHelpers.cpp revision fbb92385f2fb0ae1146bb8f3d73547d90bda6db1 
1/*
2 * Copyright (C) 2009 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define LOG_TAG "file_backup_helper"
18
19#include <utils/BackupHelpers.h>
20
21#include <utils/KeyedVector.h>
22#include <utils/ByteOrder.h>
23#include <utils/String8.h>
24
25#include <errno.h>
26#include <sys/types.h>
27#include <sys/uio.h>
28#include <sys/stat.h>
29#include <sys/time.h>  // for utimes
30#include <stdio.h>
31#include <stdlib.h>
32#include <unistd.h>
33#include <utime.h>
34#include <fcntl.h>
35#include <zlib.h>
36
37#include <cutils/log.h>
38
39namespace android {
40
41#define MAGIC0 0x70616e53 // Snap
42#define MAGIC1 0x656c6946 // File
43
44/*
45 * File entity data format (v1):
46 *
47 *   - 4-byte version number of the metadata, little endian (0x00000001 for v1)
48 *   - 12 bytes of metadata
49 *   - the file data itself
50 *
51 * i.e. a 16-byte metadata header followed by the raw file data.  If the
52 * restore code does not recognize the metadata version, it can still
53 * interpret the file data itself correctly.
54 *
55 * file_metadata_v1:
56 *
57 *   - 4 byte version number === 0x00000001 (little endian)
58 *   - 4-byte access mode (little-endian)
59 *   - undefined (8 bytes)
60 */
61
62struct file_metadata_v1 {
63    int version;
64    int mode;
65    int undefined_1;
66    int undefined_2;
67};
68
69const static int CURRENT_METADATA_VERSION = 1;
70
71// auto-free buffer management object
72class StAutoFree {
73public:
74    StAutoFree(void* buffer) { mBuf = buffer; }
75    ~StAutoFree() { free(mBuf); }
76private:
77    void* mBuf;
78};
79
80#if 0 // TEST_BACKUP_HELPERS
81#define LOGP(f, x...) printf(f "\n", x)
82#else
83#define LOGP(x...) LOGD(x)
84#endif
85
86const static int ROUND_UP[4] = { 0, 3, 2, 1 };
87
88static inline int
89round_up(int n)
90{
91    return n + ROUND_UP[n % 4];
92}
93
94static int
95read_snapshot_file(int fd, KeyedVector<String8,FileState>* snapshot)
96{
97    int bytesRead = 0;
98    int amt;
99    SnapshotHeader header;
100
101    amt = read(fd, &header, sizeof(header));
102    if (amt != sizeof(header)) {
103        return errno;
104    }
105    bytesRead += amt;
106
107    if (header.magic0 != MAGIC0 || header.magic1 != MAGIC1) {
108        LOGW("read_snapshot_file header.magic0=0x%08x magic1=0x%08x", header.magic0, header.magic1);
109        return 1;
110    }
111
112    for (int i=0; i<header.fileCount; i++) {
113        FileState file;
114        char filenameBuf[128];
115
116        amt = read(fd, &file, sizeof(FileState));
117        if (amt != sizeof(FileState)) {
118            LOGW("read_snapshot_file FileState truncated/error with read at %d bytes\n", bytesRead);
119            return 1;
120        }
121        bytesRead += amt;
122
123        // filename is not NULL terminated, but it is padded
124        int nameBufSize = round_up(file.nameLen);
125        char* filename = nameBufSize <= (int)sizeof(filenameBuf)
126                ? filenameBuf
127                : (char*)malloc(nameBufSize);
128        amt = read(fd, filename, nameBufSize);
129        if (amt == nameBufSize) {
130            snapshot->add(String8(filename, file.nameLen), file);
131        }
132        bytesRead += amt;
133        if (filename != filenameBuf) {
134            free(filename);
135        }
136        if (amt != nameBufSize) {
137            LOGW("read_snapshot_file filename truncated/error with read at %d bytes\n", bytesRead);
138            return 1;
139        }
140    }
141
142    if (header.totalSize != bytesRead) {
143        LOGW("read_snapshot_file length mismatch: header.totalSize=%d bytesRead=%d\n",
144                header.totalSize, bytesRead);
145        return 1;
146    }
147
148    return 0;
149}
150
151static int
152write_snapshot_file(int fd, const KeyedVector<String8,FileRec>& snapshot)
153{
154    int fileCount = 0;
155    int bytesWritten = sizeof(SnapshotHeader);
156    // preflight size
157    const int N = snapshot.size();
158    for (int i=0; i<N; i++) {
159        const FileRec& g = snapshot.valueAt(i);
160        if (!g.deleted) {
161            const String8& name = snapshot.keyAt(i);
162            bytesWritten += sizeof(FileState) + round_up(name.length());
163            fileCount++;
164        }
165    }
166
167    LOGP("write_snapshot_file fd=%d\n", fd);
168
169    int amt;
170    SnapshotHeader header = { MAGIC0, fileCount, MAGIC1, bytesWritten };
171
172    amt = write(fd, &header, sizeof(header));
173    if (amt != sizeof(header)) {
174        LOGW("write_snapshot_file error writing header %s", strerror(errno));
175        return errno;
176    }
177
178    for (int i=0; i<N; i++) {
179        FileRec r = snapshot.valueAt(i);
180        if (!r.deleted) {
181            const String8& name = snapshot.keyAt(i);
182            int nameLen = r.s.nameLen = name.length();
183
184            amt = write(fd, &r.s, sizeof(FileState));
185            if (amt != sizeof(FileState)) {
186                LOGW("write_snapshot_file error writing header %s", strerror(errno));
187                return 1;
188            }
189
190            // filename is not NULL terminated, but it is padded
191            amt = write(fd, name.string(), nameLen);
192            if (amt != nameLen) {
193                LOGW("write_snapshot_file error writing filename %s", strerror(errno));
194                return 1;
195            }
196            int paddingLen = ROUND_UP[nameLen % 4];
197            if (paddingLen != 0) {
198                int padding = 0xabababab;
199                amt = write(fd, &padding, paddingLen);
200                if (amt != paddingLen) {
201                    LOGW("write_snapshot_file error writing %d bytes of filename padding %s",
202                            paddingLen, strerror(errno));
203                    return 1;
204                }
205            }
206        }
207    }
208
209    return 0;
210}
211
212static int
213write_delete_file(BackupDataWriter* dataStream, const String8& key)
214{
215    LOGP("write_delete_file %s\n", key.string());
216    return dataStream->WriteEntityHeader(key, -1);
217}
218
219static int
220write_update_file(BackupDataWriter* dataStream, int fd, int mode, const String8& key,
221        char const* realFilename)
222{
223    LOGP("write_update_file %s (%s) : mode 0%o\n", realFilename, key.string(), mode);
224
225    const int bufsize = 4*1024;
226    int err;
227    int amt;
228    int fileSize;
229    int bytesLeft;
230    file_metadata_v1 metadata;
231
232    char* buf = (char*)malloc(bufsize);
233    StAutoFree _autoFree(buf);
234
235    int crc = crc32(0L, Z_NULL, 0);
236
237
238    fileSize = lseek(fd, 0, SEEK_END);
239    lseek(fd, 0, SEEK_SET);
240
241    if (sizeof(metadata) != 16) {
242        LOGE("ERROR: metadata block is the wrong size!");
243    }
244
245    bytesLeft = fileSize + sizeof(metadata);
246    err = dataStream->WriteEntityHeader(key, bytesLeft);
247    if (err != 0) {
248        return err;
249    }
250
251    // store the file metadata first
252    metadata.version = tolel(CURRENT_METADATA_VERSION);
253    metadata.mode = tolel(mode);
254    metadata.undefined_1 = metadata.undefined_2 = 0;
255    err = dataStream->WriteEntityData(&metadata, sizeof(metadata));
256    if (err != 0) {
257        return err;
258    }
259    bytesLeft -= sizeof(metadata); // bytesLeft should == fileSize now
260
261    // now store the file content
262    while ((amt = read(fd, buf, bufsize)) != 0 && bytesLeft > 0) {
263        bytesLeft -= amt;
264        if (bytesLeft < 0) {
265            amt += bytesLeft; // Plus a negative is minus.  Don't write more than we promised.
266        }
267        err = dataStream->WriteEntityData(buf, amt);
268        if (err != 0) {
269            return err;
270        }
271    }
272    if (bytesLeft != 0) {
273        if (bytesLeft > 0) {
274            // Pad out the space we promised in the buffer.  We can't corrupt the buffer,
275            // even though the data we're sending is probably bad.
276            memset(buf, 0, bufsize);
277            while (bytesLeft > 0) {
278                amt = bytesLeft < bufsize ? bytesLeft : bufsize;
279                bytesLeft -= amt;
280                err = dataStream->WriteEntityData(buf, amt);
281                if (err != 0) {
282                    return err;
283                }
284            }
285        }
286        LOGE("write_update_file size mismatch for %s. expected=%d actual=%d."
287                " You aren't doing proper locking!", realFilename, fileSize, fileSize-bytesLeft);
288    }
289
290    return NO_ERROR;
291}
292
293static int
294write_update_file(BackupDataWriter* dataStream, const String8& key, char const* realFilename)
295{
296    int err;
297    struct stat st;
298
299    err = stat(realFilename, &st);
300    if (err < 0) {
301        return errno;
302    }
303
304    int fd = open(realFilename, O_RDONLY);
305    if (fd == -1) {
306        return errno;
307    }
308
309    err = write_update_file(dataStream, fd, st.st_mode, key, realFilename);
310    close(fd);
311    return err;
312}
313
314static int
315compute_crc32(int fd)
316{
317    const int bufsize = 4*1024;
318    int amt;
319
320    char* buf = (char*)malloc(bufsize);
321    StAutoFree _autoFree(buf);
322
323    int crc = crc32(0L, Z_NULL, 0);
324
325    lseek(fd, 0, SEEK_SET);
326
327    while ((amt = read(fd, buf, bufsize)) != 0) {
328        crc = crc32(crc, (Bytef*)buf, amt);
329    }
330
331    return crc;
332}
333
334int
335back_up_files(int oldSnapshotFD, BackupDataWriter* dataStream, int newSnapshotFD,
336        char const* const* files, char const* const* keys, int fileCount)
337{
338    int err;
339    KeyedVector<String8,FileState> oldSnapshot;
340    KeyedVector<String8,FileRec> newSnapshot;
341
342    if (oldSnapshotFD != -1) {
343        err = read_snapshot_file(oldSnapshotFD, &oldSnapshot);
344        if (err != 0) {
345            // On an error, treat this as a full backup.
346            oldSnapshot.clear();
347        }
348    }
349
350    for (int i=0; i<fileCount; i++) {
351        String8 key(keys[i]);
352        FileRec r;
353        char const* file = files[i];
354        r.file = file;
355        struct stat st;
356
357        err = stat(file, &st);
358        if (err != 0) {
359            LOGW("Error stating file %s", file);
360            r.deleted = true;
361        } else {
362            r.deleted = false;
363            r.s.modTime_sec = st.st_mtime;
364            r.s.modTime_nsec = 0; // workaround sim breakage
365            //r.s.modTime_nsec = st.st_mtime_nsec;
366            r.s.mode = st.st_mode;
367            r.s.size = st.st_size;
368            // we compute the crc32 later down below, when we already have the file open.
369
370            if (newSnapshot.indexOfKey(key) >= 0) {
371                LOGP("back_up_files key already in use '%s'", key.string());
372                return -1;
373            }
374        }
375        newSnapshot.add(key, r);
376    }
377
378    int n = 0;
379    int N = oldSnapshot.size();
380    int m = 0;
381
382    while (n<N && m<fileCount) {
383        const String8& p = oldSnapshot.keyAt(n);
384        const String8& q = newSnapshot.keyAt(m);
385        FileRec& g = newSnapshot.editValueAt(m);
386        int cmp = p.compare(q);
387        if (g.deleted || cmp < 0) {
388            // file removed
389            LOGP("file removed: %s", p.string());
390            g.deleted = true; // They didn't mention the file, but we noticed that it's gone.
391            dataStream->WriteEntityHeader(p, -1);
392            n++;
393        }
394        else if (cmp > 0) {
395            // file added
396            LOGP("file added: %s", g.file.string());
397            write_update_file(dataStream, q, g.file.string());
398            m++;
399        }
400        else {
401            // both files exist, check them
402            const FileState& f = oldSnapshot.valueAt(n);
403
404            int fd = open(g.file.string(), O_RDONLY);
405            if (fd < 0) {
406                // We can't open the file.  Don't report it as a delete either.  Let the
407                // server keep the old version.  Maybe they'll be able to deal with it
408                // on restore.
409                LOGP("Unable to open file %s - skipping", g.file.string());
410            } else {
411                g.s.crc32 = compute_crc32(fd);
412
413                LOGP("%s", q.string());
414                LOGP("  new: modTime=%d,%d mode=%04o size=%-3d crc32=0x%08x",
415                        f.modTime_sec, f.modTime_nsec, f.mode, f.size, f.crc32);
416                LOGP("  old: modTime=%d,%d mode=%04o size=%-3d crc32=0x%08x",
417                        g.s.modTime_sec, g.s.modTime_nsec, g.s.mode, g.s.size, g.s.crc32);
418                if (f.modTime_sec != g.s.modTime_sec || f.modTime_nsec != g.s.modTime_nsec
419                        || f.mode != g.s.mode || f.size != g.s.size || f.crc32 != g.s.crc32) {
420                    write_update_file(dataStream, fd, g.s.mode, p, g.file.string());
421                }
422
423                close(fd);
424            }
425            n++;
426            m++;
427        }
428    }
429
430    // these were deleted
431    while (n<N) {
432        dataStream->WriteEntityHeader(oldSnapshot.keyAt(n), -1);
433        n++;
434    }
435
436    // these were added
437    while (m<fileCount) {
438        const String8& q = newSnapshot.keyAt(m);
439        FileRec& g = newSnapshot.editValueAt(m);
440        write_update_file(dataStream, q, g.file.string());
441        m++;
442    }
443
444    err = write_snapshot_file(newSnapshotFD, newSnapshot);
445
446    return 0;
447}
448
449#define RESTORE_BUF_SIZE (8*1024)
450
451RestoreHelperBase::RestoreHelperBase()
452{
453    m_buf = malloc(RESTORE_BUF_SIZE);
454}
455
456RestoreHelperBase::~RestoreHelperBase()
457{
458    free(m_buf);
459}
460
461status_t
462RestoreHelperBase::WriteFile(const String8& filename, BackupDataReader* in)
463{
464    ssize_t err;
465    size_t dataSize;
466    String8 key;
467    int fd;
468    void* buf = m_buf;
469    ssize_t amt;
470    int mode;
471    int crc;
472    struct stat st;
473    FileRec r;
474
475    err = in->ReadEntityHeader(&key, &dataSize);
476    if (err != NO_ERROR) {
477        return err;
478    }
479
480    // Get the metadata block off the head of the file entity and use that to
481    // set up the output file
482    file_metadata_v1 metadata;
483    amt = in->ReadEntityData(&metadata, sizeof(metadata));
484    if (amt != sizeof(metadata)) {
485        LOGW("Could not read metadata for %s -- %ld / %s", filename.string(),
486                (long)amt, strerror(errno));
487        return EIO;
488    }
489    metadata.version = fromlel(metadata.version);
490    metadata.mode = fromlel(metadata.mode);
491    if (metadata.version > CURRENT_METADATA_VERSION) {
492        LOGW("Restoring file with unsupported metadata version %d (currently %d)",
493                metadata.version, CURRENT_METADATA_VERSION);
494    }
495    mode = metadata.mode;
496
497    // Write the file and compute the crc
498    crc = crc32(0L, Z_NULL, 0);
499    fd = open(filename.string(), O_CREAT|O_RDWR|O_TRUNC, mode);
500    if (fd == -1) {
501        LOGW("Could not open file %s -- %s", filename.string(), strerror(errno));
502        return errno;
503    }
504
505    while ((amt = in->ReadEntityData(buf, RESTORE_BUF_SIZE)) > 0) {
506        err = write(fd, buf, amt);
507        if (err != amt) {
508            close(fd);
509            LOGW("Error '%s' writing '%s'", strerror(errno), filename.string());
510            return errno;
511        }
512        crc = crc32(crc, (Bytef*)buf, amt);
513    }
514
515    close(fd);
516
517    // Record for the snapshot
518    err = stat(filename.string(), &st);
519    if (err != 0) {
520        LOGW("Error stating file that we just created %s", filename.string());
521        return errno;
522    }
523
524    r.file = filename;
525    r.deleted = false;
526    r.s.modTime_sec = st.st_mtime;
527    r.s.modTime_nsec = 0; // workaround sim breakage
528    //r.s.modTime_nsec = st.st_mtime_nsec;
529    r.s.mode = st.st_mode;
530    r.s.size = st.st_size;
531    r.s.crc32 = crc;
532
533    m_files.add(key, r);
534
535    return NO_ERROR;
536}
537
538status_t
539RestoreHelperBase::WriteSnapshot(int fd)
540{
541    return write_snapshot_file(fd, m_files);;
542}
543
544#if TEST_BACKUP_HELPERS
545
546#define SCRATCH_DIR "/data/backup_helper_test/"
547
548static int
549write_text_file(const char* path, const char* data)
550{
551    int amt;
552    int fd;
553    int len;
554
555    fd = creat(path, 0666);
556    if (fd == -1) {
557        fprintf(stderr, "creat %s failed\n", path);
558        return errno;
559    }
560
561    len = strlen(data);
562    amt = write(fd, data, len);
563    if (amt != len) {
564        fprintf(stderr, "error (%s) writing to file %s\n", strerror(errno), path);
565        return errno;
566    }
567
568    close(fd);
569
570    return 0;
571}
572
573static int
574compare_file(const char* path, const unsigned char* data, int len)
575{
576    int fd;
577    int amt;
578
579    fd = open(path, O_RDONLY);
580    if (fd == -1) {
581        fprintf(stderr, "compare_file error (%s) opening %s\n", strerror(errno), path);
582        return errno;
583    }
584
585    unsigned char* contents = (unsigned char*)malloc(len);
586    if (contents == NULL) {
587        fprintf(stderr, "malloc(%d) failed\n", len);
588        return ENOMEM;
589    }
590    StAutoFree _autoFree(contents);
591
592    bool sizesMatch = true;
593    amt = lseek(fd, 0, SEEK_END);
594    if (amt != len) {
595        fprintf(stderr, "compare_file file length should be %d, was %d\n", len, amt);
596        sizesMatch = false;
597    }
598    lseek(fd, 0, SEEK_SET);
599
600    int readLen = amt < len ? amt : len;
601    amt = read(fd, contents, readLen);
602    if (amt != readLen) {
603        fprintf(stderr, "compare_file read expected %d bytes but got %d\n", len, amt);
604    }
605
606    bool contentsMatch = true;
607    for (int i=0; i<readLen; i++) {
608        if (data[i] != contents[i]) {
609            if (contentsMatch) {
610                fprintf(stderr, "compare_file contents are different: (index, expected, actual)\n");
611                contentsMatch = false;
612            }
613            fprintf(stderr, "  [%-2d] %02x %02x\n", i, data[i], contents[i]);
614        }
615    }
616
617    return contentsMatch && sizesMatch ? 0 : 1;
618}
619
620int
621backup_helper_test_empty()
622{
623    int err;
624    int fd;
625    KeyedVector<String8,FileRec> snapshot;
626    const char* filename = SCRATCH_DIR "backup_helper_test_empty.snap";
627
628    system("rm -r " SCRATCH_DIR);
629    mkdir(SCRATCH_DIR, 0777);
630
631    // write
632    fd = creat(filename, 0666);
633    if (fd == -1) {
634        fprintf(stderr, "error creating %s\n", filename);
635        return 1;
636    }
637
638    err = write_snapshot_file(fd, snapshot);
639
640    close(fd);
641
642    if (err != 0) {
643        fprintf(stderr, "write_snapshot_file reported error %d (%s)\n", err, strerror(err));
644        return err;
645    }
646
647    static const unsigned char correct_data[] = {
648        0x53, 0x6e, 0x61, 0x70,  0x00, 0x00, 0x00, 0x00,
649        0x46, 0x69, 0x6c, 0x65,  0x10, 0x00, 0x00, 0x00
650    };
651
652    err = compare_file(filename, correct_data, sizeof(correct_data));
653    if (err != 0) {
654        return err;
655    }
656
657    // read
658    fd = open(filename, O_RDONLY);
659    if (fd == -1) {
660        fprintf(stderr, "error opening for read %s\n", filename);
661        return 1;
662    }
663
664    KeyedVector<String8,FileState> readSnapshot;
665    err = read_snapshot_file(fd, &readSnapshot);
666    if (err != 0) {
667        fprintf(stderr, "read_snapshot_file failed %d\n", err);
668        return err;
669    }
670
671    if (readSnapshot.size() != 0) {
672        fprintf(stderr, "readSnapshot should be length 0\n");
673        return 1;
674    }
675
676    return 0;
677}
678
679int
680backup_helper_test_four()
681{
682    int err;
683    int fd;
684    KeyedVector<String8,FileRec> snapshot;
685    const char* filename = SCRATCH_DIR "backup_helper_test_four.snap";
686
687    system("rm -r " SCRATCH_DIR);
688    mkdir(SCRATCH_DIR, 0777);
689
690    // write
691    fd = creat(filename, 0666);
692    if (fd == -1) {
693        fprintf(stderr, "error opening %s\n", filename);
694        return 1;
695    }
696
697    String8 filenames[4];
698    FileState states[4];
699    FileRec r;
700    r.deleted = false;
701
702    states[0].modTime_sec = 0xfedcba98;
703    states[0].modTime_nsec = 0xdeadbeef;
704    states[0].mode = 0777; // decimal 511, hex 0x000001ff
705    states[0].size = 0xababbcbc;
706    states[0].crc32 = 0x12345678;
707    states[0].nameLen = -12;
708    r.s = states[0];
709    filenames[0] = String8("bytes_of_padding");
710    snapshot.add(filenames[0], r);
711
712    states[1].modTime_sec = 0x93400031;
713    states[1].modTime_nsec = 0xdeadbeef;
714    states[1].mode = 0666; // decimal 438, hex 0x000001b6
715    states[1].size = 0x88557766;
716    states[1].crc32 = 0x22334422;
717    states[1].nameLen = -1;
718    r.s = states[1];
719    filenames[1] = String8("bytes_of_padding3");
720    snapshot.add(filenames[1], r);
721
722    states[2].modTime_sec = 0x33221144;
723    states[2].modTime_nsec = 0xdeadbeef;
724    states[2].mode = 0744; // decimal 484, hex 0x000001e4
725    states[2].size = 0x11223344;
726    states[2].crc32 = 0x01122334;
727    states[2].nameLen = 0;
728    r.s = states[2];
729    filenames[2] = String8("bytes_of_padding_2");
730    snapshot.add(filenames[2], r);
731
732    states[3].modTime_sec = 0x33221144;
733    states[3].modTime_nsec = 0xdeadbeef;
734    states[3].mode = 0755; // decimal 493, hex 0x000001ed
735    states[3].size = 0x11223344;
736    states[3].crc32 = 0x01122334;
737    states[3].nameLen = 0;
738    r.s = states[3];
739    filenames[3] = String8("bytes_of_padding__1");
740    snapshot.add(filenames[3], r);
741
742    err = write_snapshot_file(fd, snapshot);
743
744    close(fd);
745
746    if (err != 0) {
747        fprintf(stderr, "write_snapshot_file reported error %d (%s)\n", err, strerror(err));
748        return err;
749    }
750
751    static const unsigned char correct_data[] = {
752        // header
753        0x53, 0x6e, 0x61, 0x70,  0x04, 0x00, 0x00, 0x00,
754        0x46, 0x69, 0x6c, 0x65,  0xbc, 0x00, 0x00, 0x00,
755
756        // bytes_of_padding
757        0x98, 0xba, 0xdc, 0xfe,  0xef, 0xbe, 0xad, 0xde,
758        0xff, 0x01, 0x00, 0x00,  0xbc, 0xbc, 0xab, 0xab,
759        0x78, 0x56, 0x34, 0x12,  0x10, 0x00, 0x00, 0x00,
760        0x62, 0x79, 0x74, 0x65,  0x73, 0x5f, 0x6f, 0x66,
761        0x5f, 0x70, 0x61, 0x64,  0x64, 0x69, 0x6e, 0x67,
762
763        // bytes_of_padding3
764        0x31, 0x00, 0x40, 0x93,  0xef, 0xbe, 0xad, 0xde,
765        0xb6, 0x01, 0x00, 0x00,  0x66, 0x77, 0x55, 0x88,
766        0x22, 0x44, 0x33, 0x22,  0x11, 0x00, 0x00, 0x00,
767        0x62, 0x79, 0x74, 0x65,  0x73, 0x5f, 0x6f, 0x66,
768        0x5f, 0x70, 0x61, 0x64,  0x64, 0x69, 0x6e, 0x67,
769        0x33, 0xab, 0xab, 0xab,
770
771        // bytes of padding2
772        0x44, 0x11, 0x22, 0x33,  0xef, 0xbe, 0xad, 0xde,
773        0xe4, 0x01, 0x00, 0x00,  0x44, 0x33, 0x22, 0x11,
774        0x34, 0x23, 0x12, 0x01,  0x12, 0x00, 0x00, 0x00,
775        0x62, 0x79, 0x74, 0x65,  0x73, 0x5f, 0x6f, 0x66,
776        0x5f, 0x70, 0x61, 0x64,  0x64, 0x69, 0x6e, 0x67,
777        0x5f, 0x32, 0xab, 0xab,
778
779        // bytes of padding3
780        0x44, 0x11, 0x22, 0x33,  0xef, 0xbe, 0xad, 0xde,
781        0xed, 0x01, 0x00, 0x00,  0x44, 0x33, 0x22, 0x11,
782        0x34, 0x23, 0x12, 0x01,  0x13, 0x00, 0x00, 0x00,
783        0x62, 0x79, 0x74, 0x65,  0x73, 0x5f, 0x6f, 0x66,
784        0x5f, 0x70, 0x61, 0x64,  0x64, 0x69, 0x6e, 0x67,
785        0x5f, 0x5f, 0x31, 0xab
786    };
787
788    err = compare_file(filename, correct_data, sizeof(correct_data));
789    if (err != 0) {
790        return err;
791    }
792
793    // read
794    fd = open(filename, O_RDONLY);
795    if (fd == -1) {
796        fprintf(stderr, "error opening for read %s\n", filename);
797        return 1;
798    }
799
800
801    KeyedVector<String8,FileState> readSnapshot;
802    err = read_snapshot_file(fd, &readSnapshot);
803    if (err != 0) {
804        fprintf(stderr, "read_snapshot_file failed %d\n", err);
805        return err;
806    }
807
808    if (readSnapshot.size() != 4) {
809        fprintf(stderr, "readSnapshot should be length 4 is %d\n", readSnapshot.size());
810        return 1;
811    }
812
813    bool matched = true;
814    for (size_t i=0; i<readSnapshot.size(); i++) {
815        const String8& name = readSnapshot.keyAt(i);
816        const FileState state = readSnapshot.valueAt(i);
817
818        if (name != filenames[i] || states[i].modTime_sec != state.modTime_sec
819                || states[i].modTime_nsec != state.modTime_nsec || states[i].mode != state.mode
820                || states[i].size != state.size || states[i].crc32 != states[i].crc32) {
821            fprintf(stderr, "state %d expected={%d/%d, 0x%08x, %04o, 0x%08x, %3d} '%s'\n"
822                            "          actual={%d/%d, 0x%08x, %04o, 0x%08x, %3d} '%s'\n", i,
823                    states[i].modTime_sec, states[i].modTime_nsec, states[i].mode, states[i].size,
824                    states[i].crc32, name.length(), filenames[i].string(),
825                    state.modTime_sec, state.modTime_nsec, state.mode, state.size, state.crc32,
826                    state.nameLen, name.string());
827            matched = false;
828        }
829    }
830
831    return matched ? 0 : 1;
832}
833
834// hexdump -v -e '"    " 8/1 " 0x%02x," "\n"' data_writer.data
835const unsigned char DATA_GOLDEN_FILE[] = {
836     0x44, 0x61, 0x74, 0x61, 0x0b, 0x00, 0x00, 0x00,
837     0x0c, 0x00, 0x00, 0x00, 0x6e, 0x6f, 0x5f, 0x70,
838     0x61, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x00,
839     0x6e, 0x6f, 0x5f, 0x70, 0x61, 0x64, 0x64, 0x69,
840     0x6e, 0x67, 0x5f, 0x00, 0x44, 0x61, 0x74, 0x61,
841     0x0c, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00,
842     0x70, 0x61, 0x64, 0x64, 0x65, 0x64, 0x5f, 0x74,
843     0x6f, 0x5f, 0x5f, 0x33, 0x00, 0xbc, 0xbc, 0xbc,
844     0x70, 0x61, 0x64, 0x64, 0x65, 0x64, 0x5f, 0x74,
845     0x6f, 0x5f, 0x5f, 0x33, 0x00, 0xbc, 0xbc, 0xbc,
846     0x44, 0x61, 0x74, 0x61, 0x0d, 0x00, 0x00, 0x00,
847     0x0e, 0x00, 0x00, 0x00, 0x70, 0x61, 0x64, 0x64,
848     0x65, 0x64, 0x5f, 0x74, 0x6f, 0x5f, 0x32, 0x5f,
849     0x5f, 0x00, 0xbc, 0xbc, 0x70, 0x61, 0x64, 0x64,
850     0x65, 0x64, 0x5f, 0x74, 0x6f, 0x5f, 0x32, 0x5f,
851     0x5f, 0x00, 0xbc, 0xbc, 0x44, 0x61, 0x74, 0x61,
852     0x0a, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00,
853     0x70, 0x61, 0x64, 0x64, 0x65, 0x64, 0x5f, 0x74,
854     0x6f, 0x31, 0x00, 0xbc, 0x70, 0x61, 0x64, 0x64,
855     0x65, 0x64, 0x5f, 0x74, 0x6f, 0x31, 0x00
856
857};
858const int DATA_GOLDEN_FILE_SIZE = sizeof(DATA_GOLDEN_FILE);
859
860static int
861test_write_header_and_entity(BackupDataWriter& writer, const char* str)
862{
863    int err;
864    String8 text(str);
865
866    err = writer.WriteEntityHeader(text, text.length()+1);
867    if (err != 0) {
868        fprintf(stderr, "WriteEntityHeader failed with %s\n", strerror(err));
869        return err;
870    }
871
872    err = writer.WriteEntityData(text.string(), text.length()+1);
873    if (err != 0) {
874        fprintf(stderr, "write failed for data '%s'\n", text.string());
875        return errno;
876    }
877
878    return err;
879}
880
881int
882backup_helper_test_data_writer()
883{
884    int err;
885    int fd;
886    const char* filename = SCRATCH_DIR "data_writer.data";
887
888    system("rm -r " SCRATCH_DIR);
889    mkdir(SCRATCH_DIR, 0777);
890    mkdir(SCRATCH_DIR "data", 0777);
891
892    fd = creat(filename, 0666);
893    if (fd == -1) {
894        fprintf(stderr, "error creating: %s\n", strerror(errno));
895        return errno;
896    }
897
898    BackupDataWriter writer(fd);
899
900    err = 0;
901    err |= test_write_header_and_entity(writer, "no_padding_");
902    err |= test_write_header_and_entity(writer, "padded_to__3");
903    err |= test_write_header_and_entity(writer, "padded_to_2__");
904    err |= test_write_header_and_entity(writer, "padded_to1");
905
906    close(fd);
907
908    err = compare_file(filename, DATA_GOLDEN_FILE, DATA_GOLDEN_FILE_SIZE);
909    if (err != 0) {
910        return err;
911    }
912
913    return err;
914}
915
916int
917test_read_header_and_entity(BackupDataReader& reader, const char* str)
918{
919    int err;
920    int bufSize = strlen(str)+1;
921    char* buf = (char*)malloc(bufSize);
922    StAutoFree _autoFree(buf);
923    String8 string;
924    int cookie = 0x11111111;
925    size_t actualSize;
926    bool done;
927    int type;
928    ssize_t nRead;
929
930    // printf("\n\n---------- test_read_header_and_entity -- %s\n\n", str);
931
932    err = reader.ReadNextHeader(&done, &type);
933    if (done) {
934        fprintf(stderr, "should not be done yet\n");
935        goto finished;
936    }
937    if (err != 0) {
938        fprintf(stderr, "ReadNextHeader (for app header) failed with %s\n", strerror(err));
939        goto finished;
940    }
941    if (type != BACKUP_HEADER_ENTITY_V1) {
942        err = EINVAL;
943        fprintf(stderr, "type=0x%08x expected 0x%08x\n", type, BACKUP_HEADER_ENTITY_V1);
944    }
945
946    err = reader.ReadEntityHeader(&string, &actualSize);
947    if (err != 0) {
948        fprintf(stderr, "ReadEntityHeader failed with %s\n", strerror(err));
949        goto finished;
950    }
951    if (string != str) {
952        fprintf(stderr, "ReadEntityHeader expected key '%s' got '%s'\n", str, string.string());
953        err = EINVAL;
954        goto finished;
955    }
956    if ((int)actualSize != bufSize) {
957        fprintf(stderr, "ReadEntityHeader expected dataSize 0x%08x got 0x%08x\n", bufSize,
958                actualSize);
959        err = EINVAL;
960        goto finished;
961    }
962
963    nRead = reader.ReadEntityData(buf, bufSize);
964    if (nRead < 0) {
965        err = reader.Status();
966        fprintf(stderr, "ReadEntityData failed with %s\n", strerror(err));
967        goto finished;
968    }
969
970    if (0 != memcmp(buf, str, bufSize)) {
971        fprintf(stderr, "ReadEntityData expected '%s' but got something starting with "
972                "%02x %02x %02x %02x  '%c%c%c%c'\n", str, buf[0], buf[1], buf[2], buf[3],
973                buf[0], buf[1], buf[2], buf[3]);
974        err = EINVAL;
975        goto finished;
976    }
977
978    // The next read will confirm whether it got the right amount of data.
979
980finished:
981    if (err != NO_ERROR) {
982        fprintf(stderr, "test_read_header_and_entity failed with %s\n", strerror(err));
983    }
984    return err;
985}
986
987int
988backup_helper_test_data_reader()
989{
990    int err;
991    int fd;
992    const char* filename = SCRATCH_DIR "data_reader.data";
993
994    system("rm -r " SCRATCH_DIR);
995    mkdir(SCRATCH_DIR, 0777);
996    mkdir(SCRATCH_DIR "data", 0777);
997
998    fd = creat(filename, 0666);
999    if (fd == -1) {
1000        fprintf(stderr, "error creating: %s\n", strerror(errno));
1001        return errno;
1002    }
1003
1004    err = write(fd, DATA_GOLDEN_FILE, DATA_GOLDEN_FILE_SIZE);
1005    if (err != DATA_GOLDEN_FILE_SIZE) {
1006        fprintf(stderr, "Error \"%s\" writing golden file %s\n", strerror(errno), filename);
1007        return errno;
1008    }
1009
1010    close(fd);
1011
1012    fd = open(filename, O_RDONLY);
1013    if (fd == -1) {
1014        fprintf(stderr, "Error \"%s\" opening golden file %s for read\n", strerror(errno),
1015                filename);
1016        return errno;
1017    }
1018
1019    {
1020        BackupDataReader reader(fd);
1021
1022        err = 0;
1023
1024        if (err == NO_ERROR) {
1025            err = test_read_header_and_entity(reader, "no_padding_");
1026        }
1027
1028        if (err == NO_ERROR) {
1029            err = test_read_header_and_entity(reader, "padded_to__3");
1030        }
1031
1032        if (err == NO_ERROR) {
1033            err = test_read_header_and_entity(reader, "padded_to_2__");
1034        }
1035
1036        if (err == NO_ERROR) {
1037            err = test_read_header_and_entity(reader, "padded_to1");
1038        }
1039    }
1040
1041    close(fd);
1042
1043    return err;
1044}
1045
1046static int
1047get_mod_time(const char* filename, struct timeval times[2])
1048{
1049    int err;
1050    struct stat64 st;
1051    err = stat64(filename, &st);
1052    if (err != 0) {
1053        fprintf(stderr, "stat '%s' failed: %s\n", filename, strerror(errno));
1054        return errno;
1055    }
1056    times[0].tv_sec = st.st_atime;
1057    times[1].tv_sec = st.st_mtime;
1058
1059    // If st_atime is a macro then struct stat64 uses struct timespec
1060    // to store the access and modif time values and typically
1061    // st_*time_nsec is not defined. In glibc, this is controlled by
1062    // __USE_MISC.
1063#ifdef __USE_MISC
1064#if !defined(st_atime) || defined(st_atime_nsec)
1065#error "Check if this __USE_MISC conditional is still needed."
1066#endif
1067    times[0].tv_usec = st.st_atim.tv_nsec / 1000;
1068    times[1].tv_usec = st.st_mtim.tv_nsec / 1000;
1069#else
1070    times[0].tv_usec = st.st_atime_nsec / 1000;
1071    times[1].tv_usec = st.st_mtime_nsec / 1000;
1072#endif
1073
1074    return 0;
1075}
1076
1077int
1078backup_helper_test_files()
1079{
1080    int err;
1081    int oldSnapshotFD;
1082    int dataStreamFD;
1083    int newSnapshotFD;
1084
1085    system("rm -r " SCRATCH_DIR);
1086    mkdir(SCRATCH_DIR, 0777);
1087    mkdir(SCRATCH_DIR "data", 0777);
1088
1089    write_text_file(SCRATCH_DIR "data/b", "b\nbb\n");
1090    write_text_file(SCRATCH_DIR "data/c", "c\ncc\n");
1091    write_text_file(SCRATCH_DIR "data/d", "d\ndd\n");
1092    write_text_file(SCRATCH_DIR "data/e", "e\nee\n");
1093    write_text_file(SCRATCH_DIR "data/f", "f\nff\n");
1094    write_text_file(SCRATCH_DIR "data/h", "h\nhh\n");
1095
1096    char const* files_before[] = {
1097        SCRATCH_DIR "data/b",
1098        SCRATCH_DIR "data/c",
1099        SCRATCH_DIR "data/d",
1100        SCRATCH_DIR "data/e",
1101        SCRATCH_DIR "data/f"
1102    };
1103
1104    char const* keys_before[] = {
1105        "data/b",
1106        "data/c",
1107        "data/d",
1108        "data/e",
1109        "data/f"
1110    };
1111
1112    dataStreamFD = creat(SCRATCH_DIR "1.data", 0666);
1113    if (dataStreamFD == -1) {
1114        fprintf(stderr, "error creating: %s\n", strerror(errno));
1115        return errno;
1116    }
1117
1118    newSnapshotFD = creat(SCRATCH_DIR "before.snap", 0666);
1119    if (newSnapshotFD == -1) {
1120        fprintf(stderr, "error creating: %s\n", strerror(errno));
1121        return errno;
1122    }
1123
1124    {
1125        BackupDataWriter dataStream(dataStreamFD);
1126
1127        err = back_up_files(-1, &dataStream, newSnapshotFD, files_before, keys_before, 5);
1128        if (err != 0) {
1129            return err;
1130        }
1131    }
1132
1133    close(dataStreamFD);
1134    close(newSnapshotFD);
1135
1136    sleep(3);
1137
1138    struct timeval d_times[2];
1139    struct timeval e_times[2];
1140
1141    err = get_mod_time(SCRATCH_DIR "data/d", d_times);
1142    err |= get_mod_time(SCRATCH_DIR "data/e", e_times);
1143    if (err != 0) {
1144        return err;
1145    }
1146
1147    write_text_file(SCRATCH_DIR "data/a", "a\naa\n");
1148    unlink(SCRATCH_DIR "data/c");
1149    write_text_file(SCRATCH_DIR "data/c", "c\ncc\n");
1150    write_text_file(SCRATCH_DIR "data/d", "dd\ndd\n");
1151    utimes(SCRATCH_DIR "data/d", d_times);
1152    write_text_file(SCRATCH_DIR "data/e", "z\nzz\n");
1153    utimes(SCRATCH_DIR "data/e", e_times);
1154    write_text_file(SCRATCH_DIR "data/g", "g\ngg\n");
1155    unlink(SCRATCH_DIR "data/f");
1156
1157    char const* files_after[] = {
1158        SCRATCH_DIR "data/a", // added
1159        SCRATCH_DIR "data/b", // same
1160        SCRATCH_DIR "data/c", // different mod time
1161        SCRATCH_DIR "data/d", // different size (same mod time)
1162        SCRATCH_DIR "data/e", // different contents (same mod time, same size)
1163        SCRATCH_DIR "data/g"  // added
1164    };
1165
1166    char const* keys_after[] = {
1167        "data/a", // added
1168        "data/b", // same
1169        "data/c", // different mod time
1170        "data/d", // different size (same mod time)
1171        "data/e", // different contents (same mod time, same size)
1172        "data/g"  // added
1173    };
1174
1175    oldSnapshotFD = open(SCRATCH_DIR "before.snap", O_RDONLY);
1176    if (oldSnapshotFD == -1) {
1177        fprintf(stderr, "error opening: %s\n", strerror(errno));
1178        return errno;
1179    }
1180
1181    dataStreamFD = creat(SCRATCH_DIR "2.data", 0666);
1182    if (dataStreamFD == -1) {
1183        fprintf(stderr, "error creating: %s\n", strerror(errno));
1184        return errno;
1185    }
1186
1187    newSnapshotFD = creat(SCRATCH_DIR "after.snap", 0666);
1188    if (newSnapshotFD == -1) {
1189        fprintf(stderr, "error creating: %s\n", strerror(errno));
1190        return errno;
1191    }
1192
1193    {
1194        BackupDataWriter dataStream(dataStreamFD);
1195
1196        err = back_up_files(oldSnapshotFD, &dataStream, newSnapshotFD, files_after, keys_after, 6);
1197        if (err != 0) {
1198            return err;
1199        }
1200}
1201
1202    close(oldSnapshotFD);
1203    close(dataStreamFD);
1204    close(newSnapshotFD);
1205
1206    return 0;
1207}
1208
1209int
1210backup_helper_test_null_base()
1211{
1212    int err;
1213    int oldSnapshotFD;
1214    int dataStreamFD;
1215    int newSnapshotFD;
1216
1217    system("rm -r " SCRATCH_DIR);
1218    mkdir(SCRATCH_DIR, 0777);
1219    mkdir(SCRATCH_DIR "data", 0777);
1220
1221    write_text_file(SCRATCH_DIR "data/a", "a\naa\n");
1222
1223    char const* files[] = {
1224        SCRATCH_DIR "data/a",
1225    };
1226
1227    char const* keys[] = {
1228        "a",
1229    };
1230
1231    dataStreamFD = creat(SCRATCH_DIR "null_base.data", 0666);
1232    if (dataStreamFD == -1) {
1233        fprintf(stderr, "error creating: %s\n", strerror(errno));
1234        return errno;
1235    }
1236
1237    newSnapshotFD = creat(SCRATCH_DIR "null_base.snap", 0666);
1238    if (newSnapshotFD == -1) {
1239        fprintf(stderr, "error creating: %s\n", strerror(errno));
1240        return errno;
1241    }
1242
1243    {
1244        BackupDataWriter dataStream(dataStreamFD);
1245
1246        err = back_up_files(-1, &dataStream, newSnapshotFD, files, keys, 1);
1247        if (err != 0) {
1248            return err;
1249        }
1250    }
1251
1252    close(dataStreamFD);
1253    close(newSnapshotFD);
1254
1255    return 0;
1256}
1257
1258int
1259backup_helper_test_missing_file()
1260{
1261    int err;
1262    int oldSnapshotFD;
1263    int dataStreamFD;
1264    int newSnapshotFD;
1265
1266    system("rm -r " SCRATCH_DIR);
1267    mkdir(SCRATCH_DIR, 0777);
1268    mkdir(SCRATCH_DIR "data", 0777);
1269
1270    write_text_file(SCRATCH_DIR "data/b", "b\nbb\n");
1271
1272    char const* files[] = {
1273        SCRATCH_DIR "data/a",
1274        SCRATCH_DIR "data/b",
1275        SCRATCH_DIR "data/c",
1276    };
1277
1278    char const* keys[] = {
1279        "a",
1280        "b",
1281        "c",
1282    };
1283
1284    dataStreamFD = creat(SCRATCH_DIR "null_base.data", 0666);
1285    if (dataStreamFD == -1) {
1286        fprintf(stderr, "error creating: %s\n", strerror(errno));
1287        return errno;
1288    }
1289
1290    newSnapshotFD = creat(SCRATCH_DIR "null_base.snap", 0666);
1291    if (newSnapshotFD == -1) {
1292        fprintf(stderr, "error creating: %s\n", strerror(errno));
1293        return errno;
1294    }
1295
1296    {
1297        BackupDataWriter dataStream(dataStreamFD);
1298
1299        err = back_up_files(-1, &dataStream, newSnapshotFD, files, keys, 1);
1300        if (err != 0) {
1301            return err;
1302        }
1303    }
1304
1305    close(dataStreamFD);
1306    close(newSnapshotFD);
1307
1308    return 0;
1309}
1310
1311
1312#endif // TEST_BACKUP_HELPERS
1313
1314}
1315