BackupHelpers.cpp revision ce88cb15b52998e16c3ba548a4ec49117a835e21 
1/*
2 * Copyright (C) 2009 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define LOG_TAG "file_backup_helper"
18
19#include <utils/BackupHelpers.h>
20
21#include <utils/KeyedVector.h>
22#include <utils/ByteOrder.h>
23#include <utils/String8.h>
24
25#include <errno.h>
26#include <sys/types.h>
27#include <sys/uio.h>
28#include <sys/stat.h>
29#include <sys/time.h>  // for utimes
30#include <stdio.h>
31#include <stdlib.h>
32#include <unistd.h>
33#include <utime.h>
34#include <fcntl.h>
35#include <zlib.h>
36
37#include <cutils/log.h>
38
39namespace android {
40
41#define MAGIC0 0x70616e53 // Snap
42#define MAGIC1 0x656c6946 // File
43
44#if 1 // TEST_BACKUP_HELPERS
45#define LOGP(f, x...) printf(f "\n", x)
46#else
47#define LOGP(x...) LOGD(x)
48#endif
49
50struct SnapshotHeader {
51    int magic0;
52    int fileCount;
53    int magic1;
54    int totalSize;
55};
56
57struct FileState {
58    int modTime_sec;
59    int modTime_nsec;
60    int size;
61    int crc32;
62    int nameLen;
63};
64
65struct FileRec {
66    char const* file; // this object does not own this string
67    bool deleted;
68    FileState s;
69};
70
71const static int ROUND_UP[4] = { 0, 3, 2, 1 };
72
73static inline int
74round_up(int n)
75{
76    return n + ROUND_UP[n % 4];
77}
78
79static int
80read_snapshot_file(int fd, KeyedVector<String8,FileState>* snapshot)
81{
82    int bytesRead = 0;
83    int amt;
84    SnapshotHeader header;
85
86    amt = read(fd, &header, sizeof(header));
87    if (amt != sizeof(header)) {
88        return errno;
89    }
90    bytesRead += amt;
91
92    if (header.magic0 != MAGIC0 || header.magic1 != MAGIC1) {
93        LOGW("read_snapshot_file header.magic0=0x%08x magic1=0x%08x", header.magic0, header.magic1);
94        return 1;
95    }
96
97    for (int i=0; i<header.fileCount; i++) {
98        FileState file;
99        char filenameBuf[128];
100
101        amt = read(fd, &file, sizeof(FileState));
102        if (amt != sizeof(FileState)) {
103            LOGW("read_snapshot_file FileState truncated/error with read at %d bytes\n", bytesRead);
104            return 1;
105        }
106        bytesRead += amt;
107
108        // filename is not NULL terminated, but it is padded
109        int nameBufSize = round_up(file.nameLen);
110        char* filename = nameBufSize <= (int)sizeof(filenameBuf)
111                ? filenameBuf
112                : (char*)malloc(nameBufSize);
113        amt = read(fd, filename, nameBufSize);
114        if (amt == nameBufSize) {
115            snapshot->add(String8(filename, file.nameLen), file);
116        }
117        bytesRead += amt;
118        if (filename != filenameBuf) {
119            free(filename);
120        }
121        if (amt != nameBufSize) {
122            LOGW("read_snapshot_file filename truncated/error with read at %d bytes\n", bytesRead);
123            return 1;
124        }
125    }
126
127    if (header.totalSize != bytesRead) {
128        LOGW("read_snapshot_file length mismatch: header.totalSize=%d bytesRead=%d\n",
129                header.totalSize, bytesRead);
130        return 1;
131    }
132
133    return 0;
134}
135
136static int
137write_snapshot_file(int fd, const KeyedVector<String8,FileRec>& snapshot)
138{
139    int fileCount = 0;
140    int bytesWritten = sizeof(SnapshotHeader);
141    // preflight size
142    const int N = snapshot.size();
143    for (int i=0; i<N; i++) {
144        const FileRec& g = snapshot.valueAt(i);
145        if (!g.deleted) {
146            const String8& name = snapshot.keyAt(i);
147            bytesWritten += sizeof(FileState) + round_up(name.length());
148            fileCount++;
149        }
150    }
151
152    LOGP("write_snapshot_file fd=%d\n", fd);
153
154    int amt;
155    SnapshotHeader header = { MAGIC0, fileCount, MAGIC1, bytesWritten };
156
157    amt = write(fd, &header, sizeof(header));
158    if (amt != sizeof(header)) {
159        LOGW("write_snapshot_file error writing header %s", strerror(errno));
160        return errno;
161    }
162
163    for (int i=0; i<N; i++) {
164        FileRec r = snapshot.valueAt(i);
165        if (!r.deleted) {
166            const String8& name = snapshot.keyAt(i);
167            int nameLen = r.s.nameLen = name.length();
168
169            amt = write(fd, &r.s, sizeof(FileState));
170            if (amt != sizeof(FileState)) {
171                LOGW("write_snapshot_file error writing header %s", strerror(errno));
172                return 1;
173            }
174
175            // filename is not NULL terminated, but it is padded
176            amt = write(fd, name.string(), nameLen);
177            if (amt != nameLen) {
178                LOGW("write_snapshot_file error writing filename %s", strerror(errno));
179                return 1;
180            }
181            int paddingLen = ROUND_UP[nameLen % 4];
182            if (paddingLen != 0) {
183                int padding = 0xabababab;
184                amt = write(fd, &padding, paddingLen);
185                if (amt != paddingLen) {
186                    LOGW("write_snapshot_file error writing %d bytes of filename padding %s",
187                            paddingLen, strerror(errno));
188                    return 1;
189                }
190            }
191        }
192    }
193
194    return 0;
195}
196
197static int
198write_delete_file(BackupDataWriter* dataStream, const String8& key)
199{
200    LOGP("write_delete_file %s\n", key.string());
201    return dataStream->WriteEntityHeader(key, -1);
202}
203
204static int
205write_update_file(BackupDataWriter* dataStream, int fd, const String8& key,
206        char const* realFilename)
207{
208    LOGP("write_update_file %s (%s)\n", realFilename, key.string());
209
210    const int bufsize = 4*1024;
211    int err;
212    int amt;
213    int fileSize;
214    int bytesLeft;
215
216    char* buf = (char*)malloc(bufsize);
217    int crc = crc32(0L, Z_NULL, 0);
218
219
220    bytesLeft = fileSize = lseek(fd, 0, SEEK_END);
221    lseek(fd, 0, SEEK_SET);
222
223    err = dataStream->WriteEntityHeader(key, bytesLeft);
224    if (err != 0) {
225        return err;
226    }
227
228    while ((amt = read(fd, buf, bufsize)) != 0 && bytesLeft > 0) {
229        bytesLeft -= amt;
230        if (bytesLeft < 0) {
231            amt += bytesLeft; // Plus a negative is minus.  Don't write more than we promised.
232        }
233        err = dataStream->WriteEntityData(buf, amt);
234        if (err != 0) {
235            return err;
236        }
237    }
238    if (bytesLeft != 0) {
239        if (bytesLeft > 0) {
240            // Pad out the space we promised in the buffer.  We can't corrupt the buffer,
241            // even though the data we're sending is probably bad.
242            memset(buf, 0, bufsize);
243            while (bytesLeft > 0) {
244                amt = bytesLeft < bufsize ? bytesLeft : bufsize;
245                bytesLeft -= amt;
246                err = dataStream->WriteEntityData(buf, amt);
247                if (err != 0) {
248                    return err;
249                }
250            }
251        }
252        LOGE("write_update_file size mismatch for %s. expected=%d actual=%d."
253                " You aren't doing proper locking!", realFilename, fileSize, fileSize-bytesLeft);
254    }
255
256    free(buf);
257
258    return NO_ERROR;
259}
260
261static int
262write_update_file(BackupDataWriter* dataStream, const String8& key, char const* realFilename)
263{
264    int err;
265    int fd = open(realFilename, O_RDONLY);
266    if (fd == -1) {
267        return errno;
268    }
269    err = write_update_file(dataStream, fd, key, realFilename);
270    close(fd);
271    return err;
272}
273
274static int
275compute_crc32(int fd)
276{
277    const int bufsize = 4*1024;
278    int amt;
279
280    char* buf = (char*)malloc(bufsize);
281    int crc = crc32(0L, Z_NULL, 0);
282
283    lseek(fd, 0, SEEK_SET);
284
285    while ((amt = read(fd, buf, bufsize)) != 0) {
286        crc = crc32(crc, (Bytef*)buf, amt);
287    }
288
289    free(buf);
290
291    return crc;
292}
293
294int
295back_up_files(int oldSnapshotFD, BackupDataWriter* dataStream, int newSnapshotFD,
296        char const* const* files, char const* const* keys, int fileCount)
297{
298    int err;
299    KeyedVector<String8,FileState> oldSnapshot;
300    KeyedVector<String8,FileRec> newSnapshot;
301
302    if (oldSnapshotFD != -1) {
303        err = read_snapshot_file(oldSnapshotFD, &oldSnapshot);
304        if (err != 0) {
305            // On an error, treat this as a full backup.
306            oldSnapshot.clear();
307        }
308    }
309
310    for (int i=0; i<fileCount; i++) {
311        String8 key(keys[i]);
312        FileRec r;
313        char const* file = r.file = files[i];
314        struct stat st;
315
316        err = stat(file, &st);
317        if (err != 0) {
318            LOGW("Error stating file %s", file);
319            r.deleted = true;
320        } else {
321            r.deleted = false;
322            r.s.modTime_sec = st.st_mtime;
323            r.s.modTime_nsec = 0; // workaround sim breakage
324            //r.s.modTime_nsec = st.st_mtime_nsec;
325            r.s.size = st.st_size;
326            // we compute the crc32 later down below, when we already have the file open.
327
328            if (newSnapshot.indexOfKey(key) >= 0) {
329                LOGP("back_up_files key already in use '%s'", key.string());
330                return -1;
331            }
332        }
333        newSnapshot.add(key, r);
334    }
335
336    int n = 0;
337    int N = oldSnapshot.size();
338    int m = 0;
339
340    while (n<N && m<fileCount) {
341        const String8& p = oldSnapshot.keyAt(n);
342        const String8& q = newSnapshot.keyAt(m);
343        FileRec& g = newSnapshot.editValueAt(m);
344        int cmp = p.compare(q);
345        if (g.deleted || cmp < 0) {
346            // file removed
347            LOGP("file removed: %s", p.string());
348            g.deleted = true; // They didn't mention the file, but we noticed that it's gone.
349            dataStream->WriteEntityHeader(p, -1);
350            n++;
351        }
352        else if (cmp > 0) {
353            // file added
354            LOGP("file added: %s", g.file);
355            write_update_file(dataStream, q, g.file);
356            m++;
357        }
358        else {
359            // both files exist, check them
360            const FileState& f = oldSnapshot.valueAt(n);
361
362            int fd = open(g.file, O_RDONLY);
363            if (fd < 0) {
364                // We can't open the file.  Don't report it as a delete either.  Let the
365                // server keep the old version.  Maybe they'll be able to deal with it
366                // on restore.
367                LOGP("Unable to open file %s - skipping", g.file);
368            } else {
369                g.s.crc32 = compute_crc32(fd);
370
371                LOGP("%s", q.string());
372                LOGP("  new: modTime=%d,%d size=%-3d crc32=0x%08x",
373                        f.modTime_sec, f.modTime_nsec, f.size, f.crc32);
374                LOGP("  old: modTime=%d,%d size=%-3d crc32=0x%08x",
375                        g.s.modTime_sec, g.s.modTime_nsec, g.s.size, g.s.crc32);
376                if (f.modTime_sec != g.s.modTime_sec || f.modTime_nsec != g.s.modTime_nsec
377                        || f.size != g.s.size || f.crc32 != g.s.crc32) {
378                    write_update_file(dataStream, fd, p, g.file);
379                }
380
381                close(fd);
382            }
383            n++;
384            m++;
385        }
386    }
387
388    // these were deleted
389    while (n<N) {
390        dataStream->WriteEntityHeader(oldSnapshot.keyAt(n), -1);
391        n++;
392    }
393
394    // these were added
395    while (m<fileCount) {
396        const String8& q = newSnapshot.keyAt(m);
397        FileRec& g = newSnapshot.editValueAt(m);
398        write_update_file(dataStream, q, g.file);
399        m++;
400    }
401
402    err = write_snapshot_file(newSnapshotFD, newSnapshot);
403
404    return 0;
405}
406
407#if TEST_BACKUP_HELPERS
408
409#define SCRATCH_DIR "/data/backup_helper_test/"
410
411static int
412write_text_file(const char* path, const char* data)
413{
414    int amt;
415    int fd;
416    int len;
417
418    fd = creat(path, 0666);
419    if (fd == -1) {
420        fprintf(stderr, "creat %s failed\n", path);
421        return errno;
422    }
423
424    len = strlen(data);
425    amt = write(fd, data, len);
426    if (amt != len) {
427        fprintf(stderr, "error (%s) writing to file %s\n", strerror(errno), path);
428        return errno;
429    }
430
431    close(fd);
432
433    return 0;
434}
435
436static int
437compare_file(const char* path, const unsigned char* data, int len)
438{
439    int fd;
440    int amt;
441
442    fd = open(path, O_RDONLY);
443    if (fd == -1) {
444        fprintf(stderr, "compare_file error (%s) opening %s\n", strerror(errno), path);
445        return errno;
446    }
447
448    unsigned char* contents = (unsigned char*)malloc(len);
449    if (contents == NULL) {
450        fprintf(stderr, "malloc(%d) failed\n", len);
451        return ENOMEM;
452    }
453
454    bool sizesMatch = true;
455    amt = lseek(fd, 0, SEEK_END);
456    if (amt != len) {
457        fprintf(stderr, "compare_file file length should be %d, was %d\n", len, amt);
458        sizesMatch = false;
459    }
460    lseek(fd, 0, SEEK_SET);
461
462    int readLen = amt < len ? amt : len;
463    amt = read(fd, contents, readLen);
464    if (amt != readLen) {
465        fprintf(stderr, "compare_file read expected %d bytes but got %d\n", len, amt);
466    }
467
468    bool contentsMatch = true;
469    for (int i=0; i<readLen; i++) {
470        if (data[i] != contents[i]) {
471            if (contentsMatch) {
472                fprintf(stderr, "compare_file contents are different: (index, expected, actual)\n");
473                contentsMatch = false;
474            }
475            fprintf(stderr, "  [%-2d] %02x %02x\n", i, data[i], contents[i]);
476        }
477    }
478
479    return contentsMatch && sizesMatch ? 0 : 1;
480}
481
482int
483backup_helper_test_empty()
484{
485    int err;
486    int fd;
487    KeyedVector<String8,FileRec> snapshot;
488    const char* filename = SCRATCH_DIR "backup_helper_test_empty.snap";
489
490    system("rm -r " SCRATCH_DIR);
491    mkdir(SCRATCH_DIR, 0777);
492
493    // write
494    fd = creat(filename, 0666);
495    if (fd == -1) {
496        fprintf(stderr, "error creating %s\n", filename);
497        return 1;
498    }
499
500    err = write_snapshot_file(fd, snapshot);
501
502    close(fd);
503
504    if (err != 0) {
505        fprintf(stderr, "write_snapshot_file reported error %d (%s)\n", err, strerror(err));
506        return err;
507    }
508
509    static const unsigned char correct_data[] = {
510        0x53, 0x6e, 0x61, 0x70,  0x00, 0x00, 0x00, 0x00,
511        0x46, 0x69, 0x6c, 0x65,  0x10, 0x00, 0x00, 0x00
512    };
513
514    err = compare_file(filename, correct_data, sizeof(correct_data));
515    if (err != 0) {
516        return err;
517    }
518
519    // read
520    fd = open(filename, O_RDONLY);
521    if (fd == -1) {
522        fprintf(stderr, "error opening for read %s\n", filename);
523        return 1;
524    }
525
526    KeyedVector<String8,FileState> readSnapshot;
527    err = read_snapshot_file(fd, &readSnapshot);
528    if (err != 0) {
529        fprintf(stderr, "read_snapshot_file failed %d\n", err);
530        return err;
531    }
532
533    if (readSnapshot.size() != 0) {
534        fprintf(stderr, "readSnapshot should be length 0\n");
535        return 1;
536    }
537
538    return 0;
539}
540
541int
542backup_helper_test_four()
543{
544    int err;
545    int fd;
546    KeyedVector<String8,FileRec> snapshot;
547    const char* filename = SCRATCH_DIR "backup_helper_test_four.snap";
548
549    system("rm -r " SCRATCH_DIR);
550    mkdir(SCRATCH_DIR, 0777);
551
552    // write
553    fd = creat(filename, 0666);
554    if (fd == -1) {
555        fprintf(stderr, "error opening %s\n", filename);
556        return 1;
557    }
558
559    String8 filenames[4];
560    FileState states[4];
561    FileRec r;
562    r.deleted = false;
563    r.file = NULL;
564
565    states[0].modTime_sec = 0xfedcba98;
566    states[0].modTime_nsec = 0xdeadbeef;
567    states[0].size = 0xababbcbc;
568    states[0].crc32 = 0x12345678;
569    states[0].nameLen = -12;
570    r.s = states[0];
571    filenames[0] = String8("bytes_of_padding");
572    snapshot.add(filenames[0], r);
573
574    states[1].modTime_sec = 0x93400031;
575    states[1].modTime_nsec = 0xdeadbeef;
576    states[1].size = 0x88557766;
577    states[1].crc32 = 0x22334422;
578    states[1].nameLen = -1;
579    r.s = states[1];
580    filenames[1] = String8("bytes_of_padding3");
581    snapshot.add(filenames[1], r);
582
583    states[2].modTime_sec = 0x33221144;
584    states[2].modTime_nsec = 0xdeadbeef;
585    states[2].size = 0x11223344;
586    states[2].crc32 = 0x01122334;
587    states[2].nameLen = 0;
588    r.s = states[2];
589    filenames[2] = String8("bytes_of_padding_2");
590    snapshot.add(filenames[2], r);
591
592    states[3].modTime_sec = 0x33221144;
593    states[3].modTime_nsec = 0xdeadbeef;
594    states[3].size = 0x11223344;
595    states[3].crc32 = 0x01122334;
596    states[3].nameLen = 0;
597    r.s = states[3];
598    filenames[3] = String8("bytes_of_padding__1");
599    snapshot.add(filenames[3], r);
600
601    err = write_snapshot_file(fd, snapshot);
602
603    close(fd);
604
605    if (err != 0) {
606        fprintf(stderr, "write_snapshot_file reported error %d (%s)\n", err, strerror(err));
607        return err;
608    }
609
610    static const unsigned char correct_data[] = {
611        // header
612        0x53, 0x6e, 0x61, 0x70,  0x04, 0x00, 0x00, 0x00,
613        0x46, 0x69, 0x6c, 0x65,  0xac, 0x00, 0x00, 0x00,
614
615        // bytes_of_padding
616        0x98, 0xba, 0xdc, 0xfe,  0xef, 0xbe, 0xad, 0xde,
617        0xbc, 0xbc, 0xab, 0xab,  0x78, 0x56, 0x34, 0x12,
618        0x10, 0x00, 0x00, 0x00,  0x62, 0x79, 0x74, 0x65,
619        0x73, 0x5f, 0x6f, 0x66,  0x5f, 0x70, 0x61, 0x64,
620        0x64, 0x69, 0x6e, 0x67,
621
622        // bytes_of_padding3
623        0x31, 0x00, 0x40, 0x93,  0xef, 0xbe, 0xad, 0xde,
624        0x66, 0x77, 0x55, 0x88,  0x22, 0x44, 0x33, 0x22,
625        0x11, 0x00, 0x00, 0x00,  0x62, 0x79, 0x74, 0x65,
626        0x73, 0x5f, 0x6f, 0x66,  0x5f, 0x70, 0x61, 0x64,
627        0x64, 0x69, 0x6e, 0x67,  0x33, 0xab, 0xab, 0xab,
628
629        // bytes of padding2
630        0x44, 0x11, 0x22, 0x33,  0xef, 0xbe, 0xad, 0xde,
631        0x44, 0x33, 0x22, 0x11,  0x34, 0x23, 0x12, 0x01,
632        0x12, 0x00, 0x00, 0x00,  0x62, 0x79, 0x74, 0x65,
633        0x73, 0x5f, 0x6f, 0x66,  0x5f, 0x70, 0x61, 0x64,
634        0x64, 0x69, 0x6e, 0x67,  0x5f, 0x32, 0xab, 0xab,
635
636        // bytes of padding3
637        0x44, 0x11, 0x22, 0x33,  0xef, 0xbe, 0xad, 0xde,
638        0x44, 0x33, 0x22, 0x11,  0x34, 0x23, 0x12, 0x01,
639        0x13, 0x00, 0x00, 0x00,  0x62, 0x79, 0x74, 0x65,
640        0x73, 0x5f, 0x6f, 0x66,  0x5f, 0x70, 0x61, 0x64,
641        0x64, 0x69, 0x6e, 0x67,  0x5f, 0x5f, 0x31, 0xab
642    };
643
644    err = compare_file(filename, correct_data, sizeof(correct_data));
645    if (err != 0) {
646        return err;
647    }
648
649    // read
650    fd = open(filename, O_RDONLY);
651    if (fd == -1) {
652        fprintf(stderr, "error opening for read %s\n", filename);
653        return 1;
654    }
655
656
657    KeyedVector<String8,FileState> readSnapshot;
658    err = read_snapshot_file(fd, &readSnapshot);
659    if (err != 0) {
660        fprintf(stderr, "read_snapshot_file failed %d\n", err);
661        return err;
662    }
663
664    if (readSnapshot.size() != 4) {
665        fprintf(stderr, "readSnapshot should be length 4 is %d\n", readSnapshot.size());
666        return 1;
667    }
668
669    bool matched = true;
670    for (size_t i=0; i<readSnapshot.size(); i++) {
671        const String8& name = readSnapshot.keyAt(i);
672        const FileState state = readSnapshot.valueAt(i);
673
674        if (name != filenames[i] || states[i].modTime_sec != state.modTime_sec
675                || states[i].modTime_nsec != state.modTime_nsec
676                || states[i].size != state.size || states[i].crc32 != states[i].crc32) {
677            fprintf(stderr, "state %d expected={%d/%d, 0x%08x, 0x%08x, %3d} '%s'\n"
678                            "          actual={%d/%d, 0x%08x, 0x%08x, %3d} '%s'\n", i,
679                    states[i].modTime_sec, states[i].modTime_nsec, states[i].size, states[i].crc32,
680                    name.length(), filenames[i].string(),
681                    state.modTime_sec, state.modTime_nsec, state.size, state.crc32, state.nameLen,
682                    name.string());
683            matched = false;
684        }
685    }
686
687    return matched ? 0 : 1;
688}
689
690// hexdump -v -e '"    " 8/1 " 0x%02x," "\n"' data_writer.data
691const unsigned char DATA_GOLDEN_FILE[] = {
692     0x41, 0x70, 0x70, 0x31, 0x0b, 0x00, 0x00, 0x00,
693     0xdd, 0xcc, 0xbb, 0xaa, 0x6e, 0x6f, 0x5f, 0x70,
694     0x61, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x00,
695     0x44, 0x61, 0x74, 0x61, 0x0b, 0x00, 0x00, 0x00,
696     0x0c, 0x00, 0x00, 0x00, 0x6e, 0x6f, 0x5f, 0x70,
697     0x61, 0x64, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x00,
698     0x6e, 0x6f, 0x5f, 0x70, 0x61, 0x64, 0x64, 0x69,
699     0x6e, 0x67, 0x5f, 0x00, 0x41, 0x70, 0x70, 0x31,
700     0x0c, 0x00, 0x00, 0x00, 0xdd, 0xcc, 0xbb, 0xaa,
701     0x70, 0x61, 0x64, 0x64, 0x65, 0x64, 0x5f, 0x74,
702     0x6f, 0x5f, 0x5f, 0x33, 0x00, 0xbc, 0xbc, 0xbc,
703     0x44, 0x61, 0x74, 0x61, 0x0c, 0x00, 0x00, 0x00,
704     0x0d, 0x00, 0x00, 0x00, 0x70, 0x61, 0x64, 0x64,
705     0x65, 0x64, 0x5f, 0x74, 0x6f, 0x5f, 0x5f, 0x33,
706     0x00, 0xbc, 0xbc, 0xbc, 0x70, 0x61, 0x64, 0x64,
707     0x65, 0x64, 0x5f, 0x74, 0x6f, 0x5f, 0x5f, 0x33,
708     0x00, 0xbc, 0xbc, 0xbc, 0x41, 0x70, 0x70, 0x31,
709     0x0d, 0x00, 0x00, 0x00, 0xdd, 0xcc, 0xbb, 0xaa,
710     0x70, 0x61, 0x64, 0x64, 0x65, 0x64, 0x5f, 0x74,
711     0x6f, 0x5f, 0x32, 0x5f, 0x5f, 0x00, 0xbc, 0xbc,
712     0x44, 0x61, 0x74, 0x61, 0x0d, 0x00, 0x00, 0x00,
713     0x0e, 0x00, 0x00, 0x00, 0x70, 0x61, 0x64, 0x64,
714     0x65, 0x64, 0x5f, 0x74, 0x6f, 0x5f, 0x32, 0x5f,
715     0x5f, 0x00, 0xbc, 0xbc, 0x70, 0x61, 0x64, 0x64,
716     0x65, 0x64, 0x5f, 0x74, 0x6f, 0x5f, 0x32, 0x5f,
717     0x5f, 0x00, 0xbc, 0xbc, 0x41, 0x70, 0x70, 0x31,
718     0x0a, 0x00, 0x00, 0x00, 0xdd, 0xcc, 0xbb, 0xaa,
719     0x70, 0x61, 0x64, 0x64, 0x65, 0x64, 0x5f, 0x74,
720     0x6f, 0x31, 0x00, 0xbc, 0x44, 0x61, 0x74, 0x61,
721     0x0a, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00,
722     0x70, 0x61, 0x64, 0x64, 0x65, 0x64, 0x5f, 0x74,
723     0x6f, 0x31, 0x00, 0xbc, 0x70, 0x61, 0x64, 0x64,
724     0x65, 0x64, 0x5f, 0x74, 0x6f, 0x31, 0x00, 0xbc,
725     0x46, 0x6f, 0x6f, 0x74, 0x04, 0x00, 0x00, 0x00,
726     0x99, 0x99, 0x77, 0x77
727};
728const int DATA_GOLDEN_FILE_SIZE = sizeof(DATA_GOLDEN_FILE);
729
730static int
731test_write_header_and_entity(BackupDataWriter& writer, const char* str)
732{
733    int err;
734    String8 text(str);
735
736    err = writer.WriteAppHeader(text, 0xaabbccdd);
737    if (err != 0) {
738        fprintf(stderr, "WriteAppHeader failed with %s\n", strerror(err));
739        return err;
740    }
741
742    err = writer.WriteEntityHeader(text, text.length()+1);
743    if (err != 0) {
744        fprintf(stderr, "WriteEntityHeader failed with %s\n", strerror(err));
745        return err;
746    }
747
748    err = writer.WriteEntityData(text.string(), text.length()+1);
749    if (err != 0) {
750        fprintf(stderr, "write failed for data '%s'\n", text.string());
751        return errno;
752    }
753
754    return err;
755}
756
757int
758backup_helper_test_data_writer()
759{
760    int err;
761    int fd;
762    const char* filename = SCRATCH_DIR "data_writer.data";
763
764    system("rm -r " SCRATCH_DIR);
765    mkdir(SCRATCH_DIR, 0777);
766    mkdir(SCRATCH_DIR "data", 0777);
767
768    fd = creat(filename, 0666);
769    if (fd == -1) {
770        fprintf(stderr, "error creating: %s\n", strerror(errno));
771        return errno;
772    }
773
774    BackupDataWriter writer(fd);
775
776    err = 0;
777    err |= test_write_header_and_entity(writer, "no_padding_");
778    err |= test_write_header_and_entity(writer, "padded_to__3");
779    err |= test_write_header_and_entity(writer, "padded_to_2__");
780    err |= test_write_header_and_entity(writer, "padded_to1");
781
782    writer.WriteAppFooter(0x77779999);
783
784    close(fd);
785
786    err = compare_file(filename, DATA_GOLDEN_FILE, DATA_GOLDEN_FILE_SIZE);
787    if (err != 0) {
788        return err;
789    }
790
791    return err;
792}
793
794int
795test_read_header_and_entity(BackupDataReader& reader, const char* str)
796{
797    int err;
798    int bufSize = strlen(str)+1;
799    char* buf = (char*)malloc(bufSize);
800    String8 string;
801    int cookie = 0x11111111;
802    size_t actualSize;
803
804    // printf("\n\n---------- test_read_header_and_entity -- %s\n\n", str);
805
806    err = reader.ReadNextHeader();
807    if (err != 0) {
808        fprintf(stderr, "ReadNextHeader (for app header) failed with %s\n", strerror(err));
809        goto done;
810    }
811
812    err = reader.ReadAppHeader(&string, &cookie);
813    if (err != 0) {
814        fprintf(stderr, "ReadAppHeader failed with %s\n", strerror(err));
815        goto done;
816    }
817    if (string != str) {
818        fprintf(stderr, "ReadAppHeader expected packageName '%s' got '%s'\n", str, string.string());
819        err = EINVAL;
820        goto done;
821    }
822    if (cookie != (int)0xaabbccdd) {
823        fprintf(stderr, "ReadAppHeader expected cookie 0x%08x got 0x%08x\n", 0xaabbccdd, cookie);
824        err = EINVAL;
825        goto done;
826    }
827
828    err = reader.ReadNextHeader();
829    if (err != 0) {
830        fprintf(stderr, "ReadNextHeader (for entity header) failed with %s\n", strerror(err));
831        goto done;
832    }
833
834    err = reader.ReadEntityHeader(&string, &actualSize);
835    if (err != 0) {
836        fprintf(stderr, "ReadEntityHeader failed with %s\n", strerror(err));
837        goto done;
838    }
839    if (string != str) {
840        fprintf(stderr, "ReadEntityHeader expected key '%s' got '%s'\n", str, string.string());
841        err = EINVAL;
842        goto done;
843    }
844    if ((int)actualSize != bufSize) {
845        fprintf(stderr, "ReadEntityHeader expected dataSize 0x%08x got 0x%08x\n", bufSize,
846                actualSize);
847        err = EINVAL;
848        goto done;
849    }
850
851    err = reader.ReadEntityData(buf, bufSize);
852    if (err != NO_ERROR) {
853        fprintf(stderr, "ReadEntityData failed with %s\n", strerror(err));
854        goto done;
855    }
856
857    if (0 != memcmp(buf, str, bufSize)) {
858        fprintf(stderr, "ReadEntityData expected '%s' but got something starting with "
859                "%02x %02x %02x %02x\n", str, buf[0], buf[1], buf[2], buf[3]);
860        err = EINVAL;
861        goto done;
862    }
863
864    // The next read will confirm whether it got the right amount of data.
865
866done:
867    if (err != NO_ERROR) {
868        fprintf(stderr, "test_read_header_and_entity failed with %s\n", strerror(err));
869    }
870    free(buf);
871    return err;
872}
873
874int
875backup_helper_test_data_reader()
876{
877    int err;
878    int fd;
879    const char* filename = SCRATCH_DIR "data_reader.data";
880
881    system("rm -r " SCRATCH_DIR);
882    mkdir(SCRATCH_DIR, 0777);
883    mkdir(SCRATCH_DIR "data", 0777);
884
885    fd = creat(filename, 0666);
886    if (fd == -1) {
887        fprintf(stderr, "error creating: %s\n", strerror(errno));
888        return errno;
889    }
890
891    err = write(fd, DATA_GOLDEN_FILE, DATA_GOLDEN_FILE_SIZE);
892    if (err != DATA_GOLDEN_FILE_SIZE) {
893        fprintf(stderr, "Error \"%s\" writing golden file %s\n", strerror(errno), filename);
894        return errno;
895    }
896
897    close(fd);
898
899    fd = open(filename, O_RDONLY);
900    if (fd == -1) {
901        fprintf(stderr, "Error \"%s\" opening golden file %s for read\n", strerror(errno),
902                filename);
903        return errno;
904    }
905
906    {
907        BackupDataReader reader(fd);
908
909        err = 0;
910
911        if (err == NO_ERROR) {
912            err = test_read_header_and_entity(reader, "no_padding_");
913        }
914
915        if (err == NO_ERROR) {
916            err = test_read_header_and_entity(reader, "padded_to__3");
917        }
918
919        if (err == NO_ERROR) {
920            err = test_read_header_and_entity(reader, "padded_to_2__");
921        }
922
923        if (err == NO_ERROR) {
924            err = test_read_header_and_entity(reader, "padded_to1");
925        }
926
927        if (err == NO_ERROR) {
928            err = reader.ReadNextHeader();
929            if (err != 0) {
930                fprintf(stderr, "ReadNextHeader (for app header) failed with %s\n", strerror(err));
931            }
932
933            if (err == NO_ERROR) {
934                int cookie;
935                err |= reader.ReadAppFooter(&cookie);
936                if (cookie != 0x77779999) {
937                    fprintf(stderr, "app footer cookie expected=0x%08x actual=0x%08x\n",
938                        0x77779999, cookie);
939                    err = EINVAL;
940                }
941            }
942        }
943    }
944
945    close(fd);
946
947    return err;
948}
949
950static int
951get_mod_time(const char* filename, struct timeval times[2])
952{
953    int err;
954    struct stat64 st;
955    err = stat64(filename, &st);
956    if (err != 0) {
957        fprintf(stderr, "stat '%s' failed: %s\n", filename, strerror(errno));
958        return errno;
959    }
960    times[0].tv_sec = st.st_atime;
961    times[1].tv_sec = st.st_mtime;
962
963    // If st_atime is a macro then struct stat64 uses struct timespec
964    // to store the access and modif time values and typically
965    // st_*time_nsec is not defined. In glibc, this is controlled by
966    // __USE_MISC.
967#ifdef __USE_MISC
968#if !defined(st_atime) || defined(st_atime_nsec)
969#error "Check if this __USE_MISC conditional is still needed."
970#endif
971    times[0].tv_usec = st.st_atim.tv_nsec / 1000;
972    times[1].tv_usec = st.st_mtim.tv_nsec / 1000;
973#else
974    times[0].tv_usec = st.st_atime_nsec / 1000;
975    times[1].tv_usec = st.st_mtime_nsec / 1000;
976#endif
977
978    return 0;
979}
980
981int
982backup_helper_test_files()
983{
984    int err;
985    int oldSnapshotFD;
986    int dataStreamFD;
987    int newSnapshotFD;
988
989    system("rm -r " SCRATCH_DIR);
990    mkdir(SCRATCH_DIR, 0777);
991    mkdir(SCRATCH_DIR "data", 0777);
992
993    write_text_file(SCRATCH_DIR "data/b", "b\nbb\n");
994    write_text_file(SCRATCH_DIR "data/c", "c\ncc\n");
995    write_text_file(SCRATCH_DIR "data/d", "d\ndd\n");
996    write_text_file(SCRATCH_DIR "data/e", "e\nee\n");
997    write_text_file(SCRATCH_DIR "data/f", "f\nff\n");
998    write_text_file(SCRATCH_DIR "data/h", "h\nhh\n");
999
1000    char const* files_before[] = {
1001        SCRATCH_DIR "data/b",
1002        SCRATCH_DIR "data/c",
1003        SCRATCH_DIR "data/d",
1004        SCRATCH_DIR "data/e",
1005        SCRATCH_DIR "data/f"
1006    };
1007
1008    char const* keys_before[] = {
1009        "data/b",
1010        "data/c",
1011        "data/d",
1012        "data/e",
1013        "data/f"
1014    };
1015
1016    dataStreamFD = creat(SCRATCH_DIR "1.data", 0666);
1017    if (dataStreamFD == -1) {
1018        fprintf(stderr, "error creating: %s\n", strerror(errno));
1019        return errno;
1020    }
1021
1022    newSnapshotFD = creat(SCRATCH_DIR "before.snap", 0666);
1023    if (newSnapshotFD == -1) {
1024        fprintf(stderr, "error creating: %s\n", strerror(errno));
1025        return errno;
1026    }
1027
1028    {
1029        BackupDataWriter dataStream(dataStreamFD);
1030
1031        err = back_up_files(-1, &dataStream, newSnapshotFD, files_before, keys_before, 5);
1032        if (err != 0) {
1033            return err;
1034        }
1035    }
1036
1037    close(dataStreamFD);
1038    close(newSnapshotFD);
1039
1040    sleep(3);
1041
1042    struct timeval d_times[2];
1043    struct timeval e_times[2];
1044
1045    err = get_mod_time(SCRATCH_DIR "data/d", d_times);
1046    err |= get_mod_time(SCRATCH_DIR "data/e", e_times);
1047    if (err != 0) {
1048        return err;
1049    }
1050
1051    write_text_file(SCRATCH_DIR "data/a", "a\naa\n");
1052    unlink(SCRATCH_DIR "data/c");
1053    write_text_file(SCRATCH_DIR "data/c", "c\ncc\n");
1054    write_text_file(SCRATCH_DIR "data/d", "dd\ndd\n");
1055    utimes(SCRATCH_DIR "data/d", d_times);
1056    write_text_file(SCRATCH_DIR "data/e", "z\nzz\n");
1057    utimes(SCRATCH_DIR "data/e", e_times);
1058    write_text_file(SCRATCH_DIR "data/g", "g\ngg\n");
1059    unlink(SCRATCH_DIR "data/f");
1060
1061    char const* files_after[] = {
1062        SCRATCH_DIR "data/a", // added
1063        SCRATCH_DIR "data/b", // same
1064        SCRATCH_DIR "data/c", // different mod time
1065        SCRATCH_DIR "data/d", // different size (same mod time)
1066        SCRATCH_DIR "data/e", // different contents (same mod time, same size)
1067        SCRATCH_DIR "data/g"  // added
1068    };
1069
1070    char const* keys_after[] = {
1071        "data/a", // added
1072        "data/b", // same
1073        "data/c", // different mod time
1074        "data/d", // different size (same mod time)
1075        "data/e", // different contents (same mod time, same size)
1076        "data/g"  // added
1077    };
1078
1079    oldSnapshotFD = open(SCRATCH_DIR "before.snap", O_RDONLY);
1080    if (oldSnapshotFD == -1) {
1081        fprintf(stderr, "error opening: %s\n", strerror(errno));
1082        return errno;
1083    }
1084
1085    dataStreamFD = creat(SCRATCH_DIR "2.data", 0666);
1086    if (dataStreamFD == -1) {
1087        fprintf(stderr, "error creating: %s\n", strerror(errno));
1088        return errno;
1089    }
1090
1091    newSnapshotFD = creat(SCRATCH_DIR "after.snap", 0666);
1092    if (newSnapshotFD == -1) {
1093        fprintf(stderr, "error creating: %s\n", strerror(errno));
1094        return errno;
1095    }
1096
1097    {
1098        BackupDataWriter dataStream(dataStreamFD);
1099
1100        err = back_up_files(oldSnapshotFD, &dataStream, newSnapshotFD, files_after, keys_after, 6);
1101        if (err != 0) {
1102            return err;
1103        }
1104}
1105
1106    close(oldSnapshotFD);
1107    close(dataStreamFD);
1108    close(newSnapshotFD);
1109
1110    return 0;
1111}
1112
1113int
1114backup_helper_test_null_base()
1115{
1116    int err;
1117    int oldSnapshotFD;
1118    int dataStreamFD;
1119    int newSnapshotFD;
1120
1121    system("rm -r " SCRATCH_DIR);
1122    mkdir(SCRATCH_DIR, 0777);
1123    mkdir(SCRATCH_DIR "data", 0777);
1124
1125    write_text_file(SCRATCH_DIR "data/a", "a\naa\n");
1126
1127    char const* files[] = {
1128        SCRATCH_DIR "data/a",
1129    };
1130
1131    char const* keys[] = {
1132        "a",
1133    };
1134
1135    dataStreamFD = creat(SCRATCH_DIR "null_base.data", 0666);
1136    if (dataStreamFD == -1) {
1137        fprintf(stderr, "error creating: %s\n", strerror(errno));
1138        return errno;
1139    }
1140
1141    newSnapshotFD = creat(SCRATCH_DIR "null_base.snap", 0666);
1142    if (newSnapshotFD == -1) {
1143        fprintf(stderr, "error creating: %s\n", strerror(errno));
1144        return errno;
1145    }
1146
1147    {
1148        BackupDataWriter dataStream(dataStreamFD);
1149
1150        err = back_up_files(-1, &dataStream, newSnapshotFD, files, keys, 1);
1151        if (err != 0) {
1152            return err;
1153        }
1154    }
1155
1156    close(dataStreamFD);
1157    close(newSnapshotFD);
1158
1159    return 0;
1160}
1161
1162int
1163backup_helper_test_missing_file()
1164{
1165    int err;
1166    int oldSnapshotFD;
1167    int dataStreamFD;
1168    int newSnapshotFD;
1169
1170    system("rm -r " SCRATCH_DIR);
1171    mkdir(SCRATCH_DIR, 0777);
1172    mkdir(SCRATCH_DIR "data", 0777);
1173
1174    write_text_file(SCRATCH_DIR "data/b", "b\nbb\n");
1175
1176    char const* files[] = {
1177        SCRATCH_DIR "data/a",
1178        SCRATCH_DIR "data/b",
1179        SCRATCH_DIR "data/c",
1180    };
1181
1182    char const* keys[] = {
1183        "a",
1184        "b",
1185        "c",
1186    };
1187
1188    dataStreamFD = creat(SCRATCH_DIR "null_base.data", 0666);
1189    if (dataStreamFD == -1) {
1190        fprintf(stderr, "error creating: %s\n", strerror(errno));
1191        return errno;
1192    }
1193
1194    newSnapshotFD = creat(SCRATCH_DIR "null_base.snap", 0666);
1195    if (newSnapshotFD == -1) {
1196        fprintf(stderr, "error creating: %s\n", strerror(errno));
1197        return errno;
1198    }
1199
1200    {
1201        BackupDataWriter dataStream(dataStreamFD);
1202
1203        err = back_up_files(-1, &dataStream, newSnapshotFD, files, keys, 1);
1204        if (err != 0) {
1205            return err;
1206        }
1207    }
1208
1209    close(dataStreamFD);
1210    close(newSnapshotFD);
1211
1212    return 0;
1213}
1214
1215
1216#endif // TEST_BACKUP_HELPERS
1217
1218}
1219