target_files_diff.py revision c4438d3d29389d0df0b744c339d67f4a673be96c 
1#!/usr/bin/env python
2#
3# Copyright (C) 2009 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18#
19# Finds differences between two target files packages
20#
21
22from __future__ import print_function
23
24import argparse
25import contextlib
26import os
27import re
28import subprocess
29import sys
30import tempfile
31
32def ignore(name):
33  """
34  Files to ignore when diffing
35
36  These are packages that we're already diffing elsewhere,
37  or files that we expect to be different for every build,
38  or known problems.
39  """
40
41  # We're looking at the files that make the images, so no need to search them
42  if name in ['IMAGES']:
43    return True
44  # These are packages of the recovery partition, which we're already diffing
45  if name in ['SYSTEM/etc/recovery-resource.dat',
46              'SYSTEM/recovery-from-boot.p']:
47    return True
48
49  # These files are just the BUILD_NUMBER, and will always be different
50  if name in ['BOOT/RAMDISK/selinux_version',
51              'RECOVERY/RAMDISK/selinux_version']:
52    return True
53
54  # b/26956807 .odex files are not deterministic
55  if name.endswith('.odex'):
56    return True
57
58  return False
59
60
61def rewrite_build_property(original, new):
62  """
63  Rewrite property files to remove values known to change for every build
64  """
65
66  skipped = ['ro.bootimage.build.date=',
67             'ro.bootimage.build.date.utc=',
68             'ro.bootimage.build.fingerprint=',
69             'ro.build.id=',
70             'ro.build.display.id=',
71             'ro.build.version.incremental=',
72             'ro.build.date=',
73             'ro.build.date.utc=',
74             'ro.build.host=',
75             'ro.build.user=',
76             'ro.build.description=',
77             'ro.build.fingerprint=',
78             'ro.expect.recovery_id=',
79             'ro.vendor.build.date=',
80             'ro.vendor.build.date.utc=',
81             'ro.vendor.build.fingerprint=']
82
83  for line in original:
84    skip = False
85    for s in skipped:
86      if line.startswith(s):
87        skip = True
88        break
89    if not skip:
90      new.write(line)
91
92
93def trim_install_recovery(original, new):
94  """
95  Rewrite the install-recovery script to remove the hash of the recovery
96  partition.
97  """
98  for line in original:
99    new.write(re.sub(r'[0-9a-f]{40}', '0'*40, line))
100
101def sort_file(original, new):
102  """
103  Sort the file. Some OTA metadata files are not in a deterministic order
104  currently.
105  """
106  lines = original.readlines()
107  lines.sort()
108  for line in lines:
109    new.write(line)
110
111# Map files to the functions that will modify them for diffing
112REWRITE_RULES = {
113    'BOOT/RAMDISK/default.prop': rewrite_build_property,
114    'RECOVERY/RAMDISK/default.prop': rewrite_build_property,
115    'SYSTEM/build.prop': rewrite_build_property,
116    'VENDOR/build.prop': rewrite_build_property,
117
118    'SYSTEM/bin/install-recovery.sh': trim_install_recovery,
119
120    'META/boot_filesystem_config.txt': sort_file,
121    'META/filesystem_config.txt': sort_file,
122    'META/recovery_filesystem_config.txt': sort_file,
123    'META/vendor_filesystem_config.txt': sort_file,
124}
125
126@contextlib.contextmanager
127def preprocess(name, filename):
128  """
129  Optionally rewrite files before diffing them, to remove known-variable
130  information.
131  """
132  if name in REWRITE_RULES:
133    with tempfile.NamedTemporaryFile() as newfp:
134      with open(filename, 'r') as oldfp:
135        REWRITE_RULES[name](oldfp, newfp)
136      newfp.flush()
137      yield newfp.name
138  else:
139    yield filename
140
141def diff(name, file1, file2, out_file):
142  """
143  Diff a file pair with diff, running preprocess() on the arguments first.
144  """
145  with preprocess(name, file1) as f1:
146    with preprocess(name, file2) as f2:
147      proc = subprocess.Popen(['diff', f1, f2], stdout=subprocess.PIPE,
148                              stderr=subprocess.STDOUT)
149      (stdout, _) = proc.communicate()
150      if proc.returncode == 0:
151        return
152      stdout = stdout.strip()
153      if stdout == 'Binary files %s and %s differ' % (f1, f2):
154        print("%s: Binary files differ" % name, file=out_file)
155      else:
156        for line in stdout.strip().split('\n'):
157          print("%s: %s" % (name, line), file=out_file)
158
159def recursiveDiff(prefix, dir1, dir2, out_file):
160  """
161  Recursively diff two directories, checking metadata then calling diff()
162  """
163  list1 = sorted(os.listdir(dir1))
164  list2 = sorted(os.listdir(dir2))
165
166  for entry in list1:
167    name = os.path.join(prefix, entry)
168    name1 = os.path.join(dir1, entry)
169    name2 = os.path.join(dir2, entry)
170
171    if ignore(name):
172      continue
173
174    if entry in list2:
175      if os.path.islink(name1):
176        if os.path.islink(name2):
177          link1 = os.readlink(name1)
178          link2 = os.readlink(name2)
179          if link1 != link2:
180            print("%s: Symlinks differ: %s vs %s" % (name, link1, link2),
181                  file=out_file)
182        else:
183          print("%s: File types differ, skipping compare" % name,
184                file=out_file)
185        continue
186
187      stat1 = os.stat(name1)
188      stat2 = os.stat(name2)
189      type1 = stat1.st_mode & ~0o777
190      type2 = stat2.st_mode & ~0o777
191
192      if type1 != type2:
193        print("%s: File types differ, skipping compare" % name, file=out_file)
194        continue
195
196      if stat1.st_mode != stat2.st_mode:
197        print("%s: Modes differ: %o vs %o" %
198            (name, stat1.st_mode, stat2.st_mode), file=out_file)
199
200      if os.path.isdir(name1):
201        recursiveDiff(name, name1, name2, out_file)
202      elif os.path.isfile(name1):
203        diff(name, name1, name2, out_file)
204      else:
205        print("%s: Unknown file type, skipping compare" % name, file=out_file)
206    else:
207      print("%s: Only in base package" % name, file=out_file)
208
209  for entry in list2:
210    name = os.path.join(prefix, entry)
211    name1 = os.path.join(dir1, entry)
212    name2 = os.path.join(dir2, entry)
213
214    if ignore(name):
215      continue
216
217    if entry not in list1:
218      print("%s: Only in new package" % name, file=out_file)
219
220def main():
221  parser = argparse.ArgumentParser()
222  parser.add_argument('dir1', help='The base target files package (extracted)')
223  parser.add_argument('dir2', help='The new target files package (extracted)')
224  parser.add_argument('--output',
225      help='The output file, otherwise it prints to stdout')
226  args = parser.parse_args()
227
228  if args.output:
229    out_file = open(args.output, 'w')
230  else:
231    out_file = sys.stdout
232
233  recursiveDiff('', args.dir1, args.dir2, out_file)
234
235  if args.output:
236    out_file.close()
237
238if __name__ == '__main__':
239  main()
240