target_files_diff.py revision d62c603573c53713164fda0430d477daa11d5e31 
1#!/usr/bin/env python
2#
3# Copyright (C) 2009 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18#
19# Finds differences between two target files packages
20#
21
22from __future__ import print_function
23
24import argparse
25import contextlib
26import os
27import re
28import subprocess
29import sys
30import tempfile
31
32def ignore(name):
33  """
34  Files to ignore when diffing
35
36  These are packages that we're already diffing elsewhere,
37  or files that we expect to be different for every build,
38  or known problems.
39  """
40
41  # We're looking at the files that make the images, so no need to search them
42  if name in ['IMAGES']:
43    return True
44  # These are packages of the recovery partition, which we're already diffing
45  if name in ['SYSTEM/etc/recovery-resource.dat',
46              'SYSTEM/recovery-from-boot.p']:
47    return True
48
49  # These files are just the BUILD_NUMBER, and will always be different
50  if name in ['BOOT/RAMDISK/selinux_version',
51              'RECOVERY/RAMDISK/selinux_version']:
52    return True
53
54  # b/24201956 .art/.oat/.odex files are different with every build
55  if name.endswith('.art') or name.endswith('.oat') or name.endswith('.odex'):
56    return True
57  # b/25348136 libpac.so changes with every build
58  if name in ['SYSTEM/lib/libpac.so',
59              'SYSTEM/lib64/libpac.so']:
60    return True
61
62  return False
63
64
65def rewrite_build_property(original, new):
66  """
67  Rewrite property files to remove values known to change for every build
68  """
69
70  skipped = ['ro.bootimage.build.date=',
71             'ro.bootimage.build.date.utc=',
72             'ro.bootimage.build.fingerprint=',
73             'ro.build.id=',
74             'ro.build.display.id=',
75             'ro.build.version.incremental=',
76             'ro.build.date=',
77             'ro.build.date.utc=',
78             'ro.build.host=',
79             'ro.build.description=',
80             'ro.build.fingerprint=',
81             'ro.expect.recovery_id=',
82             'ro.vendor.build.date=',
83             'ro.vendor.build.date.utc=',
84             'ro.vendor.build.fingerprint=']
85
86  for line in original:
87    skip = False
88    for s in skipped:
89      if line.startswith(s):
90        skip = True
91        break
92    if not skip:
93      new.write(line)
94
95
96def trim_install_recovery(original, new):
97  """
98  Rewrite the install-recovery script to remove the hash of the recovery
99  partition.
100  """
101  for line in original:
102    new.write(re.sub(r'[0-9a-f]{40}', '0'*40, line))
103
104def sort_file(original, new):
105  """
106  Sort the file. Some OTA metadata files are not in a deterministic order
107  currently.
108  """
109  lines = original.readlines()
110  lines.sort()
111  for line in lines:
112    new.write(line)
113
114# Map files to the functions that will modify them for diffing
115REWRITE_RULES = {
116    'BOOT/RAMDISK/default.prop': rewrite_build_property,
117    'RECOVERY/RAMDISK/default.prop': rewrite_build_property,
118    'SYSTEM/build.prop': rewrite_build_property,
119    'VENDOR/build.prop': rewrite_build_property,
120
121    'SYSTEM/bin/install-recovery.sh': trim_install_recovery,
122
123    'META/boot_filesystem_config.txt': sort_file,
124    'META/filesystem_config.txt': sort_file,
125    'META/recovery_filesystem_config.txt': sort_file,
126    'META/vendor_filesystem_config.txt': sort_file,
127}
128
129@contextlib.contextmanager
130def preprocess(name, filename):
131  """
132  Optionally rewrite files before diffing them, to remove known-variable
133  information.
134  """
135  if name in REWRITE_RULES:
136    with tempfile.NamedTemporaryFile() as newfp:
137      with open(filename, 'r') as oldfp:
138        REWRITE_RULES[name](oldfp, newfp)
139      newfp.flush()
140      yield newfp.name
141  else:
142    yield filename
143
144def diff(name, file1, file2, out_file):
145  """
146  Diff a file pair with diff, running preprocess() on the arguments first.
147  """
148  with preprocess(name, file1) as f1:
149    with preprocess(name, file2) as f2:
150      proc = subprocess.Popen(['diff', f1, f2], stdout=subprocess.PIPE,
151                              stderr=subprocess.STDOUT)
152      (stdout, _) = proc.communicate()
153      if proc.returncode == 0:
154        return
155      stdout = stdout.strip()
156      if stdout == 'Binary files %s and %s differ' % (f1, f2):
157        print("%s: Binary files differ" % name, file=out_file)
158      else:
159        for line in stdout.strip().split('\n'):
160          print("%s: %s" % (name, line), file=out_file)
161
162def recursiveDiff(prefix, dir1, dir2, out_file):
163  """
164  Recursively diff two directories, checking metadata then calling diff()
165  """
166  list1 = sorted(os.listdir(dir1))
167  list2 = sorted(os.listdir(dir2))
168
169  for entry in list1:
170    name = os.path.join(prefix, entry)
171    name1 = os.path.join(dir1, entry)
172    name2 = os.path.join(dir2, entry)
173
174    if ignore(name):
175      continue
176
177    if entry in list2:
178      if os.path.islink(name1):
179        if os.path.islink(name2):
180          link1 = os.readlink(name1)
181          link2 = os.readlink(name2)
182          if link1 != link2:
183            print("%s: Symlinks differ: %s vs %s" % (name, link1, link2),
184                  file=out_file)
185        else:
186          print("%s: File types differ, skipping compare" % name,
187                file=out_file)
188        continue
189
190      stat1 = os.stat(name1)
191      stat2 = os.stat(name2)
192      type1 = stat1.st_mode & ~0o777
193      type2 = stat2.st_mode & ~0o777
194
195      if type1 != type2:
196        print("%s: File types differ, skipping compare" % name, file=out_file)
197        continue
198
199      if stat1.st_mode != stat2.st_mode:
200        print("%s: Modes differ: %o vs %o" %
201            (name, stat1.st_mode, stat2.st_mode), file=out_file)
202
203      if os.path.isdir(name1):
204        recursiveDiff(name, name1, name2, out_file)
205      elif os.path.isfile(name1):
206        diff(name, name1, name2, out_file)
207      else:
208        print("%s: Unknown file type, skipping compare" % name, file=out_file)
209    else:
210      print("%s: Only in base package" % name, file=out_file)
211
212  for entry in list2:
213    name = os.path.join(prefix, entry)
214    name1 = os.path.join(dir1, entry)
215    name2 = os.path.join(dir2, entry)
216
217    if ignore(name):
218      continue
219
220    if entry not in list1:
221      print("%s: Only in new package" % name, file=out_file)
222
223def main():
224  parser = argparse.ArgumentParser()
225  parser.add_argument('dir1', help='The base target files package (extracted)')
226  parser.add_argument('dir2', help='The new target files package (extracted)')
227  parser.add_argument('--output',
228      help='The output file, otherwise it prints to stdout')
229  args = parser.parse_args()
230
231  if args.output:
232    out_file = open(args.output, 'w')
233  else:
234    out_file = sys.stdout
235
236  recursiveDiff('', args.dir1, args.dir2, out_file)
237
238  if args.output:
239    out_file.close()
240
241if __name__ == '__main__':
242  main()
243