bisect_driver.py revision 28683f7dacd94801e9a0d4b28e5aa8a700fc3d6f 
1# Copyright 2016 Google Inc. All Rights Reserved.
2#
3# This script is used to help the compiler wrapper in the Android build system
4# bisect for bad object files.
5
6"""Utilities for bisection of Android object files.
7
8This module contains a set of utilities to allow bisection between
9two sets (good and bad) of object files. Mostly used to find compiler
10bugs.
11
12Design doc:
13https://docs.google.com/document/d/1yDgaUIa2O5w6dc3sSTe1ry-1ehKajTGJGQCbyn0fcEM
14"""
15
16from __future__ import print_function
17
18import os
19import shutil
20import subprocess
21import sys
22
23VALID_MODES = ['POPULATE_GOOD', 'POPULATE_BAD', 'TRIAGE']
24DEP_CACHE = 'dep'
25GOOD_CACHE = 'good'
26BAD_CACHE = 'bad'
27LIST_FILE = os.path.join(GOOD_CACHE, '_LIST')
28
29CONTINUE_ON_MISSING = os.environ.get('BISECT_CONTINUE_ON_MISSING', None) == '1'
30
31
32class Error(Exception):
33  """The general compiler wrapper error class."""
34  pass
35
36
37def log_to_file(path, execargs, link_from=None, link_to=None):
38  """Common logging function.
39
40  Log current working directory, current execargs, and a from-to relationship
41  between files.
42  """
43  with open(path, 'a') as log:
44    log.write('cd: %s; %s\n' % (os.getcwd(), ' '.join(execargs)))
45    if link_from and link_to:
46      log.write('%s -> %s\n' % (link_from, link_to))
47
48
49def exec_and_return(execargs):
50  """Execute process and return.
51
52  Execute according to execargs and return immediately. Don't inspect
53  stderr or stdout.
54  """
55  return subprocess.call(execargs)
56
57
58def in_bad_set(obj_file):
59  """Check if object file is in bad set.
60
61  The binary search tool creates two files for each search iteration listing
62  the full set of bad objects and full set of good objects. We use this to
63  determine where an object file should be linked from (good or bad).
64  """
65  bad_set_file = os.environ.get('BISECT_BAD_SET')
66  ret = subprocess.call(['grep', '-x', '-q', obj_file, bad_set_file])
67  return ret == 0
68
69
70def makedirs(path):
71  """Try to create directories in path."""
72  try:
73    os.makedirs(path)
74  except os.error:
75    if not os.path.isdir(path):
76      raise
77
78
79def get_obj_path(execargs):
80  """Get the object path for the object file in the list of arguments.
81
82  Returns:
83    Tuple of object path from execution args (-o argument) and full object
84    path. If no object being outputted or output doesn't end in ".o" then return
85    empty strings.
86  """
87  try:
88    i = execargs.index('-o')
89  except ValueError:
90    return "", ""
91
92  obj_path = execargs[i+1]
93  if not obj_path.endswith(('.o',)):
94    # TODO: what suffixes do we need to contemplate
95    # TODO: add this as a warning
96    # TODO: need to handle -r compilations
97    return "", ""
98
99  return obj_path, os.path.join(os.getcwd(), obj_path)
100
101
102def get_dep_path(execargs):
103  """Get the dep file path for the dep file in the list of arguments.
104
105  Returns:
106    Tuple of dependency file path from execution args (-o argument) and full
107    dependency file path. If no dependency being outputted then return empty
108    strings.
109  """
110  try:
111    i = execargs.index('-MF')
112  except ValueError:
113    return "", ""
114
115  dep_path = execargs[i+1]
116  return dep_path, os.path.join(os.getcwd(), dep_path)
117
118
119def in_object_list(obj_name, list_filename):
120  """Check if object file name exist in file with object list."""
121  if not obj_name:
122    return False
123
124  with open(list_filename, 'r') as list_file:
125    for line in list_file:
126      if line.strip() == obj_name:
127        return True
128
129    return False
130
131
132def generate_side_effects(execargs, bisect_dir):
133  """Generate compiler side effects.
134
135  Generate and cache side effects so that we can trick make into thinking
136  the compiler is actually called during triaging.
137  """
138  # TODO(cburden): Cache .dwo files
139
140  # Cache dependency files
141  dep_path, _ = get_dep_path(execargs)
142  if not dep_path:
143    return
144
145  bisect_path = os.path.join(bisect_dir, DEP_CACHE, dep_path)
146  bisect_path_dir = os.path.dirname(bisect_path)
147  makedirs(bisect_path_dir)
148  pop_log = os.path.join(bisect_dir, DEP_CACHE, '_POPULATE_LOG')
149  log_to_file(pop_log, execargs, link_from=dep_path, link_to=bisect_path)
150
151  try:
152    if os.path.exists(dep_path):
153      shutil.copy2(dep_path, bisect_path)
154  except Exception:
155    print('Could not get dep file', file=sys.stderr)
156    raise
157
158
159def bisect_populate(execargs, bisect_dir, population_name):
160  """Add necessary information to the bisect cache for the given execution.
161
162  Extract the necessary information for bisection from the compiler
163  execution arguments and put it into the bisection cache. This
164  includes copying the created object file, adding the object
165  file path to the cache list and keeping a log of the execution.
166
167  Args:
168    execargs: compiler execution arguments.
169    bisect_dir: bisection directory.
170    population_name: name of the cache being populated (good/bad).
171  """
172  retval = exec_and_return(execargs)
173  if retval:
174    return retval
175
176  population_dir = os.path.join(bisect_dir, population_name)
177  makedirs(population_dir)
178  pop_log = os.path.join(population_dir, '_POPULATE_LOG')
179  log_to_file(pop_log, execargs)
180
181  obj_path, _ = get_obj_path(execargs)
182  if not obj_path:
183    return
184
185  bisect_path = os.path.join(population_dir, obj_path)
186  bisect_path_dir = os.path.dirname(bisect_path)
187  makedirs(bisect_path_dir)
188
189  try:
190    if os.path.exists(obj_path):
191      shutil.copy2(obj_path, bisect_path)
192      # Set cache object to be read-only so later compilations can't
193      # accidentally overwrite it.
194      os.chmod(bisect_path, 0444)
195  except Exception:
196    print('Could not populate bisect cache', file=sys.stderr)
197    raise
198
199  with open(os.path.join(population_dir, '_LIST'), 'a') as object_list:
200    object_list.write('%s\n' % obj_path)
201
202  # Cache the side effects generated by good compiler
203  if population_name == GOOD_CACHE:
204    generate_side_effects(execargs, bisect_dir)
205
206
207def bisect_triage(execargs, bisect_dir):
208  obj_path, _ = get_obj_path(execargs)
209  obj_list = os.path.join(bisect_dir, LIST_FILE)
210
211  # If the output isn't an object file just call compiler
212  if not obj_path:
213    return exec_and_return(execargs)
214
215  # If this isn't a bisected object just call compiler
216  # This shouldn't happen!
217  if not in_object_list(obj_path, obj_list):
218    if CONTINUE_ON_MISSING:
219      log_file = os.path.join(bisect_dir, '_MISSING_CACHED_OBJ_LOG')
220      log_to_file(log_file, execargs, link_from='? compiler', link_to=obj_path)
221      return exec_and_return(execargs)
222    else:
223      raise Error(('%s is missing from cache! To ignore export '
224                   'BISECT_CONTINUE_ON_MISSING=1. See documentation for more '
225                   'details on this option.' % obj_path))
226
227  # Generate compiler side effects. Trick Make into thinking compiler was
228  # actually executed.
229
230  # If dependency is generated from this call, link it from dependency cache
231  dep_path, full_dep_path = get_dep_path(execargs)
232  if dep_path:
233    cached_dep_path = os.path.join(bisect_dir, DEP_CACHE, dep_path)
234    if os.path.exists(cached_dep_path):
235      if os.path.exists(full_dep_path):
236        os.remove(full_dep_path)
237      os.link(cached_dep_path, full_dep_path)
238    else:
239      raise Error(('%s is missing from dependency cache! Unsure how to '
240                   'proceed. Make will now crash.' % cached_dep_path))
241
242  # If generated object file happened to be pruned/cleaned by Make then link it
243  # over from cache again.
244  if not os.path.exists(obj_path):
245    cache = BAD_CACHE if in_bad_set(obj_path) else GOOD_CACHE
246    cached_obj_path = os.path.join(bisect_dir, cache, obj_path)
247    if os.path.exists(cached_obj_path):
248      os.link(cached_obj_path, obj_path)
249    else:
250      raise Error('%s does not exist in %s cache' % (obj_path, cache))
251
252    # This is just used for debugging and stats gathering
253    log_file = os.path.join(bisect_dir, '_MISSING_OBJ_LOG')
254    log_to_file(log_file, execargs, link_from=cached_obj_path, link_to=obj_path)
255
256
257def bisect_driver(bisect_stage, bisect_dir, execargs):
258  """Call appropriate bisection stage according to value in bisect_stage."""
259  if bisect_stage == 'POPULATE_GOOD':
260    bisect_populate(execargs, bisect_dir, GOOD_CACHE)
261  elif bisect_stage == 'POPULATE_BAD':
262    bisect_populate(execargs, bisect_dir, BAD_CACHE)
263  elif bisect_stage == 'TRIAGE':
264    bisect_triage(execargs, bisect_dir)
265  else:
266    raise ValueError('wrong value for BISECT_STAGE: %s' % bisect_stage)
267