binary_search_perforce.py revision 77efb771911728cb6911f4f1b1741a31341ed2da 
1#!/usr/bin/python2
2"""Module of binary serch for perforce."""
3from __future__ import print_function
4
5import math
6import argparse
7import os
8import re
9import sys
10import tempfile
11
12from cros_utils import command_executer
13from cros_utils import logger
14
15verbose = True
16
17def _GetP4ClientSpec(client_name, p4_paths):
18  p4_string = ""
19  for p4_path in p4_paths:
20    if " " not in p4_path:
21      p4_string += " -a %s" % p4_path
22    else:
23      p4_string += " -a \"" + (" //" + client_name + "/").join(p4_path) + "\""
24
25  return p4_string
26
27
28def GetP4Command(client_name, p4_port, p4_paths, checkoutdir,
29                 p4_snapshot=""):
30  command = ""
31
32  if p4_snapshot:
33    command += "mkdir -p " + checkoutdir
34    for p4_path in p4_paths:
35      real_path = p4_path[1]
36      if real_path.endswith("..."):
37        real_path = real_path.replace("/...", "")
38        command += ("; mkdir -p " + checkoutdir + "/" +
39                    os.path.dirname(real_path))
40        command += ("&& rsync -lr " + p4_snapshot + "/" + real_path +
41                    " " + checkoutdir + "/" + os.path.dirname(real_path))
42    return command
43
44  command += " export P4CONFIG=.p4config"
45  command += " && mkdir -p " + checkoutdir
46  command += " && cd " + checkoutdir
47  command += " && cp ${HOME}/.p4config ."
48  command += " && chmod u+w .p4config"
49  command += " && echo \"P4PORT=" + p4_port + "\" >> .p4config"
50  command += " && echo \"P4CLIENT=" + client_name + "\" >> .p4config"
51  command += (" && g4 client " +
52              _GetP4ClientSpec(client_name, p4_paths))
53  command += " && g4 sync "
54  command += " && cd -"
55  return command
56
57class BinarySearchPoint(object):
58  """Class of binary search point."""
59  def __init__(self, revision, status, tag=None):
60    self.revision = revision
61    self.status = status
62    self.tag = tag
63
64class BinarySearcher(object):
65  """Class of binary searcher."""
66  def __init__(self, logger_to_set=None):
67    self.sorted_list = []
68    self.index_log = []
69    self.status_log = []
70    self.skipped_indices = []
71    self.current = 0
72    self.points = {}
73    self.lo = 0
74    self.hi = 0
75    if logger_to_set is not None:
76      self.logger = logger_to_set
77    else:
78      self.logger = logger.GetLogger()
79
80  def SetSortedList(self, sorted_list):
81    assert len(sorted_list) > 0
82    self.sorted_list = sorted_list
83    self.index_log = []
84    self.hi = len(sorted_list) - 1
85    self.lo = 0
86    self.points = {}
87    for i in range(len(self.sorted_list)):
88      bsp = BinarySearchPoint(self.sorted_list[i], -1, "Not yet done.")
89      self.points[i] = bsp
90
91  def SetStatus(self, status, tag=None):
92    message = ("Revision: %s index: %d returned: %d" %
93               (self.sorted_list[self.current],
94                self.current,
95                status))
96    self.logger.LogOutput(message, print_to_console=verbose)
97    assert status == 0 or status == 1 or status == 2
98    self.index_log.append(self.current)
99    self.status_log.append(status)
100    bsp = BinarySearchPoint(self.sorted_list[self.current], status, tag)
101    self.points[self.current] = bsp
102
103    if status == 2:
104      self.skipped_indices.append(self.current)
105
106    if status == 0 or status == 1:
107      if status == 0:
108        self.lo = self.current + 1
109      elif status == 1:
110        self.hi = self.current
111      self.logger.LogOutput("lo: %d hi: %d\n" % (self.lo, self.hi))
112      self.current = (self.lo + self.hi)/2
113
114    if self.lo == self.hi:
115      message = ("Search complete. First bad version: %s"
116                 " at index: %d" %
117                 (self.sorted_list[self.current],
118                  self.lo))
119      self.logger.LogOutput(message)
120      return True
121
122    for index in range(self.lo, self.hi):
123      if index not in self.skipped_indices:
124        return False
125    self.logger.LogOutput(
126        "All skipped indices between: %d and %d\n" % (self.lo, self.hi),
127        print_to_console=verbose)
128    return True
129
130  # Does a better job with chromeos flakiness.
131  def GetNextFlakyBinary(self):
132    t = (self.lo, self.current, self.hi)
133    q = [t]
134    while len(q):
135      element = q.pop(0)
136      if element[1] in self.skipped_indices:
137        # Go top
138        to_add = (element[0], (element[0] + element[1]) / 2, element[1])
139        q.append(to_add)
140        # Go bottom
141        to_add = (element[1], (element[1] + element[2]) / 2, element[2])
142        q.append(to_add)
143      else:
144        self.current = element[1]
145        return
146    assert len(q), "Queue should never be 0-size!"
147
148  def GetNextFlakyLinear(self):
149    current_hi = self.current
150    current_lo = self.current
151    while True:
152      if current_hi < self.hi and current_hi not in self.skipped_indices:
153        self.current = current_hi
154        break
155      if current_lo >= self.lo and current_lo not in self.skipped_indices:
156        self.current = current_lo
157        break
158      if current_lo < self.lo and current_hi >= self.hi:
159        break
160
161      current_hi += 1
162      current_lo -= 1
163
164  def GetNext(self):
165    self.current = (self.hi + self.lo) / 2
166    # Try going forward if current is skipped.
167    if self.current in self.skipped_indices:
168      self.GetNextFlakyBinary()
169
170    # TODO: Add an estimated time remaining as well.
171    message = ("Estimated tries: min: %d max: %d\n" %
172               (1 + math.log(self.hi - self.lo, 2),
173                self.hi - self.lo - len(self.skipped_indices)))
174    self.logger.LogOutput(message, print_to_console=verbose)
175    message = ("lo: %d hi: %d current: %d version: %s\n" %
176               (self.lo, self.hi, self.current,
177                self.sorted_list[self.current]))
178    self.logger.LogOutput(message, print_to_console=verbose)
179    self.logger.LogOutput(str(self), print_to_console=verbose)
180    return self.sorted_list[self.current]
181
182  def SetLoRevision(self, lo_revision):
183    self.lo = self.sorted_list.index(lo_revision)
184  def SetHiRevision(self, hi_revision):
185    self.hi = self.sorted_list.index(hi_revision)
186  def GetAllPoints(self):
187    to_return = ""
188    for i in range(len(self.sorted_list)):
189      to_return += ("%d %d %s\n" %
190                    (self.points[i].status,
191                     i,
192                     self.points[i].revision))
193
194    return to_return
195  def __str__(self):
196    to_return = ""
197    to_return += "Current: %d\n" % self.current
198    to_return += str(self.index_log) + "\n"
199    revision_log = []
200    for index in self.index_log:
201      revision_log.append(self.sorted_list[index])
202    to_return += str(revision_log) + "\n"
203    to_return += str(self.status_log) + "\n"
204    to_return += "Skipped indices:\n"
205    to_return += str(self.skipped_indices) + "\n"
206    to_return += self.GetAllPoints()
207    return to_return
208
209class RevisionInfo(object):
210  """Class of reversion info."""
211  def __init__(self, date, client, description):
212    self.date = date
213    self.client = client
214    self.description = description
215    self.status = -1
216
217class VCSBinarySearcher(object):
218  """Class of VCS binary searcher."""
219  def __init__(self):
220    self.bs = BinarySearcher()
221    self.rim = {}
222    self.current_ce = None
223    self.checkout_dir = None
224    self.current_revision = None
225
226  def Initialize(self):
227    pass
228  def GetNextRevision(self):
229    pass
230  def CheckoutRevision(self, revision):
231    pass
232  def SetStatus(self, status):
233    pass
234  def Cleanup(self):
235    pass
236  def SetGoodRevision(self, revision):
237    if revision is None:
238      return
239    assert revision in self.bs.sorted_list
240    self.bs.SetLoRevision(revision)
241  def SetBadRevision(self, revision):
242    if revision is None:
243      return
244    assert revision in self.bs.sorted_list
245    self.bs.SetHiRevision(revision)
246
247class P4BinarySearcher(VCSBinarySearcher):
248  """Class of P4 binary searcher."""
249  def __init__(self, p4_port, p4_paths, test_command):
250    VCSBinarySearcher.__init__(self)
251    self.p4_port = p4_port
252    self.p4_paths = p4_paths
253    self.test_command = test_command
254    self.checkout_dir = tempfile.mkdtemp()
255    self.ce = command_executer.GetCommandExecuter()
256    self.client_name = "binary-searcher-$HOSTNAME-$USER"
257    self.job_log_root = "/home/asharif/www/coreboot_triage/"
258    self.changes = None
259
260  def Initialize(self):
261    self.Cleanup()
262    command = GetP4Command(self.client_name, self.p4_port,
263                           self.p4_paths, 1, self.checkout_dir)
264    self.ce.RunCommand(command)
265    command = "cd %s && g4 changes ..." % self.checkout_dir
266    _, out, _ = self.ce.RunCommandWOutput(command)
267    self.changes = re.findall(r"Change (\d+)", out)
268    change_infos = re.findall(r"Change (\d+) on ([\d/]+) by "
269                              r"([^\s]+) ('[^']*')", out)
270    for change_info in change_infos:
271      ri = RevisionInfo(change_info[1], change_info[2], change_info[3])
272      self.rim[change_info[0]] = ri
273    # g4 gives changes in reverse chronological order.
274    self.changes.reverse()
275    self.bs.SetSortedList(self.changes)
276  def SetStatus(self, status):
277    self.rim[self.current_revision].status = status
278    return self.bs.SetStatus(status)
279  def GetNextRevision(self):
280    next_revision = self.bs.GetNext()
281    self.current_revision = next_revision
282    return next_revision
283  def CleanupCLs(self):
284    if not os.path.isfile(self.checkout_dir + "/.p4config"):
285      command = "cd %s" % self.checkout_dir
286      command += " && cp ${HOME}/.p4config ."
287      command += " && echo \"P4PORT=" + self.p4_port + "\" >> .p4config"
288      command += " && echo \"P4CLIENT=" + self.client_name + "\" >> .p4config"
289      self.ce.RunCommand(command)
290    command = "cd %s" % self.checkout_dir
291    command += "; g4 changes -c %s" % self.client_name
292    _, out, _ = self.ce.RunCommandWOUTPUOT(command)
293    changes = re.findall(r"Change (\d+)", out)
294    if len(changes) != 0:
295      command = "cd %s" % self.checkout_dir
296      for change in changes:
297        command += "; g4 revert -c %s" % change
298      self.ce.RunCommand(command)
299
300  def CleanupClient(self):
301    command = "cd %s" % self.checkout_dir
302    command += "; g4 revert ..."
303    command += "; g4 client -d %s" % self.client_name
304    self.ce.RunCommand(command)
305
306  def Cleanup(self):
307    self.CleanupCLs()
308    self.CleanupClient()
309
310  def __str__(self):
311    to_return = ""
312    for change in self.changes:
313      ri = self.rim[change]
314      if ri.status == -1:
315        to_return = "%s\t%d\n" % (change, ri.status)
316      else:
317        to_return += ("%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s\n" %
318                      (change,
319                       ri.status,
320                       ri.date,
321                       ri.client,
322                       ri.description,
323                       self.job_log_root + change + ".cmd",
324                       self.job_log_root + change + ".out",
325                       self.job_log_root + change + ".err"))
326    return to_return
327
328
329
330class P4GCCBinarySearcher(P4BinarySearcher):
331  """Class of P4 gcc binary searcher."""
332  # TODO: eventually get these patches from g4 instead of creating them manually
333  def HandleBrokenCLs(self, current_revision):
334    cr = int(current_revision)
335    problematic_ranges = []
336    problematic_ranges.append([44528, 44539])
337    problematic_ranges.append([44528, 44760])
338    problematic_ranges.append([44335, 44882])
339    command = "pwd"
340    for pr in problematic_ranges:
341      if cr in range(pr[0], pr[1]):
342        patch_file = "/home/asharif/triage_tool/%d-%d.patch" % (pr[0], pr[1])
343        f = open(patch_file)
344        patch = f.read()
345        f.close()
346        files = re.findall("--- (//.*)", patch)
347        command += "; cd %s" % self.checkout_dir
348        for f in files:
349          command += "; g4 open %s" % f
350        command += "; patch -p2 < %s" % patch_file
351    self.current_ce.RunCommand(command)
352
353  def CheckoutRevision(self, current_revision):
354    job_logger = logger.Logger(self.job_log_root,
355                               current_revision,
356                               True, subdir="")
357    self.current_ce = command_executer.GetCommandExecuter(job_logger)
358
359    self.CleanupCLs()
360    # Change the revision of only the gcc part of the toolchain.
361    command = ("cd %s/gcctools/google_vendor_src_branch/gcc "
362               "&& g4 revert ...; g4 sync @%s" %
363               (self.checkout_dir, current_revision))
364    self.current_ce.RunCommand(command)
365
366    self.HandleBrokenCLs(current_revision)
367
368
369def Main(argv):
370  """The main function."""
371  # Common initializations
372###  command_executer.InitCommandExecuter(True)
373  ce = command_executer.GetCommandExecuter()
374
375  parser = argparse.ArgumentParser()
376  parser.add_argument("-n", "--num_tries", dest="num_tries",
377                      default="100",
378                      help="Number of tries.")
379  parser.add_argument("-g", "--good_revision", dest="good_revision",
380                      help="Last known good revision.")
381  parser.add_argument("-b", "--bad_revision", dest="bad_revision",
382                      help="Last known bad revision.")
383  parser.add_argument("-s",
384                      "--script",
385                      dest="script",
386                      help="Script to run for every version.")
387  options = parser.parse_args(argv)
388  # First get all revisions
389  p4_paths = ["//depot2/gcctools/google_vendor_src_branch/gcc/gcc-4.4.3/...",
390              "//depot2/gcctools/google_vendor_src_branch/binutils/"
391              "binutils-2.20.1-mobile/...",
392              "//depot2/gcctools/google_vendor_src_branch/"
393              "binutils/binutils-20100303/..."]
394  p4gccbs = P4GCCBinarySearcher("perforce2:2666", p4_paths, "")
395
396
397  # Main loop:
398  terminated = False
399  num_tries = int(options.num_tries)
400  script = os.path.expanduser(options.script)
401
402  try:
403    p4gccbs.Initialize()
404    p4gccbs.SetGoodRevision(options.good_revision)
405    p4gccbs.SetBadRevision(options.bad_revision)
406    while not terminated and num_tries > 0:
407      current_revision = p4gccbs.GetNextRevision()
408
409      # Now run command to get the status
410      ce = command_executer.GetCommandExecuter()
411      command = "%s %s" % (script, p4gccbs.checkout_dir)
412      status = ce.RunCommand(command)
413      message = ("Revision: %s produced: %d status\n" %
414                 (current_revision, status))
415      logger.GetLogger().LogOutput(message, print_to_console=verbose)
416      terminated = p4gccbs.SetStatus(status)
417      num_tries -= 1
418      logger.GetLogger().LogOutput(str(p4gccbs), print_to_console=verbose)
419
420    if not terminated:
421      logger.GetLogger().LogOutput("Tries: %d expired." % num_tries,
422                                   print_to_console=verbose)
423    logger.GetLogger().LogOutput(str(p4gccbs.bs),
424                                 print_to_console=verbose)
425  except (KeyboardInterrupt, SystemExit):
426    logger.GetLogger().LogOutput("Cleaning up...")
427  finally:
428    logger.GetLogger().LogOutput(str(p4gccbs.bs),
429                                 print_to_console=verbose)
430    status = p4gccbs.Cleanup()
431
432
433if __name__ == "__main__":
434  Main(sys.argv[1:])
435