1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7
8"""
9submit_try: Submit a try request.
10
11This is a thin wrapper around the try request utilities in depot_tools which
12adds some validation and supports both git and svn.
13"""
14
15
16import httplib
17import json
18import os
19import re
20import shutil
21import subprocess
22import svn
23import sys
24import tempfile
25
26import retrieve_from_googlesource
27
28
29# Alias which can be used to run a try on every builder.
30ALL_BUILDERS = 'all'
31# Alias which can be used to run a try on all compile builders.
32COMPILE_BUILDERS = 'compile'
33# Alias which can be used to run a try on all builders that are run in the CQ.
34CQ_BUILDERS = 'cq'
35# Alias which can be used to specify a regex to choose builders.
36REGEX = 'regex'
37
38ALL_ALIASES = [ALL_BUILDERS, COMPILE_BUILDERS, REGEX, CQ_BUILDERS]
39
40LARGE_NUMBER_OF_BOTS = 5
41
42GIT = 'git.bat' if os.name == 'nt' else 'git'
43
44# URL of the slaves.cfg file in the Skia buildbot sources.
45SKIA_REPO = 'https://skia.googlesource.com/buildbot'
46SLAVES_CFG_PATH = 'master/slaves.cfg'
47
48# All try builders have this suffix.
49TRYBOT_SUFFIX = '-Trybot'
50
51# String for matching the svn url of the try server inside codereview.settings.
52TRYSERVER_SVN_URL = 'TRYSERVER_SVN_URL: '
53
54# Strings used for matching svn config properties.
55URL_STR = 'URL'
56REPO_ROOT_STR = 'Repository Root'
57
58
59def FindDepotTools():
60  """ Find depot_tools on the local machine and return its location. """
61  which_cmd = 'where' if os.name == 'nt' else 'which'
62  cmd = [which_cmd, 'gcl']
63  proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
64  if proc.wait() != 0:
65    raise Exception('Couldn\'t find depot_tools in PATH!')
66  gcl = proc.communicate()[0].split('\n')[0].rstrip()
67  depot_tools_dir = os.path.dirname(gcl)
68  return depot_tools_dir
69
70
71def GetCheckoutRoot():
72  """ Determine where the local checkout is rooted."""
73  cmd = ['git', 'rev-parse', '--show-toplevel']
74  proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
75                          stderr=subprocess.STDOUT)
76  if proc.wait() != 0:
77    raise Exception('Couldn\'t find checkout root!')
78  return os.path.basename(proc.communicate()[0])
79
80
81def GetTryRepo():
82  """Determine the TRYSERVER_SVN_URL from the codereview.settings file."""
83  codereview_settings_file = os.path.join(os.path.dirname(__file__), os.pardir,
84                                          'codereview.settings')
85  with open(codereview_settings_file) as f:
86    for line in f:
87      if line.startswith(TRYSERVER_SVN_URL):
88        return line[len(TRYSERVER_SVN_URL):].rstrip()
89  raise Exception('Couldn\'t determine the TRYSERVER_SVN_URL. Make sure it is '
90                  'defined in the %s file.' % codereview_settings_file)
91
92
93def RetrieveTrybotList():
94  """Retrieve the list of known trybots from the checked-in buildbot
95  configuration."""
96  # Retrieve the slaves.cfg file from the repository.
97  slaves_cfg_text = retrieve_from_googlesource.get(SKIA_REPO, SLAVES_CFG_PATH)
98
99  # Execute the slaves.cfg file to obtain the list of slaves.
100  vars = {}
101  exec(slaves_cfg_text, vars)
102  slaves_cfg = vars['slaves']
103
104  # Pull the list of known builders from the slaves list.
105  trybots = set()
106  for slave in slaves_cfg:
107    for builder in slave['builder']:
108      if not builder.endswith(TRYBOT_SUFFIX):
109        trybots.add(builder)
110
111  return list(trybots), vars['cq_trybots']
112
113
114def ValidateArgs(argv, trybots, cq_trybots, is_svn=True):
115  """ Parse and validate command-line arguments. If the arguments are valid,
116  returns a tuple of (<changelist name>, <list of trybots>).
117
118  trybots: list of strings; A list of the known try builders.
119  cq_trybots: list of strings; Trybots who get run by the commit queue.
120  is_svn: bool; whether or not we're in an svn checkout.
121  """
122
123  class CollectedArgs(object):
124    def __init__(self, bots, changelist, revision):
125      self._bots = bots
126      self._changelist = changelist
127      self._revision = revision
128
129    @property
130    def bots(self):
131      for bot in self._bots:
132        yield bot
133
134    @property
135    def changelist(self):
136      return self._changelist
137
138    @property
139    def revision(self):
140      return self._revision
141
142  usage = (
143"""submit_try: Submit a try request.
144submit_try %s--bot <buildername> [<buildername> ...]
145
146-b, --bot           Builder(s) or Alias on which to run the try. Required.
147                    Allowed aliases: %s
148-h, --help          Show this message.
149-r <revision#>      Revision from which to run the try.
150-l, --list_bots     List the available try builders and aliases and exit.
151""" % ('<changelist> ' if is_svn else '', ALL_ALIASES))
152
153  def Error(msg=None):
154    if msg:
155      print msg
156    print usage
157    sys.exit(1)
158
159  using_bots = None
160  changelist = None
161  revision = None
162
163  while argv:
164    arg = argv.pop(0)
165    if arg == '-h' or arg == '--help':
166      Error()
167    elif arg == '-l' or arg == '--list_bots':
168      format_args = ['\n  '.join(sorted(trybots))] + \
169                    ALL_ALIASES + \
170                    ['\n    '.join(sorted(cq_trybots))]
171      print (
172"""
173submit_try: Available builders:\n  %s
174
175Can also use the following aliases to run on groups of builders-
176  %s: Will run against all trybots.
177  %s: Will run against all compile trybots.
178  %s: You will be prompted to enter a regex to select builders with.
179  %s: Will run against the same trybots as the commit queue:\n    %s
180
181""" % tuple(format_args))
182      sys.exit(0)
183    elif arg == '-b' or arg == '--bot':
184      if using_bots:
185        Error('--bot specified multiple times.')
186      if len(argv) < 1:
187        Error('You must specify a builder with "--bot".')
188      using_bots = []
189      while argv and not argv[0].startswith('-'):
190        for bot in argv.pop(0).split(','):
191          if bot in ALL_ALIASES:
192            if using_bots:
193              Error('Cannot specify "%s" with additional builder names or '
194                    'aliases.' % bot)
195            elif bot == COMPILE_BUILDERS:
196              using_bots = [t for t in trybots if t.startswith('Build')]
197            elif bot == CQ_BUILDERS:
198              using_bots = cq_trybots
199            elif bot == REGEX:
200              while True:
201                regex = raw_input("Enter your trybot regex: ")
202                p = re.compile(regex)
203                using_bots = [t for t in trybots if p.match(t)]
204                print '\n\nTrybots that match your regex:\n%s\n\n' % '\n'.join(
205                    using_bots)
206                if raw_input('Re-enter regex? [y,n]: ') == 'n':
207                  break
208            break
209          else:
210            if not bot in trybots:
211              Error('Unrecognized builder: %s' % bot)
212            using_bots.append(bot)
213    elif arg == '-r':
214      if len(argv) < 1:
215        Error('You must specify a revision with "-r".')
216      revision = argv.pop(0)
217    else:
218      if changelist or not is_svn:
219        Error('Unknown argument: %s' % arg)
220      changelist = arg
221  if is_svn and not changelist:
222    Error('You must specify a changelist name.')
223  if not using_bots:
224    Error('You must specify one or more builders using --bot.')
225  if len(using_bots) > LARGE_NUMBER_OF_BOTS:
226    are_you_sure = raw_input('Running a try on a large number of bots is very '
227                             'expensive. You may be able to get enough '
228                             'information by running on a smaller set of bots. '
229                             'Are you sure you want to do this? [y,n]: ')
230    if are_you_sure != 'y':
231      Error()
232  return CollectedArgs(bots=using_bots, changelist=changelist,
233                       revision=revision)
234
235
236def SubmitTryRequest(trybots, revision=None):
237  """ Submits a try request on the given list of trybots.
238
239  Args:
240      trybots: list of strings; the names of the try builders to run.
241      revision: optional string; the revision from which to run the try.
242  """
243  botlist = ','.join(['%s%s' % (bot, TRYBOT_SUFFIX) for bot in trybots])
244  # Find depot_tools. This is needed to import git_cl and trychange.
245  sys.path.append(FindDepotTools())
246  import git_cl
247  import trychange
248
249  cmd = [GIT, 'diff', git_cl.Changelist().GetUpstreamBranch(),
250         '--no-ext-diff']
251  proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
252  git_data = proc.communicate()
253  if git_data[0] is None:
254    raise Exception('Failed to capture git diff!')
255
256  temp_dir = tempfile.mkdtemp()
257  try:
258    diff_file = os.path.join(temp_dir, 'patch.diff')
259    with open(diff_file, 'wb') as f:
260      f.write(git_data[0])
261      f.close()
262
263    try_args = ['--use_svn',
264                '--svn_repo', GetTryRepo(),
265                '--root', GetCheckoutRoot(),
266                '--bot', botlist,
267                '--diff', diff_file,
268                ]
269    if revision:
270      try_args.extend(['-r', revision])
271
272    # Submit the try request.
273    trychange.TryChange(try_args, None, False)
274  finally:
275    shutil.rmtree(temp_dir)
276
277
278def main():
279  # Retrieve the list of active try builders from the build master.
280  trybots, cq_trybots = RetrieveTrybotList()
281
282  # Determine if we're in an SVN checkout.
283  is_svn = os.path.isdir('.svn')
284
285  # Parse and validate the command-line arguments.
286  args = ValidateArgs(sys.argv[1:], trybots=trybots, cq_trybots=cq_trybots,
287                      is_svn=is_svn)
288
289  # Submit the try request.
290  SubmitTryRequest(args.bots, args.revision)
291
292
293if __name__ == '__main__':
294  sys.exit(main())
295