buildbot_json.py revision ed69676d435b7b6983271ed8fab200627a0b966e 
1#!/usr/bin/env python2
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#    * Redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer.
10#    * Redistributions in binary form must reproduce the above
11# copyright notice, this list of conditions and the following disclaimer
12# in the documentation and/or other materials provided with the
13# distribution.
14#    * Neither the name of Google Inc. nor the names of its
15# contributors may be used to endorse or promote products derived from
16# this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30# NOTE: This file is NOT under GPL.  See above.
31"""Queries buildbot through the json interface.
32"""
33
34from __future__ import print_function
35
36__author__ = 'maruel@chromium.org'
37__version__ = '1.2'
38
39import code
40import datetime
41import functools
42import json
43
44# Pylint recommends we use "from chromite.lib import cros_logging as logging".
45# Chromite specific policy message, we want to keep using the standard logging.
46# pylint: disable=cros-logging-import
47import logging
48
49# pylint: disable=deprecated-module
50import optparse
51
52import time
53import urllib
54import urllib2
55import sys
56
57try:
58  from natsort import natsorted
59except ImportError:
60  # natsorted is a simple helper to sort "naturally", e.g. "vm40" is sorted
61  # after "vm7". Defaults to normal sorting.
62  natsorted = sorted
63
64# These values are buildbot constants used for Build and BuildStep.
65# This line was copied from master/buildbot/status/builder.py.
66SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY = range(6)
67
68## Generic node caching code.
69
70
71class Node(object):
72  """Root class for all nodes in the graph.
73
74  Provides base functionality for any node in the graph, independent if it has
75  children or not or if its content can be addressed through an url or needs to
76  be fetched as part of another node.
77
78  self.printable_attributes is only used for self documentation and for str()
79  implementation.
80  """
81  printable_attributes = []
82
83  def __init__(self, parent, url):
84    self.printable_attributes = self.printable_attributes[:]
85    if url:
86      self.printable_attributes.append('url')
87      url = url.rstrip('/')
88    if parent is not None:
89      self.printable_attributes.append('parent')
90    self.url = url
91    self.parent = parent
92
93  def __str__(self):
94    return self.to_string()
95
96  def __repr__(self):
97    """Embeds key if present."""
98    key = getattr(self, 'key', None)
99    if key is not None:
100      return '<%s key=%s>' % (self.__class__.__name__, key)
101    cached_keys = getattr(self, 'cached_keys', None)
102    if cached_keys is not None:
103      return '<%s keys=%s>' % (self.__class__.__name__, cached_keys)
104    return super(Node, self).__repr__()
105
106  def to_string(self, maximum=100):
107    out = ['%s:' % self.__class__.__name__]
108    assert not 'printable_attributes' in self.printable_attributes
109
110    def limit(txt):
111      txt = str(txt)
112      if maximum > 0:
113        if len(txt) > maximum + 2:
114          txt = txt[:maximum] + '...'
115      return txt
116
117    for k in sorted(self.printable_attributes):
118      if k == 'parent':
119        # Avoid infinite recursion.
120        continue
121      out.append(limit('  %s: %r' % (k, getattr(self, k))))
122    return '\n'.join(out)
123
124  def refresh(self):
125    """Refreshes the data."""
126    self.discard()
127    return self.cache()
128
129  def cache(self):  # pragma: no cover
130    """Caches the data."""
131    raise NotImplementedError()
132
133  def discard(self):  # pragma: no cover
134    """Discards cached data.
135
136    Pretty much everything is temporary except completed Build.
137    """
138    raise NotImplementedError()
139
140
141class AddressableBaseDataNode(Node):  # pylint: disable=W0223
142  """A node that contains a dictionary of data that can be fetched with an url.
143
144  The node is directly addressable. It also often can be fetched by the parent.
145  """
146  printable_attributes = Node.printable_attributes + ['data']
147
148  def __init__(self, parent, url, data):
149    super(AddressableBaseDataNode, self).__init__(parent, url)
150    self._data = data
151
152  @property
153  def cached_data(self):
154    return self._data
155
156  @property
157  def data(self):
158    self.cache()
159    return self._data
160
161  def cache(self):
162    if self._data is None:
163      self._data = self._readall()
164      return True
165    return False
166
167  def discard(self):
168    self._data = None
169
170  def read(self, suburl):
171    assert self.url, self.__class__.__name__
172    url = self.url
173    if suburl:
174      url = '%s/%s' % (self.url, suburl)
175    return self.parent.read(url)
176
177  def _readall(self):
178    return self.read('')
179
180
181class AddressableDataNode(AddressableBaseDataNode):  # pylint: disable=W0223
182  """Automatically encodes the url."""
183
184  def __init__(self, parent, url, data):
185    super(AddressableDataNode, self).__init__(parent, urllib.quote(url), data)
186
187
188class NonAddressableDataNode(Node):  # pylint: disable=W0223
189  """A node that cannot be addressed by an unique url.
190
191  The data comes directly from the parent.
192  """
193
194  def __init__(self, parent, subkey):
195    super(NonAddressableDataNode, self).__init__(parent, None)
196    self.subkey = subkey
197
198  @property
199  def cached_data(self):
200    if self.parent.cached_data is None:
201      return None
202    return self.parent.cached_data[self.subkey]
203
204  @property
205  def data(self):
206    return self.parent.data[self.subkey]
207
208  def cache(self):
209    self.parent.cache()
210
211  def discard(self):  # pragma: no cover
212    """Avoid invalid state when parent recreate the object."""
213    raise AttributeError('Call parent discard() instead')
214
215
216class VirtualNodeList(Node):
217  """Base class for every node that has children.
218
219  Adds partial supports for keys and iterator functionality. 'key' can be a
220  string or a int. Not to be used directly.
221  """
222  printable_attributes = Node.printable_attributes + ['keys']
223
224  def __init__(self, parent, url):
225    super(VirtualNodeList, self).__init__(parent, url)
226    # Keeps the keys independently when ordering is needed.
227    self._is_cached = False
228    self._has_keys_cached = False
229
230  def __contains__(self, key):
231    """Enables 'if i in obj:'."""
232    return key in self.keys
233
234  def __iter__(self):
235    """Enables 'for i in obj:'. It returns children."""
236    self.cache_keys()
237    for key in self.keys:
238      yield self[key]
239
240  def __len__(self):
241    """Enables 'len(obj)' to get the number of childs."""
242    return len(self.keys)
243
244  def discard(self):
245    """Discards data.
246
247    The default behavior is to not invalidate cached keys. The only place where
248    keys need to be invalidated is with Builds.
249    """
250    self._is_cached = False
251    self._has_keys_cached = False
252
253  @property
254  def cached_children(self):  # pragma: no cover
255    """Returns an iterator over the children that are cached."""
256    raise NotImplementedError()
257
258  @property
259  def cached_keys(self):  # pragma: no cover
260    raise NotImplementedError()
261
262  @property
263  def keys(self):  # pragma: no cover
264    """Returns the keys for every children."""
265    raise NotImplementedError()
266
267  def __getitem__(self, key):  # pragma: no cover
268    """Returns a child, without fetching its data.
269
270    The children could be invalid since no verification is done.
271    """
272    raise NotImplementedError()
273
274  def cache(self):  # pragma: no cover
275    """Cache all the children."""
276    raise NotImplementedError()
277
278  def cache_keys(self):  # pragma: no cover
279    """Cache all children's keys."""
280    raise NotImplementedError()
281
282
283class NodeList(VirtualNodeList):  # pylint: disable=W0223
284  """Adds a cache of the keys."""
285
286  def __init__(self, parent, url):
287    super(NodeList, self).__init__(parent, url)
288    self._keys = []
289
290  @property
291  def cached_keys(self):
292    return self._keys
293
294  @property
295  def keys(self):
296    self.cache_keys()
297    return self._keys
298
299
300class NonAddressableNodeList(VirtualNodeList):  # pylint: disable=W0223
301  """A node that contains children but retrieves all its data from its parent.
302
303  I.e. there's no url to get directly this data.
304  """
305  # Child class object for children of this instance. For example, BuildSteps
306  # has BuildStep children.
307  _child_cls = None
308
309  def __init__(self, parent, subkey):
310    super(NonAddressableNodeList, self).__init__(parent, None)
311    self.subkey = subkey
312    assert (not isinstance(self._child_cls, NonAddressableDataNode) and
313            issubclass(self._child_cls, NonAddressableDataNode)), (
314                self._child_cls.__name__)
315
316  @property
317  def cached_children(self):
318    if self.parent.cached_data is not None:
319      for i in xrange(len(self.parent.cached_data[self.subkey])):
320        yield self[i]
321
322  @property
323  def cached_data(self):
324    if self.parent.cached_data is None:
325      return None
326    return self.parent.data.get(self.subkey, None)
327
328  @property
329  def cached_keys(self):
330    if self.parent.cached_data is None:
331      return None
332    return range(len(self.parent.data.get(self.subkey, [])))
333
334  @property
335  def data(self):
336    return self.parent.data[self.subkey]
337
338  def cache(self):
339    self.parent.cache()
340
341  def cache_keys(self):
342    self.parent.cache()
343
344  def discard(self):  # pragma: no cover
345    """Do not call.
346
347    Avoid infinite recursion by having the caller calls the parent's
348    discard() explicitely.
349    """
350    raise AttributeError('Call parent discard() instead')
351
352  def __iter__(self):
353    """Enables 'for i in obj:'. It returns children."""
354    if self.data:
355      for i in xrange(len(self.data)):
356        yield self[i]
357
358  def __getitem__(self, key):
359    """Doesn't cache the value, it's not needed.
360
361    TODO(maruel): Cache?
362    """
363    if isinstance(key, int) and key < 0:
364      key = len(self.data) + key
365    # pylint: disable=E1102
366    return self._child_cls(self, key)
367
368
369class AddressableNodeList(NodeList):
370  """A node that has children that can be addressed with an url."""
371
372  # Child class object for children of this instance. For example, Builders has
373  # Builder children and Builds has Build children.
374  _child_cls = None
375
376  def __init__(self, parent, url):
377    super(AddressableNodeList, self).__init__(parent, url)
378    self._cache = {}
379    assert (not isinstance(self._child_cls, AddressableDataNode) and
380            issubclass(self._child_cls, AddressableDataNode)), (
381                self._child_cls.__name__)
382
383  @property
384  def cached_children(self):
385    for item in self._cache.itervalues():
386      if item.cached_data is not None:
387        yield item
388
389  @property
390  def cached_keys(self):
391    return self._cache.keys()
392
393  def __getitem__(self, key):
394    """Enables 'obj[i]'."""
395    if self._has_keys_cached and not key in self._keys:
396      raise KeyError(key)
397
398    if not key in self._cache:
399      # Create an empty object.
400      self._create_obj(key, None)
401    return self._cache[key]
402
403  def cache(self):
404    if not self._is_cached:
405      data = self._readall()
406      for key in sorted(data):
407        self._create_obj(key, data[key])
408      self._is_cached = True
409      self._has_keys_cached = True
410
411  def cache_partial(self, children):
412    """Caches a partial number of children.
413
414    This method is more efficient since it does a single request for all the
415    children instead of one request per children.
416
417    It only grab objects not already cached.
418    """
419    # pylint: disable=W0212
420    if not self._is_cached:
421      to_fetch = [
422          child
423          for child in children
424          if not (child in self._cache and self._cache[child].cached_data)
425      ]
426      if to_fetch:
427        # Similar to cache(). The only reason to sort is to simplify testing.
428        params = '&'.join('select=%s' % urllib.quote(str(v))
429                          for v in sorted(to_fetch))
430        data = self.read('?' + params)
431        for key in sorted(data):
432          self._create_obj(key, data[key])
433
434  def cache_keys(self):
435    """Implement to speed up enumeration. Defaults to call cache()."""
436    if not self._has_keys_cached:
437      self.cache()
438      assert self._has_keys_cached
439
440  def discard(self):
441    """Discards temporary children."""
442    super(AddressableNodeList, self).discard()
443    for v in self._cache.itervalues():
444      v.discard()
445
446  def read(self, suburl):
447    assert self.url, self.__class__.__name__
448    url = self.url
449    if suburl:
450      url = '%s/%s' % (self.url, suburl)
451    return self.parent.read(url)
452
453  def _create_obj(self, key, data):
454    """Creates an object of type self._child_cls."""
455    # pylint: disable=E1102
456    obj = self._child_cls(self, key, data)
457    # obj.key and key may be different.
458    # No need to overide cached data with None.
459    if data is not None or obj.key not in self._cache:
460      self._cache[obj.key] = obj
461    if obj.key not in self._keys:
462      self._keys.append(obj.key)
463
464  def _readall(self):
465    return self.read('')
466
467
468class SubViewNodeList(VirtualNodeList):  # pylint: disable=W0223
469  """A node that shows a subset of children that comes from another structure.
470
471  The node is not addressable.
472
473  E.g. the keys are retrieved from parent but the actual data comes from
474  virtual_parent.
475  """
476
477  def __init__(self, parent, virtual_parent, subkey):
478    super(SubViewNodeList, self).__init__(parent, None)
479    self.subkey = subkey
480    self.virtual_parent = virtual_parent
481    assert isinstance(self.parent, AddressableDataNode)
482    assert isinstance(self.virtual_parent, NodeList)
483
484  @property
485  def cached_children(self):
486    if self.parent.cached_data is not None:
487      for item in self.keys:
488        if item in self.virtual_parent.keys:
489          child = self[item]
490          if child.cached_data is not None:
491            yield child
492
493  @property
494  def cached_keys(self):
495    return (self.parent.cached_data or {}).get(self.subkey, [])
496
497  @property
498  def keys(self):
499    self.cache_keys()
500    return self.parent.data.get(self.subkey, [])
501
502  def cache(self):
503    """Batch request for each child in a single read request."""
504    if not self._is_cached:
505      self.virtual_parent.cache_partial(self.keys)
506      self._is_cached = True
507
508  def cache_keys(self):
509    if not self._has_keys_cached:
510      self.parent.cache()
511      self._has_keys_cached = True
512
513  def discard(self):
514    if self.parent.cached_data is not None:
515      for child in self.virtual_parent.cached_children:
516        if child.key in self.keys:
517          child.discard()
518      self.parent.discard()
519    super(SubViewNodeList, self).discard()
520
521  def __getitem__(self, key):
522    """Makes sure the key is in our key but grab it from the virtual parent."""
523    return self.virtual_parent[key]
524
525  def __iter__(self):
526    self.cache()
527    return super(SubViewNodeList, self).__iter__()
528
529###############################################################################
530## Buildbot-specific code
531
532
533class Slave(AddressableDataNode):
534  """Buildbot slave class."""
535  printable_attributes = AddressableDataNode.printable_attributes + [
536      'name',
537      'key',
538      'connected',
539      'version',
540  ]
541
542  def __init__(self, parent, name, data):
543    super(Slave, self).__init__(parent, name, data)
544    self.name = name
545    self.key = self.name
546    # TODO(maruel): Add SlaveBuilders and a 'builders' property.
547    # TODO(maruel): Add a 'running_builds' property.
548
549  @property
550  def connected(self):
551    return self.data.get('connected', False)
552
553  @property
554  def version(self):
555    return self.data.get('version')
556
557
558class Slaves(AddressableNodeList):
559  """Buildbot slaves."""
560  _child_cls = Slave
561  printable_attributes = AddressableNodeList.printable_attributes + ['names']
562
563  def __init__(self, parent):
564    super(Slaves, self).__init__(parent, 'slaves')
565
566  @property
567  def names(self):
568    return self.keys
569
570
571class BuilderSlaves(SubViewNodeList):
572  """Similar to Slaves but only list slaves connected to a specific builder."""
573  printable_attributes = SubViewNodeList.printable_attributes + ['names']
574
575  def __init__(self, parent):
576    super(BuilderSlaves, self).__init__(parent, parent.parent.parent.slaves,
577                                        'slaves')
578
579  @property
580  def names(self):
581    return self.keys
582
583
584class BuildStep(NonAddressableDataNode):
585  """Class for a buildbot build step."""
586  printable_attributes = NonAddressableDataNode.printable_attributes + [
587      'name',
588      'number',
589      'start_time',
590      'end_time',
591      'duration',
592      'is_started',
593      'is_finished',
594      'is_running',
595      'result',
596      'simplified_result',
597  ]
598
599  def __init__(self, parent, number):
600    """Pre-loaded, since the data is retrieved via the Build object."""
601    assert isinstance(number, int)
602    super(BuildStep, self).__init__(parent, number)
603    self.number = number
604
605  @property
606  def start_time(self):
607    if self.data.get('times'):
608      return int(round(self.data['times'][0]))
609
610  @property
611  def end_time(self):
612    times = self.data.get('times')
613    if times and len(times) == 2 and times[1]:
614      return int(round(times[1]))
615
616  @property
617  def duration(self):
618    if self.start_time:
619      return (self.end_time or int(round(time.time()))) - self.start_time
620
621  @property
622  def name(self):
623    return self.data['name']
624
625  @property
626  def is_started(self):
627    return self.data.get('isStarted', False)
628
629  @property
630  def is_finished(self):
631    return self.data.get('isFinished', False)
632
633  @property
634  def is_running(self):
635    return self.is_started and not self.is_finished
636
637  @property
638  def result(self):
639    result = self.data.get('results')
640    if result is None:
641      # results may be 0, in that case with filter=1, the value won't be
642      # present.
643      if self.data.get('isFinished'):
644        result = self.data.get('results', 0)
645    while isinstance(result, list):
646      result = result[0]
647    return result
648
649  @property
650  def simplified_result(self):
651    """Returns a simplified 3 state value, True, False or None."""
652    result = self.result
653    if result in (SUCCESS, WARNINGS):
654      return True
655    elif result in (FAILURE, EXCEPTION, RETRY):
656      return False
657    assert result in (None, SKIPPED), (result, self.data)
658    return None
659
660
661class BuildSteps(NonAddressableNodeList):
662  """Duplicates keys to support lookup by both step number and step name."""
663  printable_attributes = NonAddressableNodeList.printable_attributes + [
664      'failed',
665  ]
666  _child_cls = BuildStep
667
668  def __init__(self, parent):
669    """Pre-loaded, since the data is retrieved via the Build object."""
670    super(BuildSteps, self).__init__(parent, 'steps')
671
672  @property
673  def keys(self):
674    """Returns the steps name in order."""
675    return [i['name'] for i in self.data or []]
676
677  @property
678  def failed(self):
679    """Shortcuts that lists the step names of steps that failed."""
680    return [step.name for step in self if step.simplified_result is False]
681
682  def __getitem__(self, key):
683    """Accept step name in addition to index number."""
684    if isinstance(key, basestring):
685      # It's a string, try to find the corresponding index.
686      for i, step in enumerate(self.data):
687        if step['name'] == key:
688          key = i
689          break
690      else:
691        raise KeyError(key)
692    return super(BuildSteps, self).__getitem__(key)
693
694
695class Build(AddressableDataNode):
696  """Buildbot build info."""
697  printable_attributes = AddressableDataNode.printable_attributes + [
698      'key',
699      'number',
700      'steps',
701      'blame',
702      'reason',
703      'revision',
704      'result',
705      'simplified_result',
706      'start_time',
707      'end_time',
708      'duration',
709      'slave',
710      'properties',
711      'completed',
712  ]
713
714  def __init__(self, parent, key, data):
715    super(Build, self).__init__(parent, str(key), data)
716    self.number = int(key)
717    self.key = self.number
718    self.steps = BuildSteps(self)
719
720  @property
721  def blame(self):
722    return self.data.get('blame', [])
723
724  @property
725  def builder(self):
726    """Returns the Builder object.
727
728    Goes up the hierarchy to find the Buildbot.builders[builder] instance.
729    """
730    return self.parent.parent.parent.parent.builders[self.data['builderName']]
731
732  @property
733  def start_time(self):
734    if self.data.get('times'):
735      return int(round(self.data['times'][0]))
736
737  @property
738  def end_time(self):
739    times = self.data.get('times')
740    if times and len(times) == 2 and times[1]:
741      return int(round(times[1]))
742
743  @property
744  def duration(self):
745    if self.start_time:
746      return (self.end_time or int(round(time.time()))) - self.start_time
747
748  @property
749  def eta(self):
750    return self.data.get('eta', 0)
751
752  @property
753  def completed(self):
754    return self.data.get('currentStep') is None
755
756  @property
757  def properties(self):
758    return self.data.get('properties', [])
759
760  @property
761  def reason(self):
762    return self.data.get('reason')
763
764  @property
765  def result(self):
766    result = self.data.get('results')
767    while isinstance(result, list):
768      result = result[0]
769    if result is None and self.steps:
770      # results may be 0, in that case with filter=1, the value won't be
771      # present.
772      result = self.steps[-1].result
773    return result
774
775  @property
776  def revision(self):
777    return self.data.get('sourceStamp', {}).get('revision')
778
779  @property
780  def simplified_result(self):
781    """Returns a simplified 3 state value, True, False or None."""
782    result = self.result
783    if result in (SUCCESS, WARNINGS, SKIPPED):
784      return True
785    elif result in (FAILURE, EXCEPTION, RETRY):
786      return False
787    assert result is None, (result, self.data)
788    return None
789
790  @property
791  def slave(self):
792    """Returns the Slave object.
793
794    Goes up the hierarchy to find the Buildbot.slaves[slave] instance.
795    """
796    return self.parent.parent.parent.parent.slaves[self.data['slave']]
797
798  def discard(self):
799    """Completed Build isn't discarded."""
800    if self._data and self.result is None:
801      assert not self.steps or not self.steps[-1].data.get('isFinished')
802      self._data = None
803
804
805class CurrentBuilds(SubViewNodeList):
806  """Lists of the current builds."""
807
808  def __init__(self, parent):
809    super(CurrentBuilds, self).__init__(parent, parent.builds, 'currentBuilds')
810
811
812class PendingBuilds(AddressableDataNode):
813  """List of the pending builds."""
814
815  def __init__(self, parent):
816    super(PendingBuilds, self).__init__(parent, 'pendingBuilds', None)
817
818
819class Builds(AddressableNodeList):
820  """Supports iteration.
821
822  Recommends using .cache() to speed up if a significant number of builds are
823  iterated over.
824  """
825  _child_cls = Build
826
827  def __init__(self, parent):
828    super(Builds, self).__init__(parent, 'builds')
829
830  def __getitem__(self, key):
831    """Support for negative reference and enable retrieving non-cached builds.
832
833    e.g. -1 is the last build, -2 is the previous build before the last one.
834    """
835    key = int(key)
836    if key < 0:
837      # Convert negative to positive build number.
838      self.cache_keys()
839      # Since the negative value can be outside of the cache keys range, use the
840      # highest key value and calculate from it.
841      key = max(self._keys) + key + 1
842
843    if not key in self._cache:
844      # Create an empty object.
845      self._create_obj(key, None)
846    return self._cache[key]
847
848  def __iter__(self):
849    """Returns cached Build objects in reversed order.
850
851    The most recent build is returned first and then in reverse chronological
852    order, up to the oldest cached build by the server. Older builds can be
853    accessed but will trigger significantly more I/O so they are not included by
854    default in the iteration.
855
856    To access the older builds, use self.iterall() instead.
857    """
858    self.cache()
859    return reversed(self._cache.values())
860
861  def iterall(self):
862    """Returns Build objects in decreasing order unbounded up to build 0.
863
864    The most recent build is returned first and then in reverse chronological
865    order. Older builds can be accessed and will trigger significantly more I/O
866    so use this carefully.
867    """
868    # Only cache keys here.
869    self.cache_keys()
870    if self._keys:
871      for i in xrange(max(self._keys), -1, -1):
872        yield self[i]
873
874  def cache_keys(self):
875    """Grabs the keys (build numbers) from the builder."""
876    if not self._has_keys_cached:
877      for i in self.parent.data.get('cachedBuilds', []):
878        i = int(i)
879        self._cache.setdefault(i, Build(self, i, None))
880        if i not in self._keys:
881          self._keys.append(i)
882      self._has_keys_cached = True
883
884  def discard(self):
885    super(Builds, self).discard()
886    # Can't keep keys.
887    self._has_keys_cached = False
888
889  def _readall(self):
890    return self.read('_all')
891
892
893class Builder(AddressableDataNode):
894  """Builder status."""
895  printable_attributes = AddressableDataNode.printable_attributes + [
896      'name',
897      'key',
898      'builds',
899      'slaves',
900      'pending_builds',
901      'current_builds',
902  ]
903
904  def __init__(self, parent, name, data):
905    super(Builder, self).__init__(parent, name, data)
906    self.name = name
907    self.key = name
908    self.builds = Builds(self)
909    self.slaves = BuilderSlaves(self)
910    self.current_builds = CurrentBuilds(self)
911    self.pending_builds = PendingBuilds(self)
912
913  def discard(self):
914    super(Builder, self).discard()
915    self.builds.discard()
916    self.slaves.discard()
917    self.current_builds.discard()
918
919
920class Builders(AddressableNodeList):
921  """Root list of builders."""
922  _child_cls = Builder
923
924  def __init__(self, parent):
925    super(Builders, self).__init__(parent, 'builders')
926
927
928class Buildbot(AddressableBaseDataNode):
929  """This object should be recreated on a master restart as it caches data."""
930  # Throttle fetches to not kill the server.
931  auto_throttle = None
932  printable_attributes = AddressableDataNode.printable_attributes + [
933      'slaves',
934      'builders',
935      'last_fetch',
936  ]
937
938  def __init__(self, url):
939    super(Buildbot, self).__init__(None, url.rstrip('/') + '/json', None)
940    self._builders = Builders(self)
941    self._slaves = Slaves(self)
942    self.last_fetch = None
943
944  @property
945  def builders(self):
946    return self._builders
947
948  @property
949  def slaves(self):
950    return self._slaves
951
952  def discard(self):
953    """Discards information about Builders and Slaves."""
954    super(Buildbot, self).discard()
955    self._builders.discard()
956    self._slaves.discard()
957
958  def read(self, suburl):
959    if self.auto_throttle:
960      if self.last_fetch:
961        delta = datetime.datetime.utcnow() - self.last_fetch
962        remaining = (datetime.timedelta(seconds=self.auto_throttle) - delta)
963        if remaining > datetime.timedelta(seconds=0):
964          logging.debug('Sleeping for %ss', remaining)
965          time.sleep(remaining.seconds)
966      self.last_fetch = datetime.datetime.utcnow()
967    url = '%s/%s' % (self.url, suburl)
968    if '?' in url:
969      url += '&filter=1'
970    else:
971      url += '?filter=1'
972    logging.info('read(%s)', suburl)
973    channel = urllib.urlopen(url)
974    data = channel.read()
975    try:
976      return json.loads(data)
977    except ValueError:
978      if channel.getcode() >= 400:
979        # Convert it into an HTTPError for easier processing.
980        raise urllib2.HTTPError(url, channel.getcode(), '%s:\n%s' % (url, data),
981                                channel.headers, None)
982      raise
983
984  def _readall(self):
985    return self.read('project')
986
987###############################################################################
988## Controller code
989
990
991def usage(more):
992
993  def hook(fn):
994    fn.func_usage_more = more
995    return fn
996
997  return hook
998
999
1000def need_buildbot(fn):
1001  """Post-parse args to create a buildbot object."""
1002
1003  @functools.wraps(fn)
1004  def hook(parser, args, *extra_args, **kwargs):
1005    old_parse_args = parser.parse_args
1006
1007    def new_parse_args(args):
1008      options, args = old_parse_args(args)
1009      if len(args) < 1:
1010        parser.error('Need to pass the root url of the buildbot')
1011      url = args.pop(0)
1012      if not url.startswith('http'):
1013        url = 'http://' + url
1014      buildbot = Buildbot(url)
1015      buildbot.auto_throttle = options.throttle
1016      return options, args, buildbot
1017
1018    parser.parse_args = new_parse_args
1019    # Call the original function with the modified parser.
1020    return fn(parser, args, *extra_args, **kwargs)
1021
1022  hook.func_usage_more = '[options] <url>'
1023  return hook
1024
1025
1026@need_buildbot
1027def CMDpending(parser, args):
1028  """Lists pending jobs."""
1029  parser.add_option('-b',
1030                    '--builder',
1031                    dest='builders',
1032                    action='append',
1033                    default=[],
1034                    help='Builders to filter on')
1035  options, args, buildbot = parser.parse_args(args)
1036  if args:
1037    parser.error('Unrecognized parameters: %s' % ' '.join(args))
1038  if not options.builders:
1039    options.builders = buildbot.builders.keys
1040  for builder in options.builders:
1041    builder = buildbot.builders[builder]
1042    pending_builds = builder.data.get('pendingBuilds', 0)
1043    if not pending_builds:
1044      continue
1045    print('Builder %s: %d' % (builder.name, pending_builds))
1046    if not options.quiet:
1047      for pending in builder.pending_builds.data:
1048        if 'revision' in pending['source']:
1049          print('  revision: %s' % pending['source']['revision'])
1050        for change in pending['source']['changes']:
1051          print('  change:')
1052          print('    comment: %r' % unicode(change['comments'][:50]))
1053          print('    who:     %s' % change['who'])
1054  return 0
1055
1056
1057@usage('[options] <url> [commands] ...')
1058@need_buildbot
1059def CMDrun(parser, args):
1060  """Runs commands passed as parameters.
1061
1062  When passing commands on the command line, each command will be run as if it
1063  was on its own line.
1064  """
1065  parser.add_option('-f', '--file', help='Read script from file')
1066  parser.add_option('-i',
1067                    dest='use_stdin',
1068                    action='store_true',
1069                    help='Read script on stdin')
1070  # Variable 'buildbot' is not used directly.
1071  # pylint: disable=W0612
1072  options, args, buildbot = parser.parse_args(args)
1073  if (bool(args) + bool(options.use_stdin) + bool(options.file)) != 1:
1074    parser.error('Need to pass only one of: <commands>, -f <file> or -i')
1075  if options.use_stdin:
1076    cmds = sys.stdin.read()
1077  elif options.file:
1078    cmds = open(options.file).read()
1079  else:
1080    cmds = '\n'.join(args)
1081  compiled = compile(cmds, '<cmd line>', 'exec')
1082  # pylint: disable=eval-used
1083  eval(compiled, globals(), locals())
1084  return 0
1085
1086
1087@need_buildbot
1088def CMDinteractive(parser, args):
1089  """Runs an interactive shell to run queries."""
1090  _, args, buildbot = parser.parse_args(args)
1091  if args:
1092    parser.error('Unrecognized parameters: %s' % ' '.join(args))
1093  prompt = (
1094      'Buildbot interactive console for "%s".\n'
1095      'Hint: Start with typing: \'buildbot.printable_attributes\' or '
1096      '\'print str(buildbot)\' to explore.') % buildbot.url[:-len('/json')]
1097  local_vars = {'buildbot': buildbot, 'b': buildbot}
1098  code.interact(prompt, None, local_vars)
1099
1100
1101@need_buildbot
1102def CMDidle(parser, args):
1103  """Lists idle slaves."""
1104  return find_idle_busy_slaves(parser, args, True)
1105
1106
1107@need_buildbot
1108def CMDbusy(parser, args):
1109  """Lists idle slaves."""
1110  return find_idle_busy_slaves(parser, args, False)
1111
1112
1113@need_buildbot
1114def CMDdisconnected(parser, args):
1115  """Lists disconnected slaves."""
1116  _, args, buildbot = parser.parse_args(args)
1117  if args:
1118    parser.error('Unrecognized parameters: %s' % ' '.join(args))
1119  for slave in buildbot.slaves:
1120    if not slave.connected:
1121      print(slave.name)
1122  return 0
1123
1124
1125def find_idle_busy_slaves(parser, args, show_idle):
1126  parser.add_option('-b',
1127                    '--builder',
1128                    dest='builders',
1129                    action='append',
1130                    default=[],
1131                    help='Builders to filter on')
1132  parser.add_option('-s',
1133                    '--slave',
1134                    dest='slaves',
1135                    action='append',
1136                    default=[],
1137                    help='Slaves to filter on')
1138  options, args, buildbot = parser.parse_args(args)
1139  if args:
1140    parser.error('Unrecognized parameters: %s' % ' '.join(args))
1141  if not options.builders:
1142    options.builders = buildbot.builders.keys
1143  for builder in options.builders:
1144    builder = buildbot.builders[builder]
1145    if options.slaves:
1146      # Only the subset of slaves connected to the builder.
1147      slaves = list(set(options.slaves).intersection(set(builder.slaves.names)))
1148      if not slaves:
1149        continue
1150    else:
1151      slaves = builder.slaves.names
1152    busy_slaves = [build.slave.name for build in builder.current_builds]
1153    if show_idle:
1154      slaves = natsorted(set(slaves) - set(busy_slaves))
1155    else:
1156      slaves = natsorted(set(slaves) & set(busy_slaves))
1157    if options.quiet:
1158      for slave in slaves:
1159        print(slave)
1160    else:
1161      if slaves:
1162        print('Builder %s: %s' % (builder.name, ', '.join(slaves)))
1163  return 0
1164
1165
1166def last_failure(buildbot,
1167                 builders=None,
1168                 slaves=None,
1169                 steps=None,
1170                 no_cache=False):
1171  """Returns Build object with last failure with the specific filters."""
1172  builders = builders or buildbot.builders.keys
1173  for builder in builders:
1174    builder = buildbot.builders[builder]
1175    if slaves:
1176      # Only the subset of slaves connected to the builder.
1177      builder_slaves = list(set(slaves).intersection(set(builder.slaves.names)))
1178      if not builder_slaves:
1179        continue
1180    else:
1181      builder_slaves = builder.slaves.names
1182
1183    if not no_cache and len(builder.slaves) > 2:
1184      # Unless you just want the last few builds, it's often faster to
1185      # fetch the whole thing at once, at the cost of a small hickup on
1186      # the buildbot.
1187      # TODO(maruel): Cache only N last builds or all builds since
1188      # datetime.
1189      builder.builds.cache()
1190
1191    found = []
1192    for build in builder.builds:
1193      if build.slave.name not in builder_slaves or build.slave.name in found:
1194        continue
1195      # Only add the slave for the first completed build but still look for
1196      # incomplete builds.
1197      if build.completed:
1198        found.append(build.slave.name)
1199
1200      if steps:
1201        if any(build.steps[step].simplified_result is False for step in steps):
1202          yield build
1203      elif build.simplified_result is False:
1204        yield build
1205
1206      if len(found) == len(builder_slaves):
1207        # Found all the slaves, quit.
1208        break
1209
1210
1211@need_buildbot
1212def CMDlast_failure(parser, args):
1213  """Lists all slaves that failed on that step on their last build.
1214
1215  Example: to find all slaves where their last build was a compile failure,
1216  run with --step compile
1217  """
1218  parser.add_option(
1219      '-S',
1220      '--step',
1221      dest='steps',
1222      action='append',
1223      default=[],
1224      help='List all slaves that failed on that step on their last build')
1225  parser.add_option('-b',
1226                    '--builder',
1227                    dest='builders',
1228                    action='append',
1229                    default=[],
1230                    help='Builders to filter on')
1231  parser.add_option('-s',
1232                    '--slave',
1233                    dest='slaves',
1234                    action='append',
1235                    default=[],
1236                    help='Slaves to filter on')
1237  parser.add_option('-n',
1238                    '--no_cache',
1239                    action='store_true',
1240                    help='Don\'t load all builds at once')
1241  options, args, buildbot = parser.parse_args(args)
1242  if args:
1243    parser.error('Unrecognized parameters: %s' % ' '.join(args))
1244  print_builders = not options.quiet and len(options.builders) != 1
1245  last_builder = None
1246  for build in last_failure(buildbot,
1247                            builders=options.builders,
1248                            slaves=options.slaves,
1249                            steps=options.steps,
1250                            no_cache=options.no_cache):
1251
1252    if print_builders and last_builder != build.builder:
1253      print(build.builder.name)
1254      last_builder = build.builder
1255
1256    if options.quiet:
1257      if options.slaves:
1258        print('%s: %s' % (build.builder.name, build.slave.name))
1259      else:
1260        print(build.slave.name)
1261    else:
1262      out = '%d on %s: blame:%s' % (build.number, build.slave.name,
1263                                    ', '.join(build.blame))
1264      if print_builders:
1265        out = '  ' + out
1266      print(out)
1267
1268      if len(options.steps) != 1:
1269        for step in build.steps:
1270          if step.simplified_result is False:
1271            # Assume the first line is the text name anyway.
1272            summary = ', '.join(step.data['text'][1:])[:40]
1273            out = '  %s: "%s"' % (step.data['name'], summary)
1274            if print_builders:
1275              out = '  ' + out
1276            print(out)
1277  return 0
1278
1279
1280@need_buildbot
1281def CMDcurrent(parser, args):
1282  """Lists current jobs."""
1283  parser.add_option('-b',
1284                    '--builder',
1285                    dest='builders',
1286                    action='append',
1287                    default=[],
1288                    help='Builders to filter on')
1289  parser.add_option('--blame',
1290                    action='store_true',
1291                    help='Only print the blame list')
1292  options, args, buildbot = parser.parse_args(args)
1293  if args:
1294    parser.error('Unrecognized parameters: %s' % ' '.join(args))
1295  if not options.builders:
1296    options.builders = buildbot.builders.keys
1297
1298  if options.blame:
1299    blame = set()
1300    for builder in options.builders:
1301      for build in buildbot.builders[builder].current_builds:
1302        if build.blame:
1303          for blamed in build.blame:
1304            blame.add(blamed)
1305    print('\n'.join(blame))
1306    return 0
1307
1308  for builder in options.builders:
1309    builder = buildbot.builders[builder]
1310    if not options.quiet and builder.current_builds:
1311      print(builder.name)
1312    for build in builder.current_builds:
1313      if options.quiet:
1314        print(build.slave.name)
1315      else:
1316        out = '%4d: slave=%10s' % (build.number, build.slave.name)
1317        out += '  duration=%5d' % (build.duration or 0)
1318        if build.eta:
1319          out += '  eta=%5.0f' % build.eta
1320        else:
1321          out += '           '
1322        if build.blame:
1323          out += '  blame=' + ', '.join(build.blame)
1324        print(out)
1325
1326  return 0
1327
1328
1329@need_buildbot
1330def CMDbuilds(parser, args):
1331  """Lists all builds.
1332
1333  Example: to find all builds on a single slave, run with -b bar -s foo
1334  """
1335  parser.add_option('-r',
1336                    '--result',
1337                    type='int',
1338                    help='Build result to filter on')
1339  parser.add_option('-b',
1340                    '--builder',
1341                    dest='builders',
1342                    action='append',
1343                    default=[],
1344                    help='Builders to filter on')
1345  parser.add_option('-s',
1346                    '--slave',
1347                    dest='slaves',
1348                    action='append',
1349                    default=[],
1350                    help='Slaves to filter on')
1351  parser.add_option('-n',
1352                    '--no_cache',
1353                    action='store_true',
1354                    help='Don\'t load all builds at once')
1355  options, args, buildbot = parser.parse_args(args)
1356  if args:
1357    parser.error('Unrecognized parameters: %s' % ' '.join(args))
1358  builders = options.builders or buildbot.builders.keys
1359  for builder in builders:
1360    builder = buildbot.builders[builder]
1361    for build in builder.builds:
1362      if not options.slaves or build.slave.name in options.slaves:
1363        if options.quiet:
1364          out = ''
1365          if options.builders:
1366            out += '%s/' % builder.name
1367          if len(options.slaves) != 1:
1368            out += '%s/' % build.slave.name
1369          out += '%d  revision:%s  result:%s  blame:%s' % (
1370              build.number, build.revision, build.result, ','.join(build.blame))
1371          print(out)
1372        else:
1373          print(build)
1374  return 0
1375
1376
1377@need_buildbot
1378def CMDcount(parser, args):
1379  """Count the number of builds that occured during a specific period."""
1380  parser.add_option('-o',
1381                    '--over',
1382                    type='int',
1383                    help='Number of seconds to look for')
1384  parser.add_option('-b',
1385                    '--builder',
1386                    dest='builders',
1387                    action='append',
1388                    default=[],
1389                    help='Builders to filter on')
1390  options, args, buildbot = parser.parse_args(args)
1391  if args:
1392    parser.error('Unrecognized parameters: %s' % ' '.join(args))
1393  if not options.over:
1394    parser.error(
1395        'Specify the number of seconds, e.g. --over 86400 for the last 24 '
1396        'hours')
1397  builders = options.builders or buildbot.builders.keys
1398  counts = {}
1399  since = time.time() - options.over
1400  for builder in builders:
1401    builder = buildbot.builders[builder]
1402    counts[builder.name] = 0
1403    if not options.quiet:
1404      print(builder.name)
1405    for build in builder.builds.iterall():
1406      try:
1407        start_time = build.start_time
1408      except urllib2.HTTPError:
1409        # The build was probably trimmed.
1410        print('Failed to fetch build %s/%d' % (builder.name, build.number),
1411              file=sys.stderr)
1412        continue
1413      if start_time >= since:
1414        counts[builder.name] += 1
1415      else:
1416        break
1417    if not options.quiet:
1418      print('.. %d' % counts[builder.name])
1419
1420  align_name = max(len(b) for b in counts)
1421  align_number = max(len(str(c)) for c in counts.itervalues())
1422  for builder in sorted(counts):
1423    print('%*s: %*d' % (align_name, builder, align_number, counts[builder]))
1424  print('Total: %d' % sum(counts.itervalues()))
1425  return 0
1426
1427
1428def gen_parser():
1429  """Returns an OptionParser instance with default options.
1430
1431  It should be then processed with gen_usage() before being used.
1432  """
1433  parser = optparse.OptionParser(version=__version__)
1434  # Remove description formatting
1435  parser.format_description = lambda x: parser.description
1436  # Add common parsing.
1437  old_parser_args = parser.parse_args
1438
1439  def Parse(*args, **kwargs):
1440    options, args = old_parser_args(*args, **kwargs)
1441    if options.verbose >= 2:
1442      logging.basicConfig(level=logging.DEBUG)
1443    elif options.verbose:
1444      logging.basicConfig(level=logging.INFO)
1445    else:
1446      logging.basicConfig(level=logging.WARNING)
1447    return options, args
1448
1449  parser.parse_args = Parse
1450
1451  parser.add_option('-v',
1452                    '--verbose',
1453                    action='count',
1454                    help='Use multiple times to increase logging leve')
1455  parser.add_option(
1456      '-q',
1457      '--quiet',
1458      action='store_true',
1459      help='Reduces the output to be parsed by scripts, independent of -v')
1460  parser.add_option('--throttle',
1461                    type='float',
1462                    help='Minimum delay to sleep between requests')
1463  return parser
1464
1465###############################################################################
1466## Generic subcommand handling code
1467
1468
1469def Command(name):
1470  return getattr(sys.modules[__name__], 'CMD' + name, None)
1471
1472
1473@usage('<command>')
1474def CMDhelp(parser, args):
1475  """Print list of commands or use 'help <command>'."""
1476  _, args = parser.parse_args(args)
1477  if len(args) == 1:
1478    return main(args + ['--help'])
1479  parser.print_help()
1480  return 0
1481
1482
1483def gen_usage(parser, command):
1484  """Modifies an OptionParser object with the command's documentation.
1485
1486  The documentation is taken from the function's docstring.
1487  """
1488  obj = Command(command)
1489  more = getattr(obj, 'func_usage_more')
1490  # OptParser.description prefer nicely non-formatted strings.
1491  parser.description = obj.__doc__ + '\n'
1492  parser.set_usage('usage: %%prog %s %s' % (command, more))
1493
1494
1495def main(args=None):
1496  # Do it late so all commands are listed.
1497  # pylint: disable=E1101
1498  CMDhelp.__doc__ += '\n\nCommands are:\n' + '\n'.join(
1499      '  %-12s %s' % (fn[3:], Command(fn[3:]).__doc__.split('\n', 1)[0])
1500      for fn in dir(sys.modules[__name__]) if fn.startswith('CMD'))
1501
1502  parser = gen_parser()
1503  if args is None:
1504    args = sys.argv[1:]
1505  if args:
1506    command = Command(args[0])
1507    if command:
1508      # "fix" the usage and the description now that we know the subcommand.
1509      gen_usage(parser, args[0])
1510      return command(parser, args[1:])
1511
1512  # Not a known command. Default to help.
1513  gen_usage(parser, 'help')
1514  return CMDhelp(parser, args)
1515
1516
1517if __name__ == '__main__':
1518  sys.exit(main())
1519