12a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Copyright 2013 The Chromium Authors. All rights reserved.
22a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be
32a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# found in the LICENSE file.
42a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
52a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)"""Thread and ThreadGroup that reraise exceptions on the main thread."""
6a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)# pylint: disable=W0212
72a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
8c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)import logging
92a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import sys
102a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import threading
11c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)import traceback
12c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
13a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)from pylib.utils import watchdog_timer
14c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
15c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
16c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)class TimeoutError(Exception):
17c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)  """Module-specific timeout exception."""
18c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)  pass
192a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
202a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
21f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)def LogThreadStack(thread):
22f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  """Log the stack for the given thread.
23f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)
24f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  Args:
25f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)    thread: a threading.Thread instance.
26f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  """
27f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  stack = sys._current_frames()[thread.ident]
28f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  logging.critical('*' * 80)
29f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  logging.critical('Stack dump for thread \'%s\'', thread.name)
30f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  logging.critical('*' * 80)
31f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  for filename, lineno, name, line in traceback.extract_stack(stack):
32f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)    logging.critical('File: "%s", line %d, in %s', filename, lineno, name)
33f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)    if line:
34f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)      logging.critical('  %s', line.strip())
35f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  logging.critical('*' * 80)
36f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)
37f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)
382a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)class ReraiserThread(threading.Thread):
392a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  """Thread class that can reraise exceptions."""
40c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
41a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)  def __init__(self, func, args=None, kwargs=None, name=None):
42c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    """Initialize thread.
43c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
44c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    Args:
45c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      func: callable to call on a new thread.
46c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      args: list of positional arguments for callable, defaults to empty.
47c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      kwargs: dictionary of keyword arguments for callable, defaults to empty.
48c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      name: thread name, defaults to Thread-N.
49c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    """
50c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    super(ReraiserThread, self).__init__(name=name)
51a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)    if not args:
52a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)      args = []
53a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)    if not kwargs:
54a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)      kwargs = {}
552a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    self.daemon = True
562a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    self._func = func
572a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    self._args = args
582a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    self._kwargs = kwargs
59cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    self._ret = None
602a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    self._exc_info = None
612a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
622a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  def ReraiseIfException(self):
632a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    """Reraise exception if an exception was raised in the thread."""
642a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    if self._exc_info:
652a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      raise self._exc_info[0], self._exc_info[1], self._exc_info[2]
662a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
67cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  def GetReturnValue(self):
68cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    """Reraise exception if present, otherwise get the return value."""
69cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    self.ReraiseIfException()
70cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    return self._ret
71cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)
722a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  #override
732a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  def run(self):
742a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    """Overrides Thread.run() to add support for reraising exceptions."""
752a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    try:
76cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      self._ret = self._func(*self._args, **self._kwargs)
77116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch    except: # pylint: disable=W0702
782a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      self._exc_info = sys.exc_info()
792a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
802a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
812a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)class ReraiserThreadGroup(object):
822a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  """A group of ReraiserThread objects."""
83c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
84a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)  def __init__(self, threads=None):
852a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    """Initialize thread group.
862a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
872a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    Args:
882a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      threads: a list of ReraiserThread objects; defaults to empty.
892a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    """
90a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)    if not threads:
91a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)      threads = []
922a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    self._threads = threads
932a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
942a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  def Add(self, thread):
952a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    """Add a thread to the group.
962a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
972a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    Args:
982a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      thread: a ReraiserThread object.
992a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    """
1002a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    self._threads.append(thread)
1012a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
1022a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  def StartAll(self):
1032a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    """Start all threads."""
1042a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    for thread in self._threads:
1052a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      thread.start()
1062a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
107c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)  def _JoinAll(self, watcher=watchdog_timer.WatchdogTimer(None)):
108c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    """Join all threads without stack dumps.
109c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
110c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    Reraises exceptions raised by the child threads and supports breaking
111c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    immediately on exceptions raised on the main thread.
1122a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
113c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    Args:
114c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      watcher: Watchdog object providing timeout, by default waits forever.
1152a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    """
1162a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    alive_threads = self._threads[:]
1172a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    while alive_threads:
1182a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      for thread in alive_threads[:]:
119c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)        if watcher.IsTimedOut():
120c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)          raise TimeoutError('Timed out waiting for %d of %d threads.' %
121c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)                             (len(alive_threads), len(self._threads)))
1222a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)        # Allow the main thread to periodically check for interrupts.
1232a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)        thread.join(0.1)
1242a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)        if not thread.isAlive():
1252a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)          alive_threads.remove(thread)
1262a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    # All threads are allowed to complete before reraising exceptions.
1272a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    for thread in self._threads:
1282a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      thread.ReraiseIfException()
129c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
130c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)  def JoinAll(self, watcher=watchdog_timer.WatchdogTimer(None)):
131c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    """Join all threads.
132c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
133c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    Reraises exceptions raised by the child threads and supports breaking
134c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    immediately on exceptions raised on the main thread. Unfinished threads'
135c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    stacks will be logged on watchdog timeout.
136c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
137c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    Args:
138c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      watcher: Watchdog object providing timeout, by default waits forever.
139c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    """
140c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    try:
141c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      self._JoinAll(watcher)
142c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    except TimeoutError:
143c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      for thread in (t for t in self._threads if t.isAlive()):
144f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)        LogThreadStack(thread)
145c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      raise
146cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)
147cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)  def GetAllReturnValues(self, watcher=watchdog_timer.WatchdogTimer(None)):
148cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    """Get all return values, joining all threads if necessary.
149cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)
150cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    Args:
151cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      watcher: same as in |JoinAll|. Only used if threads are alive.
152cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    """
153cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    if any([t.isAlive() for t in self._threads]):
154cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)      self.JoinAll(watcher)
155cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)    return [t.GetReturnValue() for t in self._threads]
156cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)
157