reraiser_thread.py revision f2477e01787aa58f445919b809d89e252beef54f
12a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Copyright 2013 The Chromium Authors. All rights reserved.
22a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be
32a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# found in the LICENSE file.
42a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
52a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)"""Thread and ThreadGroup that reraise exceptions on the main thread."""
62a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
7c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)import logging
82a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import sys
92a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import threading
10c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)import time
11c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)import traceback
12c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
13c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)import watchdog_timer
14c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
15c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
16c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)class TimeoutError(Exception):
17c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)  """Module-specific timeout exception."""
18c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)  pass
192a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
202a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
21f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)def LogThreadStack(thread):
22f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  """Log the stack for the given thread.
23f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)
24f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  Args:
25f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)    thread: a threading.Thread instance.
26f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  """
27f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  stack = sys._current_frames()[thread.ident]
28f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  logging.critical('*' * 80)
29f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  logging.critical('Stack dump for thread \'%s\'', thread.name)
30f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  logging.critical('*' * 80)
31f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  for filename, lineno, name, line in traceback.extract_stack(stack):
32f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)    logging.critical('File: "%s", line %d, in %s', filename, lineno, name)
33f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)    if line:
34f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)      logging.critical('  %s', line.strip())
35f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)  logging.critical('*' * 80)
36f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)
37f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)
382a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)class ReraiserThread(threading.Thread):
392a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  """Thread class that can reraise exceptions."""
40c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
41c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)  def __init__(self, func, args=[], kwargs={}, name=None):
42c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    """Initialize thread.
43c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
44c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    Args:
45c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      func: callable to call on a new thread.
46c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      args: list of positional arguments for callable, defaults to empty.
47c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      kwargs: dictionary of keyword arguments for callable, defaults to empty.
48c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      name: thread name, defaults to Thread-N.
49c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    """
50c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    super(ReraiserThread, self).__init__(name=name)
512a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    self.daemon = True
522a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    self._func = func
532a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    self._args = args
542a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    self._kwargs = kwargs
552a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    self._exc_info = None
562a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
572a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  def ReraiseIfException(self):
582a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    """Reraise exception if an exception was raised in the thread."""
592a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    if self._exc_info:
602a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      raise self._exc_info[0], self._exc_info[1], self._exc_info[2]
612a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
622a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  #override
632a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  def run(self):
642a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    """Overrides Thread.run() to add support for reraising exceptions."""
652a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    try:
662a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      self._func(*self._args, **self._kwargs)
672a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    except:
682a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      self._exc_info = sys.exc_info()
692a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      raise
702a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
712a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
722a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)class ReraiserThreadGroup(object):
732a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  """A group of ReraiserThread objects."""
74c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
752a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  def __init__(self, threads=[]):
762a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    """Initialize thread group.
772a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
782a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    Args:
792a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      threads: a list of ReraiserThread objects; defaults to empty.
802a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    """
812a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    self._threads = threads
822a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
832a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  def Add(self, thread):
842a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    """Add a thread to the group.
852a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
862a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    Args:
872a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      thread: a ReraiserThread object.
882a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    """
892a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    self._threads.append(thread)
902a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
912a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)  def StartAll(self):
922a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    """Start all threads."""
932a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    for thread in self._threads:
942a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      thread.start()
952a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
96c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)  def _JoinAll(self, watcher=watchdog_timer.WatchdogTimer(None)):
97c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    """Join all threads without stack dumps.
98c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
99c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    Reraises exceptions raised by the child threads and supports breaking
100c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    immediately on exceptions raised on the main thread.
1012a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)
102c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    Args:
103c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      watcher: Watchdog object providing timeout, by default waits forever.
1042a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    """
1052a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    alive_threads = self._threads[:]
1062a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    while alive_threads:
1072a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      for thread in alive_threads[:]:
108c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)        if watcher.IsTimedOut():
109c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)          raise TimeoutError('Timed out waiting for %d of %d threads.' %
110c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)                             (len(alive_threads), len(self._threads)))
1112a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)        # Allow the main thread to periodically check for interrupts.
1122a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)        thread.join(0.1)
1132a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)        if not thread.isAlive():
1142a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)          alive_threads.remove(thread)
1152a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    # All threads are allowed to complete before reraising exceptions.
1162a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)    for thread in self._threads:
1172a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)      thread.ReraiseIfException()
118c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
119c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)  def JoinAll(self, watcher=watchdog_timer.WatchdogTimer(None)):
120c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    """Join all threads.
121c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
122c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    Reraises exceptions raised by the child threads and supports breaking
123c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    immediately on exceptions raised on the main thread. Unfinished threads'
124c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    stacks will be logged on watchdog timeout.
125c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)
126c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    Args:
127c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      watcher: Watchdog object providing timeout, by default waits forever.
128c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    """
129c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    try:
130c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      self._JoinAll(watcher)
131c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)    except TimeoutError:
132c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      for thread in (t for t in self._threads if t.isAlive()):
133f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles)        LogThreadStack(thread)
134c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)      raise
135