autotest.py revision 548197f1292597dfa5247bcea157379680d1f5e8 
1#!/usr/bin/python
2#
3# Copyright 2007 Google Inc. Released under the GPL v2
4
5"""
6This module defines the Autotest class
7
8	Autotest: software to run tests automatically
9"""
10
11__author__ = """
12mbligh@google.com (Martin J. Bligh),
13poirier@google.com (Benjamin Poirier),
14stutsman@google.com (Ryan Stutsman)
15"""
16
17import re
18import os
19import sys
20import subprocess
21import urllib
22import tempfile
23import shutil
24import time
25
26import installable_object
27import utils
28from common import logging
29from common.error import *
30
31
32AUTOTEST_SVN  = 'svn://test.kernel.org/autotest/trunk/client'
33AUTOTEST_HTTP = 'http://test.kernel.org/svn/autotest/trunk/client'
34
35# Timeouts for powering down and up respectively
36HALT_TIME = 300
37BOOT_TIME = 1800
38
39
40class AutotestRunError(AutoservRunError):
41	pass
42
43class AutotestTimeoutError(AutoservRunError):
44	"""This exception is raised when an autotest test exceeds the timeout
45	parameter passed to run_timed_test and is killed.
46	"""
47
48
49class Autotest(installable_object.InstallableObject):
50	"""
51	This class represents the Autotest program.
52
53	Autotest is used to run tests automatically and collect the results.
54	It also supports profilers.
55
56	Implementation details:
57	This is a leaf class in an abstract class hierarchy, it must
58	implement the unimplemented methods in parent classes.
59	"""
60	job = None
61
62
63	def __init__(self, host = None):
64		self.host = host
65		self.got = False
66		self.installed = False
67		self.serverdir = utils.get_server_dir()
68		super(Autotest, self).__init__()
69
70
71	@logging.record
72	def install(self, host = None):
73		"""
74		Install autotest.  If get() was not called previously, an
75		attempt will be made to install from the autotest svn
76		repository.
77
78		Args:
79			host: a Host instance on which autotest will be
80				installed
81
82		Raises:
83			AutoservError: if a tarball was not specified and
84				the target host does not have svn installed in its path
85
86		TODO(poirier): check dependencies
87		autotest needs:
88		bzcat
89		liboptdev (oprofile)
90		binutils-dev (oprofile)
91		make
92		psutils (netperf)
93		"""
94		if not host:
95			host = self.host
96		if not self.got:
97			self.get()
98		host.ensure_up()
99		host.setup()
100		print "Installing autotest on %s" % host.hostname
101
102		autodir = _get_autodir(host)
103		host.run('mkdir -p "%s"' % utils.sh_escape(autodir))
104
105		if getattr(host, 'site_install_autotest', None):
106			if host.site_install_autotest():
107				self.installed = True
108				return
109
110		# try to install from file or directory
111		if self.source_material:
112			if os.path.isdir(self.source_material):
113				# Copy autotest recursively
114				host.send_file(self.source_material, autodir)
115			else:
116				# Copy autotest via tarball
117				raise "Not yet implemented!"
118			print "Installation of autotest completed"
119			self.installed = True
120			return
121
122		# if that fails try to install using svn
123		if utils.run('which svn').exit_status:
124			raise AutoservError('svn not found in path on \
125			target machine: %s' % host.name)
126		try:
127			host.run('svn checkout %s %s' %
128				 (AUTOTEST_SVN, autodir))
129		except AutoservRunError, e:
130			host.run('svn checkout %s %s' %
131				 (AUTOTEST_HTTP, autodir))
132		print "Installation of autotest completed"
133		self.installed = True
134
135
136	def get(self, location = None):
137		if not location:
138			location = os.path.join(self.serverdir, '../client')
139			location = os.path.abspath(location)
140		# If there's stuff run on our client directory already, it
141		# can cause problems. Try giving it a quick clean first.
142		cwd = os.getcwd()
143		os.chdir(location)
144		os.system('tools/make_clean')
145		os.chdir(cwd)
146		super(Autotest, self).get(location)
147		self.got = True
148
149
150	def run(self, control_file, results_dir = '.', host = None,
151		timeout=None):
152		"""
153		Run an autotest job on the remote machine.
154
155		Args:
156			control_file: an open file-like-obj of the control file
157			results_dir: a str path where the results should be stored
158				on the local filesystem
159			host: a Host instance on which the control file should
160				be run
161
162		Raises:
163			AutotestRunError: if there is a problem executing
164				the control file
165		"""
166		results_dir = os.path.abspath(results_dir)
167		if not host:
168			host = self.host
169		if not self.installed:
170			self.install(host)
171
172		host.ensure_up()
173
174		atrun = _Run(host, results_dir)
175		try:
176			atrun.verify_machine()
177		except:
178			print "Verify machine failed on %s. Reinstalling" % \
179								host.hostname
180			self.install(host)
181		atrun.verify_machine()
182		debug = os.path.join(results_dir, 'debug')
183		try:
184			os.makedirs(debug)
185		except:
186			pass
187
188		# Ready .... Aim ....
189		for control in [atrun.remote_control_file,
190				atrun.remote_control_file + '.state',
191				atrun.manual_control_file,
192				atrun.manual_control_file + '.state']:
193			host.run('rm -f ' + control)
194
195		# Copy control_file to remote_control_file on the host
196		tmppath = utils.get(control_file)
197		host.send_file(tmppath, atrun.remote_control_file)
198		os.remove(tmppath)
199
200		try:
201			atrun.execute_control(timeout=timeout)
202		finally:
203			# get the results
204			results = os.path.join(atrun.autodir, 'results',
205					       'default')
206			# Copy all dirs in default to results_dir
207			host.get_file(results + '/', results_dir)
208
209
210	def run_timed_test(self, test_name, results_dir = '.', host = None,
211			   timeout=None, *args, **dargs):
212		"""
213		Assemble a tiny little control file to just run one test,
214		and run it as an autotest client-side test
215		"""
216		if not host:
217			host = self.host
218		if not self.installed:
219			self.install(host)
220		opts = ["%s=%s" % (o[0], repr(o[1])) for o in dargs.items()]
221		cmd = ", ".join([repr(test_name)] + map(repr, args) + opts)
222		control = "job.run_test(%s)\n" % cmd
223		self.run(control, results_dir, host, timeout=timeout)
224
225
226	def run_test(self, test_name, results_dir = '.', host = None,
227		     *args, **dargs):
228		self.run_timed_test(test_name, results_dir, host, None,
229				    *args, **dargs)
230
231
232class _Run(object):
233	"""
234	Represents a run of autotest control file.  This class maintains
235	all the state necessary as an autotest control file is executed.
236
237	It is not intended to be used directly, rather control files
238	should be run using the run method in Autotest.
239	"""
240	def __init__(self, host, results_dir):
241		self.host = host
242		self.results_dir = results_dir
243		self.env = host.env
244
245		self.autodir = _get_autodir(self.host)
246		self.manual_control_file = os.path.join(self.autodir, 'control')
247		self.remote_control_file = os.path.join(self.autodir,
248							     'control.autoserv')
249
250
251	def verify_machine(self):
252		binary = os.path.join(self.autodir, 'bin/autotest')
253		try:
254			self.host.run('ls %s > /dev/null' % binary)
255		except:
256			raise "Autotest does not appear to be installed"
257		tmpdir = os.path.join(self.autodir, 'tmp')
258		self.host.run('umount %s' % tmpdir, ignore_status=True)
259
260
261	def __execute_section(self, section, timeout):
262		print "Executing %s/bin/autotest %s/control phase %d" % \
263					(self.autodir, self.autodir,
264					 section)
265
266		# open up the files we need for our logging
267		client_log_file = os.path.join(self.results_dir, 'debug',
268					       'client.log.%d' % section)
269		client_log = open(client_log_file, 'w', 0)
270		status_log_file = os.path.join(self.results_dir, 'status.log')
271		status_log = open(status_log_file, 'a', 0)
272
273		# create a file-like object for catching the stderr text
274		# from the autotest client and extracting status logs from it
275		class StdErrRedirector(object):
276			"""Partial file object to write to both stdout and
277			the status log file.  We only implement those methods
278			utils.run() actually calls.
279			"""
280			def __init__(self):
281				self.leftover = ""
282				self.last_line = ""
283
284			def _process_line(self, line):
285				"""Write out a line of data to the appropriate
286				stream. Status lines sent by autotest will be
287				prepended with "AUTOTEST_STATUS", and all other
288				lines are ssh error messages.
289				"""
290				if line.startswith("AUTOTEST_STATUS:"):
291					line = line[16:] + "\n"
292					sys.stdout.write(line)
293					status_log.write(line)
294					self.last_line = line
295				else:
296					sys.stderr.write(line + "\n")
297
298			def write(self, data):
299				data = self.leftover + data
300				lines = data.split("\n")
301				# process every line but the last one
302				for line in lines[:-1]:
303					self._process_line(line)
304				# save the last line for later processing
305				# since we may not have the whole line yet
306				self.leftover = lines[-1]
307
308			def flush(self):
309				sys.stdout.flush()
310				sys.stderr.flush()
311				status_log.flush()
312
313			def close(self):
314				if self.leftover:
315					self._process_line(self.leftover)
316					self.flush()
317		redirector = StdErrRedirector()
318
319		# build up the full command we want to run over the host
320		cmd = [os.path.join(self.autodir, 'bin/autotest_client')]
321		if section > 0:
322			cmd.append('-c')
323		cmd.append(self.remote_control_file)
324		full_cmd = ' '.join(cmd)
325
326		result = self.host.run(full_cmd, ignore_status=True,
327				       timeout=timeout,
328				       stdout_tee=client_log,
329				       stderr_tee=redirector)
330		redirector.close()
331
332		if result.exit_status == 1:
333			self.host.job.aborted = True
334		if not result.stderr:
335  			raise AutotestRunError(
336			    "execute_section: %s failed to return anything\n"
337			    "stdout:%s\n" % (full_cmd, result.stdout))
338
339		return redirector.last_line
340
341
342	def execute_control(self, timeout=None):
343		section = 0
344		time_left = None
345		if timeout:
346			end_time = time.time() + timeout
347			time_left = end_time - time.time()
348		while not timeout or time_left > 0:
349			last = self.__execute_section(section, time_left)
350			if timeout:
351				time_left = end_time - time.time()
352				if time_left <= 0:
353					break
354			section += 1
355			if re.match(r'^END .*\t----\t----\t.*$', last):
356				print "Client complete"
357				return
358			elif re.match('^\t*GOOD\t----\treboot\.start.*$', last):
359				print "Client is rebooting"
360				print "Waiting for client to halt"
361				if not self.host.wait_down(HALT_TIME):
362					raise AutotestRunError("%s \
363					failed to shutdown after %ds" %
364							(self.host.hostname,
365							HALT_TIME))
366				print "Client down, waiting for restart"
367				if not self.host.wait_up(BOOT_TIME):
368					# since reboot failed
369					# hardreset the machine once if possible
370					# before failing this control file
371					print "Hardresetting %s" % (
372					    self.host.hostname,)
373					try:
374						self.host.hardreset(wait=False)
375					except AutoservUnsupportedError:
376						print "Hardreset unsupported on %s" % (
377						    self.host.hostname,)
378					raise AutotestRunError("%s failed to "
379						"boot after %ds" % (
380						self.host.hostname,
381						BOOT_TIME,))
382				continue
383			raise AutotestRunError("Aborting - unknown "
384				"return code: %s\n" % last)
385
386		# should only get here if we timed out
387		assert timeout
388		raise AutotestTimeoutError()
389
390
391def _get_autodir(host):
392	try:
393		# There's no clean way to do this. readlink may not exist
394		cmd = "python -c 'import os,sys; print os.readlink(sys.argv[1])' /etc/autotest.conf"
395		dir = os.path.dirname(host.run(cmd).stdout)
396		if dir:
397			return dir
398	except AutoservRunError:
399		pass
400	for path in ['/usr/local/autotest', '/home/autotest']:
401		try:
402			host.run('ls %s > /dev/null' % \
403					 os.path.join(path, 'bin/autotest'))
404			return path
405		except AutoservRunError:
406			pass
407	raise AutotestRunError("Cannot figure out autotest directory")
408