1import os, shutil, tempfile, logging
2
3import common
4from autotest_lib.client.common_lib import utils, error, profiler_manager
5from autotest_lib.server import profiler, autotest, standalone_profiler, hosts
6
7
8PROFILER_TMPDIR = '/tmp/profilers'
9
10
11def get_profiler_results_dir(autodir):
12    """
13    Given the directory of the autotest client used to run a profiler,
14    return the remote path where profiler results will be stored.
15    """
16    return os.path.join(autodir, 'results', 'default', 'profiler_sync',
17                        'profiling')
18
19
20def get_profiler_log_path(autodir):
21    """
22    Given the directory of a profiler client, find the client log path.
23    """
24    return os.path.join(autodir, 'results', 'default', 'debug', 'client.DEBUG')
25
26
27class profilers(profiler_manager.profiler_manager):
28    def __init__(self, job):
29        super(profilers, self).__init__(job)
30        self.add_log = {}
31        self.start_delay = 0
32        # maps hostname to (host object, autotest.Autotest object, Autotest
33        # install dir), where the host object is the one created specifically
34        # for profiling
35        self.installed_hosts = {}
36        self.current_test = None
37
38
39    def set_start_delay(self, start_delay):
40        self.start_delay = start_delay
41
42
43    def load_profiler(self, profiler_name, args, dargs):
44        newprofiler = profiler.profiler_proxy(profiler_name)
45        newprofiler.initialize(*args, **dargs)
46        newprofiler.setup(*args, **dargs) # lazy setup is done client-side
47        return newprofiler
48
49
50    def add(self, profiler, *args, **dargs):
51        super(profilers, self).add(profiler, *args, **dargs)
52        self.add_log[profiler] = (args, dargs)
53
54
55    def delete(self, profiler):
56        super(profilers, self).delete(profiler)
57        if profiler in self.add_log:
58            del self.add_log[profiler]
59
60
61    def _install_clients(self):
62        """
63        Install autotest on any current job hosts.
64        """
65        in_use_hosts = set()
66        # find hosts in use but not used by us
67        for host in self.job.hosts:
68            if host.hostname not in self.job.machines:
69                # job.hosts include all host instances created on the fly.
70                # We only care DUTs in job.machines which are
71                # piped in from autoserv -m option.
72                continue
73            autodir = host.get_autodir()
74            if not (autodir and autodir.startswith(PROFILER_TMPDIR)):
75                in_use_hosts.add(host.hostname)
76        logging.debug('Hosts currently in use: %s', in_use_hosts)
77
78        # determine what valid host objects we already have installed
79        profiler_hosts = set()
80        for host, at, profiler_dir in self.installed_hosts.values():
81            if host.path_exists(profiler_dir):
82                profiler_hosts.add(host.hostname)
83            else:
84                # the profiler was wiped out somehow, drop this install
85                logging.warning('The profiler client on %s at %s was deleted',
86                                host.hostname, profiler_dir)
87                host.close()
88                del self.installed_hosts[host.hostname]
89        logging.debug('Hosts with profiler clients already installed: %s',
90                      profiler_hosts)
91
92        # install autotest on any new hosts in use
93        for hostname in in_use_hosts - profiler_hosts:
94            host = hosts.create_host(hostname)
95            tmp_dir = host.get_tmp_dir(parent=PROFILER_TMPDIR)
96            at = autotest.Autotest(host)
97            at.install_no_autoserv(autodir=tmp_dir)
98            self.installed_hosts[host.hostname] = (host, at, tmp_dir)
99
100        # drop any installs from hosts no longer in job.hosts
101        hostnames_to_drop = profiler_hosts - in_use_hosts
102        hosts_to_drop = [self.installed_hosts[hostname][0]
103                         for hostname in hostnames_to_drop]
104        for host in hosts_to_drop:
105            host.close()
106            del self.installed_hosts[host.hostname]
107
108
109    def _get_hosts(self, host=None):
110        """
111        Returns a list of (Host, Autotest, install directory) tuples for hosts
112        currently supported by this profiler. The returned Host object is always
113        the one created by this profiler, regardless of what's passed in. If
114        'host' is not None, all entries not matching that host object are
115        filtered out of the list.
116        """
117        if host is None:
118            return self.installed_hosts.values()
119        if host.hostname in self.installed_hosts:
120            return [self.installed_hosts[host.hostname]]
121        return []
122
123
124    def _get_local_profilers_dir(self, test, hostname):
125        in_machine_dir = (
126                os.path.basename(test.job.resultdir) in test.job.machines)
127        if len(test.job.machines) > 1 and not in_machine_dir:
128            local_dir = os.path.join(test.profdir, hostname)
129            if not os.path.exists(local_dir):
130                os.makedirs(local_dir)
131        else:
132            local_dir = test.profdir
133
134        return local_dir
135
136
137    def _get_failure_logs(self, autodir, test, host):
138        """
139        Collect the client logs from a profiler run and put them in a
140        file named failure-*.log.
141        """
142        try:
143            fd, path = tempfile.mkstemp(suffix='.log', prefix='failure-',
144                    dir=self._get_local_profilers_dir(test, host.hostname))
145            os.close(fd)
146            host.get_file(get_profiler_log_path(autodir), path)
147            # try to collect any partial profiler logs
148            self._get_profiler_logs(autodir, test, host)
149        except (error.AutotestError, error.AutoservError):
150            logging.exception('Profiler failure log collection failed')
151            # swallow the exception so that we don't override an existing
152            # exception being thrown
153
154
155    def _get_all_failure_logs(self, test, hosts):
156        for host, at, autodir in hosts:
157            self._get_failure_logs(autodir, test, host)
158
159
160    def _get_profiler_logs(self, autodir, test, host):
161        results_dir = get_profiler_results_dir(autodir)
162        local_dir = self._get_local_profilers_dir(test, host.hostname)
163
164        self.job.remove_client_log(host.hostname, results_dir, local_dir)
165
166        tempdir = tempfile.mkdtemp(dir=self.job.tmpdir)
167        try:
168            host.get_file(results_dir + '/', tempdir)
169        except error.AutoservRunError:
170            pass # no files to pull back, nothing we can do
171        utils.merge_trees(tempdir, local_dir)
172        shutil.rmtree(tempdir, ignore_errors=True)
173
174
175    def _run_clients(self, test, hosts):
176        """
177        We initialize the profilers just before start because only then we
178        know all the hosts involved.
179        """
180
181        hostnames = [host_info[0].hostname for host_info in hosts]
182        profilers_args = [(p.name, p.args, p.dargs)
183                          for p in self.list]
184
185        for host, at, autodir in hosts:
186            control_script = standalone_profiler.generate_test(hostnames,
187                                                               host.hostname,
188                                                               profilers_args,
189                                                               180, None)
190            try:
191                at.run(control_script, background=True)
192            except Exception:
193                self._get_failure_logs(autodir, test, host)
194                raise
195
196            remote_results_dir = get_profiler_results_dir(autodir)
197            local_results_dir = self._get_local_profilers_dir(test,
198                                                              host.hostname)
199            self.job.add_client_log(host.hostname, remote_results_dir,
200                                    local_results_dir)
201
202        try:
203            # wait for the profilers to be added
204            standalone_profiler.wait_for_profilers(hostnames)
205        except Exception:
206            self._get_all_failure_logs(test, hosts)
207            raise
208
209
210    def before_start(self, test, host=None):
211        # create host objects and install the needed clients
212        # so later in start() we don't spend too much time
213        self._install_clients()
214        self._run_clients(test, self._get_hosts(host))
215
216
217    def start(self, test, host=None):
218        hosts = self._get_hosts(host)
219
220        # wait for the profilers to start
221        hostnames = [host_info[0].hostname for host_info in hosts]
222        try:
223            standalone_profiler.start_profilers(hostnames)
224        except Exception:
225            self._get_all_failure_logs(test, hosts)
226            raise
227
228        self.current_test = test
229
230
231    def stop(self, test):
232        assert self.current_test == test
233
234        hosts = self._get_hosts()
235        # wait for the profilers to stop
236        hostnames = [host_info[0].hostname for host_info in hosts]
237        try:
238            standalone_profiler.stop_profilers(hostnames)
239        except Exception:
240            self._get_all_failure_logs(test, hosts)
241            raise
242
243
244    def report(self, test, host=None):
245        assert self.current_test == test
246
247        hosts = self._get_hosts(host)
248        # when running on specific hosts we cannot wait for the other
249        # hosts to sync with us
250        if not host:
251            hostnames = [host_info[0].hostname for host_info in hosts]
252            try:
253                standalone_profiler.finish_profilers(hostnames)
254            except Exception:
255                self._get_all_failure_logs(test, hosts)
256                raise
257
258        # pull back all the results
259        for host, at, autodir in hosts:
260            self._get_profiler_logs(autodir, test, host)
261
262
263    def handle_reboot(self, host):
264        if self.current_test:
265            test = self.current_test
266            for profiler in self.list:
267                if not profiler.supports_reboot:
268                    msg = 'profiler %s does not support rebooting during tests'
269                    msg %= profiler.name
270                    self.job.record('WARN', os.path.basename(test.outputdir),
271                                    None, msg)
272
273            self.report(test, host)
274            self.before_start(test, host)
275            self.start(test, host)
276