1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import logging
6import csv
7import cStringIO
8import random
9import re
10import collections
11
12from autotest_lib.client.common_lib.cros import path_utils
13
14class ResourceMonitorRawResult(object):
15    """Encapsulates raw resource_monitor results."""
16
17    def __init__(self, raw_results_filename):
18        self._raw_results_filename = raw_results_filename
19
20
21    def get_parsed_results(self):
22        """Constructs parsed results from the raw ones.
23
24        @return ResourceMonitorParsedResult object
25
26        """
27        return ResourceMonitorParsedResult(self.raw_results_filename)
28
29
30    @property
31    def raw_results_filename(self):
32        """@return string filename storing the raw top command output."""
33        return self._raw_results_filename
34
35
36class IncorrectTopFormat(Exception):
37    """Thrown if top output format is not as expected"""
38    pass
39
40
41def _extract_value_before_single_keyword(line, keyword):
42    """Extract word occurring immediately before the specified keyword.
43
44    @param line string the line in which to search for the keyword.
45    @param keyword string the keyword to look for. Can be a regexp.
46    @return string the word just before the keyword.
47
48    """
49    pattern = ".*?(\S+) " + keyword
50    matches = re.match(pattern, line)
51    if matches is None or len(matches.groups()) != 1:
52        raise IncorrectTopFormat
53
54    return matches.group(1)
55
56
57def _extract_values_before_keywords(line, *args):
58    """Extract the words occuring immediately before each specified
59        keyword in args.
60
61    @param line string the string to look for the keywords.
62    @param args variable number of string args the keywords to look for.
63    @return string list the words occuring just before each keyword.
64
65    """
66    line_nocomma = re.sub(",", " ", line)
67    line_singlespace = re.sub("\s+", " ", line_nocomma)
68
69    return [_extract_value_before_single_keyword(
70            line_singlespace, arg) for arg in args]
71
72
73def _find_top_output_identifying_pattern(line):
74    """Return true iff the line looks like the first line of top output.
75
76    @param line string to look for the pattern
77    @return boolean
78
79    """
80    pattern ="\s*top\s*-.*up.*users.*"
81    matches = re.match(pattern, line)
82    return matches is not None
83
84
85class ResourceMonitorParsedResult(object):
86    """Encapsulates logic to parse and represent top command results."""
87
88    _columns = ["Time", "UserCPU", "SysCPU", "NCPU", "Idle",
89            "IOWait", "IRQ", "SoftIRQ", "Steal",
90            "MemUnits", "UsedMem", "FreeMem",
91            "SwapUnits", "UsedSwap", "FreeSwap"]
92    UtilValues = collections.namedtuple('UtilValues', ' '.join(_columns))
93
94    def __init__(self, raw_results_filename):
95        """Construct a ResourceMonitorResult.
96
97        @param raw_results_filename string filename of raw batch top output.
98
99        """
100        self._raw_results_filename = raw_results_filename
101        self.parse_resource_monitor_results()
102
103
104    def parse_resource_monitor_results(self):
105        """Extract utilization metrics from output file."""
106        self._utils_over_time = []
107
108        with open(self._raw_results_filename, "r") as results_file:
109            while True:
110                curr_line = '\n'
111                while curr_line != '' and \
112                        not _find_top_output_identifying_pattern(curr_line):
113                    curr_line = results_file.readline()
114                if curr_line == '':
115                    break
116                try:
117                    time, = _extract_values_before_keywords(curr_line, "up")
118
119                    # Ignore one line.
120                    _ = results_file.readline()
121
122                    # Get the cpu usage.
123                    curr_line = results_file.readline()
124                    (cpu_user, cpu_sys, cpu_nice, cpu_idle, io_wait, irq, sirq,
125                            steal) = _extract_values_before_keywords(curr_line,
126                            "us", "sy", "ni", "id", "wa", "hi", "si", "st")
127
128                    # Get memory usage.
129                    curr_line = results_file.readline()
130                    (mem_units, mem_free,
131                            mem_used) = _extract_values_before_keywords(
132                            curr_line, "Mem", "free", "used")
133
134                    # Get swap usage.
135                    curr_line = results_file.readline()
136                    (swap_units, swap_free,
137                            swap_used) = _extract_values_before_keywords(
138                            curr_line, "Swap", "free", "used")
139
140                    curr_util_values = ResourceMonitorParsedResult.UtilValues(
141                            Time=time, UserCPU=cpu_user,
142                            SysCPU=cpu_sys, NCPU=cpu_nice, Idle=cpu_idle,
143                            IOWait=io_wait, IRQ=irq, SoftIRQ=sirq, Steal=steal,
144                            MemUnits=mem_units, UsedMem=mem_used,
145                            FreeMem=mem_free,
146                            SwapUnits=swap_units, UsedSwap=swap_used,
147                            FreeSwap=swap_free)
148                    self._utils_over_time.append(curr_util_values)
149                except IncorrectTopFormat:
150                    logging.error(
151                            "Top output format incorrect. Aborting parse.")
152                    return
153
154
155    def __repr__(self):
156        output_stringfile = cStringIO.StringIO()
157        self.save_to_file(output_stringfile)
158        return output_stringfile.getvalue()
159
160
161    def save_to_file(self, file):
162        """Save parsed top results to file
163
164        @param file file object to write to
165
166        """
167        if len(self._utils_over_time) < 1:
168            logging.warning("Tried to save parsed results, but they were "
169                    "empty. Skipping the save.")
170            return
171        csvwriter = csv.writer(file, delimiter=',')
172        csvwriter.writerow(self._utils_over_time[0]._fields)
173        for row in self._utils_over_time:
174            csvwriter.writerow(row)
175
176
177    def save_to_filename(self, filename):
178        """Save parsed top results to filename
179
180        @param filename string filepath to write to
181
182        """
183        out_file = open(filename, "wb")
184        self.save_to_file(out_file)
185        out_file.close()
186
187
188class ResourceMonitorConfig(object):
189    """Defines a single top run."""
190
191    DEFAULT_MONITOR_PERIOD = 3
192
193    def __init__(self, monitor_period=DEFAULT_MONITOR_PERIOD,
194            rawresult_output_filename=None):
195        """Construct a ResourceMonitorConfig.
196
197        @param monitor_period float seconds between successive top refreshes.
198        @param rawresult_output_filename string filename to output the raw top
199                                                results to
200
201        """
202        if monitor_period < 0.1:
203            logging.info('Monitor period must be at least 0.1s.'
204                    ' Given: %r. Defaulting to 0.1s', monitor_period)
205            monitor_period = 0.1
206
207        self._monitor_period = monitor_period
208        self._server_outfile = rawresult_output_filename
209
210
211class ResourceMonitor(object):
212    """Delegate to run top on a client.
213
214    Usage example (call from a test):
215    rmc = resource_monitor.ResourceMonitorConfig(monitor_period=1,
216            rawresult_output_filename=os.path.join(self.resultsdir,
217                                                    'topout.txt'))
218    with resource_monitor.ResourceMonitor(self.context.client.host, rmc) as rm:
219        rm.start()
220        <operation_to_monitor>
221        rm_raw_res = rm.stop()
222        rm_res = rm_raw_res.get_parsed_results()
223        rm_res.save_to_filename(
224                os.path.join(self.resultsdir, 'resource_mon.csv'))
225
226    """
227
228    def __init__(self, client_host, config):
229        """Construct a ResourceMonitor.
230
231        @param client_host: SSHHost object representing a remote ssh host
232
233        """
234        self._client_host = client_host
235        self._config = config
236        self._command_top = path_utils.must_be_installed(
237                'top', host=self._client_host)
238        self._top_pid = None
239
240
241    def __enter__(self):
242        return self
243
244
245    def __exit__(self, exc_type, exc_value, traceback):
246        if self._top_pid is not None:
247            self._client_host.run('kill %s && rm %s' %
248                    (self._top_pid, self._client_outfile), ignore_status=True)
249        return True
250
251
252    def start(self):
253        """Run top and save results to a temp file on the client."""
254        if self._top_pid is not None:
255            logging.debug("Tried to start monitoring before stopping. "
256                    "Ignoring request.")
257            return
258
259        # Decide where to write top's output to (on the client).
260        random_suffix = random.random()
261        self._client_outfile = '/tmp/topcap-%r' % random_suffix
262
263        # Run top on the client.
264        top_command = '%s -b -d%d > %s' % (self._command_top,
265                self._config._monitor_period, self._client_outfile)
266        logging.info('Running top.')
267        self._top_pid = self._client_host.run_background(top_command)
268        logging.info('Top running with pid %s', self._top_pid)
269
270
271    def stop(self):
272        """Stop running top and return the results.
273
274        @return ResourceMonitorRawResult object
275
276        """
277        logging.debug("Stopping monitor")
278        if self._top_pid is None:
279            logging.debug("Tried to stop monitoring before starting. "
280                    "Ignoring request.")
281            return
282
283        # Stop top on the client.
284        self._client_host.run('kill %s' % self._top_pid, ignore_status=True)
285
286        # Get the top output file from the client onto the server.
287        if self._config._server_outfile is None:
288            self._config._server_outfile = self._client_outfile
289        self._client_host.get_file(
290                self._client_outfile, self._config._server_outfile)
291
292        # Delete the top output file from client.
293        self._client_host.run('rm %s' % self._client_outfile,
294                ignore_status=True)
295
296        self._top_pid = None
297        logging.info("Saved resource monitor results at %s",
298                self._config._server_outfile)
299        return ResourceMonitorRawResult(self._config._server_outfile)
300