Constraint.py revision daf270b6b7176619f542a13a9403d0ed9cf4172d 
1#    Copyright 2015-2016 ARM Limited
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14#
15
16"""This module provides the Constraint class for handling
17filters and pivots in a modular fashion. This enable easy
18constraint application.
19
20An implementation of :mod:`trappy.plotter.AbstractDataPlotter`
21is expected to use the :mod:`trappy.plotter.Constraint.ConstraintManager`
22class to pivot and filter data and handle multiple column,
23trace and event inputs.
24
25The underlying object that encapsulates a unique set of
26a data column, data event and the requisite filters is
27:mod:`trappy.plotter.Constraint.Constraint`
28"""
29# pylint: disable=R0913
30from trappy.plotter.Utils import decolonize, normalize_list
31from trappy.utils import listify
32from trappy.plotter import AttrConf
33from trappy.utils import handle_duplicate_index
34
35
36class Constraint(object):
37
38    """
39    What is a Constraint?
40        It is collection of data based on two rules:
41
42        - A Pivot
43
44        - A Set of Filters
45
46        - A Data Column
47
48    For Example a :mod:`pandas.DataFrame`
49
50    =====  ======== =========
51    Time    CPU       Latency
52    =====  ======== =========
53    1       x           <val>
54    2       y           <val>
55    3       z           <val>
56    4       a           <val>
57    =====  ======== =========
58
59    The resultant data will be split for each unique pivot value
60    with the filters applied
61    ::
62
63        result["x"] = pd.Series.filtered()
64        result["y"] = pd.Series.filtered()
65        result["z"] = pd.Series.filtered()
66        result["a"] = pd.Series.filtered()
67
68
69    :param trappy_trace: Input Data
70    :type trappy_trace: :mod:`pandas.DataFrame`, :mod:`trappy.trace.FTrace`
71
72    :param column: The data column
73    :type column: str
74
75    :param template: TRAPpy Event
76    :type template: :mod:`trappy.base.Base` event
77
78    :param trace_index: The index of the trace/data in the overall constraint
79        data
80    :type trace_index: int
81
82    :param filters: A dictionary of filter values
83    :type filters: dict
84    """
85
86    def __init__(self, trappy_trace, pivot, column, template, trace_index,
87                 filters):
88        self._trappy_trace = trappy_trace
89        self._filters = filters
90        self._pivot = pivot
91        self.column = column
92        self._template = template
93        self._dup_resolved = False
94        self._data = self.populate_data_frame()
95
96        try:
97            self.result = self._apply()
98        except ValueError:
99            if not self._dup_resolved:
100                self._handle_duplicate_index()
101                try:
102                    self.result = self._apply()
103                except:
104                    raise ValueError("Unable to handle duplicates")
105
106        self.trace_index = trace_index
107
108    def _apply(self):
109        """This method applies the filter on the resultant data
110        on the input column.
111        """
112        data = self._data
113        result = {}
114
115        try:
116            values = data[self.column]
117        except KeyError:
118            return result
119
120        if self._pivot == AttrConf.PIVOT:
121            criterion = values.map(lambda x: True)
122            for key in self._filters.keys():
123                if key in data.columns:
124                    criterion = criterion & data[key].map(
125                        lambda x: x in self._filters[key])
126                    values = values[criterion]
127            result[AttrConf.PIVOT_VAL] = values
128            return result
129
130        pivot_vals = self.pivot_vals(data)
131
132        for pivot_val in pivot_vals:
133            criterion = values.map(lambda x: True)
134
135            for key in self._filters.keys():
136                if key != self._pivot and key in data.columns:
137                    criterion = criterion & data[key].map(
138                        lambda x: x in self._filters[key])
139                    values = values[criterion]
140
141            val_series = values[data[self._pivot] == pivot_val]
142            if len(val_series) != 0:
143               result[pivot_val] = val_series
144
145        return result
146
147    def _handle_duplicate_index(self):
148        """Handle duplicate values in index"""
149
150        self._data = handle_duplicate_index(self._data)
151        self._dup_resolved = True
152
153    def _uses_trappy_trace(self):
154        if not self._template:
155            return False
156        else:
157            return True
158
159    def populate_data_frame(self):
160        """Return the populated :mod:`pandas.DataFrame`"""
161        if not self._uses_trappy_trace():
162            return self._trappy_trace
163
164        data_container = getattr(
165            self._trappy_trace,
166            decolonize(self._template.name))
167        return data_container.data_frame
168
169    def pivot_vals(self, data):
170        """This method returns the unique pivot values for the
171        Constraint's pivot and the column
172
173        :param data: Input Data
174        :type data: :mod:`pandas.DataFrame`
175        """
176        if self._pivot == AttrConf.PIVOT:
177            return AttrConf.PIVOT_VAL
178
179        if self._pivot not in data.columns:
180            return []
181
182        pivot_vals = set(data[self._pivot])
183        if self._pivot in self._filters:
184            pivot_vals = pivot_vals & set(self._filters[self._pivot])
185
186        return list(pivot_vals)
187
188    def __str__(self):
189
190        name = self.get_data_name()
191
192        if not self._uses_trappy_trace():
193            return name + ":" + self.column
194
195        return name + ":" + \
196            self._template.name + ":" + self.column
197
198
199    def get_data_name(self):
200        """Get name for the data member. This method
201        relies on the "name" attribute for the name.
202        If the name attribute is absent, it associates
203        a numeric name to the respective data element
204
205        :returns: The name of the data member
206        """
207        if self._uses_trappy_trace():
208            if self._trappy_trace.name != "":
209                return self._trappy_trace.name
210            else:
211                return "Trace {}".format(self.trace_index)
212        else:
213            return "DataFrame {}".format(self.trace_index)
214
215class ConstraintManager(object):
216
217    """A class responsible for converting inputs
218    to constraints and also ensuring sanity
219
220
221    :param traces: Input Trace data
222    :type traces: :mod:`trappy.trace.FTrace`, list(:mod:`trappy.trace.FTrace`)
223    :param columns: The column values from the corresponding
224        :mod:`pandas.DataFrame`
225    :type columns: str, list(str)
226    :param pivot: The column around which the data will be
227        pivoted:
228    :type pivot: str
229    :param filters: A dictionary of values to be applied on the
230        respective columns
231    :type filters: dict
232    :param zip_constraints: Permutes the columns and traces instead
233        of a one-to-one correspondence
234    :type zip_constraints: bool
235    """
236
237    def __init__(self, traces, columns, templates, pivot, filters,
238                 zip_constraints=True):
239
240        self._ip_vec = []
241        self._ip_vec.append(listify(traces))
242        self._ip_vec.append(listify(columns))
243        self._ip_vec.append(listify(templates))
244
245        self._lens = map(len, self._ip_vec)
246        self._max_len = max(self._lens)
247        self._pivot = pivot
248        self._filters = filters
249        self._constraints = []
250
251        self._trace_expanded = False
252        self._expand()
253        if zip_constraints:
254            self._populate_zip_constraints()
255        else:
256            self._populate_constraints()
257
258    def _expand(self):
259        """This is really important. We need to
260        meet the following criteria for constraint
261        expansion:
262        ::
263
264            Len[traces] == Len[columns] == Len[templates]
265
266        Or:
267        ::
268
269            Permute(
270                Len[traces] = 1
271                Len[columns] = 1
272                Len[templates] != 1
273            )
274
275            Permute(
276                   Len[traces] = 1
277                   Len[columns] != 1
278                   Len[templates] != 1
279            )
280        """
281        min_len = min(self._lens)
282        max_pos_comp = [
283            i for i,
284            j in enumerate(
285                self._lens) if j != self._max_len]
286
287        if self._max_len == 1 and min_len != 1:
288            raise RuntimeError("Essential Arg Missing")
289
290        if self._max_len > 1:
291
292            # Are they all equal?
293            if len(set(self._lens)) == 1:
294                return
295
296            if min_len > 1:
297                raise RuntimeError("Cannot Expand a list of Constraints")
298
299            for val in max_pos_comp:
300                if val == 0:
301                    self._trace_expanded = True
302                self._ip_vec[val] = normalize_list(self._max_len,
303                                                   self._ip_vec[val])
304
305    def _populate_constraints(self):
306        """Populate the constraints creating one for each column in
307        each trace
308
309        In a multi-trace, multicolumn scenario, constraints are created for
310        all the columns in each of the traces.  _populate_constraints()
311        creates one constraint for the first trace and first column, the
312        next for the second trace and second column,...  This function
313        creates a constraint for every combination of traces and columns
314        possible.
315        """
316
317        for trace_idx, trace in enumerate(self._ip_vec[0]):
318            for col in self._ip_vec[1]:
319                template = self._ip_vec[2][trace_idx]
320                constraint = Constraint(trace, self._pivot, col, template,
321                                        trace_idx, self._filters)
322                self._constraints.append(constraint)
323
324    def get_column_index(self, constraint):
325        return self._ip_vec[1].index(constraint.column)
326
327    def _populate_zip_constraints(self):
328        """Populate the expanded constraints
329
330        In a multitrace, multicolumn scenario, create constraints for
331        the first trace and the first column, second trace and second
332        column,... that is, as if you run zip(traces, columns)
333        """
334
335        for idx in range(self._max_len):
336            if self._trace_expanded:
337                trace_idx = 0
338            else:
339                trace_idx = idx
340
341            trace = self._ip_vec[0][idx]
342            col = self._ip_vec[1][idx]
343            template = self._ip_vec[2][idx]
344            self._constraints.append(
345                Constraint(trace, self._pivot, col, template, trace_idx,
346                           self._filters))
347
348    def generate_pivots(self, permute=False):
349        """Return a union of the pivot values
350
351        :param permute: Permute the Traces and Columns
352        :type permute: bool
353        """
354        pivot_vals = []
355        for constraint in self._constraints:
356            pivot_vals += constraint.result.keys()
357
358        p_list = list(set(pivot_vals))
359        traces = range(self._lens[0])
360
361        try:
362            sorted_plist = sorted(p_list, key=int)
363        except ValueError, TypeError:
364            try:
365                sorted_plist = sorted(p_list, key=lambda x: int(x, 16))
366            except ValueError, TypeError:
367                sorted_plist = sorted(p_list)
368
369        if permute:
370            pivot_gen = ((trace_idx, pivot) for trace_idx in traces for pivot in sorted_plist)
371            return pivot_gen, len(sorted_plist) * self._lens[0]
372        else:
373            return sorted_plist, len(sorted_plist)
374
375    def constraint_labels(self):
376        """
377        :return: string to represent the
378            set of Constraints
379
380        """
381        return map(str, self._constraints)
382
383    def __len__(self):
384        return len(self._constraints)
385
386    def __iter__(self):
387        return iter(self._constraints)
388