Constraint.py revision 5122084171b205379ed288e9dd45b0ad116d2ead 
1#    Copyright 2015-2016 ARM Limited
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14#
15
16"""This module provides the Constraint class for handling
17filters and pivots in a modular fashion. This enable easy
18constraint application.
19
20An implementation of :mod:`trappy.plotter.AbstractDataPlotter`
21is expected to use the :mod:`trappy.plotter.Constraint.ConstraintManager`
22class to pivot and filter data and handle multiple column,
23trace and event inputs.
24
25The underlying object that encapsulates a unique set of
26a data column, data event and the requisite filters is
27:mod:`trappy.plotter.Constraint.Constraint`
28"""
29# pylint: disable=R0913
30from trappy.plotter.Utils import decolonize, normalize_list
31from trappy.utils import listify
32from trappy.plotter import AttrConf
33
34
35class Constraint(object):
36
37    """
38    What is a Constraint?
39        It is collection of data based on two rules:
40
41        - A Pivot
42
43        - A Set of Filters
44
45        - A Data Column
46
47    For Example a :mod:`pandas.DataFrame`
48
49    =====  ======== =========
50    Time    CPU       Latency
51    =====  ======== =========
52    1       x           <val>
53    2       y           <val>
54    3       z           <val>
55    4       a           <val>
56    =====  ======== =========
57
58    The resultant data will be split for each unique pivot value
59    with the filters applied
60    ::
61
62        result["x"] = pd.Series.filtered()
63        result["y"] = pd.Series.filtered()
64        result["z"] = pd.Series.filtered()
65        result["a"] = pd.Series.filtered()
66
67
68    :param trappy_trace: Input Data
69    :type trappy_trace: :mod:`pandas.DataFrame` or a class derived from
70        :mod:`trappy.trace.BareTrace`
71
72    :param column: The data column
73    :type column: str
74
75    :param template: TRAPpy Event
76    :type template: :mod:`trappy.base.Base` event
77
78    :param trace_index: The index of the trace/data in the overall constraint
79        data
80    :type trace_index: int
81
82    :param filters: A dictionary of filter values
83    :type filters: dict
84    """
85
86    def __init__(self, trappy_trace, pivot, column, template, trace_index,
87                 filters):
88        self._trappy_trace = trappy_trace
89        self._filters = filters
90        self._pivot = pivot
91        self.column = column
92        self._template = template
93        self._dup_resolved = False
94        self._data = self.populate_data_frame()
95
96        self.result = self._apply()
97        self.trace_index = trace_index
98
99    def _apply(self):
100        """This method applies the filter on the resultant data
101        on the input column.
102        """
103        data = self._data
104        result = {}
105
106        try:
107            values = data[self.column]
108        except KeyError:
109            return result
110
111        if self._pivot == AttrConf.PIVOT:
112            pivot_vals = [AttrConf.PIVOT_VAL]
113        else:
114            pivot_vals = self.pivot_vals(data)
115
116        for pivot_val in pivot_vals:
117            criterion = values.map(lambda x: True)
118
119            for key in self._filters.keys():
120                if key != self._pivot and key in data.columns:
121                    criterion = criterion & data[key].map(
122                        lambda x: x in self._filters[key])
123
124            if pivot_val != AttrConf.PIVOT_VAL:
125                criterion &= data[self._pivot] == pivot_val
126
127            val_series = values[criterion]
128            if len(val_series) != 0:
129                result[pivot_val] = val_series
130
131        return result
132
133    def _uses_trappy_trace(self):
134        if not self._template:
135            return False
136        else:
137            return True
138
139    def populate_data_frame(self):
140        """Return the populated :mod:`pandas.DataFrame`"""
141        if not self._uses_trappy_trace():
142            return self._trappy_trace
143
144        data_container = getattr(
145            self._trappy_trace,
146            decolonize(self._template.name))
147        return data_container.data_frame
148
149    def pivot_vals(self, data):
150        """This method returns the unique pivot values for the
151        Constraint's pivot and the column
152
153        :param data: Input Data
154        :type data: :mod:`pandas.DataFrame`
155        """
156        if self._pivot == AttrConf.PIVOT:
157            return AttrConf.PIVOT_VAL
158
159        if self._pivot not in data.columns:
160            return []
161
162        pivot_vals = set(data[self._pivot])
163        if self._pivot in self._filters:
164            pivot_vals = pivot_vals & set(self._filters[self._pivot])
165
166        return list(pivot_vals)
167
168    def __str__(self):
169
170        name = self.get_data_name()
171
172        if not self._uses_trappy_trace():
173            return name + ":" + self.column
174
175        return name + ":" + \
176            self._template.name + ":" + self.column
177
178
179    def get_data_name(self):
180        """Get name for the data member. This method
181        relies on the "name" attribute for the name.
182        If the name attribute is absent, it associates
183        a numeric name to the respective data element
184
185        :returns: The name of the data member
186        """
187        if self._uses_trappy_trace():
188            if self._trappy_trace.name != "":
189                return self._trappy_trace.name
190            else:
191                return "Trace {}".format(self.trace_index)
192        else:
193            return "DataFrame {}".format(self.trace_index)
194
195class ConstraintManager(object):
196
197    """A class responsible for converting inputs
198    to constraints and also ensuring sanity
199
200
201    :param traces: Input Trace data
202    :type traces: :mod:`trappy.trace.BareTrace`, list(:mod:`trappy.trace.BareTrace`)
203        (or a class derived from :mod:`trappy.trace.BareTrace`)
204    :param columns: The column values from the corresponding
205        :mod:`pandas.DataFrame`
206    :type columns: str, list(str)
207    :param pivot: The column around which the data will be
208        pivoted:
209    :type pivot: str
210    :param filters: A dictionary of values to be applied on the
211        respective columns
212    :type filters: dict
213    :param zip_constraints: Permutes the columns and traces instead
214        of a one-to-one correspondence
215    :type zip_constraints: bool
216    """
217
218    def __init__(self, traces, columns, templates, pivot, filters,
219                 zip_constraints=True):
220
221        self._ip_vec = []
222        self._ip_vec.append(listify(traces))
223        self._ip_vec.append(listify(columns))
224        self._ip_vec.append(listify(templates))
225
226        self._lens = map(len, self._ip_vec)
227        self._max_len = max(self._lens)
228        self._pivot = pivot
229        self._filters = filters
230        self._constraints = []
231
232        self._trace_expanded = False
233        self._expand()
234        if zip_constraints:
235            self._populate_zip_constraints()
236        else:
237            self._populate_constraints()
238
239    def _expand(self):
240        """This is really important. We need to
241        meet the following criteria for constraint
242        expansion:
243        ::
244
245            Len[traces] == Len[columns] == Len[templates]
246
247        Or:
248        ::
249
250            Permute(
251                Len[traces] = 1
252                Len[columns] = 1
253                Len[templates] != 1
254            )
255
256            Permute(
257                   Len[traces] = 1
258                   Len[columns] != 1
259                   Len[templates] != 1
260            )
261        """
262        min_len = min(self._lens)
263        max_pos_comp = [
264            i for i,
265            j in enumerate(
266                self._lens) if j != self._max_len]
267
268        if self._max_len == 1 and min_len != 1:
269            raise RuntimeError("Essential Arg Missing")
270
271        if self._max_len > 1:
272
273            # Are they all equal?
274            if len(set(self._lens)) == 1:
275                return
276
277            if min_len > 1:
278                raise RuntimeError("Cannot Expand a list of Constraints")
279
280            for val in max_pos_comp:
281                if val == 0:
282                    self._trace_expanded = True
283                self._ip_vec[val] = normalize_list(self._max_len,
284                                                   self._ip_vec[val])
285
286    def _populate_constraints(self):
287        """Populate the constraints creating one for each column in
288        each trace
289
290        In a multi-trace, multicolumn scenario, constraints are created for
291        all the columns in each of the traces.  _populate_constraints()
292        creates one constraint for the first trace and first column, the
293        next for the second trace and second column,...  This function
294        creates a constraint for every combination of traces and columns
295        possible.
296        """
297
298        for trace_idx, trace in enumerate(self._ip_vec[0]):
299            for col in self._ip_vec[1]:
300                template = self._ip_vec[2][trace_idx]
301                constraint = Constraint(trace, self._pivot, col, template,
302                                        trace_idx, self._filters)
303                self._constraints.append(constraint)
304
305    def get_column_index(self, constraint):
306        return self._ip_vec[1].index(constraint.column)
307
308    def _populate_zip_constraints(self):
309        """Populate the expanded constraints
310
311        In a multitrace, multicolumn scenario, create constraints for
312        the first trace and the first column, second trace and second
313        column,... that is, as if you run zip(traces, columns)
314        """
315
316        for idx in range(self._max_len):
317            if self._trace_expanded:
318                trace_idx = 0
319            else:
320                trace_idx = idx
321
322            trace = self._ip_vec[0][idx]
323            col = self._ip_vec[1][idx]
324            template = self._ip_vec[2][idx]
325            self._constraints.append(
326                Constraint(trace, self._pivot, col, template, trace_idx,
327                           self._filters))
328
329    def generate_pivots(self, permute=False):
330        """Return a union of the pivot values
331
332        :param permute: Permute the Traces and Columns
333        :type permute: bool
334        """
335        pivot_vals = []
336        for constraint in self._constraints:
337            pivot_vals += constraint.result.keys()
338
339        p_list = list(set(pivot_vals))
340        traces = range(self._lens[0])
341
342        try:
343            sorted_plist = sorted(p_list, key=int)
344        except (ValueError, TypeError):
345            try:
346                sorted_plist = sorted(p_list, key=lambda x: int(x, 16))
347            except (ValueError, TypeError):
348                sorted_plist = sorted(p_list)
349
350        if permute:
351            pivot_gen = ((trace_idx, pivot) for trace_idx in traces for pivot in sorted_plist)
352            return pivot_gen, len(sorted_plist) * self._lens[0]
353        else:
354            return sorted_plist, len(sorted_plist)
355
356    def constraint_labels(self):
357        """
358        :return: string to represent the
359            set of Constraints
360
361        """
362        return map(str, self._constraints)
363
364    def __len__(self):
365        return len(self._constraints)
366
367    def __iter__(self):
368        return iter(self._constraints)
369