1 The lldb-perf infrastructure for LLDB performance testing
2===========================================================
3
4lldb-perf is an infrastructure meant to simplify the creation of performance 
5tests for the LLDB debugger. It is contained in liblldbperf.a which is part of
6the standard opensource checkout of LLDB
7
8Its main concepts are:
9- Gauges: a gauge is a thing that takes a sample. Samples include elapsed time,
10  memory used, and energy consumed.
11- Metrics: a metric is a collection of samples that knows how to do statistics
12  like sum() and average(). Metrics can be extended as needed.
13- Measurements: a measurement is the thing that stores an action, a gauge and
14  a metric. You define measurements as in “take the time to run this function”,
15  “take the memory to run this block of code”, and then after you invoke it, 
16  your stats will automagically be there.
17- Tests: a test is a sequence of steps and measurements.
18
19Tests cases should be added as targets to the lldbperf.xcodeproj project. It 
20is probably easiest to duplicate one of the existing targets. In order to 
21write a test based on lldb-perf, you need to subclass  lldb_perf::TestCase:
22
23using namespace lldb_perf;
24
25class FormattersTest : public TestCase
26{
27
28Usually, you will define measurements as variables of your test case class:
29
30private:
31    // C++ formatters
32    TimeMeasurement<std::function<void(SBValue)>> m_dump_std_vector_measurement;
33    TimeMeasurement<std::function<void(SBValue)>> m_dump_std_list_measurement;
34    TimeMeasurement<std::function<void(SBValue)>> m_dump_std_map_measurement;
35    TimeMeasurement<std::function<void(SBValue)>> m_dump_std_string_measurement;
36
37    // Cocoa formatters
38    TimeMeasurement<std::function<void(SBValue)>> m_dump_nsstring_measurement;
39    TimeMeasurement<std::function<void(SBValue)>> m_dump_nsarray_measurement;
40    TimeMeasurement<std::function<void(SBValue)>> m_dump_nsdictionary_measurement;
41    TimeMeasurement<std::function<void(SBValue)>> m_dump_nsset_measurement;
42    TimeMeasurement<std::function<void(SBValue)>> m_dump_nsbundle_measurement;
43    TimeMeasurement<std::function<void(SBValue)>> m_dump_nsdate_measurement;
44
45A TimeMeasurement is, obviously, a class that measures “how much time to run 
46this block of code”. The block of code is passed as an std::function which you
47can construct with a lambda! You need to give the prototype of your block of
48code. In this example, we run blocks of code that take an SBValue and return
49nothing.
50
51These blocks look like:
52
53    m_dump_std_vector_measurement = CreateTimeMeasurement([] (SBValue value) -> void {
54        lldb_perf::Xcode::FetchVariable (value,1,false);
55    }, "std-vector", "time to dump an std::vector");
56
57Here we are saying: make me a measurement named “std-vector”, whose 
58description is “time to dump an std::vector” and that takes the time required
59to call lldb_perf::Xcode::FetchVariable(value,1,false).
60
61The Xcode class is a collection of utility functions that replicate common
62Xcode patterns (FetchVariable unsurprisingly calls API functions that Xcode
63could use when populating a variables view entry - the 1 means “expand 1 level
64of depth” and the false means “do not dump the data to stdout”)
65
66A full constructor for a TestCase looks like:
67
68FormattersTest () : TestCase()
69{
70    m_dump_std_vector_measurement = CreateTimeMeasurement([] (SBValue value) -> void {
71        lldb_perf::Xcode::FetchVariable (value,1,false);
72    }, "std-vector", "time to dump an std::vector");
73    m_dump_std_list_measurement = CreateTimeMeasurement([] (SBValue value) -> void {
74        lldb_perf::Xcode::FetchVariable (value,1,false);
75    }, "std-list", "time to dump an std::list");
76    m_dump_std_map_measurement = CreateTimeMeasurement([] (SBValue value) -> void {
77        lldb_perf::Xcode::FetchVariable (value,1,false);
78    }, "std-map", "time to dump an std::map");
79    m_dump_std_string_measurement = CreateTimeMeasurement([] (SBValue value) -> void {
80        lldb_perf::Xcode::FetchVariable (value,1,false);
81    }, "std-string", "time to dump an std::string");
82    
83    m_dump_nsstring_measurement = CreateTimeMeasurement([] (SBValue value) -> void {
84        lldb_perf::Xcode::FetchVariable (value,0,false);
85    }, "ns-string", "time to dump an NSString");
86    
87    m_dump_nsarray_measurement = CreateTimeMeasurement([] (SBValue value) -> void {
88        lldb_perf::Xcode::FetchVariable (value,1,false);
89    }, "ns-array", "time to dump an NSArray");
90    
91    m_dump_nsdictionary_measurement = CreateTimeMeasurement([] (SBValue value) -> void {
92        lldb_perf::Xcode::FetchVariable (value,1,false);
93    }, "ns-dictionary", "time to dump an NSDictionary");
94    
95    m_dump_nsset_measurement = CreateTimeMeasurement([] (SBValue value) -> void {
96        lldb_perf::Xcode::FetchVariable (value,1,false);
97    }, "ns-set", "time to dump an NSSet");
98    
99    m_dump_nsbundle_measurement = CreateTimeMeasurement([] (SBValue value) -> void {
100        lldb_perf::Xcode::FetchVariable (value,1,false);
101    }, "ns-bundle", "time to dump an NSBundle");
102    
103    m_dump_nsdate_measurement = CreateTimeMeasurement([] (SBValue value) -> void {
104        lldb_perf::Xcode::FetchVariable (value,0,false);
105    }, "ns-date", "time to dump an NSDate");
106}
107
108Once your test case is constructed, Setup() is called on it:
109
110    virtual bool
111	Setup (int argc, const char** argv)
112    {
113        m_app_path.assign(argv[1]);
114        m_out_path.assign(argv[2]);
115        m_target = m_debugger.CreateTarget(m_app_path.c_str());
116        m_target.BreakpointCreateByName("main");
117        SBLaunchInfo launch_info (argv);
118        return Launch (launch_info);
119    }
120    
121Setup() returns a boolean value that indicates if setup was successful.
122In Setup() you fill out a SBLaunchInfo with any needed settings for launching
123your process like arguments, environment variables, working directory, and
124much more.
125
126The last thing you want to do in setup is call Launch():
127
128	bool
129	Launch (coSBLaunchInfo &launch_info);
130
131This ensures your target is now alive. Make sure to have a breakpoint created.
132
133Once you launched, the event loop is entered. The event loop waits for stops, 
134and when it gets one, it calls your test case’s TestStep() function:
135
136    virtual void
137	TestStep (int counter, ActionWanted &next_action)
138
139the counter is the step id (a monotonically increasing counter). In TestStep()
140you will essentially run your measurements and then return what you want the
141driver to do by filling in the ActionWanted object named "next_action".
142
143Possible options are:
144- continue process          next_action.Continue();
145- kill process              next_action.Kill();
146- Step-out on a thread      next_action.StepOut(SBThread)
147- step-over on a thread.    next_action.StepOver(SBThread)
148
149If you use ActionWanted::Next() or ActionWanted::Finish() you need to specify
150a thread to use. By default the TestCase class will select the first thread
151that had a stop reason other than eStopReasonNone and place it into the 
152m_thread member variable of TestCase. This means if your test case hits a
153breakpoint or steps, the thread that hit the breakpoint or finished the step
154will automatically be selected in the process (m_process) and m_thread will
155be set to this thread. If you have one or more threads that will stop with a
156reason simultaneously, you will need to find those threads manually by 
157iterating through the process list and determine what to do next.
158
159For your convenience TestCase has m_debugger, m_target and m_process as member
160variables. As state above m_thread will be filled in with the first thread 
161that has a stop reason.
162
163An example:
164
165    virtual void
166	TestStep (int counter, ActionWanted &next_action)
167    {
168        case 0:
169            m_target.BreakpointCreateByLocation("fmts_tester.mm", 68);
170            next_action.Continue();
171            break;
172        case 1:
173            DoTest ();
174            next_action.Continue();
175            break;
176        case 2:
177            DoTest ();
178            next_action.StepOver(m_thread);
179            break;
180
181DoTest() is a function I define in my own class that calls the measurements:
182    void
183    DoTest ()
184    {
185        SBThread thread_main(m_thread);
186        SBFrame frame_zero(thread_main.GetFrameAtIndex(0));
187        
188        m_dump_nsarray_measurement(frame_zero.FindVariable("nsarray", lldb::eDynamicCanRunTarget));
189        m_dump_nsarray_measurement(frame_zero.FindVariable("nsmutablearray", lldb::eDynamicCanRunTarget));
190
191        m_dump_nsdictionary_measurement(frame_zero.FindVariable("nsdictionary", lldb::eDynamicCanRunTarget));
192        m_dump_nsdictionary_measurement(frame_zero.FindVariable("nsmutabledictionary", lldb::eDynamicCanRunTarget));
193        
194        m_dump_nsstring_measurement(frame_zero.FindVariable("str0", lldb::eDynamicCanRunTarget));
195        m_dump_nsstring_measurement(frame_zero.FindVariable("str1", lldb::eDynamicCanRunTarget));
196        m_dump_nsstring_measurement(frame_zero.FindVariable("str2", lldb::eDynamicCanRunTarget));
197        m_dump_nsstring_measurement(frame_zero.FindVariable("str3", lldb::eDynamicCanRunTarget));
198        m_dump_nsstring_measurement(frame_zero.FindVariable("str4", lldb::eDynamicCanRunTarget));
199        
200        m_dump_nsdate_measurement(frame_zero.FindVariable("me", lldb::eDynamicCanRunTarget));
201        m_dump_nsdate_measurement(frame_zero.FindVariable("cutie", lldb::eDynamicCanRunTarget));
202        m_dump_nsdate_measurement(frame_zero.FindVariable("mom", lldb::eDynamicCanRunTarget));
203        m_dump_nsdate_measurement(frame_zero.FindVariable("dad", lldb::eDynamicCanRunTarget));
204        m_dump_nsdate_measurement(frame_zero.FindVariable("today", lldb::eDynamicCanRunTarget));
205        
206        m_dump_nsbundle_measurement(frame_zero.FindVariable("bundles", lldb::eDynamicCanRunTarget));
207        m_dump_nsbundle_measurement(frame_zero.FindVariable("frameworks", lldb::eDynamicCanRunTarget));
208        
209        m_dump_nsset_measurement(frame_zero.FindVariable("nsset", lldb::eDynamicCanRunTarget));
210        m_dump_nsset_measurement(frame_zero.FindVariable("nsmutableset", lldb::eDynamicCanRunTarget));
211        
212        m_dump_std_vector_measurement(frame_zero.FindVariable("vector", lldb::eDynamicCanRunTarget));
213        m_dump_std_list_measurement(frame_zero.FindVariable("list", lldb::eDynamicCanRunTarget));
214        m_dump_std_map_measurement(frame_zero.FindVariable("map", lldb::eDynamicCanRunTarget));
215
216        m_dump_std_string_measurement(frame_zero.FindVariable("sstr0", lldb::eDynamicCanRunTarget));
217        m_dump_std_string_measurement(frame_zero.FindVariable("sstr1", lldb::eDynamicCanRunTarget));
218        m_dump_std_string_measurement(frame_zero.FindVariable("sstr2", lldb::eDynamicCanRunTarget));
219        m_dump_std_string_measurement(frame_zero.FindVariable("sstr3", lldb::eDynamicCanRunTarget));
220        m_dump_std_string_measurement(frame_zero.FindVariable("sstr4", lldb::eDynamicCanRunTarget));
221    }
222
223Essentially, you call your measurements as if they were functions, passing 
224them arguments and all, and they will do the right thing with gathering stats.
225
226The last step is usually to KILL the inferior and bail out:
227
228    virtual ActionWanted
229	TestStep (int counter)
230    {
231...     
232        case 9:
233            DoTest ();
234            next_action.Continue();
235            break;
236        case 10:
237            DoTest ();
238            next_action.Continue();
239            break;
240        default:
241            next_action.Kill();
242            break;
243    }
244
245
246At the end, you define a Results() function:
247
248    void
249    Results ()
250    {
251        CFCMutableArray array;
252        m_dump_std_vector_measurement.Write(array);
253        m_dump_std_list_measurement.Write(array);
254        m_dump_std_map_measurement.Write(array);
255        m_dump_std_string_measurement.Write(array);
256
257        m_dump_nsstring_measurement.Write(array);
258        m_dump_nsarray_measurement.Write(array);
259        m_dump_nsdictionary_measurement.Write(array);
260        m_dump_nsset_measurement.Write(array);
261        m_dump_nsbundle_measurement.Write(array);
262        m_dump_nsdate_measurement.Write(array);
263
264        CFDataRef xmlData = CFPropertyListCreateData (kCFAllocatorDefault, 
265                                                      array.get(), 
266                                                      kCFPropertyListXMLFormat_v1_0, 
267                                                      0, 
268                                                      NULL);
269        
270        CFURLRef file = CFURLCreateFromFileSystemRepresentation (NULL, 
271                                                                 (const UInt8*)m_out_path.c_str(), 
272                                                                 m_out_path.size(), 
273                                                                 FALSE);
274        
275        CFURLWriteDataAndPropertiesToResource(file,xmlData,NULL,NULL);
276    }
277
278For now, pretty much copy this and just call Write() on all your measurements.
279I plan to move this higher in the hierarchy (e.g. make a 
280TestCase::Write(filename) fairly soon).
281
282Your main() will look like:
283
284int main(int argc, const char * argv[])
285{
286    MyTest test;
287    TestCase::Run (test, argc, argv);
288    return 0;
289}
290
291If you are debugging your test, before Run() call
292
293    test.SetVerbose(true);
294
295Feel free to send any questions and ideas for improvements.
296