example_algorithms.py revision ed75d963c18c879b215af329888ded0b82bf0ae9 
1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""An example main file running the algorithms.
6
7Part of the Chrome build flags optimization.
8
9An example use of the framework. It parses the input json configuration file.
10Then it initiates the variables of the generation. Finally, it sets up the
11processes for different modules and runs the experiment.
12"""
13
14__author__ = 'yuhenglong@google.com (Yuheng Long)'
15
16import json
17import multiprocessing
18from optparse import OptionParser
19import sys
20
21import flags
22from genetic_algorithm import GAGeneration
23from pipeline_process import PipelineProcess
24import pipeline_worker
25from steering import Steering
26from task import BUILD_STAGE
27from task import Task
28from task import TEST_STAGE
29import testing_batch
30
31parser = OptionParser()
32
33parser.add_option('-f', '--file', dest='filename',
34                  help='configuration file FILE input', metavar='FILE')
35
36# The meta data for the genetic algorithm.
37BUILD_CMD = 'BUILD_CMD'
38TEST_CMD = 'TEST_CMD'
39OUTPUT = 'OUTPUT'
40DEFAULT_OUTPUT = 'output'
41CONF = 'CONF'
42DEFAULT_CONF = 'conf'
43NUM_BUILDER = 'NUM_BUILDER'
44DEFAULT_NUM_BUILDER = 1
45NUM_TESTER = 'NUM_TESTER'
46DEFAULT_NUM_TESTER = 1
47STOP_THRESHOLD = 'STOP_THRESHOLD'
48DEFAULT_STOP_THRESHOLD = 1
49NUM_CHROMOSOMES = 'NUM_CHROMOSOMES'
50DEFAULT_NUM_CHROMOSOMES = 20
51NUM_TRIALS = 'NUM_TRIALS'
52DEFAULT_NUM_TRIALS = 20
53MUTATION_RATE = 'MUTATION_RATE'
54DEFAULT_MUTATION_RATE = 0.01
55
56
57def _ProcessGA(meta_data):
58  """Set up the meta data for the genetic algorithm.
59
60  Args:
61    meta_data: the meta data for the genetic algorithm.
62  """
63  assert BUILD_CMD in meta_data
64  build_cmd = meta_data[BUILD_CMD]
65
66  assert TEST_CMD in meta_data
67  test_cmd = meta_data[TEST_CMD]
68
69  if OUTPUT not in meta_data:
70    output_file = DEFAULT_OUTPUT
71  else:
72    output_file = meta_data[OUTPUT]
73
74  if CONF not in meta_data:
75    conf_file = DEFAULT_CONF
76  else:
77    conf_file = meta_data[CONF]
78
79  if NUM_BUILDER not in meta_data:
80    num_builders = DEFAULT_NUM_BUILDER
81  else:
82    num_builders = meta_data[NUM_BUILDER]
83
84  if NUM_TESTER not in meta_data:
85    num_testers = DEFAULT_NUM_TESTER
86  else:
87    num_testers = meta_data[NUM_TESTER]
88
89  if STOP_THRESHOLD not in meta_data:
90    stop_threshold = DEFAULT_STOP_THRESHOLD
91  else:
92    stop_threshold = meta_data[STOP_THRESHOLD]
93
94  if NUM_CHROMOSOMES not in meta_data:
95    num_chromosomes = DEFAULT_NUM_CHROMOSOMES
96  else:
97    num_chromosomes = meta_data[NUM_CHROMOSOMES]
98
99  if NUM_TRIALS not in meta_data:
100    num_trials = DEFAULT_NUM_TRIALS
101  else:
102    num_trials = meta_data[NUM_TRIALS]
103
104  if MUTATION_RATE not in meta_data:
105    mutation_rate = DEFAULT_MUTATION_RATE
106  else:
107    mutation_rate = meta_data[MUTATION_RATE]
108
109  specs = flags.ReadConf(conf_file)
110
111  # Initiate the build/test command and the log directory.
112  Task.InitLogCommand(build_cmd, test_cmd, output_file)
113
114  # Initiate the build/test command and the log directory.
115  GAGeneration.InitMetaData(stop_threshold, num_chromosomes, num_trials,
116                            specs, mutation_rate)
117
118  # Generate the initial generations.
119  generation_tasks = testing_batch.GenerateRandomGATasks(specs, num_chromosomes,
120                                                         num_trials)
121  generations = [GAGeneration(generation_tasks, set([]), 0)]
122
123  # Execute the experiment.
124  _StartExperiment(num_builders, num_testers, generations)
125
126
127def _ParseJson(file_name):
128  """Parse the input json file.
129
130  Parse the input json file and call the proper function to perform the
131  algorithms.
132
133  Args:
134    file_name: the input json file name.
135  """
136
137  experiments = json.load(open(file_name))
138
139  for experiment in experiments:
140    if experiment == 'GA':
141      # An GA experiment
142      _ProcessGA(experiments[experiment])
143
144
145def _StartExperiment(num_builders, num_testers, generations):
146  """Set up the experiment environment and execute the framework.
147
148  Args:
149    num_builders: number of concurrent builders.
150    num_testers: number of concurrent testers.
151    generations: the initial generation for the framework.
152  """
153
154  manager = multiprocessing.Manager()
155
156  # The queue between the steering algorithm and the builder.
157  steering_build = manager.Queue()
158  # The queue between the builder and the tester.
159  build_test = manager.Queue()
160  # The queue between the tester and the steering algorithm.
161  test_steering = manager.Queue()
162
163  # Set up the processes for the builder, tester and steering algorithm module.
164  build_process = PipelineProcess(num_builders, 'builder', {}, BUILD_STAGE,
165                                  steering_build, pipeline_worker.Helper,
166                                  pipeline_worker.Worker, build_test)
167
168  test_process = PipelineProcess(num_testers, 'tester', {}, TEST_STAGE,
169                                 build_test, pipeline_worker.Helper,
170                                 pipeline_worker.Worker, test_steering)
171
172  steer_process = multiprocessing.Process(target=Steering,
173                                          args=(set([]), generations,
174                                                test_steering, steering_build))
175
176  # Start the processes.
177  build_process.start()
178  test_process.start()
179  steer_process.start()
180
181  # Wait for the processes to finish.
182  build_process.join()
183  test_process.join()
184  steer_process.join()
185
186
187def main(argv):
188  (options, _) = parser.parse_args(argv)
189  assert options.filename
190  _ParseJson(options.filename)
191
192
193if __name__ == '__main__':
194  main(sys.argv)
195