1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""An example main file running the algorithms.
5
6Part of the Chrome build flags optimization.
7
8An example use of the framework. It parses the input json configuration file.
9Then it initiates the variables of the generation. Finally, it sets up the
10processes for different modules and runs the experiment.
11"""
12
13__author__ = 'yuhenglong@google.com (Yuheng Long)'
14
15import json
16import multiprocessing
17from optparse import OptionParser
18import sys
19
20import flags
21from genetic_algorithm import GAGeneration
22from pipeline_process import PipelineProcess
23import pipeline_worker
24from steering import Steering
25from task import BUILD_STAGE
26from task import Task
27from task import TEST_STAGE
28import testing_batch
29
30parser = OptionParser()
31
32parser.add_option('-f',
33                  '--file',
34                  dest='filename',
35                  help='configuration file FILE input',
36                  metavar='FILE')
37
38# The meta data for the genetic algorithm.
39BUILD_CMD = 'BUILD_CMD'
40TEST_CMD = 'TEST_CMD'
41OUTPUT = 'OUTPUT'
42DEFAULT_OUTPUT = 'output'
43CONF = 'CONF'
44DEFAULT_CONF = 'conf'
45NUM_BUILDER = 'NUM_BUILDER'
46DEFAULT_NUM_BUILDER = 1
47NUM_TESTER = 'NUM_TESTER'
48DEFAULT_NUM_TESTER = 1
49STOP_THRESHOLD = 'STOP_THRESHOLD'
50DEFAULT_STOP_THRESHOLD = 1
51NUM_CHROMOSOMES = 'NUM_CHROMOSOMES'
52DEFAULT_NUM_CHROMOSOMES = 20
53NUM_TRIALS = 'NUM_TRIALS'
54DEFAULT_NUM_TRIALS = 20
55MUTATION_RATE = 'MUTATION_RATE'
56DEFAULT_MUTATION_RATE = 0.01
57
58
59def _ProcessGA(meta_data):
60  """Set up the meta data for the genetic algorithm.
61
62  Args:
63    meta_data: the meta data for the genetic algorithm.
64  """
65  assert BUILD_CMD in meta_data
66  build_cmd = meta_data[BUILD_CMD]
67
68  assert TEST_CMD in meta_data
69  test_cmd = meta_data[TEST_CMD]
70
71  if OUTPUT not in meta_data:
72    output_file = DEFAULT_OUTPUT
73  else:
74    output_file = meta_data[OUTPUT]
75
76  if CONF not in meta_data:
77    conf_file = DEFAULT_CONF
78  else:
79    conf_file = meta_data[CONF]
80
81  if NUM_BUILDER not in meta_data:
82    num_builders = DEFAULT_NUM_BUILDER
83  else:
84    num_builders = meta_data[NUM_BUILDER]
85
86  if NUM_TESTER not in meta_data:
87    num_testers = DEFAULT_NUM_TESTER
88  else:
89    num_testers = meta_data[NUM_TESTER]
90
91  if STOP_THRESHOLD not in meta_data:
92    stop_threshold = DEFAULT_STOP_THRESHOLD
93  else:
94    stop_threshold = meta_data[STOP_THRESHOLD]
95
96  if NUM_CHROMOSOMES not in meta_data:
97    num_chromosomes = DEFAULT_NUM_CHROMOSOMES
98  else:
99    num_chromosomes = meta_data[NUM_CHROMOSOMES]
100
101  if NUM_TRIALS not in meta_data:
102    num_trials = DEFAULT_NUM_TRIALS
103  else:
104    num_trials = meta_data[NUM_TRIALS]
105
106  if MUTATION_RATE not in meta_data:
107    mutation_rate = DEFAULT_MUTATION_RATE
108  else:
109    mutation_rate = meta_data[MUTATION_RATE]
110
111  specs = flags.ReadConf(conf_file)
112
113  # Initiate the build/test command and the log directory.
114  Task.InitLogCommand(build_cmd, test_cmd, output_file)
115
116  # Initiate the build/test command and the log directory.
117  GAGeneration.InitMetaData(stop_threshold, num_chromosomes, num_trials, specs,
118                            mutation_rate)
119
120  # Generate the initial generations.
121  generation_tasks = testing_batch.GenerateRandomGATasks(specs, num_chromosomes,
122                                                         num_trials)
123  generations = [GAGeneration(generation_tasks, set([]), 0)]
124
125  # Execute the experiment.
126  _StartExperiment(num_builders, num_testers, generations)
127
128
129def _ParseJson(file_name):
130  """Parse the input json file.
131
132  Parse the input json file and call the proper function to perform the
133  algorithms.
134
135  Args:
136    file_name: the input json file name.
137  """
138
139  experiments = json.load(open(file_name))
140
141  for experiment in experiments:
142    if experiment == 'GA':
143      # An GA experiment
144      _ProcessGA(experiments[experiment])
145
146
147def _StartExperiment(num_builders, num_testers, generations):
148  """Set up the experiment environment and execute the framework.
149
150  Args:
151    num_builders: number of concurrent builders.
152    num_testers: number of concurrent testers.
153    generations: the initial generation for the framework.
154  """
155
156  manager = multiprocessing.Manager()
157
158  # The queue between the steering algorithm and the builder.
159  steering_build = manager.Queue()
160  # The queue between the builder and the tester.
161  build_test = manager.Queue()
162  # The queue between the tester and the steering algorithm.
163  test_steering = manager.Queue()
164
165  # Set up the processes for the builder, tester and steering algorithm module.
166  build_process = PipelineProcess(num_builders, 'builder', {}, BUILD_STAGE,
167                                  steering_build, pipeline_worker.Helper,
168                                  pipeline_worker.Worker, build_test)
169
170  test_process = PipelineProcess(num_testers, 'tester', {}, TEST_STAGE,
171                                 build_test, pipeline_worker.Helper,
172                                 pipeline_worker.Worker, test_steering)
173
174  steer_process = multiprocessing.Process(
175      target=Steering,
176      args=(set([]), generations, test_steering, steering_build))
177
178  # Start the processes.
179  build_process.start()
180  test_process.start()
181  steer_process.start()
182
183  # Wait for the processes to finish.
184  build_process.join()
185  test_process.join()
186  steer_process.join()
187
188
189def main(argv):
190  (options, _) = parser.parse_args(argv)
191  assert options.filename
192  _ParseJson(options.filename)
193
194
195if __name__ == '__main__':
196  main(sys.argv)
197