1c7f1593f9af3ea1b9264b37628c36f3a70e1749aMike Frysinger#!/usr/bin/python 220082a07cd83cc4cf11ce35566e2d20266c60997kbaclawski# 320082a07cd83cc4cf11ce35566e2d20266c60997kbaclawski# Copyright 2011 Google Inc. All Rights Reserved. 420082a07cd83cc4cf11ce35566e2d20266c60997kbaclawski"""Script to divide and merge profiles.""" 53b30395f032ff7aab615625809072e855dfd5d8easharif 63b30395f032ff7aab615625809072e855dfd5d8easharifimport copy 73b30395f032ff7aab615625809072e855dfd5d8easharifimport optparse 83b30395f032ff7aab615625809072e855dfd5d8easharifimport os 93b30395f032ff7aab615625809072e855dfd5d8easharifimport pickle 103b30395f032ff7aab615625809072e855dfd5d8easharifimport re 113b30395f032ff7aab615625809072e855dfd5d8easharifimport sys 1220082a07cd83cc4cf11ce35566e2d20266c60997kbaclawskiimport tempfile 1320082a07cd83cc4cf11ce35566e2d20266c60997kbaclawski 143b30395f032ff7aab615625809072e855dfd5d8easharifimport build_chrome_browser 153b30395f032ff7aab615625809072e855dfd5d8easharifimport lock_machine 163b30395f032ff7aab615625809072e855dfd5d8easharifimport run_tests 17a8af9a7a2462b00e72deff99327bdb452a715277Caroline Ticefrom cros_utils import command_executer 18a8af9a7a2462b00e72deff99327bdb452a715277Caroline Ticefrom cros_utils import logger 193b30395f032ff7aab615625809072e855dfd5d8easharif 203b30395f032ff7aab615625809072e855dfd5d8easharif 213b30395f032ff7aab615625809072e855dfd5d8easharifclass ProfileMerger: 22f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano 23fd54f25d991c89e8f0297c9d62b0ce592825d956asharif def __init__(self, inputs, output, chunk_size, merge_program, multipliers): 243b30395f032ff7aab615625809072e855dfd5d8easharif self._inputs = inputs 253b30395f032ff7aab615625809072e855dfd5d8easharif self._output = output 263b30395f032ff7aab615625809072e855dfd5d8easharif self._chunk_size = chunk_size 273b30395f032ff7aab615625809072e855dfd5d8easharif self._merge_program = merge_program 28fd54f25d991c89e8f0297c9d62b0ce592825d956asharif self._multipliers = multipliers 293b30395f032ff7aab615625809072e855dfd5d8easharif self._ce = command_executer.GetCommandExecuter() 303b30395f032ff7aab615625809072e855dfd5d8easharif self._l = logger.GetLogger() 313b30395f032ff7aab615625809072e855dfd5d8easharif 323b30395f032ff7aab615625809072e855dfd5d8easharif def _GetFilesSetForInputDir(self, input_dir): 333b30395f032ff7aab615625809072e855dfd5d8easharif output_file = tempfile.mktemp() 34f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano command = "find %s -name '*.gcda' -o -name '*.imports' > %s" % (input_dir, 35f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano output_file) 363b30395f032ff7aab615625809072e855dfd5d8easharif self._ce.RunCommand(command) 37f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano files = open(output_file, 'r').read() 383b30395f032ff7aab615625809072e855dfd5d8easharif files_set = set([]) 393b30395f032ff7aab615625809072e855dfd5d8easharif for f in files.splitlines(): 40f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano stripped_file = f.replace(input_dir, '', 1) 41f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano stripped_file = stripped_file.lstrip('/') 423b30395f032ff7aab615625809072e855dfd5d8easharif files_set.add(stripped_file) 433b30395f032ff7aab615625809072e855dfd5d8easharif return files_set 443b30395f032ff7aab615625809072e855dfd5d8easharif 453b30395f032ff7aab615625809072e855dfd5d8easharif def _PopulateFilesSet(self): 463b30395f032ff7aab615625809072e855dfd5d8easharif self._files_set = set([]) 473b30395f032ff7aab615625809072e855dfd5d8easharif for i in self._inputs: 483b30395f032ff7aab615625809072e855dfd5d8easharif current_files_set = self._GetFilesSetForInputDir(i) 493b30395f032ff7aab615625809072e855dfd5d8easharif self._files_set.update(current_files_set) 503b30395f032ff7aab615625809072e855dfd5d8easharif 513b30395f032ff7aab615625809072e855dfd5d8easharif def _GetSubset(self): 523b30395f032ff7aab615625809072e855dfd5d8easharif ret = [] 533b30395f032ff7aab615625809072e855dfd5d8easharif for i in range(self._chunk_size): 543b30395f032ff7aab615625809072e855dfd5d8easharif if not self._files_set: 553b30395f032ff7aab615625809072e855dfd5d8easharif break 563b30395f032ff7aab615625809072e855dfd5d8easharif ret.append(self._files_set.pop()) 573b30395f032ff7aab615625809072e855dfd5d8easharif return ret 583b30395f032ff7aab615625809072e855dfd5d8easharif 593b30395f032ff7aab615625809072e855dfd5d8easharif def _CopyFilesTree(self, input_dir, files, output_dir): 603b30395f032ff7aab615625809072e855dfd5d8easharif for f in files: 613b30395f032ff7aab615625809072e855dfd5d8easharif src_file = os.path.join(input_dir, f) 623b30395f032ff7aab615625809072e855dfd5d8easharif dst_file = os.path.join(output_dir, f) 633b30395f032ff7aab615625809072e855dfd5d8easharif if not os.path.isdir(os.path.dirname(dst_file)): 64f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano command = 'mkdir -p %s' % os.path.dirname(dst_file) 653b30395f032ff7aab615625809072e855dfd5d8easharif self._ce.RunCommand(command) 66f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano command = 'cp %s %s' % (src_file, dst_file) 673b30395f032ff7aab615625809072e855dfd5d8easharif self._ce.RunCommand(command) 683b30395f032ff7aab615625809072e855dfd5d8easharif 693b30395f032ff7aab615625809072e855dfd5d8easharif def _DoChunkMerge(self, current_files): 703b30395f032ff7aab615625809072e855dfd5d8easharif temp_dirs = [] 713b30395f032ff7aab615625809072e855dfd5d8easharif for i in self._inputs: 723b30395f032ff7aab615625809072e855dfd5d8easharif temp_dir = tempfile.mkdtemp() 733b30395f032ff7aab615625809072e855dfd5d8easharif temp_dirs.append(temp_dir) 743b30395f032ff7aab615625809072e855dfd5d8easharif self._CopyFilesTree(i, current_files, temp_dir) 753b30395f032ff7aab615625809072e855dfd5d8easharif # Now do the merge. 76f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano command = ('%s --inputs=%s --output=%s' % 77f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano (self._merge_program, ','.join(temp_dirs), self._output)) 78fd54f25d991c89e8f0297c9d62b0ce592825d956asharif if self._multipliers: 79f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano command = ('%s --multipliers=%s' % (command, self._multipliers)) 803b30395f032ff7aab615625809072e855dfd5d8easharif ret = self._ce.RunCommand(command) 81f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano assert ret == 0, '%s command failed!' % command 823b30395f032ff7aab615625809072e855dfd5d8easharif for temp_dir in temp_dirs: 83f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano command = 'rm -rf %s' % temp_dir 843b30395f032ff7aab615625809072e855dfd5d8easharif self._ce.RunCommand(command) 853b30395f032ff7aab615625809072e855dfd5d8easharif 863b30395f032ff7aab615625809072e855dfd5d8easharif def DoMerge(self): 873b30395f032ff7aab615625809072e855dfd5d8easharif self._PopulateFilesSet() 883b30395f032ff7aab615625809072e855dfd5d8easharif while True: 893b30395f032ff7aab615625809072e855dfd5d8easharif current_files = self._GetSubset() 903b30395f032ff7aab615625809072e855dfd5d8easharif if not current_files: 913b30395f032ff7aab615625809072e855dfd5d8easharif break 923b30395f032ff7aab615625809072e855dfd5d8easharif self._DoChunkMerge(current_files) 933b30395f032ff7aab615625809072e855dfd5d8easharif 943b30395f032ff7aab615625809072e855dfd5d8easharif 953b30395f032ff7aab615625809072e855dfd5d8easharifdef Main(argv): 963b30395f032ff7aab615625809072e855dfd5d8easharif """The main function.""" 973b30395f032ff7aab615625809072e855dfd5d8easharif # Common initializations 98f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano ### command_executer.InitCommandExecuter(True) 993b30395f032ff7aab615625809072e855dfd5d8easharif command_executer.InitCommandExecuter() 1003b30395f032ff7aab615625809072e855dfd5d8easharif l = logger.GetLogger() 1013b30395f032ff7aab615625809072e855dfd5d8easharif ce = command_executer.GetCommandExecuter() 1023b30395f032ff7aab615625809072e855dfd5d8easharif parser = optparse.OptionParser() 103f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano parser.add_option('--inputs', 104f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano dest='inputs', 105f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano help='Comma-separated input profile directories to merge.') 106f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano parser.add_option('--output', dest='output', help='Output profile directory.') 107f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano parser.add_option('--chunk_size', 108f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano dest='chunk_size', 109f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano default='50', 110f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano help='Chunk size to divide up the profiles into.') 111f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano parser.add_option('--merge_program', 112f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano dest='merge_program', 113f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano default='/home/xur/bin/profile_merge_v15.par', 114f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano help='Merge program to use to do the actual merge.') 115f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano parser.add_option('--multipliers', 116f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano dest='multipliers', 117f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano help='multipliers to use when merging. (optional)') 1183b30395f032ff7aab615625809072e855dfd5d8easharif 1193b30395f032ff7aab615625809072e855dfd5d8easharif options, _ = parser.parse_args(argv) 1203b30395f032ff7aab615625809072e855dfd5d8easharif 121f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano if not all([options.inputs, options.output]): 122f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano l.LogError('Must supply --inputs and --output') 123c9ebd090bb6fc28887ecdae772389944e8e633b0asharif return 1 124c9ebd090bb6fc28887ecdae772389944e8e633b0asharif 1253b30395f032ff7aab615625809072e855dfd5d8easharif try: 126f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano pm = ProfileMerger( 127f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano options.inputs.split(','), options.output, int(options.chunk_size), 128f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano options.merge_program, options.multipliers) 1293b30395f032ff7aab615625809072e855dfd5d8easharif pm.DoMerge() 1303b30395f032ff7aab615625809072e855dfd5d8easharif retval = 0 1313b30395f032ff7aab615625809072e855dfd5d8easharif except: 1323b30395f032ff7aab615625809072e855dfd5d8easharif retval = 1 1333b30395f032ff7aab615625809072e855dfd5d8easharif finally: 134f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozano print 'My work is done...' 1353b30395f032ff7aab615625809072e855dfd5d8easharif return retval 1363b30395f032ff7aab615625809072e855dfd5d8easharif 1373b30395f032ff7aab615625809072e855dfd5d8easharif 138f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbeLuis Lozanoif __name__ == '__main__': 1393b30395f032ff7aab615625809072e855dfd5d8easharif retval = Main(sys.argv) 1403b30395f032ff7aab615625809072e855dfd5d8easharif sys.exit(retval) 141