1# Copyright (c) 2014 Google Inc. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5""" 6This script is intended for use as a GYP_GENERATOR. It takes as input (by way of 7the generator flag config_path) the path of a json file that dictates the files 8and targets to search for. The following keys are supported: 9files: list of paths (relative) of the files to search for. 10targets: list of targets to search for. The target names are unqualified. 11 12The following is output: 13error: only supplied if there is an error. 14warning: only supplied if there is a warning. 15targets: the set of targets passed in via targets that either directly or 16 indirectly depend upon the set of paths supplied in files. 17build_targets: minimal set of targets that directly depend on the changed 18 files and need to be built. The expectation is this set of targets is passed 19 into a build step. 20status: outputs one of three values: none of the supplied files were found, 21 one of the include files changed so that it should be assumed everything 22 changed (in this case targets and build_targets are not output) or at 23 least one file was found. 24 25If the generator flag analyzer_output_path is specified, output is written 26there. Otherwise output is written to stdout. 27""" 28 29import gyp.common 30import gyp.ninja_syntax as ninja_syntax 31import json 32import os 33import posixpath 34import sys 35 36debug = False 37 38found_dependency_string = 'Found dependency' 39no_dependency_string = 'No dependencies' 40# Status when it should be assumed that everything has changed. 41all_changed_string = 'Found dependency (all)' 42 43# MatchStatus is used indicate if and how a target depends upon the supplied 44# sources. 45# The target's sources contain one of the supplied paths. 46MATCH_STATUS_MATCHES = 1 47# The target has a dependency on another target that contains one of the 48# supplied paths. 49MATCH_STATUS_MATCHES_BY_DEPENDENCY = 2 50# The target's sources weren't in the supplied paths and none of the target's 51# dependencies depend upon a target that matched. 52MATCH_STATUS_DOESNT_MATCH = 3 53# The target doesn't contain the source, but the dependent targets have not yet 54# been visited to determine a more specific status yet. 55MATCH_STATUS_TBD = 4 56 57generator_supports_multiple_toolsets = True 58 59generator_wants_static_library_dependencies_adjusted = False 60 61generator_default_variables = { 62} 63for dirname in ['INTERMEDIATE_DIR', 'SHARED_INTERMEDIATE_DIR', 'PRODUCT_DIR', 64 'LIB_DIR', 'SHARED_LIB_DIR']: 65 generator_default_variables[dirname] = '!!!' 66 67for unused in ['RULE_INPUT_PATH', 'RULE_INPUT_ROOT', 'RULE_INPUT_NAME', 68 'RULE_INPUT_DIRNAME', 'RULE_INPUT_EXT', 69 'EXECUTABLE_PREFIX', 'EXECUTABLE_SUFFIX', 70 'STATIC_LIB_PREFIX', 'STATIC_LIB_SUFFIX', 71 'SHARED_LIB_PREFIX', 'SHARED_LIB_SUFFIX', 72 'CONFIGURATION_NAME']: 73 generator_default_variables[unused] = '' 74 75 76def _ToGypPath(path): 77 """Converts a path to the format used by gyp.""" 78 if os.sep == '\\' and os.altsep == '/': 79 return path.replace('\\', '/') 80 return path 81 82 83def _ResolveParent(path, base_path_components): 84 """Resolves |path|, which starts with at least one '../'. Returns an empty 85 string if the path shouldn't be considered. See _AddSources() for a 86 description of |base_path_components|.""" 87 depth = 0 88 while path.startswith('../'): 89 depth += 1 90 path = path[3:] 91 # Relative includes may go outside the source tree. For example, an action may 92 # have inputs in /usr/include, which are not in the source tree. 93 if depth > len(base_path_components): 94 return '' 95 if depth == len(base_path_components): 96 return path 97 return '/'.join(base_path_components[0:len(base_path_components) - depth]) + \ 98 '/' + path 99 100 101def _AddSources(sources, base_path, base_path_components, result): 102 """Extracts valid sources from |sources| and adds them to |result|. Each 103 source file is relative to |base_path|, but may contain '..'. To make 104 resolving '..' easier |base_path_components| contains each of the 105 directories in |base_path|. Additionally each source may contain variables. 106 Such sources are ignored as it is assumed dependencies on them are expressed 107 and tracked in some other means.""" 108 # NOTE: gyp paths are always posix style. 109 for source in sources: 110 if not len(source) or source.startswith('!!!') or source.startswith('$'): 111 continue 112 # variable expansion may lead to //. 113 org_source = source 114 source = source[0] + source[1:].replace('//', '/') 115 if source.startswith('../'): 116 source = _ResolveParent(source, base_path_components) 117 if len(source): 118 result.append(source) 119 continue 120 result.append(base_path + source) 121 if debug: 122 print 'AddSource', org_source, result[len(result) - 1] 123 124 125def _ExtractSourcesFromAction(action, base_path, base_path_components, 126 results): 127 if 'inputs' in action: 128 _AddSources(action['inputs'], base_path, base_path_components, results) 129 130 131def _ToLocalPath(toplevel_dir, path): 132 """Converts |path| to a path relative to |toplevel_dir|.""" 133 if path == toplevel_dir: 134 return '' 135 if path.startswith(toplevel_dir + '/'): 136 return path[len(toplevel_dir) + len('/'):] 137 return path 138 139 140def _ExtractSources(target, target_dict, toplevel_dir): 141 # |target| is either absolute or relative and in the format of the OS. Gyp 142 # source paths are always posix. Convert |target| to a posix path relative to 143 # |toplevel_dir_|. This is done to make it easy to build source paths. 144 base_path = posixpath.dirname(_ToLocalPath(toplevel_dir, _ToGypPath(target))) 145 base_path_components = base_path.split('/') 146 147 # Add a trailing '/' so that _AddSources() can easily build paths. 148 if len(base_path): 149 base_path += '/' 150 151 if debug: 152 print 'ExtractSources', target, base_path 153 154 results = [] 155 if 'sources' in target_dict: 156 _AddSources(target_dict['sources'], base_path, base_path_components, 157 results) 158 # Include the inputs from any actions. Any changes to these affect the 159 # resulting output. 160 if 'actions' in target_dict: 161 for action in target_dict['actions']: 162 _ExtractSourcesFromAction(action, base_path, base_path_components, 163 results) 164 if 'rules' in target_dict: 165 for rule in target_dict['rules']: 166 _ExtractSourcesFromAction(rule, base_path, base_path_components, results) 167 168 return results 169 170 171class Target(object): 172 """Holds information about a particular target: 173 deps: set of Targets this Target depends upon. This is not recursive, only the 174 direct dependent Targets. 175 match_status: one of the MatchStatus values. 176 back_deps: set of Targets that have a dependency on this Target. 177 visited: used during iteration to indicate whether we've visited this target. 178 This is used for two iterations, once in building the set of Targets and 179 again in _GetBuildTargets(). 180 name: fully qualified name of the target. 181 requires_build: True if the target type is such that it needs to be built. 182 See _DoesTargetTypeRequireBuild for details. 183 added_to_compile_targets: used when determining if the target was added to the 184 set of targets that needs to be built. 185 in_roots: true if this target is a descendant of one of the root nodes. 186 is_executable: true if the type of target is executable.""" 187 def __init__(self, name): 188 self.deps = set() 189 self.match_status = MATCH_STATUS_TBD 190 self.back_deps = set() 191 self.name = name 192 # TODO(sky): I don't like hanging this off Target. This state is specific 193 # to certain functions and should be isolated there. 194 self.visited = False 195 self.requires_build = False 196 self.added_to_compile_targets = False 197 self.in_roots = False 198 self.is_executable = False 199 200 201class Config(object): 202 """Details what we're looking for 203 files: set of files to search for 204 targets: see file description for details.""" 205 def __init__(self): 206 self.files = [] 207 self.targets = set() 208 209 def Init(self, params): 210 """Initializes Config. This is a separate method as it raises an exception 211 if there is a parse error.""" 212 generator_flags = params.get('generator_flags', {}) 213 config_path = generator_flags.get('config_path', None) 214 if not config_path: 215 return 216 try: 217 f = open(config_path, 'r') 218 config = json.load(f) 219 f.close() 220 except IOError: 221 raise Exception('Unable to open file ' + config_path) 222 except ValueError as e: 223 raise Exception('Unable to parse config file ' + config_path + str(e)) 224 if not isinstance(config, dict): 225 raise Exception('config_path must be a JSON file containing a dictionary') 226 self.files = config.get('files', []) 227 self.targets = set(config.get('targets', [])) 228 229 230def _WasBuildFileModified(build_file, data, files, toplevel_dir): 231 """Returns true if the build file |build_file| is either in |files| or 232 one of the files included by |build_file| is in |files|. |toplevel_dir| is 233 the root of the source tree.""" 234 if _ToLocalPath(toplevel_dir, _ToGypPath(build_file)) in files: 235 if debug: 236 print 'gyp file modified', build_file 237 return True 238 239 # First element of included_files is the file itself. 240 if len(data[build_file]['included_files']) <= 1: 241 return False 242 243 for include_file in data[build_file]['included_files'][1:]: 244 # |included_files| are relative to the directory of the |build_file|. 245 rel_include_file = \ 246 _ToGypPath(gyp.common.UnrelativePath(include_file, build_file)) 247 if _ToLocalPath(toplevel_dir, rel_include_file) in files: 248 if debug: 249 print 'included gyp file modified, gyp_file=', build_file, \ 250 'included file=', rel_include_file 251 return True 252 return False 253 254 255def _GetOrCreateTargetByName(targets, target_name): 256 """Creates or returns the Target at targets[target_name]. If there is no 257 Target for |target_name| one is created. Returns a tuple of whether a new 258 Target was created and the Target.""" 259 if target_name in targets: 260 return False, targets[target_name] 261 target = Target(target_name) 262 targets[target_name] = target 263 return True, target 264 265 266def _DoesTargetTypeRequireBuild(target_dict): 267 """Returns true if the target type is such that it needs to be built.""" 268 # If a 'none' target has rules or actions we assume it requires a build. 269 return target_dict['type'] != 'none' or \ 270 target_dict.get('actions') or target_dict.get('rules') 271 272 273def _GenerateTargets(data, target_list, target_dicts, toplevel_dir, files, 274 build_files): 275 """Returns a tuple of the following: 276 . A dictionary mapping from fully qualified name to Target. 277 . A list of the targets that have a source file in |files|. 278 . Set of root Targets reachable from the the files |build_files|. 279 This sets the |match_status| of the targets that contain any of the source 280 files in |files| to MATCH_STATUS_MATCHES. 281 |toplevel_dir| is the root of the source tree.""" 282 # Maps from target name to Target. 283 targets = {} 284 285 # Targets that matched. 286 matching_targets = [] 287 288 # Queue of targets to visit. 289 targets_to_visit = target_list[:] 290 291 # Maps from build file to a boolean indicating whether the build file is in 292 # |files|. 293 build_file_in_files = {} 294 295 # Root targets across all files. 296 roots = set() 297 298 # Set of Targets in |build_files|. 299 build_file_targets = set() 300 301 while len(targets_to_visit) > 0: 302 target_name = targets_to_visit.pop() 303 created_target, target = _GetOrCreateTargetByName(targets, target_name) 304 if created_target: 305 roots.add(target) 306 elif target.visited: 307 continue 308 309 target.visited = True 310 target.requires_build = _DoesTargetTypeRequireBuild( 311 target_dicts[target_name]) 312 target.is_executable = target_dicts[target_name]['type'] == 'executable' 313 314 build_file = gyp.common.ParseQualifiedTarget(target_name)[0] 315 if not build_file in build_file_in_files: 316 build_file_in_files[build_file] = \ 317 _WasBuildFileModified(build_file, data, files, toplevel_dir) 318 319 if build_file in build_files: 320 build_file_targets.add(target) 321 322 # If a build file (or any of its included files) is modified we assume all 323 # targets in the file are modified. 324 if build_file_in_files[build_file]: 325 print 'matching target from modified build file', target_name 326 target.match_status = MATCH_STATUS_MATCHES 327 matching_targets.append(target) 328 else: 329 sources = _ExtractSources(target_name, target_dicts[target_name], 330 toplevel_dir) 331 for source in sources: 332 if source in files: 333 print 'target', target_name, 'matches', source 334 target.match_status = MATCH_STATUS_MATCHES 335 matching_targets.append(target) 336 break 337 338 # Add dependencies to visit as well as updating back pointers for deps. 339 for dep in target_dicts[target_name].get('dependencies', []): 340 targets_to_visit.append(dep) 341 342 created_dep_target, dep_target = _GetOrCreateTargetByName(targets, dep) 343 if not created_dep_target: 344 roots.discard(dep_target) 345 346 target.deps.add(dep_target) 347 dep_target.back_deps.add(target) 348 349 return targets, matching_targets, roots & build_file_targets 350 351 352def _GetUnqualifiedToTargetMapping(all_targets, to_find): 353 """Returns a mapping (dictionary) from unqualified name to Target for all the 354 Targets in |to_find|.""" 355 result = {} 356 if not to_find: 357 return result 358 to_find = set(to_find) 359 for target_name in all_targets.keys(): 360 extracted = gyp.common.ParseQualifiedTarget(target_name) 361 if len(extracted) > 1 and extracted[1] in to_find: 362 to_find.remove(extracted[1]) 363 result[extracted[1]] = all_targets[target_name] 364 if not to_find: 365 return result 366 return result 367 368 369def _DoesTargetDependOn(target): 370 """Returns true if |target| or any of its dependencies matches the supplied 371 set of paths. This updates |matches| of the Targets as it recurses. 372 target: the Target to look for.""" 373 if target.match_status == MATCH_STATUS_DOESNT_MATCH: 374 return False 375 if target.match_status == MATCH_STATUS_MATCHES or \ 376 target.match_status == MATCH_STATUS_MATCHES_BY_DEPENDENCY: 377 return True 378 for dep in target.deps: 379 if _DoesTargetDependOn(dep): 380 target.match_status = MATCH_STATUS_MATCHES_BY_DEPENDENCY 381 return True 382 target.match_status = MATCH_STATUS_DOESNT_MATCH 383 return False 384 385 386def _GetTargetsDependingOn(possible_targets): 387 """Returns the list of Targets in |possible_targets| that depend (either 388 directly on indirectly) on the matched targets. 389 possible_targets: targets to search from.""" 390 found = [] 391 for target in possible_targets: 392 if _DoesTargetDependOn(target): 393 found.append(target) 394 return found 395 396 397def _AddBuildTargets(target, roots, add_if_no_ancestor, result): 398 """Recurses through all targets that depend on |target|, adding all targets 399 that need to be built (and are in |roots|) to |result|. 400 roots: set of root targets. 401 add_if_no_ancestor: If true and there are no ancestors of |target| then add 402 |target| to |result|. |target| must still be in |roots|. 403 result: targets that need to be built are added here.""" 404 if target.visited: 405 return 406 407 target.visited = True 408 target.in_roots = not target.back_deps and target in roots 409 410 for back_dep_target in target.back_deps: 411 _AddBuildTargets(back_dep_target, roots, False, result) 412 target.added_to_compile_targets |= back_dep_target.added_to_compile_targets 413 target.in_roots |= back_dep_target.in_roots 414 415 # Always add 'executable' targets. Even though they may be built by other 416 # targets that depend upon them it makes detection of what is going to be 417 # built easier. 418 if target.in_roots and \ 419 (target.is_executable or 420 (not target.added_to_compile_targets and 421 (add_if_no_ancestor or target.requires_build))): 422 result.add(target) 423 target.added_to_compile_targets = True 424 425 426def _GetBuildTargets(matching_targets, roots): 427 """Returns the set of Targets that require a build. 428 matching_targets: targets that changed and need to be built. 429 roots: set of root targets in the build files to search from.""" 430 result = set() 431 for target in matching_targets: 432 _AddBuildTargets(target, roots, True, result) 433 return result 434 435 436def _WriteOutput(params, **values): 437 """Writes the output, either to stdout or a file is specified.""" 438 if 'error' in values: 439 print 'Error:', values['error'] 440 if 'status' in values: 441 print values['status'] 442 if 'targets' in values: 443 values['targets'].sort() 444 print 'Supplied targets that depend on changed files:' 445 for target in values['targets']: 446 print '\t', target 447 if 'build_targets' in values: 448 values['build_targets'].sort() 449 print 'Targets that require a build:' 450 for target in values['build_targets']: 451 print '\t', target 452 453 output_path = params.get('generator_flags', {}).get( 454 'analyzer_output_path', None) 455 if not output_path: 456 print json.dumps(values) 457 return 458 try: 459 f = open(output_path, 'w') 460 f.write(json.dumps(values) + '\n') 461 f.close() 462 except IOError as e: 463 print 'Error writing to output file', output_path, str(e) 464 465 466def _WasGypIncludeFileModified(params, files): 467 """Returns true if one of the files in |files| is in the set of included 468 files.""" 469 if params['options'].includes: 470 for include in params['options'].includes: 471 if _ToGypPath(include) in files: 472 print 'Include file modified, assuming all changed', include 473 return True 474 return False 475 476 477def _NamesNotIn(names, mapping): 478 """Returns a list of the values in |names| that are not in |mapping|.""" 479 return [name for name in names if name not in mapping] 480 481 482def _LookupTargets(names, mapping): 483 """Returns a list of the mapping[name] for each value in |names| that is in 484 |mapping|.""" 485 return [mapping[name] for name in names if name in mapping] 486 487 488def CalculateVariables(default_variables, params): 489 """Calculate additional variables for use in the build (called by gyp).""" 490 flavor = gyp.common.GetFlavor(params) 491 if flavor == 'mac': 492 default_variables.setdefault('OS', 'mac') 493 elif flavor == 'win': 494 default_variables.setdefault('OS', 'win') 495 # Copy additional generator configuration data from VS, which is shared 496 # by the Windows Ninja generator. 497 import gyp.generator.msvs as msvs_generator 498 generator_additional_non_configuration_keys = getattr(msvs_generator, 499 'generator_additional_non_configuration_keys', []) 500 generator_additional_path_sections = getattr(msvs_generator, 501 'generator_additional_path_sections', []) 502 503 gyp.msvs_emulation.CalculateCommonVariables(default_variables, params) 504 else: 505 operating_system = flavor 506 if flavor == 'android': 507 operating_system = 'linux' # Keep this legacy behavior for now. 508 default_variables.setdefault('OS', operating_system) 509 510 511def GenerateOutput(target_list, target_dicts, data, params): 512 """Called by gyp as the final stage. Outputs results.""" 513 config = Config() 514 try: 515 config.Init(params) 516 if not config.files: 517 raise Exception('Must specify files to analyze via config_path generator ' 518 'flag') 519 520 toplevel_dir = _ToGypPath(os.path.abspath(params['options'].toplevel_dir)) 521 if debug: 522 print 'toplevel_dir', toplevel_dir 523 524 if _WasGypIncludeFileModified(params, config.files): 525 result_dict = { 'status': all_changed_string, 526 'targets': list(config.targets) } 527 _WriteOutput(params, **result_dict) 528 return 529 530 all_targets, matching_targets, roots = _GenerateTargets( 531 data, target_list, target_dicts, toplevel_dir, frozenset(config.files), 532 params['build_files']) 533 534 warning = None 535 unqualified_mapping = _GetUnqualifiedToTargetMapping(all_targets, 536 config.targets) 537 if len(unqualified_mapping) != len(config.targets): 538 not_found = _NamesNotIn(config.targets, unqualified_mapping) 539 warning = 'Unable to find all targets: ' + str(not_found) 540 541 if matching_targets: 542 search_targets = _LookupTargets(config.targets, unqualified_mapping) 543 matched_search_targets = _GetTargetsDependingOn(search_targets) 544 # Reset the visited status for _GetBuildTargets. 545 for target in all_targets.itervalues(): 546 target.visited = False 547 build_targets = _GetBuildTargets(matching_targets, roots) 548 matched_search_targets = [gyp.common.ParseQualifiedTarget(target.name)[1] 549 for target in matched_search_targets] 550 build_targets = [gyp.common.ParseQualifiedTarget(target.name)[1] 551 for target in build_targets] 552 else: 553 matched_search_targets = [] 554 build_targets = [] 555 556 result_dict = { 'targets': matched_search_targets, 557 'status': found_dependency_string if matching_targets else 558 no_dependency_string, 559 'build_targets': build_targets} 560 if warning: 561 result_dict['warning'] = warning 562 _WriteOutput(params, **result_dict) 563 564 except Exception as e: 565 _WriteOutput(params, error=str(e)) 566