1#!/usr/bin/env python3 2# 3# Copyright 2012 The Chromium Authors 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Compile Android resources into an intermediate APK. 8 9This can also generate an R.txt, and an .srcjar file containing the proper 10final R.java class for all resource packages the APK depends on. 11 12This will crunch images with aapt2. 13""" 14 15import argparse 16import collections 17import contextlib 18import filecmp 19import hashlib 20import logging 21import os 22import pathlib 23import re 24import shutil 25import subprocess 26import sys 27import textwrap 28from xml.etree import ElementTree 29 30from util import build_utils 31from util import diff_utils 32from util import manifest_utils 33from util import parallel 34from util import protoresources 35from util import resource_utils 36import action_helpers # build_utils adds //build to sys.path. 37import zip_helpers 38 39 40# Pngs that we shouldn't convert to webp. Please add rationale when updating. 41_PNG_WEBP_EXCLUSION_PATTERN = re.compile('|'.join([ 42 # Android requires pngs for 9-patch images. 43 r'.*\.9\.png', 44 # Daydream requires pngs for icon files. 45 r'.*daydream_icon_.*\.png' 46])) 47 48 49def _ParseArgs(args): 50 """Parses command line options. 51 52 Returns: 53 An options object as from argparse.ArgumentParser.parse_args() 54 """ 55 parser = argparse.ArgumentParser(description=__doc__) 56 57 input_opts = parser.add_argument_group('Input options') 58 output_opts = parser.add_argument_group('Output options') 59 60 input_opts.add_argument('--include-resources', 61 action='append', 62 required=True, 63 help='Paths to arsc resource files used to link ' 64 'against. Can be specified multiple times.') 65 input_opts.add_argument( 66 '--dependencies-res-zips', 67 default=[], 68 help='Resources zip archives from dependents. Required to ' 69 'resolve @type/foo references into dependent libraries.') 70 input_opts.add_argument( 71 '--extra-res-packages', 72 help='Additional package names to generate R.java files for.') 73 input_opts.add_argument( 74 '--aapt2-path', required=True, help='Path to the Android aapt2 tool.') 75 input_opts.add_argument( 76 '--android-manifest', required=True, help='AndroidManifest.xml path.') 77 input_opts.add_argument( 78 '--r-java-root-package-name', 79 default='base', 80 help='Short package name for this target\'s root R java file (ex. ' 81 'input of "base" would become gen.base_module). Defaults to "base".') 82 group = input_opts.add_mutually_exclusive_group() 83 group.add_argument( 84 '--shared-resources', 85 action='store_true', 86 help='Make all resources in R.java non-final and allow the resource IDs ' 87 'to be reset to a different package index when the apk is loaded by ' 88 'another application at runtime.') 89 group.add_argument( 90 '--app-as-shared-lib', 91 action='store_true', 92 help='Same as --shared-resources, but also ensures all resource IDs are ' 93 'directly usable from the APK loaded as an application.') 94 input_opts.add_argument( 95 '--package-id', 96 type=int, 97 help='Decimal integer representing custom package ID for resources ' 98 '(instead of 127==0x7f). Cannot be used with --shared-resources.') 99 input_opts.add_argument( 100 '--package-name', 101 help='Package name that will be used to create R class.') 102 input_opts.add_argument( 103 '--rename-manifest-package', help='Package name to force AAPT to use.') 104 input_opts.add_argument( 105 '--arsc-package-name', 106 help='Package name to set in manifest of resources.arsc file. This is ' 107 'only used for apks under test.') 108 input_opts.add_argument( 109 '--shared-resources-allowlist', 110 help='An R.txt file acting as a allowlist for resources that should be ' 111 'non-final and have their package ID changed at runtime in R.java. ' 112 'Implies and overrides --shared-resources.') 113 input_opts.add_argument( 114 '--shared-resources-allowlist-locales', 115 default='[]', 116 help='Optional GN-list of locales. If provided, all strings corresponding' 117 ' to this locale list will be kept in the final output for the ' 118 'resources identified through --shared-resources-allowlist, even ' 119 'if --locale-allowlist is being used.') 120 input_opts.add_argument( 121 '--use-resource-ids-path', 122 help='Use resource IDs generated by aapt --emit-ids.') 123 input_opts.add_argument( 124 '--debuggable', 125 action='store_true', 126 help='Whether to add android:debuggable="true".') 127 input_opts.add_argument('--static-library-version', 128 help='Version code for static library.') 129 input_opts.add_argument('--version-code', help='Version code for apk.') 130 input_opts.add_argument('--version-name', help='Version name for apk.') 131 input_opts.add_argument( 132 '--min-sdk-version', required=True, help='android:minSdkVersion for APK.') 133 input_opts.add_argument( 134 '--target-sdk-version', 135 required=True, 136 help="android:targetSdkVersion for APK.") 137 input_opts.add_argument( 138 '--max-sdk-version', 139 help="android:maxSdkVersion expected in AndroidManifest.xml.") 140 input_opts.add_argument( 141 '--manifest-package', help='Package name of the AndroidManifest.xml.') 142 input_opts.add_argument( 143 '--locale-allowlist', 144 default='[]', 145 help='GN list of languages to include. All other language configs will ' 146 'be stripped out. List may include a combination of Android locales ' 147 'or Chrome locales.') 148 input_opts.add_argument( 149 '--resource-exclusion-regex', 150 default='', 151 help='File-based filter for resources (applied before compiling)') 152 input_opts.add_argument( 153 '--resource-exclusion-exceptions', 154 default='[]', 155 help='GN list of globs that say which files to include even ' 156 'when --resource-exclusion-regex is set.') 157 input_opts.add_argument( 158 '--dependencies-res-zip-overlays', 159 help='GN list with subset of --dependencies-res-zips to use overlay ' 160 'semantics for.') 161 input_opts.add_argument( 162 '--values-filter-rules', 163 help='GN list of source_glob:regex for filtering resources after they ' 164 'are compiled. Use this to filter out entries within values/ files.') 165 input_opts.add_argument('--png-to-webp', action='store_true', 166 help='Convert png files to webp format.') 167 168 input_opts.add_argument('--webp-binary', default='', 169 help='Path to the cwebp binary.') 170 input_opts.add_argument( 171 '--webp-cache-dir', help='The directory to store webp image cache.') 172 input_opts.add_argument( 173 '--is-bundle-module', 174 action='store_true', 175 help='Whether resources are being generated for a bundle module.') 176 input_opts.add_argument( 177 '--uses-split', 178 help='Value to set uses-split to in the AndroidManifest.xml.') 179 input_opts.add_argument( 180 '--verification-version-code-offset', 181 help='Subtract this from versionCode for expectation files') 182 input_opts.add_argument( 183 '--verification-library-version-offset', 184 help='Subtract this from static-library version for expectation files') 185 input_opts.add_argument('--xml-namespaces', 186 action='store_true', 187 help='Do not pass --no-xml-namespaces') 188 189 action_helpers.add_depfile_arg(output_opts) 190 output_opts.add_argument('--arsc-path', help='Apk output for arsc format.') 191 output_opts.add_argument('--proto-path', help='Apk output for proto format.') 192 output_opts.add_argument( 193 '--info-path', help='Path to output info file for the partial apk.') 194 output_opts.add_argument( 195 '--srcjar-out', 196 help='Path to srcjar to contain generated R.java.') 197 output_opts.add_argument('--r-text-out', 198 help='Path to store the generated R.txt file.') 199 output_opts.add_argument( 200 '--proguard-file', help='Path to proguard.txt generated file.') 201 output_opts.add_argument( 202 '--emit-ids-out', help='Path to file produced by aapt2 --emit-ids.') 203 204 diff_utils.AddCommandLineFlags(parser) 205 options = parser.parse_args(args) 206 207 options.include_resources = action_helpers.parse_gn_list( 208 options.include_resources) 209 options.dependencies_res_zips = action_helpers.parse_gn_list( 210 options.dependencies_res_zips) 211 options.extra_res_packages = action_helpers.parse_gn_list( 212 options.extra_res_packages) 213 options.locale_allowlist = action_helpers.parse_gn_list( 214 options.locale_allowlist) 215 options.shared_resources_allowlist_locales = action_helpers.parse_gn_list( 216 options.shared_resources_allowlist_locales) 217 options.resource_exclusion_exceptions = action_helpers.parse_gn_list( 218 options.resource_exclusion_exceptions) 219 options.dependencies_res_zip_overlays = action_helpers.parse_gn_list( 220 options.dependencies_res_zip_overlays) 221 options.values_filter_rules = action_helpers.parse_gn_list( 222 options.values_filter_rules) 223 224 if not options.arsc_path and not options.proto_path: 225 parser.error('One of --arsc-path or --proto-path is required.') 226 227 if options.package_id and options.shared_resources: 228 parser.error('--package-id and --shared-resources are mutually exclusive') 229 230 if options.static_library_version and (options.static_library_version != 231 options.version_code): 232 assert options.static_library_version == options.version_code, ( 233 f'static_library_version={options.static_library_version} must equal ' 234 f'version_code={options.version_code}. Please verify the version code ' 235 'map for this target is defined correctly.') 236 237 return options 238 239 240def _IterFiles(root_dir): 241 for root, _, files in os.walk(root_dir): 242 for f in files: 243 yield os.path.join(root, f) 244 245 246def _RenameLocaleResourceDirs(resource_dirs, path_info): 247 """Rename locale resource directories into standard names when necessary. 248 249 This is necessary to deal with the fact that older Android releases only 250 support ISO 639-1 two-letter codes, and sometimes even obsolete versions 251 of them. 252 253 In practice it means: 254 * 3-letter ISO 639-2 qualifiers are renamed under a corresponding 255 2-letter one. E.g. for Filipino, strings under values-fil/ will be moved 256 to a new corresponding values-tl/ sub-directory. 257 258 * Modern ISO 639-1 codes will be renamed to their obsolete variant 259 for Indonesian, Hebrew and Yiddish (e.g. 'values-in/ -> values-id/). 260 261 * Norwegian macrolanguage strings will be renamed to Bokmal (main 262 Norway language). See http://crbug.com/920960. In practice this 263 means that 'values-no/ -> values-nb/' unless 'values-nb/' already 264 exists. 265 266 * BCP 47 langauge tags will be renamed to an equivalent ISO 639-1 267 locale qualifier if possible (e.g. 'values-b+en+US/ -> values-en-rUS'). 268 269 Args: 270 resource_dirs: list of top-level resource directories. 271 """ 272 for resource_dir in resource_dirs: 273 ignore_dirs = {} 274 for path in _IterFiles(resource_dir): 275 locale = resource_utils.FindLocaleInStringResourceFilePath(path) 276 if not locale: 277 continue 278 cr_locale = resource_utils.ToChromiumLocaleName(locale) 279 if not cr_locale: 280 continue # Unsupported Android locale qualifier!? 281 locale2 = resource_utils.ToAndroidLocaleName(cr_locale) 282 if locale != locale2: 283 path2 = path.replace('/values-%s/' % locale, '/values-%s/' % locale2) 284 if path == path2: 285 raise Exception('Could not substitute locale %s for %s in %s' % 286 (locale, locale2, path)) 287 288 # Ignore rather than rename when the destination resources config 289 # already exists. 290 # e.g. some libraries provide both values-nb/ and values-no/. 291 # e.g. material design provides: 292 # * res/values-rUS/values-rUS.xml 293 # * res/values-b+es+419/values-b+es+419.xml 294 config_dir = os.path.dirname(path2) 295 already_has_renamed_config = ignore_dirs.get(config_dir) 296 if already_has_renamed_config is None: 297 # Cache the result of the first time the directory is encountered 298 # since subsequent encounters will find the directory already exists 299 # (due to the rename). 300 already_has_renamed_config = os.path.exists(config_dir) 301 ignore_dirs[config_dir] = already_has_renamed_config 302 if already_has_renamed_config: 303 continue 304 305 build_utils.MakeDirectory(os.path.dirname(path2)) 306 shutil.move(path, path2) 307 path_info.RegisterRename( 308 os.path.relpath(path, resource_dir), 309 os.path.relpath(path2, resource_dir)) 310 311 312def _ToAndroidLocales(locale_allowlist): 313 """Converts the list of Chrome locales to Android config locale qualifiers. 314 315 Args: 316 locale_allowlist: A list of Chromium locale names. 317 Returns: 318 A set of matching Android config locale qualifier names. 319 """ 320 ret = set() 321 for locale in locale_allowlist: 322 locale = resource_utils.ToAndroidLocaleName(locale) 323 if locale is None or ('-' in locale and '-r' not in locale): 324 raise Exception('Unsupported Chromium locale name: %s' % locale) 325 ret.add(locale) 326 # Always keep non-regional fall-backs. 327 language = locale.split('-')[0] 328 ret.add(language) 329 330 return ret 331 332 333def _MoveImagesToNonMdpiFolders(res_root, path_info): 334 """Move images from drawable-*-mdpi-* folders to drawable-* folders. 335 336 Why? http://crbug.com/289843 337 """ 338 for src_dir_name in os.listdir(res_root): 339 src_components = src_dir_name.split('-') 340 if src_components[0] != 'drawable' or 'mdpi' not in src_components: 341 continue 342 src_dir = os.path.join(res_root, src_dir_name) 343 if not os.path.isdir(src_dir): 344 continue 345 dst_components = [c for c in src_components if c != 'mdpi'] 346 assert dst_components != src_components 347 dst_dir_name = '-'.join(dst_components) 348 dst_dir = os.path.join(res_root, dst_dir_name) 349 build_utils.MakeDirectory(dst_dir) 350 for src_file_name in os.listdir(src_dir): 351 src_file = os.path.join(src_dir, src_file_name) 352 dst_file = os.path.join(dst_dir, src_file_name) 353 assert not os.path.lexists(dst_file) 354 shutil.move(src_file, dst_file) 355 path_info.RegisterRename( 356 os.path.relpath(src_file, res_root), 357 os.path.relpath(dst_file, res_root)) 358 359 360def _DeterminePlatformVersion(aapt2_path, jar_candidates): 361 def maybe_extract_version(j): 362 try: 363 return resource_utils.ExtractBinaryManifestValues(aapt2_path, j) 364 except build_utils.CalledProcessError: 365 return None 366 367 def is_sdk_jar(jar_name): 368 if jar_name in ('android.jar', 'android_system.jar'): 369 return True 370 # Robolectric jar looks a bit different. 371 return 'android-all' in jar_name and 'robolectric' in jar_name 372 373 android_sdk_jars = [ 374 j for j in jar_candidates if is_sdk_jar(os.path.basename(j)) 375 ] 376 extract_all = [maybe_extract_version(j) for j in android_sdk_jars] 377 extract_all = [x for x in extract_all if x] 378 if len(extract_all) == 0: 379 raise Exception( 380 'Unable to find android SDK jar among candidates: %s' 381 % ', '.join(android_sdk_jars)) 382 if len(extract_all) > 1: 383 raise Exception( 384 'Found multiple android SDK jars among candidates: %s' 385 % ', '.join(android_sdk_jars)) 386 platform_version_code, platform_version_name = extract_all.pop()[:2] 387 return platform_version_code, platform_version_name 388 389 390def _FixManifest(options, temp_dir): 391 """Fix the APK's AndroidManifest.xml. 392 393 This adds any missing namespaces for 'android' and 'tools', and 394 sets certains elements like 'platformBuildVersionCode' or 395 'android:debuggable' depending on the content of |options|. 396 397 Args: 398 options: The command-line arguments tuple. 399 temp_dir: A temporary directory where the fixed manifest will be written to. 400 Returns: 401 Tuple of: 402 * Manifest path within |temp_dir|. 403 * Original package_name. 404 * Manifest package name. 405 """ 406 doc, manifest_node, app_node = manifest_utils.ParseManifest( 407 options.android_manifest) 408 409 # merge_manifest.py also sets package & <uses-sdk>. We may want to ensure 410 # manifest merger is always enabled and remove these command-line arguments. 411 manifest_utils.SetUsesSdk(manifest_node, options.target_sdk_version, 412 options.min_sdk_version, options.max_sdk_version) 413 orig_package = manifest_node.get('package') or options.manifest_package 414 fixed_package = (options.arsc_package_name or options.manifest_package 415 or orig_package) 416 manifest_node.set('package', fixed_package) 417 418 platform_version_code, platform_version_name = _DeterminePlatformVersion( 419 options.aapt2_path, options.include_resources) 420 manifest_node.set('platformBuildVersionCode', platform_version_code) 421 manifest_node.set('platformBuildVersionName', platform_version_name) 422 if options.version_code: 423 manifest_utils.NamespacedSet(manifest_node, 'versionCode', 424 options.version_code) 425 if options.version_name: 426 manifest_utils.NamespacedSet(manifest_node, 'versionName', 427 options.version_name) 428 if options.debuggable: 429 manifest_utils.NamespacedSet(app_node, 'debuggable', 'true') 430 431 if options.uses_split: 432 uses_split = ElementTree.SubElement(manifest_node, 'uses-split') 433 manifest_utils.NamespacedSet(uses_split, 'name', options.uses_split) 434 435 # Make sure the min-sdk condition is not less than the min-sdk of the bundle. 436 for min_sdk_node in manifest_node.iter('{%s}min-sdk' % 437 manifest_utils.DIST_NAMESPACE): 438 dist_value = '{%s}value' % manifest_utils.DIST_NAMESPACE 439 if int(min_sdk_node.get(dist_value)) < int(options.min_sdk_version): 440 min_sdk_node.set(dist_value, options.min_sdk_version) 441 442 debug_manifest_path = os.path.join(temp_dir, 'AndroidManifest.xml') 443 manifest_utils.SaveManifest(doc, debug_manifest_path) 444 return debug_manifest_path, orig_package, fixed_package 445 446 447def _CreateKeepPredicate(resource_exclusion_regex, 448 resource_exclusion_exceptions): 449 """Return a predicate lambda to determine which resource files to keep. 450 451 Args: 452 resource_exclusion_regex: A regular expression describing all resources 453 to exclude, except if they are mip-maps, or if they are listed 454 in |resource_exclusion_exceptions|. 455 resource_exclusion_exceptions: A list of glob patterns corresponding 456 to exceptions to the |resource_exclusion_regex|. 457 Returns: 458 A lambda that takes a path, and returns true if the corresponding file 459 must be kept. 460 """ 461 predicate = lambda path: os.path.basename(path)[0] != '.' 462 if resource_exclusion_regex == '': 463 # Do not extract dotfiles (e.g. ".gitkeep"). aapt ignores them anyways. 464 return predicate 465 466 # A simple predicate that only removes (returns False for) paths covered by 467 # the exclusion regex or listed as exceptions. 468 return lambda path: ( 469 not re.search(resource_exclusion_regex, path) or 470 build_utils.MatchesGlob(path, resource_exclusion_exceptions)) 471 472 473def _ComputeSha1(path): 474 with open(path, 'rb') as f: 475 data = f.read() 476 return hashlib.sha1(data).hexdigest() 477 478 479def _ConvertToWebPSingle(png_path, cwebp_binary, cwebp_version, webp_cache_dir): 480 sha1_hash = _ComputeSha1(png_path) 481 482 # The set of arguments that will appear in the cache key. 483 quality_args = ['-m', '6', '-q', '100', '-lossless'] 484 485 webp_cache_path = os.path.join( 486 webp_cache_dir, '{}-{}-{}'.format(sha1_hash, cwebp_version, 487 ''.join(quality_args))) 488 # No need to add .webp. Android can load images fine without them. 489 webp_path = os.path.splitext(png_path)[0] 490 491 cache_hit = os.path.exists(webp_cache_path) 492 if cache_hit: 493 os.link(webp_cache_path, webp_path) 494 else: 495 # We place the generated webp image to webp_path, instead of in the 496 # webp_cache_dir to avoid concurrency issues. 497 args = [cwebp_binary, png_path, '-o', webp_path, '-quiet'] + quality_args 498 subprocess.check_call(args) 499 500 try: 501 os.link(webp_path, webp_cache_path) 502 except OSError: 503 # Because of concurrent run, a webp image may already exists in 504 # webp_cache_path. 505 pass 506 507 os.remove(png_path) 508 original_dir = os.path.dirname(os.path.dirname(png_path)) 509 rename_tuple = (os.path.relpath(png_path, original_dir), 510 os.path.relpath(webp_path, original_dir)) 511 return rename_tuple, cache_hit 512 513 514def _ConvertToWebP(cwebp_binary, png_paths, path_info, webp_cache_dir): 515 cwebp_version = subprocess.check_output([cwebp_binary, '-version']).rstrip() 516 shard_args = [(f, ) for f in png_paths 517 if not _PNG_WEBP_EXCLUSION_PATTERN.match(f)] 518 519 build_utils.MakeDirectory(webp_cache_dir) 520 results = parallel.BulkForkAndCall(_ConvertToWebPSingle, 521 shard_args, 522 cwebp_binary=cwebp_binary, 523 cwebp_version=cwebp_version, 524 webp_cache_dir=webp_cache_dir) 525 total_cache_hits = 0 526 for rename_tuple, cache_hit in results: 527 path_info.RegisterRename(*rename_tuple) 528 total_cache_hits += int(cache_hit) 529 530 logging.debug('png->webp cache: %d/%d', total_cache_hits, len(shard_args)) 531 532 533def _RemoveImageExtensions(directory, path_info): 534 """Remove extensions from image files in the passed directory. 535 536 This reduces binary size but does not affect android's ability to load the 537 images. 538 """ 539 for f in _IterFiles(directory): 540 if (f.endswith('.png') or f.endswith('.webp')) and not f.endswith('.9.png'): 541 path_with_extension = f 542 path_no_extension = os.path.splitext(path_with_extension)[0] 543 if path_no_extension != path_with_extension: 544 shutil.move(path_with_extension, path_no_extension) 545 path_info.RegisterRename( 546 os.path.relpath(path_with_extension, directory), 547 os.path.relpath(path_no_extension, directory)) 548 549 550def _CompileSingleDep(index, dep_subdir, keep_predicate, aapt2_path, 551 partials_dir): 552 unique_name = '{}_{}'.format(index, os.path.basename(dep_subdir)) 553 partial_path = os.path.join(partials_dir, '{}.zip'.format(unique_name)) 554 555 compile_command = [ 556 aapt2_path, 557 'compile', 558 # TODO(wnwen): Turn this on once aapt2 forces 9-patch to be crunched. 559 # '--no-crunch', 560 '--dir', 561 dep_subdir, 562 '-o', 563 partial_path 564 ] 565 566 # There are resources targeting API-versions lower than our minapi. For 567 # various reasons it's easier to let aapt2 ignore these than for us to 568 # remove them from our build (e.g. it's from a 3rd party library). 569 build_utils.CheckOutput( 570 compile_command, 571 stderr_filter=lambda output: build_utils.FilterLines( 572 output, r'ignoring configuration .* for (styleable|attribute)')) 573 574 # Filtering these files is expensive, so only apply filters to the partials 575 # that have been explicitly targeted. 576 if keep_predicate: 577 logging.debug('Applying .arsc filtering to %s', dep_subdir) 578 protoresources.StripUnwantedResources(partial_path, keep_predicate) 579 return partial_path 580 581 582def _CreateValuesKeepPredicate(exclusion_rules, dep_subdir): 583 patterns = [ 584 x[1] for x in exclusion_rules 585 if build_utils.MatchesGlob(dep_subdir, [x[0]]) 586 ] 587 if not patterns: 588 return None 589 590 regexes = [re.compile(p) for p in patterns] 591 return lambda x: not any(r.search(x) for r in regexes) 592 593 594def _CompileDeps(aapt2_path, dep_subdirs, dep_subdir_overlay_set, temp_dir, 595 exclusion_rules): 596 partials_dir = os.path.join(temp_dir, 'partials') 597 build_utils.MakeDirectory(partials_dir) 598 599 job_params = [(i, dep_subdir, 600 _CreateValuesKeepPredicate(exclusion_rules, dep_subdir)) 601 for i, dep_subdir in enumerate(dep_subdirs)] 602 603 # Filtering is slow, so ensure jobs with keep_predicate are started first. 604 job_params.sort(key=lambda x: not x[2]) 605 partials = list( 606 parallel.BulkForkAndCall(_CompileSingleDep, 607 job_params, 608 aapt2_path=aapt2_path, 609 partials_dir=partials_dir)) 610 611 partials_cmd = list() 612 for i, partial in enumerate(partials): 613 dep_subdir = job_params[i][1] 614 if dep_subdir in dep_subdir_overlay_set: 615 partials_cmd += ['-R'] 616 partials_cmd += [partial] 617 return partials_cmd 618 619 620def _CreateResourceInfoFile(path_info, info_path, dependencies_res_zips): 621 for zip_file in dependencies_res_zips: 622 zip_info_file_path = zip_file + '.info' 623 if os.path.exists(zip_info_file_path): 624 path_info.MergeInfoFile(zip_info_file_path) 625 path_info.Write(info_path) 626 627 628def _RemoveUnwantedLocalizedStrings(dep_subdirs, options): 629 """Remove localized strings that should not go into the final output. 630 631 Args: 632 dep_subdirs: List of resource dependency directories. 633 options: Command-line options namespace. 634 """ 635 # Collect locale and file paths from the existing subdirs. 636 # The following variable maps Android locale names to 637 # sets of corresponding xml file paths. 638 locale_to_files_map = collections.defaultdict(set) 639 for directory in dep_subdirs: 640 for f in _IterFiles(directory): 641 locale = resource_utils.FindLocaleInStringResourceFilePath(f) 642 if locale: 643 locale_to_files_map[locale].add(f) 644 645 all_locales = set(locale_to_files_map) 646 647 # Set A: wanted locales, either all of them or the 648 # list provided by --locale-allowlist. 649 wanted_locales = all_locales 650 if options.locale_allowlist: 651 wanted_locales = _ToAndroidLocales(options.locale_allowlist) 652 653 # Set B: shared resources locales, which is either set A 654 # or the list provided by --shared-resources-allowlist-locales 655 shared_resources_locales = wanted_locales 656 shared_names_allowlist = set() 657 if options.shared_resources_allowlist_locales: 658 shared_names_allowlist = set( 659 resource_utils.GetRTxtStringResourceNames( 660 options.shared_resources_allowlist)) 661 662 shared_resources_locales = _ToAndroidLocales( 663 options.shared_resources_allowlist_locales) 664 665 # Remove any file that belongs to a locale not covered by 666 # either A or B. 667 removable_locales = (all_locales - wanted_locales - shared_resources_locales) 668 for locale in removable_locales: 669 for path in locale_to_files_map[locale]: 670 os.remove(path) 671 672 # For any locale in B but not in A, only keep the shared 673 # resource strings in each file. 674 for locale in shared_resources_locales - wanted_locales: 675 for path in locale_to_files_map[locale]: 676 resource_utils.FilterAndroidResourceStringsXml( 677 path, lambda x: x in shared_names_allowlist) 678 679 # For any locale in A but not in B, only keep the strings 680 # that are _not_ from shared resources in the file. 681 for locale in wanted_locales - shared_resources_locales: 682 for path in locale_to_files_map[locale]: 683 resource_utils.FilterAndroidResourceStringsXml( 684 path, lambda x: x not in shared_names_allowlist) 685 686 687def _FilterResourceFiles(dep_subdirs, keep_predicate): 688 # Create a function that selects which resource files should be packaged 689 # into the final output. Any file that does not pass the predicate will 690 # be removed below. 691 png_paths = [] 692 for directory in dep_subdirs: 693 for f in _IterFiles(directory): 694 if not keep_predicate(f): 695 os.remove(f) 696 elif f.endswith('.png'): 697 png_paths.append(f) 698 699 return png_paths 700 701 702def _PackageApk(options, build): 703 """Compile and link resources with aapt2. 704 705 Args: 706 options: The command-line options. 707 build: BuildContext object. 708 Returns: 709 The manifest package name for the APK. 710 """ 711 logging.debug('Extracting resource .zips') 712 dep_subdirs = [] 713 dep_subdir_overlay_set = set() 714 for dependency_res_zip in options.dependencies_res_zips: 715 extracted_dep_subdirs = resource_utils.ExtractDeps([dependency_res_zip], 716 build.deps_dir) 717 dep_subdirs += extracted_dep_subdirs 718 if dependency_res_zip in options.dependencies_res_zip_overlays: 719 dep_subdir_overlay_set.update(extracted_dep_subdirs) 720 721 logging.debug('Applying locale transformations') 722 path_info = resource_utils.ResourceInfoFile() 723 _RenameLocaleResourceDirs(dep_subdirs, path_info) 724 725 logging.debug('Applying file-based exclusions') 726 keep_predicate = _CreateKeepPredicate(options.resource_exclusion_regex, 727 options.resource_exclusion_exceptions) 728 png_paths = _FilterResourceFiles(dep_subdirs, keep_predicate) 729 730 if options.locale_allowlist or options.shared_resources_allowlist_locales: 731 logging.debug('Applying locale-based string exclusions') 732 _RemoveUnwantedLocalizedStrings(dep_subdirs, options) 733 734 if png_paths and options.png_to_webp: 735 logging.debug('Converting png->webp') 736 _ConvertToWebP(options.webp_binary, png_paths, path_info, 737 options.webp_cache_dir) 738 logging.debug('Applying drawable transformations') 739 for directory in dep_subdirs: 740 _MoveImagesToNonMdpiFolders(directory, path_info) 741 _RemoveImageExtensions(directory, path_info) 742 743 logging.debug('Running aapt2 compile') 744 exclusion_rules = [x.split(':', 1) for x in options.values_filter_rules] 745 partials = _CompileDeps(options.aapt2_path, dep_subdirs, 746 dep_subdir_overlay_set, build.temp_dir, 747 exclusion_rules) 748 749 link_command = [ 750 options.aapt2_path, 751 'link', 752 '--auto-add-overlay', 753 '--no-version-vectors', 754 '--output-text-symbols', 755 build.r_txt_path, 756 ] 757 758 for j in options.include_resources: 759 link_command += ['-I', j] 760 if options.proguard_file: 761 link_command += ['--proguard', build.proguard_path] 762 link_command += ['--proguard-minimal-keep-rules'] 763 if options.emit_ids_out: 764 link_command += ['--emit-ids', build.emit_ids_path] 765 766 # Note: only one of --proto-format, --shared-lib or --app-as-shared-lib 767 # can be used with recent versions of aapt2. 768 if options.shared_resources: 769 link_command.append('--shared-lib') 770 771 if int(options.min_sdk_version) > 21 and not options.xml_namespaces: 772 link_command.append('--no-xml-namespaces') 773 774 if options.package_id: 775 link_command += [ 776 '--package-id', 777 '0x%02x' % options.package_id, 778 '--allow-reserved-package-id', 779 ] 780 781 fixed_manifest, desired_manifest_package_name, fixed_manifest_package = ( 782 _FixManifest(options, build.temp_dir)) 783 if options.rename_manifest_package: 784 desired_manifest_package_name = options.rename_manifest_package 785 786 link_command += [ 787 '--manifest', fixed_manifest, '--rename-manifest-package', 788 desired_manifest_package_name 789 ] 790 791 if options.package_id is not None: 792 package_id = options.package_id 793 elif options.shared_resources: 794 package_id = 0 795 else: 796 package_id = 0x7f 797 _CreateStableIdsFile(options.use_resource_ids_path, build.stable_ids_path, 798 fixed_manifest_package, package_id) 799 link_command += ['--stable-ids', build.stable_ids_path] 800 801 link_command += partials 802 803 # We always create a binary arsc file first, then convert to proto, so flags 804 # such as --shared-lib can be supported. 805 link_command += ['-o', build.arsc_path] 806 807 logging.debug('Starting: aapt2 link') 808 link_proc = subprocess.Popen(link_command) 809 810 # Create .res.info file in parallel. 811 if options.info_path: 812 logging.debug('Creating .res.info file') 813 _CreateResourceInfoFile(path_info, build.info_path, 814 options.dependencies_res_zips) 815 816 exit_code = link_proc.wait() 817 assert exit_code == 0, f'aapt2 link cmd failed with {exit_code=}' 818 logging.debug('Finished: aapt2 link') 819 820 if options.shared_resources: 821 logging.debug('Resolving styleables in R.txt') 822 # Need to resolve references because unused resource removal tool does not 823 # support references in R.txt files. 824 resource_utils.ResolveStyleableReferences(build.r_txt_path) 825 826 if exit_code: 827 raise subprocess.CalledProcessError(exit_code, link_command) 828 829 if options.proguard_file and (options.shared_resources 830 or options.app_as_shared_lib): 831 # Make sure the R class associated with the manifest package does not have 832 # its onResourcesLoaded method obfuscated or removed, so that the framework 833 # can call it in the case where the APK is being loaded as a library. 834 with open(build.proguard_path, 'a') as proguard_file: 835 keep_rule = ''' 836 -keep,allowoptimization class {package}.R {{ 837 public static void onResourcesLoaded(int); 838 }} 839 '''.format(package=desired_manifest_package_name) 840 proguard_file.write(textwrap.dedent(keep_rule)) 841 842 logging.debug('Running aapt2 convert') 843 build_utils.CheckOutput([ 844 options.aapt2_path, 'convert', '--output-format', 'proto', '-o', 845 build.proto_path, build.arsc_path 846 ]) 847 848 # Workaround for b/147674078. This is only needed for WebLayer and does not 849 # affect WebView usage, since WebView does not used dynamic attributes. 850 if options.shared_resources: 851 logging.debug('Hardcoding dynamic attributes') 852 protoresources.HardcodeSharedLibraryDynamicAttributes( 853 build.proto_path, options.is_bundle_module, 854 options.shared_resources_allowlist) 855 856 build_utils.CheckOutput([ 857 options.aapt2_path, 'convert', '--output-format', 'binary', '-o', 858 build.arsc_path, build.proto_path 859 ]) 860 861 # Sanity check that the created resources have the expected package ID. 862 logging.debug('Performing sanity check') 863 _, actual_package_id = resource_utils.ExtractArscPackage( 864 options.aapt2_path, 865 build.arsc_path if options.arsc_path else build.proto_path) 866 # When there are no resources, ExtractArscPackage returns (None, None), in 867 # this case there is no need to check for matching package ID. 868 if actual_package_id is not None and actual_package_id != package_id: 869 raise Exception('Invalid package ID 0x%x (expected 0x%x)' % 870 (actual_package_id, package_id)) 871 872 return desired_manifest_package_name 873 874 875def _CreateStableIdsFile(in_path, out_path, package_name, package_id): 876 """Transforms a file generated by --emit-ids from another package. 877 878 --stable-ids is generally meant to be used by different versions of the same 879 package. To make it work for other packages, we need to transform the package 880 name references to match the package that resources are being generated for. 881 """ 882 if in_path: 883 data = pathlib.Path(in_path).read_text() 884 else: 885 # Force IDs to use 0x01 for the type byte in order to ensure they are 886 # different from IDs generated by other apps. https://crbug.com/1293336 887 data = 'pkg:id/fake_resource_id = 0x7f010000\n' 888 # Replace "pkg:" with correct package name. 889 data = re.sub(r'^.*?:', package_name + ':', data, flags=re.MULTILINE) 890 # Replace "0x7f" with correct package id. 891 data = re.sub(r'0x..', '0x%02x' % package_id, data) 892 pathlib.Path(out_path).write_text(data) 893 894 895def _WriteOutputs(options, build): 896 possible_outputs = [ 897 (options.srcjar_out, build.srcjar_path), 898 (options.r_text_out, build.r_txt_path), 899 (options.arsc_path, build.arsc_path), 900 (options.proto_path, build.proto_path), 901 (options.proguard_file, build.proguard_path), 902 (options.emit_ids_out, build.emit_ids_path), 903 (options.info_path, build.info_path), 904 ] 905 906 for final, temp in possible_outputs: 907 # Write file only if it's changed. 908 if final and not (os.path.exists(final) and filecmp.cmp(final, temp)): 909 shutil.move(temp, final) 910 911 912def _CreateNormalizedManifestForVerification(options): 913 with build_utils.TempDir() as tempdir: 914 fixed_manifest, _, _ = _FixManifest(options, tempdir) 915 with open(fixed_manifest) as f: 916 return manifest_utils.NormalizeManifest( 917 f.read(), options.verification_version_code_offset, 918 options.verification_library_version_offset) 919 920 921def main(args): 922 build_utils.InitLogging('RESOURCE_DEBUG') 923 args = build_utils.ExpandFileArgs(args) 924 options = _ParseArgs(args) 925 926 if options.expected_file: 927 actual_data = _CreateNormalizedManifestForVerification(options) 928 diff_utils.CheckExpectations(actual_data, options) 929 if options.only_verify_expectations: 930 return 931 932 path = options.arsc_path or options.proto_path 933 debug_temp_resources_dir = os.environ.get('TEMP_RESOURCES_DIR') 934 if debug_temp_resources_dir: 935 path = os.path.join(debug_temp_resources_dir, os.path.basename(path)) 936 else: 937 # Use a deterministic temp directory since .pb files embed the absolute 938 # path of resources: crbug.com/939984 939 path = path + '.tmpdir' 940 build_utils.DeleteDirectory(path) 941 942 with resource_utils.BuildContext( 943 temp_dir=path, keep_files=bool(debug_temp_resources_dir)) as build: 944 945 manifest_package_name = _PackageApk(options, build) 946 947 # If --shared-resources-allowlist is used, all the resources listed in the 948 # corresponding R.txt file will be non-final, and an onResourcesLoaded() 949 # will be generated to adjust them at runtime. 950 # 951 # Otherwise, if --shared-resources is used, the all resources will be 952 # non-final, and an onResourcesLoaded() method will be generated too. 953 # 954 # Otherwise, all resources will be final, and no method will be generated. 955 # 956 rjava_build_options = resource_utils.RJavaBuildOptions() 957 if options.shared_resources_allowlist: 958 rjava_build_options.ExportSomeResources( 959 options.shared_resources_allowlist) 960 rjava_build_options.GenerateOnResourcesLoaded() 961 if options.shared_resources: 962 # The final resources will only be used in WebLayer, so hardcode the 963 # package ID to be what WebLayer expects. 964 rjava_build_options.SetFinalPackageId( 965 protoresources.SHARED_LIBRARY_HARDCODED_ID) 966 elif options.shared_resources or options.app_as_shared_lib: 967 rjava_build_options.ExportAllResources() 968 rjava_build_options.GenerateOnResourcesLoaded() 969 970 custom_root_package_name = options.r_java_root_package_name 971 grandparent_custom_package_name = None 972 973 # Always generate an R.java file for the package listed in 974 # AndroidManifest.xml because this is where Android framework looks to find 975 # onResourcesLoaded() for shared library apks. While not actually necessary 976 # for application apks, it also doesn't hurt. 977 apk_package_name = manifest_package_name 978 979 if options.package_name and not options.arsc_package_name: 980 # Feature modules have their own custom root package name and should 981 # inherit from the appropriate base module package. This behaviour should 982 # not be present for test apks with an apk under test. Thus, 983 # arsc_package_name is used as it is only defined for test apks with an 984 # apk under test. 985 custom_root_package_name = options.package_name 986 grandparent_custom_package_name = options.r_java_root_package_name 987 # Feature modules have the same manifest package as the base module but 988 # they should not create an R.java for said manifest package because it 989 # will be created in the base module. 990 apk_package_name = None 991 992 if options.srcjar_out: 993 logging.debug('Creating R.srcjar') 994 resource_utils.CreateRJavaFiles(build.srcjar_dir, apk_package_name, 995 build.r_txt_path, 996 options.extra_res_packages, 997 rjava_build_options, options.srcjar_out, 998 custom_root_package_name, 999 grandparent_custom_package_name) 1000 with action_helpers.atomic_output(build.srcjar_path) as f: 1001 zip_helpers.zip_directory(f, build.srcjar_dir) 1002 1003 logging.debug('Copying outputs') 1004 _WriteOutputs(options, build) 1005 1006 if options.depfile: 1007 assert options.srcjar_out, 'Update first output below and remove assert.' 1008 depfile_deps = (options.dependencies_res_zips + 1009 options.dependencies_res_zip_overlays + 1010 options.include_resources) 1011 action_helpers.write_depfile(options.depfile, options.srcjar_out, 1012 depfile_deps) 1013 1014 1015if __name__ == '__main__': 1016 main(sys.argv[1:]) 1017