Updating how we roll android_deps
Bug: 668211
Change-Id: I77bdd5ac4caeb3b9ffb175aa3e2f29eb908803e4
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1557858
Commit-Queue: Sam Maier <smaier@chromium.org>
Reviewed-by: Peter Wen <wnwen@chromium.org>
Cr-Original-Commit-Position: refs/heads/master@{#657245}
Cr-Mirrored-From: https://chromium.googlesource.com/chromium/src
Cr-Mirrored-Commit: e367a3f08518be6516802bdef0306c46649c9378
diff --git a/roll/android_deps/build.gradle b/roll/android_deps/build.gradle
index 141aa0d..c3dc08e 100644
--- a/roll/android_deps/build.gradle
+++ b/roll/android_deps/build.gradle
@@ -91,6 +91,9 @@
task setUpRepository(type: BuildConfigGenerator) {
// Paths are relative to the chromium source root.
repositoryPath 'third_party/android_deps'
+ depsPath 'DEPS'
+ chromiumSourceRoot '../../../..'
+ cipdBucket 'chromium'
}
task wrapper(type: Wrapper) {
diff --git a/roll/android_deps/buildSrc/src/main/groovy/BuildConfigGenerator.groovy b/roll/android_deps/buildSrc/src/main/groovy/BuildConfigGenerator.groovy
index fd5593b..21a6ae2 100644
--- a/roll/android_deps/buildSrc/src/main/groovy/BuildConfigGenerator.groovy
+++ b/roll/android_deps/buildSrc/src/main/groovy/BuildConfigGenerator.groovy
@@ -51,9 +51,41 @@
*/
String repositoryPath
+ /**
+ * Relative path to the DEPS file where the cipd packages are specified.
+ */
+ String depsPath
+
+ /**
+ * Relative path to the Chromium source root from the build.gradle file.
+ */
+ String chromiumSourceRoot
+
+ /**
+ * Name of the cipd root package.
+ */
+ String cipdBucket
+
+ /**
+ * Prefix of path to strip before uploading to CIPD.
+ */
+ String stripFromCipdPath
+
+ /**
+ * Skips license file import.
+ */
+ boolean skipLicenses
+
+ /**
+ * Only pull play services targets into BUILD.gn file.
+ * If the play services target depends on a non-play services target, it will use the target in
+ * //third_party/android_deps/BUILD.gn.
+ */
+ boolean onlyPlayServices
+
@TaskAction
void main() {
- def graph = new ChromiumDepGraph(project: project)
+ def graph = new ChromiumDepGraph(project: project, skipLicenses: skipLicenses)
def normalisedRepoPath = normalisePath(repositoryPath)
def rootDirPath = normalisePath(".")
@@ -63,7 +95,7 @@
// 2. Import artifacts into the local repository
def dependencyDirectories = []
graph.dependencies.values().each { dependency ->
- if (dependency.exclude || EXISTING_LIBS.get(dependency.id) != null) {
+ if (excludeDependency(dependency, onlyPlayServices)) {
return
}
logger.debug "Processing ${dependency.name}: \n${jsonDump(dependency)}"
@@ -83,25 +115,31 @@
}
new File("${absoluteDepDir}/README.chromium").write(makeReadme(dependency))
- new File("${absoluteDepDir}/cipd.yaml").write(makeCipdYaml(dependency, repositoryPath))
+ new File("${absoluteDepDir}/cipd.yaml").write(makeCipdYaml(dependency, cipdBucket,
+ stripFromCipdPath,
+ repositoryPath))
new File("${absoluteDepDir}/OWNERS").write(makeOwners())
- if (!dependency.licensePath?.trim()?.isEmpty()) {
- new File("${absoluteDepDir}/LICENSE").write(
- new File("${normalisedRepoPath}/${dependency.licensePath}").text)
- } else if (!dependency.licenseUrl?.trim()?.isEmpty()) {
- downloadFile(dependency.licenseUrl, new File("${absoluteDepDir}/LICENSE"))
+ if (!skipLicenses) {
+ if (!dependency.licensePath?.trim()?.isEmpty()) {
+ new File("${absoluteDepDir}/LICENSE").write(
+ new File("${normalisedRepoPath}/${dependency.licensePath}").text)
+ } else if (!dependency.licenseUrl?.trim()?.isEmpty()) {
+ downloadFile(dependency.licenseUrl, new File("${absoluteDepDir}/LICENSE"))
+ }
}
}
// 3. Generate the root level build files
- updateBuildTargetDeclaration(graph, "${normalisedRepoPath}/BUILD.gn")
- updateDepsDeclaration(graph, repositoryPath, "${rootDirPath}/DEPS")
+ updateBuildTargetDeclaration(graph, "${normalisedRepoPath}/BUILD.gn", onlyPlayServices)
+ updateDepsDeclaration(graph, cipdBucket, stripFromCipdPath, repositoryPath,
+ "${rootDirPath}/${depsPath}", onlyPlayServices)
dependencyDirectories.sort { path1, path2 -> return path1.compareTo(path2) }
updateReadmeReferenceFile(dependencyDirectories,
"${normalisedRepoPath}/additional_readme_paths.json")
}
- private static void updateBuildTargetDeclaration(ChromiumDepGraph depGraph, String path) {
+ private static void updateBuildTargetDeclaration(ChromiumDepGraph depGraph, String path,
+ boolean onlyPlayServices) {
File buildFile = new File(path)
def sb = new StringBuilder()
@@ -115,39 +153,43 @@
}
depGraph.dependencies.values().sort(dependencyComparator).each { dependency ->
- if (dependency.exclude || EXISTING_LIBS.get(dependency.id) != null) {
+ if (excludeDependency(dependency, onlyPlayServices)) {
return
}
def depsStr = ""
if (!dependency.children.isEmpty()) {
dependency.children.each { childDep ->
- def dep = depGraph.dependencies[childDep];
+ def dep = depGraph.dependencies[childDep]
if (dep.exclude) {
return
}
// Special case: If a child dependency is an existing lib, rather than skipping
// it, replace the child dependency with the existing lib.
def existingLib = EXISTING_LIBS.get(dep.id)
+ def targetName = translateTargetName(dep.id) + "_java"
if (existingLib != null) {
- depsStr += "\"${existingLib}\","
+ depsStr += "\"${existingLib}\","
+ } else if (onlyPlayServices && !isPlayServicesTarget(dep.id)) {
+ depsStr += "\"//third_party/android_deps:${targetName}\","
} else {
- depsStr += "\":${dep.id}_java\","
+ depsStr += "\":${targetName}\","
}
}
}
def libPath = "${DOWNLOAD_DIRECTORY_NAME}/${dependency.id}"
+ def targetName = translateTargetName(dependency.id) + "_java"
sb.append(BUILD_GN_GEN_REMINDER)
if (dependency.extension == 'jar') {
sb.append("""\
- java_prebuilt("${dependency.id}_java") {
+ java_prebuilt("${targetName}") {
jar_path = "${libPath}/${dependency.fileName}"
output_name = "${dependency.id}"
""".stripIndent())
if (dependency.supportsAndroid) sb.append(" supports_android = true\n")
} else if (dependency.extension == 'aar') {
sb.append("""\
- android_aar_prebuilt("${dependency.id}_java") {
+ android_aar_prebuilt("${targetName}") {
aar_path = "${libPath}/${dependency.fileName}"
info_path = "${libPath}/${dependency.id}.info"
""".stripIndent())
@@ -173,15 +215,28 @@
"${BUILD_GN_TOKEN_START}\n${sb.toString()}\n${BUILD_GN_TOKEN_END}"))
}
+ public static String translateTargetName(String targetName) {
+ if (isPlayServicesTarget(targetName)) {
+ return targetName.replaceFirst("com_", "").replaceFirst("android_gms_", "")
+ }
+ return targetName
+ }
+
+ public static boolean isPlayServicesTarget(String dependencyId) {
+ // Firebase has historically been treated as a part of play services, so it counts here for
+ // backwards compatibility.
+ return Pattern.matches(".*google.*(play_services|firebase).*", dependencyId)
+ }
+
private static void addSpecialTreatment(StringBuilder sb, String dependencyId) {
- if (Pattern.matches(".*google.*play_services.*", dependencyId)) {
- if (Pattern.matches(".*cast_framework.*", dependencyId)) {
+ if (isPlayServicesTarget(dependencyId)) {
+ if (Pattern.matches(".*cast_framework.*", dependencyId)) {
sb.append(' # Removing all resources from cast framework as they are unused bloat.\n')
sb.append(' strip_resources = true\n')
- } else {
+ } else {
sb.append(' # Removing drawables from GMS .aars as they are unused bloat.\n')
sb.append(' strip_drawables = true\n')
- }
+ }
}
switch(dependencyId) {
case 'com_android_support_support_compat':
@@ -224,8 +279,9 @@
}
}
- private static void updateDepsDeclaration(ChromiumDepGraph depGraph, String repoPath,
- String depsFilePath) {
+ private static void updateDepsDeclaration(ChromiumDepGraph depGraph, String cipdBucket,
+ String stripFromCipdPath, String repoPath,
+ String depsFilePath, boolean onlyPlayServices) {
File depsFile = new File(depsFilePath)
def sb = new StringBuilder()
// Note: The string we're inserting is nested 1 level, hence the 2 leading spaces. Same
@@ -238,16 +294,24 @@
}
depGraph.dependencies.values().sort(dependencyComparator).each { dependency ->
- if (dependency.exclude || EXISTING_LIBS.get(dependency.id) != null) {
+ if (excludeDependency(dependency, onlyPlayServices)) {
return
}
def depPath = "${DOWNLOAD_DIRECTORY_NAME}/${dependency.id}"
+ def cipdPath = "${cipdBucket}/"
+ if (stripFromCipdPath) {
+ assert repoPath.startsWith(stripFromCipdPath)
+ cipdPath += repoPath.substring(stripFromCipdPath.length() + 1)
+ } else {
+ cipdPath += repoPath
+ }
+ cipdPath += "/${depPath}"
sb.append("""\
|
| 'src/${repoPath}/${depPath}': {
| 'packages': [
| {
- | 'package': 'chromium/${repoPath}/${depPath}',
+ | 'package': '${cipdPath}',
| 'version': 'version:${dependency.version}-${CIPD_SUFFIX}',
| },
| ],
@@ -267,9 +331,14 @@
refFile.write(JsonOutput.prettyPrint(JsonOutput.toJson(directories)) + "\n")
}
+ public static boolean excludeDependency(ChromiumDepGraph.DependencyDescription dependency,
+ boolean onlyPlayServices) {
+ return dependency.exclude || EXISTING_LIBS.get(dependency.id) != null ||
+ (onlyPlayServices && !isPlayServicesTarget(dependency.id))
+ }
+
private String normalisePath(String pathRelativeToChromiumRoot) {
- def pathToChromiumRoot = "../../../.." // Relative to build.gradle, the project root.
- return project.file("${pathToChromiumRoot}/${pathRelativeToChromiumRoot}").absolutePath
+ return project.file("${chromiumSourceRoot}/${pathRelativeToChromiumRoot}").absolutePath
}
static String makeOwners() {
@@ -308,8 +377,20 @@
""".stripIndent()
}
- static String makeCipdYaml(ChromiumDepGraph.DependencyDescription dependency, String repoPath) {
+ static String makeCipdYaml(ChromiumDepGraph.DependencyDescription dependency, String cipdBucket,
+ String stripFromCipdPath, String repoPath) {
+ if (!stripFromCipdPath) {
+ stripFromCipdPath = ''
+ }
def cipdVersion = "${dependency.version}-${CIPD_SUFFIX}"
+ def cipdPath = "${cipdBucket}/"
+ if (stripFromCipdPath) {
+ assert repoPath.startsWith(stripFromCipdPath)
+ cipdPath += repoPath.substring(stripFromCipdPath.length() + 1)
+ } else {
+ cipdPath += repoPath
+ }
+ cipdPath += "/${DOWNLOAD_DIRECTORY_NAME}/${dependency.id}"
// NOTE: the fetch_all.py script relies on the format of this file!
// See fetch_all.py:GetCipdPackageInfo().
@@ -320,7 +401,7 @@
# To create CIPD package run the following command.
# cipd create --pkg-def cipd.yaml -tag version:${cipdVersion}
- package: chromium/${repoPath}/${DOWNLOAD_DIRECTORY_NAME}/${dependency.id}
+ package: ${cipdPath}
description: "${dependency.displayName}"
data:
- file: ${dependency.fileName}
diff --git a/roll/android_deps/buildSrc/src/main/groovy/ChromiumDepGraph.groovy b/roll/android_deps/buildSrc/src/main/groovy/ChromiumDepGraph.groovy
index af3f7be..f639b45 100644
--- a/roll/android_deps/buildSrc/src/main/groovy/ChromiumDepGraph.groovy
+++ b/roll/android_deps/buildSrc/src/main/groovy/ChromiumDepGraph.groovy
@@ -58,6 +58,7 @@
]
Project project
+ boolean skipLicenses
void collectDependencies() {
def compileConfig = project.configurations.getByName('compile').resolvedConfiguration
@@ -84,7 +85,8 @@
}
compileConfig.resolvedArtifacts.each { artifact ->
- def dep = dependencies.get(makeModuleId(artifact))
+ def id = makeModuleId(artifact)
+ def dep = dependencies.get(id)
assert dep != null : "No dependency collected for artifact ${artifact.name}"
dep.supportsAndroid = true
dep.testOnly = false
@@ -149,9 +151,11 @@
List<String> childModules) {
def pom = getPomFromArtifact(artifact.id.componentIdentifier).file
def pomContent = new XmlSlurper(false, false).parse(pom)
- String licenseName
- String licenseUrl
- (licenseName, licenseUrl) = resolveLicenseInformation(id, pomContent)
+ String licenseName = ''
+ String licenseUrl = ''
+ if (!skipLicenses) {
+ (licenseName, licenseUrl) = resolveLicenseInformation(id, pomContent)
+ }
// Get rid of irrelevant indent that might be present in the XML file.
def description = pomContent.description?.text()?.trim()?.replaceAll(/\s+/, " ")
@@ -208,6 +212,15 @@
}
}
+ if (skipLicenses) {
+ dep.licenseName = ''
+ dep.licensePath = ''
+ dep.licenseUrl = ''
+ if (dep.id?.endsWith('license')) {
+ dep.exclude = true
+ }
+ }
+
return dep
}
diff --git a/roll/android_deps/fetch_all.py b/roll/android_deps/fetch_all.py
index cea4b71..e537896 100755
--- a/roll/android_deps/fetch_all.py
+++ b/roll/android_deps/fetch_all.py
@@ -61,9 +61,6 @@
# Location of the android_deps libs directory from a root checkout.
_ANDROID_DEPS_LIBS_SUBDIR = _ANDROID_DEPS_SUBDIR + '/libs'
-# Location of the build.gradle file used to configure our dependencies.
-_BUILD_GRADLE_PATH = 'tools/android/roll/android_deps/build.gradle'
-
# Location of the buildSrc directory used implement our gradle task.
_GRADLE_BUILDSRC_PATH = 'tools/android/roll/android_deps/buildSrc'
@@ -74,14 +71,6 @@
_ANDROID_DEPS_ADDITIONAL_README_PATHS,
]
-# The list of files and dirs that are copied to the build directory by this
-# script. Should not include _UPDATED_GIT_FILES.
-_COPIED_PATHS = [
- _ANDROID_DEPS_LICENSE_SUBDIR,
- _BUILD_GRADLE_PATH,
- _GRADLE_BUILDSRC_PATH,
-]
-
# If this file exists in an aar file then it is appended to LICENSE
_THIRD_PARTY_LICENSE_FILENAME = 'third_party_licenses.txt'
@@ -274,7 +263,7 @@
return (package_name, package_tag)
-def ParseDeps(root_dir):
+def ParseDeps(root_dir, libs_dir):
"""Parse an android_deps/libs and retrieve package information.
Args:
@@ -286,8 +275,7 @@
and |package_name| and |package_tag| are the extracted from it.
"""
result = {}
- libs_dir = os.path.abspath(os.path.join(
- root_dir, _ANDROID_DEPS_LIBS_SUBDIR))
+ libs_dir = os.path.abspath(os.path.join(root_dir, libs_dir))
for cipd_file in FindInDirectory(libs_dir, 'cipd.yaml'):
pkg_name, pkg_tag = GetCipdPackageInfo(cipd_file)
cipd_path = os.path.dirname(cipd_file)
@@ -335,6 +323,16 @@
help='Path to Chromium source tree (auto-detect by default).')
parser.add_argument('--gradle-wrapper',
help='Path to custom gradle wrapper (auto-detect by default).')
+ parser.add_argument(
+ '--build-gradle',
+ help='Path to build.gradle relative to src/.',
+ default='tools/android/roll/android_deps/build.gradle')
+ parser.add_argument(
+ '--git-dir', help='Path to git subdir from chromium-dir.', default='.')
+ parser.add_argument(
+ '--ignore-licenses',
+ help='Ignores licenses for these deps.',
+ action='store_true')
parser.add_argument('--update-all', action='store_true',
help='Update current checkout in case of build.gradle changes.'
'This will also print a list of commands to upload new and updated '
@@ -346,21 +344,37 @@
'\'debug\')')
args = parser.parse_args()
- logging.basicConfig(format='%(message)s')
- logger = logging.getLogger()
- if args.log_level:
- logger.setLevel(args.log_level.upper())
-
# Determine Chromium source tree.
chromium_src = args.chromium_dir
if not chromium_src:
# Assume this script is stored under tools/android/roll/android_deps/
chromium_src = _CHROMIUM_SRC
+ chromium_src = os.path.abspath(chromium_src)
+
+ abs_git_dir = os.path.normpath(os.path.join(chromium_src, args.git_dir))
+
if not os.path.isdir(chromium_src):
raise Exception('Not a directory: ' + chromium_src)
+ if not os.path.isdir(abs_git_dir):
+ raise Exception('Not a directory: ' + abs_git_dir)
- chromium_src = os.path.abspath(chromium_src)
+ # The list of files and dirs that are copied to the build directory by this
+ # script. Should not include _UPDATED_GIT_FILES.
+ copied_paths = {
+ args.build_gradle:
+ args.build_gradle,
+ _GRADLE_BUILDSRC_PATH:
+ os.path.join(os.path.dirname(args.build_gradle), "buildSrc"),
+ }
+
+ if not args.ignore_licenses:
+ copied_paths[_ANDROID_DEPS_LICENSE_SUBDIR] = _ANDROID_DEPS_LICENSE_SUBDIR
+
+ logging.basicConfig(format='%(message)s')
+ logger = logging.getLogger()
+ if args.log_level:
+ logger.setLevel(args.log_level.upper())
# Handle --reset-workspace here.
if args.reset_workspace:
@@ -370,7 +384,7 @@
RunCommand(['rm', '-rf', cipd_dir])
print '# Saving build.gradle content'
- build_gradle_path = os.path.join(chromium_src, _BUILD_GRADLE_PATH)
+ build_gradle_path = os.path.join(chromium_src, args.build_gradle)
build_gradle = ReadFile(build_gradle_path)
print '# Resetting and re-syncing workspace. (may take a while).'
@@ -381,10 +395,12 @@
return
missing_files = []
- for src_path in _UPDATED_GIT_FILES + _COPIED_PATHS:
+ for src_path in copied_paths.keys():
if not os.path.exists(os.path.join(chromium_src, src_path)):
missing_files.append(src_path)
-
+ for src_path in _UPDATED_GIT_FILES:
+ if not os.path.exists(os.path.join(abs_git_dir, src_path)):
+ missing_files.append(src_path)
if missing_files:
raise Exception('Missing files from %s: %s' % (chromium_src, missing_files))
@@ -401,12 +417,12 @@
print '# Setup build directory.'
logging.debug('Using build directory: ' + build_dir)
for git_file in _UPDATED_GIT_FILES:
- git_data = ReadGitHeadFile(chromium_src, git_file)
- WriteFile(os.path.join(build_dir, git_file), git_data)
+ git_data = ReadGitHeadFile(abs_git_dir, git_file)
+ WriteFile(os.path.join(build_dir, args.git_dir, git_file), git_data)
- for path in _COPIED_PATHS:
- CopyFileOrDirectory(os.path.join(chromium_src, path),
- os.path.join(build_dir, path))
+ for path, dest in copied_paths.iteritems():
+ CopyFileOrDirectory(
+ os.path.join(chromium_src, path), os.path.join(build_dir, dest))
print '# Use Gradle to download packages and edit/create relevant files.'
# This gradle command generates the new DEPS and BUILD.gn files, it can also
@@ -414,16 +430,20 @@
# for such cases.
gradle_cmd = [
gradle_wrapper_path,
- '-b', os.path.join(build_dir, _BUILD_GRADLE_PATH),
+ '-b',
+ os.path.join(build_dir, args.build_gradle),
'setupRepository',
'--stacktrace',
]
RunCommand(gradle_cmd)
- libs_dir = os.path.join(build_dir, _ANDROID_DEPS_LIBS_SUBDIR)
+ libs_dir = os.path.join(build_dir, args.git_dir, _ANDROID_DEPS_LIBS_SUBDIR)
print '# Reformat %s.' % _ANDROID_DEPS_BUILD_GN
- gn_args = ['gn', 'format', os.path.join(build_dir, _ANDROID_DEPS_BUILD_GN)]
+ gn_args = [
+ 'gn', 'format',
+ os.path.join(build_dir, args.git_dir, _ANDROID_DEPS_BUILD_GN)
+ ]
RunCommand(gn_args)
print '# Generate Android .aar info and third-party license files.'
@@ -435,17 +455,19 @@
if not os.path.exists(aar_info_path):
logging.info('- %s' % aar_info_name)
RunCommand([aar_py, 'list', aar_file, '--output', aar_info_path])
- with zipfile.ZipFile(aar_file) as z:
- if _THIRD_PARTY_LICENSE_FILENAME in z.namelist():
- license_path = os.path.join(aar_dirname, 'LICENSE')
- # Make sure to append as we don't want to lose the existing license.
- with open(license_path, 'a') as f:
- f.write(z.read(_THIRD_PARTY_LICENSE_FILENAME))
+ if not args.ignore_licenses:
+ with zipfile.ZipFile(aar_file) as z:
+ if _THIRD_PARTY_LICENSE_FILENAME in z.namelist():
+ license_path = os.path.join(aar_dirname, 'LICENSE')
+ # Make sure to append as we don't want to lose the existing license.
+ with open(license_path, 'a') as f:
+ f.write(z.read(_THIRD_PARTY_LICENSE_FILENAME))
print '# Compare CIPD packages.'
- existing_packages = ParseDeps(chromium_src)
- build_packages = ParseDeps(build_dir)
+ existing_packages = ParseDeps(abs_git_dir, _ANDROID_DEPS_LIBS_SUBDIR)
+ build_packages = ParseDeps(
+ build_dir, os.path.join(args.git_dir, _ANDROID_DEPS_LIBS_SUBDIR))
deleted_packages = []
updated_packages = []
@@ -496,12 +518,13 @@
# Copy updated DEPS and BUILD.gn to build directory.
update_cmds = []
for updated_file in _UPDATED_GIT_FILES:
- CopyFileOrDirectory(os.path.join(build_dir, updated_file),
- os.path.join(chromium_src, updated_file))
+ CopyFileOrDirectory(
+ os.path.join(build_dir, args.git_dir, updated_file),
+ os.path.join(abs_git_dir, updated_file))
# Delete obsolete or updated package directories.
for pkg in deleted_packages + updated_packages:
- pkg_path = os.path.join(chromium_src, existing_packages[pkg].path)
+ pkg_path = os.path.join(abs_git_dir, existing_packages[pkg].path)
DeleteDirectory(pkg_path)
# Copy new and updated packages from build directory.
