Script to push download_file_types.pb to all platforms, via Component Updater.
BUG=596555
CQ_INCLUDE_TRYBOTS=tryserver.chromium.linux:closure_compilation
Review-Url: https://codereview.chromium.org/2003323003
Cr-Commit-Position: refs/heads/master@{#396114}
diff --git a/chrome/browser/resources/safe_browsing/BUILD.gn b/chrome/browser/resources/safe_browsing/BUILD.gn
index 6793358..6149d24 100644
--- a/chrome/browser/resources/safe_browsing/BUILD.gn
+++ b/chrome/browser/resources/safe_browsing/BUILD.gn
@@ -2,6 +2,8 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+# TODO(nparker): reduce the duplication between these two, somehow.
+
# Generate the binary proto form of "file_types" from the ascii proto.
action("make_file_types_protobuf") {
script = "gen_file_type_proto.py"
@@ -10,7 +12,8 @@
# chrome/browser/browser_resources.grd will look for it.
input_filename = "download_file_types.asciipb"
- output_filename = "$target_gen_dir/download_file_types.pb"
+ output_dir = target_gen_dir
+ output_basename = "download_file_types.pb"
python_path_root = "$root_build_dir/pyproto"
python_path_safe_browsing = "$python_path_root/chrome/common/safe_browsing"
@@ -32,18 +35,16 @@
}
inputs = [
- script,
input_filename,
]
- # This script requires the generated python proto code
deps = [
"//chrome/common/safe_browsing:proto",
"//third_party/protobuf:py_proto",
]
outputs = [
- output_filename,
+ "$output_dir/$output_basename",
]
args = [
@@ -52,8 +53,51 @@
target_arch,
"-i",
rebase_path(input_filename, root_build_dir),
+ "-d",
+ rebase_path(output_dir, root_build_dir),
"-o",
- rebase_path(output_filename, root_build_dir),
+ output_basename,
+ "-p",
+ rebase_path(python_path_root, root_build_dir),
+ "-p",
+ rebase_path(python_path_safe_browsing, root_build_dir),
+ ]
+}
+
+# Generate the binary proto for ALL platforms. This is only run manually
+# when pushing the files to GCS for the component-updater to pick up.
+action("make_all_file_types_protobuf") {
+ script = "gen_file_type_proto.py"
+
+ input_filename = "download_file_types.asciipb"
+ output_dir = "$target_gen_dir/all"
+ output_basename = "download_file_types.pb"
+ python_path_root = "$root_build_dir/pyproto"
+ python_path_safe_browsing = "$python_path_root/chrome/common/safe_browsing"
+
+ inputs = [
+ input_filename,
+ ]
+
+ deps = [
+ "//chrome/common/safe_browsing:proto",
+ "//third_party/protobuf:py_proto",
+ ]
+
+ # A directory, since we can't derive the actual file names here.
+ outputs = [
+ output_dir,
+ ]
+
+ args = [
+ "-w",
+ "-a",
+ "-i",
+ rebase_path(input_filename, root_build_dir),
+ "-d",
+ rebase_path(output_dir, root_build_dir),
+ "-o",
+ output_basename,
"-p",
rebase_path(python_path_root, root_build_dir),
"-p",
diff --git a/chrome/browser/resources/safe_browsing/README.md b/chrome/browser/resources/safe_browsing/README.md
index 503b016..61f2096 100644
--- a/chrome/browser/resources/safe_browsing/README.md
+++ b/chrome/browser/resources/safe_browsing/README.md
@@ -3,16 +3,26 @@
This describes how to adjust file-type download behavior in
Chrome including interactions with Safe Browsing. The metadata described
here, and stored in `download_file_types.asciipb`, will be both baked into
-Chrome released and pushable to Chrome between releases. http://crbug.com/596555
+Chrome released and pushable to Chrome between releases (via
+`FileTypePolicies` class). http://crbug.com/596555
Rendered version of this file: https://chromium.googlesource.com/chromium/src/+/master/chrome/browser/resources/safe_browsing/README.md
-## Procedure for adding a new type
- * Edit `download_file_types.asciipb`, edit `download_stats.cc` (necessary
- until it gets replaced), and update `histograms.xml`
- * Get it reviewed, submit.
- * Push via component update (PROCEDURE TBD)
+## Procedure for adding/modifying file type(s)
+ * **Edit** `download_file_types.asciipb` and update `histograms.xml`
+ * Get it reviewed, **submit.**
+ * **Push** it to all users via component update:
+ * Wait 1-3 day for this to run on Canary to verify it doesn't crash Chrome.
+ * In a synced checkout, generate protos for all platforms:
+ * % `ninja -C out-gn/Debug
+ chrome/browser/resources/safe_browsing:make_all_file_types_protobuf`
+ * That will instruct you to run another command to push the files to GCS.
+ You must a member of chrome-file-type-policies-pushers@google.com to have
+ access to the GCS bucket.
+ * The Component Updater system will notice those files and push them to
+ users withing ~6 hours. If not, contact `waffles@.`
+
## Guidelines for a DownloadFileType entry:
See `download_file_types.proto` for all fields.
diff --git a/chrome/browser/resources/safe_browsing/gen_file_type_proto.py b/chrome/browser/resources/safe_browsing/gen_file_type_proto.py
index 1e55f9a1..63ed686 100755
--- a/chrome/browser/resources/safe_browsing/gen_file_type_proto.py
+++ b/chrome/browser/resources/safe_browsing/gen_file_type_proto.py
@@ -11,9 +11,11 @@
"""
import optparse
+import os
import re
import subprocess
import sys
+import traceback
def ImportProtoModules(paths):
@@ -93,6 +95,7 @@
def FilterPbForPlatform(full_pb, platform_type):
""" Return a filtered protobuf for this platform_type """
+ assert type(platform_type) is int, "Bad platform_type type"
new_pb = config_pb2.DownloadFileTypeConfig();
new_pb.CopyFrom(full_pb)
@@ -127,25 +130,59 @@
return new_pb
-def GeneratBinaryProtos(opts):
- # Read the ASCII
- ifile = open(opts.infile, 'r')
- ascii_pb_str = ifile.read()
- ifile.close()
-
- # Parse it into a structure PB
- pb = config_pb2.DownloadFileTypeConfig()
- text_format.Merge(ascii_pb_str, pb)
-
- ValidatePb(pb);
- platform_type = PlatformTypes()[opts.type]
- filtered_pb = FilterPbForPlatform(pb, platform_type);
+def FilterForPlatformAndWrite(full_pb, platform_type, outfile):
+ """ Filter and write out a file for this platform """
+ # Filter it
+ filtered_pb = FilterPbForPlatform(full_pb, platform_type);
# Serialize it
binary_pb_str = filtered_pb.SerializeToString()
# Write it to disk
- open(opts.outfile, 'wb').write(binary_pb_str)
+ open(outfile, 'wb').write(binary_pb_str)
+
+
+def MakeSubDirs(outfile):
+ """ Make the subdirectories needed to create file |outfile| """
+ dirname = os.path.dirname(outfile)
+ if not os.path.exists(dirname):
+ os.makedirs(dirname)
+
+
+def GenerateBinaryProtos(opts):
+ """ Read the ASCII proto and generate one or more binary protos. """
+ # Read the ASCII
+ ifile = open(opts.infile, 'r')
+ ascii_pb_str = ifile.read()
+ ifile.close()
+
+ # Parse it into a structured PB
+ full_pb = config_pb2.DownloadFileTypeConfig()
+ text_format.Merge(ascii_pb_str, full_pb)
+
+ ValidatePb(full_pb);
+
+ if opts.type is not None:
+ # Just one platform type
+ platform_enum = PlatformTypes()[opts.type]
+ outfile = os.path.join(opts.outdir, opts.outbasename)
+ FilterForPlatformAndWrite(full_pb, platform_enum, outfile)
+ else:
+ # Make a separate file for each platform
+ for platform_type, platform_enum in PlatformTypes().iteritems():
+ # e.g. .../all/77/chromeos/download_file_types.pb
+ outfile = os.path.join(opts.outdir,
+ str(full_pb.version_id),
+ platform_type,
+ opts.outbasename)
+ MakeSubDirs(outfile)
+ FilterForPlatformAndWrite(full_pb, platform_enum, outfile)
+
+ print "\n\nTo push these files, run the following:"
+ print ("python " +
+ "chrome/browser/resources/safe_browsing/push_file_type_proto.py " +
+ "-d " + os.path.abspath(opts.outdir))
+ print "\n\n"
def main():
@@ -155,44 +192,59 @@
help='Wrap this script in another python '
'execution to disable site-packages. This is a '
'fix for http://crbug.com/605592')
+
+ parser.add_option('-a', '--all', action="store_true", default=False,
+ help='Write a separate file for every platform. '
+ 'Outfile must have a %d for version and %s for platform.')
parser.add_option('-t', '--type',
help='The platform type. One of android, chromeos, ' +
'linux, mac, win')
parser.add_option('-i', '--infile',
help='The ASCII DownloadFileType-proto file to read.')
- parser.add_option('-o', '--outfile',
- help='The binary file to write to.')
+ parser.add_option('-d', '--outdir',
+ help='Directory underwhich binary file(s) will be written')
+ parser.add_option('-o', '--outbasename',
+ help='Basename of the binary file to write to.')
parser.add_option('-p', '--path', action="append",
help='Repeat this as needed. Directory(s) containing ' +
'the download_file_types_pb2.py and ' +
'google.protobuf.text_format modules')
(opts, args) = parser.parse_args()
- if opts.infile is None or opts.outfile is None:
+ if opts.infile is None or opts.outdir is None or opts.outbasename is None:
parser.print_help()
return 1
if opts.wrap:
# Run this script again with different args to the interpreter.
command = [sys.executable, '-S', '-s', sys.argv[0]]
- command += ['-t', opts.type]
+ if opts.type is not None:
+ command += ['-t', opts.type]
+ if opts.all:
+ command += ['-a']
command += ['-i', opts.infile]
- command += ['-o', opts.outfile]
+ command += ['-d', opts.outdir]
+ command += ['-o', opts.outbasename]
for path in opts.path:
command += ['-p', path]
sys.exit(subprocess.call(command))
ImportProtoModules(opts.path)
- if (opts.type not in PlatformTypes()):
+ if (not opts.all and opts.type not in PlatformTypes()):
print "ERROR: Unknown platform type '%s'" % opts.type
parser.print_help()
return 1
+ if (bool(opts.all) == bool(opts.type)):
+ print "ERROR: Need exactly one of --type or --all"
+ parser.print_help()
+ return 1
+
try:
- GeneratBinaryProtos(opts)
+ GenerateBinaryProtos(opts)
except Exception as e:
- print "ERROR: Failed to render binary version of %s:\n %s\n" % (
- opts.infile, str(e))
+ print "ERROR: Failed to render binary version of %s:\n %s\n%s" % (
+ opts.infile, str(e), traceback.format_exc())
return 1
diff --git a/chrome/browser/resources/safe_browsing/push_file_type_proto.py b/chrome/browser/resources/safe_browsing/push_file_type_proto.py
new file mode 100755
index 0000000..0256ee8
--- /dev/null
+++ b/chrome/browser/resources/safe_browsing/push_file_type_proto.py
@@ -0,0 +1,65 @@
+#!/usr/bin/python
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Push the {vers}/{plaform}/download_file_types.pb files to GCS so
+# that the component update system will pick them up and push them
+# to users. See README.md before running this.
+
+import optparse
+import os
+import subprocess
+import sys
+
+
+DEST_BUCKET = 'gs://chrome-component-file-type-policies'
+
+
+def main():
+ parser = optparse.OptionParser()
+ parser.add_option('-d', '--dir',
+ help='The directory containing '
+ '{vers}/{platform}/download_file_types.pb files.')
+
+ (opts, args) = parser.parse_args()
+ if opts.dir is None:
+ parser.print_help()
+ return 1
+
+ os.chdir(opts.dir)
+
+ # Sanity check that we're in the right place
+ assert opts.dir.endswith('/all'), '--dir should end with /all'
+ dirs = os.listdir('.')
+ assert (len(dirs) == 1 and dirs[0].isdigit()), (
+ 'Should be exactly one version directory. Please delete the contents '
+ 'of the target dir and regenerate the protos.')
+
+ # Push the files with their directories, in the form
+ # {vers}/{platform}/download_file_types.pb
+ # Don't overwrite existing files, incase we forgot to increment the
+ # version.
+ vers_dir = dirs[0]
+ command = ['gsutil', 'cp', '-Rn', vers_dir, DEST_BUCKET]
+
+ print 'Going to run the following command'
+ print ' ', ' '.join(command)
+ print '\nIn directory'
+ print ' ', opts.dir
+ print '\nWhich should push the following files'
+ expected_files = [os.path.join(dp, f) for dp, dn, fn in
+ os.walk(vers_dir) for f in fn]
+ for f in expected_files:
+ print ' ', f
+
+ shall = raw_input('\nAre you sure (y/N) ').lower() == 'y'
+ if not shall:
+ print 'aborting'
+ return 1
+ return subprocess.call(command)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
+
