| # Copyright 2015 The Chromium Authors | |
| # Use of this source code is governed by a BSD-style license that can be | |
| # found in the LICENSE file. | |
| """Presubmit script validating field trial configs. | |
| See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts | |
| for more details on the presubmit API built into depot_tools. | |
| """ | |
| import copy | |
| import io | |
| import json | |
| import sys | |
| from collections import OrderedDict | |
| USE_PYTHON3 = True | |
| VALID_EXPERIMENT_KEYS = [ | |
| 'name', 'forcing_flag', 'params', 'enable_features', 'disable_features', | |
| 'min_os_version', '//0', '//1', '//2', '//3', '//4', '//5', '//6', '//7', | |
| '//8', '//9' | |
| ] | |
| FIELDTRIAL_CONFIG_FILE_NAME = 'fieldtrial_testing_config.json' | |
| def PrettyPrint(contents): | |
| """Pretty prints a fieldtrial configuration. | |
| Args: | |
| contents: File contents as a string. | |
| Returns: | |
| Pretty printed file contents. | |
| """ | |
| # We have a preferred ordering of the fields (e.g. platforms on top). This | |
| # code loads everything into OrderedDicts and then tells json to dump it out. | |
| # The JSON dumper will respect the dict ordering. | |
| # | |
| # The ordering is as follows: | |
| # { | |
| # 'StudyName Alphabetical': [ | |
| # { | |
| # 'platforms': [sorted platforms] | |
| # 'groups': [ | |
| # { | |
| # name: ... | |
| # forcing_flag: "forcing flag string" | |
| # params: {sorted dict} | |
| # enable_features: [sorted features] | |
| # disable_features: [sorted features] | |
| # (Unexpected extra keys will be caught by the validator) | |
| # } | |
| # ], | |
| # .... | |
| # }, | |
| # ... | |
| # ] | |
| # ... | |
| # } | |
| config = json.loads(contents) | |
| ordered_config = OrderedDict() | |
| for key in sorted(config.keys()): | |
| study = copy.deepcopy(config[key]) | |
| ordered_study = [] | |
| for experiment_config in study: | |
| ordered_experiment_config = OrderedDict([('platforms', | |
| experiment_config['platforms']), | |
| ('experiments', [])]) | |
| for experiment in experiment_config['experiments']: | |
| ordered_experiment = OrderedDict() | |
| for index in range(0, 10): | |
| comment_key = '//' + str(index) | |
| if comment_key in experiment: | |
| ordered_experiment[comment_key] = experiment[comment_key] | |
| ordered_experiment['name'] = experiment['name'] | |
| if 'forcing_flag' in experiment: | |
| ordered_experiment['forcing_flag'] = experiment['forcing_flag'] | |
| if 'params' in experiment: | |
| ordered_experiment['params'] = OrderedDict( | |
| sorted(experiment['params'].items(), key=lambda t: t[0])) | |
| if 'enable_features' in experiment: | |
| ordered_experiment['enable_features'] = \ | |
| sorted(experiment['enable_features']) | |
| if 'disable_features' in experiment: | |
| ordered_experiment['disable_features'] = \ | |
| sorted(experiment['disable_features']) | |
| ordered_experiment_config['experiments'].append(ordered_experiment) | |
| if 'min_os_version' in experiment: | |
| ordered_experiment['min_os_version'] = experiment['min_os_version'] | |
| ordered_study.append(ordered_experiment_config) | |
| ordered_config[key] = ordered_study | |
| return json.dumps( | |
| ordered_config, sort_keys=False, indent=4, separators=(',', ': ')) + '\n' | |
| def ValidateData(json_data, file_path, message_type): | |
| """Validates the format of a fieldtrial configuration. | |
| Args: | |
| json_data: Parsed JSON object representing the fieldtrial config. | |
| file_path: String representing the path to the JSON file. | |
| message_type: Type of message from |output_api| to return in the case of | |
| errors/warnings. | |
| Returns: | |
| A list of |message_type| messages. In the case of all tests passing with no | |
| warnings/errors, this will return []. | |
| """ | |
| def _CreateMessage(message_format, *args): | |
| return _CreateMalformedConfigMessage(message_type, file_path, | |
| message_format, *args) | |
| if not isinstance(json_data, dict): | |
| return _CreateMessage('Expecting dict') | |
| for (study, experiment_configs) in iter(json_data.items()): | |
| warnings = _ValidateEntry(study, experiment_configs, _CreateMessage) | |
| if warnings: | |
| return warnings | |
| return [] | |
| def _ValidateEntry(study, experiment_configs, create_message_fn): | |
| """Validates one entry of the field trial configuration.""" | |
| if not isinstance(study, str): | |
| return create_message_fn('Expecting keys to be string, got %s', type(study)) | |
| if not isinstance(experiment_configs, list): | |
| return create_message_fn('Expecting list for study %s', study) | |
| # Add context to other messages. | |
| def _CreateStudyMessage(message_format, *args): | |
| suffix = ' in Study[%s]' % study | |
| return create_message_fn(message_format + suffix, *args) | |
| for experiment_config in experiment_configs: | |
| warnings = _ValidateExperimentConfig(experiment_config, _CreateStudyMessage) | |
| if warnings: | |
| return warnings | |
| return [] | |
| def _ValidateExperimentConfig(experiment_config, create_message_fn): | |
| """Validates one config in a configuration entry.""" | |
| if not isinstance(experiment_config, dict): | |
| return create_message_fn('Expecting dict for experiment config') | |
| if not 'experiments' in experiment_config: | |
| return create_message_fn('Missing valid experiments for experiment config') | |
| if not isinstance(experiment_config['experiments'], list): | |
| return create_message_fn('Expecting list for experiments') | |
| for experiment_group in experiment_config['experiments']: | |
| warnings = _ValidateExperimentGroup(experiment_group, create_message_fn) | |
| if warnings: | |
| return warnings | |
| if not 'platforms' in experiment_config: | |
| return create_message_fn('Missing valid platforms for experiment config') | |
| if not isinstance(experiment_config['platforms'], list): | |
| return create_message_fn('Expecting list for platforms') | |
| supported_platforms = [ | |
| 'android', 'android_weblayer', 'android_webview', 'chromeos', | |
| 'chromeos_lacros', 'ios', 'linux', 'mac', 'windows' | |
| ] | |
| experiment_platforms = experiment_config['platforms'] | |
| unsupported_platforms = list( | |
| set(experiment_platforms).difference(supported_platforms)) | |
| if unsupported_platforms: | |
| return create_message_fn('Unsupported platforms %s', unsupported_platforms) | |
| return [] | |
| def _ValidateExperimentGroup(experiment_group, create_message_fn): | |
| """Validates one group of one config in a configuration entry.""" | |
| name = experiment_group.get('name', '') | |
| if not name or not isinstance(name, str): | |
| return create_message_fn('Missing valid name for experiment') | |
| # Add context to other messages. | |
| def _CreateGroupMessage(message_format, *args): | |
| suffix = ' in Group[%s]' % name | |
| return create_message_fn(message_format + suffix, *args) | |
| if 'params' in experiment_group: | |
| params = experiment_group['params'] | |
| if not isinstance(params, dict): | |
| return _CreateGroupMessage('Expected dict for params') | |
| for (key, value) in iter(params.items()): | |
| if not isinstance(key, str) or not isinstance(value, str): | |
| return _CreateGroupMessage('Invalid param (%s: %s)', key, value) | |
| for key in experiment_group.keys(): | |
| if key not in VALID_EXPERIMENT_KEYS: | |
| return _CreateGroupMessage('Key[%s] is not a valid key', key) | |
| return [] | |
| def _CreateMalformedConfigMessage(message_type, file_path, message_format, | |
| *args): | |
| """Returns a list containing one |message_type| with the error message. | |
| Args: | |
| message_type: Type of message from |output_api| to return in the case of | |
| errors/warnings. | |
| message_format: The error message format string. | |
| file_path: The path to the config file. | |
| *args: The args for message_format. | |
| Returns: | |
| A list containing a message_type with a formatted error message and | |
| 'Malformed config file [file]: ' prepended to it. | |
| """ | |
| error_message_format = 'Malformed config file %s: ' + message_format | |
| format_args = (file_path,) + args | |
| return [message_type(error_message_format % format_args)] | |
| def CheckPretty(contents, file_path, message_type): | |
| """Validates the pretty printing of fieldtrial configuration. | |
| Args: | |
| contents: File contents as a string. | |
| file_path: String representing the path to the JSON file. | |
| message_type: Type of message from |output_api| to return in the case of | |
| errors/warnings. | |
| Returns: | |
| A list of |message_type| messages. In the case of all tests passing with no | |
| warnings/errors, this will return []. | |
| """ | |
| pretty = PrettyPrint(contents) | |
| if contents != pretty: | |
| return [ | |
| message_type('Pretty printing error: Run ' | |
| 'python3 testing/variations/PRESUBMIT.py %s' % file_path) | |
| ] | |
| return [] | |
| def _GetStudyConfigFeatures(study_config): | |
| """Gets the set of features overridden in a study config.""" | |
| features = set() | |
| for experiment in study_config.get("experiments", []): | |
| features.update(experiment.get("enable_features", [])) | |
| features.update(experiment.get("disable_features", [])) | |
| return features | |
| def _GetDuplicatedFeatures(study1, study2): | |
| """Gets the set of features that are overridden in two overlapping studies.""" | |
| duplicated_features = set() | |
| for study_config1 in study1: | |
| features = _GetStudyConfigFeatures(study_config1) | |
| platforms = set(study_config1.get("platforms", [])) | |
| for study_config2 in study2: | |
| # If the study configs do not specify any common platform, they do not | |
| # overlap, so we can skip them. | |
| if platforms.isdisjoint(set(study_config2.get("platforms", []))): | |
| continue | |
| common_features = features & _GetStudyConfigFeatures(study_config2) | |
| duplicated_features.update(common_features) | |
| return duplicated_features | |
| def CheckDuplicatedFeatures(new_json_data, old_json_data, message_type): | |
| """Validates that features are not specified in multiple studies. | |
| Note that a feature may be specified in different studies that do not overlap. | |
| For example, if they specify different platforms. In such a case, this will | |
| not give a warning/error. However, it is possible that this incorrectly | |
| gives an error, as it is possible for studies to have complex filters (e.g., | |
| if they make use of additional filters such as form_factors, | |
| is_low_end_device, etc.). In those cases, the PRESUBMIT check can be bypassed. | |
| Since this will only check for studies that were changed in this particular | |
| commit, bypassing the PRESUBMIT check will not block future commits. | |
| Args: | |
| new_json_data: Parsed JSON object representing the new fieldtrial config. | |
| old_json_data: Parsed JSON object representing the old fieldtrial config. | |
| message_type: Type of message from |output_api| to return in the case of | |
| errors/warnings. | |
| Returns: | |
| A list of |message_type| messages. In the case of all tests passing with no | |
| warnings/errors, this will return []. | |
| """ | |
| # Get list of studies that changed. | |
| changed_studies = [] | |
| for study_name in new_json_data: | |
| if (study_name not in old_json_data or | |
| new_json_data[study_name] != old_json_data[study_name]): | |
| changed_studies.append(study_name) | |
| # A map between a feature name and the name of studies that use it. E.g., | |
| # duplicated_features_to_studies_map["FeatureA"] = {"StudyA", "StudyB"}. | |
| # Only features that are defined in multiple studies are added to this map. | |
| duplicated_features_to_studies_map = dict() | |
| # Compare the changed studies against all studies defined. | |
| for changed_study_name in changed_studies: | |
| for study_name in new_json_data: | |
| if changed_study_name == study_name: | |
| continue | |
| duplicated_features = _GetDuplicatedFeatures( | |
| new_json_data[changed_study_name], new_json_data[study_name]) | |
| for feature in duplicated_features: | |
| if feature not in duplicated_features_to_studies_map: | |
| duplicated_features_to_studies_map[feature] = set() | |
| duplicated_features_to_studies_map[feature].update( | |
| [changed_study_name, study_name]) | |
| if len(duplicated_features_to_studies_map) == 0: | |
| return [] | |
| duplicated_features_strings = [ | |
| "%s (in studies %s)" % (feature, ', '.join(studies)) | |
| for feature, studies in duplicated_features_to_studies_map.items() | |
| ] | |
| return [ | |
| message_type('The following feature(s) were specified in multiple ' | |
| 'studies: %s' % ', '.join(duplicated_features_strings)) | |
| ] | |
| def CommonChecks(input_api, output_api): | |
| affected_files = input_api.AffectedFiles( | |
| include_deletes=False, | |
| file_filter=lambda x: x.LocalPath().endswith('.json')) | |
| for f in affected_files: | |
| if not f.LocalPath().endswith(FIELDTRIAL_CONFIG_FILE_NAME): | |
| return [ | |
| output_api.PresubmitError( | |
| '%s is the only json file expected in this folder. If new jsons ' | |
| 'are added, please update the presubmit process with proper ' | |
| 'validation. ' % FIELDTRIAL_CONFIG_FILE_NAME | |
| ) | |
| ] | |
| contents = input_api.ReadFile(f) | |
| try: | |
| json_data = input_api.json.loads(contents) | |
| result = ValidateData( | |
| json_data, | |
| f.AbsoluteLocalPath(), | |
| output_api.PresubmitError) | |
| if result: | |
| return result | |
| result = CheckPretty(contents, f.LocalPath(), output_api.PresubmitError) | |
| if result: | |
| return result | |
| result = CheckDuplicatedFeatures( | |
| json_data, | |
| input_api.json.loads('\n'.join(f.OldContents())), | |
| output_api.PresubmitError) | |
| if result: | |
| return result | |
| except ValueError: | |
| return [ | |
| output_api.PresubmitError('Malformed JSON file: %s' % f.LocalPath()) | |
| ] | |
| return [] | |
| def CheckChangeOnUpload(input_api, output_api): | |
| return CommonChecks(input_api, output_api) | |
| def CheckChangeOnCommit(input_api, output_api): | |
| return CommonChecks(input_api, output_api) | |
| def main(argv): | |
| with io.open(argv[1], encoding='utf-8') as f: | |
| content = f.read() | |
| pretty = PrettyPrint(content) | |
| io.open(argv[1], 'wb').write(pretty.encode('utf-8')) | |
| if __name__ == '__main__': | |
| sys.exit(main(sys.argv)) |