blob: 13510d3ec7bf2a77d19fcbe30a61abe1b40cc518 [file] [log] [blame]
#!/usr/bin/env python3
# Copyright 2022 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import argparse
import glob
import json
import copy
import os
import sys
# This script runs in Chromium and ChromiumOS.
try:
# Chromium.
_SRC_PATH = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', '..', '..'))
sys.path.append(os.path.join(_SRC_PATH, 'third_party'))
import pyyaml
except ImportError:
# ChromiumOS.
# Some consumers (ChromiumOS ebuilds) of this script have pyyaml already in
# their environment and may not have access to the full Chromium repository
# nor the exact file structure. Import based on the Chromium third_party may
# not always be possible.
# Those consumers may install pyyaml in their environment and import it with
# the name of the module instead as per the library documentation.
# For compatibility reasons, please refer to
# https://source.chromium.org/chromium/chromium/src/+/main:third_party/pyyaml/README.chromium
# to know which version of pyyaml to use.
import yaml as pyyaml
def _SafeListDir(directory):
'''Wrapper around os.listdir() that ignores files created by Finder.app.'''
# On macOS, Finder.app creates .DS_Store files when a user visit a
# directory causing failure of the script laters on because there
# are no such group as .DS_Store. Skip the file to prevent the error.
return filter(lambda name:(name != '.DS_Store'),sorted(os.listdir(directory)))
TEMPLATES_PATH = os.path.join(
os.path.dirname(__file__), 'templates')
DEFAULT_TEMPLATES_GEN_PATH = os.path.join(
os.path.dirname(__file__), 'policy_templates.json')
POLICY_DEFINITIONS_KEY = 'policy_definitions'
def _SubstituteSchemaRefNames(node, child_key, common_schema, parent_refs,
refs_seen):
'''Converts recursively objects with the key '$ref' into their actual schema.
'''
if (not isinstance(node, dict) or not child_key in node
or not isinstance(node[child_key], dict)):
return
if '$ref' in node[child_key]:
ref_name = node[child_key]['$ref']
# If the parent has the same id as child, leave the child's ref to avoid
# infinite loop.
if ref_name not in parent_refs:
node[child_key] = copy.deepcopy(common_schema[ref_name])
parent_refs.add(ref_name)
# If the schema has been seen already, only keep a reference to the first to
# avoid having the same ref defined at multiple places.
if ref_name not in refs_seen:
node[child_key]['id'] = ref_name
refs_seen.add(ref_name)
for ck in sorted(node[child_key].keys()):
# Copy parents ref so that parents are unique for each child branch and do
# not mix with sibling nodes.
_SubstituteSchemaRefNames(node[child_key], ck, common_schema,
parent_refs.copy(), refs_seen)
def _SubstituteSchemaRefs(policies, common_schema):
'''Converts objects with the key '$ref' into their actual schema.
Args:
policies: List of policies.
common_schema: Dictionary of schemas by their ref names.'''
policy_list = [policy for policy in policies if 'schema' in policy]
refs_seen = set()
for policy in sorted(policy_list, key=lambda policy: policy['id']):
parent_refs = set()
_SubstituteSchemaRefNames(policy, 'schema', common_schema, parent_refs,
refs_seen)
parent_refs = set()
_SubstituteSchemaRefNames(policy, 'validation_schema', common_schema,
parent_refs, refs_seen)
def _BuildPolicyTemplate(data):
'''
Converts data into the format of the old policy_templates.json so that it
can be used in policy_templates.grd
Schema : {
"policy_definitions": {
"type": "list",
"items": {
"id": { "type": "number" },
"name": "string",
"policies": { "type": "list", "items": "string" } // for policy groups.
// See components/policy/resources/new_policy_templates/policy.yaml
// for the other variables/
}
},
"messages": {
"type": "Object",
"patternProperties": {
"[A-Za-z]": { "desc": "string", "text": "String" }
}
},
//components/policy/resources/templates/manual_device_policy_proto_map.yaml
// Includes policies where generate_device_proto is true.
"device_policy_proto_map": {
"type": "Object",
"patternProperties": {
"[A-Za-z]": { "type": "Object", "properties": "String" }
}
},
// Only lists of 2 items
"legacy_device_policy_proto_map": { "type": "list", "items": "String" },
//components/policy/resources/templates/messages.yaml
"messages": {
"type": "Object",
"patternProperties": {
"[A-Za-z]": {
"type": "Object",
"properties": { "text": string, "description": "string" }
}
},
"risk_tag_definitions": {
"type": "list",
"items": {
"name": "string",
"description": "string",
"user-description": "string"
}
},
"policy_atomic_group_definitions": {
"type": "list",
"items": {
"id": { "type": "number" },
"name": "string",
"caption": "string",
"policies": { "type": "list", "items": "string" } // for policy groups.
// See components/policy/resources/new_policy_templates/policy.yaml
// for the other variables/
}
},
"placeholders": { "type": "list" },
"deleted_policy_ids": { "type": "list", "items": { "type": "number" } },
"deleted_atomic_policy_group_ids": {
"type": "list",
"items": { "type": "number" }
},
"highest_id_currently_used": { "type": "number" },
"highest_atomic_group_id_currently_used": { "type": "number" },
}
'''
policy_name_id = { name: id for id, name
in data['policies']['policies'].items() if name }
atomic_group_name_id = { name: id
for id, name in data['policies']['atomic_groups'].items() if name }
policy_groups = [{
'name': group_name,
'type': 'group',
'caption': group['caption'],
'desc': group['desc'],
'policies': list(group['policies'])
} for group_name, group in data[POLICY_DEFINITIONS_KEY].items()]
policies = []
atomic_groups = []
for group in data[POLICY_DEFINITIONS_KEY].values():
for policy_name, policy in group['policies'].items():
policies.append({
'id': policy_name_id[policy_name],
'name': policy_name, **policy
})
for name, atomic_group in group['policy_atomic_groups'].items():
atomic_groups.append({
'id': atomic_group_name_id[name],
'name': name, **atomic_group
})
device_policy_proto_map = data['manual_device_policy_proto_map'].copy()
for policy in policies:
if not policy.get('device_only', False):
continue
if not policy.get('generate_device_proto', True):
continue
device_policy_proto_map[policy['name']] = policy['name'] + '.value'
result = {
POLICY_DEFINITIONS_KEY: policies + policy_groups,
'deleted_policy_ids':
[id for id, name in data['policies']['policies'].items() if not name],
'highest_id_currently_used': len(data['policies']['policies']),
'policy_atomic_group_definitions': atomic_groups,
'deleted_atomic_policy_group_ids': [
id for id, name in data['policies']['atomic_groups'].items()
if not name
],
'highest_atomic_group_id_currently_used':
len(data['policies']['atomic_groups']),
'placeholders': [],
'legacy_device_policy_proto_map': [],
'device_policy_proto_map': device_policy_proto_map,
'messages': data['messages'],
'risk_tag_definitions': [{'name': name, **value}
for name, value in data['risk_tag_definitions'].items()]
}
for key, values in data['legacy_device_policy_proto_map'].items():
for item in values:
result['legacy_device_policy_proto_map'].append([key, item])
_SubstituteSchemaRefs(result[POLICY_DEFINITIONS_KEY], data['common_schemas'])
return result
def _GetMetadata():
'''Returns an object containing the policy metadata in order to build the
policy definition template.'''
result = {}
for file in _SafeListDir(TEMPLATES_PATH):
filename = os.fsdecode(file)
file_basename, file_extension = os.path.splitext(filename)
if not file_extension == ".yaml":
continue
with open(os.path.join(TEMPLATES_PATH, filename), encoding='utf-8') as f:
result[file_basename] = pyyaml.safe_load(f)
return result
def _GetPoliciesAndGroups():
'''Returns an object containing the policy groups with their details, policies
and atomic policy groups in order to build the policy definition template.
'''
result = {}
policy_definitions_path = os.path.join(TEMPLATES_PATH, POLICY_DEFINITIONS_KEY)
for group_name in _SafeListDir(policy_definitions_path):
result[group_name] = {'policies': {}, 'policy_atomic_groups': {}}
group_path = os.path.join(policy_definitions_path, group_name)
if not os.path.isdir(group_path):
continue
for file in _SafeListDir(group_path):
filename = os.fsdecode(file)
file_basename, file_extension = os.path.splitext(filename)
file_path = os.path.join(group_path, filename)
if file_extension != '.yaml':
continue
with open(file_path, encoding='utf-8') as f:
data = pyyaml.safe_load(f)
if file_basename == '.group.details':
result[group_name].update(data)
elif file_basename == 'policy_atomic_groups':
result[group_name]['policy_atomic_groups'].update(data)
else:
result[group_name]['policies'][file_basename] = data
return result
def _LoadPolicies():
'''
Loads all the yaml files used to define policies and their metadata into a
single object. This is a direct representation of the data structure of the
directories and their files.
Schema : {
//components/policy/resources/templates/policies.yaml
"policies":{
// Map of policy atomic group ID to policy atomic group name.
"atomic_groups": {
"type": "Object",
"patternProperties": {
"[A-Za-z]": { "type": "Object", "properties": "String" }
}
}
// Map of policy ID to policy name.
"policies": {
"type": "Object",
"patternProperties": {
"[A-Za-z]": { "type": "Object", "properties": "String" }
}
}
}
"policy_definitions": {
"type": "Object",
"patternProperties": {
// Dictionary of policy groups
"[A-Za-z]": {
"type": "Object",
// c/policy/resources/new_policy_templates/.groups.details.yaml
"properties": {
"caption": {"type": "string"},
"description": {"type": "string"}
},
"patternProperties": {
// Dictionary of policies
"[A-Za-z]": {
"type": "Object",
"properties": {
// c/policy/resources/new_policy_templates/policy.yaml
}
}
}
}
}
}
},
// components/policy/resources/templates/manual_device_policy_proto_map.yaml
"manual_device_policy_proto_map": {
"type": "Object",
"patternProperties": {
"[A-Za-z]": { "type": "Object", "properties": "String" }
}
},
// components/policy/resources/templates/legacy_device_policy_proto_map.yaml
"legacy_device_policy_proto_map": {
"type": "Object",
"patternProperties": {
"[A-Za-z]": { "type": "list", "items": "String" }
}
},
// components/policy/resources/templates/messages.yaml
"messages": {
"type": "Object",
"patternProperties": {
"[A-Za-z]": {
"type": "Object",
"properties": { "text": string, "description": "string" }
}
}
},
// components/policy/resources/templates/risk_tag_definitions.yaml
"risk_tag_definitions": {
"type": "Object",
"patternProperties": {
"[A-Za-z]": {
"type": "Object",
"properties": { "description": string, "user-description": "string" }
}
}
}
}
'''
return {
POLICY_DEFINITIONS_KEY: _GetPoliciesAndGroups(),
**_GetMetadata()
}
def GetPolicyTemplates():
'''Returns an object containing the policy templates.
'''
template = _LoadPolicies()
return _BuildPolicyTemplate(template)
def _WriteDepFile(dep_file, target, source_files):
'''Writes a dep file for `target` at `dep_file` with `source_files` as the
dependencies.
Args:
dep_file: A path to the dependencies file for this script.
target: The build target.
source_files: A list of the dependencies for the build target.
'''
with open(dep_file, "w") as f:
f.write(target)
f.write(": ")
f.write(' '.join(source_files))
def main():
'''Generates the a JSON file at `dest` with all the policy definitions.
If `dest` is not specified, a file name 'policy_templates.json' will be
generated in the same directory as the script.
Args:
dest: A path to the policy templates generated definitions.
depfile: A path to the dependencies file for this script.
'''
parser = argparse.ArgumentParser()
parser.add_argument('--dest', dest='dest')
parser.add_argument('--depfile', dest='deps_file')
args = parser.parse_args()
if args.dest:
path = os.path.join(args.dest)
else:
path = DEFAULT_TEMPLATES_GEN_PATH
policy_templates = GetPolicyTemplates()
with open(path, 'w+', encoding='utf-8') as dest:
json.dump(policy_templates, dest, indent=2, sort_keys=True)
files = sorted([f.replace('\\', '/')
for f in glob.glob(TEMPLATES_PATH + '/**/*.yaml', recursive=True)])
if args.deps_file:
_WriteDepFile(args.deps_file, args.dest, files)
if '__main__' == __name__:
sys.exit(main())