| # Copyright 2018 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Helpers for dealing with translation files.""" |
| |
| import ast |
| import os |
| import re |
| import xml.etree.cElementTree as ElementTree |
| |
| |
| class GRDFile(object): |
| """Class representing a grd xml file. |
| |
| Attributes: |
| path: the path to the grd file. |
| dir: the path to the the grd's parent directery. |
| name: the base name of the grd file. |
| grdp_paths: the list of grdp files included in the grd via <part>. |
| structure_paths: the paths of any <structure> elements in the grd file. |
| xtb_paths: the xtb paths where the grd's translations live. |
| lang_to_xtb_path: maps each language to the xtb path for that language. |
| appears_translatable: whether the contents of the grd indicate that it's |
| supposed to be translated. |
| expected_languages: the languages that this grd is expected to have |
| translations for, based on the translation expectations file. |
| """ |
| |
| def __init__(self, path): |
| self.path = path |
| self.dir, self.name = os.path.split(path) |
| dom, self.grdp_paths = _parse_grd_file(path) |
| self.structure_paths = [os.path.join(self.dir, s.get('file')) |
| for s in dom.findall('.//structure')] |
| self.xtb_paths = [os.path.join(self.dir, f.get('path')) |
| for f in dom.findall('.//file')] |
| self.lang_to_xtb_path = {} |
| self.appears_translatable = (len(self.xtb_paths) != 0 or |
| dom.find('.//message') is not None) |
| self.expected_languages = None |
| |
| def _populate_lang_to_xtb_path(self, errors): |
| """Populates the lang_to_xtb_path attribute.""" |
| grd_root = os.path.splitext(self.name)[0] |
| lang_pattern = re.compile(r'%s_([^_]+)\.xtb$' % re.escape(grd_root)) |
| for xtb_path in self.xtb_paths: |
| xtb_basename = os.path.basename(xtb_path) |
| xtb_lang_match = re.match(lang_pattern, xtb_basename) |
| if not xtb_lang_match: |
| errors.append('%s: invalid xtb name: %s. xtb name must be %s_<lang>' |
| '.xtb where <lang> is the language code.' % |
| (self.name, xtb_basename, grd_root)) |
| continue |
| xtb_lang = xtb_lang_match.group(1) |
| if xtb_lang in self.lang_to_xtb_path: |
| errors.append('%s: %s is listed twice' % (self.name, xtb_basename)) |
| continue |
| self.lang_to_xtb_path[xtb_lang] = xtb_path |
| |
| return errors |
| |
| |
| def get_translatable_grds(repo_root, all_grd_paths, |
| translation_expectations_path): |
| """Returns all the grds that should be translated as a list of GRDFiles. |
| |
| This verifies that every grd file that appears translatable is listed in |
| the translation expectations, and that every grd in the translation |
| expectations actually exists. |
| |
| Args: |
| repo_root: The path to the root of the repository. |
| all_grd_paths: All grd paths in the repository relative to repo_root. |
| translation_expectations_path: The path to the translation expectations |
| file, which specifies which grds to translate and into which languages. |
| """ |
| grd_to_langs, untranslated_grds = _parse_translation_expectations( |
| translation_expectations_path) |
| |
| # Check that every grd that appears translatable is listed in |
| # the translation expectations. |
| grds_with_expectations = set(grd_to_langs.keys()).union(untranslated_grds) |
| all_grds = {p: GRDFile(os.path.join(repo_root, p)) for p in all_grd_paths} |
| errors = [] |
| for path, grd in all_grds.iteritems(): |
| if grd.appears_translatable: |
| if path not in grds_with_expectations: |
| errors.append('%s appears to be translatable (because it contains ' |
| '<file> or <message> elements), but is not listed in the ' |
| 'translation expectations.' % path) |
| |
| # Check that every file in translation_expectations exists. |
| for path in grds_with_expectations: |
| if path not in all_grd_paths: |
| errors.append('%s is listed in the translation expectations, but this ' |
| 'grd file does not exist.' % path) |
| |
| if errors: |
| raise Exception('%s needs to be updated. Please fix these issues:\n - %s' % |
| (translation_expectations_path, '\n - '.join(errors))) |
| |
| translatable_grds = [] |
| for path, expected_languages_list in grd_to_langs.iteritems(): |
| grd = all_grds[path] |
| grd.expected_languages = expected_languages_list |
| grd._populate_lang_to_xtb_path(errors) |
| translatable_grds.append(grd) |
| |
| # Ensure each grd lists the expected languages. |
| expected_languages = set(expected_languages_list) |
| actual_languages = set(grd.lang_to_xtb_path.keys()) |
| if expected_languages.difference(actual_languages): |
| errors.append('%s: missing translations for these languages: %s. Add ' |
| '<file> and <output> elements to the grd file, or update ' |
| 'the translation expectations.' % (grd.name, |
| sorted(expected_languages.difference(actual_languages)))) |
| if actual_languages.difference(expected_languages): |
| errors.append('%s: references translations for unexpected languages: %s. ' |
| 'Remove the offending <file> and <output> elements from the' |
| ' grd file, or update the translation expectations.' |
| % (grd.name, |
| sorted(actual_languages.difference(expected_languages)))) |
| |
| if errors: |
| raise Exception('Please fix these issues:\n - %s' % |
| ('\n - '.join(errors))) |
| |
| return translatable_grds |
| |
| |
| def _parse_grd_file(grd_path): |
| """Reads a grd(p) file and any subfiles included via <part file="..." />. |
| |
| Args: |
| grd_path: The path of the .grd or .grdp file. |
| Returns: |
| A tuple (grd_dom, grdp_paths). dom is an ElementTree DOM for the grd file, |
| with the <part> elements inlined. grdp_paths is the list of grdp files that |
| were included via <part> elements. |
| """ |
| grdp_paths = [] |
| grd_dom = ElementTree.parse(grd_path) |
| # We modify grd in the loop, so listify this iterable to be safe. |
| part_nodes = list(grd_dom.findall('.//part')) |
| for part_node in part_nodes: |
| grdp_rel_path = part_node.get('file') |
| grdp_path = os.path.join(os.path.dirname(grd_path), grdp_rel_path) |
| grdp_paths.append(grdp_path) |
| grdp_dom, grdp_grpd_paths = _parse_grd_file(grdp_path) |
| grdp_paths.extend(grdp_grpd_paths) |
| part_node.append(grdp_dom.getroot()) |
| return grd_dom, grdp_paths |
| |
| |
| def _parse_translation_expectations(path): |
| """Parses a translations expectations file. |
| |
| Example translations expectations file: |
| { |
| "desktop_grds": { |
| "languages": ["es", "fr"], |
| "files": [ |
| "ash/ash_strings.grd", |
| "ui/strings/ui_strings.grd", |
| ], |
| }, |
| "android_grds": { |
| "languages": ["de", "pt-BR"], |
| "files": [ |
| "chrome/android/android_chrome_strings.grd", |
| }, |
| "untranslated_grds": { |
| "chrome/locale_settings.grd": "Not UI strings; localized separately", |
| "chrome/locale_settings_mac.grd": "Not UI strings; localized separately", |
| }, |
| } |
| |
| Returns: |
| A tuple (grd_to_langs, untranslated_grds). grd_to_langs maps each grd path |
| to the list of languages into which that grd should be translated. |
| untranslated_grds is a list of grds that "appear translatable" but should |
| not be translated. |
| """ |
| with open(path) as f: |
| file_contents = f.read() |
| |
| def assert_list_of_strings(l, name): |
| assert isinstance(l, list) and all(isinstance(s, basestring) for s in l), ( |
| '%s must be a list of strings' % name) |
| |
| try: |
| translations_expectations = ast.literal_eval(file_contents) |
| assert isinstance(translations_expectations, dict), ( |
| '%s must be a python dict' % path) |
| |
| grd_to_langs = {} |
| untranslated_grds = [] |
| |
| for group_name, settings in translations_expectations.iteritems(): |
| if group_name == 'untranslated_grds': |
| untranslated_grds = list(settings.keys()) |
| assert_list_of_strings(untranslated_grds, 'untranslated_grds') |
| continue |
| |
| languages = settings['languages'] |
| files = settings['files'] |
| assert_list_of_strings(languages, group_name + '.languages') |
| assert_list_of_strings(files, group_name + '.files') |
| for grd in files: |
| grd_to_langs[grd] = languages |
| |
| return grd_to_langs, untranslated_grds |
| |
| except Exception: |
| print 'Error: failed to parse', path |
| raise |
