blob: d2da05709029e312fbeb414d6db19b661fcfdd9a [file] [log] [blame]
#!/usr/bin/env python
# pylint: disable=E0602,E1101,W0201
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Visualize and/or modify HWID and related component data."""
import copy
import logging
import os
import re
import sys
from itertools import chain
from random import shuffle
from string import uppercase # pylint: disable=W0402
from zlib import crc32
import factory_common # pylint: disable=W0611
from cros.factory.hwid.v2.bom_names import BOM_NAME_SET
from cros.factory.hwid.v2.yaml_datastore import InvalidDataError
from cros.factory.hwid.v2.yaml_datastore import MakeDatastoreClass
from cros.factory.hwid.v2.yaml_datastore import YamlDatastore
from cros.factory.utils.argparse_utils import CmdArg
from cros.factory.utils.argparse_utils import Command
from cros.factory.utils.argparse_utils import ParseCmdline
from cros.factory.utils.argparse_utils import verbosity_cmd_arg
from cros.factory.utils.debug_utils import SetupLogging
from cros.factory.utils import sys_utils
from cros.factory.utils.type_utils import Error
from cros.factory.utils.type_utils import Obj
# The expected location of HWID data within a factory image or the
# chroot.
'src', 'platform', 'chromeos-hwid', 'v2')
if sys_utils.InChroot()
else '/usr/local/factory/hwid')
# File that contains component data shared by all boards.
COMPONENT_DB_FILENAME = 'component_db'
# The size the board and bom name characters.
# Regular expression raw strings to match each of HWID component.
# Regular expresions to match just BOARD and BOM
BOARD_RE = re.compile(r'^(%s)$' % BOARD_RE_PATTERN)
BOM_RE = re.compile(r'^(%s)$' % BOM_RE_PATTERN)
BVV_GLOB_RE = re.compile(
r'^(%s)\s+(%s|\*)-(%s|\*)$' %
BBVV_GLOB_RE = re.compile(r'^(%s)\s+(%s|\*)\s+(%s|\*)-(%s|\*)$' % (
# HWID regexp.
HWID_RE = re.compile(
r'^(%s) (%s) (%s)-(%s) ([0-9]+)$' %
# Key to the concatenated probed string in probe results.
COMPACT_PROBE_STR = 'compact_str'
# Possible life cycle stages (status) for components and HWIDs.
MakeDatastoreClass('StatusData', dict(
(status_name, (list, str))
for status_name in LIFE_CYCLE_STAGES))
MakeDatastoreClass('ComponentRegistry', {
'probeable_components': (dict, (dict, str)),
'opaque_components': (dict, (list, str)),
'status': StatusData,
# To keep the database more human readable, the below components field
# contains a dict mapping component class to _either_ a single
# component name string for singletons _or_ a list of component name
# strings for when multiple components should exist for a single
# class. This will allow single line database entries for the many
# singleton cases, only expanding to multi-line lists when there are
# actually multiple entries to track.
MakeDatastoreClass('ComponentSpec', {
'classes_dontcare': (list, str),
'classes_missing': (list, str),
'components': (dict, [str, (list, str)]),
# ComponentData is the matching data for ComponentSpec -- in other
# words, ComponentSpec expresses an expectation (including, for
# example, dontcare), whereas ComponentData expresses an actual
# configuration.
MakeDatastoreClass('ComponentData', {
'classes_missing': (list, str),
'extant_components': (list, str),
MakeDatastoreClass('BomSpec', {
'primary': ComponentSpec,
'variants': (list, str),
MakeDatastoreClass('InitialConfigSpec', {
'constraints': (dict, str),
'enforced_for_boms': (list, str),
# TODO(tammo): Consider creating an explicit common-for-all-boms
# component-spec at the device level, to reduce the per-bom noise.
# The contents of this could be derived automatically.
MakeDatastoreClass('DeviceSpec', {
'boms': (dict, BomSpec),
'hwid_status': StatusData,
'initial_configs': (dict, InitialConfigSpec),
'variants': (dict, ComponentSpec),
'volatiles': (dict, (dict, str)),
'volatile_values': (dict, str),
'vpd_ro_fields': (list, str),
'vpd_rw_fields': (list, str),
MakeDatastoreClass('ProbeResults', {
'found_probe_value_map': (dict, [(dict, str),
(list, (dict, [str]))]),
'missing_component_classes': (list, str),
'found_volatile_values': (dict, [str, (dict, str)]),
'initial_configs': (dict, str),
def HwidChecksum(text):
return ('%04u' % (crc32(text) & 0xffffffffL))[-4:]
def ParseHwid(hwid):
"""Parse HWID string details. See the hwid spec for details."""
match = HWID_RE.findall(hwid)
if not match:
raise Error, ('illegal hwid %r, does not match ' % hwid +
board, bom, variant, volatile, checksum = match.pop()
expected_checksum = HwidChecksum(
'%s %s %s-%s' % (board, bom, variant, volatile))
if checksum != expected_checksum:
raise Error, 'bad checksum for hwid %r (expected %s)' % (
hwid, expected_checksum)
return Obj(hwid=hwid, board=board, bom=bom,
variant=variant, volatile=volatile)
def AlphaIndex(num):
"""Generate an alphabetic value corresponding to the input number.
Translate 0->A, 1->B, .. 25->Z, 26->AA, 27->AB, and so on.
result = ''
alpha_count = len(uppercase)
while True:
result = uppercase[num % alpha_count] + result
num /= alpha_count
if num == 0:
num -= 1
return result
def FmtRightAlignedDict(d):
max_key_width = max(len(k) for k in d) if d else 0
return ['%s%s: %s' % ((max_key_width - len(k)) * ' ', k, v)
for k, v in sorted((k, v) for k, v in d.items())]
def FmtLeftAlignedDict(d):
max_key_width = max(len(k) for k in d) if d else 0
return ['%s%s: %s' % (k, (max_key_width - len(k)) * ' ', v)
for k, v in sorted((k, v) for k, v in d.items())]
def ComponentSpecClasses(component_spec):
"""Return full set of component classes referenced anywhere in the spec."""
return (set(component_spec.classes_dontcare) |
set(component_spec.classes_missing) |
def ComponentSpecClassCompsMap(component_spec):
"""Return comp_class:[comp_name] dict.
Since the ComponentSpec allows for mappings from component class to
either lists or singletons, this routine generates a more useful
dict which converts all singletons to lists of one element. This
allows traversal of the datastructure without needing to inspect any
return dict(
(comp_class, comp_data if isinstance(comp_data, list) else [comp_data])
for comp_class, comp_data in component_spec.components.items())
def ComponentSpecCompClassMap(component_spec):
"""Return comp_name:comp_class dict, for lookup of component class by name."""
return dict(
(comp, comp_class)
for comp_class, comps in ComponentSpecClassCompsMap(component_spec).items()
for comp in comps)
def CombineComponentSpecs(a, b):
"""Return the union of two ComponentSpecs, ignoring any conflicts.
Creates a new ComponentSpec which is input A with B merged into it,
meaning that for conflicting component mappings (meaning different
mappings for a given component class), the result will contain the
mapping from B (the A mapping gets clobbered). For dontcare and
missing, the result contains the unions of the respective component
class sets.
components = {}
return ComponentSpec(
classes_dontcare=list(set(a.classes_dontcare) | set(b.classes_dontcare)),
classes_missing=list(set(a.classes_missing) | set(b.classes_missing)),
def ComponentSpecsConflict(a, b):
"""Determine if the specs refer to any overlapping classes.
In other words, determine if combining them would result in any
conflict (loss of information).
return (ComponentSpecClasses(a) & ComponentSpecClasses(b)) != set()
def ComponentSpecsEqual(a, b):
return ((set(a.classes_dontcare) == set(b.classes_dontcare)) and
(set(a.classes_missing) == set(b.classes_missing)) and
a.components == b.components)
class Validate(object): # pylint: disable=W0232
def HwidPart(cls, tag, name, matching_re):
if not matching_re.match(name):
raise Error, ('%s name %s does not match %s' % (
tag, name, matching_re.pattern))
def BoardName(cls, name):
cls.HwidPart('board', name, BOARD_RE)
def BomName(cls, name):
cls.HwidPart('bom', name, BOM_RE)
def Status(cls, status):
if not status in LIFE_CYCLE_STAGES:
raise Error, ('status must be one of {%s}, not %r' %
(', '.join(LIFE_CYCLE_STAGES), status))
def ComponentName(cls, name):
regexp = r'^([a-zA-Z0-9_]+)$'
if not re.match(regexp, name):
raise Error, ('component names must match the regexp %r, not %r' %
(regexp, name))
def VolatileName(cls, name):
regexp = r'^([a-zA-Z0-9_\./]+)$'
if not re.match(regexp, name):
raise Error, ('volatile result names must match the regexp %r, not %r' %
(regexp, name))
def InitialConfigContraintName(cls, name):
regexp = r'^([a-zA-Z0-9_]+)$'
if not re.match(regexp, name):
raise Error, ('initial config constraint names must match the '
'regexp %r, not %r' % (regexp, name))
def InitialConfigTag(cls, tag):
if not tag.isdigit():
raise Error, 'initial config tags must be digits, not %r' % tag
class CompDb(YamlDatastore):
def _BuildNameResultMap(self):
self.name_result_map = dict(
(comp_name, probe_result)
for comp_class, comp_map in self.probeable_components.items()
for comp_name, probe_result in comp_map.items())
def _BuildResultNameMap(self):
self.result_name_map = dict(
(probe_result, comp_name)
for comp_class, comp_map in self.probeable_components.items()
for comp_name, probe_result in comp_map.items())
def _BuildNameClassMaps(self):
self.name_class_map = {}
(comp_name, comp_class)
for comp_class, comps in self.opaque_components.items()
for comp_name in comps))
(comp_name, comp_class)
for comp_class, comp_map in self.probeable_components.items()
for comp_name in comp_map))
self.class_name_map = {}
for name, comp_class in self.name_class_map.items():
self.class_name_map.setdefault(comp_class, set()).add(name)
def _PreprocessData(self):
self.all_comp_classes = (set(self.opaque_components) |
self.all_comp_names = set(self.name_class_map)
self.opaque_comp_names = set(
for comp_class, comps in self.opaque_components.items()
for comp_name in comps)
def _EnforceProbeResultUniqueness(self):
if len(self.result_name_map) < len(self.name_result_map):
extra = set(self.name_result_map) - set(self.result_name_map.values())
raise Error, ('probe results are not all unique; '
'components [%s] are redundant' % ', '.join(extra))
def _EnforceCompNameUniqueness(self):
names = set()
overlap = set()
for comp_map in self.probeable_components.values():
for name in comp_map.values():
(names if name not in names else overlap).add(name)
for comps in self.opaque_components.values():
for name in comps:
(names if name not in names else overlap).add(name)
if overlap:
raise Error, ('component names [%s] are not unique' % ', '.join(overlap))
def EnforceInvariants(self):
def CompExists(self, comp):
if comp not in self.all_comp_names:
raise Error, 'unknown component named %r' % comp
def CompClassExists(self, comp_class):
if comp_class not in self.all_comp_classes:
raise Error, 'unknown component class %r' % comp_class
def AddComponent(self, comp_class, probe_result=None, comp_name=None):
if comp_name is None:
comp_count = len(self.class_name_map.get(comp_class, set()))
comp_name = '%s_%d' % (comp_class, comp_count)
assert comp_name not in self.name_class_map
if probe_result is not None:
comp_map = self.probeable_components.setdefault(comp_class, {})
comp_map[comp_name] = probe_result
self.opaque_components.setdefault(comp_class, []).append(comp_name)
return comp_name
def __init__(self, data): # pylint: disable=W0231
def CreateComponentSpec(self, components=None, dontcare=None, missing=None,
"""Build spec from a lists of component names and classes.
This properly builds the ComponentSpec.components mapping (see
schema documenation) -- create a singleton if there previously was
nothing, create a list of two elts if there was a singleton, and
otherwise just add to an existing list.
components = components if components is not None else []
dontcare = dontcare if dontcare is not None else []
missing = missing if missing is not None else []
comp_map = {}
for comp in components:
comp_class = self.name_class_map[comp]
if filter_component_classes and comp_class in filter_component_classes:
comp_data = comp_map.get(comp_class, None)
if comp_data is None:
comp_map[comp_class] = comp
elif isinstance(comp_data, list):
comp_map[comp_class] = [comp_data, comp]
if filter_component_classes:
missing = list(set(missing) - set(filter_component_classes))
dontcare = list(set(dontcare) - set(filter_component_classes))
class_conflict = set(dontcare) & set(missing) & set(comp_map)
if class_conflict:
raise Error, ('illegal component specification, conflicting data for '
'component classes: %s' % ', '.join(class_conflict))
return ComponentSpec(
def MatchComponentSpecWithData(self, component_spec, component_data,
"""Does the specification match the actual configuration data?
For all extant components check they either match exactly or are
specified as dontcare. For all actually missing components, check
they are specified as either missing or dontcare.
mismatches: If not None, will be populated with a list of reasons why
the component spec does not match.
True if the component spec matches.
if mismatches is None:
mismatches = []
spec_class_comps_map = ComponentSpecClassCompsMap(component_spec)
for comp_class in component_data.classes_missing:
if comp_class in component_spec.components:
mismatches.append('component class %s is missing' % comp_class)
comp_class_to_match = set(component_spec.components.keys())
comp_class_matched = set()
for comp in component_data.extant_components:
comp_class = self.name_class_map[comp]
if comp_class in component_spec.classes_missing:
mismatches.append('component class %s should be missing '
'but was detected as %s' % (comp_class, comp))
expected_comps = spec_class_comps_map.get(comp_class, None)
if expected_comps is not None and comp not in expected_comps:
mismatches.append('component class %s should be one of %s but was '
'detected as %s' % (comp_class, expected_comps, comp))
# add one item to comp_class_matched
comp_class_not_matched = comp_class_to_match - comp_class_matched
if len(comp_class_not_matched) != 0:
mismatches.append('these component classes did not match '
'the probed result: %s' %
', '.join(comp_class_not_matched))
return not mismatches
def ComponentDataClasses(self, component_data):
return (set(component_data.classes_missing) |
for comp in component_data.extant_components))
def ComponentDataIsComplete(self, component_data):
return self.ComponentDataClasses(component_data) == self.all_comp_classes
def MatchComponentProbeValues(self, found_probe_value_map):
"""Resolve component probe results into canonical names.
Returns the extant_components part of a ComponentData, along with
a dict of {comp_class:probe_value} for all of the unidentifiable
components from the input.
result = Obj(matched=[], unmatched={})
# Modify HWID v2 to look at probe.COMPACT_PROBE_STR field of probe results.
for probe_class, pr_data in found_probe_value_map.items():
if isinstance(pr_data, list):
probe_values = [pr[COMPACT_PROBE_STR] for pr in pr_data]
probe_values = [pr_data[COMPACT_PROBE_STR]]
for probe_value in probe_values:
component_name = self.result_name_map.get(probe_value, None)
if component_name is not None:
pr_list = result.unmatched.setdefault(probe_class, [])
return result
def Read(cls, path):
full_path = os.path.join(path, COMPONENT_DB_FILENAME)
if not os.path.isfile(full_path):
raise InvalidDataError, (
'ComponentDB not found (expected path is %r).' % full_path)
with open(full_path, 'r') as f:
return cls(ComponentRegistry.Decode(
def Write(self, path):
"""Write the component_db and all device data files."""
data = ComponentRegistry(**dict(
(field_name, getattr(self, field_name))
for field_name in ComponentRegistry.FieldNames()))
# Create copy (with known up to date internal data) to re-check invariants.
self.WriteOnDiff(path, COMPONENT_DB_FILENAME, data.Encode())
class CookedBoms(object):
def _BuildCompBomsMap(self):
"""Build dict of (component: bom name set) mappings.
Match each component with the set of boms containing it.
self.comp_boms_map = {}
for bom_name, bom in self.bom_map.items():
for comp in ComponentSpecCompClassMap(bom.primary):
self.comp_boms_map.setdefault(comp, set()).add(bom_name)
def _BuildCommonCompMap(self):
"""Return (comp_class: [comp]) dict for components common to all boms."""
self.comp_map = {}
for bom in self.bom_map.values():
for comp, comp_class in ComponentSpecCompClassMap(bom.primary).items():
if self.comp_boms_map[comp] == self.names:
self.comp_map.setdefault(comp_class, set()).add(comp)
def _BuildCommonComps(self):
self.common_comps = set()
for comps in self.comp_map.values():
self.common_comps |= set(comps)
def _BuildHierarchy(self):
self.hierarchy = []
def AddBom(bom_names):
(bom_name, self.bom_map[bom_name])
for bom_name in bom_names)))
uncommon_comp_boms_map = dict(
(comp, bom_names) for comp, bom_names in self.comp_boms_map.items()
if comp not in self.common_comps)
uncommon_bom_names = set().union(*uncommon_comp_boms_map.values())
if len(self.names) > 1:
for bom_name in self.names - uncommon_bom_names:
while uncommon_bom_names:
related_bom_sets = [
bom_subset & uncommon_bom_names
for bom_subset in uncommon_comp_boms_map.values()]
most_related = sorted([(len(rbs), rbs) for rbs in related_bom_sets],
uncommon_bom_names -= most_related
def __init__(self, bom_map):
self.bom_map = bom_map
self.names = set(bom_map)
class Device(YamlDatastore):
def _BuildReverseIcMap(self):
self.reverse_ic_map = {}
for index, data in self.initial_configs.items():
for bom_name in data.enforced_for_boms:
self.reverse_ic_map.setdefault(bom_name, set()).add(index)
def _BuildReverseVolValueMap(self):
self.reverse_vol_value_map = dict(
(v, k) for k, v in self.volatile_values.items())
def UpdateHwidStatusPatterns(self):
status_tree = dict((status, {}) for status in LIFE_CYCLE_STAGES)
for bom_name, var_status in self._hwid_status_map.items():
for var_code, vol_status in var_status.items():
for vol_code, status in vol_status.items():
bom_subtree = status_tree[status].setdefault(bom_name, {})
bom_subtree.setdefault(vol_code, set()).add(var_code)
for status in LIFE_CYCLE_STAGES:
patterns = set()
for bom_name, bom_subtree in status_tree[status].items():
star_var_vols = set(
vol_code for vol_code, var_set in bom_subtree.items()
if var_set == set(self.boms[bom_name].variants))
if star_var_vols == set(self.volatiles):
patterns.add('%s *-*' % bom_name)
for vol_code, var_codes in bom_subtree.items():
if vol_code in star_var_vols:
patterns.add('%s *-%s' % (bom_name, vol_code))
for var_code in var_codes:
patterns.add('%s %s-%s' % (bom_name, var_code, vol_code))
setattr(self.hwid_status, status, sorted(patterns))
def UpdateHwidStatusMaps(self, bom, variant, volatile, status):
target_boms = set([bom]) if bom != '*' else set(self.boms)
target_vols = set([volatile]) if volatile != '*' else set(self.volatiles)
for bom_name in target_boms:
var_status = self._hwid_status_map.setdefault(bom_name, {})
all_vars = set(self.boms[bom_name].variants)
target_vars = [variant] if variant != '*' else all_vars
for var_code in target_vars:
vol_status = var_status.setdefault(var_code, {})
for vol_code in target_vols:
prev_status = vol_status.get(vol_code, None)
if prev_status is not None:'hwid %s %s-%s status change %r -> %r',
bom_name, var_code, vol_code, prev_status, status)
vol_status[vol_code] = status
hwid = self.FmtHwid(bom_name, var_code, vol_code)
self.flat_hwid_status_map[hwid] = status
def _BuildHwidStatusMaps(self):
self._hwid_status_map = {}
self.flat_hwid_status_map = {}
status_globs = [(pattern, status)
for status in LIFE_CYCLE_STAGES
for pattern in getattr(self.hwid_status, status)]
for pattern, status in status_globs:
match = BVV_GLOB_RE.findall(pattern)
if not match:
raise Error, 'illegal hwid_status pattern %r' % pattern
bom, variant, volatile = match.pop()
self.UpdateHwidStatusMaps(bom, variant, volatile, status)
def _BuildClassSets(self):
self.primary_classes = set().union(*[
ComponentSpecClasses(bom.primary) for bom in self.boms.values()])
self.variant_classes = (self._comp_db.all_comp_classes -
def _PreprocessData(self):
self.cooked_boms = CookedBoms(self.boms)
def _EnforceVariantClassesAllMatch(self):
if not self.boms or not self.variants:
variant_classes = set().union(*[
ComponentSpecClasses(variant) for variant in self.variants.values()])
if self.variant_classes != variant_classes:
missing = self.variant_classes - variant_classes
extra = variant_classes - self.variant_classes
msg = ('%r primary and variant classes are incomplete; '
'primary + variant != all classes' % self.board_name)
msg += '; missing [%s]' % ', '.join(missing) if missing else ''
msg += '; extra [%s]' % ', '.join(extra) if extra else ''
raise Error, msg
for var_code, variant in self.variants.items():
if ComponentSpecClasses(variant) != variant_classes:
raise Error, ('%r variant classes are [%s]; variant %r does not match' %
(self.board_name, ', '.join(variant_classes), var_code))
def _EnforceCompClassesAllMatch(self):
"""Verify that all boms and variants have the same class coverage.
Class coverage for boms and variants must be the same to allow
arbitrary combinations between them. The set of variant classes
is implicitly the set of all possible classes minus those classes
used in bom primaries.
Also make sure that all classes are known, meaning that they occur
in the component_db.
for comp_class in self.primary_classes:
if comp_class not in self._comp_db.all_comp_classes:
raise Error, ('%s refers to unknown component class %r' % (
self.board_name, comp_class))
for bom_name, bom in self.boms.items():
if ComponentSpecClasses(bom.primary) != self.primary_classes:
raise Error, ('%s primary classes are [%s]; bom %r does not match' % (
self.board_name, ', '.join(self.primary_classes), bom_name))
for var_code, variant in self.variants.items():
if ComponentSpecClasses(variant) != self.variant_classes:
raise Error, ('%s variant classes are [%s]; variant %r does not match' %
(self.board_name, ', '.join(self.variant_classes),
def _EnforceVolatileUniqueness(self):
if len(self.volatile_values) < len(self.reverse_vol_value_map):
extra = (set(self.reverse_vol_value_map) -
raise Error, ('volatiles are not all unique; '
'volatiles [%s] are redundant' % ', '.join(extra))
def _EnforceInitialConfigsDontConflict(self):
for bom_name in self.boms:
combined_constraints = {}
for ic_tag in self.reverse_ic_map.get(bom_name, []):
for key, value in self.initial_configs[ic_tag].constraints.items():
existing_value = combined_constraints.get(key, None)
if existing_value is not None and existing_value != value:
raise Error, ('initial configs for bom %r conflict; enforced '
'tags specify more than one differing constraint '
'for %r' % (bom_name, key))
def EnforceInvariants(self):
# TODO(tammo): prevent hwid and contained component status conflicts
# TODO(tammo): for all status values, make sure the corresponding
# hwid exists ; that the bom has those variant and volatiles
# assigned
def BomExists(self, bom_name):
if bom_name not in self.boms:
raise Error, 'unknown bom %r for board %r' % (bom_name, self.board_name)
def VariantExists(self, var_code):
if var_code not in self.variants:
raise Error, ('unknown variant %r for board %r' %
(var_code, self.board_name))
def VolatileExists(self, vol_code):
if vol_code not in self.volatiles:
raise Error, ('unknown volatile %r for board %r' %
(vol_code, self.board_name))
def InitialConfigExists(self, ic_code):
if ic_code not in self.initial_configs:
raise Error, ('unknown initial_config %r for board %r' %
(ic_code, self.board_name))
def CommonInitialConfigs(self, target_bom_names):
"""Return all initial_config indices shared by the target boms."""
return set.intersection(*[
self.reverse_ic_map.get(bom_name, set())
for bom_name in target_bom_names]) if target_bom_names else set()
def CommonMissingClasses(self, target_bom_names):
return set.intersection(*[
for bom_name in target_bom_names]) if target_bom_names else set()
def CommonDontcareClasses(self, target_bom_names):
return set.intersection(*[
for bom_name in target_bom_names]) if target_bom_names else set()
def GetVolatileCodes(self, bom_name, variant_code, status_mask):
variant_status_map = self._hwid_status_map.get(bom_name, {})
volatile_status_map = variant_status_map.get(variant_code, {})
return set(volatile_code for volatile_code, status
in volatile_status_map.items()
if status in status_mask)
def GetInitialConfigConstraints(self, bom_name):
constraints = {}
for ic_code in self.reverse_ic_map.get(bom_name, []):
return constraints
def SetHwidStatus(self, bom, variant, volatile, status):
self.UpdateHwidStatusMaps(bom, variant, volatile, status)
def GetHwidStatus(self, bom_name, variant_code, volatile_code):
variant_status_map = self._hwid_status_map.get(bom_name, {})
volatile_status_map = variant_status_map.get(variant_code, {})
return volatile_status_map.get(volatile_code, None)
def AvailableBomNames(self, count):
"""Return count random bom names that are not yet used by board."""
existing_names = set(bom_name for bom_name in self.boms)
available_names = [bom_name for bom_name in BOM_NAME_SET
if bom_name not in existing_names]
if len(available_names) < count:
raise Error('too few available bom names (%d left)' %
return available_names[:count]
def CreateBom(self, bom_name, component_spec):
if bom_name in self.boms:
raise Error, '%s bom %s already exists' % (self.board_name, bom_name)
if self.boms:
existing_primary_classes = set().union(*[
ComponentSpecClasses(bom.primary) for bom in self.boms.values()])
new_primary_classes = ComponentSpecClasses(component_spec)
if new_primary_classes != existing_primary_classes:
msg = ('proposed bom has different component class '
'coverage than existing %s boms' % self.board_name)
missing = existing_primary_classes - new_primary_classes
if missing:
msg += ', missing [%s]' % ', '.join(sorted(missing))
extra = new_primary_classes - existing_primary_classes
if extra:
msg += ', extra [%s]' % ', '.join(sorted(extra))
raise Error, msg
bom_data = BomSpec(primary=component_spec, variants=[])
self.boms[bom_name] = bom_data
def CreateVariant(self, component_spec):
for existing_var_code, existing_variant in self.variants.items():
if component_spec.__dict__ == existing_variant.__dict__:
raise Error, ('%s equivalent variant %s already exists' %
(self.board_name, existing_var_code))
if self.variants:
variant_classes = set().union(*[
ComponentSpecClasses(variant) for variant in self.variants.values()])
if ComponentSpecClasses(component_spec) != variant_classes:
raise Error, ('proposed variant component data has different class '
'coverage than existing %s variants' % self.board_name)
var_code = AlphaIndex(len(self.variants))
self.variants[var_code] = component_spec
return var_code
def AddVolatileValue(self, vol_class, vol_value, vol_name=None):
if vol_name is None:
vol_name = '%s_%d' % (vol_class, len(self.volatile_values))
assert vol_name not in self.volatile_values
self.volatile_values[vol_name] = vol_value
return vol_name
def AddVolatile(self, spec):
vol_tag = AlphaIndex(len(self.volatiles))
self.volatiles[vol_tag] = spec
return vol_tag
def AddInitialConfig(self, constraints):
map(Validate.InitialConfigContraintName, constraints)
ic = InitialConfigSpec(constraints=constraints, enforced_for_boms=[])
ic_tag = str(len(self.initial_configs))
self.initial_configs[ic_tag] = ic
return ic_tag
def MatchVolatileValues(self, value_map):
result = Obj(
# Modify HWID v2 to look at probe.COMPACT_PROBE_STR field of probe results.
from cros.factory.gooftool import probe
for probe_class, pr_data in value_map.items():
probe_value = pr_data[probe.COMPACT_PROBE_STR]
volatile_name = self.reverse_vol_value_map.get(probe_value, None)
if volatile_name is not None:
result.matched_volatiles[probe_class] = volatile_name
result.unmatched_values[probe_class] = probe_value
result.matched_tags = sorted(
tag for tag, volatile in self.volatiles.items()
if (volatile == result.matched_volatiles
or not volatile))
return result
def MatchInitialConfigValues(self, value_map):
return sorted(
tag for tag, ic in self.initial_configs.items()
if ic.constraints == value_map)
def MatchBoms(self, component_data):
ret = set()
for bom_name, bom in self.boms.items():
mismatches = []
if self._comp_db.MatchComponentSpecWithData(bom.primary, component_data,
logging.debug('%s does not match: ', bom_name)
for m in mismatches:
logging.debug(' - %s', m)
return ret
def MatchVariants(self, bom_name, component_data):
matches = set()
bom = self.boms[bom_name]
for var_code, variant in self.variants.items():
if var_code not in bom.variants:
variant_spec = CombineComponentSpecs(bom.primary, variant)
if self._comp_db.MatchComponentSpecWithData(variant_spec, component_data):
return matches
def BuildMatchTree(self, component_data, volatile_tags=None):
"""Return nesting dicts with matches for component and volatile data.
Tree looks like {bom_name: {var_code: {vol_code: (hwid,
status)}}}, containing those boms that match the component_data,
then for those boms the variants that match the component_data,
and finally those volatiles that match the specified set of tags
and also have non-None status.
volatile_tags = volatile_tags if volatile_tags is not None else []
match_tree = dict((bom_name, {}) for bom_name in
for bom_name, variant_tree in match_tree.items():
matching_variants = self.MatchVariants(bom_name, component_data)
for var_code in matching_variants:
volatile_tree = variant_tree.setdefault(var_code, {})
for vol_tag in volatile_tags:
status = self.GetHwidStatus(bom_name, var_code, vol_tag)
if status is not None:
volatile_tree[vol_tag] = status
return match_tree
def GetMatchTreeHwids(self, match_tree):
"""Return a {hwid: status} dict built from a MatchTree."""
return dict(
(self.FmtHwid(bom_name, variant_code, volatile_code), status)
for bom_name, variant_tree in match_tree.items()
for variant_code, volatile_tree in variant_tree.items()
for volatile_code, status in volatile_tree.items())
def IntersectBomsAndInitialConfigs(self, initial_config_tags):
"""Return bom_name list for which specified initial_configs are enforced."""
return set(
bom_name for bom_name in self.boms
if set(self.reverse_ic_map.get(bom_name, [])) <= set(
def FmtHwid(self, bom, variant, volatile):
"""Generate HWID string. See the hwid spec for details."""
text = '%s %s %s-%s' % (self.board_name, bom, variant, volatile)
assert text.isupper(), 'HWID cannot have lower case text parts.'
return str(text + ' ' + HwidChecksum(text))
def __init__(self, comp_db, board_name, device_data): # pylint: disable=W0231
self._comp_db = comp_db
self.board_name = board_name
def Read(cls, path, comp_db, board_name):
full_path = os.path.join(path, board_name)
if not os.path.isfile(full_path):
raise InvalidDataError, 'path %r is not a board file' % full_path
with open(full_path, 'r') as f:
return cls(comp_db, board_name, DeviceSpec.Decode(
def Write(self, path):
device_data = DeviceSpec(**dict(
(field_name, getattr(self, field_name))
for field_name in DeviceSpec.FieldNames()))
# Create copy (with known up to date internal data) to re-check invariants.
Device(self._comp_db, self.board_name, device_data)
self.WriteOnDiff(path, self.board_name, device_data.Encode())
class HardwareDb(object):
def __init__(self, path):
"""Read the component_db and all device data files."""
self.path = path
self.comp_db = CompDb.Read(path)
self.devices = dict((entry, Device.Read(path, self.comp_db, entry))
for entry in os.listdir(path)
if entry.isalpha() and entry.isupper())
def CreateDevice(self, board_name):
if board_name in self.devices:
raise Error, ('board %r already exists' % board_name)
device = Device(self.comp_db, board_name, DeviceSpec.New())
self.devices[board_name] = device
return device
def GetDevice(self, board_name=None):
if board_name is None and len(self.devices) == 1:
return self.devices[self.devices.keys().pop()]
if board_name not in self.devices:
raise Error, ('board %r does not exist' % board_name)
return self.devices[board_name]
def Write(self):
"""Write the component_db and all device data files."""
for device in self.devices.values():
def PrintHwidHierarchy(device, cooked_boms, status_mask):
"""Hierarchically show all details for all specified BOMs.
Details include both primary and variant component configurations,
initial config, and status.
def FmtList(depth, l):
if len(l) == 1:
return str(list(l)[0])
elts = [((depth + 2) * ' ') + str(x) for x in sorted(l)]
return '\n' + '\n'.join(elts)
def ShowHwids(depth, bom_name):
bom = device.boms[bom_name]
for variant_code in sorted(bom.variants):
for volatile_code in sorted(device.GetVolatileCodes(
bom_name, variant_code, status_mask)):
variant = device.variants[variant_code]
hwid = device.FmtHwid(bom_name, variant_code, volatile_code)
status = device.GetHwidStatus(bom_name, variant_code, volatile_code)
print (depth * ' ') + '%s [%s]' % (hwid, status)
variant_comps = (
(comp_class, ', '.join(comps))
for comp_class, comps in
for line in FmtRightAlignedDict(variant_comps):
print (depth * ' ') + ' (variant) ' + line
extra_class_data = {'classes missing': variant.classes_missing,
'classes dontcare': variant.classes_dontcare}
extra_class_output = dict(
(k, FmtList(depth, v)) for k, v in extra_class_data.items() if v)
for line in FmtLeftAlignedDict(extra_class_output):
print (depth * ' ') + ' ' + line
print ''
def TraverseBomHierarchy(boms, depth, masks):
print (depth * ' ') + '-'.join(sorted(boms.names))
common_ic = device.CommonInitialConfigs(boms.names) - masks.ic
common_missing = device.CommonMissingClasses(boms.names) - masks.missing
common_wild = device.CommonDontcareClasses(boms.names) - masks.wild
common_data = {'initial_config': common_ic,
'classes missing': common_missing,
'classes dontcare': common_wild}
common_output = dict(
(k, FmtList(depth, v)) for k, v in common_data.items() if v)
for line in FmtLeftAlignedDict(common_output):
print (depth * ' ') + ' ' + line
common_present = dict(
(comp_class, ', '.join(x for x in (comps - masks.present)))
for comp_class, comps in boms.comp_map.items()
if comps - masks.present)
for line in FmtRightAlignedDict(common_present):
print (depth * ' ') + ' (primary) ' + line
print ''
if len(boms.names) == 1:
ShowHwids(depth + 1, list(boms.names)[0])
for sub_boms in boms.hierarchy:
depth + 1,
Obj(ic=masks.ic | common_ic,
missing=masks.missing | common_missing,
wild=masks.wild | common_wild,
present=masks.present | boms.common_comps))
Obj(ic=set(), present=set(), missing=set(), wild=set()))
# TODO(tammo): Add examples to the command line function docstrings.
def CreateBoard(config, hw_db):
"""Create empty device data file for specified board."""
'create_bom', CmdArg('-b', '--board', required=True),
'-c', '--comps', nargs='*', default=[],
help='list of component names'),
'-m', '--missing', nargs='*', default=[],
help='list of component classes, or "*"'),
'-d', '--dontcare', nargs='*', default=[],
help='list of component classes, or "*"'),
'--variant_classes', nargs='*', default=[],
help='list of component classes'),
'-n', '--name',
help='optional bom name; '
'will be automatically provided if not specified'))
def CreateBom(config, hw_db):
"""Create a new bom with specified components.
Either --missing or --dontcare can optionally be passed '*' to
indicate all components are either missing or always-matched.
The '*' wildcard will automatically cover all non-variant component
classes for the specified board. Correspondingly, if this is the
first bom for the board, then you likely also want to specify
--variant_classes to explicitly enunerate which classes will be
covered by variants (and should not be included in the missing or
dontcare sets).
If no name is specified for the new bom, a name will be
automatically provided from a pool of unused bom names.
device = hw_db.GetDevice(config.board)
map(hw_db.comp_db.CompExists, config.comps)
map(hw_db.comp_db.CompClassExists, config.variant_classes)
if config.variant_classes:
variant_classes = set(config.variant_classes)
elif not device.boms:
raise Error, 'variant classes must be declared for the first bom'
variant_classes = hw_db.comp_db.all_comp_classes - device.primary_classes
if config.missing == ['*'] and config.dontcare == ['*']:
raise Error, 'missing and dontcase cannot be simultaneously wildcarded (*)'
if config.missing == ['*']:
config.missing = hw_db.comp_db.all_comp_classes - variant_classes
map(hw_db.comp_db.CompClassExists, config.missing)
if config.dontcare == ['*']:
config.dontcare = hw_db.comp_db.all_comp_classes - variant_classes
map(hw_db.comp_db.CompClassExists, config.dontcare)
bom_name = if else device.AvailableBomNames(1)[0]
component_spec = hw_db.comp_db.CreateComponentSpec(
print 'creating %s bom %s' % (config.board, bom_name)
device.CreateBom(bom_name, component_spec)
'create_bom_matrix', CmdArg('-b', '--board', required=True),
'--cross_comps', nargs='*', default=[],
help='list of component names'),
'--fixed_comps', nargs='*', default=[],
help='list of component names'),
'-m', '--missing', nargs='*', default=[],
help='list of component classes'),
'-d', '--dontcare', nargs='*', default=[],
help='list of component classes'))
def CreateBomMatrix(config, hw_db):
"""Create all possible boms from the specified components.
Enough components need to be specified to avoid any ambiguity in
component configurations. Specifically, component class coverage
must be complete. To make this as easy as possible, this command
will assume that any component classes not specified on the command
line should match the bindings of existing boms. If there are no
existing boms, or if the existing boms do not all have exactly the
same component bindings for the classes in question, this will fail.
In other words, it is necessary to specify components on the command
line for all component classes that do not share common mappings
across all existing boms.
NOTE: This routine will only assign a single component per component
class, and hence is not useful for creating boms where more than one
component should be present for a single component class.
// Create all of the 18 boms possible with 3 cpus, 3 tpms, and 2 keyboards:
create_bom_matrix -b FOO --missing %s --cross_comps cpu_0 \
cpu_1 cpu_2 tpm_0 tpm_1 tpm_2 kbd_0 kbd_1
def DoCrossproduct(target_comps_list, accumulator=[]): # pylint: disable=W0102
return (list(chain.from_iterable(
[DoCrossproduct(target_comps_list[1:], accumulator + [comp])
for comp in target_comps_list[0]]))
if target_comps_list else [accumulator])
comp_db = hw_db.comp_db
device = hw_db.GetDevice(config.board)
map(comp_db.CompExists, config.cross_comps)
map(comp_db.CompExists, config.fixed_comps)
map(comp_db.CompClassExists, config.dontcare)
map(comp_db.CompClassExists, config.missing)
fixed_component_spec = comp_db.CreateComponentSpec(
common_component_spec = comp_db.CreateComponentSpec(
fixed_component_spec = CombineComponentSpecs(
fixed_component_spec, common_component_spec)
print 'fixed component spec:\n%s' % fixed_component_spec.Encode()
cross_component_spec = comp_db.CreateComponentSpec(
cross_class_comps_map = ComponentSpecClassCompsMap(cross_component_spec)
total_classes = (ComponentSpecClasses(fixed_component_spec) |
if total_classes != device.primary_classes:
raise Error, ('component specification insufficient, also need '
'specification for component classes: %s' %
', '.join(sorted(device.primary_classes - total_classes)))
crossproduct = DoCrossproduct(cross_class_comps_map.values())
target_component_specs = []
for comps in crossproduct:
component_spec = comp_db.CreateComponentSpec(components=comps)
component_spec = CombineComponentSpecs(fixed_component_spec, component_spec)
def Unique((bom_name, bom)):
if not ComponentSpecsEqual(component_spec, bom.primary):
return True
print 'existing bom matches one config: %s' % bom_name
if not all(map(Unique, device.boms.items())):
print 'creating %d new boms\n' % len(target_component_specs)
bom_names = device.AvailableBomNames(len(target_component_specs))
for bom_name, component_spec in zip(bom_names, target_component_specs):
print bom_name
print component_spec.Encode()
device.CreateBom(bom_name, component_spec)
'create_variant', CmdArg('-b', '--board', required=True),
'-c', '--comps', nargs='*', default=[],
help='list of component names'),
'-m', '--missing', nargs='*', default=[],
help='list of component classes'),
'-d', '--dontcare', nargs='*', default=[],
help='list of component classes'))
def CreateVariant(config, hw_db):
"""Create a new variant with specified components.
For the specified board, create a new variant from given compontent
// Create board FOO variant for the 'logitec_us_ext' keyboard.
create_variant -b FOO -c logitec_ex_ext
// Create an empty variant for board FOO.
create_variant -b FOO
// Create a variant that matches all possible keyboard for board FOO.
create_variant -b FOO --dontcare keyboard
device = hw_db.GetDevice(config.board)
map(hw_db.comp_db.CompExists, config.comps)
map(hw_db.comp_db.CompClassExists, config.missing)
map(hw_db.comp_db.CompClassExists, config.dontcare)
component_spec = hw_db.comp_db.CreateComponentSpec(
config.comps, config.dontcare, config.missing)
variant = device.CreateVariant(component_spec)
print 'created %s variant %s' % (config.board, variant)
CmdArg('-b', '--board', required=True),
CmdArg('--bom', required=True),
CmdArg('--variant', required=True))
# TODO(tammo): Make --bom into a list --boms and assign to all.
def AssignVariant(config, hw_db):
"""Associate variant with bom."""
device = hw_db.GetDevice(config.board)
bom = device.boms[]
if config.variant in bom.variants:
print '%s bom %s already uses variant %s' % (
config.board,, config.variant)
print 'added variant %s for %s bom %s' % (
config.board,, config.variant)
CmdArg('-b', '--board', required=True),
CmdArg('--bom', required=True),
CmdArg('--ic', required=True),
CmdArg('--cancel', action='store_true'))
def AssignInitialConfig(config, hw_db):
"""Start or cancel initial config enforcement.
Make sure that the specified initial_config is enforced for the
specified board-bom combination. Unless --cancel is specified, in
which case any matching enforcement is terminated.
device = hw_db.GetDevice(config.board)
ic = device.initial_configs[config.ic]
if config.cancel:
if not in ic.enforced_for_boms:
print 'initial config %s already not enforced for bom %s' % (
print 'not enforcing initial config %s for bom %s' % (
if in ic.enforced_for_boms:
print 'initial config %s already enforced for bom %s' % (
print 'enforcing initial config %s for bom %s' % (config.ic,
def SetHwidStatus(config, hw_db):
"""(Re)Assign status to HWIDs.
Set the status for a single HWID or for a group of HWIDs specified
using '*'-based glob expressions over BOM, VARIANT, and VOLATILE
fields. The '*' value implies all possible values for the
corresponding field.
For all of the affected HWIDs, their status will be reset as
specified. This will clobber any existing status, if any.
// This sets supported status for all variants of device FOO bom BAR
// with volatile code X.
set_hwid_status 'FOO BAR *-X' supported
// This sets 'eol' status just for 'FOO BAR A-B'
set_hwid_status 'FOO BAR A-B' eol
// This sets 'deprecated' status for all FOO boms and variants with
// volatile code C.
set_hwid_status 'FOO * *-C' deprecated
match = BBVV_GLOB_RE.findall(config.pattern)
if not match:
raise Error, ('illegal input pattern %r, expected '
'BOARD BOM VARIANT-VOLATILE' % config.pattern)
board, bom, variant, volatile = match.pop()
device = hw_db.GetDevice(board)
if bom != '*':
if variant != '*':
if volatile != '*':
if not device.boms:
raise Error, 'cannot assign status, %s has no BOMs' % device.board_name
if not device.variants:
raise Error, 'cannot assign status, %s has no variants' % device.board_name
if not device.volatiles:
raise Error, 'cannot assign status, %s has no volatiles' % device.board_name
device.SetHwidStatus(bom, variant, volatile, config.status)
CmdArg('-b', '--board', required=True),
CmdArg('--create_bom', nargs='?', default=False, metavar='BOM_NAME'))
def AssimilateProbeResults(config, hw_db):
"""Merge new data from stdin, optionally create new bom.
Any new data is added to the hardware database, including component
probe results, volatile result, and initial_config data. Canonical
names are automatically chosen for new data, which can be changed
later by renaming.
If a bom name is specified, and if a bom of that name does not
already exist, attempt to create it, and associate the components
specified by the input data. If there is already a bom with the
exact same component mappings, the request will fail.
Variant data that cannot be derived from the input data must be
added to the bom later using other commands.
Boms created using this command do not have any status, and hence
there is no binding made with any volatile or initial_config
properties (including any in the input data).
device = hw_db.GetDevice(config.board)
probe_results = ProbeResults.Decode(
for comp_class in (set(probe_results.found_probe_value_map) |
cooked_components = hw_db.comp_db.MatchComponentProbeValues(
for comp in cooked_components.matched:
print 'found matching %r component %r' % (
hw_db.comp_db.name_class_map[comp], comp)
for comp_class, comp_prs in cooked_components.unmatched.items():
for comp_probe_result in comp_prs:
comp_name = hw_db.comp_db.AddComponent(comp_class, comp_probe_result)
print 'added component/probe_result %r : %r' % (
comp_name, comp_probe_result)
cooked_volatiles = device.MatchVolatileValues(
for vol_class, vol_name in cooked_volatiles.matched_volatiles.items():
print 'found matching %r %r volatile %r' % (
device.board_name, vol_class, vol_name)
for vol_class, vol_value in cooked_volatiles.unmatched_values.items():
vol_name = device.AddVolatileValue(vol_class, vol_value)
print 'added volatile_value/probe_result %r : %r' % (
vol_name, vol_value)
cooked_initial_configs = device.MatchInitialConfigValues(
if cooked_initial_configs:
print 'matching initial config tags: %s' % ', '.join(cooked_initial_configs)
ic_tag = device.AddInitialConfig(probe_results.initial_configs)
print 'added initial config spec as tag %s' % ic_tag
# Cook components and volatiles again, to pick up new mappings.
recooked_components = hw_db.comp_db.MatchComponentProbeValues(
component_data = ComponentData(
recooked_volatiles = device.MatchVolatileValues(
if recooked_volatiles.matched_tags:
print 'matching volatile tags: %s' % ', '.join(
vol_tag = device.AddVolatile(recooked_volatiles.matched_volatiles)
print 'added volatile spec as tag %s' % vol_tag
match_tree = device.BuildMatchTree(
component_data, recooked_volatiles.matched_tags)
if match_tree:
is_complete = hw_db.comp_db.ComponentDataIsComplete(component_data)
print '%s matching boms: %s' % (
'exactly' if is_complete else 'partially', ', '.join(sorted(match_tree)))
if config.create_bom != False:
missing_classes = (
hw_db.comp_db.all_comp_classes - device.variant_classes -
if missing_classes:
print ('ignoring create_bom argument; component data missing [%s] classes'
% ', '.join(missing_classes))
component_spec = hw_db.comp_db.CreateComponentSpec(
for bom_name in match_tree:
bom = device.boms[bom_name]
if bom.primary == component_spec.components:
print ('ignoring create_bom argument; identical bom %r already exists' %
if config.create_bom in device.boms:
print ('bom %r exists, but component list differs from this data' %
bom_name = (config.create_bom if config.create_bom
else device.AvailableBomNames(1)[0])
print 'creating %s bom %s' % (config.board, bom_name)
device.CreateBom(bom_name, component_spec)
# TODO(tammo): add_component and set_component_status commands
CmdArg('--status', nargs='*'),
CmdArg('-b', '--board'))
def HwidHierarchyViewCommand(config, hw_db):
"""Show HWIDs in visually efficient hierarchical manner.
Starting with the set of all HWIDs for each board or a selected
board, show the set of common components and data values, then find
subsets of HWIDs with maximally shared data and repeat until there
are only singleton sets, at which point print the full HWID strings.
map(Validate.Status, config.status if config.status else [])
status_mask = config.status if config.status else LIFE_CYCLE_STAGES
for board, device in hw_db.devices.items():
if config.board:
if not config.board == board:
print '---- %s ----\n' % board
PrintHwidHierarchy(device, device.cooked_boms, status_mask)
CmdArg('-b', '--board'),
CmdArg('-s', '--status', nargs='*'),
CmdArg('-v', '--verbose', action='store_true',
help='show status in addition to the HWID string itself'))
def ListHwidsCommand(config, hw_db):
"""Print sorted list of existing HWIDs.
Optionally list HWIDs for specific status values (default is for all
HWIDs which have some kind of status to be shown). Optionally show
the status of each HWID. Optionally limit the list to a specific
status_mask = config.status if config.status else LIFE_CYCLE_STAGES
for board, device in hw_db.devices.items():
if config.board:
if not config.board == board:
filtered_hwid_status_map = dict(
(hwid, status) for hwid, status in device.flat_hwid_status_map.items()
if status in status_mask)
max_hwid_len = (max(len(x) for x in filtered_hwid_status_map)
if filtered_hwid_status_map else 0)
for hwid, status in sorted(filtered_hwid_status_map.items()):
if config.verbose:
print '%s%s [%s]' % (hwid, (max_hwid_len - len(hwid)) * ' ', status)
print hwid
CmdArg('-b', '--board'),
CmdArg('-s', '--status', nargs='*'))
def ListHwidsCSVCommand(config, hw_db):
"""Print sorted list of existing HWIDs as CSV format.
Optionally list HWIDs for specific status values (default is for all
HWIDs which have some kind of status to be shown).
status_mask = config.status if config.status else LIFE_CYCLE_STAGES
for board, device in hw_db.devices.iteritems():
if config.board:
if not config.board == board:
filtered_hwid_map = (
(hwid, status)
for hwid, status in device.flat_hwid_status_map.iteritems()
if status in status_mask))
# Prepare CSV header
header = ['hwid']
var_header = []
for var_code in device.variants:
var_header = sorted(device.variants[var_code].classes_missing +
# Because variant of BOMs are the same, so we generate each variant first.
# Components described in classes_missing and components, we add these
# components into tmp_dict first and sort tmp_dict to make output
# consistently.
var_comps_dict = {}
for var_code in device.variants:
var_comps_dict[var_code] = []
tmp_dict = copy.copy(device.variants[var_code].components)
# For missing class, shows 'None'
for comp_class in device.variants[var_code].classes_missing:
tmp_dict[comp_class] = 'None'
for comp_class, comps in sorted(tmp_dict.iteritems()):
# For each bom combination, output components composition
for bom_name, bom in sorted(device.boms.iteritems()):
# Output header at once, primary components first and then variant
# components
if len(header) == 1:
for comp_class, comps in sorted(bom.primary.components.iteritems()):
header += var_header
print ','.join(header)
for var_code in bom.variants:
for vol_code in device.volatiles:
hwid = device.FmtHwid(bom_name, var_code, vol_code)
if hwid in filtered_hwid_map:
comp_list = []
# Find out components for this hwid and add variant components
for comp_class, comps in sorted(bom.primary.components.iteritems()):
if isinstance(comps, list):
comp_list.append(' '.join(comps))
comp_list += var_comps_dict[var_code]
print '%s,%s' % (hwid, ','.join(comp_list))
CmdArg('-b', '--board'))
def ComponentBreakdownCommand(config, hw_db):
"""Map components to HWIDs, organized by component.
For all boards, or for a specified board, first show the set of
common components. For all the non-common components, show a list
of BOM names that use them.
for board, device in hw_db.devices.items():
if config.board:
if not config.board == board:
print '---- %s ----' % board
common_comp_map = dict(
(comp_class, ', '.join(comps))
for comp_class, comps in device.cooked_boms.comp_map.items())
if common_comp_map:
print '[common]'
for line in FmtRightAlignedDict(common_comp_map):
print ' ' + line
uncommon_comps = (set(device.cooked_boms.comp_boms_map) -
uncommon_comp_map = {}
for comp in uncommon_comps:
comp_class = hw_db.comp_db.name_class_map[comp]
bom_names = device.cooked_boms.comp_boms_map[comp]
comp_map = uncommon_comp_map.setdefault(comp_class, {})
comp_map[comp] = ', '.join(sorted(bom_names))
for comp_class, comp_map in uncommon_comp_map.items():
print comp_class + ':'
for line in FmtRightAlignedDict(comp_map):
print ' ' + line
CmdArg('-b', '--board', required=True),
CmdArg('-d', '--dest_dir'),
CmdArg('-s', '--status', nargs='*', default=['supported']))
def FilterDatabase(config, hw_db):
"""Filter board and component_db files based on status.
Generate a board data file containing only those boms matching the
specified status, and only that portion of the related board data
that is used by those boms. Also produce a component_db which
contains entries only for those components used by the selected
If dest_dir is not specified, a subdirectory of the current database
directory will automatically be created with the name
'filtered_db_BOARD', for the corresponding board name.
device = hw_db.GetDevice(config.board)
if not config.dest_dir:
config.dest_dir = os.path.join(hw_db.path, 'filtered_db_' + config.board)
if not os.path.isdir(config.dest_dir):
elif not os.path.isdir(config.dest_dir):
raise Error, ('target directory %r does not exist' % config.dest_dir)
map(Validate.Status, config.status)
target_status = {}
target_hwids = set()
target_boms = set()
target_variants = set()
target_volatiles = set()
for hwid, status in device.flat_hwid_status_map.items():
parsed_hwid = ParseHwid(hwid)
if status not in config.status:
target_status.setdefault(status, []).append(parsed_hwid)
target_components = (
set(comp for comp, boms in device.cooked_boms.comp_boms_map.items()
if target_boms & boms) |
for var_code in target_variants
for comp in ComponentSpecCompClassMap(device.variants[var_code])))
target_volatile_names = set(
for vol_code in target_volatiles
for vol_name in device.volatiles[vol_code].values())
comp_db = hw_db.comp_db
filtered_comp_db = CompDb(ComponentRegistry(
(comp_class, dict(
(comp_name, probe_result)
for comp_name, probe_result in comp_map.items()
if comp_name in target_components))
for comp_class, comp_map in comp_db.probeable_components.items()),
(comp_class, [comp_name for comp_name in comps
if comp_name in target_components])
for comp_class, comps in comp_db.opaque_components.items()),
(status, [comp_name for comp_name in getattr(comp_db.status, status)
if comp_name in target_components])
for status in LIFE_CYCLE_STAGES))))
filtered_device = Device(filtered_comp_db, config.board, DeviceSpec(
boms=dict((bom_name, bom) for bom_name, bom in device.boms.items()
if bom_name in target_boms),
(status, ['%s %s-%s' % (, hwid.variant, hwid.volatile)
for hwid in target_status.get(status, set())])
for status in LIFE_CYCLE_STAGES)),
(ic_tag, InitialConfigSpec(
enforced_for_boms=list(set(ic.enforced_for_boms) & target_boms)))
for ic_tag, ic in device.initial_configs.items()
if set(ic.enforced_for_boms) & target_boms),
variants=dict((var_code, device.variants[var_code])
for var_code in target_variants),
volatiles=dict((vol_code, vol_spec)
for vol_code, vol_spec in device.volatiles.items()
if vol_code in target_volatiles),
volatile_values=dict((vol_name, device.volatile_values[vol_name])
for vol_name in target_volatile_names),
filtered_hw_db = HardwareDb(config.dest_dir)
filtered_hw_db.devices[config.board] = filtered_device
# TODO(tammo): If someone is using this, make it work; otherwise delete.
# @Command('legacy_export',
# CmdArg('-b', '--board', required=True),
# CmdArg('-d', '--dest_dir', required=True),
# CmdArg('-e', '--extra'),
# CmdArg('-s', '--status', default='supported'))
def LegacyExport(config, data):
"""Generate legacy-format 'components_<HWID>' files.
For the specified board, in the specified destination directory,
this will create a hash.db file and one file per HWID. All of these
files should be compatible with the pre-hwid-tool era code.
The goal of this command is to enable maintaining data in the new
format for use with factory branches that can only consume the older
The 'extra' argument can specify a file that contains extra dict
extries that will be included in each of the hwid files. This is
useful for specifying the legacy fields that no longer exist in the
new data format.
This command will be removed once we are no longer supporting any
boards that depend on the old-style data formatting.
from pprint import pprint # pylint: disable=W0404
if config.board not in data.devices:
print 'ERROR: unknown board %r.' % config.board
if not os.path.exists(config.dest_dir):
print 'ERROR: destination directory %r does not exist.' % config.dest_dir
extra_fields = eval(open(config.extra).read()) if config.extra else None
device = data.devices[config.board]
hash_db_path = os.path.join(config.dest_dir, 'hash.db')
with open(hash_db_path, 'w') as f:
pprint(device.volatile_value_map, f)
ic_reverse_map = {}
for ic_index, bom_list in device.initial_config_use_map.items():
for bom in bom_list:
ic_reverse_map[bom] = ic_index
def WriteLegacyHwidFile(bom, volind, variant, hwid):
hwid_str = device.FmtHwid(bom, volind, variant)
export_data = {'part_id_hwqual': [hwid_str]}
for comp_class, comp_name in hwid.component_map.items():
if comp_name == 'NONE':
probe_result = ''
elif comp_name == 'ANY':
probe_result = '*'
probe_result = data.comp_db.registry[comp_class][comp_name]
export_data['part_id_' + comp_class] = [probe_result]
for vol_class, vol_name in device.volatile_map[volind].items():
export_data['hash_' + vol_class] = [vol_name]
variant_data = device.variant_map[variant]
if len(variant_data) not in [0, 1]:
print ('ERROR: legacy_export expects zero or one variants, '
'hwid %s has %d.' % (hwid_str, len(variant_data)))
for variant_value in variant_data:
export_data['part_id_keyboard'] = [
initial_config = device.initial_config_map[ic_reverse_map[bom]]
for ic_class, ic_value in initial_config.items():
export_data['version_' + ic_class] = [ic_value]
export_data['config_factory_initial'] = sorted(
'version_' + ic_class for ic_class in initial_config)
hwid_file_name = ('components ' + hwid_str).replace(' ', '_')
hwid_file_path = os.path.join(config.dest_dir, hwid_file_name)
with open(hwid_file_path, 'w') as f:
pprint(export_data, f)
for bom, hwid in device.hwid_map.items():
for volind in device.volatile_map:
for variant in hwid.variant_list:
status = device.GetHwidStatus(bom, volind, variant)
if (config.status != '' and
(status is None or config.status != status)):
WriteLegacyHwidFile(bom, volind, variant, hwid)
def RenameComponents(config, hw_db): # pylint: disable=W0613
"""Change canonical component names.
Given a list of old-new name pairs on stdin, replace each instance
of each old name with the corresponding new name in the
component_db and in all board files. The expected stdin format is
one pair per line, and the two words in each pair are whitespace
pairs = [line.strip().split() for line in sys.stdin]
comp_db = hw_db.comp_db
for pair in pairs:
if len(pair) != 2:
raise Error, ('each line of input must have exactly 2 words, '
'found line with %r' % pair)
old_name, new_name = pair
for old_name, new_name in pairs:
comp_class = comp_db.name_class_map[old_name]
if old_name in comp_db.opaque_comp_names:
comps = comp_db.opaque_components[comp_class]
comp_map = comp_db.probeable_components[comp_class]
comp_map[new_name] = comp_map.pop(old_name)
def UpdateComponentSpec(spec):
for comp_class, comp_data in spec.components.items():
if isinstance(comp_data, list) and old_name in comp_data:
elif comp_data == old_name:
spec.components[comp_class] = new_name
for device in hw_db.devices.values():
for bom in device.boms.values():
for variant in device.variants.values():
def Main():
"""Run sub-command specified by the command line args."""
config = ParseCmdline(
'Visualize and/or modify HWID and related component data.',
CmdArg('-p', '--data_path', metavar='PATH',
CmdArg('-l', '--log', metavar='PATH',
help='Write logs to this file.'),
SetupLogging(config.verbosity, config.log)
hw_db = HardwareDb(config.data_path)
config.command(config, hw_db)
except Error, e:
sys.exit('ERROR: %s' % e)
except Exception, e:
if __name__ == '__main__':