blob: 18f3858de71376e88ed8dc25bd8a36283c2e75b5 [file] [log] [blame]
# Copyright 2016 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# This Makefile provides different targets:
# - closure: Build static resources in Closure (js, css)
# - par: Archived python files, with minimal resources.
# - toolkit: Installer for factory test DUT software.
# - overlord: Overlord the remote monitoring daemon.
# - doc: HTML documentation.
# - bundle: everything for deployment, including doc, toolkit, setup, ... etc.
# Some targets, including 'par' and 'toolkit', are using a 'resource system'.
# Source files from factory repo and factory-board/files were collected
# when building the 'resource' target. Other portage packages can also put what
# should be added into toolkit and par into BOARD_RESOURCES_DIR
# (/build/$BOARD/var/lib/factory/resources) - for example files downloaded from
# CPFE or localmirror.
#
# Resource files should be named as TARGET-NAME.tar. For example,
# 'toolkit-webgl.tar' refers to a webgl resource only for toolkit (not par),
# 'par-shopfloor.tar' refers to a shopfloor resource only for PAR (not toolkit),
# 'resource-l10n.tar' refers to a resource that toolkit and PAR will all have.
# For debug and testing purposes, the Makefile also supports other virtual
# targets like presubmit-*, lint, test, overlay-* ... etc.
# Local environment settings
MK_DIR := devtools/mk
include $(MK_DIR)/common.mk
BUILD_DIR ?= build
RESOURCE_DIR ?= $(BUILD_DIR)/resource
RESOURCE_PATH ?= $(RESOURCE_DIR)/factory.tar
RESOURCE_SRC_DIR ?= resources
BUNDLE_DIR ?= \
$(if $(DESTDIR),$(DESTDIR)/$(TARGET_DIR)/bundle,$(BUILD_DIR)/bundle)
TEMP_DIR ?= $(BUILD_DIR)/tmp
# Global environment settings
SHELL := bash
OUTOFTREE_BUILD ?=
PYTHON ?= python2
TARGET_DIR = /usr/local/factory
# Build and board config settings
STATIC ?= false
BOARD_PACKAGE_NAME ?= $(notdir $(realpath $(dir $(BOARD_EBUILD))))
BOARD_PACKAGE_FILE ?= \
$(if $(BOARD_EBUILD),$(SYSROOT)/packages/chromeos-base/$(basename $(notdir \
$(BOARD_EBUILD))).tbz2)
BOARD_RESOURCES_DIR ?= $(SYSROOT)/var/lib/factory/resources
BOARD_TARGET_DIR ?= $(SYSROOT)$(TARGET_DIR)
SYSROOT ?= $(if $(BOARD),/build/$(BOARD),/)
# The SETUP_BIN is for setup/ in factory bundle, for setting up and preparing
# images that partners usually run on an x86_64 server (same environment running
# cros_sdk or chroot). So this is / instead of SYSROOT / ROOT.
SETUP_BIN_ROOT ?= /
PAR_TEMP_DIR = $(TEMP_DIR)/par
PAR_OUTPUT_DIR = $(BUILD_DIR)/par
PAR_NAME = factory.par
SETUP_PAR_NAME = setup_tools.par
SETUP_PAR_MODULES = $(shell cat $(MK_DIR)/setup_tools_modules.lst)
TOOLKIT_VERSION ?= $(shell $(MK_DIR)/toolkit_version.sh)
TOOLKIT_FILENAME ?= install_factory_toolkit.run
TOOLKIT_TEMP_DIR = $(TEMP_DIR)/toolkit
TOOLKIT_OUTPUT_DIR = $(BUILD_DIR)
DOC_TEMP_DIR = $(TEMP_DIR)/docsrc
DOC_ARCHIVE_PATH = $(BUILD_DIR)/doc.zip
DOC_OUTPUT_DIR = $(BUILD_DIR)/doc
DOC_PUBLISH_URL = gs://chromeos-factory-docs/sdk
CLOSURE_DIR = py/goofy/static
CLOSURE_OUTPUT_FILENAMES = js/goofy.js css/closure.css
CLOSURE_OUTPUT_DIR ?= \
$(abspath $(if $(OUTOFTREE_BUILD),$(BUILD_DIR)/closure,$(CLOSURE_DIR)))
CROS_REGIONS_DATABASE ?= $(SYSROOT)/usr/share/misc/cros-regions.json
# External dependency.
OVERLORD_DEPS_URL ?= \
gs://chromeos-localmirror/distfiles/overlord-deps-0.0.3.tar.gz
OVERLORD_DEPS_DIR ?= $(BUILD_DIR)/dist/go
# This must match chromeos-base/factory/factory-9999.ebuild.
WEBGL_AQUARIUM_URI ?= \
gs://chromeos-localmirror/distfiles/webgl-aquarium-20130524.tar.bz2
WEBGL_AQUARIUM_DIR ?= $(BUILD_DIR)/dist/webgl_aquarium_static
# Following versions must match dev-libs/closure-library/*.ebuild.
CLOSURE_LIB_GITREV ?= 7744b75c637054ed38b14901d091ec3196b2ef9b
CLOSURE_LIB_URL ?= \
gs://chromeos-localmirror/distfiles/closure-library-201702.tar.gz
CLOSURE_LIB_DIR ?= $(BUILD_DIR)/dist/closure-library-$(CLOSURE_LIB_GITREV)
# TODO(hungte) Remove the ifdef when chromeos-base/factory ebuild has changed to
# always specify CLOSURE_LIB_DIR.
ifdef CROS_WORKON_SRCROOT
CLOSURE_LIB_DIR = /opt/closure-library
endif
LINT_BLACKLIST=$(shell cat $(MK_DIR)/pylint.blacklist | grep -v '^\#')
LINT_FILES=$(shell find py go po devtools -name '*.py' -type f | sort)
LINT_WHITELIST=$(filter-out $(LINT_BLACKLIST),$(wildcard $(LINT_FILES)))
UNITTESTS=$(shell find py go po -name '*_unittest.py' | sort)
UNITTESTS_BLACKLIST=$(shell cat $(MK_DIR)/unittests.blacklist)
UNITTESTS_WHITELIST= \
$(filter-out $(UNITTESTS_BLACKLIST),$(wildcard $(UNITTESTS)))
TEST_EXTRA_FLAGS=
# Substitute PRESUBMIT_FILES to relative path (similar to
# GNU realpath "--relative-to=.", but works on non-GNU realpath).
PRESUBMIT_FILES := \
$(if $(PRESUBMIT_FILES), \
$(shell realpath $$PRESUBMIT_FILES | sed "s'^$$(realpath $$(pwd))/''g"))
PRESUBMIT_TARGETS := \
presubmit-deps presubmit-lint presubmit-shebang presubmit-test
# Virtual targets. The '.phony' is a special hack to allow making targets with
# wildchar (for instance, overlay-%) to be treated as .PHONY.
.PHONY: \
.phony default clean closure proto overlord ovl-bin par doc resource toolkit \
bundle presubmit presubmit-chroot $(PRESUBMIT_TARGETS) \
lint smartlint smart_lint test testall overlay check-board-resources \
publish-docs po
# This must be the first rule.
default: closure
clean:
$(MAKE) -C $(CLOSURE_DIR) OUTPUT_DIR=$(CLOSURE_OUTPUT_DIR) $@
rm -rf $(RESOURCE_DIR) $(TEMP_DIR) $(BUILD_DIR) $(BUNDLE_DIR)
# Currently the only programs using Closure is in Goofy.
closure: $(CLOSURE_LIB_DIR)
$(MAKE) -C $(CLOSURE_DIR) OUTPUT_DIR=$(CLOSURE_OUTPUT_DIR) \
CLOSURE_LIB_DIR=$(realpath $(CLOSURE_LIB_DIR))
# Regenerates the reg code proto.
# TODO(jsalz): Integrate this as a "real" part of the build, rather than
# relying on regenerating it only if/when it changes. This is OK for now since
# this proto should change infrequently or never.
proto:
protoc proto/reg_code.proto --python_out=py
func-extract-from-url = @\
mkdir -p $(1) ;\
gsutil cp $(2) $(1)/. ;\
tar -xf $(1)/$(notdir $(2)) -C $(1) ; \
test -d $@
$(OVERLORD_DEPS_DIR):
$(call func-extract-from-url,$(dir $@),$(OVERLORD_DEPS_URL))
$(WEBGL_AQUARIUM_DIR):
$(call func-extract-from-url,$(dir $@),$(WEBGL_AQUARIUM_URI))
$(CLOSURE_LIB_DIR):
$(call func-extract-from-url,$(dir $@),$(CLOSURE_LIB_URL))
# TODO(hungte) Change overlord to build out-of-tree.
overlord: $(OVERLORD_DEPS_DIR)
$(MAKE) -C go/src/overlord DEPS=false STATIC=$(STATIC) \
GOPATH=$(realpath $(OVERLORD_DEPS_DIR)):$(realpath go)
# To install, get go/bin/{overlordd,ghost}, and go/src/overlord/app.
ovl-bin:
# Create virtualenv environment
rm -rf $(BUILD_DIR)/.env
virtualenv $(BUILD_DIR)/.env
# Build ovl binary with pyinstaller
cd $(BUILD_DIR); \
source .env/bin/activate; \
pip install jsonrpclib ws4py pyinstaller pyyaml; \
pyinstaller --onefile $(CURDIR)/py/tools/ovl.py
# Checks if a package is properly installed.
# Usage: $(call func-check-package,PACKAGE,TEST_RULE)
func-check-package = @\
if ! $(2); then \
$(MK_DIR)/die.sh "Need to run 'emerge-$(BOARD) $(1)' for rule '$(2)'." ; \
fi ${\n}
# Checks if all board resources (from packages) are ready.
# $(BOARD_PACKAGE_NAME) is checked by comparing ebuild and package file
# timestamp, but 'git checkout' does not keep file timestamps, and portage looks
# at version instead of timestamp. So pre-built package may be older than ebuild
# files, and this will be a problem for fresh checkout.
# The solution is to do timestamp comparison only in an interactive quickfix
# (modify, make, test) cycle, by checking if the build is triggered by ebuild
# ($(FROM_EBUILD) is set in factory-9999.ebuild).
check-board-resources:
$(if $(BOARD_EBUILD),\
$(if $(FROM_EBUILD),\
$(info Ignore $(BOARD_PACKAGE_NAME) check.), \
$(call func-check-package,$(BOARD_PACKAGE_NAME), \
[ "$(realpath $(BOARD_EBUILD))" -ot "$(BOARD_PACKAGE_FILE)" ])) \
$(call func-check-package,chromeos-regions, \
[ -e "$(CROS_REGIONS_DATABASE)" ] ))
# Prepare files from source folder into resource folder.
resource: closure check-board-resources po
@$(info Create resource $(if $(BOARD),for [$(BOARD)],without board).)
mkdir -p $(RESOURCE_DIR)
tar -cf $(RESOURCE_PATH) -X $(MK_DIR)/resource_exclude.lst \
bin misc py py_pkg sh init \
$(if $(wildcard $(BOARD_FILES_DIR)), \
--exclude '$(RESOURCE_SRC_DIR)' -C $(BOARD_FILES_DIR) .)
tar -rf $(RESOURCE_PATH) -C $(BUILD_DIR) locale
$(if $(OUTOFTREE_BUILD),\
tar -rf $(RESOURCE_PATH) --transform 's"^"./py/goofy/static/"' \
-C "$(CLOSURE_OUTPUT_DIR)" $(CLOSURE_OUTPUT_FILENAMES))
$(if $(wildcard $(CROS_REGIONS_DATABASE)),\
tar -rf $(RESOURCE_PATH) --transform 's"^"./py/test/l10n/"' \
-C $(dir $(CROS_REGIONS_DATABASE)) $(notdir $(CROS_REGIONS_DATABASE)))
$(foreach file,\
$(wildcard $(BOARD_RESOURCES_DIR)/$@-*.tar \
$(BOARD_RESOURCES_DIR)/factory-*.tar),\
$(info - Found board resource file $(file)) \
tar -Af $(RESOURCE_PATH) $(file)${\n})
$(MK_DIR)/create_resources.py -v --output_dir $(RESOURCE_DIR) \
--sysroot $(SYSROOT) --resources $(RESOURCE_SRC_DIR) \
--board_resources $(BOARD_FILES_DIR)/$(RESOURCE_SRC_DIR)
# Apply files from BOARD_RESOURCES_DIR to particular folder.
# Usage: $(call func-apply-board-resources,RESOURCE_TYPE,OUTPUT_FOLDER)
func-apply-board-resources = @\
$(foreach file,$(wildcard \
$(BOARD_RESOURCES_DIR)/$(1)-*.tar $(RESOURCE_DIR)/$(1)-*.tar),\
$(info - Found board resource file $(file))${\n} \
tar -xf $(file) -C $(2)${\n})
# Make and test a PAR file. The PAR will be tested by importing state and run as
# gooftool.
# Usage: $(call func-make-par,OUTPUT.par,OPTIONS,INPUT_DIR)
func-make-par = @\
@echo "Building PAR $(1)..." && \
mkdir -p "$(dir $(1))" && \
$(3)/bin/make_par $(2) -o $(1) && \
echo -n "Checking PAR invocation..." && \
PYTHONPATH=$(1) $(PYTHON) -c 'import cros.factory.test.state' && \
$(1) gooftool --help | grep -q '^usage: gooftool' && \
echo " Good."
# Builds executable python archives.
par: resource
rm -rf $(PAR_TEMP_DIR); mkdir -p $(PAR_TEMP_DIR)
tar -xf $(RESOURCE_PATH) -C $(PAR_TEMP_DIR)
mkdir -p "$(PAR_OUTPUT_DIR)"
bin/tiny_par --pkg py_pkg -o "$(PAR_OUTPUT_DIR)/$(SETUP_PAR_NAME)" \
$(foreach module,$(SETUP_PAR_MODULES),-m $(module))
@echo -n "Checking $(SETUP_PAR_NAME) invocation..."
@"$(PAR_OUTPUT_DIR)/$(SETUP_PAR_NAME)" image_tool help >/dev/null 2>&1 \
&& echo "Good"
$(call func-apply-board-resources,par,$(PAR_TEMP_DIR))
$(call func-make-par,$(PAR_OUTPUT_DIR)/$(PAR_NAME),,$(PAR_TEMP_DIR))
$(call func-make-par,$(PAR_OUTPUT_DIR)/factory-mini.par,--mini,\
$(PAR_TEMP_DIR))
# Builds factory toolkit from resources.
toolkit: $(WEBGL_AQUARIUM_DIR) resource par
rm -rf $(TOOLKIT_TEMP_DIR) $(TOOLKIT_OUTPUT_DIR)/$(TOOLKIT_FILENAME)
mkdir -p $(TOOLKIT_TEMP_DIR)$(TARGET_DIR) $(TOOLKIT_OUTPUT_DIR)
tar -xf $(RESOURCE_PATH) -C $(TOOLKIT_TEMP_DIR)$(TARGET_DIR)
cp -r $(WEBGL_AQUARIUM_DIR)/* \
$(TOOLKIT_TEMP_DIR)$(TARGET_DIR)/py/test/pytests/webgl_aquarium_static
$(call func-apply-board-resources,toolkit,\
$(TOOLKIT_TEMP_DIR)$(TARGET_DIR))
cp "$(PAR_OUTPUT_DIR)/factory.par" "$(TOOLKIT_TEMP_DIR)$(TARGET_DIR)/"
if [ -f /usr/bin/makeself.sh ]; then \
cp -fL /usr/bin/makeself*.sh $(TOOLKIT_TEMP_DIR)/. ; \
else \
cp -fL /usr/bin/makeself $(TOOLKIT_TEMP_DIR)/makeself.sh && \
cp -fL /usr/share/makeself/makeself*.sh $(TOOLKIT_TEMP_DIR)/. ; \
fi
# TODO(hungte) Figure out a way to get repo status in OUTOFTREE_BUILD.
$(if $(OUTOFTREE_BUILD),,$(if $(BOARD),\
py/toolkit/print_repo_status.py -b $(BOARD) \
>$(TOOLKIT_TEMP_DIR)/REPO_STATUS))
# Install factory test enabled flag.
touch $(TOOLKIT_TEMP_DIR)$(TARGET_DIR)/enabled
chmod -R go=rX $(TOOLKIT_TEMP_DIR)$(TARGET_DIR)
$(TOOLKIT_TEMP_DIR)$(TARGET_DIR)/py/toolkit/installer.py \
--pack-into $(TOOLKIT_OUTPUT_DIR)/$(TOOLKIT_FILENAME) \
--version "$(BOARD) Factory Toolkit $(TOOLKIT_VERSION)"
# Creates build/doc and build/doc.zip, containing the factory SDK docs.
doc:
rm -rf $(DOC_TEMP_DIR); mkdir -p $(DOC_TEMP_DIR)
# Do the actual build in the DOC_TEMP_DIR directory, since we need to
# munge the docs a bit.
rsync -a doc/ $(DOC_TEMP_DIR)
# Generate rst sources for test cases
bin/generate_rsts -o $(DOC_TEMP_DIR)
CROS_FACTORY_PY_ROOT=$(realpath py_pkg) $(MAKE) -C $(DOC_TEMP_DIR) html
mkdir -p $(dir $(DOC_ARCHIVE_PATH))
rm -rf $(DOC_OUTPUT_DIR)
cp -r $(DOC_TEMP_DIR)/_build/html $(DOC_OUTPUT_DIR)
(cd $(DOC_OUTPUT_DIR)/..; zip -qr9 - $(notdir $(DOC_OUTPUT_DIR))) \
>$(DOC_ARCHIVE_PATH)
# Publishes doc to https://storage.googleapis.com/chromeos-factory-docs/sdk/
publish-docs: clean
# Force using an empty database to load whole region set from source
CROS_REGIONS_DATABASE="/dev/null" $(MAKE) doc
gsutil -h "Cache-Control:public, max-age=3600" -m rsync -c -d -r \
$(DOC_OUTPUT_DIR) $(DOC_PUBLISH_URL)
# Builds everything needed and create the proper bundle folder.
# Note there may be already few files like HWID, README, and MANIFEST.yaml
# already installed into $(SYSROOT)/usr/local/factory/bundle.
bundle: par toolkit
$(MK_DIR)/bundle.sh \
"$(BUNDLE_DIR)" \
"$(TOOLKIT_OUTPUT_DIR)/$(TOOLKIT_FILENAME)" \
"$(PAR_OUTPUT_DIR)/$(SETUP_PAR_NAME)" \
"setup" \
"$(SETUP_BIN_ROOT)"
$(call func-apply-board-resources,bundle,$(BUNDLE_DIR))
$(info Bundle is created in $(abspath $(BUNDLE_DIR)))
lint:
$(MK_DIR)/pylint.sh $(LINT_WHITELIST)
# Target to lint only files that have changed. (We allow either
# "smartlint" or "smart_lint".)
smartlint smart_lint:
bin/smart_lint $(if $(BOARD),--overlay $(BOARD))
# Target to lint only files that have changed, including files from
# the given overlay.
smart_lint-%: .phony
bin/smart_lint --overlay $(@:smart_lint-%=%)
presubmit-chroot:
$(foreach target,$(PRESUBMIT_TARGETS),\
PYTHONDONTWRITEBYTECODE=true $(MAKE) -s $(target)${\n})
presubmit-lint:
@$(MAKE) lint LINT_FILES="$(filter %.py,$(PRESUBMIT_FILES))" 2>/dev/null
presubmit-shebang:
@$(MK_DIR)/presubmit-shebang.py -a $(MK_DIR)/presubmit-shebang.json \
$(PRESUBMIT_FILES)
presubmit-deps:
@if ! py/tools/deps.py -p $(filter py/%,$(PRESUBMIT_FILES)); then \
$(MK_DIR)/die.sh "Dependency check failed." \
"Please read py/tools/deps.conf for more information." ; \
fi
presubmit-test:
@$(MK_DIR)/$@.py $(PRESUBMIT_FILES)
presubmit:
ifeq ($(wildcard /etc/debian_chroot),)
$(info Running presubmit checks inside chroot...)
@cros_sdk PRESUBMIT_FILES="$(PRESUBMIT_FILES)" -- \
$(MAKE) -C ../platform/factory -s $@-chroot
else
@$(MAKE) -s $@-chroot
endif
test:
@TEST_EXTRA_FLAGS=$(TEST_EXTRA_FLAGS) \
$(MK_DIR)/test.sh $(UNITTESTS_WHITELIST)
testall:
@$(MAKE) --no-print-directory test TEST_EXTRA_FLAGS=--nofilter
# Builds an overlay of the given board. Use "private" to overlay
# factory-private (e.g., to build private API docs).
overlay:
$(if $(BOARD),,$(error "You must specify a board to build overlay."))
$(if $(BOARD_FILES_DIR),, \
$(error "You have to first run: setup_board --board $(BOARD)"))
rm -rf $@-$(BOARD)
mkdir -p $@-$(BOARD)
rsync -aK --exclude build --exclude overlay-\* ./ $@-$(BOARD)/
rsync -aK $(if $(filter $(BOARD),private), \
--exclude Makefile ../factory-private/ $@-$(BOARD)/, \
"$(BOARD_FILES_DIR)/" $@-$(BOARD)/)
overlay-%: .phony
$(MAKE) overlay BOARD=$(subst overlay-,,$@)
# Tests the overlay of the given board.
test-overlay-%: overlay-%
$(MAKE) -C $< test && touch .tests-passed
# Lints the overlay of the given board.
lint-overlay-%: overlay-%
$(MAKE) -C $< lint
# Create par of the given board.
par-overlay-%: overlay-%
$(MAKE) -C $< par
po:
$(MAKE) -C po build BOARD=$(BOARD) BUILD_DIR=$(abspath $(BUILD_DIR))