blob: 33c207aace597c568f13d32ec9e6a72f49e04709 [file] [log] [blame]
#!/usr/bin/env python
#
# Copyright 2007 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""An abstraction around the source and executable for a Go application."""
import atexit
import errno
import logging
import os
import os.path
import shutil
import subprocess
import sys
import tempfile
import google
from google.appengine.tools.devappserver2 import errors
from google.appengine.tools.devappserver2 import safe_subprocess
_SDKROOT = os.path.dirname(os.path.dirname(google.__file__))
GOROOT = os.path.join(_SDKROOT, 'goroot')
_GAB_PATH = os.path.join(GOROOT, 'bin', 'go-app-builder')
if sys.platform.startswith('win'):
_GAB_PATH += '.exe'
def _rmtree(directory):
try:
shutil.rmtree(directory)
except:
pass
def _escape_tool_flags(*flags):
"""Escapes a list of flags for consumption by gab.
This is reverse to the encoding in //apphosting/runtime/go/builder/flags.go.
Args:
*flags: A list of flag arguments to be escaped.
Returns:
A single escaped string.
"""
return ','.join([f.replace('\\', r'\\').replace(',', r'\,') for f in flags])
def _get_base_gab_args(application_root, nobuild_files, arch):
"""Returns the base arguments for invoking go-app-builder.
Args:
application_root: string path to the root dir of the application.
nobuild_files: regexp identifying which files to not build.
arch: The one-character architecture designator (5, 6, or 8).
Returns:
List of strings of arguments for invoking go-app-builder.
"""
# Go's regexp package does not implicitly anchor to the start.
gab_args = [
_GAB_PATH,
'-app_base', application_root,
'-arch', arch,
'-dynamic',
'-goroot', GOROOT,
'-nobuild_files', '^' + str(nobuild_files),
'-unsafe',
]
if 'GOPATH' in os.environ:
gab_args.extend(['-gopath', os.environ['GOPATH']])
return gab_args
def list_go_files(application_root, nobuild_files, skip_files):
"""Returns a list of all Go files under the application root.
Args:
application_root: string path to the root dir of the application.
nobuild_files: regexp identifying which files to not build.
skip_files: regexp identifying which files to omit from app.
Returns:
A list of every .go file under the application root, relative to
that root.
"""
go_files = []
for root, _, file_names in os.walk(application_root):
for file_name in file_names:
if not file_name.endswith('.go'):
continue
full_path = os.path.join(root, file_name)
rel_path = os.path.relpath(full_path, application_root)
if skip_files.match(rel_path):
continue
if nobuild_files.match(rel_path):
continue
go_files.append(rel_path)
return go_files
def get_app_extras_for_vm(application_root, nobuild_files, skip_files):
"""Returns an iterable describing extra Go files needed to build VM apps.
The Go files are decided based on the production environment linux/amd64.
Args:
application_root: string path to the root dir of the application.
nobuild_files: regexp identifying which files to not build.
skip_files: regexp identifying which files to omit from app.
Returns:
An iterable of pairs, one per extra Go file. The first pair element
is the relative path at which to import the Go file; the second is its
absolute path.
Raises:
BuildError: if the go application builder fails.
"""
gab_args = _get_base_gab_args(application_root, nobuild_files, '6')
gab_args.extend(['-print_extras', '-vm'])
gab_args.extend(list_go_files(application_root, nobuild_files, skip_files))
env = {
'GOOS': 'linux',
'GOARCH': 'amd64',
}
gab_process = safe_subprocess.start_process(gab_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
gab_stdout, gab_stderr = gab_process.communicate()
if gab_process.returncode:
raise BuildError(
'(Executed command: %s)\n\n%s' % (' '.join(gab_args),
gab_stderr))
return [l.split('|') for l in gab_stdout.split('\n') if l]
class BuildError(errors.Error):
"""Building the GoApplication failed."""
class GoApplication(object):
"""An abstraction around the source and executable for a Go application."""
def __init__(self, module_configuration):
"""Initializer for Module.
Args:
module_configuration: An application_configuration.ModuleConfiguration
instance storing the configuration data for a module.
"""
self._module_configuration = module_configuration
self._go_file_to_mtime = {}
self._extras_hash = None
self._go_executable = None
self._work_dir = None
self._arch = self._get_architecture()
self._pkg_path = self._get_pkg_path()
@property
def go_executable(self):
"""The path to the Go executable. None if it has not been built."""
return self._go_executable
def get_environment(self):
"""Return the environment that used be used to run the Go executable."""
environ = {'GOROOT': GOROOT,
'PWD': self._module_configuration.application_root,
'TZ': 'UTC',
'RUN_WITH_DEVAPPSERVER': '1'}
if 'SYSTEMROOT' in os.environ:
environ['SYSTEMROOT'] = os.environ['SYSTEMROOT']
if 'USER' in os.environ:
environ['USER'] = os.environ['USER']
return environ
@staticmethod
def _get_architecture():
architecture_map = {
'arm': '5',
'amd64': '6',
'386': '8',
}
for platform in os.listdir(os.path.join(GOROOT, 'pkg', 'tool')):
# Look for 'linux_amd64', 'windows_386', etc.
if '_' not in platform:
continue
architecture = platform.split('_', 1)[1]
if architecture in architecture_map:
return architecture_map[architecture]
raise BuildError('No known compiler found in goroot (%s)' % GOROOT)
@staticmethod
def _get_pkg_path():
for n in os.listdir(os.path.join(GOROOT, 'pkg')):
# Look for 'linux_amd64_appengine', 'windows_386_appengine', etc.
if n.endswith('_appengine'):
return os.path.join(GOROOT, 'pkg', n)
raise BuildError('No package path found in goroot (%s)' % GOROOT)
def _get_go_files_to_mtime(self):
"""Returns a dict mapping all Go files to their mtimes.
Returns:
A dict mapping the path relative to the application root of every .go
file in the application root, or any of its subdirectories, to the file's
modification time.
"""
app_root = self._module_configuration.application_root
go_file_to_mtime = {}
for rel_path in list_go_files(
app_root, self._module_configuration.nobuild_files,
self._module_configuration.skip_files):
full_path = os.path.join(app_root, rel_path)
try:
go_file_to_mtime[rel_path] = os.path.getmtime(full_path)
except OSError as e:
# Ignore deleted files.
if e.errno != errno.ENOENT:
raise
return go_file_to_mtime
def _get_extras_hash(self):
"""Returns a hash of the names and mtimes of package dependencies.
Returns:
Returns a string representing a hash.
Raises:
BuildError: if the go application builder fails.
"""
gab_args = _get_base_gab_args(
self._module_configuration.application_root,
self._module_configuration.nobuild_files, self._arch)
gab_args.append('-print_extras_hash')
gab_args.extend(self._go_file_to_mtime)
gab_process = safe_subprocess.start_process(gab_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={})
gab_stdout, gab_stderr = gab_process.communicate()
if gab_process.returncode:
raise BuildError(
'(Executed command: %s)\n\n%s' % (' '.join(gab_args),
gab_stderr))
else:
return gab_stdout
def _build(self):
assert self._go_file_to_mtime, 'no .go files'
logging.debug('Building Go application')
gab_args = _get_base_gab_args(
self._module_configuration.application_root,
self._module_configuration.nobuild_files, self._arch)
gab_args.extend([
'-binary_name', '_go_app',
'-extra_imports', 'appengine_internal/init',
'-work_dir', self._work_dir,
'-gcflags', _escape_tool_flags('-I', self._pkg_path),
'-ldflags', _escape_tool_flags('-L', self._pkg_path),
])
gab_args.extend(self._go_file_to_mtime)
gab_process = safe_subprocess.start_process(gab_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={})
gab_stdout, gab_stderr = gab_process.communicate()
if gab_process.returncode:
raise BuildError(
'(Executed command: %s)\n\n%s\n%s' % (' '.join(gab_args),
gab_stdout,
gab_stderr))
else:
logging.debug('Build succeeded:\n%s\n%s', gab_stdout, gab_stderr)
self._go_executable = os.path.join(self._work_dir, '_go_app')
def maybe_build(self, maybe_modified_since_last_build):
"""Builds an executable for the application if necessary.
Args:
maybe_modified_since_last_build: True if any files in the application root
or the GOPATH have changed since the last call to maybe_build, False
otherwise. This argument is used to decide whether a build is Required
or not.
Returns:
True if compilation was successfully performed (will raise
an exception if compilation was attempted but failed).
False if compilation was not attempted.
Raises:
BuildError: if building the executable fails for any reason.
"""
if not self._work_dir:
self._work_dir = tempfile.mkdtemp('appengine-go-bin')
atexit.register(_rmtree, self._work_dir)
if not os.path.exists(_GAB_PATH):
# TODO: This message should be more useful i.e. point the
# user to an SDK that does have the right components.
raise BuildError('Required Go components are missing from the SDK.')
if self._go_executable and not maybe_modified_since_last_build:
return False
(self._go_file_to_mtime,
old_go_file_to_mtime) = (self._get_go_files_to_mtime(),
self._go_file_to_mtime)
if not self._go_file_to_mtime:
raise BuildError('no .go files found in %s' %
self._module_configuration.application_root)
self._extras_hash, old_extras_hash = (self._get_extras_hash(),
self._extras_hash)
if (self._go_executable and
self._go_file_to_mtime == old_go_file_to_mtime and
self._extras_hash == old_extras_hash):
return False
if self._go_file_to_mtime != old_go_file_to_mtime:
logging.debug('Rebuilding Go application due to source modification')
elif self._extras_hash != old_extras_hash:
logging.debug('Rebuilding Go application due to GOPATH modification')
else:
logging.debug('Building Go application')
self._build()
return True