blob: c6b6e53a63bfcc32a5c74226e534740f3c046951 [file] [log] [blame]
#!/usr/bin/env python
# Copyright 2007 Google Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.
"""Appcfg logic specific to Java apps."""
from __future__ import with_statement
import collections
import os.path
import re
import shutil
import stat
import subprocess
import sys
import tempfile
from google.appengine.datastore import datastore_index
from google.appengine.datastore import datastore_index_xml
from import app_engine_web_xml_parser
from import backends_xml_parser
from import cron_xml_parser
from import dispatch_xml_parser
from import dos_xml_parser
from import jarfile
from import queue_xml_parser
from import web_xml_parser
from import yaml_translator
_CLASSES_JAR_NAME_PREFIX = '_ah_webinf_classes'
_COMPILED_JSP_JAR_NAME_PREFIX = '_ah_compiled_jsps'
_MAX_COMPILED_JSP_JAR_SIZE = 1024 * 1024 * 5
class Error(Exception):
class ConfigurationError(Error):
"""There was a configuration error in the application being uploaded."""
class CompileError(Error):
"""There was a compilation error in a JSP file or its generated Java code."""
def IsWarFileWithoutYaml(dir_path):
if os.path.isfile(os.path.join(dir_path, 'app.yaml')):
return False
web_inf = os.path.join(dir_path, 'WEB-INF')
return (os.path.isdir(web_inf) and
set(['appengine-web.xml', 'web.xml']).issubset(os.listdir(web_inf)))
def AddUpdateOptions(parser):
"""Adds options specific to the 'update' command on Java apps to 'parser'.
parser: An instance of OptionsParser.
parser.add_option('--retain_upload_dir', action='store_true',
dest='retain_upload_dir', default=False,
help='Do not delete temporary (staging) directory used '
'in uploading Java apps')
parser.add_option('--no_symlinks', action='store_true',
dest='no_symlinks', default=False,
help='Do not use symbolic links when making the temporary '
'(staging) directory for uploading Java apps')
parser.add_option('--compile_encoding', action='store',
dest='compile_encoding', default='UTF-8',
help='Set the encoding to be used when compiling Java '
'source files (default "UTF-8").')
parser.add_option('--disable_jar_jsps', action='store_false',
dest='jar_jsps', default=True,
help='Do not jar the classes generated from JSPs.')
parser.add_option('--delete_jsps', action='store_true',
dest='delete_jsps', default=False,
help='Delete the JSP source files after compilation.')
parser.add_option('--enable_jar_classes', action='store_true',
dest='do_jar_classes', default=False,
help='Jar the WEB-INF/classes content.')
parser.add_option('--enable_jar_splitting', action='store_true',
dest='do_jar_splitting', default=False,
help='Split large jar files (> 32M) into smaller '
parser.add_option('--jar_splitting_excludes', action='store',
dest='jar_splitting_exclude_suffixes', default='',
help='When --enable_jar_splitting is specified and '
'--jar_splitting_excludes specifies a comma-separated list '
'of suffixes, a file in a jar whose name ends with one '
'of the suffixes will not be included in the split jar '
class JavaAppUpdate(object):
"""Performs Java-specific update configurations."""
_JSP_REGEX = re.compile('.*\\.jspx?')
_xml_parser = collections.namedtuple(
'_xml_parser', ['xml_name', 'yaml_name', 'xml_to_yaml_function'])
_xml_parser('backends.xml', 'backends.yaml',
_xml_parser('cron.xml', 'cron.yaml', cron_xml_parser.GetCronYaml),
_xml_parser('dispatch.xml', 'dispatch.yaml',
_xml_parser('dos.xml', 'dos.yaml', dos_xml_parser.GetDosYaml),
_xml_parser('queue.xml', 'queue.yaml', queue_xml_parser.GetQueueYaml),
def __init__(self, basepath, options):
self.basepath = os.path.abspath(basepath)
self.options = options
if not hasattr(self.options, 'no_symlinks'):
self.options.no_symlinks = True
java_home, exec_suffix = JavaHomeAndSuffix()
self.java_command = os.path.join(java_home, 'bin', 'java' + exec_suffix)
self.javac_command = os.path.join(java_home, 'bin', 'javac' + exec_suffix)
self.app_engine_web_xml = self._ReadAppEngineWebXml()
self.app_engine_web_xml.app_root = self.basepath
if self.options.app_id:
self.app_engine_web_xml.app_id = self.options.app_id
if self.options.version:
self.app_engine_web_xml.version_id = self.options.version
self.web_xml = self._ReadWebXml()
def _ValidateXmlFiles(self):
sdk_dir = os.path.dirname(jarfile.__file__)
xml_validator_jar = os.path.join(
sdk_dir, 'java', 'lib', 'impl', 'libxmlvalidator.jar')
if not os.path.exists(xml_validator_jar):
print >>sys.stderr, ('Not validating XML files because %s does not '
'exist' % xml_validator_jar)
validator_args = []
schema_dir = os.path.join(sdk_dir, 'java', 'docs')
for schema_name in os.listdir(schema_dir):
basename, extension = os.path.splitext(schema_name)
if extension == '.xsd':
schema_file = os.path.join(schema_dir, schema_name)
xml_file = os.path.join(self.basepath, 'WEB-INF', basename + '.xml')
if os.path.exists(xml_file):
validator_args += [xml_file, schema_file]
if validator_args:
command_and_args = [
] + validator_args
status =
if status:
raise ConfigurationError('XML validation failed')
def _ReadAppEngineWebXml(self):
return self._ReadAndParseXml(
def _ReadWebXml(self, basepath=None):
if not basepath:
basepath = self.basepath
return self._ReadAndParseXml(
def _ReadAndParseXml(self, basepath, file_name, parser):
with open(os.path.join(basepath, 'WEB-INF', file_name)) as file_handle:
return parser().ProcessXml(
def CreateStagingDirectory(self, tools_dir):
"""Creates a staging directory for uploading.
This is where we perform the necessary actions to create an application
directory for the update command to work properly - files are organized
into the static folder, and yaml files are generated where they can be
found later.
tools_dir: Path to the SDK tools directory
(typically .../google/appengine/tools)
The path to a new temporary directory which contains generated yaml files
and a static file directory. For the most part, the rest of the update and
upload flow can resume identically to Python/PHP/Go applications.
CompileError: if compilation of JSPs failed.
ConfigurationError: if the app to be staged has a configuration error.
IOError: if there was an I/O problem, for example when scanning jar files.
stage_dir = tempfile.mkdtemp(prefix='appcfgpy')
static_dir = os.path.join(stage_dir, '__static__')
self._CopyOrLink(self.basepath, stage_dir, static_dir, False)
self.app_engine_web_xml.app_root = stage_dir
if self.options.compile_jsps:
self._CompileJspsIfAny(tools_dir, stage_dir)
web_inf = os.path.join(stage_dir, 'WEB-INF')
web_inf_lib = os.path.join(web_inf, 'lib')
api_jar_dict = _FindApiJars(web_inf_lib)
api_versions = set(api_jar_dict.values())
if not api_versions:
api_version = None
elif len(api_versions) == 1:
api_version = api_versions.pop()
raise ConfigurationError('API jars have inconsistent versions: %s' %
for staged_api_jar in api_jar_dict:
appengine_generated = os.path.join(
stage_dir, 'WEB-INF', 'appengine-generated')
self._GenerateAppYaml(stage_dir, api_version, appengine_generated)
app_id = self.options.app_id or self.app_engine_web_xml.app_id
assert app_id, 'Missing app id'
for parser in self._XML_PARSERS:
xml_name = os.path.join(web_inf, parser.xml_name)
if os.path.exists(xml_name):
with open(xml_name) as xml_file:
xml_string =
yaml_string = parser.xml_to_yaml_function(app_id, xml_string)
yaml_file = os.path.join(appengine_generated, parser.yaml_name)
with open(yaml_file, 'w') as yaml:
indexes = []
for xml_name in (
os.path.join('appengine-generated', 'datastore-indexes-auto.xml')):
xml_name = os.path.join(self.basepath, 'WEB-INF', xml_name)
if os.path.exists(xml_name):
with open(xml_name) as xml_file:
xml_string =
index_definitions = datastore_index_xml.IndexesXmlToIndexDefinitions(
if indexes:
yaml_string = datastore_index.IndexDefinitions(indexes=indexes).ToYAML()
yaml_file = os.path.join(appengine_generated, 'index.yaml')
with open(yaml_file, 'w') as yaml:
return stage_dir
def GenerateAppYamlString(self, static_file_list, api_version=None):
"""Constructs an app.yaml string equivalent to the XML files under WEB-INF.
static_file_list: a list of strings that are the absolute path names of
static file resources.
api_version: a string that is the Java API version number, or None if
not known or relevant.
A string that would have the same effect as the XML files under WEB-INF
if it were the contents of an app.yaml file.
return yaml_translator.AppYamlTranslator(
def _GenerateAppYaml(self, stage_dir, api_version, appengine_generated):
"""Creates the app.yaml file in WEB-INF/appengine-generated/.
The path to the WEB-INF/appengine-generated directory.
static_file_list = self._GetStaticFileList(stage_dir)
yaml_str = self.GenerateAppYamlString(static_file_list, api_version)
if not os.path.isdir(appengine_generated):
with open(os.path.join(appengine_generated, 'app.yaml'), 'w') as handle:
def _CopyOrLink(self, source_dir, stage_dir, static_dir, inside_web_inf):
source_dir = os.path.abspath(source_dir)
stage_dir = os.path.abspath(stage_dir)
static_dir = os.path.abspath(static_dir)
for file_name in os.listdir(source_dir):
file_path = os.path.join(source_dir, file_name)
if file_name.startswith('.') or file_name == 'appengine-generated':
if os.path.isdir(file_path):
os.path.join(stage_dir, file_name),
os.path.join(static_dir, file_name),
inside_web_inf or file_name == 'WEB-INF')
if (inside_web_inf
or self.app_engine_web_xml.IncludesResource(file_path)
or (self.options.compile_jsps
and file_path.lower().endswith('.jsp'))):
self._CopyOrLinkFile(file_path, os.path.join(stage_dir, file_name))
if (not inside_web_inf
and self.app_engine_web_xml.IncludesStatic(file_path)):
self._CopyOrLinkFile(file_path, os.path.join(static_dir, file_name))
def _CopyOrLinkFile(self, source, dest):
destdir = os.path.dirname(dest)
if not os.path.exists(destdir):
if self._ShouldSplitJar(source):
self._SplitJar(source, destdir)
elif source.endswith('web.xml'):
shutil.copy(source, dest)
os.chmod(dest, os.stat(dest).st_mode | stat.S_IWRITE)
elif self.options.no_symlinks:
shutil.copy(source, dest)
os.symlink(source, dest)
def _MoveDirectoryContents(self, source_dir, dest_dir):
"""Move the contents of source_dir to dest_dir, which might not exist.
IOError: if the dest_dir hierarchy already contains a file where the
source_dir hierarchy has a file or directory of the same name, or if
the dest_dir hierarchy already contains a directory where the source_dir
hierarchy has a file of the same name.
if not os.path.exists(dest_dir):
for entry in os.listdir(source_dir):
source_entry = os.path.join(source_dir, entry)
dest_entry = os.path.join(dest_dir, entry)
if os.path.exists(dest_entry):
if os.path.isdir(source_entry) and os.path.isdir(dest_entry):
self._MoveDirectoryContents(source_entry, dest_entry)
raise IOError('Cannot overwrite existing %s' % dest_entry)
shutil.move(source_entry, dest_entry)
_MAX_SIZE = 32 * 1000 * 1000
def _ShouldSplitJar(self, path):
return (path.lower().endswith('.jar') and self.options.do_jar_splitting and
os.path.getsize(path) >= self._MAX_SIZE)
def _SplitJar(self, jar_path, dest_dir):
"""Split a source jar into two or more jars in the given dest_dir.
jar_path: string that is the path to jar to be split. The contents of this
jar will be copied into the output jars, but the jar itself will not be
dest_dir: directory into which to put the jars that result from splitting
the input jar.
IOError: if the jar cannot be split.
exclude_suffixes = (
set(self.options.jar_splitting_exclude_suffixes.split(',')) - set(['']))
include = lambda name: not any(name.endswith(s) for s in exclude_suffixes)
jarfile.SplitJar(jar_path, dest_dir, self._MAX_SIZE, include)
def _GetStaticFileList(staging_dir):
return _FilesMatching(os.path.join(staging_dir, '__static__'))
def _CompileJspsIfAny(self, tools_dir, staging_dir):
"""Compiles JSP files, if any, into .class files.."""
if self._MatchingFileExists(self._JSP_REGEX, staging_dir):
gen_dir = tempfile.mkdtemp()
self._CompileJspsWithGenDir(tools_dir, staging_dir, gen_dir)
def _CompileJspsWithGenDir(self, tools_dir, staging_dir, gen_dir):
staging_web_inf = os.path.join(staging_dir, 'WEB-INF')
lib_dir = os.path.join(staging_web_inf, 'lib')
for jar_file in GetUserJspLibFiles(tools_dir):
jar_file, os.path.join(lib_dir, os.path.basename(jar_file)))
for jar_file in GetSharedJspLibFiles(tools_dir):
jar_file, os.path.join(lib_dir, os.path.basename(jar_file)))
classes_dir = os.path.join(staging_web_inf, 'classes')
generated_web_xml = os.path.join(staging_web_inf, 'generated_web.xml')
classpath = self._GetJspClasspath(tools_dir, classes_dir, gen_dir)
command_and_args = [
'-classpath', classpath,
'-uriroot', staging_dir,
'-p', 'org.apache.jsp',
'-webinc', generated_web_xml,
'-d', gen_dir,
'-javaEncoding', self.options.compile_encoding,
status =
if status:
raise CompileError(
'Compilation of JSPs exited with status %d' % status)
self._CompileJavaFiles(classpath, staging_web_inf, gen_dir)
self.web_xml = self._ReadWebXml(staging_dir)
def _CompileJavaFiles(self, classpath, web_inf, jsp_class_dir):
"""Compile all *.java files found under jsp_class_dir."""
java_files = _FilesMatching(jsp_class_dir, lambda f: f.endswith('.java'))
if not java_files:
command_and_args = [
'-classpath', classpath,
'-d', jsp_class_dir,
'-encoding', self.options.compile_encoding,
] + java_files
status =
if status:
raise CompileError(
'Compilation of JSP-generated code exited with status %d' % status)
if self.options.jar_jsps:
self._ZipJasperGeneratedFiles(web_inf, jsp_class_dir)
web_inf_classes = os.path.join(web_inf, 'classes')
self._MoveDirectoryContents(jsp_class_dir, web_inf_classes)
if self.options.delete_jsps:
jsps = _FilesMatching(os.path.dirname(web_inf),
lambda f: f.endswith('.jsp'))
for f in jsps:
if self.options.do_jar_classes:
def _ZipJasperGeneratedFiles(web_inf, jsp_class_dir):
lib_dir = os.path.join(web_inf, 'lib')
jarfile.Make(jsp_class_dir, lib_dir, _COMPILED_JSP_JAR_NAME_PREFIX,
include_predicate=lambda name: not name.endswith('.java'))
def _ZipWebInfClassesFiles(web_inf):
lib_dir = os.path.join(web_inf, 'lib')
classes_dir = os.path.join(web_inf, 'classes')
jarfile.Make(classes_dir, lib_dir, _CLASSES_JAR_NAME_PREFIX,
def _GetJspClasspath(tools_dir, classes_dir, gen_dir):
"""Builds the classpath for the JSP Compilation system call."""
lib_dir = os.path.join(os.path.dirname(classes_dir), 'lib')
elements = (
GetImplLibs(tools_dir) + GetSharedLibFiles(tools_dir) +
[classes_dir, gen_dir] +
lib_dir, lambda f: f.endswith('.jar') or f.endswith('.zip')))
return (os.pathsep).join(elements)
def _MatchingFileExists(regex, dir_path):
for _, _, files in os.walk(dir_path):
for f in files:
if, f):
return True
return False
def GetImplLibs(tools_dir):
return _GetLibsShallow(os.path.join(tools_dir, 'java', 'lib', 'impl'))
def GetSharedLibFiles(tools_dir):
return _GetLibsRecursive(os.path.join(tools_dir, 'java', 'lib', 'shared'))
def GetUserJspLibFiles(tools_dir):
return _GetLibsRecursive(
os.path.join(tools_dir, 'java', 'lib', 'tools', 'jsp'))
def GetSharedJspLibFiles(tools_dir):
return _GetLibsRecursive(
os.path.join(tools_dir, 'java', 'lib', 'shared', 'jsp'))
def _GetLibsRecursive(dir_path):
return _FilesMatching(dir_path, lambda f: f.endswith('.jar'))
def _GetLibsShallow(dir_path):
libs = []
for f in os.listdir(dir_path):
if os.path.isfile(os.path.join(dir_path, f)) and f.endswith('.jar'):
libs.append(os.path.join(dir_path, f))
return libs
def _FilesMatching(root, predicate=lambda f: True):
"""Finds all files under the given root that match the given predicate.
root: a string that is the absolute or relative path to a directory.
predicate: a function that takes a file name (without a directory) and
returns a truth value.
A list of strings that are the paths of every file under the given root
that satisfies the given predicate. The paths are absolute if and only if
the input root is absolute.
matches = []
for path, _, files in os.walk(root):
matches += [os.path.join(path, f) for f in files if predicate(f)]
return matches
def JavaHomeAndSuffix():
"""Find the directory that the JDK is installed in.
The JDK install directory is expected to have a bin directory that contains
at a minimum the java and javac executables. If the environment variable
JAVA_HOME is set then it must point to such a directory. Otherwise, we look
for javac on the PATH and check that it is inside a JDK install directory.
A tuple where the first element is the JDK install directory and the second
element is a suffix that must be appended to executables in that directory
('' on Unix-like systems, '.exe' on Windows).
RuntimeError: If JAVA_HOME is set but is not a JDK install directory, or
otherwise if a JDK install directory cannot be found based on the PATH.
def ResultForJdkAt(path):
"""Return (path, suffix) if path is a JDK install directory, else None."""
def IsExecutable(binary):
return os.path.isfile(binary) and os.access(binary, os.X_OK)
def ResultFor(path):
for suffix in ['', '.exe']:
if all(IsExecutable(os.path.join(path, 'bin', binary + suffix))
for binary in ['java', 'javac', 'jar']):
return (path, suffix)
return None
result = ResultFor(path)
if not result:
head, tail = os.path.split(path)
if tail == 'jre':
result = ResultFor(head)
return result
java_home = os.getenv('JAVA_HOME')
if java_home:
result = ResultForJdkAt(java_home)
if result:
return result
raise RuntimeError(
'JAVA_HOME is set but does not reference a valid JDK: %s' % java_home)
for path_dir in os.environ['PATH'].split(os.pathsep):
maybe_root, last = os.path.split(path_dir)
if last == 'bin':
result = ResultForJdkAt(maybe_root)
if result:
return result
raise RuntimeError('Did not find JDK in PATH and JAVA_HOME is not set')
def _FindApiJars(lib_dir):
"""Find the appengine-api-*.jar and its version.
The version of an appengine-api-*.jar is the Specification-Version attribute
in the jar's manifest section whose Name is 'com/google/appengine/api/'.
lib_dir: the base directory under which jars are to be found.
A dict from string to string, mapping all found API jars to their
corresponding versions.
IOError: if there was a problem reading the jars.
result = {}
for jar_file in _FilesMatching(lib_dir, lambda f: f.endswith('.jar')):
manifest = jarfile.ReadManifest(jar_file)
if manifest:
section = manifest.sections.get('com/google/appengine/api/')
if section and 'Specification-Version' in section:
result[jar_file] = section['Specification-Version']
return result