Merge pull request #86 from WebAssembly/de-wasmate
Delete wasmate
diff --git a/buildbot/build.py b/buildbot/build.py
index b95d3fe..1b0401b 100755
--- a/buildbot/build.py
+++ b/buildbot/build.py
@@ -24,6 +24,8 @@
import assemble_files
import compile_torture_tests
import link_assembly_files
+# Use proc instead of subprocess to get logged subprocess invocation.
+import proc
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -110,7 +112,6 @@
# should be manually updated when convenient.
GCC_REVISION = 'b6125c702850488ac3bfb1079ae5c9db89989406'
-
# Magic annotations:
# https://chromium.googlesource.com/chromium/tools/build/+/master/scripts/common/annotator.py
def BuildStep(name):
@@ -168,7 +169,7 @@
tar = os.path.join(up_directory, basename + '.tbz2')
Remove(tar)
print 'Creating %s from %s/%s' % (tar, up_directory, basename)
- subprocess.check_call(['tar', 'cjf', tar, basename], cwd=up_directory)
+ proc.check_call(['tar', 'cjf', tar, basename], cwd=up_directory)
return tar
@@ -178,7 +179,7 @@
return
remote = CLOUD_STORAGE_PATH + remote
print 'Uploading %s to %s' % (local, remote)
- subprocess.check_call(
+ proc.check_call(
['gsutil', 'cp', '-a', 'public-read', local, 'gs://' + remote])
StepLink(link_name, CLOUD_STORAGE_BASE_URL + remote)
@@ -190,7 +191,7 @@
copy_from = CLOUD_STORAGE_PATH + copy_from
copy_to = CLOUD_STORAGE_PATH + copy_to
print 'Copying %s to %s' % (copy_from, copy_to)
- subprocess.check_call(
+ proc.check_call(
['gsutil', 'cp', '-a', 'public-read',
'gs://' + copy_from, 'gs://' + copy_to])
StepLink(link_name, CLOUD_STORAGE_BASE_URL + copy_to)
@@ -209,14 +210,14 @@
def GitRemoteUrl(cwd, remote):
"""Get the URL of a remote."""
- return subprocess.check_output(['git', 'config', '--get', 'remote.%s.url' %
- remote], cwd=cwd).strip()
+ return proc.check_output(['git', 'config', '--get', 'remote.%s.url' %
+ remote], cwd=cwd).strip()
def HasRemote(cwd, remote):
""""Checked whether the named remote exists."""
- remotes = subprocess.check_output(['git', 'remote'],
- cwd=cwd).strip().splitlines()
+ remotes = proc.check_output(['git', 'remote'],
+ cwd=cwd).strip().splitlines()
return remote in remotes
@@ -233,8 +234,8 @@
return
remote = GITHUB_SSH + '/'.join(GitRemoteUrl(cwd, 'origin').split('/')[-2:])
print '%s has no github remote, adding %s' % (cwd, remote)
- subprocess.check_call(['git', 'remote', 'add', GITHUB_REMOTE, remote],
- cwd=cwd)
+ proc.check_call(['git', 'remote', 'add', GITHUB_REMOTE, remote],
+ cwd=cwd)
def GitCloneFetchCheckout(name, work_dir, git_repo, checkout='origin/master'):
@@ -243,11 +244,11 @@
print '%s directory already exists' % name
else:
print 'Cloning %s from %s into %s' % (name, git_repo, work_dir)
- subprocess.check_call(['git', 'clone', git_repo, work_dir])
+ proc.check_call(['git', 'clone', git_repo, work_dir])
print 'Syncing %s' % name
- subprocess.check_call(['git', 'fetch'], cwd=work_dir)
+ proc.check_call(['git', 'fetch'], cwd=work_dir)
print 'Checking out %s' % checkout
- subprocess.check_call(['git', 'checkout', checkout], cwd=work_dir)
+ proc.check_call(['git', 'checkout', checkout], cwd=work_dir)
PrintCurrentGitRev(work_dir)
AddGithubRemote(work_dir)
@@ -258,24 +259,24 @@
The upstream repository is in Subversion. Use `git pull --rebase` instead of
git pull: llvm.org/docs/GettingStarted.html#git-mirror
"""
- subprocess.check_call(
+ proc.check_call(
['git', 'config', 'branch.master.rebase', 'true'], cwd=cwd)
def PrintCurrentGitRev(cwd):
- log = subprocess.check_output(
+ log = proc.check_output(
['git', 'log', '--oneline', '-n1'], cwd=cwd).strip()
- remote = subprocess.check_output(
+ remote = proc.check_output(
['git', 'config', '--get', 'remote.origin.url'], cwd=cwd).strip()
sys.stdout.write('%s from remote %s is at revision %s\n' %
(cwd, remote, log))
def FindPriorRev(path, goal):
- revs = subprocess.check_output(
+ revs = proc.check_output(
['git', 'rev-list', 'origin/master'], cwd=path).splitlines()
for rev in revs:
- num = subprocess.check_output(
+ num = proc.check_output(
['git', 'svn', 'find-rev', rev], cwd=path).strip()
if int(num) <= goal:
return rev
@@ -288,23 +289,23 @@
print 'LLVM and Clang directories already exist'
else:
print 'Cloning LLVM and Clang'
- subprocess.check_call(['git', 'clone', LLVM_GIT, LLVM_SRC_DIR])
+ proc.check_call(['git', 'clone', LLVM_GIT, LLVM_SRC_DIR])
GitConfigRebaseMaster(LLVM_SRC_DIR)
- subprocess.check_call(['git', 'clone', CLANG_GIT, CLANG_SRC_DIR])
+ proc.check_call(['git', 'clone', CLANG_GIT, CLANG_SRC_DIR])
GitConfigRebaseMaster(CLANG_SRC_DIR)
print 'Syncing LLVM'
- subprocess.check_call(['git', 'fetch'], cwd=LLVM_SRC_DIR)
- subprocess.check_call(['git', 'checkout', LLVM_REVISION], cwd=LLVM_SRC_DIR)
+ proc.check_call(['git', 'fetch'], cwd=LLVM_SRC_DIR)
+ proc.check_call(['git', 'checkout', LLVM_REVISION], cwd=LLVM_SRC_DIR)
print 'Getting SVN rev'
global LLVM_SVN_REV
- LLVM_SVN_REV = int(subprocess.check_output(
+ LLVM_SVN_REV = int(proc.check_output(
['git', 'svn', 'find-rev', 'HEAD'], cwd=LLVM_SRC_DIR).strip())
print 'SVN REV: %d' % LLVM_SVN_REV
print 'Finding prior Clang rev'
- subprocess.check_call(['git', 'fetch'], cwd=CLANG_SRC_DIR)
+ proc.check_call(['git', 'fetch'], cwd=CLANG_SRC_DIR)
prior_rev = FindPriorRev(CLANG_SRC_DIR, LLVM_SVN_REV)
print 'Checking out Clang rev: %s' % prior_rev
- subprocess.check_call(['git', 'checkout', prior_rev], cwd=CLANG_SRC_DIR)
+ proc.check_call(['git', 'checkout', prior_rev], cwd=CLANG_SRC_DIR)
PrintCurrentGitRev(LLVM_SRC_DIR)
PrintCurrentGitRev(CLANG_SRC_DIR)
@@ -316,12 +317,12 @@
print 'Cloning Prebuilt Chromium Clang directory'
Mkdir(PREBUILT_CLANG)
Mkdir(PREBUILT_CLANG_TOOLS)
- subprocess.check_call(
+ proc.check_call(
['git', 'clone', PREBUILT_CLANG_GIT, PREBUILT_CLANG_TOOLS_CLANG])
print 'Syncing Prebuilt Chromium Clang scripts'
- subprocess.check_call(['git', 'fetch'], cwd=PREBUILT_CLANG_TOOLS_CLANG)
+ proc.check_call(['git', 'fetch'], cwd=PREBUILT_CLANG_TOOLS_CLANG)
print 'Syncing Prebuilt Chromium Clang'
- subprocess.check_call(
+ proc.check_call(
[os.path.join(PREBUILT_CLANG_TOOLS_CLANG, 'scripts', 'update.py')])
assert os.path.isfile(CC), 'Expect clang at %s' % CC
assert os.path.isfile(CXX), 'Expect clang++ at %s' % CXX
@@ -339,7 +340,7 @@
with open(OCAML_TAR, 'wb') as out:
out.write(f.read())
print 'Download done, untar %s' % OCAML_TAR
- subprocess.check_call(['tar', '-xvf', OCAML_TAR], cwd=WORK_DIR)
+ proc.check_call(['tar', '-xvf', OCAML_TAR], cwd=WORK_DIR)
assert os.path.isdir(OCAML_DIR), 'Untar should produce %s' % OCAML_DIR
@@ -370,7 +371,7 @@
BuildStep('Build LLVM')
print 'Running cmake on llvm'
Mkdir(LLVM_OUT_DIR)
- subprocess.check_call(
+ proc.check_call(
['cmake', '-G', 'Ninja', LLVM_SRC_DIR,
'-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
'-DLLVM_BUILD_TESTS=ON',
@@ -382,29 +383,29 @@
'-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly',
'-DLLVM_TARGETS_TO_BUILD=X86'], cwd=LLVM_OUT_DIR)
print 'Running ninja'
- subprocess.check_call(['ninja'], cwd=LLVM_OUT_DIR)
+ proc.check_call(['ninja'], cwd=LLVM_OUT_DIR)
def TestLLVM():
BuildStep('Test LLVM')
- subprocess.check_call(['ninja', 'check-all'], cwd=LLVM_OUT_DIR)
+ proc.check_call(['ninja', 'check-all'], cwd=LLVM_OUT_DIR)
def InstallLLVM():
BuildStep('Install LLVM')
Remove(LLVM_INSTALL_DIR)
- subprocess.check_call(['ninja', 'install'], cwd=LLVM_OUT_DIR)
+ proc.check_call(['ninja', 'install'], cwd=LLVM_OUT_DIR)
def BuildSexpr():
BuildStep('Build Sexpr')
# sexpr-wasm builds in its own in-tree out/ folder. The build is fast, so
# always clobber.
- subprocess.check_call(['make', 'clean'], cwd=SEXPR_SRC_DIR)
- subprocess.check_call(['make',
- 'CC=%s' % CC,
- 'CXX=%s' % CXX],
- cwd=SEXPR_SRC_DIR)
+ proc.check_call(['make', 'clean'], cwd=SEXPR_SRC_DIR)
+ proc.check_call(['make',
+ 'CC=%s' % CC,
+ 'CXX=%s' % CXX],
+ cwd=SEXPR_SRC_DIR)
sexpr = os.path.join(SEXPR_OUT_DIR, 'sexpr-wasm')
CopyBinaryToArchive(sexpr)
@@ -414,10 +415,10 @@
makefile = os.path.join(OCAML_DIR, 'config', 'Makefile')
if not os.path.isfile(makefile):
configure = os.path.join(OCAML_DIR, 'configure')
- subprocess.check_call(
+ proc.check_call(
[configure, '-prefix', OCAML_OUT_DIR, '-cc', CC], cwd=OCAML_DIR)
- subprocess.check_call(['make', 'world.opt', '-j%s' % NPROC], cwd=OCAML_DIR)
- subprocess.check_call(['make', 'install'], cwd=OCAML_DIR)
+ proc.check_call(['make', 'world.opt', '-j%s' % NPROC], cwd=OCAML_DIR)
+ proc.check_call(['make', 'install'], cwd=OCAML_DIR)
ocamlbuild = os.path.join(OCAML_BIN_DIR, 'ocamlbuild')
assert os.path.isfile(ocamlbuild), 'Expected installed %s' % ocamlbuild
os.environ['PATH'] = OCAML_BIN_DIR + os.pathsep + os.environ['PATH']
@@ -426,8 +427,8 @@
def BuildSpec():
BuildStep('Build spec')
# Spec builds in-tree. Always clobber and run the tests.
- subprocess.check_call(['make', 'clean'], cwd=ML_DIR)
- subprocess.check_call(['make', 'all'], cwd=ML_DIR)
+ proc.check_call(['make', 'clean'], cwd=ML_DIR)
+ proc.check_call(['make', 'all'], cwd=ML_DIR)
wasm = os.path.join(ML_DIR, 'wasm.opt')
CopyBinaryToArchive(wasm)
@@ -435,13 +436,13 @@
def BuildBinaryen():
BuildStep('Build binaryen')
Mkdir(BINARYEN_OUT_DIR)
- subprocess.check_call(
+ proc.check_call(
['cmake', '-G', 'Ninja', BINARYEN_SRC_DIR,
'-DCMAKE_C_COMPILER=' + CC,
'-DCMAKE_CXX_COMPILER=' + CXX],
cwd=BINARYEN_OUT_DIR)
print 'Running ninja'
- subprocess.check_call(['ninja'], cwd=BINARYEN_OUT_DIR)
+ proc.check_call(['ninja'], cwd=BINARYEN_OUT_DIR)
assert os.path.isdir(BINARYEN_BIN_DIR), 'Expected %s' % BINARYEN_BIN_DIR
for node in os.listdir(BINARYEN_BIN_DIR):
f = os.path.join(BINARYEN_BIN_DIR, node)
diff --git a/buildbot/proc.py b/buildbot/proc.py
new file mode 100755
index 0000000..8ca0c67
--- /dev/null
+++ b/buildbot/proc.py
@@ -0,0 +1,40 @@
+#! /usr/bin/env python
+
+# Copyright 2016 WebAssembly Community Group participants
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This module is intended to be a drop-in replacement for the standard
+# subprocess module, with the difference that it logs commands before it runs
+# them. Everything not overriden should pass through to the subprocess module
+# via the import trick below.
+
+# Imports subprocess in its own namespace so we can always refer directly to
+# its attributes.
+import subprocess
+# Imports all of subprocess into the current namespace, effectively re-exporting
+# everything.
+from subprocess import *
+
+
+# Now we can override any parts of subprocess we want, while leaving the rest.
+def check_call(cmd, **kwargs):
+ print 'check_call Running command:'
+ print ' '.join('"' + c + '"' if ' ' in c else c for c in cmd)
+ return subprocess.check_call(cmd, **kwargs)
+
+
+def check_output(cmd, **kwargs):
+ print 'check_output Running command:'
+ print ' '.join('"' + c + '"' if ' ' in c else c for c in cmd)
+ return subprocess.check_output(cmd, **kwargs)
diff --git a/buildbot/testing.py b/buildbot/testing.py
index 0fa76dd..15d078a 100755
--- a/buildbot/testing.py
+++ b/buildbot/testing.py
@@ -15,13 +15,14 @@
# limitations under the License.
import difflib
-import os
-import os.path
import math
import multiprocessing
-import subprocess
+import os
+import os.path
import sys
+# Use proc instead of subprocess to get logged subprocess invocation.
+import proc
class Result:
"""Result from a single test that was run."""
@@ -62,16 +63,15 @@
"""Execute a single test."""
basename = os.path.basename(test_file)
outfile = self.outname_ctor(self.outdir, test_file)
- # Print something simple so buildbots don't think the script is dead.
- sys.stdout.write('.')
- sys.stdout.flush()
try:
- output = subprocess.check_output(
+ output = proc.check_output(
self.command_ctor(test_file, outfile, self.extras),
- stderr=subprocess.STDOUT, cwd=self.outdir)
+ stderr=proc.STDOUT, cwd=self.outdir)
+ # Flush the logged command sobuildbots don't think the script is dead.
+ sys.stdout.flush()
assert os.path.isfile(outfile), 'Missing output file %s' % outfile
return Result(test=basename, success=True, output=output)
- except subprocess.CalledProcessError as e:
+ except proc.CalledProcessError as e:
return Result(test=basename, success=False, output=e.output)