Merge pull request #86 from WebAssembly/de-wasmate

Delete wasmate
diff --git a/buildbot/build.py b/buildbot/build.py
index b95d3fe..1b0401b 100755
--- a/buildbot/build.py
+++ b/buildbot/build.py
@@ -24,6 +24,8 @@
 import assemble_files
 import compile_torture_tests
 import link_assembly_files
+# Use proc instead of subprocess to get logged subprocess invocation.
+import proc
 
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -110,7 +112,6 @@
 # should be manually updated when convenient.
 GCC_REVISION = 'b6125c702850488ac3bfb1079ae5c9db89989406'
 
-
 # Magic annotations:
 # https://chromium.googlesource.com/chromium/tools/build/+/master/scripts/common/annotator.py
 def BuildStep(name):
@@ -168,7 +169,7 @@
   tar = os.path.join(up_directory, basename + '.tbz2')
   Remove(tar)
   print 'Creating %s from %s/%s' % (tar, up_directory, basename)
-  subprocess.check_call(['tar', 'cjf', tar, basename], cwd=up_directory)
+  proc.check_call(['tar', 'cjf', tar, basename], cwd=up_directory)
   return tar
 
 
@@ -178,7 +179,7 @@
     return
   remote = CLOUD_STORAGE_PATH + remote
   print 'Uploading %s to %s' % (local, remote)
-  subprocess.check_call(
+  proc.check_call(
       ['gsutil', 'cp', '-a', 'public-read', local, 'gs://' + remote])
   StepLink(link_name, CLOUD_STORAGE_BASE_URL + remote)
 
@@ -190,7 +191,7 @@
   copy_from = CLOUD_STORAGE_PATH + copy_from
   copy_to = CLOUD_STORAGE_PATH + copy_to
   print 'Copying %s to %s' % (copy_from, copy_to)
-  subprocess.check_call(
+  proc.check_call(
       ['gsutil', 'cp', '-a', 'public-read',
        'gs://' + copy_from, 'gs://' + copy_to])
   StepLink(link_name, CLOUD_STORAGE_BASE_URL + copy_to)
@@ -209,14 +210,14 @@
 
 def GitRemoteUrl(cwd, remote):
   """Get the URL of a remote."""
-  return subprocess.check_output(['git', 'config', '--get', 'remote.%s.url' %
-                                  remote], cwd=cwd).strip()
+  return proc.check_output(['git', 'config', '--get', 'remote.%s.url' %
+                            remote], cwd=cwd).strip()
 
 
 def HasRemote(cwd, remote):
   """"Checked whether the named remote exists."""
-  remotes = subprocess.check_output(['git', 'remote'],
-                                    cwd=cwd).strip().splitlines()
+  remotes = proc.check_output(['git', 'remote'],
+                              cwd=cwd).strip().splitlines()
   return remote in remotes
 
 
@@ -233,8 +234,8 @@
     return
   remote = GITHUB_SSH + '/'.join(GitRemoteUrl(cwd, 'origin').split('/')[-2:])
   print '%s has no github remote, adding %s' % (cwd, remote)
-  subprocess.check_call(['git', 'remote', 'add', GITHUB_REMOTE, remote],
-                        cwd=cwd)
+  proc.check_call(['git', 'remote', 'add', GITHUB_REMOTE, remote],
+                  cwd=cwd)
 
 
 def GitCloneFetchCheckout(name, work_dir, git_repo, checkout='origin/master'):
@@ -243,11 +244,11 @@
     print '%s directory already exists' % name
   else:
     print 'Cloning %s from %s into %s' % (name, git_repo, work_dir)
-    subprocess.check_call(['git', 'clone', git_repo, work_dir])
+    proc.check_call(['git', 'clone', git_repo, work_dir])
   print 'Syncing %s' % name
-  subprocess.check_call(['git', 'fetch'], cwd=work_dir)
+  proc.check_call(['git', 'fetch'], cwd=work_dir)
   print 'Checking out %s' % checkout
-  subprocess.check_call(['git', 'checkout', checkout], cwd=work_dir)
+  proc.check_call(['git', 'checkout', checkout], cwd=work_dir)
   PrintCurrentGitRev(work_dir)
   AddGithubRemote(work_dir)
 
@@ -258,24 +259,24 @@
   The upstream repository is in Subversion. Use `git pull --rebase` instead of
   git pull: llvm.org/docs/GettingStarted.html#git-mirror
   """
-  subprocess.check_call(
+  proc.check_call(
       ['git', 'config', 'branch.master.rebase', 'true'], cwd=cwd)
 
 
 def PrintCurrentGitRev(cwd):
-  log = subprocess.check_output(
+  log = proc.check_output(
       ['git', 'log', '--oneline', '-n1'], cwd=cwd).strip()
-  remote = subprocess.check_output(
+  remote = proc.check_output(
       ['git', 'config', '--get', 'remote.origin.url'], cwd=cwd).strip()
   sys.stdout.write('%s from remote %s is at revision %s\n' %
                    (cwd, remote, log))
 
 
 def FindPriorRev(path, goal):
-  revs = subprocess.check_output(
+  revs = proc.check_output(
       ['git', 'rev-list', 'origin/master'], cwd=path).splitlines()
   for rev in revs:
-    num = subprocess.check_output(
+    num = proc.check_output(
         ['git', 'svn', 'find-rev', rev], cwd=path).strip()
     if int(num) <= goal:
       return rev
@@ -288,23 +289,23 @@
     print 'LLVM and Clang directories already exist'
   else:
     print 'Cloning LLVM and Clang'
-    subprocess.check_call(['git', 'clone', LLVM_GIT, LLVM_SRC_DIR])
+    proc.check_call(['git', 'clone', LLVM_GIT, LLVM_SRC_DIR])
     GitConfigRebaseMaster(LLVM_SRC_DIR)
-    subprocess.check_call(['git', 'clone', CLANG_GIT, CLANG_SRC_DIR])
+    proc.check_call(['git', 'clone', CLANG_GIT, CLANG_SRC_DIR])
     GitConfigRebaseMaster(CLANG_SRC_DIR)
   print 'Syncing LLVM'
-  subprocess.check_call(['git', 'fetch'], cwd=LLVM_SRC_DIR)
-  subprocess.check_call(['git', 'checkout', LLVM_REVISION], cwd=LLVM_SRC_DIR)
+  proc.check_call(['git', 'fetch'], cwd=LLVM_SRC_DIR)
+  proc.check_call(['git', 'checkout', LLVM_REVISION], cwd=LLVM_SRC_DIR)
   print 'Getting SVN rev'
   global LLVM_SVN_REV
-  LLVM_SVN_REV = int(subprocess.check_output(
+  LLVM_SVN_REV = int(proc.check_output(
       ['git', 'svn', 'find-rev', 'HEAD'], cwd=LLVM_SRC_DIR).strip())
   print 'SVN REV: %d' % LLVM_SVN_REV
   print 'Finding prior Clang rev'
-  subprocess.check_call(['git', 'fetch'], cwd=CLANG_SRC_DIR)
+  proc.check_call(['git', 'fetch'], cwd=CLANG_SRC_DIR)
   prior_rev = FindPriorRev(CLANG_SRC_DIR, LLVM_SVN_REV)
   print 'Checking out Clang rev: %s' % prior_rev
-  subprocess.check_call(['git', 'checkout', prior_rev], cwd=CLANG_SRC_DIR)
+  proc.check_call(['git', 'checkout', prior_rev], cwd=CLANG_SRC_DIR)
   PrintCurrentGitRev(LLVM_SRC_DIR)
   PrintCurrentGitRev(CLANG_SRC_DIR)
 
@@ -316,12 +317,12 @@
     print 'Cloning Prebuilt Chromium Clang directory'
     Mkdir(PREBUILT_CLANG)
     Mkdir(PREBUILT_CLANG_TOOLS)
-    subprocess.check_call(
+    proc.check_call(
         ['git', 'clone', PREBUILT_CLANG_GIT, PREBUILT_CLANG_TOOLS_CLANG])
   print 'Syncing Prebuilt Chromium Clang scripts'
-  subprocess.check_call(['git', 'fetch'], cwd=PREBUILT_CLANG_TOOLS_CLANG)
+  proc.check_call(['git', 'fetch'], cwd=PREBUILT_CLANG_TOOLS_CLANG)
   print 'Syncing Prebuilt Chromium Clang'
-  subprocess.check_call(
+  proc.check_call(
       [os.path.join(PREBUILT_CLANG_TOOLS_CLANG, 'scripts', 'update.py')])
   assert os.path.isfile(CC), 'Expect clang at %s' % CC
   assert os.path.isfile(CXX), 'Expect clang++ at %s' % CXX
@@ -339,7 +340,7 @@
     with open(OCAML_TAR, 'wb') as out:
       out.write(f.read())
     print 'Download done, untar %s' % OCAML_TAR
-    subprocess.check_call(['tar', '-xvf', OCAML_TAR], cwd=WORK_DIR)
+    proc.check_call(['tar', '-xvf', OCAML_TAR], cwd=WORK_DIR)
     assert os.path.isdir(OCAML_DIR), 'Untar should produce %s' % OCAML_DIR
 
 
@@ -370,7 +371,7 @@
   BuildStep('Build LLVM')
   print 'Running cmake on llvm'
   Mkdir(LLVM_OUT_DIR)
-  subprocess.check_call(
+  proc.check_call(
       ['cmake', '-G', 'Ninja', LLVM_SRC_DIR,
        '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
        '-DLLVM_BUILD_TESTS=ON',
@@ -382,29 +383,29 @@
        '-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly',
        '-DLLVM_TARGETS_TO_BUILD=X86'], cwd=LLVM_OUT_DIR)
   print 'Running ninja'
-  subprocess.check_call(['ninja'], cwd=LLVM_OUT_DIR)
+  proc.check_call(['ninja'], cwd=LLVM_OUT_DIR)
 
 
 def TestLLVM():
   BuildStep('Test LLVM')
-  subprocess.check_call(['ninja', 'check-all'], cwd=LLVM_OUT_DIR)
+  proc.check_call(['ninja', 'check-all'], cwd=LLVM_OUT_DIR)
 
 
 def InstallLLVM():
   BuildStep('Install LLVM')
   Remove(LLVM_INSTALL_DIR)
-  subprocess.check_call(['ninja', 'install'], cwd=LLVM_OUT_DIR)
+  proc.check_call(['ninja', 'install'], cwd=LLVM_OUT_DIR)
 
 
 def BuildSexpr():
   BuildStep('Build Sexpr')
   # sexpr-wasm builds in its own in-tree out/ folder. The build is fast, so
   # always clobber.
-  subprocess.check_call(['make', 'clean'], cwd=SEXPR_SRC_DIR)
-  subprocess.check_call(['make',
-                         'CC=%s' % CC,
-                         'CXX=%s' % CXX],
-                        cwd=SEXPR_SRC_DIR)
+  proc.check_call(['make', 'clean'], cwd=SEXPR_SRC_DIR)
+  proc.check_call(['make',
+                   'CC=%s' % CC,
+                   'CXX=%s' % CXX],
+                  cwd=SEXPR_SRC_DIR)
   sexpr = os.path.join(SEXPR_OUT_DIR, 'sexpr-wasm')
   CopyBinaryToArchive(sexpr)
 
@@ -414,10 +415,10 @@
   makefile = os.path.join(OCAML_DIR, 'config', 'Makefile')
   if not os.path.isfile(makefile):
     configure = os.path.join(OCAML_DIR, 'configure')
-    subprocess.check_call(
+    proc.check_call(
         [configure, '-prefix', OCAML_OUT_DIR, '-cc', CC], cwd=OCAML_DIR)
-  subprocess.check_call(['make', 'world.opt', '-j%s' % NPROC], cwd=OCAML_DIR)
-  subprocess.check_call(['make', 'install'], cwd=OCAML_DIR)
+  proc.check_call(['make', 'world.opt', '-j%s' % NPROC], cwd=OCAML_DIR)
+  proc.check_call(['make', 'install'], cwd=OCAML_DIR)
   ocamlbuild = os.path.join(OCAML_BIN_DIR, 'ocamlbuild')
   assert os.path.isfile(ocamlbuild), 'Expected installed %s' % ocamlbuild
   os.environ['PATH'] = OCAML_BIN_DIR + os.pathsep + os.environ['PATH']
@@ -426,8 +427,8 @@
 def BuildSpec():
   BuildStep('Build spec')
   # Spec builds in-tree. Always clobber and run the tests.
-  subprocess.check_call(['make', 'clean'], cwd=ML_DIR)
-  subprocess.check_call(['make', 'all'], cwd=ML_DIR)
+  proc.check_call(['make', 'clean'], cwd=ML_DIR)
+  proc.check_call(['make', 'all'], cwd=ML_DIR)
   wasm = os.path.join(ML_DIR, 'wasm.opt')
   CopyBinaryToArchive(wasm)
 
@@ -435,13 +436,13 @@
 def BuildBinaryen():
   BuildStep('Build binaryen')
   Mkdir(BINARYEN_OUT_DIR)
-  subprocess.check_call(
+  proc.check_call(
       ['cmake', '-G', 'Ninja', BINARYEN_SRC_DIR,
        '-DCMAKE_C_COMPILER=' + CC,
        '-DCMAKE_CXX_COMPILER=' + CXX],
       cwd=BINARYEN_OUT_DIR)
   print 'Running ninja'
-  subprocess.check_call(['ninja'], cwd=BINARYEN_OUT_DIR)
+  proc.check_call(['ninja'], cwd=BINARYEN_OUT_DIR)
   assert os.path.isdir(BINARYEN_BIN_DIR), 'Expected %s' % BINARYEN_BIN_DIR
   for node in os.listdir(BINARYEN_BIN_DIR):
     f = os.path.join(BINARYEN_BIN_DIR, node)
diff --git a/buildbot/proc.py b/buildbot/proc.py
new file mode 100755
index 0000000..8ca0c67
--- /dev/null
+++ b/buildbot/proc.py
@@ -0,0 +1,40 @@
+#! /usr/bin/env python
+
+#   Copyright 2016 WebAssembly Community Group participants
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+# This module is intended to be a drop-in replacement for the standard
+# subprocess module, with the difference that it logs commands before it runs
+# them. Everything not overriden should pass through to the subprocess module
+# via the import trick below.
+
+# Imports subprocess in its own namespace so we can always refer directly to
+# its attributes.
+import subprocess
+# Imports all of subprocess into the current namespace, effectively re-exporting
+# everything.
+from subprocess import *
+
+
+# Now we can override any parts of subprocess we want, while leaving the rest.
+def check_call(cmd, **kwargs):
+  print 'check_call Running command:'
+  print ' '.join('"' + c + '"' if ' ' in c else c  for c in cmd)
+  return subprocess.check_call(cmd, **kwargs)
+
+
+def check_output(cmd, **kwargs):
+  print 'check_output Running command:'
+  print ' '.join('"' + c + '"' if ' ' in c else c  for c in cmd)
+  return subprocess.check_output(cmd, **kwargs)
diff --git a/buildbot/testing.py b/buildbot/testing.py
index 0fa76dd..15d078a 100755
--- a/buildbot/testing.py
+++ b/buildbot/testing.py
@@ -15,13 +15,14 @@
 #   limitations under the License.
 
 import difflib
-import os
-import os.path
 import math
 import multiprocessing
-import subprocess
+import os
+import os.path
 import sys
 
+# Use proc instead of subprocess to get logged subprocess invocation.
+import proc
 
 class Result:
   """Result from a single test that was run."""
@@ -62,16 +63,15 @@
     """Execute a single test."""
     basename = os.path.basename(test_file)
     outfile = self.outname_ctor(self.outdir, test_file)
-    # Print something simple so buildbots don't think the script is dead.
-    sys.stdout.write('.')
-    sys.stdout.flush()
     try:
-      output = subprocess.check_output(
+      output = proc.check_output(
           self.command_ctor(test_file, outfile, self.extras),
-          stderr=subprocess.STDOUT, cwd=self.outdir)
+          stderr=proc.STDOUT, cwd=self.outdir)
+      # Flush the logged command sobuildbots don't think the script is dead.
+      sys.stdout.flush()
       assert os.path.isfile(outfile), 'Missing output file %s' % outfile
       return Result(test=basename, success=True, output=output)
-    except subprocess.CalledProcessError as e:
+    except proc.CalledProcessError as e:
       return Result(test=basename, success=False, output=e.output)