Parallelize binaryen js spec tests
diff --git a/check.py b/check.py index 0618c59..c6ae21e 100755 --- a/check.py +++ b/check.py
@@ -393,6 +393,60 @@ shared.num_failures += len(result.errors) + len(result.failures) +def run_binaryenjs_test_with_wrapped_stdout(s: Path): + """Return (bool, str) where the first element is whether the test was + successful and the second is the combined stdout and stderr of the test. + """ + out = io.StringIO() + try: + binaryenjs.run_one_binaryen_js_test(str(s), stdout=out) + except Exception as e: + shared.num_failures += 1 + # Serialize exceptions into the output string buffer + # so they can be reported on the main thread. + print(e, file=out) + return False, out.getvalue() + return True, out.getvalue() + + +def run_binaryenjs_tests(): + if not (shared.MOZJS or shared.NODEJS): + shared.fail_with_error('no vm to run binaryen.js tests') + + if not os.path.exists(shared.BINARYEN_JS): + shared.fail_with_error('no ' + shared.BINARYEN_JS + ' build to test') + + print('\n[ checking binaryen.js testcases (' + shared.BINARYEN_JS + ')... ]\n') + + worker_count = os.cpu_count() + print("Running with", worker_count, "workers") + tests = (Path(x) for x in shared.get_tests(shared.get_test_dir('binaryen.js'), ['.js'])) + + failed_stdouts = [] + with ThreadPool(processes=worker_count) as pool: + try: + for success, stdout in pool.imap_unordered(run_binaryenjs_test_with_wrapped_stdout, tests): + if success: + print(stdout, end="") + continue + + failed_stdouts.append(stdout) + if shared.options.abort_on_first_failure: + with red_stderr(): + print("Aborted binaryen.js test suite execution after first failure. Set --no-fail-fast to disable this.", file=sys.stderr) + break + except KeyboardInterrupt: + # Hard exit to avoid threads continuing to run after Ctrl-C. + # There's no concern of deadlocking during shutdown here. + os._exit(1) + + if failed_stdouts: + with red_stderr(): + print("Failed tests:", file=sys.stderr) + for failed in failed_stdouts: + print(failed, end="", file=sys.stderr) + + @shared.with_pass_debug() def run_lit(): lit_script = os.path.join(shared.options.binaryen_bin, 'binaryen-lit') @@ -441,7 +495,7 @@ 'validator': run_validator_tests, 'example': run_example_tests, 'unit': run_unittest, - 'binaryenjs': binaryenjs.test_binaryen_js, + 'binaryenjs': run_binaryenjs_tests, 'lit': run_lit, 'gtest': run_gtest, }
diff --git a/scripts/test/binaryenjs.py b/scripts/test/binaryenjs.py index 97d84bb..ff0083b 100644 --- a/scripts/test/binaryenjs.py +++ b/scripts/test/binaryenjs.py
@@ -14,6 +14,7 @@ import os import subprocess +from pathlib import Path from . import shared, support @@ -34,9 +35,12 @@ ''' -def make_js_test(input_js_file, binaryen_js): - basename = os.path.basename(input_js_file) - outname = os.path.splitext(basename)[0] + '.mjs' +def make_js_test(input_js_file, binaryen_js, base_name=None): + if base_name: + outname = base_name + '.mjs' + else: + basename = os.path.basename(input_js_file) + outname = os.path.splitext(basename)[0] + '.mjs' with open(outname, 'w') as f: f.write(make_js_test_header(binaryen_js)) test_src = open(input_js_file).read() @@ -44,38 +48,50 @@ return outname +def run_one_binaryen_js_test(s, stdout=None): + node_has_wasm = shared.NODEJS and support.node_has_webassembly(shared.NODEJS) + if not os.path.exists(shared.BINARYEN_JS): + shared.fail_with_error('no ' + shared.BINARYEN_JS + ' build to test') + + # /path/to/binaryen/test/binaryen.js/foo.js -> test-binaryen.js-foo + base_name = "-".join(Path(s).relative_to(Path(shared.options.binaryen_root)).with_suffix("").parts) + + print('..', s, file=stdout) + outname = make_js_test(s, shared.BINARYEN_JS, base_name=base_name) + + def test(cmd): + if 'fatal' not in s: + out = support.run_command(cmd, stderr=subprocess.STDOUT, stdout=stdout) + else: + # expect an error - the specific error code will depend on the vm + out = support.run_command(cmd, stderr=subprocess.STDOUT, expected_status=None, stdout=stdout) + expected_file = s + '.txt' + expected = open(expected_file).read() + if expected not in out: + shared.fail(out, expected) + + # run in all possible shells + if shared.MOZJS: + test([shared.MOZJS, '-m', outname]) + if shared.NODEJS: + test_src = open(s).read() + if node_has_wasm or 'WebAssembly.' not in test_src: + test([shared.NODEJS, outname]) + else: + print('Skipping ' + s + ' because WebAssembly might not be supported', file=stdout) + + def test_binaryen_js(): if not (shared.MOZJS or shared.NODEJS): shared.fail_with_error('no vm to run binaryen.js tests') - node_has_wasm = shared.NODEJS and support.node_has_webassembly(shared.NODEJS) if not os.path.exists(shared.BINARYEN_JS): shared.fail_with_error('no ' + shared.BINARYEN_JS + ' build to test') print('\n[ checking binaryen.js testcases (' + shared.BINARYEN_JS + ')... ]\n') for s in shared.get_tests(shared.get_test_dir('binaryen.js'), ['.js']): - outname = make_js_test(s, shared.BINARYEN_JS) - - def test(cmd): - if 'fatal' not in s: - out = support.run_command(cmd, stderr=subprocess.STDOUT) - else: - # expect an error - the specific error code will depend on the vm - out = support.run_command(cmd, stderr=subprocess.STDOUT, expected_status=None) - expected = open(os.path.join(shared.options.binaryen_test, 'binaryen.js', s + '.txt')).read() - if expected not in out: - shared.fail(out, expected) - - # run in all possible shells - if shared.MOZJS: - test([shared.MOZJS, '-m', outname]) - if shared.NODEJS: - test_src = open(s).read() - if node_has_wasm or 'WebAssembly.' not in test_src: - test([shared.NODEJS, outname]) - else: - print('Skipping ' + s + ' because WebAssembly might not be supported') + run_one_binaryen_js_test(s) def update_binaryen_js_tests():