| #!/usr/bin/env python3 | |
| # Copyright (C) 2018 Bocoup LLC. All rights reserved. | |
| # Copyright (C) 2026 Apple Inc. All rights reserved. | |
| # | |
| # Redistribution and use in source and binary forms, with or without | |
| # modification, are permitted provided that the following conditions | |
| # are met: | |
| # | |
| # 1. Redistributions of source code must retain the above | |
| # copyright notice, this list of conditions and the following | |
| # disclaimer. | |
| # 2. Redistributions in binary form must reproduce the above | |
| # copyright notice, this list of conditions and the following | |
| # disclaimer in the documentation and/or other materials | |
| # provided with the distribution. | |
| # | |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER "AS IS" AND ANY | |
| # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |
| # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |
| # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE | |
| # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, | |
| # OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, | |
| # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR | |
| # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR | |
| # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF | |
| # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | |
| # SUCH DAMAGE. | |
| """Test262 test runner for WebKit's JavaScriptCore.""" | |
| import argparse | |
| import html as html_module | |
| import os | |
| import re | |
| import shutil | |
| import subprocess | |
| import sys | |
| import tempfile | |
| import time | |
| from pathlib import Path | |
| from typing import Any, Dict, List, NoReturn, Optional, Set, Tuple | |
| # Make webkitcorepy/webkitpy importable | |
| _MODULE_DIR = Path(__file__).resolve().parent | |
| _SCRIPTS_DIR = _MODULE_DIR.parent | |
| _WEBKIT_ROOT = _SCRIPTS_DIR.parent.parent | |
| for _lib_path in (str(_SCRIPTS_DIR), str(_SCRIPTS_DIR / "libraries" / "webkitcorepy")): | |
| if _lib_path not in sys.path: | |
| sys.path.insert(0, _lib_path) | |
| from webkitcorepy import TaskPool, Version # noqa: E402 | |
| from webkitpy.common.system.executive import Executive # noqa: E402 | |
| from webkitpy.common.system.filesystem import FileSystem # noqa: E402 | |
| from webkitpy.common.system.platforminfo import PlatformInfo # noqa: E402 | |
| from webkitpy.common.webkit_finder import WebKitFinder # noqa: E402 | |
| try: | |
| import yaml | |
| from yaml.events import AliasEvent, ScalarEvent | |
| except ImportError: | |
| print( | |
| "Error: PyYAML is required. Install with: pip3 install pyyaml", | |
| file=sys.stderr, | |
| ) | |
| sys.exit(1) | |
| try: | |
| _YamlLoader = yaml.CSafeLoader | |
| except AttributeError: | |
| _YamlLoader = yaml.SafeLoader | |
| class _YamlDumper(yaml.SafeDumper): | |
| analysis: Any | |
| def check_simple_key(self) -> bool: | |
| if isinstance(self.event, AliasEvent): | |
| return True | |
| if isinstance(self.event, ScalarEvent): | |
| if self.analysis is None: | |
| self.analysis = self.analyze_scalar(self.event.value) | |
| return not self.analysis.empty and not self.analysis.multiline | |
| return self.check_empty_sequence() or self.check_empty_mapping() | |
| def choose_scalar_style(self) -> str: | |
| # When PyYAML would emit a string as single-quoted (e.g. because of | |
| # a `: ` that blocks plain style), and the string itself contains a | |
| # single quote, prefer double-quoted style so we get | |
| # `"...'foo'..."` rather than the ugly `'...''foo''...'` doubling. | |
| style = super().choose_scalar_style() | |
| if style == "'" and "'" in self.event.value: | |
| return '"' | |
| return style | |
| # --- Path Constants --- | |
| _DEFAULT_TEST262_DIR = _WEBKIT_ROOT / "JSTests" / "test262" | |
| _DEFAULT_CONFIG = _DEFAULT_TEST262_DIR / "config.yaml" | |
| _DEFAULT_EXPECTATIONS = _DEFAULT_TEST262_DIR / ("expectations-linux.yaml" if sys.platform.startswith("linux") else "expectations.yaml") | |
| _REPORT_CSS = _MODULE_DIR / "report.css" | |
| DEFAULT_HARNESS_FILES = ["sta.js", "assert.js"] | |
| ASYNC_HARNESS_FILES = ["doneprintHandle.js"] | |
| _METADATA_RE = re.compile( | |
| r"/\*(---[\r\n]+[\S\s]*)[\r\n]+\s*---\*/", re.MULTILINE | |
| ) | |
| _ERROR_RE = re.compile(r"^Exception: ([\w\d]+: .*)", re.MULTILINE) | |
| _TEST_FILE_RE = re.compile(r"(?<!_FIXTURE)\.[jJ][sS]$") | |
| # --- Worker Process State --- | |
| # Set once per worker by _worker_setup (which TaskPool runs inside each worker | |
| # process) and read by _run_batch. A module global works across both fork and | |
| # spawn start methods because the setup function runs in-process in every | |
| # worker rather than relying on inherited parent state. | |
| _WORKER_CTX: Optional[Dict[str, Any]] = None | |
| def _worker_setup(ctx: Dict[str, Any]) -> None: | |
| global _WORKER_CTX | |
| _WORKER_CTX = ctx | |
| def _run_batch(filepaths: List[str]) -> List[Dict[str, Any]]: | |
| assert _WORKER_CTX is not None | |
| results: List[Dict[str, Any]] = [] | |
| for fp in filepaths: | |
| try: | |
| results.extend(_process_test_file(fp, _WORKER_CTX)) | |
| except Exception as e: | |
| rel_path = os.path.relpath(fp, _WORKER_CTX["test262_dir"]) | |
| results.append( | |
| { | |
| "path": rel_path, | |
| "mode": "default", | |
| "time": 0, | |
| "passed": False, | |
| "error": f"Worker error: {e}", | |
| "output": str(e), | |
| "features": [], | |
| "exit_code": 1, | |
| "skipped": False, | |
| "is_crash": True, | |
| } | |
| ) | |
| return results | |
| # --- Utilities --- | |
| def _die(msg: str) -> NoReturn: | |
| print(f"Error: {msg}", file=sys.stderr) | |
| sys.exit(1) | |
| def _load_yaml(path: str) -> Any: | |
| with open(path, "r", encoding="utf-8") as f: | |
| return yaml.load(f, Loader=_YamlLoader) | |
| def _read_file(path: str) -> str: | |
| with open(path, "r", encoding="utf-8", errors="replace") as f: | |
| return f.read() | |
| def _parse_metadata(contents: str, filepath: str) -> Dict[str, Any]: | |
| m = _METADATA_RE.search(contents) | |
| if not m: | |
| return {} | |
| try: | |
| return yaml.load(m.group(1), Loader=_YamlLoader) or {} | |
| except yaml.YAMLError as e: | |
| print(f"Warning: YAML parse error in {filepath}: {e}", file=sys.stderr) | |
| return {} | |
| def _parse_error(output: Optional[str]) -> Optional[str]: | |
| if not output: | |
| return None | |
| m = _ERROR_RE.search(output) | |
| if m: | |
| return m.group(1) | |
| return output.split("\n")[0] | |
| # --- Platform Detection --- | |
| def _detect_platform(release: bool, sanitizer: Optional[str]) -> Dict[str, str]: | |
| info = PlatformInfo() | |
| # Normalize webkitpy's short os names to the labels used in config.yaml's | |
| # skip conditions (comparisons are case-insensitive, but keep them tidy). | |
| os_label = {"mac": "macOS", "linux": "Linux", "win": "Windows"}.get( | |
| info.os_name, info.os_name | |
| ) | |
| return { | |
| "os": os_label, | |
| "os_version": str(info.os_version) if info.os_version else "", | |
| "build": "release" if release else "debug", | |
| "sanitizer": sanitizer or "", | |
| } | |
| # --- Skip / Filter Logic --- | |
| def _evaluate_condition(condition: Dict[str, str], platform_info: Dict[str, str]) -> bool: | |
| for key, value in condition.items(): | |
| if key == "os": | |
| if platform_info["os"].lower() != value.lower(): | |
| return False | |
| elif key == "os_version": | |
| if not platform_info["os_version"]: | |
| return False | |
| try: | |
| if not Version.from_string(platform_info["os_version"]).matches(str(value)): | |
| return False | |
| except (ValueError, TypeError): | |
| return False | |
| elif key == "build": | |
| if platform_info["build"] != value.lower(): | |
| return False | |
| elif key == "sanitizer": | |
| if platform_info["sanitizer"].lower() != value.lower(): | |
| return False | |
| return True | |
| def _should_skip( | |
| rel_path: str, | |
| metadata: Dict[str, Any], | |
| config: Dict[str, Any], | |
| skip_set: Set[str], | |
| skip_paths: List[str], | |
| skip_features_set: Set[str], | |
| filter_features: Set[str], | |
| platform_info: Dict[str, str], | |
| conditional_skips: List[Dict[str, Any]], | |
| ) -> bool: | |
| skip_section = config.get("skip") if config else None | |
| if skip_section: | |
| if rel_path in skip_set: | |
| return True | |
| for pattern in skip_paths: | |
| if re.search(pattern, rel_path): | |
| return True | |
| test_features = metadata.get("features") or [] | |
| for feature in test_features: | |
| if feature in skip_features_set: | |
| if not filter_features or feature not in filter_features: | |
| return True | |
| if filter_features: | |
| if not any(f in filter_features for f in test_features): | |
| return True | |
| elif filter_features: | |
| test_features = metadata.get("features") or [] | |
| if not any(f in filter_features for f in test_features): | |
| return True | |
| for cond in conditional_skips: | |
| if_clause = cond.get("if", {}) | |
| if not _evaluate_condition(if_clause, platform_info): | |
| continue | |
| cond_files = set(cond.get("files") or []) | |
| if rel_path in cond_files: | |
| return True | |
| cond_paths = cond.get("paths") or [] | |
| for pattern in cond_paths: | |
| if re.search(pattern, rel_path): | |
| return True | |
| cond_features = set(cond.get("features") or []) | |
| test_features = metadata.get("features") or [] | |
| for feature in test_features: | |
| if feature in cond_features: | |
| if not filter_features or feature not in filter_features: | |
| return True | |
| return False | |
| # --- Scenarios --- | |
| def _get_scenarios(flags: List[str]) -> List[str]: | |
| if not flags: | |
| return ["strict mode", "default"] | |
| if "raw" in flags: | |
| return ["raw"] | |
| if "noStrict" in flags: | |
| return ["default"] | |
| if "onlyStrict" in flags: | |
| return ["strict mode"] | |
| if "module" in flags: | |
| return ["module"] | |
| return ["strict mode", "default"] | |
| # --- Harness --- | |
| def _compile_harness(filenames: List[str], harness_dir: str) -> str: | |
| content = "" | |
| for name in filenames: | |
| path = os.path.join(harness_dir, name) | |
| with open(path, "r", encoding="utf-8", errors="replace") as f: | |
| content += f.read() | |
| return content | |
| def _compile_includes( | |
| includes: List[str], harness_dir: str, temp_dir: str | |
| ) -> Optional[str]: | |
| content = "" | |
| for name in includes: | |
| path = os.path.join(harness_dir, name) | |
| try: | |
| with open(path, "r", encoding="utf-8", errors="replace") as f: | |
| content += f.read() | |
| except FileNotFoundError: | |
| pass | |
| if not content: | |
| return None | |
| fd, path = tempfile.mkstemp(dir=temp_dir, suffix=".js") | |
| with os.fdopen(fd, "w", encoding="utf-8") as f: | |
| f.write(content) | |
| return path | |
| def _get_feature_flags(config: Dict[str, Any], metadata: Dict[str, Any]) -> List[str]: | |
| flags = [] | |
| flag_map = config.get("flags") if config else None | |
| if flag_map and metadata.get("features"): | |
| for feature in metadata["features"]: | |
| flag_name = flag_map.get(feature) | |
| if flag_name: | |
| flags.append(f"--{flag_name}=1") | |
| test_flags = metadata.get("flags") or [] | |
| if "CanBlockIsFalse" in test_flags: | |
| flags.append("--can-block-is-false") | |
| return flags | |
| # --- JSC Execution --- | |
| def _run_jsc( | |
| jsc: str, | |
| filepath: str, | |
| scenario: str, | |
| metadata: Dict[str, Any], | |
| feature_flags: List[str], | |
| default_harness_file: Optional[str], | |
| async_harness_file: Optional[str], | |
| includes_file: Optional[str], | |
| timeout: Optional[int], | |
| env: Dict[str, str], | |
| ) -> Tuple[int, Optional[str], float]: | |
| cmd = [jsc] | |
| cmd.extend(feature_flags) | |
| if timeout: | |
| cmd.append(f"--watchdog={timeout}") | |
| negative = metadata.get("negative") | |
| if negative and negative.get("type"): | |
| cmd.append(f"--exception={negative['type']}") | |
| test_flags = metadata.get("flags") or [] | |
| is_async = "async" in test_flags | |
| if is_async: | |
| cmd.append("--test262-async") | |
| if scenario != "raw" and default_harness_file: | |
| cmd.append(default_harness_file) | |
| if is_async and scenario != "raw" and async_harness_file: | |
| cmd.append(async_harness_file) | |
| if includes_file: | |
| cmd.append(includes_file) | |
| if scenario == "module": | |
| cmd.append(f"--module-file={filepath}") | |
| elif scenario == "strict mode": | |
| cmd.append(f"--strict-file={filepath}") | |
| else: | |
| cmd.append(filepath) | |
| process_timeout = (timeout / 1000 + 60) if timeout else 300 | |
| start = time.monotonic() | |
| try: | |
| proc = subprocess.run( | |
| cmd, | |
| stdout=subprocess.PIPE, | |
| stderr=subprocess.STDOUT, | |
| text=True, | |
| env=env, | |
| timeout=process_timeout, | |
| ) | |
| elapsed = time.monotonic() - start | |
| # Python reports signal kills as negative returncodes; normalize to | |
| # the shell convention (128+N) so `& 0x7f` crash detection works. | |
| returncode = proc.returncode | |
| if returncode < 0: | |
| returncode = 128 + (-returncode) | |
| if returncode != 0: | |
| # Match Perl's `chomp`: remove trailing newlines but preserve | |
| # trailing spaces on the last line (some Test262Error messages | |
| # have a meaningful trailing space). | |
| output = (proc.stdout or "").rstrip("\n\r") | |
| return returncode, output if output else None, elapsed | |
| return 0, None, elapsed | |
| except subprocess.TimeoutExpired: | |
| elapsed = time.monotonic() - start | |
| return 1, "Timeout", elapsed | |
| except Exception as e: | |
| elapsed = time.monotonic() - start | |
| return 1, str(e), elapsed | |
| # --- Test File Processing (Worker) --- | |
| def _process_test_file( | |
| filepath: str, ctx: Dict[str, Any] | |
| ) -> List[Dict[str, Any]]: | |
| test262_dir = ctx["test262_dir"] | |
| rel_path = os.path.relpath(filepath, test262_dir) | |
| try: | |
| contents = _read_file(filepath) | |
| except Exception as e: | |
| return [ | |
| { | |
| "path": rel_path, | |
| "mode": "default", | |
| "time": 0, | |
| "passed": False, | |
| "error": f"Read error: {e}", | |
| "output": str(e), | |
| "features": [], | |
| "exit_code": 1, | |
| "skipped": False, | |
| "is_crash": True, | |
| } | |
| ] | |
| metadata = _parse_metadata(contents, filepath) | |
| features = metadata.get("features") or [] | |
| skip = _should_skip( | |
| rel_path, | |
| metadata, | |
| ctx["config"], | |
| ctx["skip_set"], | |
| ctx["skip_paths"], | |
| ctx["skip_features"], | |
| ctx["filter_features"], | |
| ctx["platform_info"], | |
| ctx["conditional_skips"], | |
| ) | |
| if ctx["skipped_only"]: | |
| skip = not skip | |
| if skip: | |
| if not ctx["skipped_only"]: | |
| return [ | |
| { | |
| "path": rel_path, | |
| "mode": "skip", | |
| "time": 0, | |
| "passed": False, | |
| "error": None, | |
| "output": None, | |
| "features": features, | |
| "exit_code": 0, | |
| "skipped": True, | |
| "is_crash": False, | |
| } | |
| ] | |
| return [] | |
| scenarios = _get_scenarios(metadata.get("flags") or []) | |
| feature_flags = _get_feature_flags(ctx["config"], metadata) | |
| includes = metadata.get("includes") or [] | |
| includes_file = None | |
| if includes: | |
| includes_file = _compile_includes( | |
| includes, ctx["harness_dir"], ctx["temp_dir"] | |
| ) | |
| results = [] | |
| for scenario in scenarios: | |
| exit_code, output, exec_time = _run_jsc( | |
| ctx["jsc"], | |
| filepath, | |
| scenario, | |
| metadata, | |
| feature_flags, | |
| ctx["default_harness_file"], | |
| ctx["async_harness_file"], | |
| includes_file, | |
| ctx["timeout"], | |
| ctx["env"], | |
| ) | |
| error = None | |
| passed = True | |
| is_crash = False | |
| exit_signal = exit_code & 0x7f | |
| if exit_signal == 3: | |
| exit_signal = 0 | |
| is_crash = exit_signal != 0 | |
| if output or is_crash: | |
| error = _parse_error(output) if output else None | |
| if is_crash: | |
| error = f"Bad exit code: {exit_code}" | |
| passed = False | |
| results.append( | |
| { | |
| "path": rel_path, | |
| "mode": scenario, | |
| "time": exec_time, | |
| "passed": passed, | |
| "error": error, | |
| "output": output if not passed else None, | |
| "features": features, | |
| "exit_code": exit_code, | |
| "skipped": False, | |
| "is_crash": is_crash, | |
| } | |
| ) | |
| if includes_file: | |
| try: | |
| os.unlink(includes_file) | |
| except OSError: | |
| pass | |
| return results | |
| # --- Result Classification --- | |
| def _classify_results( | |
| raw_results: List[Dict[str, Any]], | |
| expectations: Optional[Dict[str, Any]], | |
| ) -> List[Dict[str, Any]]: | |
| classified = [] | |
| for r in raw_results: | |
| if r["skipped"]: | |
| r["result"] = "skip" | |
| classified.append(r) | |
| continue | |
| path = r["path"] | |
| mode = r["mode"] | |
| expected_exit = None | |
| if expectations and path in expectations: | |
| expected_exit = expectations[path].get(mode) | |
| if not r["passed"]: | |
| is_new_failure = not ( | |
| expectations is not None | |
| and expected_exit is not None | |
| and r.get("exit_code") == expected_exit | |
| ) | |
| r["result"] = "unexpected_fail" if is_new_failure else "expected_fail" | |
| else: | |
| r["result"] = "unexpected_pass" if expected_exit is not None else "expected_pass" | |
| classified.append(r) | |
| return classified | |
| # --- Test Discovery --- | |
| def _discover_tests(test262_dir: str, test_dirs: List[str]) -> List[str]: | |
| files = [] | |
| for test_dir in test_dirs: | |
| root = os.path.join(test262_dir, test_dir) | |
| # --test-only accepts a single test file as well as a directory. | |
| if os.path.isfile(root): | |
| if _TEST_FILE_RE.search(os.path.basename(root)): | |
| files.append(root) | |
| else: | |
| print(f"Warning: not a test file: {root}", file=sys.stderr) | |
| continue | |
| if not os.path.isdir(root): | |
| print(f"Warning: test path not found: {root}", file=sys.stderr) | |
| continue | |
| for dirpath, _, filenames in os.walk(root): | |
| for f in sorted(filenames): | |
| if _TEST_FILE_RE.search(f): | |
| files.append(os.path.join(dirpath, f)) | |
| return files | |
| def _load_import_file(test262_dir: str) -> List[str]: | |
| import_file = os.path.join(test262_dir, "latest-changes-summary.txt") | |
| if not os.path.exists(import_file): | |
| _die(f"Import file not found: {import_file}") | |
| files = [] | |
| with open(import_file, "r") as f: | |
| for line in f: | |
| line = line.strip() | |
| if line and line[0] in ("A", "M") and "test/" in line: | |
| parts = line.split(None, 1) | |
| if len(parts) == 2: | |
| files.append(os.path.join(test262_dir, parts[1])) | |
| return files | |
| def _find_failing(results_file: str, test262_dir: str) -> List[str]: | |
| if not os.path.exists(results_file): | |
| _die(f"Results file not found: {results_file}") | |
| all_results = _load_yaml(results_file) | |
| if not all_results: | |
| _die("No results found in results file.") | |
| paths = set() | |
| for r in all_results: | |
| if r.get("result", "").endswith("fail"): | |
| paths.add(r["path"]) | |
| return [os.path.join(test262_dir, p) for p in paths] | |
| # --- JSC Finder --- | |
| # Disambiguates which jsc binary to run. | |
| _PORT_BUILD_DIR_NAMES: Dict[str, str] = {"gtk": "GTK", "wpe": "WPE", "jsc-only": "JSCOnly"} | |
| def _find_jsc(jsc_path: Optional[str], release: bool, port: Optional[str] = None) -> str: | |
| if jsc_path: | |
| jsc = os.path.abspath(jsc_path) | |
| if not os.path.exists(jsc): | |
| _die(f"--jsc path does not exist: {jsc}") | |
| return jsc | |
| config_name = "Release" if release else "Debug" | |
| finder = WebKitFinder(FileSystem()) | |
| if port: | |
| # These ports build the executable into a "bin" subdirectory rather | |
| # than directly into the configuration directory. | |
| config_dir = Path(finder.path_from_webkit_outputdir(_PORT_BUILD_DIR_NAMES[port], config_name)) | |
| candidates = [config_dir / "bin" / "jsc"] | |
| else: | |
| # WebKitFinder.path_from_webkit_outputdir honors WEBKIT_OUTPUTDIR and | |
| # otherwise falls back to <webkit>/WebKitBuild. | |
| config_dir = Path(finder.path_from_webkit_outputdir(config_name)) | |
| candidates = [ | |
| config_dir / "bin" / "jsc", | |
| config_dir / "jsc", | |
| # Apple ports ship jsc inside the framework, see jscPath() in | |
| # webkitdirs.pm and the equivalent probe in run-jsc-stress-tests. | |
| config_dir / "JavaScriptCore.framework" / "Helpers" / "jsc", | |
| config_dir / "Debug" / "jsc", | |
| config_dir / "Release" / "jsc", | |
| ] | |
| for c in candidates: | |
| if c.exists(): | |
| return str(c) | |
| if port: | |
| _die( | |
| f"Cannot find jsc in {config_dir}.\n" | |
| "Build it, or specify an existing binary with --jsc <path>." | |
| ) | |
| try: | |
| result = subprocess.run( | |
| ["which", "jsc"], capture_output=True, text=True | |
| ) | |
| if result.returncode == 0 and result.stdout.strip(): | |
| return result.stdout.strip() | |
| except Exception: | |
| pass | |
| _die( | |
| f"Cannot find jsc in {config_dir}.\n" | |
| "Try --release or specify with --jsc <path>." | |
| ) | |
| # --- Save Expectations --- | |
| # Written at the top of expectations.yaml on every save. | |
| _EXPECTATIONS_HEADER = """\ | |
| # Expected test262 failures. Generated by Tools/Scripts/test262-runner --save. | |
| # | |
| # Each entry maps a test file to the modes it is expected to fail in ("default", | |
| # "strict mode", "module" or "raw"), and each mode to the exit code jsc is | |
| # expected to terminate with: | |
| # | |
| # 3 The test ran to completion and failed: an assertion in the test or | |
| # harness failed, or the test threw an uncaught exception. This is | |
| # jsc's EXIT_EXCEPTION. | |
| # 128 + N jsc was killed by signal N, i.e. it crashed. Common values are 134 | |
| # (SIGABRT, which includes assertion failures in debug builds), 138 | |
| # (SIGBUS) and 139 (SIGSEGV). | |
| # 1 jsc could not run the test at all, or the runner timed out waiting | |
| # for it. | |
| # | |
| # A failing test counts as expected only if it fails with the exit code recorded | |
| # here, so a test that starts crashing where it previously failed an assertion | |
| # is still reported as a new failure. | |
| """ | |
| def _save_expectations( | |
| expectations_file: str, results: List[Dict[str, Any]], running_all: bool | |
| ) -> None: | |
| failed: Dict[str, Dict[str, int]] = {} | |
| for r in results: | |
| if r["result"].endswith("fail"): | |
| failed.setdefault(r["path"], {})[r["mode"]] = r["exit_code"] | |
| if not running_all: | |
| old: Dict[str, Dict[str, int]] = {} | |
| if os.path.exists(expectations_file): | |
| old = _load_yaml(expectations_file) or {} | |
| run_tests: Dict[str, Set[str]] = {} | |
| for r in results: | |
| if not r["skipped"]: | |
| run_tests.setdefault(r["path"], set()).add(r["mode"]) | |
| for path, modes in run_tests.items(): | |
| if path in old: | |
| for mode in modes: | |
| old[path].pop(mode, None) | |
| if not old[path]: | |
| del old[path] | |
| for path, mode_exits in failed.items(): | |
| old.setdefault(path, {}).update(mode_exits) | |
| failed = old | |
| with open(expectations_file, "w", encoding="utf-8") as f: | |
| f.write(_EXPECTATIONS_HEADER) | |
| yaml.dump( | |
| failed, | |
| f, | |
| Dumper=_YamlDumper, | |
| default_flow_style=False, | |
| allow_unicode=True, | |
| sort_keys=True, | |
| explicit_start=True, | |
| width=2**31 - 1, | |
| ) | |
| print(f"Saved expectation file in: {expectations_file}") | |
| # --- Save Results --- | |
| def _save_results(results_file: str, results: List[Dict[str, Any]]) -> None: | |
| data = [] | |
| for r in results: | |
| entry = { | |
| "path": r["path"], | |
| "mode": r["mode"], | |
| "time": r.get("time", 0), | |
| "result": r["result"], | |
| } | |
| if r.get("error"): | |
| entry["error"] = r["error"] | |
| if r.get("output"): | |
| entry["output"] = r["output"] | |
| if r.get("features"): | |
| entry["features"] = r["features"] | |
| data.append(entry) | |
| with open(results_file, "w", encoding="utf-8") as f: | |
| yaml.dump( | |
| data, | |
| f, | |
| Dumper=_YamlDumper, | |
| default_flow_style=False, | |
| allow_unicode=True, | |
| explicit_start=True, | |
| width=2**31 - 1, | |
| ) | |
| print(f"Saved all the results in {results_file}") | |
| # --- Summary Generation --- | |
| def _summarize_results( | |
| results: List[Dict[str, Any]], results_dir: str | |
| ) -> None: | |
| print("Summarizing results...") | |
| os.makedirs(results_dir, exist_ok=True) | |
| summary_txt = os.path.join(results_dir, "summary.txt") | |
| summary_yaml = os.path.join(results_dir, "summary.yaml") | |
| summary_html = os.path.join(results_dir, "summary.html") | |
| by_feature: Dict[str, List[float]] = {} | |
| by_path: Dict[str, List[float]] = {} | |
| for test in results: | |
| result = test.get("result", "skip") | |
| test_time = test.get("time", 0) or 0 | |
| if test.get("features"): | |
| for feature in test["features"]: | |
| stats = by_feature.setdefault(feature, [0, 0, 0, 0.0]) | |
| if result.endswith("pass"): | |
| stats[0] += 1 | |
| if result.endswith("fail"): | |
| stats[1] += 1 | |
| if result == "skip": | |
| stats[2] += 1 | |
| stats[3] += test_time | |
| parts = test["path"].split("/") | |
| # Skip the first element ("test") and the filename | |
| folder_parts = parts[1:-1] | |
| for i in range(len(folder_parts)): | |
| partial = "/".join(folder_parts[: i + 1]) | |
| stats = by_path.setdefault(partial, [0, 0, 0, 0.0]) | |
| if result.endswith("pass"): | |
| stats[0] += 1 | |
| if result.endswith("fail"): | |
| stats[1] += 1 | |
| if result == "skip": | |
| stats[2] += 1 | |
| stats[3] += test_time | |
| # Write summary.txt | |
| with open(summary_txt, "w") as f: | |
| f.write( | |
| f"{'TOTAL':<6} {'RUN':<6} {'PASS-%':<6} {'PASS':<6} " | |
| f"{'FAIL':<6} {'SKIP':<6} {'TIME':<7} {'AVG':<6} FEATURE\n" | |
| ) | |
| for key in sorted(by_feature): | |
| s = by_feature[key] | |
| total_run = s[0] + s[1] | |
| total = total_run + s[2] | |
| pct = f"{(s[0] / total * 100):.0f}%" if total else "0%" | |
| t = f"{s[3]:.1f}s" | |
| avg = f"{s[3] / total_run:.2f}s" if total_run else "0s" | |
| f.write( | |
| f"{total:<6} {total_run:<6} {pct:<6} {s[0]:<6} " | |
| f"{s[1]:<6} {s[2]:<6} {t:<7} {avg:<6} {key}\n" | |
| ) | |
| f.write( | |
| f"\n\n{'TOTAL':<6} {'RUN':<6} {'PASS-%':<6} {'PASS':<6} " | |
| f"{'FAIL':<6} {'SKIP':<6} {'TIME':<7} {'AVG':<6} FOLDER\n" | |
| ) | |
| for key in sorted(by_path): | |
| s = by_path[key] | |
| total_run = s[0] + s[1] | |
| total = total_run + s[2] | |
| pct = f"{(s[0] / total * 100):.0f}%" if total else "0%" | |
| t = f"{s[3]:.1f}s" | |
| avg = f"{s[3] / total_run:.2f}s" if total_run else "0s" | |
| f.write( | |
| f"{total:<6} {total_run:<6} {pct:<6} {s[0]:<6} " | |
| f"{s[1]:<6} {s[2]:<6} {t:<7} {avg:<6} {key}\n" | |
| ) | |
| # Write summary.html | |
| with open(summary_html, "w") as f: | |
| f.write( | |
| '<html><head><title>Test262 Summaries</title>\n' | |
| '<link rel="stylesheet" href="report.css">\n' | |
| "</head><body>\n" | |
| "<h1>Test262 Summaries</h1>\n" | |
| '<div class="visit">Visit <a href="index.html">the index</a>' | |
| " for a report of failures.</div>\n" | |
| "<h2>By Features</h2>\n" | |
| '<table class="summary-table"><thead>\n' | |
| "<th>Feature</th><th>%</th><th>Total</th><th>Run</th>" | |
| "<th>Passed</th><th>Failed</th><th>Skipped</th>" | |
| "<th>Exec. time</th><th>Avg. time</th>\n" | |
| "</thead><tbody>\n" | |
| ) | |
| for key in sorted(by_feature): | |
| s = by_feature[key] | |
| total_run = s[0] + s[1] | |
| total = total_run + s[2] | |
| iper = (s[0] / total * 100) if total else 0 | |
| pct = f"{iper:.0f}%" | |
| t = f"{s[3]:.1f}s" | |
| avg = f"{s[3] / total_run:.2f}s" if total_run else "0s" | |
| f.write( | |
| f'<tr class="per-{iper}">' | |
| f"<td>{html_module.escape(key)}</td><td>{pct}</td>" | |
| f"<td>{total}</td><td>{total_run}</td>" | |
| f"<td>{s[0]}</td><td>{s[1]}</td><td>{s[2]}</td>" | |
| f"<td>{t}</td><td>{avg}</td></tr>\n" | |
| ) | |
| f.write( | |
| "</tbody></table>\n" | |
| "<h2>By Path</h2>\n" | |
| '<table class="summary-table"><thead>\n' | |
| "<th>Folder</th><th>%</th><th>Total</th><th>Run</th>" | |
| "<th>Passed</th><th>Failed</th><th>Skipped</th>" | |
| "<th>Exec. time</th><th>Avg. time</th>\n" | |
| "</thead><tbody>\n" | |
| ) | |
| for key in sorted(by_path): | |
| s = by_path[key] | |
| total_run = s[0] + s[1] | |
| total = total_run + s[2] | |
| iper = (s[0] / total * 100) if total else 0 | |
| pct = f"{iper:.0f}%" | |
| t = f"{s[3]:.1f}s" | |
| avg = f"{s[3] / total_run:.2f}s" if total_run else "0s" | |
| f.write( | |
| f'<tr class="per-{iper}">' | |
| f"<td>{html_module.escape(key)}</td><td>{pct}</td>" | |
| f"<td>{total}</td><td>{total_run}</td>" | |
| f"<td>{s[0]}</td><td>{s[1]}</td><td>{s[2]}</td>" | |
| f"<td>{t}</td><td>{avg}</td></tr>\n" | |
| ) | |
| f.write( | |
| "</tbody></table>\n" | |
| '<div class="visit">Visit <a href="index.html">the index</a>' | |
| " for a report of failures.</div>\n" | |
| "</body></html>\n" | |
| ) | |
| # Write summary.yaml | |
| summary_data = { | |
| "byFeature": {k: v[:] for k, v in by_feature.items()}, | |
| "byFolder": {k: v[:] for k, v in by_path.items()}, | |
| } | |
| with open(summary_yaml, "w", encoding="utf-8") as f: | |
| yaml.dump( | |
| summary_data, | |
| f, | |
| Dumper=_YamlDumper, | |
| default_flow_style=False, | |
| allow_unicode=True, | |
| width=2**31 - 1, | |
| ) | |
| # --- HTML Index Report --- | |
| def _generate_html_index( | |
| failed: Dict[str, Dict[str, Any]], | |
| results_dir: str, | |
| total_run: int, | |
| fail_count: int, | |
| new_fail_count: int, | |
| skip_count: int, | |
| ) -> None: | |
| os.makedirs(results_dir, exist_ok=True) | |
| index_html = os.path.join(results_dir, "index.html") | |
| with open(index_html, "w") as f: | |
| f.write( | |
| '<html><head><title>Test262 Results</title>\n' | |
| '<link rel="stylesheet" href="report.css">\n' | |
| "</head><body>\n" | |
| "<h1>Test262 Results</h1>\n" | |
| '<div class="visit">Visit <a href="summary.html">the summary</a>' | |
| " for statistics.</div>\n" | |
| "<h2>Stats</h2><ul>\n" | |
| ) | |
| failed_files = len(failed) | |
| total_plus = total_run + skip_count | |
| f.write( | |
| f"<li>{total_run} test files run from {total_plus} files, " | |
| f"{skip_count} skipped test files</li>\n" | |
| f"<li>{fail_count} failures from {failed_files} distinct files, " | |
| f"{new_fail_count} new failures</li>\n" | |
| ) | |
| f.write("</ul><h2>Failures</h2><ul>\n") | |
| for path in sorted(failed): | |
| scenarios = failed[path] | |
| escaped_path = html_module.escape(path) | |
| f.write( | |
| f'<li class="list-item">' | |
| f'<label for="{escaped_path}" class="expander-control">' | |
| f"{escaped_path}</label>\n" | |
| f'<input type="checkbox" id="{escaped_path}" class="expander">\n' | |
| f'<ul class="expand">\n' | |
| ) | |
| for scenario, value in sorted(scenarios.items()): | |
| escaped_value = html_module.escape(str(value)) if value else "" | |
| f.write( | |
| f"<li>{html_module.escape(scenario)}: {escaped_value}</li>\n" | |
| ) | |
| f.write("</ul></li>\n") | |
| f.write( | |
| "</ul>\n" | |
| '<div class="visit">Visit <a href="summary.html">the summary</a>' | |
| " for statistics.</div>\n" | |
| "</body></html>\n" | |
| ) | |
| # --- CLI --- | |
| def _parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser( | |
| description="Test262 runner for WebKit's JavaScriptCore." | |
| ) | |
| parser.add_argument( | |
| "-j", "--jsc", help="Path to JSC binary." | |
| ) | |
| parser.add_argument( | |
| "-t", "--t262", help="Root test262 directory." | |
| ) | |
| parser.add_argument( | |
| "-o", | |
| "--test-only", | |
| action="append", | |
| dest="test_only", | |
| help="Run specific test262 subdirectory (can be repeated).", | |
| ) | |
| parser.add_argument( | |
| "-p", | |
| "--child-processes", | |
| type=int, | |
| dest="child_processes", | |
| help="Number of child processes for parallel execution.", | |
| ) | |
| build_group = parser.add_mutually_exclusive_group() | |
| build_group.add_argument( | |
| "--release", action="store_true", default=False, help="Use Release build of JSC (default)." | |
| ) | |
| build_group.add_argument( | |
| "--debug", action="store_true", default=False, help="Use Debug build of JSC." | |
| ) | |
| port_group = parser.add_mutually_exclusive_group() | |
| port_group.add_argument( | |
| "--gtk", action="store_const", const="gtk", dest="port", | |
| help="Find jsc under the GTK port's build directory (WebKitBuild/GTK).", | |
| ) | |
| port_group.add_argument( | |
| "--wpe", action="store_const", const="wpe", dest="port", | |
| help="Find jsc under the WPE port's build directory (WebKitBuild/WPE).", | |
| ) | |
| port_group.add_argument( | |
| "--jsc-only", action="store_const", const="jsc-only", dest="port", | |
| help="Find jsc under the JSCOnly port's build directory (WebKitBuild/JSCOnly).", | |
| ) | |
| parser.add_argument( | |
| "-v", "--verbose", action="store_true", help="Verbose output." | |
| ) | |
| parser.add_argument( | |
| "-f", | |
| "--features", | |
| action="append", | |
| help="Filter tests by feature (can be repeated).", | |
| ) | |
| parser.add_argument( | |
| "-c", "--config", help="Path to config file." | |
| ) | |
| parser.add_argument( | |
| "-i", | |
| "--ignore-config", | |
| action="store_true", | |
| dest="ignore_config", | |
| help="Ignore config file.", | |
| ) | |
| parser.add_argument( | |
| "--save", | |
| action="store_true", | |
| help="Save current failures to expectations file.", | |
| ) | |
| parser.add_argument( | |
| "-e", "--expectations", help="Path to expectations file." | |
| ) | |
| parser.add_argument( | |
| "-x", | |
| "--ignore-expectations", | |
| action="store_true", | |
| dest="ignore_expectations", | |
| help="Ignore expectations file; report all failures.", | |
| ) | |
| parser.add_argument( | |
| "-F", | |
| "--failing-files", | |
| action="store_true", | |
| dest="failing_files", | |
| help="Re-run only tests that failed in a previous results file.", | |
| ) | |
| parser.add_argument( | |
| "-l", | |
| "--latest-import", | |
| action="store_true", | |
| dest="latest_import", | |
| help="Run only files from the latest import.", | |
| ) | |
| parser.add_argument( | |
| "--stats", | |
| action="store_true", | |
| help="Calculate statistics from a results file.", | |
| ) | |
| parser.add_argument( | |
| "-r", "--results", help="Path to results file (for --stats or --failing-files)." | |
| ) | |
| parser.add_argument( | |
| "--timeout", type=int, help="Timeout per test in milliseconds." | |
| ) | |
| parser.add_argument( | |
| "-S", | |
| "--skipped-files", | |
| action="store_true", | |
| dest="skipped_files", | |
| help="Run only tests skipped by config.", | |
| ) | |
| parser.add_argument( | |
| "--no-progress", | |
| action="store_true", | |
| dest="no_progress", | |
| help="Suppress progress output.", | |
| ) | |
| # --- New features --- | |
| parser.add_argument( | |
| "--filter", | |
| help="Regex filter on test paths. Only tests whose path matches are run.", | |
| ) | |
| parser.add_argument( | |
| "--sanitizer", | |
| help="Declare sanitizer in use (e.g. asan, tsan) for conditional skip rules.", | |
| ) | |
| args = parser.parse_args() | |
| # A build configuration is optional and defaults to Release, matching the | |
| # original runner. --release/--debug are mutually exclusive, so --debug is | |
| # the only thing that moves us off Release; everything downstream reads | |
| # args.release. | |
| args.release = not args.debug | |
| return args | |
| # --- Main --- | |
| def main() -> None: | |
| args = _parse_args() | |
| results_dir = os.path.join(os.getcwd(), "test262-results") | |
| # --- Stats-only mode --- | |
| if args.stats: | |
| results_file = args.results or os.path.join(results_dir, "results.yaml") | |
| if not os.path.exists(results_file): | |
| _die(f"Results file not found: {results_file}") | |
| results = _load_yaml(results_file) | |
| if not results: | |
| _die("No results in results file.") | |
| _summarize_results(results, results_dir) | |
| return | |
| # --- Failing-files: validate results file early --- | |
| if args.failing_files: | |
| results_file = args.results or os.path.join(results_dir, "results.yaml") | |
| if not os.path.exists(results_file): | |
| _die("Cannot find results file. Specify with --results.") | |
| # --- Find JSC --- | |
| jsc = _find_jsc(args.jsc, args.release, args.port) | |
| # --- Resolve paths --- | |
| if args.latest_import: | |
| test262_dir = str(_DEFAULT_TEST262_DIR) | |
| elif args.t262: | |
| test262_dir = os.path.abspath(args.t262) | |
| else: | |
| test262_dir = str(_DEFAULT_TEST262_DIR) | |
| harness_dir = os.path.join(test262_dir, "harness") | |
| if not os.path.exists(harness_dir): | |
| harness_dir = str(_DEFAULT_TEST262_DIR / "harness") | |
| config_file = args.config or str(_DEFAULT_CONFIG) | |
| expectations_file = args.expectations or str(_DEFAULT_EXPECTATIONS) | |
| if args.expectations: | |
| expectations_file = os.path.abspath(args.expectations) | |
| # --- Load config --- | |
| config: Dict[str, Any] = {} | |
| if not args.ignore_config: | |
| if os.path.exists(config_file): | |
| config = _load_yaml(config_file) or {} | |
| elif args.config: | |
| _die(f"Config file {config_file} does not exist.") | |
| # --- Load expectations --- | |
| expectations = None | |
| if ( | |
| not args.ignore_expectations | |
| and not args.skipped_files | |
| and os.path.exists(expectations_file) | |
| ): | |
| expectations = _load_yaml(expectations_file) or {} | |
| # --- Platform info --- | |
| platform_info = _detect_platform(args.release, args.sanitizer) | |
| # --- Build skip sets --- | |
| skip_set: Set[str] = set() | |
| skip_paths: List[str] = [] | |
| skip_features: Set[str] = set() | |
| conditional_skips: List[Dict[str, Any]] = [] | |
| if config and "skip" in config: | |
| skip_section = config["skip"] | |
| skip_set = set(skip_section.get("files") or []) | |
| skip_paths = skip_section.get("paths") or [] | |
| skip_features_raw = skip_section.get("features") or [] | |
| skip_features = set() | |
| for f in skip_features_raw: | |
| m = re.match(r"(\S+)", str(f)) | |
| if m: | |
| skip_features.add(m.group(1)) | |
| conditional_skips = skip_section.get("conditions") or [] | |
| filter_features = set(args.features) if args.features else set() | |
| # --- Environment --- | |
| dyld_path = os.path.dirname(os.path.abspath(jsc)) | |
| env = os.environ.copy() | |
| if "TZ" not in env: | |
| env["TZ"] = "PST" | |
| if platform_info["os"] == "macOS": | |
| env["DYLD_FRAMEWORK_PATH"] = dyld_path | |
| elif platform_info["os"] == "Linux": | |
| lib_dir = os.path.normpath(os.path.join(dyld_path, os.pardir, "lib")) | |
| existing_ld_path = env.get("LD_LIBRARY_PATH") | |
| env["LD_LIBRARY_PATH"] = ( | |
| f"{lib_dir}{os.pathsep}{existing_ld_path}" if existing_ld_path else lib_dir | |
| ) | |
| # --- Process count --- | |
| num_processes = args.child_processes or Executive().cpu_count() or 1 | |
| # --- Print settings --- | |
| print(f"\nSettings:") | |
| print(f"Test262 Dir: {os.path.relpath(test262_dir)}") | |
| print(f"JSC: {os.path.relpath(jsc)}") | |
| print(f"Child Processes: {num_processes}") | |
| if args.timeout: | |
| print(f"Test timeout: {args.timeout}ms") | |
| if args.port: | |
| print(f"Port: {args.port}") | |
| if platform_info["os"] == "macOS": | |
| print(f"DYLD_FRAMEWORK_PATH: {dyld_path}") | |
| elif platform_info["os"] == "Linux": | |
| print(f"LD_LIBRARY_PATH: {env['LD_LIBRARY_PATH']}") | |
| if filter_features: | |
| print(f"Features to include: {', '.join(sorted(filter_features))}") | |
| if args.test_only: | |
| print(f"Paths: {', '.join(args.test_only)}") | |
| if config: | |
| print(f"Config file: {os.path.relpath(config_file)}") | |
| if expectations: | |
| print(f"Expectations file: {os.path.relpath(expectations_file)}") | |
| if args.filter: | |
| print(f"Filter: {args.filter}") | |
| if args.sanitizer: | |
| print(f"Sanitizer: {args.sanitizer}") | |
| if args.failing_files: | |
| print("Running only the failing files in results") | |
| if args.latest_import: | |
| print("Running only the latest imported files") | |
| if args.skipped_files: | |
| print("Running only the skipped files in config.yaml") | |
| if args.verbose: | |
| print("Verbose mode") | |
| print("---\n") | |
| # --- Create temp directory and run --- | |
| with tempfile.TemporaryDirectory() as temp_dir: | |
| # Compile harness | |
| default_content = _compile_harness(DEFAULT_HARNESS_FILES, harness_dir) | |
| default_harness_file = os.path.join(temp_dir, "default_harness.js") | |
| with open(default_harness_file, "w", encoding="utf-8") as f: | |
| f.write(default_content) | |
| async_content = _compile_harness(ASYNC_HARNESS_FILES, harness_dir) | |
| async_harness_file = os.path.join(temp_dir, "async_harness.js") | |
| with open(async_harness_file, "w", encoding="utf-8") as f: | |
| f.write(async_content) | |
| # --- Discover test files --- | |
| test_dirs = args.test_only or ["test"] | |
| running_all = ( | |
| not args.test_only | |
| and not args.latest_import | |
| and not args.failing_files | |
| and not args.filter | |
| and not filter_features | |
| ) | |
| if args.latest_import: | |
| files = _load_import_file(test262_dir) | |
| elif args.failing_files: | |
| rf = args.results or os.path.join(results_dir, "results.yaml") | |
| files = _find_failing(rf, test262_dir) | |
| else: | |
| files = _discover_tests(test262_dir, test_dirs) | |
| # Apply regex filter | |
| if args.filter: | |
| try: | |
| pattern = re.compile(args.filter) | |
| except re.error as e: | |
| _die(f"Invalid --filter regex: {e}") | |
| files = [ | |
| f | |
| for f in files | |
| if pattern.search(os.path.relpath(f, test262_dir)) | |
| ] | |
| if not files: | |
| # Running nothing is never a success: exiting 0 here makes a typo in | |
| # --test-only or --filter look like a clean run. | |
| print("No test files found.", file=sys.stderr) | |
| sys.exit(1) | |
| total_files = len(files) | |
| # --- Build worker context --- | |
| ctx = { | |
| "test262_dir": test262_dir, | |
| "config": config, | |
| "harness_dir": harness_dir, | |
| "jsc": jsc, | |
| "default_harness_file": default_harness_file, | |
| "async_harness_file": async_harness_file, | |
| "timeout": args.timeout, | |
| "env": env, | |
| "skip_set": skip_set, | |
| "skip_paths": skip_paths, | |
| "skip_features": skip_features, | |
| "filter_features": filter_features, | |
| "skipped_only": args.skipped_files, | |
| "temp_dir": temp_dir, | |
| "platform_info": platform_info, | |
| "conditional_skips": conditional_skips, | |
| "verbose": args.verbose, | |
| } | |
| # --- Run tests --- | |
| start_time = time.monotonic() | |
| all_results: List[Dict[str, Any]] = [] | |
| completed = 0 | |
| last_print = 0.0 | |
| # Let's not spam log files too hard | |
| progress_interval = 0.1 if sys.stdout.isatty() else 5.0 | |
| # Batch files per task to amortize IPC/pickle overhead. Aim for ~4 | |
| # batches per worker so there's still some load-balancing headroom. | |
| batch_size = max(1, min(50, total_files // (num_processes * 4) or 1)) | |
| batches = [ | |
| files[i:i + batch_size] | |
| for i in range(0, total_files, batch_size) | |
| ] | |
| def _print_progress(final: bool = False) -> None: | |
| elapsed = time.monotonic() - start_time | |
| rate = completed / elapsed if elapsed > 0 else 0 | |
| remaining = total_files - completed | |
| eta = remaining / rate if rate > 0 else 0 | |
| em, es = divmod(int(elapsed), 60) | |
| etm, ets = divmod(int(eta), 60) | |
| end_char = "\n" if final else "" | |
| line_char = "\r" if sys.stdout.isatty() else "\n" | |
| print( | |
| f"{line_char}[{completed}/{total_files}] " | |
| f"{rate:5.1f}/s {em}m{es:02d}s elapsed " | |
| f"ETA {etm}m{ets:02d}s ", | |
| end=end_char, | |
| flush=True, | |
| ) | |
| def _on_batch_done(results: List[Dict[str, Any]], batch_len: int) -> None: | |
| nonlocal completed, last_print | |
| all_results.extend(results) | |
| completed += batch_len | |
| if not args.no_progress: | |
| now = time.monotonic() | |
| if now - last_print >= progress_interval or completed >= total_files: | |
| _print_progress(final=completed >= total_files) | |
| last_print = now | |
| # TaskPool runs _worker_setup once per worker to seed the per-worker | |
| # context, then invokes each batch's callback back in this process (so | |
| # progress printing and result collection stay single-threaded). Its | |
| # context manager terminates workers on exit, including when a | |
| # KeyboardInterrupt unwinds through it. | |
| try: | |
| with TaskPool( | |
| workers=num_processes, | |
| name="test262", | |
| setup=_worker_setup, | |
| setupargs=(ctx,), | |
| ) as pool: | |
| for batch in batches: | |
| pool.do( | |
| _run_batch, | |
| batch, | |
| callback=lambda results, n=len(batch): _on_batch_done(results, n), | |
| ) | |
| pool.wait() | |
| except KeyboardInterrupt: | |
| if not args.no_progress: | |
| print() | |
| print("\nInterrupted. Exiting...", file=sys.stderr) | |
| sys.exit(130) | |
| # --- Sort and classify results --- | |
| all_results.sort(key=lambda r: (r["path"], r["mode"])) | |
| all_results = _classify_results(all_results, expectations) | |
| # --- Compute statistics --- | |
| failed: Dict[str, Dict[str, Any]] = {} | |
| fail_count = 0 | |
| new_fail_count = 0 | |
| new_pass_count = 0 | |
| skip_count = 0 | |
| new_failure_report = "" | |
| new_pass_report = "" | |
| for test in all_results: | |
| path = test["path"] | |
| mode = test["mode"] | |
| result = test["result"] | |
| if result.endswith("fail"): | |
| fail_count += 1 | |
| failed.setdefault(path, {})[mode] = test.get("error") | |
| if result == "unexpected_fail": | |
| new_fail_count += 1 | |
| output = test.get("output", "") | |
| exit_code = test.get("exit_code", 0) | |
| features_str = "" | |
| if args.verbose and test.get("features"): | |
| features_str = ( | |
| "Features: " | |
| + ", ".join(test["features"]) | |
| + "\n" | |
| ) | |
| msg = ( | |
| f"! NEW FAIL {path} ({mode}) " | |
| f"(Exit code: {exit_code})\n" | |
| f"{features_str}{output}\n" | |
| ) | |
| if args.verbose: | |
| new_failure_report += msg + "\n" | |
| # --skipped-files runs the tests the config skips, which are | |
| # expected to fail; reporting every one of them as a new | |
| # failure is pure noise, so only --verbose asks for it. | |
| suppressed = args.skipped_files and not args.verbose | |
| if not suppressed and (not expectations or not args.verbose): | |
| print(msg) | |
| elif result.endswith("pass"): | |
| is_new_pass = result == "unexpected_pass" | |
| # --skipped-files deliberately runs without an expectations | |
| # file, so nothing is ever classified unexpected_pass there. | |
| # Every test that passes is one the config no longer needs to | |
| # skip, which is the entire point of the mode. | |
| if is_new_pass or args.skipped_files: | |
| new_pass_count += 1 | |
| new_pass_report += f"PASS {path} ({mode})\n" | |
| if is_new_pass: | |
| if not args.verbose: | |
| print(f"NEW PASS {path} ({mode})") | |
| elif args.verbose: | |
| print(f"PASS {path} ({mode})") | |
| elif result == "skip": | |
| skip_count += 1 | |
| if args.verbose: | |
| print(f"SKIP {path}") | |
| # --- Print summaries --- | |
| if args.verbose and expectations and (new_failure_report or new_pass_report): | |
| print() | |
| if new_failure_report: | |
| print("---------------NEW FAILING TESTS SUMMARY---------------\n") | |
| print(new_failure_report) | |
| if new_pass_report: | |
| print("---------------NEW PASSING TESTS SUMMARY---------------\n") | |
| print(new_pass_report) | |
| print("---------------------------------------------------------\n") | |
| if args.skipped_files and new_pass_report: | |
| print("---------------NEW PASSING TESTS SUMMARY---------------") | |
| print(f"\n{new_pass_report}") | |
| print("---------------------------------------------------------") | |
| total_run = len(all_results) - skip_count | |
| print(f"\n{total_run} tests run") | |
| print(f"{skip_count} test files skipped") | |
| if not expectations: | |
| print(f"{fail_count} tests failed") | |
| if args.skipped_files: | |
| print(f"{new_pass_count} tests newly pass") | |
| else: | |
| print(f"{fail_count} tests failed in total") | |
| print(f"{new_fail_count} tests newly fail") | |
| print(f"{new_pass_count} tests newly pass") | |
| # --- Save expectations --- | |
| if args.save: | |
| _save_expectations(expectations_file, all_results, running_all) | |
| # --- Save results and reports --- | |
| os.makedirs(results_dir, exist_ok=True) | |
| results_file = os.path.join(results_dir, "results.yaml") | |
| _save_results(results_file, all_results) | |
| if _REPORT_CSS.exists(): | |
| shutil.copy(str(_REPORT_CSS), results_dir) | |
| _summarize_results(all_results, results_dir) | |
| _generate_html_index( | |
| failed, results_dir, total_run, fail_count, new_fail_count, skip_count | |
| ) | |
| elapsed = time.monotonic() - start_time | |
| print(f"See the summaries and results in the {results_dir}.") | |
| print(f"\nDone in {elapsed:.2f} seconds!") | |
| total_failures = new_fail_count if expectations else fail_count | |
| sys.exit(1 if total_failures else 0) |