| #!/usr/bin/env python3 | |
| # Copyright (C) 2024 Apple Inc. All rights reserved. | |
| # | |
| # Redistribution and use in source and binary forms, with or without | |
| # modification, are permitted provided that the following conditions | |
| # are met: | |
| # 1. Redistributions of source code must retain the above copyright | |
| # notice, this list of conditions and the following disclaimer. | |
| # 2. Redistributions in binary form must reproduce the above copyright | |
| # notice, this list of conditions and the following disclaimer in the | |
| # documentation and/or other materials provided with the distribution. | |
| # | |
| # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS "AS IS" AND | |
| # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED | |
| # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE | |
| # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR | |
| # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |
| # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR | |
| # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER | |
| # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, | |
| # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| import os | |
| import subprocess | |
| import argparse | |
| import json | |
| import sys | |
| import re | |
| from webkitpy.results.upload import Upload | |
| from webkitpy.results.options import upload_args | |
| from webkitpy.safer_cpp.checkers import Checker | |
| from libraries.webkitscmpy.webkitscmpy import Commit | |
| from libraries.webkitscmpy.webkitscmpy import local, log, remote | |
| STATIC_ANALYZER_UNEXPECTED = 'StaticAnalyzerUnexpectedRegressions' | |
| UNEXPECTED_PASS = {'actual': 'PASS', 'expected': 'FAIL'} | |
| UNEXPECTED_FAILURE = {'actual': 'FAIL', 'expected': 'PASS'} | |
| EXPECTED_FAILURE = {'actual': 'FAIL', 'expected': 'FAIL'} | |
| # Generated JS bindings, in the Xcode and CMake build layouts. | |
| DERIVED_SOURCES_RE = r'(^|.*\/)(DerivedSources\/WebCore\/JS|WebCore\/DerivedSources\/JS).*' | |
| EXPECTATION_LINE_RE = r'(\s*\[\s*(?P<platform>\w+)\s*\]\s*)?(?P<path>.+)' | |
| def normalize_platform(platform): | |
| if not platform: | |
| return None | |
| platform = platform.lower() | |
| if platform == "mac" or platform == "macos": | |
| return "macOS" | |
| elif platform == "ios": | |
| return "iOS" | |
| return None | |
| def parser(): | |
| parser = argparse.ArgumentParser(parents=[upload_args()], description='Finds new regressions and fixes between two smart pointer static analysis results') | |
| parser.add_argument( | |
| 'new_dir', | |
| help='Path to directory of results from new build' | |
| ) | |
| parser.add_argument( | |
| '--archived-dir', | |
| dest='archived_dir', | |
| help='Path to directory of previous results for comparison' | |
| ) | |
| parser.add_argument( | |
| '--build-output', | |
| dest='build_output', | |
| help='Path to the static analyzer output from the new build', | |
| required='--generate-results-only' in sys.argv | |
| ) | |
| parser.add_argument( | |
| '--scan-build-path', | |
| dest='scan_build', | |
| help='Path to scan-build executable', | |
| required='--generate-results-only' in sys.argv | |
| ) | |
| parser.add_argument( | |
| '--generate-results-only', | |
| dest='generate_filtered_results', | |
| action='store_true', | |
| default=False | |
| ) | |
| parser.add_argument( | |
| '--check-expectations', | |
| dest='check_expectations', | |
| action='store_true', | |
| default=False, | |
| help='Compare new results to expectations (instead of a previous run)' | |
| ) | |
| parser.add_argument( | |
| '--delete-results', | |
| dest='delete_results', | |
| action='store_true', | |
| default=False, | |
| help='Delete static analyzer results' | |
| ) | |
| config_options = parser.add_argument_group('Configuration options') | |
| config_options.add_argument('--architecture') | |
| config_options.add_argument('--platform', default='mac') | |
| config_options.add_argument('--version') | |
| config_options.add_argument('--version-name') | |
| config_options.add_argument('--style') | |
| config_options.add_argument('--sdk') | |
| return parser.parse_args() | |
| def find_diff(args, expectation_file_path, results_file_path): | |
| # Create empty file if the corresponding one doesn't exist - this happens if a checker is added or removed | |
| if not os.path.exists(expectation_file_path): | |
| os.makedirs(os.path.dirname(expectation_file_path), exist_ok=True) | |
| f = open(expectation_file_path, 'a') | |
| f.close() | |
| if not os.path.exists(results_file_path): | |
| os.makedirs(os.path.dirname(results_file_path), exist_ok=True) | |
| f = open(results_file_path, 'a') | |
| f.close() | |
| with open(expectation_file_path) as baseline_file, open(results_file_path) as new_file: | |
| baseline_list = [] | |
| expectation_line_re = re.compile(EXPECTATION_LINE_RE) | |
| for line in baseline_file.read().splitlines(): | |
| if line.startswith('//') or line.startswith('#'): | |
| continue | |
| match = expectation_line_re.match(line) | |
| if not match: | |
| print(f'Unexpected line: {line}') | |
| continue | |
| expectations_platform = normalize_platform(match.group('platform')) | |
| if expectations_platform and normalize_platform(args.platform) != expectations_platform: | |
| continue | |
| baseline_list.append(match.group('path')) | |
| new_file_list = new_file.read().splitlines() | |
| # Find new regressions | |
| diff_new_from_baseline = set(new_file_list) - set(baseline_list) | |
| # Find fixes | |
| diff_baseline_from_new = set(baseline_list) - set(new_file_list) | |
| # FIXME: Ignore DerivedSources until autogenerator scripts are clean | |
| filtered_regressions = [file for file in diff_new_from_baseline if not re.match(DERIVED_SOURCES_RE, file)] | |
| filtered_fixes = [file for file in diff_baseline_from_new if not re.match(DERIVED_SOURCES_RE, file)] | |
| filtered_new_files = [file for file in new_file_list if not re.match(DERIVED_SOURCES_RE, file)] | |
| return set(filtered_regressions), set(filtered_fixes), filtered_new_files | |
| def create_filtered_results_dir(args, project, result_paths, category='StaticAnalyzerRegressions'): | |
| # Create symlinks to new issues only so that we can run scan-build to generate new index.html files | |
| prefix_path = os.path.abspath(f'{args.build_output}/{category}/{project}/StaticAnalyzerReports') | |
| subprocess.run(['mkdir', '-p', prefix_path]) | |
| for path_to_report in result_paths: | |
| report = path_to_report.split('/')[-1] | |
| path_to_report_new = os.path.join(prefix_path, report) | |
| subprocess.run(['ln', '-s', os.path.abspath(path_to_report), path_to_report_new]) | |
| print('\n') | |
| path_to_project = f'{args.build_output}/{category}/{project}' | |
| subprocess.run([args.scan_build, '--generate-index-only', os.path.abspath(path_to_project)]) | |
| def compare_project_results_to_expectations(args, new_path, project, upload_only=False): | |
| unexpected_issues_total = set() | |
| unexpected_result_paths_total = set() | |
| unexpected_buggy_files = set() | |
| unexpected_clean_files = set() | |
| project_results_passes = {} | |
| project_results_failures = {} | |
| project_results_for_upload = {} | |
| # Compare the list of dirty files to the expectations list of files | |
| for checker in Checker.enumerate(): | |
| checker_name = checker.name() | |
| # Get unexpected clean and buggy files per checker | |
| buggy_files, clean_files, current_results = find_diff(args, checker.expectations_path(project), f'{new_path}/{project}/{checker_name}Files') | |
| unexpected_clean_files.update(clean_files) | |
| unexpected_buggy_files.update(buggy_files) | |
| # Get unexpected issues per checker | |
| unexpected_issues = set() | |
| unexpected_result_paths = set() | |
| with open(f'{new_path}/issues_per_file.json') as f: | |
| issues_per_file = json.load(f) | |
| for file_name in buggy_files: | |
| unexpected_issues.update(list(issues_per_file[checker_name][file_name].keys())) | |
| unexpected_result_paths.update(list(issues_per_file[checker_name][file_name].values())) | |
| unexpected_result_paths_total.update(unexpected_result_paths) | |
| unexpected_issues_total.update(unexpected_issues) | |
| if not upload_only: | |
| # Set up JSON object | |
| project_results_passes[checker_name] = list(clean_files) | |
| project_results_failures[checker_name] = list(buggy_files) | |
| # Create sorted files for each unexpected list - these need the .txt to be displayed in browser | |
| subprocess.run(['mkdir', '-p', f'{args.build_output}/{STATIC_ANALYZER_UNEXPECTED}/{project}']) | |
| with open(f'{args.build_output}/{STATIC_ANALYZER_UNEXPECTED}/{project}/UnexpectedPasses{checker_name}.txt', 'a') as f: | |
| f.write('\n'.join(sorted(clean_files))) | |
| with open(f'{args.build_output}/{STATIC_ANALYZER_UNEXPECTED}/{project}/UnexpectedFailures{checker_name}.txt', 'a') as f: | |
| f.write('\n'.join(sorted(buggy_files))) | |
| with open(f'{new_path}/{project}/UnexpectedIssues{checker_name}', 'a') as f: | |
| f.write('\n'.join(unexpected_issues)) | |
| if clean_files or buggy_files or unexpected_issues: | |
| print(f'\n{checker_name}:') | |
| print(f' Unexpected passing files: {len(clean_files)}') | |
| print(f' Unexpected failing files: {len(buggy_files)}') | |
| print(f' Unexpected issues: {len(unexpected_issues)}') | |
| for file in current_results: | |
| if not file in project_results_for_upload.keys(): | |
| project_results_for_upload[file] = {} | |
| if file in buggy_files: | |
| project_results_for_upload[file][checker_name] = UNEXPECTED_FAILURE | |
| else: | |
| project_results_for_upload[file][checker_name] = EXPECTED_FAILURE | |
| for file in clean_files: | |
| if not file in project_results_for_upload.keys(): | |
| project_results_for_upload[file] = {} | |
| project_results_for_upload[file][checker_name] = UNEXPECTED_PASS | |
| if unexpected_issues_total and args.scan_build and not upload_only: | |
| create_filtered_results_dir(args, project, unexpected_result_paths_total, STATIC_ANALYZER_UNEXPECTED) | |
| if not unexpected_buggy_files and not unexpected_clean_files and not unexpected_issues_total and not upload_only: | |
| print('No unexpected results!') | |
| return unexpected_buggy_files, unexpected_clean_files, unexpected_issues_total, project_results_passes, project_results_failures, project_results_for_upload | |
| def compare_project_results_by_run(args, archive_path, new_path, project): | |
| new_issues_total = set() | |
| new_files_total = set() | |
| fixed_issues_total = set() | |
| fixed_files_total = set() | |
| unexpected_result_paths_total = set() | |
| project_results_passes = {} | |
| project_results_failures = {} | |
| for checker in Checker.enumerate(): | |
| checker_name = checker.name() | |
| _, fixed_issues, _ = find_diff(args, f'{archive_path}/{checker_name}Issues', f'{new_path}/{project}/{checker_name}Issues') | |
| new_files, fixed_files, _ = find_diff(args, f'{archive_path}/{checker_name}Files', f'{new_path}/{project}/{checker_name}Files') | |
| fixed_issues_total.update(fixed_issues) | |
| fixed_files_total.update(fixed_files) | |
| new_files_total.update(new_files) | |
| new_issues = set() | |
| unexpected_result_paths = set() | |
| # Get unexpected issues per checker | |
| with open(f'{new_path}/issues_per_file.json') as f: | |
| issues_per_file = json.load(f) | |
| for file_name in new_files: | |
| new_issues.update(list(issues_per_file[checker_name][file_name].keys())) | |
| unexpected_result_paths.update(list(issues_per_file[checker_name][file_name].values())) | |
| unexpected_result_paths_total.update(unexpected_result_paths) | |
| new_issues_total.update(new_issues) | |
| # JSON | |
| project_results_passes[checker_name] = list(fixed_files) | |
| project_results_failures[checker_name] = list(new_files) | |
| if fixed_issues or fixed_files or new_issues or new_files: | |
| print(f'\n{checker_name}:') | |
| print(f' Issues fixed: {len(fixed_issues)}') | |
| print(f' Files fixed: {len(fixed_files)}') | |
| print(f' Unexpected issues: {len(new_issues)}') | |
| print(f' Unexpected failing files: {len(new_files)}') | |
| if new_issues_total and args.scan_build: | |
| create_filtered_results_dir(args, project, unexpected_result_paths_total, 'StaticAnalyzerRegressions') | |
| if not new_issues_total and not new_files_total and not fixed_issues_total and not fixed_files_total: | |
| print('No unexpected results!') | |
| return new_issues_total, new_files_total, fixed_files_total, fixed_issues_total, project_results_passes, project_results_failures | |
| def upload_results(options, results): | |
| print('\nPreparing to upload results...') | |
| config = Upload.create_configuration( | |
| architecture=options.architecture, | |
| platform=options.platform, | |
| version=options.version, | |
| version_name=options.version_name, | |
| style=options.style, | |
| sdk=options.sdk | |
| ) | |
| repo = os.getcwd() | |
| if repo.startswith(('https://', 'http://')): | |
| repository = remote.Scm.from_url(repo) | |
| else: | |
| try: | |
| repository = local.Scm.from_path(path=repo,) | |
| except OSError: | |
| log.warning("No repository found at '{}'".format(repo)) | |
| repository = None | |
| commit = repository.commit() | |
| commit = commit.Encoder().default(commit) | |
| commit['repository_id'] = 'webkit' | |
| upload = Upload( | |
| suite=options.suite or 'safer-cpp-checks', | |
| configuration=config, | |
| details=Upload.create_details(options=options), | |
| commits=[commit], | |
| run_stats=Upload.create_run_stats(), results=results, | |
| ) | |
| for url in options.report_urls: | |
| if not upload.upload(url): | |
| sys.stderr.write(f'Failed to upload results\n') | |
| def compare_results(args): | |
| new_issues_total = set() | |
| new_files_total = set() | |
| fixed_files_total = set() | |
| fixed_issues_total = set() | |
| unexpected_passes_total = set() | |
| unexpected_failures_total = set() | |
| unexpected_issues_total = set() | |
| unexpected_results_data = {'passes': {}, 'failures': {}} | |
| results_for_upload = {} | |
| new_path = os.path.abspath(f'{args.new_dir}') | |
| archive_path = os.path.abspath(f'{args.archived_dir}') if args.archived_dir else '' | |
| for project in Checker.projects(): | |
| project_results_for_upload = None | |
| print(f'\n------ {project} ------') | |
| if args.check_expectations: | |
| unexpected_failures, unexpected_passes, unexpected_issues, project_results_passes, project_results_failures, project_results_for_upload = compare_project_results_to_expectations(args, new_path, project) | |
| unexpected_failures_total.update(unexpected_failures) | |
| unexpected_passes_total.update(unexpected_passes) | |
| unexpected_issues_total.update(unexpected_issues) | |
| else: | |
| path_to_project = f'{archive_path}/{project}' | |
| new_issues, new_files, fixed_files, fixed_issues, project_results_passes, project_results_failures = compare_project_results_by_run(args, path_to_project, new_path, project) | |
| new_issues_total.update(new_issues) | |
| new_files_total.update(new_files) | |
| fixed_files_total.update(fixed_files) | |
| fixed_issues_total.update(fixed_issues) | |
| # JSON | |
| unexpected_results_data['passes'][project] = project_results_passes | |
| unexpected_results_data['failures'][project] = project_results_failures | |
| if args.report_urls: | |
| path_for_upload = archive_path or new_path | |
| if not project_results_for_upload: | |
| print(f'\nChecking against expectations for results to upload for {project}...') | |
| _, _, _, _, _, project_results_for_upload = compare_project_results_to_expectations(args, path_for_upload, project, upload_only=True) | |
| results_for_upload[project] = project_results_for_upload | |
| if args.build_output: | |
| results_data_file = os.path.abspath(f"{args.build_output}/unexpected_results.json") | |
| with open(results_data_file, "w") as f: | |
| results_data_obj = json.dumps(unexpected_results_data, indent=4) | |
| f.write(results_data_obj) | |
| print('\n') | |
| for type, type_total in { | |
| 'new issues': new_issues_total, | |
| 'new files': new_files_total, | |
| 'fixed files': fixed_files_total, | |
| 'fixed issues': fixed_issues_total, | |
| 'unexpected failing files': unexpected_failures_total, | |
| 'unexpected passing files': unexpected_passes_total, | |
| 'unexpected issues': unexpected_issues_total | |
| }.items(): | |
| if type_total: | |
| print(f'Total {type}: {len(type_total)}') | |
| return results_for_upload | |
| def generate_filtered_results(args): | |
| report_paths = [] | |
| print(f'Generating new results index with results from {args.build_output}...') | |
| with open(os.path.abspath(f"{args.build_output}/unexpected_results.json"), "r") as f: | |
| results_data = json.load(f) | |
| with open(f'{args.new_dir}/issues_per_file.json') as f: | |
| issues_per_file = json.load(f) | |
| for project, checkers in results_data.get('failures').items(): | |
| for checker, files in checkers.items(): | |
| for file in files: | |
| report_paths += [path_to_report for path_to_report in issues_per_file[checker][file].values()] | |
| create_filtered_results_dir(args, project, report_paths, category='StaticAnalyzerRegressions') | |
| def main(): | |
| args = parser() | |
| if not args.generate_filtered_results: | |
| results_for_upload = compare_results(args) | |
| if args.report_urls: | |
| upload_results(args, results_for_upload) | |
| else: | |
| generate_filtered_results(args) | |
| # We don't need the full results for EWS runs. Delete full results if option enabled. | |
| if args.delete_results: | |
| path_to_delete = os.path.abspath(f'{args.build_output}/StaticAnalyzer') | |
| print(f'\nDeleting results from {path_to_delete}...') | |
| subprocess.run(['rm', '-r', path_to_delete]) | |
| return 0 | |
| if __name__ == '__main__': | |
| main() |