| #!/usr/bin/env python3 |
| # Copyright (C) 2024 Apple Inc. All rights reserved. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions |
| # are met: |
| # 1. Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # 2. Redistributions in binary form must reproduce the above copyright |
| # notice, this list of conditions and the following disclaimer in the |
| # documentation and/or other materials provided with the distribution. |
| # |
| # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS "AS IS" AND |
| # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| # DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR |
| # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
| # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR |
| # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
| # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, |
| # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| import os |
| import subprocess |
| import argparse |
| import json |
| import sys |
| import re |
| |
| from webkitpy.results.upload import Upload |
| from webkitpy.results.options import upload_args |
| from webkitpy.safer_cpp.checkers import Checker |
| from libraries.webkitscmpy.webkitscmpy import Commit |
| from libraries.webkitscmpy.webkitscmpy import local, log, remote |
| |
| STATIC_ANALYZER_UNEXPECTED = 'StaticAnalyzerUnexpectedRegressions' |
| UNEXPECTED_PASS = {'actual': 'PASS', 'expected': 'FAIL'} |
| UNEXPECTED_FAILURE = {'actual': 'FAIL', 'expected': 'PASS'} |
| EXPECTED_FAILURE = {'actual': 'FAIL', 'expected': 'FAIL'} |
| DERIVED_SOURCES_RE = r'(^|.*\/)DerivedSources\/WebCore\/JS.*' |
| EXPECTATION_LINE_RE = r'(\s*\[\s*(?P<platform>\w+)\s*\]\s*)?(?P<path>.+)' |
| |
| |
| def parser(): |
| parser = argparse.ArgumentParser(parents=[upload_args()], description='Finds new regressions and fixes between two smart pointer static analysis results') |
| parser.add_argument( |
| 'new_dir', |
| help='Path to directory of results from new build' |
| ) |
| parser.add_argument( |
| '--archived-dir', |
| dest='archived_dir', |
| help='Path to directory of previous results for comparison' |
| ) |
| parser.add_argument( |
| '--build-output', |
| dest='build_output', |
| help='Path to the static analyzer output from the new build', |
| required='--generate-results-only' in sys.argv |
| ) |
| parser.add_argument( |
| '--scan-build-path', |
| dest='scan_build', |
| help='Path to scan-build executable', |
| required='--generate-results-only' in sys.argv |
| ) |
| parser.add_argument( |
| '--generate-results-only', |
| dest='generate_filtered_results', |
| action='store_true', |
| default=False |
| ) |
| parser.add_argument( |
| '--check-expectations', |
| dest='check_expectations', |
| action='store_true', |
| default=False, |
| help='Compare new results to expectations (instead of a previous run)' |
| ) |
| parser.add_argument( |
| '--delete-results', |
| dest='delete_results', |
| action='store_true', |
| default=False, |
| help='Delete static analyzer results' |
| ) |
| config_options = parser.add_argument_group('Configuration options') |
| config_options.add_argument('--architecture') |
| config_options.add_argument('--platform', default='mac') |
| config_options.add_argument('--version') |
| config_options.add_argument('--version-name') |
| config_options.add_argument('--style') |
| config_options.add_argument('--sdk') |
| |
| return parser.parse_args() |
| |
| |
| def find_diff(args, expectation_file_path, results_file_path): |
| # Create empty file if the corresponding one doesn't exist - this happens if a checker is added or removed |
| if not os.path.exists(expectation_file_path): |
| os.makedirs(os.path.dirname(expectation_file_path), exist_ok=True) |
| f = open(expectation_file_path, 'a') |
| f.close() |
| |
| if not os.path.exists(results_file_path): |
| os.makedirs(os.path.dirname(results_file_path), exist_ok=True) |
| f = open(results_file_path, 'a') |
| f.close() |
| |
| with open(expectation_file_path) as baseline_file, open(results_file_path) as new_file: |
| baseline_list = [] |
| expectation_line_re = re.compile(EXPECTATION_LINE_RE) |
| for line in baseline_file.read().splitlines(): |
| if line.startswith('//') or line.startswith('#'): |
| continue |
| match = expectation_line_re.match(line) |
| if not match: |
| print(f'Unexpected line: {line}') |
| continue |
| platform = match.group('platform') |
| if platform and args.platform.lower() != platform.lower(): |
| continue |
| baseline_list.append(match.group('path')) |
| new_file_list = new_file.read().splitlines() |
| # Find new regressions |
| diff_new_from_baseline = set(new_file_list) - set(baseline_list) |
| # Find fixes |
| diff_baseline_from_new = set(baseline_list) - set(new_file_list) |
| |
| # FIXME: Ignore DerivedSources until autogenerator scripts are clean |
| filtered_regressions = [file for file in diff_new_from_baseline if not re.match(DERIVED_SOURCES_RE, file)] |
| filtered_fixes = [file for file in diff_baseline_from_new if not re.match(DERIVED_SOURCES_RE, file)] |
| filtered_new_files = [file for file in new_file_list if not re.match(DERIVED_SOURCES_RE, file)] |
| |
| return set(filtered_regressions), set(filtered_fixes), filtered_new_files |
| |
| |
| def create_filtered_results_dir(args, project, result_paths, category='StaticAnalyzerRegressions'): |
| # Create symlinks to new issues only so that we can run scan-build to generate new index.html files |
| prefix_path = os.path.abspath(f'{args.build_output}/{category}/{project}/StaticAnalyzerReports') |
| subprocess.run(['mkdir', '-p', prefix_path]) |
| for path_to_report in result_paths: |
| report = path_to_report.split('/')[-1] |
| path_to_report_new = os.path.join(prefix_path, report) |
| subprocess.run(['ln', '-s', os.path.abspath(path_to_report), path_to_report_new]) |
| print('\n') |
| path_to_project = f'{args.build_output}/{category}/{project}' |
| subprocess.run([args.scan_build, '--generate-index-only', os.path.abspath(path_to_project)]) |
| |
| |
| def compare_project_results_to_expectations(args, new_path, project, upload_only=False): |
| unexpected_issues_total = set() |
| unexpected_result_paths_total = set() |
| unexpected_buggy_files = set() |
| unexpected_clean_files = set() |
| project_results_passes = {} |
| project_results_failures = {} |
| project_results_for_upload = {} |
| |
| # Compare the list of dirty files to the expectations list of files |
| for checker in Checker.enumerate(): |
| checker_name = checker.name() |
| # Get unexpected clean and buggy files per checker |
| buggy_files, clean_files, current_results = find_diff(args, checker.expectations_path(project), f'{new_path}/{project}/{checker_name}Files') |
| unexpected_clean_files.update(clean_files) |
| unexpected_buggy_files.update(buggy_files) |
| |
| # Get unexpected issues per checker |
| unexpected_issues = set() |
| unexpected_result_paths = set() |
| |
| with open(f'{new_path}/issues_per_file.json') as f: |
| issues_per_file = json.load(f) |
| for file_name in buggy_files: |
| unexpected_issues.update(list(issues_per_file[checker_name][file_name].keys())) |
| unexpected_result_paths.update(list(issues_per_file[checker_name][file_name].values())) |
| unexpected_result_paths_total.update(unexpected_result_paths) |
| unexpected_issues_total.update(unexpected_issues) |
| |
| if not upload_only: |
| # Set up JSON object |
| project_results_passes[checker_name] = list(clean_files) |
| project_results_failures[checker_name] = list(buggy_files) |
| |
| # Create sorted files for each unexpected list - these need the .txt to be displayed in browser |
| subprocess.run(['mkdir', '-p', f'{args.build_output}/{STATIC_ANALYZER_UNEXPECTED}/{project}']) |
| with open(f'{args.build_output}/{STATIC_ANALYZER_UNEXPECTED}/{project}/UnexpectedPasses{checker_name}.txt', 'a') as f: |
| f.write('\n'.join(sorted(clean_files))) |
| with open(f'{args.build_output}/{STATIC_ANALYZER_UNEXPECTED}/{project}/UnexpectedFailures{checker_name}.txt', 'a') as f: |
| f.write('\n'.join(sorted(buggy_files))) |
| with open(f'{new_path}/{project}/UnexpectedIssues{checker_name}', 'a') as f: |
| f.write('\n'.join(unexpected_issues)) |
| |
| if clean_files or buggy_files or unexpected_issues: |
| print(f'\n{checker_name}:') |
| print(f' Unexpected passing files: {len(clean_files)}') |
| print(f' Unexpected failing files: {len(buggy_files)}') |
| print(f' Unexpected issues: {len(unexpected_issues)}') |
| |
| for file in current_results: |
| if not file in project_results_for_upload.keys(): |
| project_results_for_upload[file] = {} |
| if file in buggy_files: |
| project_results_for_upload[file][checker_name] = UNEXPECTED_FAILURE |
| else: |
| project_results_for_upload[file][checker_name] = EXPECTED_FAILURE |
| for file in clean_files: |
| if not file in project_results_for_upload.keys(): |
| project_results_for_upload[file] = {} |
| project_results_for_upload[file][checker_name] = UNEXPECTED_PASS |
| |
| if unexpected_issues_total and args.scan_build and not upload_only: |
| create_filtered_results_dir(args, project, unexpected_result_paths_total, STATIC_ANALYZER_UNEXPECTED) |
| |
| if not unexpected_buggy_files and not unexpected_clean_files and not unexpected_issues_total and not upload_only: |
| print('No unexpected results!') |
| |
| return unexpected_buggy_files, unexpected_clean_files, unexpected_issues_total, project_results_passes, project_results_failures, project_results_for_upload |
| |
| |
| def compare_project_results_by_run(args, archive_path, new_path, project): |
| new_issues_total = set() |
| new_files_total = set() |
| fixed_issues_total = set() |
| fixed_files_total = set() |
| unexpected_result_paths_total = set() |
| project_results_passes = {} |
| project_results_failures = {} |
| |
| for checker in Checker.enumerate(): |
| checker_name = checker.name() |
| _, fixed_issues, _ = find_diff(args, f'{archive_path}/{checker_name}Issues', f'{new_path}/{project}/{checker_name}Issues') |
| new_files, fixed_files, _ = find_diff(args, f'{archive_path}/{checker_name}Files', f'{new_path}/{project}/{checker_name}Files') |
| fixed_issues_total.update(fixed_issues) |
| fixed_files_total.update(fixed_files) |
| new_files_total.update(new_files) |
| new_issues = set() |
| unexpected_result_paths = set() |
| |
| # Get unexpected issues per checker |
| with open(f'{new_path}/issues_per_file.json') as f: |
| issues_per_file = json.load(f) |
| for file_name in new_files: |
| new_issues.update(list(issues_per_file[checker_name][file_name].keys())) |
| unexpected_result_paths.update(list(issues_per_file[checker_name][file_name].values())) |
| unexpected_result_paths_total.update(unexpected_result_paths) |
| new_issues_total.update(new_issues) |
| |
| # JSON |
| project_results_passes[checker_name] = list(fixed_files) |
| project_results_failures[checker_name] = list(new_files) |
| |
| if fixed_issues or fixed_files or new_issues or new_files: |
| print(f'\n{checker_name}:') |
| print(f' Issues fixed: {len(fixed_issues)}') |
| print(f' Files fixed: {len(fixed_files)}') |
| print(f' Unexpected issues: {len(new_issues)}') |
| print(f' Unexpected failing files: {len(new_files)}') |
| |
| if new_issues_total and args.scan_build: |
| create_filtered_results_dir(args, project, unexpected_result_paths_total, 'StaticAnalyzerRegressions') |
| |
| if not new_issues_total and not new_files_total and not fixed_issues_total and not fixed_files_total: |
| print('No unexpected results!') |
| |
| return new_issues_total, new_files_total, fixed_files_total, fixed_issues_total, project_results_passes, project_results_failures |
| |
| |
| def upload_results(options, results): |
| print('\nPreparing to upload results...') |
| |
| config = Upload.create_configuration( |
| architecture=options.architecture, |
| platform=options.platform, |
| version=options.version, |
| version_name=options.version_name, |
| style=options.style, |
| sdk=options.sdk |
| ) |
| |
| repo = os.getcwd() |
| if repo.startswith(('https://', 'http://')): |
| repository = remote.Scm.from_url(repo) |
| else: |
| try: |
| repository = local.Scm.from_path(path=repo,) |
| except OSError: |
| log.warning("No repository found at '{}'".format(repo)) |
| repository = None |
| |
| commit = repository.commit() |
| commit = commit.Encoder().default(commit) |
| commit['repository_id'] = 'webkit' |
| |
| upload = Upload( |
| suite=options.suite or 'safer-cpp-checks', |
| configuration=config, |
| details=Upload.create_details(options=options), |
| commits=[commit], |
| run_stats=Upload.create_run_stats(), results=results, |
| ) |
| |
| for url in options.report_urls: |
| if not upload.upload(url): |
| sys.stderr.write(f'Failed to upload results\n') |
| |
| |
| def compare_results(args): |
| new_issues_total = set() |
| new_files_total = set() |
| fixed_files_total = set() |
| fixed_issues_total = set() |
| unexpected_passes_total = set() |
| unexpected_failures_total = set() |
| unexpected_issues_total = set() |
| unexpected_results_data = {'passes': {}, 'failures': {}} |
| results_for_upload = {} |
| new_path = os.path.abspath(f'{args.new_dir}') |
| archive_path = os.path.abspath(f'{args.archived_dir}') if args.archived_dir else '' |
| |
| for project in Checker.projects(): |
| project_results_for_upload = None |
| print(f'\n------ {project} ------') |
| if args.check_expectations: |
| unexpected_failures, unexpected_passes, unexpected_issues, project_results_passes, project_results_failures, project_results_for_upload = compare_project_results_to_expectations(args, new_path, project) |
| unexpected_failures_total.update(unexpected_failures) |
| unexpected_passes_total.update(unexpected_passes) |
| unexpected_issues_total.update(unexpected_issues) |
| else: |
| path_to_project = f'{archive_path}/{project}' |
| new_issues, new_files, fixed_files, fixed_issues, project_results_passes, project_results_failures = compare_project_results_by_run(args, path_to_project, new_path, project) |
| new_issues_total.update(new_issues) |
| new_files_total.update(new_files) |
| fixed_files_total.update(fixed_files) |
| fixed_issues_total.update(fixed_issues) |
| # JSON |
| unexpected_results_data['passes'][project] = project_results_passes |
| unexpected_results_data['failures'][project] = project_results_failures |
| |
| if args.report_urls: |
| path_for_upload = archive_path or new_path |
| if not project_results_for_upload: |
| print(f'\nChecking against expectations for results to upload for {project}...') |
| _, _, _, _, _, project_results_for_upload = compare_project_results_to_expectations(args, path_for_upload, project, upload_only=True) |
| results_for_upload[project] = project_results_for_upload |
| |
| if args.build_output: |
| results_data_file = os.path.abspath(f"{args.build_output}/unexpected_results.json") |
| with open(results_data_file, "w") as f: |
| results_data_obj = json.dumps(unexpected_results_data, indent=4) |
| f.write(results_data_obj) |
| |
| print('\n') |
| for type, type_total in { |
| 'new issues': new_issues_total, |
| 'new files': new_files_total, |
| 'fixed files': fixed_files_total, |
| 'fixed issues': fixed_issues_total, |
| 'unexpected failing files': unexpected_failures_total, |
| 'unexpected passing files': unexpected_passes_total, |
| 'unexpected issues': unexpected_issues_total |
| }.items(): |
| if type_total: |
| print(f'Total {type}: {len(type_total)}') |
| |
| return results_for_upload |
| |
| |
| def generate_filtered_results(args): |
| report_paths = [] |
| print(f'Generating new results index with results from {args.build_output}...') |
| |
| with open(os.path.abspath(f"{args.build_output}/unexpected_results.json"), "r") as f: |
| results_data = json.load(f) |
| with open(f'{args.new_dir}/issues_per_file.json') as f: |
| issues_per_file = json.load(f) |
| |
| for project, checkers in results_data.get('failures').items(): |
| for checker, files in checkers.items(): |
| for file in files: |
| report_paths += [path_to_report for path_to_report in issues_per_file[checker][file].values()] |
| |
| create_filtered_results_dir(args, project, report_paths, category='StaticAnalyzerRegressions') |
| |
| |
| def main(): |
| args = parser() |
| |
| if not args.generate_filtered_results: |
| results_for_upload = compare_results(args) |
| if args.report_urls: |
| upload_results(args, results_for_upload) |
| else: |
| generate_filtered_results(args) |
| |
| # We don't need the full results for EWS runs. Delete full results if option enabled. |
| if args.delete_results: |
| path_to_delete = os.path.abspath(f'{args.build_output}/StaticAnalyzer') |
| print(f'\nDeleting results from {path_to_delete}...') |
| subprocess.run(['rm', '-r', path_to_delete]) |
| |
| return 0 |
| |
| |
| if __name__ == '__main__': |
| main() |
