| #!/usr/bin/env python3 |
| # Copyright (c) 2021-2025 The Khronos Group Inc. |
| # Copyright (c) 2021-2025 Valve Corporation |
| # Copyright (c) 2021-2025 LunarG, Inc. |
| # Copyright (c) 2021-2024 Google Inc. |
| # Copyright (c) 2023-2024 RasterGrid Kft. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import argparse |
| import filecmp |
| import os |
| import re |
| import shutil |
| import subprocess |
| import sys |
| import tempfile |
| import difflib |
| import common_ci |
| import pickle |
| from xml.etree import ElementTree |
| from generate_spec_error_message import GenerateSpecErrorMessage |
| |
| def RunGenerators(api: str, registry: str, grammar: str, directory: str, styleFile: str, targetFilter: str, caching: bool): |
| |
| try: |
| code = common_ci.RunShellCmd(f'clang-format --version') |
| has_clang_format = True |
| except: |
| has_clang_format = False |
| |
| if not has_clang_format: |
| print("WARNING: Unable to find clang-format!") |
| |
| # These live in the Vulkan-Docs repo, but are pulled in via the |
| # Vulkan-Headers/registry folder |
| # At runtime we inject python path to find these helper scripts |
| scripts = os.path.dirname(registry) |
| scripts_directory_path = os.path.dirname(os.path.abspath(__file__)) |
| registry_headers_path = os.path.join(scripts_directory_path, scripts) |
| sys.path.insert(0, registry_headers_path) |
| try: |
| from reg import Registry |
| except: |
| print("ModuleNotFoundError: No module named 'reg'") # normal python error message |
| print(f'{registry_headers_path} is not pointing to the Vulkan-Headers registry directory.') |
| print("Inside Vulkan-Headers there is a registry/reg.py file that is used.") |
| sys.exit(1) # Return without call stack so easy to spot error |
| |
| from base_generator import BaseGeneratorOptions |
| from generators.thread_safety_generator import ThreadSafetyOutputGenerator |
| from generators.stateless_validation_helper_generator import StatelessValidationHelperOutputGenerator |
| from generators.object_tracker_generator import ObjectTrackerOutputGenerator |
| from generators.dispatch_table_helper_generator import DispatchTableHelperOutputGenerator |
| from generators.extension_helper_generator import ExtensionHelperOutputGenerator |
| from generators.api_version_generator import ApiVersionOutputGenerator |
| from generators.layer_dispatch_table_generator import LayerDispatchTableOutputGenerator |
| from generators.layer_chassis_generator import LayerChassisOutputGenerator |
| from generators.dispatch_object_generator import DispatchObjectGenerator |
| from generators.dispatch_vector_generator import DispatchVectorGenerator |
| from generators.function_pointers_generator import FunctionPointersOutputGenerator |
| from generators.best_practices_generator import BestPracticesOutputGenerator |
| from generators.legacy_generator import LegacyGenerator |
| from generators.spirv_validation_generator import SpirvValidationHelperOutputGenerator |
| from generators.spirv_grammar_generator import SpirvGrammarHelperOutputGenerator |
| from generators.command_validation_generator import CommandValidationOutputGenerator |
| from generators.dynamic_state_generator import DynamicStateOutputGenerator |
| from generators.sync_validation_generator import SyncValidationOutputGenerator |
| from generators.object_types_generator import ObjectTypesOutputGenerator |
| from generators.enum_flag_bits_generator import EnumFlagBitsOutputGenerator |
| from generators.valid_enum_values_generator import ValidEnumValuesOutputGenerator |
| from generators.valid_flag_values_generator import ValidFlagValuesOutputGenerator |
| from generators.spirv_tool_commit_id_generator import SpirvToolCommitIdOutputGenerator |
| from generators.error_location_helper_generator import ErrorLocationHelperOutputGenerator |
| from generators.pnext_chain_extraction_generator import PnextChainExtractionGenerator |
| from generators.device_features_generator import DeviceFeaturesOutputGenerator |
| from generators.feature_requirements import FeatureRequirementsGenerator |
| from generators.feature_not_present import FeatureNotPresentGenerator |
| from generators.test_icd_generator import TestIcdGenerator |
| |
| # These set fields that are needed by both OutputGenerator and BaseGenerator, |
| # but are uniform and don't need to be set at a per-generated file level |
| from base_generator import SetOutputDirectory, SetTargetApiName, SetMergedApiNames, EnableCaching |
| SetOutputDirectory(directory) |
| SetTargetApiName(api) |
| |
| valid_usage_file = os.path.join(scripts, 'validusage.json') |
| |
| # Build up a list of all generators |
| # Note: Options variable names MUST match order of constructor variable in generator |
| generators = { |
| 'thread_safety_instance_defs.h' : { |
| 'generator' : ThreadSafetyOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'thread_safety_device_defs.h' : { |
| 'generator' : ThreadSafetyOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'thread_safety.cpp' : { |
| 'generator' : ThreadSafetyOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'stateless_device_methods.h' : { |
| 'generator' : StatelessValidationHelperOutputGenerator, |
| 'genCombined': False, |
| 'options' : [valid_usage_file], |
| }, |
| 'stateless_instance_methods.h' : { |
| 'generator' : StatelessValidationHelperOutputGenerator, |
| 'genCombined': False, |
| 'options' : [valid_usage_file], |
| }, |
| 'stateless_validation_helper.cpp' : { |
| 'generator' : StatelessValidationHelperOutputGenerator, |
| 'genCombined': False, |
| 'options' : [valid_usage_file], |
| 'regenerate' : True |
| }, |
| 'enum_flag_bits.h' : { |
| 'generator' : EnumFlagBitsOutputGenerator, |
| 'genCombined': False, |
| }, |
| 'valid_enum_values.h' : { |
| 'generator' : ValidEnumValuesOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'valid_enum_values.cpp' : { |
| 'generator' : ValidEnumValuesOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'valid_flag_values.cpp' : { |
| 'generator' : ValidFlagValuesOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'object_tracker_device_methods.h' : { |
| 'generator' : ObjectTrackerOutputGenerator, |
| 'genCombined': True, |
| 'options' : [valid_usage_file], |
| }, |
| 'object_tracker_instance_methods.h' : { |
| 'generator' : ObjectTrackerOutputGenerator, |
| 'genCombined': True, |
| 'options' : [valid_usage_file], |
| }, |
| 'object_tracker.cpp' : { |
| 'generator' : ObjectTrackerOutputGenerator, |
| 'genCombined': True, |
| 'options' : [valid_usage_file], |
| }, |
| 'error_location_helper.h' : { |
| 'generator' : ErrorLocationHelperOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'error_location_helper.cpp' : { |
| 'generator' : ErrorLocationHelperOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'vk_dispatch_table_helper.h' : { |
| 'generator' : DispatchTableHelperOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'vk_dispatch_table_helper.cpp' : { |
| 'generator' : DispatchTableHelperOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'vk_function_pointers.h' : { |
| 'generator' : FunctionPointersOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'vk_function_pointers.cpp' : { |
| 'generator' : FunctionPointersOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'vk_layer_dispatch_table.h' : { |
| 'generator' : LayerDispatchTableOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'vk_object_types.h' : { |
| 'generator' : ObjectTypesOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'vk_object_types.cpp' : { |
| 'generator' : ObjectTypesOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'vk_extension_helper.h' : { |
| 'generator' : ExtensionHelperOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'vk_extension_helper.cpp' : { |
| 'generator' : ExtensionHelperOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'vk_api_version.h' : { |
| 'generator' : ApiVersionOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'validation_object_instance_methods.h' : { |
| 'generator' : LayerChassisOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'validation_object_device_methods.h' : { |
| 'generator' : LayerChassisOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'validation_object.cpp' : { |
| 'generator' : LayerChassisOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'chassis.cpp' : { |
| 'generator' : LayerChassisOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'dispatch_object_device_methods.h' : { |
| 'generator' : DispatchObjectGenerator, |
| 'genCombined': True, |
| }, |
| 'dispatch_object_instance_methods.h' : { |
| 'generator' : DispatchObjectGenerator, |
| 'genCombined': True, |
| }, |
| 'dispatch_functions.h' : { |
| 'generator' : DispatchObjectGenerator, |
| 'genCombined': True, |
| }, |
| 'dispatch_object.cpp' : { |
| 'generator' : DispatchObjectGenerator, |
| 'genCombined': True, |
| }, |
| 'dispatch_vector.h' : { |
| 'generator' : DispatchVectorGenerator, |
| 'genCombined': True, |
| }, |
| 'dispatch_vector.cpp' : { |
| 'generator' : DispatchVectorGenerator, |
| 'genCombined': True, |
| }, |
| 'best_practices_device_methods.h' : { |
| 'generator' : BestPracticesOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'best_practices_instance_methods.h' : { |
| 'generator' : BestPracticesOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'best_practices.cpp' : { |
| 'generator' : BestPracticesOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'legacy.cpp' : { |
| 'generator' : LegacyGenerator, |
| 'genCombined': True, |
| }, |
| 'legacy.h' : { |
| 'generator' : LegacyGenerator, |
| 'genCombined': True, |
| }, |
| 'sync_validation_types.h' : { |
| 'generator' : SyncValidationOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'sync_validation_types.cpp' : { |
| 'generator' : SyncValidationOutputGenerator, |
| 'genCombined': True, |
| 'regenerate' : True |
| }, |
| 'spirv_validation_helper.h' : { |
| 'generator' : SpirvValidationHelperOutputGenerator, |
| 'genCombined': False, |
| 'options' : [grammar], |
| }, |
| 'spirv_validation_helper.cpp' : { |
| 'generator' : SpirvValidationHelperOutputGenerator, |
| 'genCombined': False, |
| 'options' : [grammar], |
| }, |
| 'spirv_grammar_helper.h' : { |
| 'generator' : SpirvGrammarHelperOutputGenerator, |
| 'genCombined': False, |
| 'options' : [grammar], |
| }, |
| 'spirv_grammar_helper.cpp' : { |
| 'generator' : SpirvGrammarHelperOutputGenerator, |
| 'genCombined': False, |
| 'options' : [grammar], |
| }, |
| 'spirv_tools_commit_id.h' : { |
| 'genCombined': False, |
| 'generator' : SpirvToolCommitIdOutputGenerator, |
| }, |
| 'command_validation.h' : { |
| 'generator' : CommandValidationOutputGenerator, |
| 'genCombined': True, |
| 'options' : [valid_usage_file], |
| }, |
| 'command_validation.cpp' : { |
| 'generator' : CommandValidationOutputGenerator, |
| 'genCombined': True, |
| 'options' : [valid_usage_file], |
| }, |
| 'dynamic_state_helper.h' : { |
| 'generator' : DynamicStateOutputGenerator, |
| 'genCombined': False, |
| }, |
| 'dynamic_state_helper.cpp' : { |
| 'generator' : DynamicStateOutputGenerator, |
| 'genCombined': False, |
| }, |
| 'pnext_chain_extraction.h' : { |
| 'generator' : PnextChainExtractionGenerator, |
| 'genCombined': True, |
| }, |
| 'pnext_chain_extraction.cpp' : { |
| 'generator' : PnextChainExtractionGenerator, |
| 'genCombined': True, |
| }, |
| 'device_features.h' : { |
| 'generator' : DeviceFeaturesOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'device_features.cpp' : { |
| 'generator' : DeviceFeaturesOutputGenerator, |
| 'genCombined': True, |
| }, |
| 'feature_requirements_helper.h' : { |
| 'generator' : FeatureRequirementsGenerator, |
| 'genCombined': True, |
| }, |
| 'feature_requirements_helper.cpp' : { |
| 'generator' : FeatureRequirementsGenerator, |
| 'genCombined': True, |
| }, |
| 'feature_not_present.cpp' : { |
| 'generator' : FeatureNotPresentGenerator, |
| 'genCombined': True, |
| }, |
| 'test_icd_helper.h' : { |
| 'generator' : TestIcdGenerator, |
| 'genCombined': False, |
| }, |
| } |
| |
| unknownTargets = [x for x in (targetFilter if targetFilter else []) if x not in generators.keys()] |
| if unknownTargets: |
| print(f'ERROR: No generator options for unknown target(s): {", ".join(unknownTargets)}', file=sys.stderr) |
| return 1 |
| |
| # Filter if --target is passed in |
| targets = [x for x in generators.keys() if not targetFilter or x in targetFilter] |
| |
| cacheVkObjectData = None |
| cachePath = os.path.join(tempfile.gettempdir(), f'vkobject_{os.getpid()}') |
| if caching: |
| EnableCaching() |
| |
| for index, target in enumerate(targets, start=1): |
| print(f'[{index}|{len(targets)}] Generating {target}') |
| |
| # First grab a class contructor object and create an instance |
| targetGenerator = generators[target]['generator'] |
| generatorOptions = generators[target]['options'] if 'options' in generators[target] else [] |
| generator = targetGenerator(*generatorOptions) |
| |
| # This code and the 'genCombined' generator metadata is used by downstream |
| # users to generate code with all Vulkan APIs merged into the target API variant |
| # (e.g. Vulkan SC) when needed. The constructed apiList is also used to filter |
| # out non-applicable extensions later below. |
| apiList = [api] |
| if api != 'vulkan' and generators[target]['genCombined']: |
| SetMergedApiNames('vulkan') |
| apiList.append('vulkan') |
| else: |
| SetMergedApiNames(None) |
| |
| baseOptions = BaseGeneratorOptions(customFileName = target) |
| |
| # Create the registry object with the specified generator and generator |
| # options. The options are set before XML loading as they may affect it. |
| reg = Registry(generator, baseOptions) |
| |
| # Parse the specified registry XML into an ElementTree object |
| tree = ElementTree.parse(registry) |
| |
| # Load the XML tree into the registry object |
| reg.loadElementTree(tree) |
| |
| # The cached data is saved inside the BaseGenerator, so search for it and try |
| # to reuse the parsing for each generator file. |
| if caching and not cacheVkObjectData: |
| if os.path.isfile(cachePath): |
| file = open(cachePath, 'rb') |
| cacheVkObjectData = pickle.load(file) |
| file.close() |
| |
| if caching and cacheVkObjectData and ('regenerate' not in generators[target] or not generators[target]['regenerate']): |
| # TODO - We shouldn't have to regenerate any files, need to investigate why we some scripts need it |
| reg.gen.generateFromCache(cacheVkObjectData, reg.genOpts) |
| else: |
| # Finally, use the output generator to create the requested target |
| reg.apiGen() |
| |
| # Run clang-format on the file |
| if has_clang_format: |
| common_ci.RunShellCmd(f'clang-format -i --style=file:{styleFile} {os.path.join(directory, target)}') |
| |
| if os.path.isfile(cachePath): |
| os.remove(cachePath) |
| |
| # helper to define paths relative to the repo root |
| def repo_relative(path): |
| return os.path.abspath(os.path.join(os.path.dirname(__file__), '..', path)) |
| |
| def main(argv): |
| # files to exclude from --verify check |
| # The shaders requires glslangvalidator, so they are updated manually with generate_spirv when needed |
| verify_exclude = [ |
| '.clang-format', |
| 'gpuav_offline_spirv.h', |
| 'gpuav_offline_spirv.cpp', |
| 'feature_requirements_helper.h', # https://github.com/KhronosGroup/Vulkan-ValidationLayers/issues/8969 |
| 'feature_requirements_helper.cpp', |
| 'feature_not_present.cpp', # this single funciton really fails with various clang-format versions |
| ] |
| |
| parser = argparse.ArgumentParser(description='Generate source code for this repository') |
| parser.add_argument('--api', |
| default='vulkan', |
| choices=['vulkan'], |
| help='Specify API name to generate') |
| parser.add_argument('paths', nargs='+', |
| help='Either: Paths to the Vulkan-Headers registry directory and the SPIRV-Headers grammar directory' |
| + ' OR path to the base directory containing the Vulkan-Headers and SPIRV-Headers repositories') |
| parser.add_argument('--generated-version', help='sets the header version used to generate the repo') |
| parser.add_argument('-o', help='Create target and related files in specified directory.', dest='output_directory') |
| group = parser.add_mutually_exclusive_group() |
| group.add_argument('--target', nargs='+', help='only generate file names passed in') |
| group.add_argument('-i', '--incremental', action='store_true', help='only update repo files that change') |
| group.add_argument('-v', '--verify', action='store_true', help='verify repo files match generator output') |
| group.add_argument('--no-caching', action='store_true', help='Do not try to cache generator objects') |
| args = parser.parse_args(argv) |
| |
| repo_dir = repo_relative(f'layers/{args.api}/generated') |
| |
| # Need pass style file incase running with --verify and it can't find the file automatically in the temp directory |
| styleFile = os.path.join(repo_dir, '.clang-format') |
| if common_ci.IsGHA() and args.verify: |
| # Have found that sometimes (~5%) the 20.04 Ubuntu machines have clang-format v11 but we need v14 to |
| # use a dedicated styleFile location. For these case there we can survive just skipping the verify check |
| stdout = subprocess.check_output(['clang-format', '--version']).decode("utf-8") |
| version = stdout[stdout.index('version') + 8:][:2] |
| if int(version) < 14: |
| return 0 # Success |
| |
| # Update the api_version in the respective json files |
| if args.generated_version: |
| json_files = [] |
| json_files.append(repo_relative('layers/VkLayer_khronos_validation.json.in')) |
| json_files.append(repo_relative('tests/layers/VkLayer_device_profile_api.json.in')) |
| for json_file in json_files: |
| with open(json_file, 'r') as file: |
| json_str = file.read() |
| with open(json_file, 'w') as file: |
| # Update json at the string-level so it doesn't get reformatted |
| file.write(re.sub(r'("api_version" *: *)".*?"', fr'\1"{args.generated_version}"', json_str)) |
| |
| # get directory where generators will run |
| if args.verify or args.incremental: |
| # generate in temp directory so we can compare or copy later |
| temp_obj = tempfile.TemporaryDirectory(prefix='vvl_codegen_') |
| temp_dir = temp_obj.name |
| gen_dir = temp_dir |
| else: |
| # generate directly in the repo |
| gen_dir = repo_dir |
| |
| if args.output_directory is not None: |
| gen_dir = args.output_directory |
| |
| if len(args.paths) == 1: |
| base = args.paths[0] |
| registry = os.path.join(base, 'Vulkan-Headers/registry') |
| grammar = os.path.join(base, 'SPIRV-Headers/include/spirv/unified1') |
| elif len(args.paths) == 2: |
| registry = args.paths[0] |
| grammar = args.paths[1] |
| else: |
| args.print_help() |
| return -1 |
| |
| registry = os.path.abspath(os.path.join(registry, 'vk.xml')) |
| if not os.path.isfile(registry): |
| print(f'{registry} does not exist') |
| return -1 |
| grammar = os.path.abspath(os.path.join(grammar, 'spirv.core.grammar.json')) |
| if not os.path.isfile(grammar): |
| print(f'{grammar} does not exist') |
| return -1 |
| |
| caching = not args.no_caching |
| RunGenerators(args.api, registry, grammar, gen_dir, styleFile, args.target, caching) |
| |
| # Generate vk_validation_error_messages.h (ignore if targeting a single generator) |
| if (not args.target): |
| valid_usage_file = os.path.abspath(os.path.join(os.path.dirname(registry), "validusage.json")) |
| error_message_file = os.path.join(gen_dir, 'vk_validation_error_messages.h') |
| GenerateSpecErrorMessage(args.api, valid_usage_file, error_message_file) |
| |
| # optional post-generation steps |
| if args.verify: |
| # compare contents of temp dir and repo |
| temp_files = set(os.listdir(temp_dir)) |
| repo_files = set(os.listdir(repo_dir)) |
| for filename in sorted((temp_files | repo_files) - set(verify_exclude)): |
| temp_filename = os.path.join(temp_dir, filename) |
| repo_filename = os.path.join(repo_dir, filename) |
| if filename not in repo_files: |
| print('ERROR: Missing repo file', filename) |
| return 2 |
| elif filename not in temp_files: |
| print('ERROR: Missing generator for', filename) |
| return 3 |
| elif not filecmp.cmp(temp_filename, repo_filename, shallow=False): |
| print('ERROR: Repo files do not match generator output for', filename) |
| # print line diff on file mismatch |
| with open(temp_filename) as temp_file, open(repo_filename) as repo_file: |
| print(''.join(difflib.unified_diff(temp_file.readlines(), |
| repo_file.readlines(), |
| fromfile='temp/' + filename, |
| tofile= 'repo/' + filename))) |
| return 4 |
| |
| # return code for test scripts |
| print('SUCCESS: Repo files match generator output') |
| |
| elif args.incremental: |
| # copy missing or differing files from temp directory to repo |
| for filename in os.listdir(temp_dir): |
| temp_filename = os.path.join(temp_dir, filename) |
| repo_filename = os.path.join(repo_dir, filename) |
| if not os.path.exists(repo_filename) or \ |
| not filecmp.cmp(temp_filename, repo_filename, shallow=False): |
| print('update', repo_filename) |
| shutil.copyfile(temp_filename, repo_filename) |
| |
| # write out the header version used to generate the code to a checked in CMake file |
| if args.generated_version: |
| # Update the CMake project version |
| with open(repo_relative('CMakeLists.txt'), "r+") as f: |
| data = f.read() |
| f.seek(0) |
| f.write(re.sub("project.*VERSION.*", f"project(VVL VERSION {args.generated_version} LANGUAGES CXX)", data)) |
| f.truncate() |
| |
| return 0 |
| |
| if __name__ == '__main__': |
| sys.exit(main(sys.argv[1:])) |
| |
