From 75c680ad48daac344deb6c5639baa7bb944903e9 Mon Sep 17 00:00:00 2001 From: aharon-abramson Date: Mon, 6 Nov 2023 14:53:00 +0200 Subject: [PATCH 001/202] Update cl_khr_command_buffer_mutable_dispatch.asciidoc Add type cl_mutable_dispatch_promises_khr and its possible values --- ...r_command_buffer_mutable_dispatch.asciidoc | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc index 2fef8ca19..53dd351f5 100644 --- a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc +++ b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc @@ -101,6 +101,9 @@ typedef cl_uint cl_mutable_command_info_khr; // Identifies the type of a structure to allow structure pointer chains typedef cl_uint cl_command_buffer_structure_type_khr; + +// Bitfield covering certain promises by the user to the implementation, enabling possible optimizations +typedef cl_bitfield cl_mutable_dispatch_promises_khr; ---- Struct type for setting kernel arguments normally passed using {clSetKernelArg} @@ -233,6 +236,7 @@ CL_DEVICE_MUTABLE_DISPATCH_CAPABILITIES_KHR 0x12B0 // Property to cl_ndrange_kernel_command_properties_khr CL_MUTABLE_DISPATCH_UPDATABLE_FIELDS_KHR 0x12B1 +CL_MUTABLE_DISPATCH_PROMISES_KHR 0x12B2 // Bits for cl_mutable_dispatch_fields_khr bitfield CL_MUTABLE_DISPATCH_GLOBAL_OFFSET_KHR (0x1 << 0) @@ -254,6 +258,9 @@ CL_MUTABLE_COMMAND_COMMAND_TYPE_KHR 0x12AD // Bits for cl_command_buffer_flags_khr CL_COMMAND_BUFFER_MUTABLE_KHR (0x1 << 1) + +// Bits for cl_mutable_dispatch_promises_khr bitfield +CL_MUTABLE_DISPATCH_PROMISE_NO_ADDITIONAL_WORK_GROUPS_KHR (0x1 << 0) ---- Enum values for {cl_command_buffer_structure_type_khr_TYPE} allowing the structure @@ -321,6 +328,14 @@ description of property values. | {CL_COMMAND_BUFFER_MUTABLE_KHR} - Enables modification of the command-buffer, by default command-buffers are immutable. If set, commands in the command-buffer may be updated via {clUpdateMutableCommandsKHR}. + +| {CL_MUTABLE_DISPATCH_PROMISES_KHR} +| {cl_mutable_dispatch_promises_khr_TYPE} +| This is a bitfield and can be set to a combination of the following values: + + {CL_MUTABLE_DISPATCH_PROMISE_NO_ADDITIONAL_WORK_GROUPS_KHR} + A promise by the user that the number of work-groups of any ND-range kernel recorded in this command + buffer will not be updated beyond the number defined when the ND-range kernel was recorded. |==== ==== Modifications to clCommandNDRangeKernelKHR @@ -392,6 +407,14 @@ in the table below. If {CL_MUTABLE_DISPATCH_UPDATABLE_FIELDS_KHR} is not specified then it defaults to the value returned by the {CL_DEVICE_MUTABLE_DISPATCH_CAPABILITIES_KHR} device query. + +| {CL_MUTABLE_DISPATCH_PROMISES_KHR} +| {cl_mutable_dispatch_promises_khr_TYPE} +| This is a bitfield and can be set to a combination of the following values: + + {CL_MUTABLE_DISPATCH_PROMISE_NO_ADDITIONAL_WORK_GROUPS_KHR} + A promise by the user that the number of work-groups of this ND-range kernel will not be updated beyond + the number defined when the ND-range kernel was recorded. |==== ===== Mutable Handle Parameter From c37e00f1423379a882c173c12ed1aa5ca9d8e51d Mon Sep 17 00:00:00 2001 From: aharon-abramson Date: Tue, 7 Nov 2023 11:50:25 +0200 Subject: [PATCH 002/202] Update cl_khr_command_buffer_mutable_dispatch.asciidoc --- ext/cl_khr_command_buffer_mutable_dispatch.asciidoc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc index 53dd351f5..ab85fb6cf 100644 --- a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc +++ b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc @@ -20,6 +20,7 @@ commands between command-buffer enqueues. |==== | *Date* | *Version* | *Description* | 2022-08-31 | 0.9.0 | First assigned version (provisional). +| 2023-11-07 | 0.9.1 | Add type cl_mutable_dispatch_promises_khr and its possible values |==== ==== Dependencies @@ -236,7 +237,7 @@ CL_DEVICE_MUTABLE_DISPATCH_CAPABILITIES_KHR 0x12B0 // Property to cl_ndrange_kernel_command_properties_khr CL_MUTABLE_DISPATCH_UPDATABLE_FIELDS_KHR 0x12B1 -CL_MUTABLE_DISPATCH_PROMISES_KHR 0x12B2 +CL_MUTABLE_DISPATCH_PROMISES_KHR 0x12B7 // Bits for cl_mutable_dispatch_fields_khr bitfield CL_MUTABLE_DISPATCH_GLOBAL_OFFSET_KHR (0x1 << 0) @@ -556,9 +557,9 @@ the array violates the defined conditions: * {CL_INVALID_VALUE} if _type_ is not {CL_STRUCTURE_TYPE_MUTABLE_DISPATCH_CONFIG_KHR}. -* {CL_INVALID_OPERATION} if values of _local_work_size_ and/or - _global_work_size_ result in an increase to the number of work-groups in the - ND-range. +* {CL_INVALID_OPERATION} if {CL_MUTABLE_DISPATCH_PROMISE_NO_ADDITIONAL_WORK_GROUPS_KHR} is specified, + and values of _local_work_size_ and/or _global_work_size_ result in an increase to the number of work- + groups in the ND-range over the number specified when the ND-range kernel was recorded. * {CL_INVALID_OPERATION} if the values of _local_work_size_ and/or _global_work_size_ result in a change to work-group uniformity. From ed629a972fd6a2377b120b64e78fa1ceb2b9f8d3 Mon Sep 17 00:00:00 2001 From: Aharon Abramson Date: Tue, 7 Nov 2023 12:20:56 +0200 Subject: [PATCH 003/202] Update ext/cl_khr_command_buffer_mutable_dispatch.asciidoc Co-authored-by: Ewan Crawford --- ext/cl_khr_command_buffer_mutable_dispatch.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc index ab85fb6cf..6f2c8a3b4 100644 --- a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc +++ b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc @@ -20,7 +20,7 @@ commands between command-buffer enqueues. |==== | *Date* | *Version* | *Description* | 2022-08-31 | 0.9.0 | First assigned version (provisional). -| 2023-11-07 | 0.9.1 | Add type cl_mutable_dispatch_promises_khr and its possible values +| 2023-11-07 | 0.9.1 | Add type cl_mutable_dispatch_promises_khr and its possible values (provisional). |==== ==== Dependencies From e7d3343d95113e93e59eb76e4458896b07c7eded Mon Sep 17 00:00:00 2001 From: aharon-abramson Date: Fri, 10 Nov 2023 16:12:31 +0200 Subject: [PATCH 004/202] changes to cl.xml --- ...r_command_buffer_mutable_dispatch.asciidoc | 36 ++++++++++--------- xml/cl.xml | 11 +++++- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc index 6f2c8a3b4..a8b234df7 100644 --- a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc +++ b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc @@ -20,7 +20,7 @@ commands between command-buffer enqueues. |==== | *Date* | *Version* | *Description* | 2022-08-31 | 0.9.0 | First assigned version (provisional). -| 2023-11-07 | 0.9.1 | Add type cl_mutable_dispatch_promises_khr and its possible values (provisional). +| 2023-11-07 | 0.9.1 | Add type cl_mutable_dispatch_asserts_khr and its possible values (provisional). |==== ==== Dependencies @@ -103,8 +103,8 @@ typedef cl_uint cl_mutable_command_info_khr; // Identifies the type of a structure to allow structure pointer chains typedef cl_uint cl_command_buffer_structure_type_khr; -// Bitfield covering certain promises by the user to the implementation, enabling possible optimizations -typedef cl_bitfield cl_mutable_dispatch_promises_khr; +// Bitfield covering certain asserts by the user to the implementation, enabling possible optimizations +typedef cl_bitfield cl_mutable_dispatch_asserts_khr; ---- Struct type for setting kernel arguments normally passed using {clSetKernelArg} @@ -237,7 +237,7 @@ CL_DEVICE_MUTABLE_DISPATCH_CAPABILITIES_KHR 0x12B0 // Property to cl_ndrange_kernel_command_properties_khr CL_MUTABLE_DISPATCH_UPDATABLE_FIELDS_KHR 0x12B1 -CL_MUTABLE_DISPATCH_PROMISES_KHR 0x12B7 +CL_MUTABLE_DISPATCH_ASSERTS_KHR 0x12B7 // Bits for cl_mutable_dispatch_fields_khr bitfield CL_MUTABLE_DISPATCH_GLOBAL_OFFSET_KHR (0x1 << 0) @@ -260,8 +260,8 @@ CL_MUTABLE_COMMAND_COMMAND_TYPE_KHR 0x12AD // Bits for cl_command_buffer_flags_khr CL_COMMAND_BUFFER_MUTABLE_KHR (0x1 << 1) -// Bits for cl_mutable_dispatch_promises_khr bitfield -CL_MUTABLE_DISPATCH_PROMISE_NO_ADDITIONAL_WORK_GROUPS_KHR (0x1 << 0) +// Bits for cl_mutable_dispatch_asserts_khr bitfield +CL_MUTABLE_DISPATCH_ASSERT_NO_ADDITIONAL_WORK_GROUPS_KHR (0x1 << 0) ---- Enum values for {cl_command_buffer_structure_type_khr_TYPE} allowing the structure @@ -330,12 +330,12 @@ description of property values. command-buffer, by default command-buffers are immutable. If set, commands in the command-buffer may be updated via {clUpdateMutableCommandsKHR}. -| {CL_MUTABLE_DISPATCH_PROMISES_KHR} -| {cl_mutable_dispatch_promises_khr_TYPE} +| {CL_MUTABLE_DISPATCH_ASSERTS_KHR} +| {cl_mutable_dispatch_asserts_khr_TYPE} | This is a bitfield and can be set to a combination of the following values: - {CL_MUTABLE_DISPATCH_PROMISE_NO_ADDITIONAL_WORK_GROUPS_KHR} - A promise by the user that the number of work-groups of any ND-range kernel recorded in this command + {CL_MUTABLE_DISPATCH_ASSERT_NO_ADDITIONAL_WORK_GROUPS_KHR} + An assertion by the user that the number of work-groups of any ND-range kernel recorded in this command buffer will not be updated beyond the number defined when the ND-range kernel was recorded. |==== @@ -409,13 +409,17 @@ in the table below. defaults to the value returned by the {CL_DEVICE_MUTABLE_DISPATCH_CAPABILITIES_KHR} device query. -| {CL_MUTABLE_DISPATCH_PROMISES_KHR} -| {cl_mutable_dispatch_promises_khr_TYPE} +| {CL_MUTABLE_DISPATCH_ASSERTS_KHR} +| {cl_mutable_dispatch_asserts_khr_TYPE} | This is a bitfield and can be set to a combination of the following values: - {CL_MUTABLE_DISPATCH_PROMISE_NO_ADDITIONAL_WORK_GROUPS_KHR} - A promise by the user that the number of work-groups of this ND-range kernel will not be updated beyond - the number defined when the ND-range kernel was recorded. + {CL_MUTABLE_DISPATCH_ASSERT_NO_ADDITIONAL_WORK_GROUPS_KHR} + An assertion by the user that the number of work-groups of this ND-range kernel will not be updated beyond + the number defined when the ND-range kernel was recorded. The number of work-groups is + defined as the product for each _i_ from _0_ to _work_dim - 1_ of + _ceil((global_work_size[i] - global_work_offset[i])/local_work_size[i])_ (if + _global_work_offset_ is NULL, _global_work_offset[i]_ should be replaced with _0_). + In case _local_work_size_ is NULL, the effect of this flag is undefined. |==== ===== Mutable Handle Parameter @@ -557,7 +561,7 @@ the array violates the defined conditions: * {CL_INVALID_VALUE} if _type_ is not {CL_STRUCTURE_TYPE_MUTABLE_DISPATCH_CONFIG_KHR}. -* {CL_INVALID_OPERATION} if {CL_MUTABLE_DISPATCH_PROMISE_NO_ADDITIONAL_WORK_GROUPS_KHR} is specified, +* {CL_INVALID_OPERATION} if {CL_MUTABLE_DISPATCH_ASSERT_NO_ADDITIONAL_WORK_GROUPS_KHR} is specified, and values of _local_work_size_ and/or _global_work_size_ result in an increase to the number of work- groups in the ND-range over the number specified when the ND-range kernel was recorded. diff --git a/xml/cl.xml b/xml/cl.xml index 69a63d931..f4597b7c6 100644 --- a/xml/cl.xml +++ b/xml/cl.xml @@ -252,6 +252,7 @@ server's OpenCL/api-docs repository. typedef cl_bitfield cl_device_fp_atomic_capabilities_ext; typedef cl_uint cl_image_requirements_info_ext; typedef cl_bitfield cl_platform_command_buffer_capabilities_khr; + typedef cl_bitfield cl_mutable_dispatch_promises_khr Structure types @@ -1343,6 +1344,9 @@ server's OpenCL/api-docs repository. + + + @@ -1779,7 +1783,8 @@ server's OpenCL/api-docs repository. - + + @@ -7277,6 +7282,7 @@ server's OpenCL/api-docs repository. + @@ -7313,6 +7319,9 @@ server's OpenCL/api-docs repository. + + + From 392d97b7fba824bfa4ece034003e1a8172df6c98 Mon Sep 17 00:00:00 2001 From: aharon-abramson Date: Sun, 12 Nov 2023 09:19:41 +0200 Subject: [PATCH 005/202] update extension's version --- xml/cl.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xml/cl.xml b/xml/cl.xml index f4597b7c6..063e2e3f2 100644 --- a/xml/cl.xml +++ b/xml/cl.xml @@ -1783,7 +1783,7 @@ server's OpenCL/api-docs repository. - + @@ -7270,7 +7270,7 @@ server's OpenCL/api-docs repository. - + From 995886a6f41ef24cd2e88f4fbdf7b3397c9a8258 Mon Sep 17 00:00:00 2001 From: aharon-abramson Date: Mon, 13 Nov 2023 13:19:23 +0200 Subject: [PATCH 006/202] rename "promise" to "assert" --- .asciidoctorconfig.adoc | 12 + .project | 11 + xml/cgenerator.py | 420 +++++++++ xml/checklinks.py | 71 ++ xml/cl.xml | 14 +- xml/clconventions.py | 241 ++++++ xml/conventions.py | 358 ++++++++ xml/docgenerator.py | 454 ++++++++++ xml/extensionmetadocgenerator.py | 659 ++++++++++++++ xml/genRef.py | 1019 ++++++++++++++++++++++ xml/gen_dictionaries.py | 258 ++++++ xml/gen_version_notes.py | 127 +++ xml/gencl.py | 464 ++++++++++ xml/generator.py | 1186 +++++++++++++++++++++++++ xml/pygenerator.py | 365 ++++++++ xml/realign.py | 47 + xml/reflib.py | 663 ++++++++++++++ xml/reg.py | 1397 ++++++++++++++++++++++++++++++ 18 files changed, 7759 insertions(+), 7 deletions(-) create mode 100644 .asciidoctorconfig.adoc create mode 100644 .project create mode 100644 xml/cgenerator.py create mode 100644 xml/checklinks.py create mode 100644 xml/clconventions.py create mode 100644 xml/conventions.py create mode 100644 xml/docgenerator.py create mode 100644 xml/extensionmetadocgenerator.py create mode 100644 xml/genRef.py create mode 100644 xml/gen_dictionaries.py create mode 100644 xml/gen_version_notes.py create mode 100644 xml/gencl.py create mode 100644 xml/generator.py create mode 100644 xml/pygenerator.py create mode 100644 xml/realign.py create mode 100644 xml/reflib.py create mode 100644 xml/reg.py diff --git a/.asciidoctorconfig.adoc b/.asciidoctorconfig.adoc new file mode 100644 index 000000000..53b403dbc --- /dev/null +++ b/.asciidoctorconfig.adoc @@ -0,0 +1,12 @@ +// +++++++++++++++++++++++++++++++++++++++++++++++++++++++ +// + Initial AsciiDoc editor configuration file - V1.0 + +// ++++++++++++++++++++++++++++++++++++++++++++++++++++++ +// +// Did not found any configuration files, so create this at project root level. +// If you do not like those files to be generated - you can turn it off inside Asciidoctor Editor preferences. +// +// You can define editor specific parts here. +// For example: with next line you could set imagesdir attribute to subfolder "images" relative to the folder where this config file is located. +// :imagesdir: {asciidoctorconfigdir}/images +// +// For more information please take a look at https://github.com/de-jcup/eclipse-asciidoctor-editor/wiki/Asciidoctor-configfiles diff --git a/.project b/.project new file mode 100644 index 000000000..9877aec6c --- /dev/null +++ b/.project @@ -0,0 +1,11 @@ + + + OpenCL-Docs + + + + + + + + diff --git a/xml/cgenerator.py b/xml/cgenerator.py new file mode 100644 index 000000000..4b2a8f1e2 --- /dev/null +++ b/xml/cgenerator.py @@ -0,0 +1,420 @@ +#!/usr/bin/python3 -i +# +# Copyright 2013-2023 The Khronos Group Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +import re +from generator import (GeneratorOptions, OutputGenerator, noneStr, + regSortFeatures, write) + + +class CGeneratorOptions(GeneratorOptions): + """CGeneratorOptions - subclass of GeneratorOptions. + + Adds options used by COutputGenerator objects during C language header + generation.""" + + def __init__(self, + prefixText="", + genFuncPointers=True, + protectFile=True, + protectFeature=True, + protectProto=None, + protectProtoStr=None, + apicall='', + apientry='', + apientryp='', + indentFuncProto=True, + indentFuncPointer=False, + alignFuncParam=0, + genEnumBeginEndRange=False, + genAliasMacro=False, + aliasMacro='', + misracstyle=False, + misracppstyle=False, + **kwargs + ): + """Constructor. + Additional parameters beyond parent class: + + - prefixText - list of strings to prefix generated header with + (usually a copyright statement + calling convention macros). + - protectFile - True if multiple inclusion protection should be + generated (based on the filename) around the entire header. + - protectFeature - True if #ifndef..#endif protection should be + generated around a feature interface in the header file. + - genFuncPointers - True if function pointer typedefs should be + generated + - protectProto - If conditional protection should be generated + around prototype declarations, set to either '#ifdef' + to require opt-in (#ifdef protectProtoStr) or '#ifndef' + to require opt-out (#ifndef protectProtoStr). Otherwise + set to None. + - protectProtoStr - #ifdef/#ifndef symbol to use around prototype + declarations, if protectProto is set + - apicall - string to use for the function declaration prefix, + such as APICALL on Windows. + - apientry - string to use for the calling convention macro, + in typedefs, such as APIENTRY. + - apientryp - string to use for the calling convention macro + in function pointer typedefs, such as APIENTRYP. + - indentFuncProto - True if prototype declarations should put each + parameter on a separate line + - indentFuncPointer - True if typedefed function pointers should put each + parameter on a separate line + - alignFuncParam - if nonzero and parameters are being put on a + separate line, align parameter names at the specified column + - genEnumBeginEndRange - True if BEGIN_RANGE / END_RANGE macros should + be generated for enumerated types + - genAliasMacro - True if the OpenXR alias macro should be generated + for aliased types (unclear what other circumstances this is useful) + - aliasMacro - alias macro to inject when genAliasMacro is True + - misracstyle - generate MISRA C-friendly headers + - misracppstyle - generate MISRA C++-friendly headers""" + + GeneratorOptions.__init__(self, **kwargs) + + self.prefixText = prefixText + """list of strings to prefix generated header with (usually a copyright statement + calling convention macros).""" + + self.genFuncPointers = genFuncPointers + """True if function pointer typedefs should be generated""" + + self.protectFile = protectFile + """True if multiple inclusion protection should be generated (based on the filename) around the entire header.""" + + self.protectFeature = protectFeature + """True if #ifndef..#endif protection should be generated around a feature interface in the header file.""" + + self.protectProto = protectProto + """If conditional protection should be generated around prototype declarations, set to either '#ifdef' to require opt-in (#ifdef protectProtoStr) or '#ifndef' to require opt-out (#ifndef protectProtoStr). Otherwise set to None.""" + + self.protectProtoStr = protectProtoStr + """#ifdef/#ifndef symbol to use around prototype declarations, if protectProto is set""" + + self.apicall = apicall + """string to use for the function declaration prefix, such as APICALL on Windows.""" + + self.apientry = apientry + """string to use for the calling convention macro, in typedefs, such as APIENTRY.""" + + self.apientryp = apientryp + """string to use for the calling convention macro in function pointer typedefs, such as APIENTRYP.""" + + self.indentFuncProto = indentFuncProto + """True if prototype declarations should put each parameter on a separate line""" + + self.indentFuncPointer = indentFuncPointer + """True if typedefed function pointers should put each parameter on a separate line""" + + self.alignFuncParam = alignFuncParam + """if nonzero and parameters are being put on a separate line, align parameter names at the specified column""" + + self.genEnumBeginEndRange = genEnumBeginEndRange + """True if BEGIN_RANGE / END_RANGE macros should be generated for enumerated types""" + + self.genAliasMacro = genAliasMacro + """True if the OpenXR alias macro should be generated for aliased types (unclear what other circumstances this is useful)""" + + self.aliasMacro = aliasMacro + """alias macro to inject when genAliasMacro is True""" + + self.misracstyle = misracstyle + """generate MISRA C-friendly headers""" + + self.misracppstyle = misracppstyle + """generate MISRA C++-friendly headers""" + + self.codeGenerator = True + """True if this generator makes compilable code""" + + +class COutputGenerator(OutputGenerator): + """Generates C-language API interfaces.""" + + # This is an ordered list of sections in the header file. + TYPE_SECTIONS = ['include', 'define', 'basetype', 'handle', 'enum', + 'group', 'bitmask', 'funcpointer', 'struct'] + ALL_SECTIONS = TYPE_SECTIONS + ['commandPointer', 'command'] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Internal state - accumulators for different inner block text + self.sections = {section: [] for section in self.ALL_SECTIONS} + self.feature_not_empty = False + self.may_alias = None + + def beginFile(self, genOpts): + OutputGenerator.beginFile(self, genOpts) + # C-specific + # + # Multiple inclusion protection & C++ wrappers. + if genOpts.protectFile and self.genOpts.filename: + headerSym = re.sub(r'\.h', '_h_', + os.path.basename(self.genOpts.filename)).upper() + write('#ifndef', headerSym, file=self.outFile) + write('#define', headerSym, '1', file=self.outFile) + self.newline() + + # User-supplied prefix text, if any (list of strings) + if genOpts.prefixText: + for s in genOpts.prefixText: + write(s, file=self.outFile) + + # C++ extern wrapper - after prefix lines so they can add includes. + self.newline() + write('#ifdef __cplusplus', file=self.outFile) + write('extern "C" {', file=self.outFile) + write('#endif', file=self.outFile) + self.newline() + + def endFile(self): + # C-specific + # Finish C++ wrapper and multiple inclusion protection + self.newline() + write('#ifdef __cplusplus', file=self.outFile) + write('}', file=self.outFile) + write('#endif', file=self.outFile) + if self.genOpts.protectFile and self.genOpts.filename: + self.newline() + write('#endif', file=self.outFile) + # Finish processing in superclass + OutputGenerator.endFile(self) + + def beginFeature(self, interface, emit): + # Start processing in superclass + OutputGenerator.beginFeature(self, interface, emit) + # C-specific + # Accumulate includes, defines, types, enums, function pointer typedefs, + # end function prototypes separately for this feature. They're only + # printed in endFeature(). + self.sections = {section: [] for section in self.ALL_SECTIONS} + self.feature_not_empty = False + + def endFeature(self): + "Actually write the interface to the output file." + # C-specific + if self.emit: + if self.feature_not_empty: + if self.genOpts.conventions.writeFeature(self.featureExtraProtect, self.genOpts.filename): + self.newline() + if self.genOpts.protectFeature: + write('#ifndef', self.featureName, file=self.outFile) + # If type declarations are needed by other features based on + # this one, it may be necessary to suppress the ExtraProtect, + # or move it below the 'for section...' loop. + if self.featureExtraProtect is not None: + write('#ifdef', self.featureExtraProtect, file=self.outFile) + self.newline() + write('#define', self.featureName, '1', file=self.outFile) + for section in self.TYPE_SECTIONS: + contents = self.sections[section] + if contents: + write('\n'.join(contents), file=self.outFile) + if self.genOpts.genFuncPointers and self.sections['commandPointer']: + write('\n'.join(self.sections['commandPointer']), file=self.outFile) + self.newline() + if self.sections['command']: + if self.genOpts.protectProto: + write(self.genOpts.protectProto, + self.genOpts.protectProtoStr, file=self.outFile) + write('\n'.join(self.sections['command']), end='', file=self.outFile) + if self.genOpts.protectProto: + write('#endif', file=self.outFile) + else: + self.newline() + if self.featureExtraProtect is not None: + write('#endif /*', self.featureExtraProtect, '*/', file=self.outFile) + if self.genOpts.protectFeature: + write('#endif /*', self.featureName, '*/', file=self.outFile) + # Finish processing in superclass + OutputGenerator.endFeature(self) + + def appendSection(self, section, text): + "Append a definition to the specified section" + # self.sections[section].append('SECTION: ' + section + '\n') + self.sections[section].append(text) + self.feature_not_empty = True + + def genType(self, typeinfo, name, alias): + "Generate type." + OutputGenerator.genType(self, typeinfo, name, alias) + typeElem = typeinfo.elem + + # Vulkan: + # Determine the category of the type, and the type section to add + # its definition to. + # 'funcpointer' is added to the 'struct' section as a workaround for + # internal issue #877, since structures and function pointer types + # can have cross-dependencies. + category = typeElem.get('category') + if category == 'funcpointer': + section = 'struct' + else: + section = category + + if category in ('struct', 'union'): + # If the type is a struct type, generate it using the + # special-purpose generator. + self.genStruct(typeinfo, name, alias) + else: + # OpenXR: this section was not under 'else:' previously, just fell through + if alias: + # If the type is an alias, just emit a typedef declaration + body = 'typedef ' + alias + ' ' + name + ';\n' + else: + # Replace tags with an APIENTRY-style string + # (from self.genOpts). Copy other text through unchanged. + # If the resulting text is an empty string, don't emit it. + body = noneStr(typeElem.text) + for elem in typeElem: + if elem.tag == 'apientry': + body += self.genOpts.apientry + noneStr(elem.tail) + else: + body += noneStr(elem.text) + noneStr(elem.tail) + if body: + # Add extra newline after multi-line entries. + if '\n' in body[0:-1]: + body += '\n' + self.appendSection(section, body) + + def genProtectString(self, protect_str): + """Generate protection string. + + Protection strings are the strings defining the OS/Platform/Graphics + requirements for a given OpenXR command. When generating the + language header files, we need to make sure the items specific to a + graphics API or OS platform are properly wrapped in #ifs.""" + protect_if_str = '' + protect_end_str = '' + if not protect_str: + return (protect_if_str, protect_end_str) + + if ',' in protect_str: + protect_list = protect_str.split(",") + protect_defs = ('defined(%s)' % d for d in protect_list) + protect_def_str = ' && '.join(protect_defs) + protect_if_str = '#if %s\n' % protect_def_str + protect_end_str = '#endif // %s\n' % protect_def_str + else: + protect_if_str = '#ifdef %s\n' % protect_str + protect_end_str = '#endif // %s\n' % protect_str + + return (protect_if_str, protect_end_str) + + def typeMayAlias(self, typeName): + if not self.may_alias: + # First time we've asked if a type may alias. + # So, let's populate the set of all names of types that may. + + # Everyone with an explicit mayalias="true" + self.may_alias = set(typeName + for typeName, data in self.registry.typedict.items() + if data.elem.get('mayalias') == 'true') + + # Every type mentioned in some other type's parentstruct attribute. + parent_structs = (otherType.elem.get('parentstruct') + for otherType in self.registry.typedict.values()) + self.may_alias.update(set(x for x in parent_structs + if x is not None)) + return typeName in self.may_alias + + def genStruct(self, typeinfo, typeName, alias): + """Generate struct (e.g. C "struct" type). + + This is a special case of the tag where the contents are + interpreted as a set of tags instead of freeform C + C type declarations. The tags are just like + tags - they are a declaration of a struct or union member. + Only simple member declarations are supported (no nested + structs etc.) + + If alias is not None, then this struct aliases another; just + generate a typedef of that alias.""" + OutputGenerator.genStruct(self, typeinfo, typeName, alias) + + typeElem = typeinfo.elem + + if alias: + body = 'typedef ' + alias + ' ' + typeName + ';\n' + else: + body = '' + (protect_begin, protect_end) = self.genProtectString(typeElem.get('protect')) + if protect_begin: + body += protect_begin + body += 'typedef ' + typeElem.get('category') + + # This is an OpenXR-specific alternative where aliasing refers + # to an inheritance hierarchy of types rather than C-level type + # aliases. + if self.genOpts.genAliasMacro and self.typeMayAlias(typeName): + body += ' ' + self.genOpts.aliasMacro + + body += ' ' + typeName + ' {\n' + + targetLen = self.getMaxCParamTypeLength(typeinfo) + for member in typeElem.findall('.//member'): + body += self.makeCParamDecl(member, targetLen + 4) + body += ';\n' + body += '} ' + typeName + ';\n' + if protect_end: + body += protect_end + + self.appendSection('struct', body) + + def genGroup(self, groupinfo, groupName, alias=None): + """Generate groups (e.g. C "enum" type). + + These are concatenated together with other types. + + If alias is not None, it is the name of another group type + which aliases this type; just generate that alias.""" + OutputGenerator.genGroup(self, groupinfo, groupName, alias) + groupElem = groupinfo.elem + + # After either enumerated type or alias paths, add the declaration + # to the appropriate section for the group being defined. + if groupElem.get('type') == 'bitmask': + section = 'bitmask' + else: + section = 'group' + + if alias: + # If the group name is aliased, just emit a typedef declaration + # for the alias. + body = 'typedef ' + alias + ' ' + groupName + ';\n' + self.appendSection(section, body) + else: + (section, body) = self.buildEnumCDecl(self.genOpts.genEnumBeginEndRange, groupinfo, groupName) + self.appendSection(section, "\n" + body) + + def genEnum(self, enuminfo, name, alias): + """Generate the C declaration for a constant (a single value).""" + + OutputGenerator.genEnum(self, enuminfo, name, alias) + + body = self.buildConstantCDecl(enuminfo, name, alias) + self.appendSection('enum', body) + + def genCmd(self, cmdinfo, name, alias): + "Command generation" + OutputGenerator.genCmd(self, cmdinfo, name, alias) + + # if alias: + # prefix = '// ' + name + ' is an alias of command ' + alias + '\n' + # else: + # prefix = '' + + prefix = '' + decls = self.makeCDecls(cmdinfo.elem) + self.appendSection('command', prefix + decls[0] + '\n') + if self.genOpts.genFuncPointers: + self.appendSection('commandPointer', decls[1]) + + def misracstyle(self): + return self.genOpts.misracstyle; + + def misracppstyle(self): + return self.genOpts.misracppstyle; diff --git a/xml/checklinks.py b/xml/checklinks.py new file mode 100644 index 000000000..94b650dad --- /dev/null +++ b/xml/checklinks.py @@ -0,0 +1,71 @@ +#!/usr/bin/python3 +# +# Copyright 2013-2023 The Khronos Group Inc. +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import os +import re + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument('-d', action='store', dest='directory', + default='../api', + help='Directory containing files to check') + parser.add_argument('--unlinked', action='store_true', + help='Check for unlinked APIs and enums (may have false positives!)') + + args = parser.parse_args() + + links = set() + anchors = set() + + for filename in os.listdir(args.directory): + filename = args.directory + '/' + filename + sourcefile = open(filename, 'r') + sourcetext = sourcefile.read() + sourcefile.close() + + # We're not going to check API links. + #filelinks = re.findall(r"{((cl\w+)|(CL\w+))}", sourcetext) + filelinks = re.findall(r"{((CL\w+))}", sourcetext) + fileanchors = re.findall(r"{((cl\w+)|(CL\w+))_anchor}", sourcetext) + + filelinks = [re.sub(r"_anchor\b", "", link[0]) for link in filelinks] + fileanchors = [anchor[0] for anchor in fileanchors] + + links = links.union(set(filelinks) - set(fileanchors)) + anchors = anchors.union(set(fileanchors)) + + #print("=== " + filename) + #print("links:") + #print(' '.join(filelinks)) + #print("anchors:") + #print(' '.join(fileanchors)) + + if args.unlinked: + # Look for APIs and enums that do not begin with: + # { = asciidoctor attribute link + # character = middle of word + # < = asciidoctor link + # ' = refpage description + # / = proto include + fileunlinkedapi = sorted(list(set(re.findall(r"[^{\w<'/](cl[A-Z]\w+)\b[^'](?!.')", sourcetext)))) + fileunlinkedenums = sorted(list(set(re.findall("r[^{\w<](CL_\w+)", sourcetext)))) + fileunlinkedtypes = sorted(list(set(re.findall("r[^{\w<](cl_\w+)", sourcetext)))) + + if len(fileunlinkedapi) != 0: + print("unlinked APIs in " + filename + ":\n\t" + '\n\t'.join(fileunlinkedapi)) + + if len(fileunlinkedenums) != 0: + print("unlinked enums in " + filename + ":\n\t" + '\n\t'.join(fileunlinkedenums)) + + if len(fileunlinkedtypes) != 0: + print("unlinked types in " + filename + ":\n\t" + '\n\t'.join(fileunlinkedtypes)) + + linkswithoutanchors = sorted(list(links - anchors)) + anchorswithoutlinks = sorted(list(anchors - links)) + + print("links without anchors:\n\t" + '\n\t'.join(linkswithoutanchors)) + #print("anchors without links:\n\t" + '\n\t'.join(anchorswithoutlinks)) diff --git a/xml/cl.xml b/xml/cl.xml index 063e2e3f2..283b068d5 100644 --- a/xml/cl.xml +++ b/xml/cl.xml @@ -252,7 +252,7 @@ server's OpenCL/api-docs repository. typedef cl_bitfield cl_device_fp_atomic_capabilities_ext; typedef cl_uint cl_image_requirements_info_ext; typedef cl_bitfield cl_platform_command_buffer_capabilities_khr; - typedef cl_bitfield cl_mutable_dispatch_promises_khr + typedef cl_bitfield cl_mutable_dispatch_asserts_khr Structure types @@ -1344,8 +1344,8 @@ server's OpenCL/api-docs repository. - - + + @@ -1783,7 +1783,7 @@ server's OpenCL/api-docs repository. - + @@ -7282,7 +7282,7 @@ server's OpenCL/api-docs repository. - + @@ -7319,8 +7319,8 @@ server's OpenCL/api-docs repository. - - + + diff --git a/xml/clconventions.py b/xml/clconventions.py new file mode 100644 index 000000000..f4df49d2d --- /dev/null +++ b/xml/clconventions.py @@ -0,0 +1,241 @@ +#!/usr/bin/python3 -i +# +# Copyright 2013-2023 The Khronos Group Inc. +# SPDX-License-Identifier: Apache-2.0 + +# Working-group-specific style conventions, +# used in generation. + +import re + +from conventions import ConventionsBase + + +class OpenCLConventions(ConventionsBase): + def formatExtension(self, name): + """Mark up a name as an extension for the spec.""" + return '`<<{}>>`'.format(name) + + @property + def null(self): + """Preferred spelling of NULL.""" + return '`NULL`' + + @property + def constFlagBits(self): + """Returns True if static const flag bits should be generated, False if an enumerated type should be generated.""" + return False + + @property + def struct_macro(self): + return 'sname:' + + @property + def external_macro(self): + return 'code:' + + @property + def structtype_member_name(self): + """Return name of the structure type member""" + return 'sType' + + @property + def nextpointer_member_name(self): + """Return name of the structure pointer chain member""" + return 'pNext' + + @property + def valid_pointer_prefix(self): + """Return prefix to pointers which must themselves be valid""" + return 'valid' + + def is_structure_type_member(self, paramtype, paramname): + """Determine if member type and name match the structure type member.""" + return False + + def is_nextpointer_member(self, paramtype, paramname): + """Determine if member type and name match the next pointer chain member.""" + return paramtype == 'void' and paramname == self.nextpointer_member_name + + def generate_structure_type_from_name(self, structname): + """Generate a structure type name token from a structure name. + This should never be called for OpenCL, just other APIs.""" + return '' + + @property + def warning_comment(self): + """Return warning comment to be placed in header of generated Asciidoctor files""" + return '// WARNING: DO NOT MODIFY! This file is automatically generated from the cl.xml registry' + + @property + def file_suffix(self): + """Return suffix of generated Asciidoctor files""" + return '.txt' + + def api_name(self, spectype='api'): + """Return API or specification name for citations in ref pages.ref + pages should link to for + + spectype is the spec this refpage is for: 'api' is the OpenCL API + Specification, 'clang' is the OpenCL C Language specification. + Defaults to 'api'. If an unrecognized spectype is given, returns + None. + """ + if spectype == 'api' or spectype is None: + return 'OpenCL' + elif spectype == 'clang': + return 'OpenCL C' + else: + return None + + @property + def xml_supported_name_of_api(self): + """Return the supported= attribute used in API XML""" + return 'opencl' + + @property + def api_prefix(self): + """Return API token prefix""" + return 'CL_' + + @property + def api_version_prefix(self): + """Return API core version token prefix""" + return 'CL_VERSION_' + + @property + def KHR_prefix(self): + """Return extension name prefix for KHR extensions""" + return 'cl_khr_' + + @property + def EXT_prefix(self): + """Return extension name prefix for EXT extensions""" + return 'cl_ext_' + + @property + def write_contacts(self): + """Return whether contact list should be written to extension appendices""" + return True + + @property + def write_refpage_include(self): + """Return whether refpage include should be written to extension appendices""" + return False + + def writeFeature(self, featureExtraProtect, filename): + """Returns True if OutputGenerator.endFeature should write this feature. + Used in COutputGenerator + """ + return True + + def requires_error_validation(self, return_type): + """Returns True if the return_type element is an API result code + requiring error validation. + """ + return False + + @property + def required_errors(self): + """Return a list of required error codes for validation.""" + return [] + + def is_externsync_command(self, protoname): + """Returns True if the protoname element is an API command requiring + external synchronization + """ + return False + + def is_api_name(self, name): + """Returns True if name is in the reserved API namespace. + For OpenCL, these are names with a case-insensitive 'cl' prefix. + """ + return name[0:2].lower() == 'cl' + + def is_voidpointer_alias(self, tag, text, tail): + """Return True if the declaration components (tag,text,tail) of an + element represents a void * type + """ + return tag == 'type' and text == 'void' and tail.startswith('*') + + def make_voidpointer_alias(self, tail): + """Reformat a void * declaration to include the API alias macro. + Vulkan doesn't have an API alias macro, so do nothing. + """ + return tail + + def specURL(self, spectype = 'api'): + """Return public registry URL which ref pages should link to for + full Specification, so xrefs in the asciidoc source that aren't + to ref pages can link into it instead. + + spectype is the spec this refpage is for: 'api' is the OpenCL API + Specification, 'clang' is the OpenCL C Language specification. + Defaults to 'api'. If an unrecognized spectype is given, returns + None. + """ + if spectype == 'api' or spectype is None: + return 'https://www.khronos.org/registry/OpenCL/specs/3.0-unified/html/OpenCL_API.html' + elif spectype == 'clang': + return 'https://www.khronos.org/registry/OpenCL/specs/3.0-unified/html/OpenCL_C.html' + else: + return None + + @property + def xml_api_name(self): + """Return the name used in the default API XML registry for the default API""" + return 'opencl' + + @property + def registry_path(self): + """Return relpath to the default API XML registry in this project.""" + return 'xml/cl.xml' + + @property + def specification_path(self): + """Return relpath to the Asciidoctor specification sources in this project.""" + return '../appendices/meta' + + @property + def extra_refpage_headers(self): + """Return any extra text to add to refpage headers.""" + return 'include::{config}/attribs.txt[]\n' + \ + 'include::{config}/opencl.asciidoc[]\n' + \ + 'include::{apispec}/footnotes.asciidoc[]\n' + \ + 'include::{cspec}/footnotes.asciidoc[]\n' + \ + 'include::{cspec}/feature-dictionary.asciidoc[]\n' + \ + 'include::{generated}/api/api-dictionary-no-links.asciidoc[]' + + @property + def extension_index_prefixes(self): + """Return a list of extension prefixes used to group extension refpages.""" + return ['cl_khr', 'cl_ext', 'cl'] + + @property + def unified_flag_refpages(self): + """Return True if Flags/FlagBits refpages are unified, False if + they're separate. + """ + return False + + @property + def spec_reflow_path(self): + """Return the relative path to the spec source folder to reflow""" + return '.' + + @property + def spec_no_reflow_dirs(self): + """Return a set of directories not to automatically descend into + when reflowing spec text + """ + return ('scripts', 'style') + + @property + def should_skip_checking_codes(self): + """Return True if more than the basic validation of return codes should + be skipped for a command. + + OpenCL has a different style of error handling than OpenXR or + Vulkan, so these checks are not appropriate.""" + + return True diff --git a/xml/conventions.py b/xml/conventions.py new file mode 100644 index 000000000..6b6b23d14 --- /dev/null +++ b/xml/conventions.py @@ -0,0 +1,358 @@ +#!/usr/bin/python3 -i +# +# Copyright 2013-2023 The Khronos Group Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +# Base class for working-group-specific style conventions, +# used in generation. + +from enum import Enum + +# Type categories that respond "False" to isStructAlwaysValid +# basetype is home to typedefs like ..Bool32 +CATEGORIES_REQUIRING_VALIDATION = set(('handle', + 'enum', + 'bitmask', + 'basetype', + None)) + +# These are basic C types pulled in via openxr_platform_defines.h +TYPES_KNOWN_ALWAYS_VALID = set(('char', + 'float', + 'int8_t', 'uint8_t', + 'int32_t', 'uint32_t', + 'int64_t', 'uint64_t', + 'size_t', + 'uintptr_t', + 'int', + )) + + +class ProseListFormats(Enum): + """A connective, possibly with a quantifier.""" + AND = 0 + EACH_AND = 1 + OR = 2 + ANY_OR = 3 + + @classmethod + def from_string(cls, s): + if s == 'or': + return cls.OR + if s == 'and': + return cls.AND + return None + + @property + def connective(self): + if self in (ProseListFormats.OR, ProseListFormats.ANY_OR): + return 'or' + return 'and' + + def quantifier(self, n): + """Return the desired quantifier for a list of a given length.""" + if self == ProseListFormats.ANY_OR: + if n > 1: + return 'any of ' + elif self == ProseListFormats.EACH_AND: + if n > 2: + return 'each of ' + if n == 2: + return 'both of ' + return '' + + +class ConventionsBase: + """WG-specific conventions.""" + + def __init__(self): + self._command_prefix = None + self._type_prefix = None + + def formatExtension(self, name): + """Mark up an extension name as a link the spec.""" + return '`apiext:{}`'.format(name) + + @property + def null(self): + """Preferred spelling of NULL.""" + raise NotImplementedError + + def makeProseList(self, elements, fmt=ProseListFormats.AND, with_verb=False, *args, **kwargs): + """Make a (comma-separated) list for use in prose. + + Adds a connective (by default, 'and') + before the last element if there are more than 1. + + Adds the right one of "is" or "are" to the end if with_verb is true. + + Optionally adds a quantifier (like 'any') before a list of 2 or more, + if specified by fmt. + + Override with a different method or different call to + _implMakeProseList if you want to add a comma for two elements, + or not use a serial comma. + """ + return self._implMakeProseList(elements, fmt, with_verb, *args, **kwargs) + + @property + def struct_macro(self): + """Get the appropriate format macro for a structure. + + May override. + """ + return 'slink:' + + @property + def external_macro(self): + """Get the appropriate format macro for an external type like uint32_t. + + May override. + """ + return 'code:' + + def makeStructName(self, name): + """Prepend the appropriate format macro for a structure to a structure type name. + + Uses struct_macro, so just override that if you want to change behavior. + """ + return self.struct_macro + name + + def makeExternalTypeName(self, name): + """Prepend the appropriate format macro for an external type like uint32_t to a type name. + + Uses external_macro, so just override that if you want to change behavior. + """ + return self.external_macro + name + + def _implMakeProseList(self, elements, fmt, with_verb, comma_for_two_elts=False, serial_comma=True): + """Internal-use implementation to make a (comma-separated) list for use in prose. + + Adds a connective (by default, 'and') + before the last element if there are more than 1, + and only includes commas if there are more than 2 + (if comma_for_two_elts is False). + + Adds the right one of "is" or "are" to the end if with_verb is true. + + Optionally adds a quantifier (like 'any') before a list of 2 or more, + if specified by fmt. + + Don't edit these defaults, override self.makeProseList(). + """ + assert(serial_comma) # didn't implement what we didn't need + if isinstance(fmt, str): + fmt = ProseListFormats.from_string(fmt) + + my_elts = list(elements) + if len(my_elts) > 1: + my_elts[-1] = '{} {}'.format(fmt.connective, my_elts[-1]) + + if not comma_for_two_elts and len(my_elts) <= 2: + prose = ' '.join(my_elts) + else: + prose = ', '.join(my_elts) + + quantifier = fmt.quantifier(len(my_elts)) + + parts = [quantifier, prose] + + if with_verb: + if len(my_elts) > 1: + parts.append(' are') + else: + parts.append(' is') + return ''.join(parts) + + @property + def file_suffix(self): + """Return suffix of generated Asciidoctor files""" + raise NotImplementedError + + def api_name(self, spectype=None): + """Return API or specification name for citations in ref pages. + + spectype is the spec this refpage is for. + 'api' (the default value) is the main API Specification. + If an unrecognized spectype is given, returns None. + + Must implement.""" + raise NotImplementedError + + def should_insert_may_alias_macro(self, genOpts): + """Return true if we should insert a "may alias" macro in this file. + + Only used by OpenXR right now.""" + return False + + @property + def command_prefix(self): + """Return the expected prefix of commands/functions. + + Implemented in terms of api_prefix.""" + if not self._command_prefix: + self._command_prefix = self.api_prefix[:].replace('_', '').lower() + return self._command_prefix + + @property + def type_prefix(self): + """Return the expected prefix of type names. + + Implemented in terms of command_prefix (and in turn, api_prefix).""" + if not self._type_prefix: + self._type_prefix = ''.join( + (self.command_prefix[0:1].upper(), self.command_prefix[1:])) + return self._type_prefix + + @property + def api_prefix(self): + """Return API token prefix. + + Typically two uppercase letters followed by an underscore. + + Must implement.""" + raise NotImplementedError + + @property + def api_version_prefix(self): + """Return API core version token prefix. + + Implemented in terms of api_prefix. + + May override.""" + return self.api_prefix + 'VERSION_' + + @property + def KHR_prefix(self): + """Return extension name prefix for KHR extensions. + + Implemented in terms of api_prefix. + + May override.""" + return self.api_prefix + 'KHR_' + + @property + def EXT_prefix(self): + """Return extension name prefix for EXT extensions. + + Implemented in terms of api_prefix. + + May override.""" + return self.api_prefix + 'EXT_' + + def writeFeature(self, featureExtraProtect, filename): + """Return True if OutputGenerator.endFeature should write this feature. + + Defaults to always True. + Used in COutputGenerator. + + May override.""" + return True + + def requires_error_validation(self, return_type): + """Return True if the return_type element is an API result code + requiring error validation. + + Defaults to always False. + + May override.""" + return False + + @property + def required_errors(self): + """Return a list of required error codes for validation. + + Defaults to an empty list. + + May override.""" + return [] + + def is_voidpointer_alias(self, tag, text, tail): + """Return True if the declaration components (tag,text,tail) of an + element represents a void * type. + + Defaults to a reasonable implementation. + + May override.""" + return tag == 'type' and text == 'void' and tail.startswith('*') + + def make_voidpointer_alias(self, tail): + """Reformat a void * declaration to include the API alias macro. + + Defaults to a no-op. + + Must override if you actually want to use this feature in your project.""" + return tail + + def category_requires_validation(self, category): + """Return True if the given type 'category' always requires validation. + + Defaults to a reasonable implementation. + + May override.""" + return category in CATEGORIES_REQUIRING_VALIDATION + + def type_always_valid(self, typename): + """Return True if the given type name is always valid (never requires validation). + + This is for things like integers. + + Defaults to a reasonable implementation. + + May override.""" + return typename in TYPES_KNOWN_ALWAYS_VALID + + @property + def should_skip_checking_codes(self): + """Return True if more than the basic validation of return codes should + be skipped for a command.""" + + return False + + @property + def generate_index_terms(self): + """Return True if asiidoctor index terms should be generated as part + of an API interface from the docgenerator.""" + + return False + + @property + def generate_enum_table(self): + """Return True if asciidoctor tables describing enumerants in a + group should be generated as part of group generation.""" + return False + + @property + def generate_max_enum_in_docs(self): + """Return True if MAX_ENUM tokens should be generated in + documentation includes.""" + return False + + + def extension_include_string(self, ext): + """Return format string for include:: line for an extension appendix + file. ext is an object with the following members: + - name - extension string string + - vendor - vendor portion of name + - barename - remainder of name + + Must implement.""" + raise NotImplementedError + + @property + def refpage_generated_include_path(self): + """Return path relative to the generated reference pages, to the + generated API include files. + + Must implement.""" + raise NotImplementedError + + def valid_flag_bit(self, bitpos): + """Return True if bitpos is an allowed numeric bit position for + an API flag. + + Behavior depends on the data type used for flags (which may be 32 + or 64 bits), and may depend on assumptions about compiler + handling of sign bits in enumerated types, as well.""" + return True diff --git a/xml/docgenerator.py b/xml/docgenerator.py new file mode 100644 index 000000000..073552534 --- /dev/null +++ b/xml/docgenerator.py @@ -0,0 +1,454 @@ +#!/usr/bin/python3 -i +# +# Copyright 2013-2023 The Khronos Group Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +from generator import GeneratorOptions, OutputGenerator, noneStr, write + +ENUM_TABLE_PREFIX = """ +[cols=",",options="header",] +|======================================================================= +|Enum |Description""" + +ENUM_TABLE_SUFFIX = """|=======================================================================""" + +FLAG_BLOCK_PREFIX = """.Flag Descriptions +****""" + +FLAG_BLOCK_SUFFIX = """****""" + + +class DocGeneratorOptions(GeneratorOptions): + """DocGeneratorOptions - subclass of GeneratorOptions for + generating declaration snippets for the spec. + + Shares many members with CGeneratorOptions, since + both are writing C-style declarations.""" + + def __init__(self, + prefixText="", + apicall='', + apientry='', + apientryp='', + indentFuncProto=True, + indentFuncPointer=False, + alignFuncParam=0, + secondaryInclude=False, + expandEnumerants=True, + extEnumerantAdditions=False, + extEnumerantFormatString=" (Added by the {} extension)", + **kwargs): + """Constructor. + + Since this generator outputs multiple files at once, + the filename is just a "stamp" to indicate last generation time. + + Shares many parameters/members with CGeneratorOptions, since + both are writing C-style declarations: + + - prefixText - list of strings to prefix generated header with + (usually a copyright statement + calling convention macros). + - apicall - string to use for the function declaration prefix, + such as APICALL on Windows. + - apientry - string to use for the calling convention macro, + in typedefs, such as APIENTRY. + - apientryp - string to use for the calling convention macro + in function pointer typedefs, such as APIENTRYP. + - indentFuncProto - True if prototype declarations should put each + parameter on a separate line + - indentFuncPointer - True if typedefed function pointers should put each + parameter on a separate line + - alignFuncParam - if nonzero and parameters are being put on a + separate line, align parameter names at the specified column + + Additional parameters/members: + + - expandEnumerants - if True, add BEGIN/END_RANGE macros in enumerated + type declarations + - secondaryInclude - if True, add secondary (no xref anchor) versions + of generated files + - extEnumerantAdditions - if True, include enumerants added by extensions + in comment tables for core enumeration types. + - extEnumerantFormatString - A format string for any additional message for + enumerants from extensions if extEnumerantAdditions is True. The correctly- + marked-up extension name will be passed. + """ + GeneratorOptions.__init__(self, **kwargs) + self.prefixText = prefixText + """list of strings to prefix generated header with (usually a copyright statement + calling convention macros).""" + + self.apicall = apicall + """string to use for the function declaration prefix, such as APICALL on Windows.""" + + self.apientry = apientry + """string to use for the calling convention macro, in typedefs, such as APIENTRY.""" + + self.apientryp = apientryp + """string to use for the calling convention macro in function pointer typedefs, such as APIENTRYP.""" + + self.indentFuncProto = indentFuncProto + """True if prototype declarations should put each parameter on a separate line""" + + self.indentFuncPointer = indentFuncPointer + """True if typedefed function pointers should put each parameter on a separate line""" + + self.alignFuncParam = alignFuncParam + """if nonzero and parameters are being put on a separate line, align parameter names at the specified column""" + + self.secondaryInclude = secondaryInclude + """if True, add secondary (no xref anchor) versions of generated files""" + + self.expandEnumerants = expandEnumerants + """if True, add BEGIN/END_RANGE macros in enumerated type declarations""" + + self.extEnumerantAdditions = extEnumerantAdditions + """if True, include enumerants added by extensions in comment tables for core enumeration types.""" + + self.extEnumerantFormatString = extEnumerantFormatString + """A format string for any additional message for + enumerants from extensions if extEnumerantAdditions is True. The correctly- + marked-up extension name will be passed.""" + + +class DocOutputGenerator(OutputGenerator): + """DocOutputGenerator - subclass of OutputGenerator. + + Generates AsciiDoc includes with C-language API interfaces, for reference + pages and the corresponding specification. Similar to COutputGenerator, + but each interface is written into a different file as determined by the + options, only actual C types are emitted, and none of the boilerplate + preprocessor code is emitted.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Keep track of all extension numbers + self.extension_numbers = set() + + def beginFile(self, genOpts): + OutputGenerator.beginFile(self, genOpts) + + # This should be a separate conventions property rather than an + # inferred type name pattern for different APIs. + self.result_type = genOpts.conventions.type_prefix + "Result" + + def endFile(self): + OutputGenerator.endFile(self) + + def beginFeature(self, interface, emit): + # Start processing in superclass + OutputGenerator.beginFeature(self, interface, emit) + + # Decide if we're in a core or an + self.in_core = (interface.tag == 'feature') + + # Verify that each has a unique number during doc + # generation + # TODO move this to consistency_tools + if not self.in_core: + extension_number = interface.get('number') + if extension_number is not None and extension_number != "0": + if extension_number in self.extension_numbers: + self.logMsg('error', 'Duplicate extension number ', extension_number, ' detected in feature ', interface.get('name'), '\n') + exit(1) + else: + self.extension_numbers.add(extension_number) + + def endFeature(self): + # Finish processing in superclass + OutputGenerator.endFeature(self) + + def genRequirements(self, name, mustBeFound = True): + """Generate text showing what core versions and extensions introduce + an API. This relies on the map in api.py, which may be loaded at + runtime into self.apidict. If not present, no message is + generated. + + - name - name of the API + - mustBeFound - If True, when requirements for 'name' cannot be + determined, a warning comment is generated. + """ + + if self.apidict: + if name in self.apidict.requiredBy: + features = [] + for (base,dependency) in self.apidict.requiredBy[name]: + if dependency is not None: + features.append('{} with {}'.format(base, dependency)) + else: + features.append(base) + return '// Provided by {}\n'.format(', '.join(features)) + else: + if mustBeFound: + self.logMsg('warn', 'genRequirements: API {} not found'.format(name)) + return '' + else: + # No API dictionary available, return nothing + return '' + + def writeInclude(self, directory, basename, contents): + """Generate an include file. + + - directory - subdirectory to put file in + - basename - base name of the file + - contents - contents of the file (Asciidoc boilerplate aside)""" + # Create subdirectory, if needed + directory = self.genOpts.directory + '/' + directory + self.makeDir(directory) + + # Create file + filename = directory + '/' + basename + '.txt' + self.logMsg('diag', '# Generating include file:', filename) + fp = open(filename, 'w', encoding='utf-8') + + # Asciidoc anchor + write(self.genOpts.conventions.warning_comment, file=fp) + write('[[{0},{0}]]'.format(basename), file=fp) + + if self.genOpts.conventions.generate_index_terms: + index_terms = [] + if basename.startswith(self.conventions.command_prefix): + index_terms.append(basename[2:] + " (function)") + elif basename.startswith(self.conventions.type_prefix): + index_terms.append(basename[2:] + " (type)") + elif basename.startswith(self.conventions.api_prefix): + index_terms.append(basename[len(self.conventions.api_prefix):] + " (define)") + index_terms.append(basename) + write('indexterm:[{}]'.format(','.join(index_terms)), file=fp) + + write('[source,opencl]', file=fp) + write('----', file=fp) + write(contents, file=fp) + write('----', file=fp) + fp.close() + + if self.genOpts.secondaryInclude: + # Create secondary no cross-reference include file + filename = directory + '/' + basename + '.no-xref.txt' + self.logMsg('diag', '# Generating include file:', filename) + fp = open(filename, 'w', encoding='utf-8') + + # Asciidoc anchor + write(self.genOpts.conventions.warning_comment, file=fp) + write('// Include this no-xref version without cross reference id for multiple includes of same file', file=fp) + write('[source,opencl]', file=fp) + write('----', file=fp) + write(contents, file=fp) + write('----', file=fp) + fp.close() + + def writeTable(self, basename, values): + """Output a table of enumerants.""" + directory = Path(self.genOpts.directory) / 'enums' + self.makeDir(str(directory)) + + filename = str(directory / '{}.comments.txt'.format(basename)) + self.logMsg('diag', '# Generating include file:', filename) + + with open(filename, 'w', encoding='utf-8') as fp: + write(self.conventions.warning_comment, file=fp) + write(ENUM_TABLE_PREFIX, file=fp) + + for data in values: + write("|ename:{}".format(data['name']), file=fp) + write("|{}".format(data['comment']), file=fp) + + write(ENUM_TABLE_SUFFIX, file=fp) + + def writeFlagBox(self, basename, values): + """Output a box of flag bit comments.""" + directory = Path(self.genOpts.directory) / 'enums' + self.makeDir(str(directory)) + + filename = str(directory / '{}.comments.txt'.format(basename)) + self.logMsg('diag', '# Generating include file:', filename) + + with open(filename, 'w', encoding='utf-8') as fp: + write(self.conventions.warning_comment, file=fp) + write(FLAG_BLOCK_PREFIX, file=fp) + + for data in values: + write("* ename:{} -- {}".format(data['name'], + data['comment']), + file=fp) + + write(FLAG_BLOCK_SUFFIX, file=fp) + + def genType(self, typeinfo, name, alias): + """Generate type.""" + OutputGenerator.genType(self, typeinfo, name, alias) + typeElem = typeinfo.elem + # If the type is a struct type, traverse the embedded tags + # generating a structure. Otherwise, emit the tag text. + category = typeElem.get('category') + + if category in ('struct', 'union'): + # If the type is a struct type, generate it using the + # special-purpose generator. + self.genStruct(typeinfo, name, alias) + else: + body = self.genRequirements(name) + if alias: + # If the type is an alias, just emit a typedef declaration + body += 'typedef ' + alias + ' ' + name + ';\n' + self.writeInclude(OutputGenerator.categoryToPath[category], + name, body) + else: + # Replace tags with an APIENTRY-style string + # (from self.genOpts). Copy other text through unchanged. + # If the resulting text is an empty string, don't emit it. + body += noneStr(typeElem.text) + for elem in typeElem: + if elem.tag == 'apientry': + body += self.genOpts.apientry + noneStr(elem.tail) + else: + body += noneStr(elem.text) + noneStr(elem.tail) + + if body: + if category in OutputGenerator.categoryToPath: + self.writeInclude(OutputGenerator.categoryToPath[category], + name, body + '\n') + else: + self.logMsg('diag', '# NOT writing include file for type:', + name, '- bad category: ', category) + else: + self.logMsg('diag', '# NOT writing empty include file for type', name) + + def genStruct(self, typeinfo, typeName, alias): + """Generate struct.""" + OutputGenerator.genStruct(self, typeinfo, typeName, alias) + + typeElem = typeinfo.elem + + body = self.genRequirements(typeName) + if alias: + body += 'typedef ' + alias + ' ' + typeName + ';\n' + else: + body += 'typedef ' + typeElem.get('category') + ' ' + typeName + ' {\n' + + targetLen = self.getMaxCParamTypeLength(typeinfo) + for member in typeElem.findall('.//member'): + body += self.makeCParamDecl(member, targetLen + 4) + body += ';\n' + body += '} ' + typeName + ';' + + self.writeInclude('structs', typeName, body) + + def genEnumTable(self, groupinfo, groupName): + """Generate tables of enumerant values and short descriptions from + the XML.""" + + values = [] + got_comment = False + missing_comments = [] + for elem in groupinfo.elem.findall('enum'): + if not elem.get('required'): + continue + name = elem.get('name') + + data = { + 'name': name, + } + + (numVal, strVal) = self.enumToValue(elem, True) + data['value'] = numVal + + extname = elem.get('extname') + + added_by_extension_to_core = (extname is not None and self.in_core) + if added_by_extension_to_core and not self.genOpts.extEnumerantAdditions: + # We're skipping such values + continue + + comment = elem.get('comment') + if comment: + got_comment = True + elif name.endswith('_UNKNOWN') and numVal == 0: + # This is a placeholder for 0-initialization to be clearly invalid. + # Just skip this silently + continue + else: + # Skip but record this in case it's an odd-one-out missing a comment. + missing_comments.append(name) + continue + + if added_by_extension_to_core and self.genOpts.extEnumerantFormatString: + # Add a note to the comment + comment += self.genOpts.extEnumerantFormatString.format( + self.conventions.formatExtension(extname)) + + data['comment'] = comment + values.append(data) + + if got_comment: + # If any had a comment, output it. + + if missing_comments: + self.logMsg('warn', 'The following values for', groupName, + 'were omitted from the table due to missing comment attributes:', + ', '.join(missing_comments)) + + group_type = groupinfo.elem.get('type') + if groupName == self.result_type: + # Split this into success and failure + self.writeTable(groupName + '.success', + (data for data in values + if data['value'] >= 0)) + self.writeTable(groupName + '.error', + (data for data in values + if data['value'] < 0)) + elif group_type == 'bitmask': + self.writeFlagBox(groupName, values) + elif group_type == 'enum': + self.writeTable(groupName, values) + else: + raise RuntimeError("Unrecognized enums type: " + str(group_type)) + + def genGroup(self, groupinfo, groupName, alias): + """Generate group (e.g. C "enum" type).""" + OutputGenerator.genGroup(self, groupinfo, groupName, alias) + + body = self.genRequirements(groupName) + if alias: + # If the group name is aliased, just emit a typedef declaration + # for the alias. + body += 'typedef ' + alias + ' ' + groupName + ';\n' + else: + expand = self.genOpts.expandEnumerants + (_, enumbody) = self.buildEnumCDecl(expand, groupinfo, groupName) + body += enumbody + if self.genOpts.conventions.generate_enum_table: + self.genEnumTable(groupinfo, groupName) + + self.writeInclude('enums', groupName, body) + + def genEnum(self, enuminfo, name, alias): + """Generate the C declaration for a constant (a single value).""" + + OutputGenerator.genEnum(self, enuminfo, name, alias) + + body = self.buildConstantCDecl(enuminfo, name, alias) + + self.writeInclude('enums', name, body) + + def genCmd(self, cmdinfo, name, alias): + "Generate command." + OutputGenerator.genCmd(self, cmdinfo, name, alias) + + return_type = cmdinfo.elem.find('proto/type') + if self.genOpts.conventions.requires_error_validation(return_type): + # This command returns an API result code, so check that it + # returns at least the required errors. + # TODO move this to consistency_tools + required_errors = set(self.genOpts.conventions.required_errors) + errorcodes = cmdinfo.elem.get('errorcodes').split(',') + if not required_errors.issubset(set(errorcodes)): + self.logMsg('error', 'Missing required error code for command: ', name, '\n') + exit(1) + + body = self.genRequirements(name) + decls = self.makeCDecls(cmdinfo.elem) + body += decls[0] + self.writeInclude('protos', name, body) diff --git a/xml/extensionmetadocgenerator.py b/xml/extensionmetadocgenerator.py new file mode 100644 index 000000000..d6243889d --- /dev/null +++ b/xml/extensionmetadocgenerator.py @@ -0,0 +1,659 @@ +#!/usr/bin/python3 -i +# +# Copyright 2013-2023 The Khronos Group Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +import os +import re +import sys +from functools import total_ordering +from generator import GeneratorOptions, OutputGenerator, regSortFeatures, write + +class ExtensionMetaDocGeneratorOptions(GeneratorOptions): + """ExtensionMetaDocGeneratorOptions - subclass of GeneratorOptions. + + Represents options during extension metainformation generation for Asciidoc""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +EXT_NAME_DECOMPOSE_RE = re.compile(r'[A-Z]+_(?P[A-Z]+)_(?P[\w_]+)') + + +@total_ordering +class Extension: + def __init__(self, + generator, # needed for logging and API conventions + filename, + name, + number, + ext_type, + requires, + requiresCore, + contact, + promotedTo, + deprecatedBy, + obsoletedBy, + provisional, + revision, + specialuse ): + self.generator = generator + self.conventions = generator.genOpts.conventions + self.filename = filename + self.name = name + self.number = number + self.ext_type = ext_type + self.requires = requires + self.requiresCore = requiresCore + self.contact = contact + self.promotedTo = promotedTo + self.deprecatedBy = deprecatedBy + self.obsoletedBy = obsoletedBy + self.provisional = provisional + self.revision = revision + self.specialuse = specialuse + + self.deprecationType = None + self.supercedingAPIVersion = None + self.supercedingExtension = None + + if self.promotedTo is not None and self.deprecatedBy is not None and self.obsoletedBy is not None: + self.generator.logMsg('warn', 'All \'promotedto\', \'deprecatedby\' and \'obsoletedby\' attributes used on extension ' + self.name + '! Ignoring \'promotedto\' and \'deprecatedby\'.') + elif self.promotedTo is not None and self.deprecatedBy is not None: + self.generator.logMsg('warn', 'Both \'promotedto\' and \'deprecatedby\' attributes used on extension ' + self.name + '! Ignoring \'deprecatedby\'.') + elif self.promotedTo is not None and self.obsoletedBy is not None: + self.generator.logMsg('warn', 'Both \'promotedto\' and \'obsoletedby\' attributes used on extension ' + self.name + '! Ignoring \'promotedto\'.') + elif self.deprecatedBy is not None and self.obsoletedBy is not None: + self.generator.logMsg('warn', 'Both \'deprecatedby\' and \'obsoletedby\' attributes used on extension ' + self.name + '! Ignoring \'deprecatedby\'.') + + supercededBy = None + if self.promotedTo is not None: + self.deprecationType = 'promotion' + supercededBy = promotedTo + elif self.deprecatedBy is not None: + self.deprecationType = 'deprecation' + supercededBy = deprecatedBy + elif self.obsoletedBy is not None: + self.deprecationType = 'obsoletion' + supercededBy = obsoletedBy + + if supercededBy is not None: + if supercededBy == '' and not self.deprecationType == 'promotion': + pass # supercedingAPIVersion, supercedingExtension is None + elif supercededBy.startswith(self.conventions.api_version_prefix): + self.supercedingAPIVersion = supercededBy + elif supercededBy.startswith(self.conventions.api_prefix): + self.supercedingExtension = supercededBy + else: + self.generator.logMsg('error', 'Unrecognized ' + self.deprecationType + ' attribute value \'' + supercededBy + '\'!') + + match = EXT_NAME_DECOMPOSE_RE.match(self.name) + self.vendor = match.group('tag') + self.bare_name = match.group('name') + + def __str__(self): + return self.name + def __eq__(self, other): + return self.name == other.name + def __ne__(self, other): + return self.name != other.name + + def __lt__(self, other): + self_is_KHR = self.name.startswith(self.conventions.KHR_prefix) + self_is_EXT = self.name.startswith(self.conventions.EXT_prefix) + other_is_KHR = other.name.startswith(self.conventions.KHR_prefix) + other_is_EXT = other.name.startswith(self.conventions.EXT_prefix) + + swap = False + if self_is_KHR and not other_is_KHR: + return not swap + if other_is_KHR and not self_is_KHR: + return swap + if self_is_EXT and not other_is_EXT: + return not swap + if other_is_EXT and not self_is_EXT: + return swap + + return self.name < other.name + + def typeToStr(self): + if self.ext_type == 'instance': + return 'Instance extension' + if self.ext_type == 'device': + return 'Device extension' + + if self.ext_type is not None: + self.generator.logMsg('warn', 'The type attribute of ' + self.name + ' extension is neither \'instance\' nor \'device\'. That is invalid (at the time this script was written).') + else: # should be unreachable + self.generator.logMsg('error', 'Logic error in typeToStr(): Missing type attribute!') + return None + + def specLink(self, xrefName, xrefText, isRefpage = False): + """Generate a string containing a link to a specification anchor in + asciidoctor markup form. + + - xrefName - anchor name in the spec + - xrefText - text to show for the link, or None + - isRefpage = True if generating a refpage include, False if + generating a specification extension appendix include""" + + if isRefpage: + # Always link into API spec + specURL = self.conventions.specURL('api') + return 'link:{}#{}[{}^]'.format(specURL, xrefName, xrefText) + else: + return '<<' + xrefName + ', ' + xrefText + '>>' + + def conditionalLinkCoreAPI(self, apiVersion, linkSuffix, isRefpage): + versionMatch = re.match(self.conventions.api_version_prefix + r'(\d+)_(\d+)', apiVersion) + major = versionMatch.group(1) + minor = versionMatch.group(2) + + dottedVersion = major + '.' + minor + + xrefName = 'versions-' + dottedVersion + linkSuffix + xrefText = self.conventions.api_name() + ' ' + dottedVersion + + doc = 'ifdef::' + apiVersion + '[]\n' + doc += ' ' + self.specLink(xrefName, xrefText, isRefpage) + '\n' + doc += 'endif::' + apiVersion + '[]\n' + doc += 'ifndef::' + apiVersion + '[]\n' + doc += ' ' + self.conventions.api_name() + ' ' + dottedVersion + '\n' + doc += 'endif::' + apiVersion + '[]\n' + + return doc + + def conditionalLinkExt(self, extName, indent = ' '): + doc = 'ifdef::' + extName + '[]\n' + doc += indent + self.conventions.formatExtension(extName) + '\n' + doc += 'endif::' + extName + '[]\n' + doc += 'ifndef::' + extName + '[]\n' + doc += indent + '`' + extName + '`\n' + doc += 'endif::' + extName + '[]\n' + + return doc + + def resolveDeprecationChain(self, extensionsList, succeededBy, isRefpage, file): + ext = next(x for x in extensionsList if x.name == succeededBy) + + if ext.deprecationType: + if ext.deprecationType == 'promotion': + if ext.supercedingAPIVersion: + write(' ** Which in turn was _promoted_ to\n' + ext.conditionalLinkCoreAPI(ext.supercedingAPIVersion, '-promotions', isRefpage), file=file) + else: # ext.supercedingExtension + write(' ** Which in turn was _promoted_ to extension\n' + ext.conditionalLinkExt(ext.supercedingExtension), file=file) + ext.resolveDeprecationChain(extensionsList, ext.supercedingExtension, file) + elif ext.deprecationType == 'deprecation': + if ext.supercedingAPIVersion: + write(' ** Which in turn was _deprecated_ by\n' + ext.conditionalLinkCoreAPI(ext.supercedingAPIVersion, '-new-feature', isRefpage), file=file) + elif ext.supercedingExtension: + write(' ** Which in turn was _deprecated_ by\n' + ext.conditionalLinkExt(ext.supercedingExtension) + ' extension', file=file) + ext.resolveDeprecationChain(extensionsList, ext.supercedingExtension, file) + else: + write(' ** Which in turn was _deprecated_ without replacement', file=file) + elif ext.deprecationType == 'obsoletion': + if ext.supercedingAPIVersion: + write(' ** Which in turn was _obsoleted_ by\n' + ext.conditionalLinkCoreAPI(ext.supercedingAPIVersion, '-new-feature', isRefpage), file=file) + elif ext.supercedingExtension: + write(' ** Which in turn was _obsoleted_ by\n' + ext.conditionalLinkExt(ext.supercedingExtension) + ' extension', file=file) + ext.resolveDeprecationChain(extensionsList, ext.supercedingExtension, file) + else: + write(' ** Which in turn was _obsoleted_ without replacement', file=file) + else: # should be unreachable + self.generator.logMsg('error', 'Logic error in resolveDeprecationChain(): deprecationType is neither \'promotion\', \'deprecation\' nor \'obsoletion\'!') + + + def writeTag(self, tag, value, isRefpage, fp): + """Write a tag and (if non-None) a tag value to a file. + + - tag - string tag name + - value - tag value, or None + - isRefpage - controls style in which the tag is marked up + - fp - open file pointer to write to""" + + if isRefpage: + # Use subsection headers for the tag name + tagPrefix = '== ' + tagSuffix = '' + else: + # Use an bolded item list for the tag name + tagPrefix = '*' + tagSuffix = '*::' + + write(tagPrefix + tag + tagSuffix, file=fp) + if value is not None: + write(value, file=fp) + + if isRefpage: + write('', file=fp) + + def makeMetafile(self, extensionsList, isRefpage = False): + """Generate a file containing extension metainformation in + asciidoctor markup form. + + - extensionsList - list of extensions spec is being generated against + - isRefpage - True if generating a refpage include, False if + generating a specification extension appendix include""" + + if isRefpage: + filename = self.filename.replace('meta/', 'meta/refpage.') + else: + filename = self.filename + + fp = self.generator.newFile(filename) + + if not isRefpage: + write('[[' + self.name + ']]', file=fp) + write('=== ' + self.name, file=fp) + write('', file=fp) + + self.writeTag('Name String', '`' + self.name + '`', isRefpage, fp) + self.writeTag('Extension Type', self.typeToStr(), isRefpage, fp) + + self.writeTag('Registered Extension Number', self.number, isRefpage, fp) + self.writeTag('Revision', self.revision, isRefpage, fp) + + # Only API extension dependencies are coded in XML, others are explicit + self.writeTag('Extension and Version Dependencies', None, isRefpage, fp) + + write(' * Requires ' + self.conventions.api_name() + ' ' + self.requiresCore, file=fp) + if self.requires: + for dep in self.requires.split(','): + write(' * Requires', self.conventions.formatExtension(dep), + file=fp) + if self.provisional == 'true': + write(' * *This is a _provisional_ extension and must: be used with caution.', file=fp) + write(' See the ' + + self.specLink(xrefName = 'boilerplate-provisional-header', + xrefText = 'description', + isRefpage = isRefpage) + + ' of provisional header files for enablement and stability details.*', file=fp) + write('', file=fp) + + if self.deprecationType: + self.writeTag('Deprecation state', None, isRefpage, fp) + + if self.deprecationType == 'promotion': + if self.supercedingAPIVersion: + write(' * _Promoted_ to\n' + self.conditionalLinkCoreAPI(self.supercedingAPIVersion, '-promotions', isRefpage), file=fp) + else: # ext.supercedingExtension + write(' * _Promoted_ to\n' + self.conditionalLinkExt(self.supercedingExtension) + ' extension', file=fp) + self.resolveDeprecationChain(extensionsList, self.supercedingExtension, isRefpage, fp) + elif self.deprecationType == 'deprecation': + if self.supercedingAPIVersion: + write(' * _Deprecated_ by\n' + self.conditionalLinkCoreAPI(self.supercedingAPIVersion, '-new-features', isRefpage), file=fp) + elif self.supercedingExtension: + write(' * _Deprecated_ by\n' + self.conditionalLinkExt(self.supercedingExtension) + ' extension' , file=fp) + self.resolveDeprecationChain(extensionsList, self.supercedingExtension, isRefpage, fp) + else: + write(' * _Deprecated_ without replacement' , file=fp) + elif self.deprecationType == 'obsoletion': + if self.supercedingAPIVersion: + write(' * _Obsoleted_ by\n' + self.conditionalLinkCoreAPI(self.supercedingAPIVersion, '-new-features', isRefpage), file=fp) + elif self.supercedingExtension: + write(' * _Obsoleted_ by\n' + self.conditionalLinkExt(self.supercedingExtension) + ' extension' , file=fp) + self.resolveDeprecationChain(extensionsList, self.supercedingExtension, isRefpage, fp) + else: + # TODO: Does not make sense to retroactively ban use of extensions from 1.0. + # Needs some tweaks to the semantics and this message, when such extension(s) occur. + write(' * _Obsoleted_ without replacement' , file=fp) + else: # should be unreachable + self.generator.logMsg('error', 'Logic error in makeMetafile(): deprecationType is neither \'promotion\', \'deprecation\' nor \'obsoletion\'!') + write('', file=fp) + + if self.specialuse is not None: + specialuses = self.specialuse.split(',') + if len(specialuses) > 1: + header = 'Special Uses' + else: + header = 'Special Use' + self.writeTag(header, None, isRefpage, fp) + + for use in specialuses: + # Each specialuse attribute value expands an asciidoctor + # attribute of the same name, instead of using the shorter, + # and harder to understand attribute + write('* {}'.format( + self.specLink( + xrefName = self.conventions.special_use_section_anchor, + xrefText = '{' + use + '}', + isRefpage = isRefpage)), file=fp) + write('', file=fp) + + if self.conventions.write_contacts: + self.writeTag('Contact', None, isRefpage, fp) + + contacts = self.contact.split(',') + for contact in contacts: + contactWords = contact.strip().split() + name = ' '.join(contactWords[:-1]) + handle = contactWords[-1] + if handle.startswith('gitlab:'): + prettyHandle = 'icon:gitlab[alt=GitLab, role="red"]' + handle.replace('gitlab:@', '') + elif handle.startswith('@'): + issuePlaceholderText = '[' + self.name + '] ' + handle + issuePlaceholderText += '%0A<>' + trackerLink = 'link:++https://github.com/KhronosGroup/Vulkan-Docs/issues/new?body=' + issuePlaceholderText + '++' + prettyHandle = trackerLink + '[icon:github[alt=GitHub,role="black"]' + handle[1:] + ', window=_blank]' + else: + prettyHandle = handle + + write(' * ' + name + ' ' + prettyHandle, file=fp) + write('', file=fp) + + # Check if a proposal document for this extension exists in the + # current repository, and link to the same document (parameterized + # by a URL prefix attribute) if it does. + # The assumption is that a proposal document for an extension + # VK_name will be located in 'proposals/VK_name.asciidoc' relative + # to the repository root, and that this script will be invoked from + # the repository root. + path = 'proposals/{}.asciidoc'.format(self.name) + if os.path.exists(path) and os.access(path, os.R_OK): + self.writeTag('Extension Proposal', + 'link:{{specRepositoryURL}}/{}[{}]'.format(path, self.name), isRefpage, fp) + + fp.close() + +class ExtensionMetaDocOutputGenerator(OutputGenerator): + """ExtensionMetaDocOutputGenerator - subclass of OutputGenerator. + + Generates AsciiDoc includes with metainformation for the API extension + appendices. The fields used from tags in the API XML are: + + - name extension name string + - number extension number (optional) + - contact name and GitHub login or email address (optional) + - type 'instance' | 'device' (optional) + - requires list of comma-separated required API extensions (optional) + - requiresCore required core version of API (optional) + - promotedTo extension or API version it was promoted to + - deprecatedBy extension or API version which deprecated this extension, + or empty string if deprecated without replacement + - obsoletedBy extension or API version which obsoleted this extension, + or empty string if obsoleted without replacement + - provisional 'true' if this extension is released provisionally""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.extensions = [] + # List of strings containing all vendor tags + self.vendor_tags = [] + self.file_suffix = '' + + def newFile(self, filename): + self.logMsg('diag', '# Generating include file:', filename) + fp = open(filename, 'w', encoding='utf-8') + write(self.genOpts.conventions.warning_comment, file=fp) + return fp + + def beginFile(self, genOpts): + OutputGenerator.beginFile(self, genOpts) + + self.directory = self.genOpts.directory + self.file_suffix = self.genOpts.conventions.file_suffix + + # Iterate over all 'tag' Elements and add the names of all the valid vendor + # tags to the list + root = self.registry.tree.getroot() + for tag in root.findall('tags/tag'): + self.vendor_tags.append(tag.get('name')) + + # Create subdirectory, if needed + self.makeDir(self.directory) + + def conditionalExt(self, extName, content, ifdef = None, condition = None): + doc = '' + + innerdoc = 'ifdef::' + extName + '[]\n' + innerdoc += content + '\n' + innerdoc += 'endif::' + extName + '[]\n' + + if ifdef: + if ifdef == 'ifndef': + if condition: + doc += 'ifndef::' + condition + '[]\n' + doc += innerdoc + doc += 'endif::' + condition + '[]\n' + else: # no condition is as if condition is defined; "nothing" is always defined :p + pass # so no output + elif ifdef == 'ifdef': + if condition: + doc += 'ifdef::' + condition + '+' + extName + '[]\n' + doc += content + '\n' # does not include innerdoc; the ifdef was merged with the one above + doc += 'endif::' + condition + '+' + extName + '[]\n' + else: # no condition is as if condition is defined; "nothing" is always defined :p + doc += innerdoc + else: # should be unreachable + raise RuntimeError('Should be unreachable: ifdef is neither \'ifdef \' nor \'ifndef\'!') + else: + doc += innerdoc + + return doc + + def makeExtensionInclude(self, ext): + return self.conventions.extension_include_string(ext) + + def endFile(self): + self.extensions.sort() + + # Generate metadoc extension files, in refpage and non-refpage form + for ext in self.extensions: + ext.makeMetafile(self.extensions, isRefpage = False) + if self.conventions.write_refpage_include: + ext.makeMetafile(self.extensions, isRefpage = True) + + # Generate list of promoted extensions + promotedExtensions = {} + for ext in self.extensions: + if ext.deprecationType == 'promotion' and ext.supercedingAPIVersion: + promotedExtensions.setdefault(ext.supercedingAPIVersion, []).append(ext) + + for coreVersion, extensions in promotedExtensions.items(): + promoted_extensions_fp = self.newFile(self.directory + '/promoted_extensions_' + coreVersion + self.file_suffix) + + for ext in extensions: + indent = '' + write(' * {blank}\n+\n' + ext.conditionalLinkExt(ext.name, indent), file=promoted_extensions_fp) + + promoted_extensions_fp.close() + + # Re-sort to match earlier behavior + # TODO: Remove this extra sort when re-arranging section order OK. + + def makeSortKey(ext): + name = ext.name.lower() + prefixes = self.conventions.extension_index_prefixes + for i, prefix in enumerate(prefixes): + if ext.name.startswith(prefix): + return (i, name) + return (len(prefixes), name) + + self.extensions.sort(key=makeSortKey) + + # Generate include directives for the extensions appendix, grouping + # extensions by status (current, deprecated, provisional, etc.) + with self.newFile(self.directory + '/current_extensions_appendix' + self.file_suffix) as current_extensions_appendix_fp, \ + self.newFile(self.directory + '/deprecated_extensions_appendix' + self.file_suffix) as deprecated_extensions_appendix_fp, \ + self.newFile(self.directory + '/current_extension_appendices' + self.file_suffix) as current_extension_appendices_fp, \ + self.newFile(self.directory + '/current_extension_appendices_toc' + self.file_suffix) as current_extension_appendices_toc_fp, \ + self.newFile(self.directory + '/deprecated_extension_appendices' + self.file_suffix) as deprecated_extension_appendices_fp, \ + self.newFile(self.directory + '/deprecated_extension_appendices_toc' + self.file_suffix) as deprecated_extension_appendices_toc_fp, \ + self.newFile(self.directory + '/deprecated_extensions_guard_macro' + self.file_suffix) as deprecated_extensions_guard_macro_fp, \ + self.newFile(self.directory + '/provisional_extensions_appendix' + self.file_suffix) as provisional_extensions_appendix_fp, \ + self.newFile(self.directory + '/provisional_extension_appendices' + self.file_suffix) as provisional_extension_appendices_fp, \ + self.newFile(self.directory + '/provisional_extension_appendices_toc' + self.file_suffix) as provisional_extension_appendices_toc_fp, \ + self.newFile(self.directory + '/provisional_extensions_guard_macro' + self.file_suffix) as provisional_extensions_guard_macro_fp: + + write('', file=current_extensions_appendix_fp) + write('include::deprecated_extensions_guard_macro' + self.file_suffix + '[]', file=current_extensions_appendix_fp) + write('', file=current_extensions_appendix_fp) + write('ifndef::HAS_DEPRECATED_EXTENSIONS[]', file=current_extensions_appendix_fp) + write('[[extension-appendices-list]]', file=current_extensions_appendix_fp) + write('== List of Extensions', file=current_extensions_appendix_fp) + write('endif::HAS_DEPRECATED_EXTENSIONS[]', file=current_extensions_appendix_fp) + write('ifdef::HAS_DEPRECATED_EXTENSIONS[]', file=current_extensions_appendix_fp) + write('[[extension-appendices-list]]', file=current_extensions_appendix_fp) + write('== List of Current Extensions', file=current_extensions_appendix_fp) + write('endif::HAS_DEPRECATED_EXTENSIONS[]', file=current_extensions_appendix_fp) + write('', file=current_extensions_appendix_fp) + write('include::current_extension_appendices_toc' + self.file_suffix + '[]', file=current_extensions_appendix_fp) + write('\n<<<\n', file=current_extensions_appendix_fp) + write('include::current_extension_appendices' + self.file_suffix + '[]', file=current_extensions_appendix_fp) + + write('', file=deprecated_extensions_appendix_fp) + write('include::deprecated_extensions_guard_macro' + self.file_suffix + '[]', file=deprecated_extensions_appendix_fp) + write('', file=deprecated_extensions_appendix_fp) + write('ifdef::HAS_DEPRECATED_EXTENSIONS[]', file=deprecated_extensions_appendix_fp) + write('[[deprecated-extension-appendices-list]]', file=deprecated_extensions_appendix_fp) + write('== List of Deprecated Extensions', file=deprecated_extensions_appendix_fp) + write('include::deprecated_extension_appendices_toc' + self.file_suffix + '[]', file=deprecated_extensions_appendix_fp) + write('\n<<<\n', file=deprecated_extensions_appendix_fp) + write('include::deprecated_extension_appendices' + self.file_suffix + '[]', file=deprecated_extensions_appendix_fp) + write('endif::HAS_DEPRECATED_EXTENSIONS[]', file=deprecated_extensions_appendix_fp) + + # add include guards to allow multiple includes + write('ifndef::DEPRECATED_EXTENSIONS_GUARD_MACRO_INCLUDE_GUARD[]', file=deprecated_extensions_guard_macro_fp) + write(':DEPRECATED_EXTENSIONS_GUARD_MACRO_INCLUDE_GUARD:\n', file=deprecated_extensions_guard_macro_fp) + write('ifndef::PROVISIONAL_EXTENSIONS_GUARD_MACRO_INCLUDE_GUARD[]', file=provisional_extensions_guard_macro_fp) + write(':PROVISIONAL_EXTENSIONS_GUARD_MACRO_INCLUDE_GUARD:\n', file=provisional_extensions_guard_macro_fp) + + write('', file=provisional_extensions_appendix_fp) + write('include::provisional_extensions_guard_macro' + self.file_suffix + '[]', file=provisional_extensions_appendix_fp) + write('', file=provisional_extensions_appendix_fp) + write('ifdef::HAS_PROVISIONAL_EXTENSIONS[]', file=provisional_extensions_appendix_fp) + write('[[provisional-extension-appendices-list]]', file=provisional_extensions_appendix_fp) + write('== List of Provisional Extensions', file=provisional_extensions_appendix_fp) + write('include::provisional_extension_appendices_toc' + self.file_suffix + '[]', file=provisional_extensions_appendix_fp) + write('\n<<<\n', file=provisional_extensions_appendix_fp) + write('include::provisional_extension_appendices' + self.file_suffix + '[]', file=provisional_extensions_appendix_fp) + write('endif::HAS_PROVISIONAL_EXTENSIONS[]', file=provisional_extensions_appendix_fp) + + for ext in self.extensions: + include = self.makeExtensionInclude(ext) + link = ' * ' + self.conventions.formatExtension(ext.name) + if ext.provisional == 'true': + write(self.conditionalExt(ext.name, include), file=provisional_extension_appendices_fp) + write(self.conditionalExt(ext.name, link), file=provisional_extension_appendices_toc_fp) + write(self.conditionalExt(ext.name, ':HAS_PROVISIONAL_EXTENSIONS:'), file=provisional_extensions_guard_macro_fp) + elif ext.deprecationType is None: + write(self.conditionalExt(ext.name, include), file=current_extension_appendices_fp) + write(self.conditionalExt(ext.name, link), file=current_extension_appendices_toc_fp) + else: + condition = ext.supercedingAPIVersion if ext.supercedingAPIVersion else ext.supercedingExtension # potentially None too + + write(self.conditionalExt(ext.name, include, 'ifndef', condition), file=current_extension_appendices_fp) + write(self.conditionalExt(ext.name, link, 'ifndef', condition), file=current_extension_appendices_toc_fp) + + write(self.conditionalExt(ext.name, include, 'ifdef', condition), file=deprecated_extension_appendices_fp) + write(self.conditionalExt(ext.name, link, 'ifdef', condition), file=deprecated_extension_appendices_toc_fp) + + write(self.conditionalExt(ext.name, ':HAS_DEPRECATED_EXTENSIONS:', 'ifdef', condition), file=deprecated_extensions_guard_macro_fp) + + write('endif::DEPRECATED_EXTENSIONS_GUARD_MACRO_INCLUDE_GUARD[]', file=deprecated_extensions_guard_macro_fp) + write('endif::PROVISIONAL_EXTENSIONS_GUARD_MACRO_INCLUDE_GUARD[]', file=provisional_extensions_guard_macro_fp) + + OutputGenerator.endFile(self) + + def beginFeature(self, interface, emit): + # Start processing in superclass + OutputGenerator.beginFeature(self, interface, emit) + + if interface.tag != 'extension': + self.logMsg('diag', 'beginFeature: ignoring non-extension feature', self.featureName) + return + + # These attributes must exist + name = self.featureName + number = self.getAttrib(interface, 'number') + ext_type = self.getAttrib(interface, 'type') + revision = self.getSpecVersion(interface, name) + + # These attributes are optional + OPTIONAL = False + requires = self.getAttrib(interface, 'requires', OPTIONAL) + requiresCore = self.getAttrib(interface, 'requiresCore', OPTIONAL, '1.0') # TODO update this line with update_version.py + contact = self.getAttrib(interface, 'contact', OPTIONAL) + promotedTo = self.getAttrib(interface, 'promotedto', OPTIONAL) + deprecatedBy = self.getAttrib(interface, 'deprecatedby', OPTIONAL) + obsoletedBy = self.getAttrib(interface, 'obsoletedby', OPTIONAL) + provisional = self.getAttrib(interface, 'provisional', OPTIONAL, 'false') + specialuse = self.getAttrib(interface, 'specialuse', OPTIONAL) + + filename = self.directory + '/' + name + self.file_suffix + + extdata = Extension( + generator = self, + filename = filename, + name = name, + number = number, + ext_type = ext_type, + requires = requires, + requiresCore = requiresCore, + contact = contact, + promotedTo = promotedTo, + deprecatedBy = deprecatedBy, + obsoletedBy = obsoletedBy, + provisional = provisional, + revision = revision, + specialuse = specialuse) + self.extensions.append(extdata) + + + def endFeature(self): + # Finish processing in superclass + OutputGenerator.endFeature(self) + + def getAttrib(self, elem, attribute, required=True, default=None): + """Query an attribute from an element, or return a default value + + - elem - element to query + - attribute - attribute name + - required - whether attribute must exist + - default - default value if attribute not present""" + attrib = elem.get(attribute, default) + if required and (attrib is None): + name = elem.get('name', 'UNKNOWN') + self.logMsg('error', 'While processing \'' + self.featureName + ', <' + elem.tag + '> \'' + name + '\' does not contain required attribute \'' + attribute + '\'') + return attrib + + def numbersToWords(self, name): + allowlist = ['WIN32', 'INT16', 'D3D1'] + + # temporarily replace allowlist items + for i, w in enumerate(allowlist): + name = re.sub(w, '{' + str(i) + '}', name) + + name = re.sub(r'(?<=[A-Z])(\d+)(?![A-Z])', r'_\g<1>', name) + + # undo allowlist substitution + for i, w in enumerate(allowlist): + name = re.sub('\\{' + str(i) + '}', w, name) + + return name + + def getSpecVersion(self, elem, extname, default=None): + """Determine the extension revision from the EXTENSION_NAME_SPEC_VERSION + enumerant. + + - elem - element to query + - extname - extension name from the 'name' attribute + - default - default value if SPEC_VERSION token not present""" + # The literal enumerant name to match + versioningEnumName = self.numbersToWords(extname.upper()) + '_SPEC_VERSION' + + for enum in elem.findall('./require/enum'): + enumName = self.getAttrib(enum, 'name') + if enumName == versioningEnumName: + return self.getAttrib(enum, 'value') + + #if not found: + for enum in elem.findall('./require/enum'): + enumName = self.getAttrib(enum, 'name') + if enumName.find('SPEC_VERSION') != -1: + self.logMsg('diag', 'Missing ' + versioningEnumName + '! Potential misnamed candidate ' + enumName + '.') + return self.getAttrib(enum, 'value') + + self.logMsg('error', 'Missing ' + versioningEnumName + '!') + return default diff --git a/xml/genRef.py b/xml/genRef.py new file mode 100644 index 000000000..87d8d7a60 --- /dev/null +++ b/xml/genRef.py @@ -0,0 +1,1019 @@ +#!/usr/bin/python3 +# +# Copyright 2016-2023 The Khronos Group Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +# genRef.py - create API ref pages from spec source files +# +# Usage: genRef.py files + +import argparse +import io +import os +import re +import sys +from collections import OrderedDict +from reflib import (findRefs, fixupRefs, loadFile, logDiag, logWarn, + printPageInfo, setLogFile) +from reg import Registry +from clconventions import OpenCLConventions as APIConventions + + +def makeExtensionInclude(name): + """Return an include command, given an extension name.""" + return 'include::{}/refpage.{}{}[]'.format( + conventions.specification_path, + name, + conventions.file_suffix) + + +def makeAPIInclude(type, name): + """Return an include command for a generated API interface + - type - type of the API, e.g. 'flags', 'handles', etc + - name - name of the API""" + + return 'include::{}/api/{}/{}{}\n'.format( + conventions.refpage_generated_include_path, + type, name, conventions.file_suffix) + + +def isextension(name): + """Return True if name is an API extension name (ends with an upper-case + author ID). + + This assumes that author IDs are at least two characters.""" + return name[-2:].isalpha() and name[-2:].isupper() + + +def printCopyrightSourceComments(fp): + """Print Khronos CC-BY copyright notice on open file fp. + + Writes an asciidoc comment block, which copyrights the source + file.""" + print('// Copyright 2014-2023 The Khronos Group, Inc.', file=fp) + print('//', file=fp) + # This works around constraints of the 'reuse' tool + print('// SPDX' + '-License-Identifier: CC-BY-4.0', file=fp) + print('', file=fp) + + +def printFooter(fp): + """Print footer material at the end of each refpage on open file fp. + + If generating separate refpages, adds the copyright. + If generating the single combined refpage, just add a separator.""" + + print('ifdef::doctype-manpage[]', + '== Copyright', + '', + 'include::{config}/copyright-ccby.txt[]', + 'endif::doctype-manpage[]', + '', + 'ifndef::doctype-manpage[]', + '<<<', + 'endif::doctype-manpage[]', + '', + sep='\n', file=fp) + + +def macroPrefix(name): + """Add a spec asciidoc macro prefix to an API name, depending on its type + (protos, structs, enums, etc.). + + If the name is not recognized, use the generic link macro 'reflink:'.""" + if name in api.basetypes: + return 'basetype:' + name + if name in api.defines: + return 'dlink:' + name + if name in api.enums: + return 'elink:' + name + if name in api.flags: + return 'elink:' + name + if name in api.funcpointers: + return 'tlink:' + name + if name in api.handles: + return 'slink:' + name + if name in api.protos: + return 'flink:' + name + if name in api.structs: + return 'slink:' + name + if name == 'TBD': + return 'No cross-references are available' + return 'reflink:' + name + + +def seeAlsoList(apiName, explicitRefs=None, apiAliases=[]): + """Return an asciidoc string with a list of 'See Also' references for the + API entity 'apiName', based on the relationship mapping in the api module. + + 'explicitRefs' is a list of additional cross-references. + + If apiAliases is not None, it is a list of aliases of apiName whose + cross-references will also be included. + + If no relationships are available, return None.""" + + refs = set(()) + + # apiName and its aliases are treated equally + allApis = apiAliases.copy() + allApis.append(apiName) + + # Add all the implicit references to refs + for name in allApis: + if name in api.mapDict: + refs.update(api.mapDict[name]) + + # Add all the explicit references + if explicitRefs is not None: + if isinstance(explicitRefs, str): + explicitRefs = explicitRefs.split() + refs.update(name for name in explicitRefs) + + # Add extensions / core versions based on dependencies + for name in allApis: + if name in api.requiredBy: + for (base,dependency) in api.requiredBy[name]: + refs.add(base) + if dependency is not None: + refs.add(dependency) + + if len(refs) == 0: + return None + else: + return ', '.join(macroPrefix(name) for name in sorted(refs)) + '\n' + + +def remapIncludes(lines, baseDir, specDir): + """Remap include directives in a list of lines so they can be extracted to a + different directory. + + Returns remapped lines. + + - lines - text to remap + - baseDir - target directory + - specDir - source directory""" + # This should be compiled only once + includePat = re.compile(r'^include::(?P.*)\[\]') + + newLines = [] + for line in lines: + matches = includePat.search(line) + if matches is not None: + path = matches.group('path') + + if path[0] != '{': + # Relative path to include file from here + incPath = specDir + '/' + path + # Remap to be relative to baseDir + newPath = os.path.relpath(incPath, baseDir) + newLine = 'include::' + newPath + '[]\n' + logDiag('remapIncludes: remapping', line, '->', newLine) + newLines.append(newLine) + else: + # An asciidoctor variable starts the path. + # This must be an absolute path, not needing to be rewritten. + newLines.append(line) + else: + newLines.append(line) + return newLines + + +def refPageShell(pageName, pageDesc, fp, head_content = None, sections=None, tail_content=None, man_section=3): + """Generate body of a reference page. + + - pageName - string name of the page + - pageDesc - string short description of the page + - fp - file to write to + - head_content - text to include before the sections + - sections - iterable returning (title,body) for each section. + - tail_content - text to include after the sections + - man_section - Unix man page section""" + + printCopyrightSourceComments(fp) + + print(':data-uri:', + ':icons: font', + conventions.extra_refpage_headers, + '', + sep='\n', file=fp) + + s = '{}({})'.format(pageName, man_section) + print('= ' + s, + '', + sep='\n', file=fp) + if pageDesc.strip() == '': + pageDesc = 'NO SHORT DESCRIPTION PROVIDED' + logWarn('refPageHead: no short description provided for', pageName) + + print('== Name', + '{} - {}'.format(pageName, pageDesc), + '', + sep='\n', file=fp) + + if head_content is not None: + print(head_content, + '', + sep='\n', file=fp) + + if sections is not None: + for title, content in sections.items(): + print('== {}'.format(title), + '', + content, + '', + sep='\n', file=fp) + + if tail_content is not None: + print(tail_content, + '', + sep='\n', file=fp) + + +def refPageHead(pageName, pageDesc, specText, fieldName, fieldText, descText, fp): + """Generate header of a reference page. + + - pageName - string name of the page + - pageDesc - string short description of the page + - specType - string containing 'spec' field from refpage open block, or None. + Used to determine containing spec name and URL. + - specText - string that goes in the "C Specification" section + - fieldName - string heading an additional section following specText, if not None + - fieldText - string that goes in the additional section + - descText - string that goes in the "Description" section + - fp - file to write to""" + sections = OrderedDict() + + if specText is not None: + sections['C Specification'] = specText + + if fieldName is not None: + sections[fieldName] = fieldText + + if descText is None or descText.strip() == '': + logWarn('refPageHead: no description provided for', pageName) + + if descText is not None: + sections['Description'] = descText + + refPageShell(pageName, pageDesc, fp, head_content=None, sections=sections) + + +def refPageTail(pageName, + specType=None, + specAnchor=None, + seeAlso=None, + fp=None, + auto=False): + """Generate end boilerplate of a reference page. + + - pageName - name of the page + - specType - None or the 'spec' attribute from the refpage block, + identifying the specification name and URL this refpage links to. + - specAnchor - None or the 'anchor' attribute from the refpage block, + identifying the anchor in the specification this refpage links to. If + None, the pageName is assumed to be a valid anchor.""" + + specName = conventions.api_name(specType) + specURL = conventions.specURL(specType) + if specAnchor is None: + specAnchor = pageName + + if seeAlso is None: + seeAlso = 'No cross-references are available\n' + + notes = [ + 'For more information, see the {}#{}[{} Specification^]'.format( + specURL, specAnchor, specName), + '', + ] + + if auto: + notes.extend(( + 'This page is a generated document.', + 'Fixes and changes should be made to the generator scripts, ' + 'not directly.', + )) + else: + notes.extend(( + 'This page is extracted from the ' + specName + ' Specification. ', + 'Fixes and changes should be made to the Specification, ' + 'not directly.', + )) + + print('== See Also', + '', + seeAlso, + '', + sep='\n', file=fp) + + print('== Document Notes', + '', + '\n'.join(notes), + '', + sep='\n', file=fp) + + printFooter(fp) + + +def xrefRewriteInitialize(): + """Initialize substitution patterns for asciidoctor xrefs.""" + + global refLinkPattern, refLinkSubstitute + global refLinkTextPattern, refLinkTextSubstitute + global specLinkPattern, specLinkSubstitute + + # These are xrefs to Vulkan API entities, rewritten to link to refpages + # The refLink variants are for xrefs with only an anchor and no text. + # The refLinkText variants are for xrefs with both anchor and text + refLinkPattern = re.compile(r'<<([Vv][Kk][^>,]+)>>') + refLinkSubstitute = r'link:\1.html[\1^]' + + refLinkTextPattern = re.compile(r'<<([Vv][Kk][^>,]+)[,]?[ \t\n]*([^>,]*)>>') + refLinkTextSubstitute = r'link:\1.html[\2^]' + + # These are xrefs to other anchors, rewritten to link to the spec + specLinkPattern = re.compile(r'<<([^>,]+)[,]?[ \t\n]*([^>,]*)>>') + + # Unfortunately, specLinkSubstitute depends on the link target, + # so can't be constructed in advance. + specLinkSubstitute = None + + +def xrefRewrite(text, specURL): + """Rewrite asciidoctor xrefs in text to resolve properly in refpages. + Xrefs which are to Vulkan refpages are rewritten to link to those + refpages. The remainder are rewritten to generate external links into + the supplied specification document URL. + + - text - string to rewrite, or None + - specURL - URL to target + + Returns rewritten text, or None, respectively""" + + global refLinkPattern, refLinkSubstitute + global refLinkTextPattern, refLinkTextSubstitute + global specLinkPattern, specLinkSubstitute + + specLinkSubstitute = r'link:{}#\1[\2^]'.format(specURL) + + if text is not None: + text, _ = refLinkPattern.subn(refLinkSubstitute, text) + text, _ = refLinkTextPattern.subn(refLinkTextSubstitute, text) + text, _ = specLinkPattern.subn(specLinkSubstitute, text) + + return text + +def emitPage(baseDir, specDir, pi, file): + """Extract a single reference page into baseDir. + + - baseDir - base directory to emit page into + - specDir - directory extracted page source came from + - pi - pageInfo for this page relative to file + - file - list of strings making up the file, indexed by pi""" + pageName = baseDir + '/' + pi.name + '.txt' + + # Add a dictionary entry for this page + global genDict + genDict[pi.name] = None + logDiag('emitPage:', pageName) + + # Short description + if pi.desc is None: + pi.desc = '(no short description available)' + + # Member/parameter section label and text, if there is one + field = None + fieldText = None + + if pi.type != 'freeform' and pi.type != 'spirv': + if pi.include is None: + # Not sure how this happens yet + logWarn('emitPage:', pageName, 'INCLUDE is None, no page generated') + return + + # Specification text from beginning to just before the parameter + # section. This covers the description, the prototype, the version + # note, and any additional version note text. If a parameter section + # is absent then go a line beyond the include. + remap_end = pi.include + 1 if pi.param is None else pi.param + lines = remapIncludes(file[pi.begin:remap_end], baseDir, specDir) + specText = ''.join(lines) + + if pi.param is not None: + if pi.type == 'structs': + field = 'Members' + elif pi.type in ['protos', 'funcpointers']: + field = 'Parameters' + else: + logWarn('emitPage: unknown field type:', pi.type, + 'for', pi.name) + lines = remapIncludes(file[pi.param:pi.body], baseDir, specDir) + fieldText = ''.join(lines) + + # Description text + if pi.body != pi.include: + lines = remapIncludes(file[pi.body:pi.end + 1], baseDir, specDir) + descText = ''.join(lines) + else: + descText = None + logWarn('emitPage: INCLUDE == BODY, so description will be empty for', pi.name) + if pi.begin != pi.include: + logWarn('emitPage: Note: BEGIN != INCLUDE, so the description might be incorrectly located before the API include!') + else: + specText = None + descText = ''.join(file[pi.begin:pi.end + 1]) + + # Rewrite asciidoctor xrefs to resolve properly in refpages + specURL = conventions.specURL(pi.spec) + + specText = xrefRewrite(specText, specURL) + fieldText = xrefRewrite(fieldText, specURL) + descText = xrefRewrite(descText, specURL) + + fp = open(pageName, 'w', encoding='utf-8') + refPageHead(pi.name, + pi.desc, + specText, + field, fieldText, + descText, + fp) + refPageTail(pageName=pi.name, + specType=pi.spec, + specAnchor=pi.anchor, + seeAlso=seeAlsoList(pi.name, pi.refs, pi.alias.split()), + fp=fp, + auto=False) + fp.close() + + +def autoGenEnumsPage(baseDir, pi, file): + """Autogenerate a single reference page in baseDir. + + Script only knows how to do this for /enums/ pages, at present. + + - baseDir - base directory to emit page into + - pi - pageInfo for this page relative to file + - file - list of strings making up the file, indexed by pi""" + pageName = baseDir + '/' + pi.name + '.txt' + fp = open(pageName, 'w', encoding='utf-8') + + # Add a dictionary entry for this page + global genDict + genDict[pi.name] = None + logDiag('autoGenEnumsPage:', pageName) + + # Short description + if pi.desc is None: + pi.desc = '(no short description available)' + + # Description text. Allow for the case where an enum definition + # is not embedded. + if not pi.embed: + embedRef = '' + else: + embedRef = ''.join(( + ' * The reference page for ', + macroPrefix(pi.embed), + ', where this interface is defined.\n')) + + txt = ''.join(( + 'For more information, see:\n\n', + embedRef, + ' * The See Also section for other reference pages using this type.\n', + ' * The ' + apiName + ' Specification.\n')) + + refPageHead(pi.name, + pi.desc, + ''.join(file[pi.begin:pi.include + 1]), + None, None, + txt, + fp) + refPageTail(pageName=pi.name, + specType=pi.spec, + specAnchor=pi.anchor, + seeAlso=seeAlsoList(pi.name, pi.refs, pi.alias.split()), + fp=fp, + auto=True) + fp.close() + + +# Pattern to break apart an API *Flags{authorID} name, used in +# autoGenFlagsPage. +flagNamePat = re.compile(r'(?P\w+)Flags(?P[A-Z]*)') + + +def autoGenFlagsPage(baseDir, flagName): + """Autogenerate a single reference page in baseDir for an API *Flags type. + + - baseDir - base directory to emit page into + - flagName - API *Flags name""" + pageName = baseDir + '/' + flagName + '.txt' + fp = open(pageName, 'w', encoding='utf-8') + + # Add a dictionary entry for this page + global genDict + genDict[flagName] = None + logDiag('autoGenFlagsPage:', pageName) + + # Short description + matches = flagNamePat.search(flagName) + if matches is not None: + name = matches.group('name') + author = matches.group('author') + logDiag('autoGenFlagsPage: split name into', name, 'Flags', author) + flagBits = name + 'FlagBits' + author + desc = 'Bitmask of ' + flagBits + else: + logWarn('autoGenFlagsPage:', pageName, 'does not end in "Flags{author ID}". Cannot infer FlagBits type.') + flagBits = None + desc = 'Unknown ' + apiName + ' flags type' + + # Description text + if flagBits is not None: + txt = ''.join(( + 'etext:' + flagName, + ' is a mask of zero or more elink:' + flagBits + '.\n', + 'It is used as a member and/or parameter of the structures and commands\n', + 'in the See Also section below.\n')) + else: + txt = ''.join(( + 'etext:' + flagName, + ' is an unknown ' + apiName + ' type, assumed to be a bitmask.\n')) + + refPageHead(flagName, + desc, + makeAPIInclude('flags', flagName), + None, None, + txt, + fp) + refPageTail(pageName=flagName, + specType=pi.spec, + specAnchor=pi.anchor, + seeAlso=seeAlsoList(flagName, None), + fp=fp, + auto=True) + fp.close() + + +def autoGenHandlePage(baseDir, handleName): + """Autogenerate a single handle page in baseDir for an API handle type. + + - baseDir - base directory to emit page into + - handleName - API handle name""" + # @@ Need to determine creation function & add handles/ include for the + # @@ interface in generator.py. + pageName = baseDir + '/' + handleName + '.txt' + fp = open(pageName, 'w', encoding='utf-8') + + # Add a dictionary entry for this page + global genDict + genDict[handleName] = None + logDiag('autoGenHandlePage:', pageName) + + # Short description + desc = apiName + ' object handle' + + descText = ''.join(( + 'sname:' + handleName, + ' is an object handle type, referring to an object used\n', + 'by the ' + apiName + ' implementation. These handles are created or allocated\n', + 'by the @@ TBD @@ function, and used by other ' + apiName + ' structures\n', + 'and commands in the See Also section below.\n')) + + refPageHead(handleName, + desc, + makeAPIInclude('handles', handleName), + None, None, + descText, + fp) + refPageTail(pageName=handleName, + specType=pi.spec, + specAnchor=pi.anchor, + seeAlso=seeAlsoList(handleName, None), + fp=fp, + auto=True) + fp.close() + + +def genRef(specFile, baseDir): + """Extract reference pages from a spec asciidoc source file. + + - specFile - filename to extract from + - baseDir - output directory to generate page in""" + file = loadFile(specFile) + if file is None: + return + + # Save the path to this file for later use in rewriting relative includes + specDir = os.path.dirname(os.path.abspath(specFile)) + + pageMap = findRefs(file, specFile) + logDiag(specFile + ': found', len(pageMap.keys()), 'potential pages') + + sys.stderr.flush() + + # Fix up references in pageMap + fixupRefs(pageMap, specFile, file) + + # Create each page, if possible + pages = {} + + for name in sorted(pageMap): + pi = pageMap[name] + + printPageInfo(pi, file) + + if pi.Warning: + logDiag('genRef:', pi.name + ':', pi.Warning) + + if pi.extractPage: + emitPage(baseDir, specDir, pi, file) + elif pi.type == 'enums': + autoGenEnumsPage(baseDir, pi, file) + elif pi.type == 'flags': + autoGenFlagsPage(baseDir, pi.name) + else: + # Don't extract this page + logWarn('genRef: Cannot extract or autogenerate:', pi.name) + + pages[pi.name] = pi + for alias in pi.alias.split(): + pages[alias] = pi + + return pages + + +def genSinglePageRef(baseDir): + """Generate baseDir/apispec.txt, the single-page version of the ref pages. + + This assumes there's a page for everything in the api module dictionaries. + Extensions (KHR, EXT, etc.) are currently skipped""" + # Accumulate head of page + head = io.StringIO() + + printCopyrightSourceComments(head) + + print('= ' + apiName + ' API Reference Pages', + ':data-uri:', + ':icons: font', + ':doctype: book', + ':numbered!:', + ':max-width: 200', + ':data-uri:', + ':toc2:', + ':toclevels: 2', + '', + sep='\n', file=head) + + print('== Copyright', file=head) + print('', file=head) + print('include::{config}/copyright-ccby.txt[]', file=head) + print('', file=head) + # Inject the table of contents. Asciidoc really ought to be generating + # this for us. + + sections = [ + [api.protos, 'protos', apiName + ' Commands'], + [api.handles, 'handles', 'Object Handles'], + [api.structs, 'structs', 'Structures'], + [api.enums, 'enums', 'Enumerations'], + [api.flags, 'flags', 'Flags'], + [api.funcpointers, 'funcpointers', 'Function Pointer Types'], + [api.basetypes, 'basetypes', apiName + ' Scalar types'], + [api.defines, 'defines', 'C Macro Definitions'], + [extensions, 'extensions', apiName + ' Extensions'] + ] + + # Accumulate body of page + body = io.StringIO() + + for (apiDict, label, title) in sections: + # Add section title/anchor header to body + anchor = '[[' + label + ',' + title + ']]' + print(anchor, + '== ' + title, + '', + ':leveloffset: 2', + '', + sep='\n', file=body) + + if label == 'extensions': + # preserve order of extensions since we already sorted the way we want. + keys = apiDict.keys() + else: + keys = sorted(apiDict.keys()) + + for refPage in keys: + # Don't generate links for aliases, which are included with the + # aliased page + if refPage not in api.alias: + # Add page to body + if 'FlagBits' in refPage and conventions.unified_flag_refpages: + # OpenXR does not create separate ref pages for FlagBits: + # the FlagBits includes go in the Flags refpage. + # Previously the Vulkan script would only emit non-empty + # Vk*Flags pages, via the logic + # if refPage not in api.flags or api.flags[refPage] is not None + # emit page + # Now, all are emitted. + continue + else: + print('include::' + refPage + '.txt[]', file=body) + else: + # Alternatively, we could (probably should) link to the + # aliased refpage + logWarn('(Benign) Not including', refPage, + 'in single-page reference', + 'because it is an alias of', api.alias[refPage]) + + print('\n' + ':leveloffset: 0' + '\n', file=body) + + # Write head and body to the output file + pageName = baseDir + '/apispec.txt' + fp = open(pageName, 'w', encoding='utf-8') + + print(head.getvalue(), file=fp, end='') + print(body.getvalue(), file=fp, end='') + + head.close() + body.close() + fp.close() + + +def genExtension(baseDir, extpath, name, info): + """Generate refpage, and add dictionary entry for an extension + + - baseDir - output directory to generate page in + - extpath - None, or path to per-extension specification sources if + those are to be included in extension refpages + - name - extension name + - info - Element from XML""" + + # Add a dictionary entry for this page + global genDict + genDict[name] = None + declares = [] + elem = info.elem + + # Type of extension (instance, device, etc.) + ext_type = elem.get('type') + + # Autogenerate interfaces from entry + for required in elem.find('require'): + req_name = required.get('name') + if not req_name: + # This isn't what we're looking for + continue + if req_name.endswith('_SPEC_VERSION') or req_name.endswith('_EXTENSION_NAME'): + # Don't link to spec version or extension name - those ref pages aren't created. + continue + + if required.get('extends'): + # These are either extensions of enumerated types, or const enum + # values: neither of which get a ref page - although we could + # include the enumerated types in the See Also list. + continue + + if req_name not in genDict: + logWarn('ERROR: {} (in extension {}) does not have a ref page.'.format(req_name, name)) + + declares.append(req_name) + + # import pdb + # pdb.set_trace() + + appbody = None + if extpath is not None: + appfp = open('{}/{}.txt'.format(extpath, name), 'r', encoding='utf-8') + if appfp is not None: + appbody = appfp.read() + + # Transform internal links to crosslinks + specURL = conventions.specURL() + appbody = xrefRewrite(appbody, specURL) + else: + logWarn('Cannot find extension appendix for', name) + + # Fall through to autogenerated page + extpath = None + appbody = None + appfp.close() + + # Include the extension appendix without an extra title + # head_content = 'include::{{appendices}}/{}.txt[]'.format(name) + + # Write the extension refpage + pageName = baseDir + '/' + name + '.txt' + logDiag('genExtension:', pageName) + fp = open(pageName, 'w', encoding='utf-8') + + # There are no generated titled sections + sections = None + + # 'See link:{html_spec_relative}#%s[ %s] in the main specification for complete information.' % ( + # name, name) + refPageShell(name, + "{} extension".format(ext_type), + fp, + appbody, + sections=sections) + refPageTail(pageName=name, + specType=None, + specAnchor=name, + seeAlso=seeAlsoList(name, declares), + fp=fp, + auto=True) + fp.close() + + +if __name__ == '__main__': + global genDict, extensions, conventions, apiName + genDict = {} + extensions = OrderedDict() + conventions = APIConventions() + apiName = conventions.api_name('api') + + parser = argparse.ArgumentParser() + + parser.add_argument('-diag', action='store', dest='diagFile', + help='Set the diagnostic file') + parser.add_argument('-warn', action='store', dest='warnFile', + help='Set the warning file') + parser.add_argument('-log', action='store', dest='logFile', + help='Set the log file for both diagnostics and warnings') + parser.add_argument('-genpath', action='store', + default='gen', + help='Path to directory containing generated files') + parser.add_argument('-basedir', action='store', dest='baseDir', + default=None, + help='Set the base directory in which pages are generated') + parser.add_argument('-noauto', action='store_true', + help='Don\'t generate inferred ref pages automatically') + parser.add_argument('files', metavar='filename', nargs='*', + help='a filename to extract ref pages from') + parser.add_argument('--version', action='version', version='%(prog)s 1.0') + parser.add_argument('-extension', action='append', + default=[], + help='Specify an extension or extensions to add to targets') + parser.add_argument('-rewrite', action='store', + default=None, + help='Name of output file to write Apache mod_rewrite directives to') + parser.add_argument('-toc', action='store', + default=None, + help='Name of output file to write an alphabetical TOC to') + parser.add_argument('-registry', action='store', + default=conventions.registry_path, + help='Use specified registry file instead of default') + parser.add_argument('-extpath', action='store', + default=None, + help='Use extension descriptions from this directory instead of autogenerating extension refpages') + + results = parser.parse_args() + + # Look for api.py in the specified directory + if results.genpath is not None: + sys.path.insert(0, results.genpath) + import api + + setLogFile(True, True, results.logFile) + setLogFile(True, False, results.diagFile) + setLogFile(False, True, results.warnFile) + + # Initialize static rewrite patterns for spec xrefs + xrefRewriteInitialize() + + if results.baseDir is None: + baseDir = results.genpath + '/ref' + else: + baseDir = results.baseDir + + # Dictionary of pages & aliases + pages = {} + + for file in results.files: + d = genRef(file, baseDir) + pages.update(d) + + # Now figure out which pages *weren't* generated from the spec. + # This relies on the dictionaries of API constructs in the api module. + + if not results.noauto: + registry = Registry() + registry.loadFile(results.registry) + + if conventions.write_refpage_include: + # Only extensions with a supported="..." attribute in this set + # will be considered for extraction/generation. + supported_strings = set((conventions.xml_api_name,)) + ext_names = set(k for k, v in registry.extdict.items() + if v.supported in supported_strings) + + desired_extensions = ext_names.intersection(set(results.extension)) + for prefix in conventions.extension_index_prefixes: + # Splits up into chunks, sorted within each chunk. + filtered_extensions = sorted( + [name for name in desired_extensions + if name.startswith(prefix) and name not in extensions]) + for name in filtered_extensions: + # logWarn('NOT autogenerating extension refpage for', name) + extensions[name] = None + genExtension(baseDir, results.extpath, name, registry.extdict[name]) + + # autoGenFlagsPage is no longer needed because they are added to + # the spec sources now. + # for page in api.flags: + # if page not in genDict: + # autoGenFlagsPage(baseDir, page) + + # autoGenHandlePage is no longer needed because they are added to + # the spec sources now. + # for page in api.structs: + # if typeCategory[page] == 'handle': + # autoGenHandlePage(baseDir, page) + + sections = [ + (api.flags, 'Flag Types'), + (api.enums, 'Enumerated Types'), + (api.structs, 'Structures'), + (api.protos, 'Prototypes'), + (api.funcpointers, 'Function Pointers'), + (api.basetypes, apiName + ' Scalar Types'), + (extensions, apiName + ' Extensions'), + ] + + # Summarize pages that weren't generated, for good or bad reasons + + for (apiDict, title) in sections: + # OpenXR was keeping a 'flagged' state which only printed out a + # warning for the first non-generated page, but was otherwise + # unused. This doesn't seem helpful. + for page in apiDict: + if page not in genDict: + # Page was not generated - why not? + if page in api.alias: + logWarn('(Benign, is an alias) Ref page for', title, page, 'is aliased into', api.alias[page]) + elif page in api.flags and api.flags[page] is None: + logWarn('(Benign, no FlagBits defined) No ref page generated for ', title, + page) + else: + # Could introduce additional logic to detect + # external types and not emit them. + logWarn('No ref page generated for ', title, page) + + genSinglePageRef(baseDir) + + if results.rewrite: + # Generate Apache rewrite directives for refpage aliases + fp = open(results.rewrite, 'w', encoding='utf-8') + + for page in sorted(pages): + p = pages[page] + rewrite = p.name + + if page != rewrite: + print('RewriteRule ^', page, '.html$ ', rewrite, '.html', + sep='', file=fp) + fp.close() + + if results.toc: + # Generate dynamic portion of refpage TOC + fp = open(results.toc, 'w', encoding='utf-8') + + # Run through dictionary of pages generating an TOC + print(12 * ' ', '
  • Alphabetic Contents', sep='', file=fp) + print(16 * ' ', '
      ', sep='', file=fp) + lastLetter = None + + for page in sorted(pages, key=str.upper): + p = pages[page] + letter = page[0:1].upper() + + if letter != lastLetter: + if lastLetter: + # End previous block + print(24 * ' ', '
    ', sep='', file=fp) + print(20 * ' ', '
    • ', sep='', file=fp) + # Start new block + print(20 * ' ', '
  • ', letter, sep='', file=fp) + print(24 * ' ', '
      ', sep='', file=fp) + lastLetter = letter + + # Add this page to the list + print(28 * ' ', '
    • ', page, '
      • ', + sep='', file=fp) + + if lastLetter: + # Close the final letter block + print(24 * ' ', '
    ', sep='', file=fp) + print(20 * ' ', '
    • ', sep='', file=fp) + + # Close the list + print(16 * ' ', '', sep='', file=fp) + print(12 * ' ', '', sep='', file=fp) + + # print('name {} -> page {}'.format(page, pages[page].name)) + + fp.close() diff --git a/xml/gen_dictionaries.py b/xml/gen_dictionaries.py new file mode 100644 index 000000000..069800dbb --- /dev/null +++ b/xml/gen_dictionaries.py @@ -0,0 +1,258 @@ +#!/usr/bin/python3 + +# Copyright 2019-2023 The Khronos Group Inc. +# SPDX-License-Identifier: Apache-2.0 + +from collections import OrderedDict + +import argparse +import sys +import urllib +import xml.etree.ElementTree as etree +import urllib.request + +def parse_xml(path): + file = urllib.request.urlopen(path) if path.startswith("http") else open(path, 'r') + with file: + tree = etree.parse(file) + return tree + +# File Header: +def GetHeader(): + return """// Copyright 2017-2023 The Khronos Group. This work is licensed under a +// Creative Commons Attribution 4.0 International License; see +// http://creativecommons.org/licenses/by/4.0/ + +""" + +# File Footer: +def GetFooter(): + return """ +""" + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument('-registry', action='store', + default='cl.xml', + help='Use specified registry file instead of cl.xml') + parser.add_argument('-o', action='store', dest='directory', + default='.', + help='Create target and related files in specified directory') + + args = parser.parse_args() + + linkFileName = args.directory + '/api-dictionary.asciidoc' + nolinkFileName = args.directory + '/api-dictionary-no-links.asciidoc' + typeFileName = args.directory + '/api-types.txt' + + specpath = args.registry + #specpath = "https://raw.githubusercontent.com/KhronosGroup/OpenCL-Registry/main/xml/cl.xml" + + print('Generating dictionaries from: ' + specpath) + + spec = parse_xml(specpath) + + linkFile = open(linkFileName, 'w') + nolinkFile = open(nolinkFileName, 'w') + linkFile.write( GetHeader() ) + nolinkFile.write( GetHeader() ) + typeFile = open(typeFileName, 'w') + + # Generate the API functions dictionaries: + + numberOfFuncs = 0 + + # Add core API functions with and without links: + for feature in spec.findall('feature/require'): + for api in feature.findall('command'): + name = api.get('name') + #print('found api: ' + name) + + # Example with link: + # + # // clEnqueueNDRangeKernel + # :clEnqueueNDRangeKernel_label: pass:q[*clEnqueueNDRangeKernel*] + # :clEnqueueNDRangeKernel: <> + linkFile.write('// ' + name + '\n') + linkFile.write(':' + name + '_label: pass:q[*' + name + '*]\n') + linkFile.write(':' + name + ': <<' + name + ',{' + name + '_label}>>\n') + linkFile.write('\n') + + # Example without link: + # + # // clEnqueueNDRangeKernel + # :clEnqueueNDRangeKernel: pass:q[*clEnqueueNDRangeKernel*] + nolinkFile.write('// ' + name + '\n') + nolinkFile.write(':' + name + ': pass:q[*' + name + '*]\n') + nolinkFile.write('\n') + + numberOfFuncs = numberOfFuncs + 1 + + # Add extension API functions without links: + for extension in spec.findall('extensions/extension/require'): + for api in extension.findall('command'): + name = api.get('name') + #print('found extension api: ' +name) + + # Example without link: + # + # // clGetGLObjectInfo + # :clGetGLObjectInfo: pass:q[*clGetGLObjectInfo*] + linkFile.write('// ' + name + '\n') + linkFile.write(':' + name + ': pass:q[*' + name + '*]\n') + linkFile.write('\n') + + nolinkFile.write('// ' + name + '\n') + nolinkFile.write(':' + name + ': pass:q[*' + name + '*]\n') + nolinkFile.write('\n') + + numberOfFuncs = numberOfFuncs + 1 + + print('Found ' + str(numberOfFuncs) + ' API functions.') + + # Generate the API enums dictionaries: + + numberOfEnums = 0 + + for enums in spec.findall('enums'): + name = enums.get('name') + for enum in enums.findall('enum'): + name = enum.get('name') + #print('found enum: ' + name) + + # Create a variant of the name that precedes underscores with + # "zero width" spaces. This causes some long names to be + # broken at more intuitive places. + htmlName = name[:3] + name[3:].replace("_", "_") + otherName = name[:3] + name[3:].replace("_", "_​") + + # Example with link: + # + # // CL_MEM_READ_ONLY + #:CL_MEM_READ_ONLY_label: pass:q[`CL_MEM_READ_ONLY`] + #:CL_MEM_READ_ONLY: <> + #:CL_MEM_READ_ONLY_anchor: [[CL_MEM_READ_ONLY]]{CL_MEM_READ_ONLY} + linkFile.write('// ' + name + '\n') + linkFile.write('ifdef::backend-html5[]\n') + linkFile.write(':' + name + '_label: pass:q[`' + htmlName + '`]\n') + linkFile.write('endif::[]\n') + linkFile.write('ifndef::backend-html5[]\n') + linkFile.write(':' + name + '_label: pass:q[`' + otherName + '`]\n') + linkFile.write('endif::[]\n') + linkFile.write(':' + name + ': <<' + name + ',{' + name + '_label}>>\n') + linkFile.write(':' + name + '_anchor: [[' + name + ']]{' + name + '}\n') + linkFile.write('\n') + + # Example without link: + # + # // CL_MEM_READ_ONLY + #:CL_MEM_READ_ONLY: pass:q[`CL_MEM_READ_ONLY`] + #:CL_MEM_READ_ONLY_anchor: {CL_MEM_READ_ONLY} + nolinkFile.write('// ' + name + '\n') + nolinkFile.write('ifdef::backend-html5[]\n') + nolinkFile.write(':' + name + ': pass:q[`' + htmlName + '`]\n') + nolinkFile.write('endif::[]\n') + nolinkFile.write('ifndef::backend-html5[]\n') + nolinkFile.write(':' + name + ': pass:q[`' + otherName + '`]\n') + nolinkFile.write('endif::[]\n') + nolinkFile.write(':' + name + '_anchor: {' + name + '}\n') + nolinkFile.write('\n') + + numberOfEnums = numberOfEnums + 1 + + print('Found ' + str(numberOfEnums) + ' API enumerations.') + + # Generate the API types dictionaries: + + numberOfTypes = 0 + + for types in spec.findall('types'): + for type in types.findall('type'): + addLink = False + name = "" + category = type.get('category') + if category == 'basetype': + name = type.get('name') + elif category == 'struct': + addLink = True + name = type.get('name') + elif category == 'define': + name = type.find('name').text + else: + continue + + #print('found type: ' +name) + + # Create a variant of the name that precedes underscores with + # "zero width" spaces. This causes some long names to be + # broken at more intuitive places. + if name.endswith('_t'): + htmlName = name + otherName = name + else: + htmlName = name[:3] + name[3:].replace("_", "_") + otherName = name[:3] + name[3:].replace("_", "_​") + + # Some types can have spaces in the name (such as unsigned char), + # but Asciidoctor attributes cannot. So, replace spaces with + # underscores for the attribute name. + attribName = name.replace(" ", "_") + + # Append the type suffix for disambiguation, since asciidoctor + # attributes are not case-sensitive (currently). + attribName = attribName + "_TYPE" + + # Example with link: + # + # // cl_image_desc + # :cl_image_desc_TYPE_label: pass:q[`cl_image_desc`] + # :cl_image_desc_TYPE: <> + linkFile.write('// ' + name + '\n') + if addLink: + linkFile.write('ifdef::backend-html5[]\n') + linkFile.write(':' + attribName + '_label: pass:q[`' + htmlName + '`]\n') + linkFile.write('endif::[]\n') + linkFile.write('ifndef::backend-html5[]\n') + linkFile.write(':' + attribName + '_label: pass:q[`' + otherName + '`]\n') + linkFile.write('endif::[]\n') + linkFile.write(':' + attribName + ': <<' + name + ',{' + attribName + '_label}>>\n') + else: + linkFile.write('ifdef::backend-html5[]\n') + linkFile.write(':' + attribName + ': pass:q[`' + htmlName + '`]\n') + linkFile.write('endif::[]\n') + linkFile.write('ifndef::backend-html5[]\n') + linkFile.write(':' + attribName + ': pass:q[`' + otherName + '`]\n') + linkFile.write('endif::[]\n') + linkFile.write('\n') + + # // cl_image_desc + # :cl_image_desc_TYPE: pass:q[`cl_image_desc`] + nolinkFile.write('// ' + name + '\n') + nolinkFile.write('ifdef::backend-html5[]\n') + nolinkFile.write(':' + attribName + ': pass:q[`' + htmlName + '`]\n') + nolinkFile.write('endif::[]\n') + nolinkFile.write('ifndef::backend-html5[]\n') + nolinkFile.write(':' + attribName + ': pass:q[`' + otherName + '`]\n') + nolinkFile.write('endif::[]\n') + nolinkFile.write('\n') + + # Print the type list to a file for custom syntax highlighting. + # For this we only care about CL types, not base types. + if category != 'basetype': + typeFile.write(' ' + name + '\n') + + numberOfTypes = numberOfTypes + 1 + + print('Found ' + str(numberOfTypes) + ' API types.') + + linkFile.write( GetFooter() ) + linkFile.close() + nolinkFile.write( GetFooter() ) + nolinkFile.close() + typeFile.close() + + print('Successfully generated file: ' + linkFileName) + print('Successfully generated file: ' + nolinkFileName) + print('Successfully generated file: ' + typeFileName) + diff --git a/xml/gen_version_notes.py b/xml/gen_version_notes.py new file mode 100644 index 000000000..9fed05178 --- /dev/null +++ b/xml/gen_version_notes.py @@ -0,0 +1,127 @@ +#!/usr/bin/python3 + +# Copyright 2019-2023 The Khronos Group Inc. +# SPDX-License-Identifier: Apache-2.0 + +from collections import OrderedDict + +import argparse +import sys +import os +import urllib +import xml.etree.ElementTree as etree +import urllib.request + + +def parse_xml(path): + file = urllib.request.urlopen(path) if path.startswith("http") else open( + path, 'r') + with file: + tree = etree.parse(file) + return tree + + +# File Header: +def GetHeader(): + return """// Copyright 2017-2023 The Khronos Group. This work is licensed under a +// Creative Commons Attribution 4.0 International License; see +// http://creativecommons.org/licenses/by/4.0/ +""" + + +# File Footer: +def GetFooter(): + return """ +""" + +def FullNote(name, added_in, deprecated_by): + # Four patterns: (1) always present in OpenCL, (2) added after 1.0, (3) in + # 1.0 but now deprecated, and (4) added after 1.0 but now deprecated. + if added_in == "1.0" and deprecated_by == None: + return "\n// Intentionally empty, %s has always been present." % name + if added_in != "1.0" and deprecated_by == None: + return "\nIMPORTANT: {%s} is <> version %s." % (name, added_in) + if added_in == "1.0" and deprecated_by != None: + return "\nIMPORTANT: {%s} is <> version %s." % (name, deprecated_by) + if added_in != "1.0" and deprecated_by != None: + return "\nIMPORTANT: {%s} is <> version %s and <> version %s." % (name, added_in, deprecated_by) + +def ShortNote(name, added_in, deprecated_by): + # Four patterns: (1) always present in OpenCL, (2) added after 1.0, (3) in + # 1.0 but now deprecated, and (4) added after 1.0 but now deprecated. + if added_in == "1.0" and deprecated_by == None: + return "// Intentionally empty, %s has always been present." % name + if added_in != "1.0" and deprecated_by == None: + return "<> version %s." % added_in + if added_in == "1.0" and deprecated_by != None: + return "<> version %s." % deprecated_by + if added_in != "1.0" and deprecated_by != None: + return "<> version %s and <> version %s." % (added_in, deprecated_by) + +# Find feature groups that are parents of a feature/require/${entry_type} +# hierarchy, and then find all the ${entry_type} within each hierarchy: +def process_xml(spec, entry_type, note_printer): + numberOfEntries = 0 + numberOfNewEntries = 0 + numberOfDeprecatedEntries = 0 + + for feature in spec.findall('.//feature/require/%s/../..' % entry_type): + for entry in feature.findall('.//%s' % entry_type): + name = entry.get('name') + + numberOfEntries += 1 + added_in = feature.get('number') + deprecated_by = None + + # All the groups that this specific API ${entry_type} belongs. + categories = spec.findall( + './/require[@comment]/%s[@name="%s"]/..' % (entry_type, name)) + for category in categories: + comment = category.get('comment') + if "deprecated in OpenCL" in comment: + words = comment.split(" ") + assert " ".join(words[-4:-1]) == "deprecated in OpenCL" + assert deprecated_by == None # Can't deprecate something twice. + deprecated_by = words[-1] + + versionFileName = os.path.join(args.directory, name + ".asciidoc") + with open(versionFileName, 'w') as versionFile: + versionFile.write(GetHeader()) + versionFile.write(note_printer(name, added_in, deprecated_by)) + versionFile.write(GetFooter()) + + numberOfNewEntries += 0 if added_in == "1.0" else 1 + numberOfDeprecatedEntries += 0 if deprecated_by == None else 1 + + print('Found ' + str(numberOfEntries) + ' API ' + entry_type + 's, ' + + str(numberOfNewEntries) + " newer than 1.0, " + + str(numberOfDeprecatedEntries) + " are deprecated.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + '-registry', + action='store', + default='cl.xml', + help='Use specified registry file instead of cl.xml') + parser.add_argument( + '-o', + action='store', + dest='directory', + default='.', + help='Create target and related files in specified directory') + + args = parser.parse_args() + + specpath = args.registry + + print('Generating version notes from: ' + specpath) + + spec = parse_xml(specpath) + + # Generate the API functions dictionaries: + + process_xml(spec, "command", FullNote) + process_xml(spec, "enum", ShortNote) diff --git a/xml/gencl.py b/xml/gencl.py new file mode 100644 index 000000000..eb77a8cdf --- /dev/null +++ b/xml/gencl.py @@ -0,0 +1,464 @@ +#!/usr/bin/python3 +# +# Copyright 2013-2023 The Khronos Group Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import pdb +import re +import sys +import time +import xml.etree.ElementTree as etree + +from cgenerator import CGeneratorOptions, COutputGenerator +from docgenerator import DocGeneratorOptions, DocOutputGenerator +from extensionmetadocgenerator import (ExtensionMetaDocGeneratorOptions, + ExtensionMetaDocOutputGenerator) + +from generator import write + + +from pygenerator import PyOutputGenerator +from reflib import logDiag, logWarn, setLogFile +from reg import Registry + +from clconventions import OpenCLConventions as APIConventions + + +# Simple timer functions +startTime = None + + +def startTimer(timeit): + global startTime + if timeit: + startTime = time.process_time() + + +def endTimer(timeit, msg): + global startTime + if timeit: + endTime = time.process_time() + logDiag(msg, endTime - startTime) + startTime = None + + +def makeREstring(strings, default=None, strings_are_regex=False): + """Turn a list of strings into a regexp string matching exactly those strings.""" + if strings or default is None: + if not strings_are_regex: + strings = (re.escape(s) for s in strings) + return '^(' + '|'.join(strings) + ')$' + return default + +def makeGenOpts(args): + """Returns a directory of [ generator function, generator options ] indexed + by specified short names. The generator options incorporate the following + parameters: + + args is an parsed argument object; see below for the fields that are used.""" + global genOpts + genOpts = {} + + # Default class of extensions to include, or None + defaultExtensions = args.defaultExtensions + + # Additional extensions to include (list of extensions) + extensions = args.extension + + # Extensions to remove (list of extensions) + removeExtensions = args.removeExtensions + + # Extensions to emit (list of extensions) + emitExtensions = args.emitExtensions + + # SPIR-V capabilities / features to emit (list of extensions & capabilities) + # emitSpirv = args.emitSpirv + + # Features to include (list of features) + features = args.feature + + # Whether to disable inclusion protect in headers + protect = args.protect + + # Output target directory + directory = args.directory + + # Path to generated files, particularly api.py + genpath = args.genpath + + # Generate MISRA C-friendly headers + misracstyle = args.misracstyle; + + # Generate MISRA C++-friendly headers + misracppstyle = args.misracppstyle; + + # Descriptive names for various regexp patterns used to select + # versions and extensions + allSpirv = allFeatures = allExtensions = r'.*' + + # Turn lists of names/patterns into matching regular expressions + addExtensionsPat = makeREstring(extensions, None) + removeExtensionsPat = makeREstring(removeExtensions, None) + emitExtensionsPat = makeREstring(emitExtensions, allExtensions) + # emitSpirvPat = makeREstring(emitSpirv, allSpirv) + featuresPat = makeREstring(features, allFeatures) + + # Copyright text prefixing all headers (list of strings). + # The SPDX formatting below works around constraints of the 'reuse' tool + prefixStrings = [ + '/*', + '** Copyright 2015-2023 The Khronos Group Inc.', + '**', + '** SPDX' + '-License-Identifier: Apache-2.0', + '*/', + '' + ] + + # Text specific to OpenCL headers + clPrefixStrings = [ + '/*', + '** This header is generated from the Khronos OpenCL XML API Registry.', + '**', + '*/', + '' + ] + + # Defaults for generating re-inclusion protection wrappers (or not) + protectFile = protect + + # An API style conventions object + conventions = APIConventions() + + # API include files for spec and ref pages + # Overwrites include subdirectories in spec source tree + # The generated include files do not include the calling convention + # macros (apientry etc.), unlike the header files. + # Because the 1.0 core branch includes ref pages for extensions, + # all the extension interfaces need to be generated, even though + # none are used by the core spec itself. + genOpts['apiinc'] = [ + DocOutputGenerator, + DocGeneratorOptions( + conventions = conventions, + filename = 'timeMarker', + directory = directory, + genpath = genpath, + apiname = 'opencl', + profile = None, + versions = featuresPat, + emitversions = featuresPat, + defaultExtensions = defaultExtensions, + addExtensions = addExtensionsPat, + removeExtensions = removeExtensionsPat, + emitExtensions = emitExtensionsPat, + prefixText = prefixStrings + clPrefixStrings, + apicall = '', + apientry = '', + apientryp = '*', + alignFuncParam = 0, + expandEnumerants = False) + ] + + # Python representation of API information, used by scripts that + # don't need to load the full XML. + genOpts['api.py'] = [ + PyOutputGenerator, + DocGeneratorOptions( + conventions = conventions, + filename = 'api.py', + directory = directory, + genpath = genpath, + apiname = 'opencl', + profile = None, + versions = featuresPat, + emitversions = featuresPat, + defaultExtensions = None, + addExtensions = addExtensionsPat, + removeExtensions = removeExtensionsPat, + emitExtensions = emitExtensionsPat, + reparentEnums = False) + ] + + # Extension metainformation for spec extension appendices + # Includes all extensions by default, but only so that the generated + # 'promoted_extensions_*' files refer to all extensions that were + # promoted to a core version. + genOpts['extinc'] = [ + ExtensionMetaDocOutputGenerator, + ExtensionMetaDocGeneratorOptions( + conventions = conventions, + filename = 'timeMarker', + directory = directory, + genpath = None, + apiname = 'opencl', + profile = None, + versions = featuresPat, + emitversions = None, + defaultExtensions = defaultExtensions, + addExtensions = addExtensionsPat, + removeExtensions = None, + emitExtensions = emitExtensionsPat) + ] + + # Platform extensions, in their own header files + # Each element of the platforms[] array defines information for + # generating a single platform: + # [0] is the generated header file name + # [1] is the set of platform extensions to generate + # [2] is additional extensions whose interfaces should be considered, + # but suppressed in the output, to avoid duplicate definitions of + # dependent types like VkDisplayKHR and VkSurfaceKHR which come from + # non-platform extensions. + + # Track all platform extensions, for exclusion from vulkan_core.h + allPlatformExtensions = [] + + # # Extensions suppressed for all platforms. + # # Covers common WSI extension types. + # commonSuppressExtensions = [ 'VK_KHR_display', 'VK_KHR_swapchain' ] + # + # platforms = [ + # [ 'vulkan_android.h', [ 'VK_KHR_android_surface', + # 'VK_ANDROID_external_memory_android_hardware_buffer' + # ], commonSuppressExtensions ], + # [ 'vulkan_fuchsia.h', [ 'VK_FUCHSIA_imagepipe_surface'], commonSuppressExtensions ], + # [ 'vulkan_ios.h', [ 'VK_MVK_ios_surface' ], commonSuppressExtensions ], + # [ 'vulkan_macos.h', [ 'VK_MVK_macos_surface' ], commonSuppressExtensions ], + # [ 'vulkan_vi.h', [ 'VK_NN_vi_surface' ], commonSuppressExtensions ], + # [ 'vulkan_wayland.h', [ 'VK_KHR_wayland_surface' ], commonSuppressExtensions ], + # [ 'vulkan_win32.h', [ 'VK_.*_win32(|_.*)' ], commonSuppressExtensions + [ 'VK_KHR_external_semaphore', 'VK_KHR_external_memory_capabilities', 'VK_KHR_external_fence', 'VK_KHR_external_fence_capabilities', 'VK_NV_external_memory_capabilities' ] ], + # [ 'vulkan_xcb.h', [ 'VK_KHR_xcb_surface' ], commonSuppressExtensions ], + # [ 'vulkan_xlib.h', [ 'VK_KHR_xlib_surface' ], commonSuppressExtensions ], + # [ 'vulkan_xlib_xrandr.h', [ 'VK_EXT_acquire_xlib_display' ], commonSuppressExtensions ], + # ] + # + # for platform in platforms: + # headername = platform[0] + # + # allPlatformExtensions += platform[1] + # + # addPlatformExtensionsRE = makeREstring(platform[1] + platform[2]) + # emitPlatformExtensionsRE = makeREstring(platform[1]) + # + # opts = CGeneratorOptions( + # filename = headername, + # directory = directory, + # apiname = 'vulkan', + # profile = None, + # versions = featuresPat, + # emitversions = None, + # defaultExtensions = None, + # addExtensions = addPlatformExtensionsRE, + # removeExtensions = None, + # emitExtensions = emitPlatformExtensionsRE, + # prefixText = prefixStrings + clPrefixStrings, + # genFuncPointers = True, + # protectFile = protectFile, + # protectFeature = False, + # protectProto = '#ifndef', + # protectProtoStr = 'VK_NO_PROTOTYPES', + # apicall = 'VKAPI_ATTR ', + # apientry = 'VKAPI_CALL ', + # apientryp = 'VKAPI_PTR *', + # alignFuncParam = 0) + # + # genOpts[headername] = [ COutputGenerator, opts ] + + # Header for core API + extensions. + # To generate just the core API, + # change to 'defaultExtensions = None' below. + # + # By default this adds all enabled, non-platform extensions. + # It removes all platform extensions (from the platform headers options + # constructed above) as well as any explicitly specified removals. + + removeExtensionsPat = makeREstring( + allPlatformExtensions + removeExtensions, None, strings_are_regex=True) + + genOpts['cl.h'] = [ + COutputGenerator, + CGeneratorOptions( + conventions = conventions, + filename = 'cl.h', + directory = directory, + genpath = None, + apiname = 'opencl', + profile = None, + versions = featuresPat, + emitversions = featuresPat, + defaultExtensions = defaultExtensions, + addExtensions = None, + removeExtensions = removeExtensionsPat, + emitExtensions = emitExtensionsPat, + prefixText = prefixStrings + clPrefixStrings, + genFuncPointers = False, + protectFile = protectFile, + protectFeature = False, + protectProto = '#ifndef', + protectProtoStr = 'CL_NO_PROTOTYPES', + apicall = 'CL_API_ENTRY ', + apientry = 'CL_API_CALL ', + apientryp = 'CL_API_CALL *', + alignFuncParam = 0, + misracstyle = misracstyle, + misracppstyle = misracppstyle) + ] + +def genTarget(args): + """Create an API generator and corresponding generator options based on + the requested target and command line options. + + This is encapsulated in a function so it can be profiled and/or timed. + The args parameter is an parsed argument object containing the following + fields that are used: + + - target - target to generate + - directory - directory to generate it in + - protect - True if re-inclusion wrappers should be created + - extensions - list of additional extensions to include in generated interfaces""" + + # Create generator options with parameters specified on command line + makeGenOpts(args) + + # pdb.set_trace() + + # Select a generator matching the requested target + if args.target in genOpts: + createGenerator = genOpts[args.target][0] + options = genOpts[args.target][1] + + logDiag('* Building', options.filename) + logDiag('* options.versions =', options.versions) + logDiag('* options.emitversions =', options.emitversions) + logDiag('* options.defaultExtensions =', options.defaultExtensions) + logDiag('* options.addExtensions =', options.addExtensions) + logDiag('* options.removeExtensions =', options.removeExtensions) + logDiag('* options.emitExtensions =', options.emitExtensions) + + gen = createGenerator(errFile=errWarn, + warnFile=errWarn, + diagFile=diag) + return (gen, options) + else: + logErr('No generator options for unknown target:', args.target) + return None + + +# -feature name +# -extension name +# For both, "name" may be a single name, or a space-separated list +# of names, or a regular expression. +if __name__ == '__main__': + parser = argparse.ArgumentParser() + + parser.add_argument('-defaultExtensions', action='store', + default='opencl', + help='Specify a single class of extensions to add to targets') + parser.add_argument('-extension', action='append', + default=[], + help='Specify an extension or extensions to add to targets') + parser.add_argument('-removeExtensions', action='append', + default=[], + help='Specify an extension or extensions to remove from targets') + parser.add_argument('-emitExtensions', action='append', + default=[], + help='Specify an extension or extensions to emit in targets') + + + + parser.add_argument('-feature', action='append', + default=[], + help='Specify a core API feature name or names to add to targets') + parser.add_argument('-debug', action='store_true', + help='Enable debugging') + parser.add_argument('-dump', action='store_true', + help='Enable dump to stderr') + parser.add_argument('-diagfile', action='store', + default=None, + help='Write diagnostics to specified file') + parser.add_argument('-errfile', action='store', + default=None, + help='Write errors and warnings to specified file instead of stderr') + parser.add_argument('-noprotect', dest='protect', action='store_false', + help='Disable inclusion protection in output headers') + parser.add_argument('-profile', action='store_true', + help='Enable profiling') + parser.add_argument('-registry', action='store', + default='cl.xml', + help='Use specified registry file instead of cl.xml') + parser.add_argument('-time', action='store_true', + help='Enable timing') + parser.add_argument('-validate', action='store_true', + help='Validate the registry properties and exit') + parser.add_argument('-genpath', action='store', default='gen', + help='Path to generated files') + parser.add_argument('-o', action='store', dest='directory', + default='.', + help='Create target and related files in specified directory') + parser.add_argument('target', metavar='target', nargs='?', + help='Specify target') + parser.add_argument('-quiet', action='store_true', default=True, + help='Suppress script output during normal execution.') + parser.add_argument('-verbose', action='store_false', dest='quiet', default=True, + help='Enable script output during normal execution.') + parser.add_argument('-misracstyle', dest='misracstyle', action='store_true', + help='generate MISRA C-friendly headers') + parser.add_argument('-misracppstyle', dest='misracppstyle', action='store_true', + help='generate MISRA C++-friendly headers') + + args = parser.parse_args() + + # This splits arguments which are space-separated lists + args.feature = [name for arg in args.feature for name in arg.split()] + args.extension = [name for arg in args.extension for name in arg.split()] + + # create error/warning & diagnostic files + if args.errfile: + errWarn = open(args.errfile, 'w', encoding='utf-8') + else: + errWarn = sys.stderr + + if args.diagfile: + diag = open(args.diagfile, 'w', encoding='utf-8') + else: + diag = None + + (gen, options) = (None, None) + if not args.validate: + # Create the API generator & generator options + (gen, options) = genTarget(args) + + # Create the registry object with the specified generator and generator + # options. The options are set before XML loading as they may affect it. + reg = Registry(gen, options) + + # Parse the specified registry XML into an ElementTree object + startTimer(args.time) + tree = etree.parse(args.registry) + endTimer(args.time, '* Time to make ElementTree =') + + # Load the XML tree into the registry object + startTimer(args.time) + reg.loadElementTree(tree) + endTimer(args.time, '* Time to parse ElementTree =') + + if args.validate: + success = reg.validateRegistry() + sys.exit(0 if success else 1) + + if args.dump: + logDiag('* Dumping registry to regdump.txt') + reg.dumpReg(filehandle=open('regdump.txt', 'w', encoding='utf-8')) + + # Finally, use the output generator to create the requested target + if args.debug: + pdb.run('reg.apiGen()') + else: + startTimer(args.time) + reg.apiGen() + endTimer(args.time, '* Time to generate ' + options.filename + ' =') + + if not args.quiet: + logDiag('* Generated', options.filename) diff --git a/xml/generator.py b/xml/generator.py new file mode 100644 index 000000000..c7c460d95 --- /dev/null +++ b/xml/generator.py @@ -0,0 +1,1186 @@ +#!/usr/bin/python3 -i +# +# Copyright 2013-2023 The Khronos Group Inc. +# +# SPDX-License-Identifier: Apache-2.0 +"""Base class for source/header/doc generators, as well as some utility functions.""" + +from __future__ import unicode_literals + +import io +import os +import pdb +import re +import shutil +import sys +import tempfile +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path + +from spec_tools.util import getElemName, getElemType + + +def write(*args, **kwargs): + file = kwargs.pop('file', sys.stdout) + end = kwargs.pop('end', '\n') + file.write(' '.join(str(arg) for arg in args)) + file.write(end) + + +def noneStr(s): + """Return string argument, or "" if argument is None. + + Used in converting etree Elements into text. + s - string to convert""" + if s: + return s + return "" + + +def enquote(s): + """Return string argument with surrounding quotes, + for serialization into Python code.""" + if s: + return "'{}'".format(s) + return None + + +def regSortCategoryKey(feature): + """Sort key for regSortFeatures. + Sorts by category of the feature name string: + + - Core API features (those defined with a `` tag) + - ARB/KHR/OES (Khronos extensions) + - other (EXT/vendor extensions)""" + + if feature.elem.tag == 'feature': + return 0 + if (feature.category == 'ARB' + or feature.category == 'KHR' + or feature.category == 'OES'): + return 1 + + return 2 + + +def regSortOrderKey(feature): + """Sort key for regSortFeatures - key is the sortorder attribute.""" + + # print("regSortOrderKey {} -> {}".format(feature.name, feature.sortorder)) + return feature.sortorder + + +def regSortFeatureVersionKey(feature): + """Sort key for regSortFeatures - key is the feature version. + `` elements all have version number 0.""" + + return float(feature.versionNumber) + + +def regSortExtensionNumberKey(feature): + """Sort key for regSortFeatures - key is the extension number. + `` elements all have extension number 0.""" + + return int(feature.number) + + +def regSortFeatures(featureList): + """Default sort procedure for features. + + - Sorts by explicit sort order (default 0) relative to other features + - then by feature category ('feature' or 'extension'), + - then by version number (for features) + - then by extension number (for extensions)""" + featureList.sort(key=regSortExtensionNumberKey) + featureList.sort(key=regSortFeatureVersionKey) + featureList.sort(key=regSortCategoryKey) + featureList.sort(key=regSortOrderKey) + + +class GeneratorOptions: + """Base class for options used during header/documentation production. + + These options are target language independent, and used by + Registry.apiGen() and by base OutputGenerator objects.""" + + def __init__(self, + conventions=None, + filename=None, + directory='.', + genpath=None, + apiname=None, + profile=None, + versions='.*', + emitversions='.*', + defaultExtensions=None, + addExtensions=None, + removeExtensions=None, + emitExtensions=None, + emitSpirv=None, + reparentEnums=True, + sortProcedure=regSortFeatures): + """Constructor. + + Arguments: + + - conventions - may be mandatory for some generators: + an object that implements ConventionsBase + - filename - basename of file to generate, or None to write to stdout. + - directory - directory in which to generate files + - genpath - path to previously generated files, such as api.py + - apiname - string matching `` 'apiname' attribute, e.g. 'gl'. + - profile - string specifying API profile , e.g. 'core', or None. + - versions - regex matching API versions to process interfaces for. + Normally `'.*'` or `'[0-9][.][0-9]'` to match all defined versions. + - emitversions - regex matching API versions to actually emit + interfaces for (though all requested versions are considered + when deciding which interfaces to generate). For GL 4.3 glext.h, + this might be `'1[.][2-5]|[2-4][.][0-9]'`. + - defaultExtensions - If not None, a string which must in its + entirety match the pattern in the "supported" attribute of + the ``. Defaults to None. Usually the same as apiname. + - addExtensions - regex matching names of additional extensions + to include. Defaults to None. + - removeExtensions - regex matching names of extensions to + remove (after defaultExtensions and addExtensions). Defaults + to None. + - emitExtensions - regex matching names of extensions to actually emit + interfaces for (though all requested versions are considered when + deciding which interfaces to generate). + to None. + - emitSpirv - regex matching names of extensions and capabilities + to actually emit interfaces for. + - reparentEnums - move elements which extend an enumerated + type from or elements to the target + element. This is required for almost all purposes, but the + InterfaceGenerator relies on the list of interfaces in the + or being complete. Defaults to True. + - sortProcedure - takes a list of FeatureInfo objects and sorts + them in place to a preferred order in the generated output. + Default is core API versions, ARB/KHR/OES extensions, all other + extensions, by core API version number or extension number in each + group. + + The regex patterns can be None or empty, in which case they match + nothing.""" + self.conventions = conventions + """may be mandatory for some generators: + an object that implements ConventionsBase""" + + self.filename = filename + "basename of file to generate, or None to write to stdout." + + self.genpath = genpath + """path to previously generated files, such as api.py""" + + self.directory = directory + "directory in which to generate filename" + + self.apiname = apiname + "string matching `` 'apiname' attribute, e.g. 'gl'." + + self.profile = profile + "string specifying API profile , e.g. 'core', or None." + + self.versions = self.emptyRegex(versions) + """regex matching API versions to process interfaces for. + Normally `'.*'` or `'[0-9][.][0-9]'` to match all defined versions.""" + + self.emitversions = self.emptyRegex(emitversions) + """regex matching API versions to actually emit + interfaces for (though all requested versions are considered + when deciding which interfaces to generate). For GL 4.3 glext.h, + this might be `'1[.][2-5]|[2-4][.][0-9]'`.""" + + self.defaultExtensions = defaultExtensions + """If not None, a string which must in its + entirety match the pattern in the "supported" attribute of + the ``. Defaults to None. Usually the same as apiname.""" + + self.addExtensions = self.emptyRegex(addExtensions) + """regex matching names of additional extensions + to include. Defaults to None.""" + + self.removeExtensions = self.emptyRegex(removeExtensions) + """regex matching names of extensions to + remove (after defaultExtensions and addExtensions). Defaults + to None.""" + + self.emitExtensions = self.emptyRegex(emitExtensions) + """regex matching names of extensions to actually emit + interfaces for (though all requested versions are considered when + deciding which interfaces to generate).""" + + self.emitSpirv = self.emptyRegex(emitSpirv) + """regex matching names of extensions and capabilities + to actually emit interfaces for.""" + + self.reparentEnums = reparentEnums + """boolean specifying whether to remove elements from + or when extending an type.""" + + self.sortProcedure = sortProcedure + """takes a list of FeatureInfo objects and sorts + them in place to a preferred order in the generated output. + Default is core API versions, ARB/KHR/OES extensions, all + other extensions, alphabetically within each group.""" + + self.codeGenerator = False + """True if this generator makes compilable code""" + + def emptyRegex(self, pat): + """Substitute a regular expression which matches no version + or extension names for None or the empty string.""" + if not pat: + return '_nomatch_^' + + return pat + + +class OutputGenerator: + """Generate specified API interfaces in a specific style, such as a C header. + + Base class for generating API interfaces. + Manages basic logic, logging, and output file control. + Derived classes actually generate formatted output. + """ + + # categoryToPath - map XML 'category' to include file directory name + categoryToPath = { + 'bitmask': 'flags', + 'enum': 'enums', + 'funcpointer': 'funcpointers', + 'handle': 'handles', + 'define': 'defines', + 'basetype': 'basetypes', + } + + def __init__(self, errFile=sys.stderr, warnFile=sys.stderr, diagFile=sys.stdout): + """Constructor + + - errFile, warnFile, diagFile - file handles to write errors, + warnings, diagnostics to. May be None to not write.""" + self.outFile = None + self.errFile = errFile + self.warnFile = warnFile + self.diagFile = diagFile + # Internal state + self.featureName = None + self.genOpts = None + self.registry = None + self.featureDictionary = {} + # Used for extension enum value generation + self.extBase = 1000000000 + self.extBlockSize = 1000 + self.madeDirs = {} + + # API dictionary, which may be loaded by the beginFile method of + # derived generators. + self.apidict = None + + def logMsg(self, level, *args): + """Write a message of different categories to different + destinations. + + - `level` + - 'diag' (diagnostic, voluminous) + - 'warn' (warning) + - 'error' (fatal error - raises exception after logging) + + - `*args` - print()-style arguments to direct to corresponding log""" + if level == 'error': + strfile = io.StringIO() + write('ERROR:', *args, file=strfile) + if self.errFile is not None: + write(strfile.getvalue(), file=self.errFile) + raise UserWarning(strfile.getvalue()) + elif level == 'warn': + if self.warnFile is not None: + write('WARNING:', *args, file=self.warnFile) + elif level == 'diag': + if self.diagFile is not None: + write('DIAG:', *args, file=self.diagFile) + else: + raise UserWarning( + '*** FATAL ERROR in Generator.logMsg: unknown level:' + level) + + def enumToValue(self, elem, needsNum, bitwidth = 32, forceSuffix = False): + """Parse and convert an `` tag into a value. + + Returns a list: + + - first element - integer representation of the value, or None + if needsNum is False. The value must be a legal number + if needsNum is True. + - second element - string representation of the value + + There are several possible representations of values. + + - A 'value' attribute simply contains the value. + - A 'bitpos' attribute defines a value by specifying the bit + position which is set in that value. + - An 'offset','extbase','extends' triplet specifies a value + as an offset to a base value defined by the specified + 'extbase' extension name, which is then cast to the + typename specified by 'extends'. This requires probing + the registry database, and imbeds knowledge of the + API extension enum scheme in this function. + - An 'alias' attribute contains the name of another enum + which this is an alias of. The other enum must be + declared first when emitting this enum.""" + name = elem.get('name') + numVal = None + if 'value' in elem.keys(): + value = elem.get('value') + # print('About to translate value =', value, 'type =', type(value)) + if needsNum: + numVal = int(value, 0) + # If there's a non-integer, numeric 'type' attribute (e.g. 'u' or + # 'ull'), append it to the string value. + # t = enuminfo.elem.get('type') + # if t is not None and t != '' and t != 'i' and t != 's': + # value += enuminfo.type + if forceSuffix: + if bitwidth == 64: + value = value + 'ULL' + else: + value = value + 'U' + self.logMsg('diag', 'Enum', name, '-> value [', numVal, ',', value, ']') + return [numVal, value] + if 'bitpos' in elem.keys(): + value = elem.get('bitpos') + bitpos = int(value, 0) + numVal = 1 << bitpos + value = '0x%08x' % numVal + if bitwidth == 64: + value = value + 'ULL' + elif forceSuffix: + value = value + 'U' + self.logMsg('diag', 'Enum', name, '-> bitpos [', numVal, ',', value, ']') + return [numVal, value] + if 'offset' in elem.keys(): + # Obtain values in the mapping from the attributes + enumNegative = False + offset = int(elem.get('offset'), 0) + extnumber = int(elem.get('extnumber'), 0) + extends = elem.get('extends') + if 'dir' in elem.keys(): + enumNegative = True + self.logMsg('diag', 'Enum', name, 'offset =', offset, + 'extnumber =', extnumber, 'extends =', extends, + 'enumNegative =', enumNegative) + # Now determine the actual enumerant value, as defined + # in the "Layers and Extensions" appendix of the spec. + numVal = self.extBase + (extnumber - 1) * self.extBlockSize + offset + if enumNegative: + numVal *= -1 + value = '%d' % numVal + # More logic needed! + self.logMsg('diag', 'Enum', name, '-> offset [', numVal, ',', value, ']') + return [numVal, value] + if 'alias' in elem.keys(): + return [None, elem.get('alias')] + return [None, None] + + def checkDuplicateEnums(self, enums): + """Check enumerated values for duplicates. + + - enums - list of `` Elements + + returns the list with duplicates stripped""" + # Dictionaries indexed by name and numeric value. + # Entries are [ Element, numVal, strVal ] matching name or value + + nameMap = {} + valueMap = {} + + stripped = [] + for elem in enums: + name = elem.get('name') + (numVal, strVal) = self.enumToValue(elem, True) + + if name in nameMap: + # Duplicate name found; check values + (name2, numVal2, strVal2) = nameMap[name] + + # Duplicate enum values for the same name are benign. This + # happens when defining the same enum conditionally in + # several extension blocks. + if (strVal2 == strVal or (numVal is not None + and numVal == numVal2)): + True + # self.logMsg('info', 'checkDuplicateEnums: Duplicate enum (' + name + + # ') found with the same value:' + strVal) + else: + self.logMsg('warn', 'checkDuplicateEnums: Duplicate enum (' + name + + ') found with different values:' + strVal + + ' and ' + strVal2) + + # Don't add the duplicate to the returned list + continue + elif numVal in valueMap: + # Duplicate value found (such as an alias); report it, but + # still add this enum to the list. + (name2, numVal2, strVal2) = valueMap[numVal] + + msg = 'Two enums found with the same value: {} = {} = {}'.format( + name, name2.get('name'), strVal) + self.logMsg('error', msg) + + # Track this enum to detect followon duplicates + nameMap[name] = [elem, numVal, strVal] + if numVal is not None: + valueMap[numVal] = [elem, numVal, strVal] + + # Add this enum to the list + stripped.append(elem) + + # Return the list + return stripped + + def misracstyle(self): + return False; + + def misracppstyle(self): + return False; + + def buildEnumCDecl(self, expand, groupinfo, groupName): + """Generate the C declaration for an enum""" + groupElem = groupinfo.elem + + # Determine the required bit width for the enum group. + # 32 is the default, which generates C enum types for the values. + bitwidth = 32 + + # If the constFlagBits preference is set, 64 is the default for bitmasks + if self.genOpts.conventions.constFlagBits and groupElem.get('type') == 'bitmask': + bitwidth = 64 + + # Check for an explicitly defined bitwidth, which will override any defaults. + if groupElem.get('bitwidth'): + try: + bitwidth = int(groupElem.get('bitwidth')) + except ValueError as ve: + self.logMsg('error', 'Invalid value for bitwidth attribute (', groupElem.get('bitwidth'), ') for ', groupName, ' - must be an integer value\n') + exit(1) + + usebitmask = False + usedefine = False + + # Bitmask flags can be generated as either "static const uint{32,64}_t" values, + # or as 32-bit C enums. 64-bit types must use uint64_t values. + if groupElem.get('type') == 'bitmask': + if bitwidth > 32 or self.misracppstyle(): + usebitmask = True + if self.misracstyle(): + usedefine = True + + if usedefine or usebitmask: + # Validate the bitwidth and generate values appropriately + if bitwidth > 64: + self.logMsg('error', 'Invalid value for bitwidth attribute (', groupElem.get('bitwidth'), ') for bitmask type ', groupName, ' - must be less than or equal to 64\n') + exit(1) + else: + return self.buildEnumCDecl_BitmaskOrDefine(groupinfo, groupName, bitwidth, usedefine) + else: + # Validate the bitwidth and generate values appropriately + if bitwidth > 32: + self.logMsg('error', 'Invalid value for bitwidth attribute (', groupElem.get('bitwidth'), ') for enum type ', groupName, ' - must be less than or equal to 32\n') + exit(1) + else: + return self.buildEnumCDecl_Enum(expand, groupinfo, groupName) + + def buildEnumCDecl_BitmaskOrDefine(self, groupinfo, groupName, bitwidth, usedefine): + """Generate the C declaration for an "enum" that is actually a + set of flag bits""" + groupElem = groupinfo.elem + flagTypeName = groupElem.get('name') + + # Prefix + body = "// Flag bits for " + flagTypeName + "\n" + + if bitwidth == 64: + body += "typedef VkFlags64 %s;\n" % flagTypeName; + else: + body += "typedef VkFlags %s;\n" % flagTypeName; + + # Maximum allowable value for a flag (unsigned 64-bit integer) + maxValidValue = 2**(64) - 1 + minValidValue = 0 + + # Get a list of nested 'enum' tags. + enums = groupElem.findall('enum') + + # Check for and report duplicates, and return a list with them + # removed. + enums = self.checkDuplicateEnums(enums) + + # Accumulate non-numeric enumerant values separately and append + # them following the numeric values, to allow for aliases. + # NOTE: this doesn't do a topological sort yet, so aliases of + # aliases can still get in the wrong order. + aliasText = '' + + # Loop over the nested 'enum' tags. + for elem in enums: + # Convert the value to an integer and use that to track min/max. + # Values of form -(number) are accepted but nothing more complex. + # Should catch exceptions here for more complex constructs. Not yet. + (numVal, strVal) = self.enumToValue(elem, True, bitwidth, True) + name = elem.get('name') + + # Range check for the enum value + if numVal is not None and (numVal > maxValidValue or numVal < minValidValue): + self.logMsg('error', 'Allowable range for flag types in C is [', minValidValue, ',', maxValidValue, '], but', name, 'flag has a value outside of this (', strVal, ')\n') + exit(1) + + decl = self.genRequirements(name, mustBeFound = False) + + if self.isEnumRequired(elem): + protect = elem.get('protect') + if protect is not None: + body += '#ifdef {}\n'.format(protect) + + if usedefine: + decl += "#define {} {}\n".format(name, strVal) + elif self.misracppstyle(): + decl += "static constexpr {} {} {{{}}};\n".format(flagTypeName, name, strVal) + else: + # Some C compilers only allow initializing a 'static const' variable with a literal value. + # So initializing an alias from another 'static const' value would fail to compile. + # Work around this by chasing the aliases to get the actual value. + while numVal is None: + alias = self.registry.tree.find("enums/enum[@name='" + strVal + "']") + (numVal, strVal) = self.enumToValue(alias, True, bitwidth, True) + decl += "static const {} {} = {};\n".format(flagTypeName, name, strVal) + + if numVal is not None: + body += decl + else: + aliasText += decl + + if protect is not None: + body += '#endif\n' + + # Now append the non-numeric enumerant values + body += aliasText + + # Postfix + + return ("bitmask", body) + + def buildEnumCDecl_Enum(self, expand, groupinfo, groupName): + """Generate the C declaration for an enumerated type""" + groupElem = groupinfo.elem + + # Break the group name into prefix and suffix portions for range + # enum generation + expandName = re.sub(r'([0-9]+|[a-z_])([A-Z0-9])', r'\1_\2', groupName).upper() + expandPrefix = expandName + expandSuffix = '' + expandSuffixMatch = re.search(r'[A-Z][A-Z]+$', groupName) + if expandSuffixMatch: + expandSuffix = '_' + expandSuffixMatch.group() + # Strip off the suffix from the prefix + expandPrefix = expandName.rsplit(expandSuffix, 1)[0] + + # Prefix + body = ["typedef enum %s {" % groupName] + + # @@ Should use the type="bitmask" attribute instead + isEnum = ('FLAG_BITS' not in expandPrefix) + + # Allowable range for a C enum - which is that of a signed 32-bit integer + maxValidValue = 2**(32 - 1) - 1 + minValidValue = (maxValidValue * -1) - 1 + + + # Get a list of nested 'enum' tags. + enums = groupElem.findall('enum') + + # Check for and report duplicates, and return a list with them + # removed. + enums = self.checkDuplicateEnums(enums) + + # Loop over the nested 'enum' tags. Keep track of the minimum and + # maximum numeric values, if they can be determined; but only for + # core API enumerants, not extension enumerants. This is inferred + # by looking for 'extends' attributes. + minName = None + + # Accumulate non-numeric enumerant values separately and append + # them following the numeric values, to allow for aliases. + # NOTE: this doesn't do a topological sort yet, so aliases of + # aliases can still get in the wrong order. + aliasText = [] + + for elem in enums: + # Convert the value to an integer and use that to track min/max. + # Values of form -(number) are accepted but nothing more complex. + # Should catch exceptions here for more complex constructs. Not yet. + (numVal, strVal) = self.enumToValue(elem, True) + name = elem.get('name') + + # Extension enumerants are only included if they are required + if self.isEnumRequired(elem): + decl = '' + + protect = elem.get('protect') + if protect is not None: + decl += '#ifdef {}\n'.format(protect) + + # Indent requirements comment, if there is one + requirements = self.genRequirements(name, mustBeFound = False) + if requirements != '': + requirements = ' ' + requirements + decl += requirements + decl += ' {} = {},'.format(name, strVal) + + if protect is not None: + decl += '\n#endif' + + if numVal is not None: + body.append(decl) + else: + aliasText.append(decl) + + # Range check for the enum value + if numVal is not None and (numVal > maxValidValue or numVal < minValidValue): + self.logMsg('error', 'Allowable range for C enum types is [', minValidValue, ',', maxValidValue, '], but', name, 'has a value outside of this (', strVal, ')\n') + exit(1) + + # Don't track min/max for non-numbers (numVal is None) + if isEnum and numVal is not None and elem.get('extends') is None: + if minName is None: + minName = maxName = name + minValue = maxValue = numVal + elif numVal < minValue: + minName = name + minValue = numVal + elif numVal > maxValue: + maxName = name + maxValue = numVal + + # Now append the non-numeric enumerant values + body.extend(aliasText) + + # Generate min/max value tokens - legacy use case. + if isEnum and expand: + body.extend((" {}_BEGIN_RANGE{} = {},".format(expandPrefix, expandSuffix, minName), + " {}_END_RANGE{} = {},".format( + expandPrefix, expandSuffix, maxName), + " {}_RANGE_SIZE{} = ({} - {} + 1),".format(expandPrefix, expandSuffix, maxName, minName))) + + # Generate a range-padding value to ensure the enum is 32 bits, but + # only in code generators, so it doesn't appear in documentation + if (self.genOpts.codeGenerator or + self.conventions.generate_max_enum_in_docs): + body.append(" {}_MAX_ENUM{} = 0x7FFFFFFF".format( + expandPrefix, expandSuffix)) + + # Postfix + body.append("} %s;" % groupName) + + # Determine appropriate section for this declaration + if groupElem.get('type') == 'bitmask': + section = 'bitmask' + else: + section = 'group' + + return (section, '\n'.join(body)) + + def buildConstantCDecl(self, enuminfo, name, alias): + """Generate the C declaration for a constant (a single + value). + + tags may specify their values in several ways, but are + usually just integers or floating-point numbers.""" + + (_, strVal) = self.enumToValue(enuminfo.elem, False) + + if self.misracppstyle() and enuminfo.elem.get('type') and not alias: + # Generate e.g.: static constexpr uint32_t x = ~static_cast(1U); + # This appeases MISRA "underlying type" rules. + typeStr = enuminfo.elem.get('type'); + invert = '~' in strVal + number = strVal.strip("()~UL") + if typeStr != "float": + number += 'U' + strVal = "~" if invert else "" + strVal += "static_cast<" + typeStr + ">(" + number + ")" + body = 'static constexpr ' + typeStr.ljust(9) + name.ljust(33) + ' {' + strVal + '};' + elif enuminfo.elem.get('type') and not alias: + # Generate e.g.: #define x (~0ULL) + typeStr = enuminfo.elem.get('type'); + invert = '~' in strVal + paren = '(' in strVal + number = strVal.strip("()~UL") + if typeStr != "float": + if typeStr == "uint64_t": + number += 'ULL' + else: + number += 'U' + strVal = "~" if invert else "" + strVal += number + if paren: + strVal = "(" + strVal + ")"; + body = '#define ' + name.ljust(33) + ' ' + strVal; + else: + body = '#define ' + name.ljust(33) + ' ' + strVal + + return body + + def makeDir(self, path): + """Create a directory, if not already done. + + Generally called from derived generators creating hierarchies.""" + self.logMsg('diag', 'OutputGenerator::makeDir(' + path + ')') + if path not in self.madeDirs: + # This can get race conditions with multiple writers, see + # https://stackoverflow.com/questions/273192/ + if not os.path.exists(path): + os.makedirs(path) + self.madeDirs[path] = None + + def beginFile(self, genOpts): + """Start a new interface file + + - genOpts - GeneratorOptions controlling what's generated and how""" + self.genOpts = genOpts + self.should_insert_may_alias_macro = \ + self.genOpts.conventions.should_insert_may_alias_macro(self.genOpts) + + # Try to import the API dictionary, api.py, if it exists. Nothing in + # api.py cannot be extracted directly from the XML, and in the + # future we should do that. + if self.genOpts.genpath is not None: + try: + sys.path.insert(0, self.genOpts.genpath) + import api + self.apidict = api + except ImportError: + self.apidict = None + + self.conventions = genOpts.conventions + + # Open a temporary file for accumulating output. + if self.genOpts.filename is not None: + self.outFile = tempfile.NamedTemporaryFile(mode='w', encoding='utf-8', newline='\n', delete=False) + else: + self.outFile = sys.stdout + + def endFile(self): + if self.errFile: + self.errFile.flush() + if self.warnFile: + self.warnFile.flush() + if self.diagFile: + self.diagFile.flush() + self.outFile.flush() + if self.outFile != sys.stdout and self.outFile != sys.stderr: + self.outFile.close() + + # On successfully generating output, move the temporary file to the + # target file. + if self.genOpts.filename is not None: + if sys.platform == 'win32': + directory = Path(self.genOpts.directory) + if not Path.exists(directory): + os.makedirs(directory) + shutil.copy(self.outFile.name, self.genOpts.directory + '/' + self.genOpts.filename) + os.remove(self.outFile.name) + self.genOpts = None + + def beginFeature(self, interface, emit): + """Write interface for a feature and tag generated features as having been done. + + - interface - element for the `` / `` to generate + - emit - actually write to the header only when True""" + self.emit = emit + self.featureName = interface.get('name') + # If there's an additional 'protect' attribute in the feature, save it + self.featureExtraProtect = interface.get('protect') + + def endFeature(self): + """Finish an interface file, closing it when done. + + Derived classes responsible for emitting feature""" + self.featureName = None + self.featureExtraProtect = None + + def genRequirements(self, name, mustBeFound = True): + """Generate text showing what core versions and extensions introduce + an API. This exists in the base Generator class because it's used by + the shared enumerant-generating interfaces (buildEnumCDecl, etc.). + Here it returns an empty string for most generators, but can be + overridden by e.g. DocGenerator. + + - name - name of the API + - mustBeFound - If True, when requirements for 'name' cannot be + determined, a warning comment is generated. + """ + + return '' + + def validateFeature(self, featureType, featureName): + """Validate we're generating something only inside a `` tag""" + if self.featureName is None: + raise UserWarning('Attempt to generate', featureType, + featureName, 'when not in feature') + + def genType(self, typeinfo, name, alias): + """Generate interface for a type + + - typeinfo - TypeInfo for a type + + Extend to generate as desired in your derived class.""" + self.validateFeature('type', name) + + def genStruct(self, typeinfo, typeName, alias): + """Generate interface for a C "struct" type. + + - typeinfo - TypeInfo for a type interpreted as a struct + + Extend to generate as desired in your derived class.""" + self.validateFeature('struct', typeName) + + # The mixed-mode tags may contain no-op tags. + # It is convenient to remove them here where all output generators + # will benefit. + for member in typeinfo.elem.findall('.//member'): + for comment in member.findall('comment'): + member.remove(comment) + + def genGroup(self, groupinfo, groupName, alias): + """Generate interface for a group of enums (C "enum") + + - groupinfo - GroupInfo for a group. + + Extend to generate as desired in your derived class.""" + + self.validateFeature('group', groupName) + + def genEnum(self, enuminfo, typeName, alias): + """Generate interface for an enum (constant). + + - enuminfo - EnumInfo for an enum + - name - enum name + + Extend to generate as desired in your derived class.""" + self.validateFeature('enum', typeName) + + def genCmd(self, cmd, cmdinfo, alias): + """Generate interface for a command. + + - cmdinfo - CmdInfo for a command + + Extend to generate as desired in your derived class.""" + self.validateFeature('command', cmdinfo) + + def genSpirv(self, spirv, spirvinfo, alias): + """Generate interface for a spirv element. + + - spirvinfo - SpirvInfo for a command + + Extend to generate as desired in your derived class.""" + return + + def makeProtoName(self, name, tail): + """Turn a `` `` into C-language prototype + and typedef declarations for that name. + + - name - contents of `` tag + - tail - whatever text follows that tag in the Element""" + return self.genOpts.apientry + name + tail + + def makeTypedefName(self, name, tail): + """Make the function-pointer typedef name for a command.""" + return '(' + self.genOpts.apientryp + 'PFN_' + name + tail + ')' + + def makeCParamDecl(self, param, aligncol): + """Return a string which is an indented, formatted + declaration for a `` or `` block (e.g. function parameter + or structure/union member). + + - param - Element (`` or ``) to format + - aligncol - if non-zero, attempt to align the nested `` element + at this column""" + indent = ' ' + paramdecl = indent + prefix = noneStr(param.text) + + for elem in param: + text = noneStr(elem.text) + tail = noneStr(elem.tail) + + if self.should_insert_may_alias_macro and self.genOpts.conventions.is_voidpointer_alias(elem.tag, text, tail): + # OpenXR-specific macro insertion - but not in apiinc for the spec + tail = self.genOpts.conventions.make_voidpointer_alias(tail) + if elem.tag == 'name' and aligncol > 0: + self.logMsg('diag', 'Aligning parameter', elem.text, 'to column', self.genOpts.alignFuncParam) + # Align at specified column, if possible + paramdecl = paramdecl.rstrip() + oldLen = len(paramdecl) + # This works around a problem where very long type names - + # longer than the alignment column - would run into the tail + # text. + paramdecl = paramdecl.ljust(aligncol - 1) + ' ' + newLen = len(paramdecl) + self.logMsg('diag', 'Adjust length of parameter decl from', oldLen, 'to', newLen, ':', paramdecl) + + if (self.misracppstyle() and prefix.find('const ') != -1): + # Change pointer type order from e.g. "const void *" to "void const *". + # If the string starts with 'const', reorder it to be after the first type. + paramdecl += prefix.replace('const ', '') + text + ' const' + tail + else: + paramdecl += prefix + text + tail + + # Clear prefix for subsequent iterations + prefix = '' + + # If prefix was originally non-empty and the param has no elements + # (e.g. is nothing but text), preserve it. + paramdecl = paramdecl + prefix + + if aligncol == 0: + # Squeeze out multiple spaces other than the indentation + paramdecl = indent + ' '.join(paramdecl.split()) + return paramdecl + + def getCParamTypeLength(self, param): + """Return the length of the type field is an indented, formatted + declaration for a `` or `` block (e.g. function parameter + or structure/union member). + + - param - Element (`` or ``) to identify""" + + # Allow for missing tag + newLen = 0 + paramdecl = ' ' + noneStr(param.text) + for elem in param: + text = noneStr(elem.text) + tail = noneStr(elem.tail) + + if self.should_insert_may_alias_macro and self.genOpts.conventions.is_voidpointer_alias(elem.tag, text, tail): + # OpenXR-specific macro insertion + tail = self.genOpts.conventions.make_voidpointer_alias(tail) + if elem.tag == 'name': + # Align at specified column, if possible + newLen = len(paramdecl.rstrip()) + self.logMsg('diag', 'Identifying length of', elem.text, 'as', newLen) + paramdecl += text + tail + + return newLen + + def getMaxCParamTypeLength(self, info): + """Return the length of the longest type field for a member/parameter. + + - info - TypeInfo or CommandInfo. + """ + lengths = (self.getCParamTypeLength(member) + for member in info.getMembers()) + return max(lengths) + + def getHandleParent(self, typename): + """Get the parent of a handle object.""" + info = self.registry.typedict.get(typename) + if info is None: + return None + + elem = info.elem + if elem is not None: + return elem.get('parent') + + return None + + def iterateHandleAncestors(self, typename): + """Iterate through the ancestors of a handle type.""" + current = self.getHandleParent(typename) + while current is not None: + yield current + current = self.getHandleParent(current) + + def getHandleAncestors(self, typename): + """Get the ancestors of a handle object.""" + return list(self.iterateHandleAncestors(typename)) + + def getTypeCategory(self, typename): + """Get the category of a type.""" + info = self.registry.typedict.get(typename) + if info is None: + return None + + elem = info.elem + if elem is not None: + return elem.get('category') + return None + + def isStructAlwaysValid(self, structname): + """Try to do check if a structure is always considered valid (i.e. there's no rules to its acceptance).""" + # A conventions object is required for this call. + if not self.conventions: + raise RuntimeError("To use isStructAlwaysValid, be sure your options include a Conventions object.") + + if self.conventions.type_always_valid(structname): + return True + + category = self.getTypeCategory(structname) + if self.conventions.category_requires_validation(category): + return False + + info = self.registry.typedict.get(structname) + assert(info is not None) + + members = info.getMembers() + + for member in members: + member_name = getElemName(member) + if member_name in (self.conventions.structtype_member_name, + self.conventions.nextpointer_member_name): + return False + + if member.get('noautovalidity'): + return False + + member_type = getElemType(member) + + if member_type in ('void', 'char') or self.paramIsArray(member) or self.paramIsPointer(member): + return False + + if self.conventions.type_always_valid(member_type): + continue + + member_category = self.getTypeCategory(member_type) + + if self.conventions.category_requires_validation(member_category): + return False + + if member_category in ('struct', 'union'): + if self.isStructAlwaysValid(member_type) is False: + return False + + return True + + def isEnumRequired(self, elem): + """Return True if this `` element is + required, False otherwise + + - elem - `` element to test""" + required = elem.get('required') is not None + self.logMsg('diag', 'isEnumRequired:', elem.get('name'), + '->', required) + return required + + # @@@ This code is overridden by equivalent code now run in + # @@@ Registry.generateFeature + + required = False + + extname = elem.get('extname') + if extname is not None: + # 'supported' attribute was injected when the element was + # moved into the group in Registry.parseTree() + if self.genOpts.defaultExtensions == elem.get('supported'): + required = True + elif re.match(self.genOpts.addExtensions, extname) is not None: + required = True + elif elem.get('version') is not None: + required = re.match(self.genOpts.emitversions, elem.get('version')) is not None + else: + required = True + + return required + + def makeCDecls(self, cmd): + """Return C prototype and function pointer typedef for a + `` Element, as a two-element list of strings. + + - cmd - Element containing a `` tag""" + proto = cmd.find('proto') + params = cmd.findall('param') + # Begin accumulating prototype and typedef strings + pdecl = self.genOpts.apicall + tdecl = 'typedef ' + + # Insert the function return type/name. + # For prototypes, add APIENTRY macro before the name + # For typedefs, add (APIENTRY *) around the name and + # use the PFN_cmdnameproc naming convention. + # Done by walking the tree for element by element. + # etree has elem.text followed by (elem[i], elem[i].tail) + # for each child element and any following text + # Leading text + pdecl += noneStr(proto.text) + tdecl += noneStr(proto.text) + # For each child element, if it's a wrap in appropriate + # declaration. Otherwise append its contents and tail contents. + for elem in proto: + text = noneStr(elem.text) + tail = noneStr(elem.tail) + if elem.tag == 'name': + pdecl += self.makeProtoName(text, tail) + tdecl += self.makeTypedefName(text, tail) + else: + pdecl += text + tail + tdecl += text + tail + + if self.genOpts.alignFuncParam == 0: + # Squeeze out multiple spaces - there is no indentation + pdecl = ' '.join(pdecl.split()) + tdecl = ' '.join(tdecl.split()) + + # Now add the parameter declaration list, which is identical + # for prototypes and typedefs. Concatenate all the text from + # a node without the tags. No tree walking required + # since all tags are ignored. + # Uses: self.indentFuncProto + # self.indentFuncPointer + # self.alignFuncParam + n = len(params) + # Indented parameters + if n > 0: + indentdecl = '(\n' + indentdecl += ',\n'.join(self.makeCParamDecl(p, self.genOpts.alignFuncParam) + for p in params) + indentdecl += ');' + else: + indentdecl = '(void);' + # Non-indented parameters + paramdecl = '(' + if n > 0: + paramnames = [] + if self.misracppstyle(): + for p in params: + param = '' + firstIter = True; + for t in p.itertext(): + if (firstIter): + prefix = t + firstIter = False + else: + # Change pointer type order from e.g. "const void *" to "void const *". + # If the string starts with 'const', reorder it to be after the first type. + if (prefix.find('const ') != -1): + param += prefix.replace('const ', '') + t + ' const ' + else: + param += prefix + t + # Clear prefix for subsequent iterations + prefix = '' + paramnames.append(param); + else: + paramnames = (''.join(t for t in p.itertext()) + for p in params) + paramdecl += ', '.join(paramnames) + else: + paramdecl += 'void' + paramdecl += ");" + return [pdecl + indentdecl, tdecl + paramdecl] + + def newline(self): + """Print a newline to the output file (utility function)""" + write('', file=self.outFile) + + def setRegistry(self, registry): + self.registry = registry diff --git a/xml/pygenerator.py b/xml/pygenerator.py new file mode 100644 index 000000000..b2e76e66d --- /dev/null +++ b/xml/pygenerator.py @@ -0,0 +1,365 @@ +#!/usr/bin/python3 -i +# +# Copyright 2013-2023 The Khronos Group Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +import sys +from generator import OutputGenerator, enquote, noneStr, write +import pprint + +class PyOutputGenerator(OutputGenerator): + """PyOutputGenerator - subclass of OutputGenerator. + Generates Python data structures describing API names and relationships. + Similar to DocOutputGenerator, but writes a single file.""" + + def apiName(self, name): + """Return True if name is in the reserved API namespace. + + Delegates to the conventions object. """ + return self.genOpts.conventions.is_api_name(name) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Track features being generated + self.features = [] + + # Reverse map from interface names to features requiring them + self.apimap = {} + + def beginFile(self, genOpts): + OutputGenerator.beginFile(self, genOpts) + # + # Dictionaries are keyed by the name of the entity (e.g. + # self.structs is keyed by structure names). Values are + # the names of related entities (e.g. structs contain + # a list of type names of members, enums contain a list + # of enumerants belong to the enumerated type, etc.), or + # just None if there are no directly related entities. + # + # Collect the mappings, then emit the Python script in endFile + self.basetypes = {} + self.consts = {} + self.enums = {} + self.flags = {} + self.funcpointers = {} + self.protos = {} + self.structs = {} + self.handles = {} + self.defines = {} + self.alias = {} + # Dictionary containing the type of a type name + # (e.g. the string name of the dictionary with its contents). + self.typeCategory = {} + self.mapDict = {} + + def addInterfaceMapping(self, api, feature, required): + """Add a reverse mapping in self.apimap from an API to a feature + requiring that API. + + - api - name of the API + - feature - name of the feature requiring it + - required - None, or an additional feature dependency within + 'feature' """ + + # Each entry in self.apimap contains one or more + # ( feature, required ) tuples. + deps = ( feature, required ) + + if api in self.apimap: + self.apimap[api].append(deps) + else: + self.apimap[api] = [ deps ] + + def mapInterfaceKeys(self, feature, key): + """Construct reverse mapping of APIs to features requiring them in + self.apimap. + + - feature - name of the feature being generated + - key - API category - 'define', 'basetype', etc.""" + + dict = self.featureDictionary[feature][key] + + if dict: + # Not clear why handling of command vs. type APIs is different - + # see interfacedocgenerator.py, which this was based on. + if key == 'command': + for required in dict: + for api in dict[required]: + self.addInterfaceMapping(api, feature, required) + else: + for required in dict: + for parent in dict[required]: + for api in dict[required][parent]: + self.addInterfaceMapping(api, feature, required) + + def mapInterfaces(self, feature): + """Construct reverse mapping of APIs to features requiring them in + self.apimap. + + - feature - name of the feature being generated""" + + # Map each category of interface + self.mapInterfaceKeys(feature, 'basetype') + self.mapInterfaceKeys(feature, 'bitmask') + self.mapInterfaceKeys(feature, 'command') + self.mapInterfaceKeys(feature, 'define') + self.mapInterfaceKeys(feature, 'enum') + self.mapInterfaceKeys(feature, 'enumconstant') + self.mapInterfaceKeys(feature, 'funcpointer') + self.mapInterfaceKeys(feature, 'handle') + self.mapInterfaceKeys(feature, 'include') + self.mapInterfaceKeys(feature, 'struct') + self.mapInterfaceKeys(feature, 'union') + + def endFile(self): + # Print out all the dictionaries as Python strings. + # Could just print(dict) but that's not human-readable + dicts = ( [ self.basetypes, 'basetypes' ], + [ self.consts, 'consts' ], + [ self.enums, 'enums' ], + [ self.flags, 'flags' ], + [ self.funcpointers, 'funcpointers' ], + [ self.protos, 'protos' ], + [ self.structs, 'structs' ], + [ self.handles, 'handles' ], + [ self.defines, 'defines' ], + [ self.typeCategory, 'typeCategory' ], + [ self.alias, 'alias' ] ) + for (entry_dict, name) in dicts: + write(name + ' = {}', file=self.outFile) + for key in sorted(entry_dict.keys()): + write(name + '[' + enquote(key) + '] = ', entry_dict[key], + file=self.outFile) + + # Dictionary containing the relationships of a type + # (e.g. a dictionary with each related type as keys). + write('mapDict = {}', file=self.outFile) + + # Could just print(self.mapDict), but prefer something + # human-readable and stable-ordered + for baseType in sorted(self.mapDict.keys()): + write('mapDict[' + enquote(baseType) + '] = ', file=self.outFile, end='') + pprint.pprint(self.mapDict[baseType], self.outFile) + + # Generate feature <-> interface mappings + for feature in self.features: + self.mapInterfaces(feature) + + # Write out the reverse map from APIs to requiring features + write('requiredBy = {}', file=self.outFile) + + for api in sorted(self.apimap): + # Construct list of requirements as Python list arguments + ##reqs = ', '.join('({}, {})'.format(enquote(dep[0]), enquote(dep[1])) for dep in self.apimap[api]) + ##write('requiredBy[{}] = ( {} )'.format(enquote(api), reqs), file=self.outFile) + + # Ideally these would be sorted by dep[0] as well + reqs = ', '.join('({}, {})'.format(enquote(dep[0]), enquote(dep[1])) for dep in self.apimap[api]) + write('requiredBy[{}] = {}'.format(enquote(api), pprint.saferepr(self.apimap[api])), file=self.outFile) + + OutputGenerator.endFile(self) + + def beginFeature(self, interface, emit): + # Start processing in superclass + OutputGenerator.beginFeature(self, interface, emit) + + # Add this feature to the list being tracked + self.features.append( self.featureName ) + + def endFeature(self): + # Finish processing in superclass + OutputGenerator.endFeature(self) + + def addName(self, entry_dict, name, value): + """Add a string entry to the dictionary, quoting it so it gets printed + out correctly in self.endFile().""" + entry_dict[name] = enquote(value) + + def addMapping(self, baseType, refType): + """Add a mapping between types to mapDict. + + Only include API types, so we don't end up with a lot of useless uint32_t and void types.""" + if not self.apiName(baseType) or not self.apiName(refType): + self.logMsg('diag', 'PyOutputGenerator::addMapping: IGNORE map from', baseType, '<->', refType) + return + + self.logMsg('diag', 'PyOutputGenerator::addMapping: map from', + baseType, '<->', refType) + + if baseType not in self.mapDict: + baseDict = {} + self.mapDict[baseType] = baseDict + else: + baseDict = self.mapDict[baseType] + if refType not in self.mapDict: + refDict = {} + self.mapDict[refType] = refDict + else: + refDict = self.mapDict[refType] + + baseDict[refType] = None + refDict[baseType] = None + + def genType(self, typeinfo, name, alias): + """Generate type. + + - For 'struct' or 'union' types, defer to genStruct() to + add to the dictionary. + - For 'bitmask' types, add the type name to the 'flags' dictionary, + with the value being the corresponding 'enums' name defining + the acceptable flag bits. + - For 'enum' types, add the type name to the 'enums' dictionary, + with the value being '@STOPHERE@' (because this case seems + never to happen). + - For 'funcpointer' types, add the type name to the 'funcpointers' + dictionary. + - For 'handle' and 'define' types, add the handle or #define name + to the 'struct' dictionary, because that's how the spec sources + tag these types even though they aren't structs.""" + OutputGenerator.genType(self, typeinfo, name, alias) + typeElem = typeinfo.elem + # If the type is a struct type, traverse the embedded tags + # generating a structure. Otherwise, emit the tag text. + category = typeElem.get('category') + + # Add a typeCategory{} entry for the category of this type. + self.addName(self.typeCategory, name, category) + + if category in ('struct', 'union'): + self.genStruct(typeinfo, name, alias) + else: + if alias: + # Add name -> alias mapping + self.addName(self.alias, name, alias) + + # Always emit an alias (?!) + count = 1 + + # May want to only emit full type definition when not an alias? + else: + # Extract the type name + # (from self.genOpts). Copy other text through unchanged. + # If the resulting text is an empty string, don't emit it. + count = len(noneStr(typeElem.text)) + for elem in typeElem: + count += len(noneStr(elem.text)) + len(noneStr(elem.tail)) + + if count > 0: + if category == 'bitmask': + requiredEnum = typeElem.get('requires') + self.addName(self.flags, name, requiredEnum) + + # This happens when the Flags type is defined, but no + # FlagBits are defined yet. + if requiredEnum is not None: + self.addMapping(name, requiredEnum) + elif category == 'enum': + # This case does not seem to come up. It nominally would + # result from + # , + # but the output generator doesn't emit them directly. + self.logMsg('warn', 'PyOutputGenerator::genType: invalid \'enum\' category for name:', name) + elif category == 'funcpointer': + self.funcpointers[name] = None + elif category == 'handle': + self.handles[name] = None + elif category == 'define': + self.defines[name] = None + elif category == 'basetype': + # Don't add an entry for base types that are not API types + # e.g. an API Bool type gets an entry, uint32_t does not + if self.apiName(name): + self.basetypes[name] = None + self.addName(self.typeCategory, name, 'basetype') + else: + self.logMsg('diag', 'PyOutputGenerator::genType: unprocessed type:', name, 'category:', category) + else: + self.logMsg('diag', 'PyOutputGenerator::genType: unprocessed type:', name) + + def genStruct(self, typeinfo, typeName, alias): + """Generate struct (e.g. C "struct" type). + + Add the struct name to the 'structs' dictionary, with the + value being an ordered list of the struct member names.""" + OutputGenerator.genStruct(self, typeinfo, typeName, alias) + + if alias: + # Add name -> alias mapping + self.addName(self.alias, typeName, alias) + else: + # May want to only emit definition on this branch + True + + members = [member.text for member in typeinfo.elem.findall('.//member/name')] + self.structs[typeName] = members + memberTypes = [member.text for member in typeinfo.elem.findall('.//member/type')] + for member_type in memberTypes: + self.addMapping(typeName, member_type) + + def genGroup(self, groupinfo, groupName, alias): + """Generate group (e.g. C "enum" type). + + These are concatenated together with other types. + + - Add the enum type name to the 'enums' dictionary, with + the value being an ordered list of the enumerant names. + - Add each enumerant name to the 'consts' dictionary, with + the value being the enum type the enumerant is part of.""" + OutputGenerator.genGroup(self, groupinfo, groupName, alias) + groupElem = groupinfo.elem + + if alias: + # Add name -> alias mapping + self.addName(self.alias, groupName, alias) + else: + # May want to only emit definition on this branch + True + + # Loop over the nested 'enum' tags. + enumerants = [elem.get('name') for elem in groupElem.findall('enum')] + for name in enumerants: + self.addName(self.consts, name, groupName) + self.enums[groupName] = enumerants + + def genEnum(self, enuminfo, name, alias): + """Generate enumerant (compile-time constants). + + - Add the constant name to the 'consts' dictionary, with the + value being None to indicate that the constant isn't + an enumeration value.""" + OutputGenerator.genEnum(self, enuminfo, name, alias) + + if name not in self.consts: + # Add a typeCategory{} entry for the category of this type. + self.addName(self.typeCategory, name, 'consts') + self.consts[name] = None + # Otherwise, don't add it to the consts dictionary because it's + # already present. This happens due to the generator 'reparentEnums' + # parameter being False, so each extension enum appears in both the + # type and in the or it originally + # came from. + + def genCmd(self, cmdinfo, name, alias): + """Generate command. + + - Add the command name to the 'protos' dictionary, with the + value being an ordered list of the parameter names.""" + OutputGenerator.genCmd(self, cmdinfo, name, alias) + + if alias: + # Add name -> alias mapping + self.addName(self.alias, name, alias) + else: + # May want to only emit definition on this branch + True + + # Add a typeCategory{} entry for the category of this type. + self.addName(self.typeCategory, name, 'protos') + + params = [param.text for param in cmdinfo.elem.findall('param/name')] + self.protos[name] = params + paramTypes = [param.text for param in cmdinfo.elem.findall('param/type')] + for param_type in paramTypes: + self.addMapping(name, param_type) diff --git a/xml/realign.py b/xml/realign.py new file mode 100644 index 000000000..b59865b3d --- /dev/null +++ b/xml/realign.py @@ -0,0 +1,47 @@ +#!/usr/bin/python3 +# +# Copyright 2013-2023 The Khronos Group Inc. +# SPDX-License-Identifier: Apache-2.0 + +# Usage: realign [infile] > outfile +# Used to realign XML tags in the Vulkan registry after it's operated on by +# some other filter, since whitespace inside a tag isn't part of the +# internal representation. + +import copy, sys, string, re + +def realignXML(fp): + patterns = [ + [ '(^ *\ 1): + realignXML(open(sys.argv[1], 'r', encoding='utf-8')) + else: + realignXML(sys.stdin) diff --git a/xml/reflib.py b/xml/reflib.py new file mode 100644 index 000000000..426a1811b --- /dev/null +++ b/xml/reflib.py @@ -0,0 +1,663 @@ +#!/usr/bin/python3 +# +# Copyright 2016-2023 The Khronos Group Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +# Utility functions for automatic ref page generation and other script stuff + +import io +import re +import sys +import subprocess + +# global errFile, warnFile, diagFile + +errFile = sys.stderr +warnFile = sys.stdout +diagFile = None +logSourcefile = None +logProcname = None +logLine = None + +def unescapeQuotes(s): + """Remove \' escape sequences in a string (refpage description)""" + return s.replace('\\\'', '\'') + +def write(*args, **kwargs ): + file = kwargs.pop('file',sys.stdout) + end = kwargs.pop('end','\n') + file.write(' '.join(str(arg) for arg in args)) + file.write(end) + +def setLogSourcefile(filename): + """Metadata which may be printed (if not None) for diagnostic messages""" + global logSourcefile + logSourcefile = filename + +def setLogProcname(procname): + global logProcname + logProcname = procname + +def setLogLine(line): + global logLine + logLine = line + +def logHeader(severity): + """Generate prefix for a diagnostic line using metadata and severity""" + global logSourcefile, logProcname, logLine + + msg = severity + ': ' + if logProcname: + msg = msg + ' in ' + logProcname + if logSourcefile: + msg = msg + ' for ' + logSourcefile + if logLine: + msg = msg + ' line ' + str(logLine) + return msg + ' ' + +def setLogFile(setDiag, setWarn, filename): + """Set the file handle to log either or both warnings and diagnostics to. + + - setDiag and setWarn are True if the corresponding handle is to be set. + - filename is None for no logging, '-' for stdout, or a pathname.""" + global diagFile, warnFile + + if filename is None: + return + + if filename == '-': + fp = sys.stdout + else: + fp = open(filename, 'w', encoding='utf-8') + + if setDiag: + diagFile = fp + if setWarn: + warnFile = fp + +def logDiag(*args, **kwargs): + file = kwargs.pop('file', diagFile) + end = kwargs.pop('end','\n') + if file is not None: + file.write(logHeader('DIAG') + ' '.join(str(arg) for arg in args)) + file.write(end) + +def logWarn(*args, **kwargs): + file = kwargs.pop('file', warnFile) + end = kwargs.pop('end','\n') + if file is not None: + file.write(logHeader('WARN') + ' '.join(str(arg) for arg in args)) + file.write(end) + +def logErr(*args, **kwargs): + file = kwargs.pop('file', errFile) + end = kwargs.pop('end','\n') + + strfile = io.StringIO() + strfile.write(logHeader('ERROR') + ' '.join(str(arg) for arg in args)) + strfile.write(end) + + if file is not None: + file.write(strfile.getvalue()) + sys.exit(1) + +def isempty(s): + """Return True if s is nothing but white space, False otherwise""" + return len(''.join(s.split())) == 0 + +class pageInfo: + """Information about a ref page relative to the file it's extracted from.""" + def __init__(self): + self.extractPage = True + """True if page should be extracted""" + + self.Warning = None + """string warning if page is suboptimal or can't be generated""" + + self.embed = False + """False or the name of the ref page this include is embedded within""" + + self.type = None + """'structs', 'protos', 'funcpointers', 'flags', 'enums'""" + + self.name = None + """struct/proto/enumerant/etc. name""" + + self.desc = None + """short description of ref page""" + + self.begin = None + """index of first line of the page (heuristic or // refBegin)""" + + self.include = None + """index of include:: line defining the page""" + + self.param = None + """index of first line of parameter/member definitions""" + + self.body = None + """index of first line of body text""" + + self.validity = None + """index of validity include""" + + self.end = None + """index of last line of the page (heuristic validity include, or // refEnd)""" + + self.alias = '' + """aliases of this name, if supplied, or ''""" + + self.refs = '' + """cross-references on // refEnd line, if supplied""" + + self.spec = None + """'spec' attribute in refpage open block, if supplied, or None for the default ('api') type""" + + self.anchor = None + """'anchor' attribute in refpage open block, if supplied, or inferred to be the same as the 'name'""" + +def printPageInfoField(desc, line, file): + """Print a single field of a pageInfo struct, possibly None. + + - desc - string description of field + - line - field value or None + - file - indexed by line""" + if line is not None: + logDiag(desc + ':', line + 1, '\t-> ', file[line], end='') + else: + logDiag(desc + ':', line) + +def printPageInfo(pi, file): + """Print out fields of a pageInfo struct + + - pi - pageInfo + - file - indexed by pageInfo""" + logDiag('TYPE: ', pi.type) + logDiag('NAME: ', pi.name) + logDiag('WARNING:', pi.Warning) + logDiag('EXTRACT:', pi.extractPage) + logDiag('EMBED: ', pi.embed) + logDiag('DESC: ', pi.desc) + printPageInfoField('BEGIN ', pi.begin, file) + printPageInfoField('INCLUDE ', pi.include, file) + printPageInfoField('PARAM ', pi.param, file) + printPageInfoField('BODY ', pi.body, file) + printPageInfoField('VALIDITY', pi.validity, file) + printPageInfoField('END ', pi.end, file) + logDiag('REFS: "' + pi.refs + '"') + +def prevPara(file, line): + """Go back one paragraph from the specified line and return the line number + of the first line of that paragraph. + + Paragraphs are delimited by blank lines. It is assumed that the + current line is the first line of a paragraph. + + - file is an array of strings + - line is the starting point (zero-based)""" + # Skip over current paragraph + while (line >= 0 and not isempty(file[line])): + line = line - 1 + # Skip over white space + while (line >= 0 and isempty(file[line])): + line = line - 1 + # Skip to first line of previous paragraph + while (line >= 1 and not isempty(file[line-1])): + line = line - 1 + return line + +def nextPara(file, line): + """Go forward one paragraph from the specified line and return the line + number of the first line of that paragraph. + + Paragraphs are delimited by blank lines. It is assumed that the + current line is standalone (which is bogus). + + - file is an array of strings + - line is the starting point (zero-based)""" + maxLine = len(file) - 1 + # Skip over current paragraph + while (line != maxLine and not isempty(file[line])): + line = line + 1 + # Skip over white space + while (line != maxLine and isempty(file[line])): + line = line + 1 + return line + +def lookupPage(pageMap, name): + """Return (creating if needed) the pageInfo entry in pageMap for name""" + if name not in pageMap: + pi = pageInfo() + pi.name = name + pageMap[name] = pi + else: + pi = pageMap[name] + return pi + +def loadFile(filename): + """Load a file into a list of strings. Return the list or None on failure""" + try: + fp = open(filename, 'r', encoding='utf-8') + except: + logWarn('Cannot open file', filename, ':', sys.exc_info()[0]) + return None + + file = fp.readlines() + fp.close() + + return file + +def clampToBlock(line, minline, maxline): + """Clamp a line number to be in the range [minline,maxline]. + + If the line number is None, just return it. + If minline is None, don't clamp to that value.""" + if line is None: + return line + if minline and line < minline: + return minline + if line > maxline: + return maxline + + return line + +def fixupRefs(pageMap, specFile, file): + """Fill in missing fields in pageInfo structures, to the extent they can be + inferred. + + - pageMap - dictionary of pageInfo structures + - specFile - filename + - file - list of strings making up the file, indexed by pageInfo""" + # All potential ref pages are now in pageMap. Process them to + # identify actual page start/end/description boundaries, if + # not already determined from the text. + for name in sorted(pageMap.keys()): + pi = pageMap[name] + + # # If nothing is found but an include line with no begin, validity, + # # or end, this is not intended as a ref page (yet). Set the begin + # # line to the include line, so autogeneration can at least + # # pull the include out, but mark it not to be extracted. + # # Examples include the host sync table includes in + # # chapters/fundamentals.txt and the table of Vk*Flag types in + # # appendices/boilerplate.txt. + # if pi.begin is None and pi.validity is None and pi.end is None: + # pi.begin = pi.include + # pi.extractPage = False + # pi.Warning = 'No begin, validity, or end lines identified' + # continue + + # Using open block delimiters, ref pages must *always* have a + # defined begin and end. If either is undefined, that's fatal. + if pi.begin is None: + pi.extractPage = False + pi.Warning = 'Can\'t identify begin of ref page open block' + continue + + if pi.end is None: + pi.extractPage = False + pi.Warning = 'Can\'t identify end of ref page open block' + continue + + # If there's no description of the page, infer one from the type + if pi.desc is None: + if pi.type is not None: + # pi.desc = pi.type[0:len(pi.type)-1] + ' (no short description available)' + pi.Warning = 'No short description available; could infer from the type and name' + else: + pi.extractPage = False + pi.Warning = 'No short description available, cannot infer from the type' + continue + + # Try to determine where the parameter and body sections of the page + # begin. funcpointer, proto, and struct pages infer the location of + # the parameter and body sections. Other pages infer the location of + # the body, but have no parameter sections. + if pi.include is not None: + if pi.type in ['funcpointers', 'protos', 'structs']: + pi.param = nextPara(file, pi.include) + if pi.body is None: + pi.body = nextPara(file, pi.param) + else: + if pi.body is None: + pi.body = nextPara(file, pi.include) + else: + pi.Warning = 'Page does not have an API definition include::' + + # It's possible for the inferred param and body lines to run past + # the end of block, if, for example, there is no parameter section. + pi.param = clampToBlock(pi.param, pi.include, pi.end) + pi.body = clampToBlock(pi.body, pi.param, pi.end) + + # We can get to this point with .include, .param, and .validity + # all being None, indicating those sections weren't found. + + logDiag('fixupRefs: after processing,', pi.name, 'looks like:') + printPageInfo(pi, file) + + # Now that all the valid pages have been found, try to make some + # inferences about invalid pages. + # + # If a reference without a .end is entirely inside a valid reference, + # then it's intentionally embedded - may want to create an indirect + # page that links into the embedding page. This is done by a very + # inefficient double loop, but the loop depth is small. + for name in sorted(pageMap.keys()): + pi = pageMap[name] + + if pi.end is None: + for embedName in sorted(pageMap.keys()): + logDiag('fixupRefs: comparing', pi.name, 'to', embedName) + embed = pageMap[embedName] + # Don't check embeddings which are themselves invalid + if not embed.extractPage: + logDiag('Skipping check for embedding in:', embed.name) + continue + if embed.begin is None or embed.end is None: + logDiag('fixupRefs:', name + ':', + 'can\'t compare to unanchored ref:', embed.name, + 'in', specFile, 'at line', pi.include ) + printPageInfo(pi, file) + printPageInfo(embed, file) + # If an embed is found, change the error to a warning + elif (pi.include is not None and pi.include >= embed.begin and + pi.include <= embed.end): + logDiag('fixupRefs: Found embed for:', name, + 'inside:', embedName, + 'in', specFile, 'at line', pi.include ) + pi.embed = embed.name + pi.Warning = 'Embedded in definition for ' + embed.name + break + else: + logDiag('fixupRefs: No embed match for:', name, + 'inside:', embedName, 'in', specFile, + 'at line', pi.include) + + +# Patterns used to recognize interesting lines in an asciidoc source file. +# These patterns are only compiled once. +INCSVAR_DEF = re.compile(r':INCS-VAR: (?P.*)') +endifPat = re.compile(r'^endif::(?P[\w_+,]+)\[\]') +beginPat = re.compile(r'^\[open,(?Prefpage=.*)\]') +# attribute key/value pairs of an open block +attribStr = r"([a-z]+)='([^'\\]*(?:\\.[^'\\]*)*)'" +attribPat = re.compile(attribStr) +bodyPat = re.compile(r'^// *refBody') +errorPat = re.compile(r'^// *refError') + +# This regex transplanted from check_spec_links +# It looks for either OpenXR or Vulkan generated file conventions, and for +# the api/validity include (generated_type), protos/struct/etc path +# (category), and API name (entity_name). It could be put into the API +# conventions object. +INCLUDE = re.compile( + r'include::(?P((../){1,4}|\{INCS-VAR\}/|\{generated\}/)(generated/)?)(?P[\w]+)/(?P\w+)/(?P[^./]+).txt[\[][\]]') + + +def findRefs(file, filename): + """Identify reference pages in a list of strings, returning a dictionary of + pageInfo entries for each one found, or None on failure.""" + setLogSourcefile(filename) + setLogProcname('findRefs') + + # To reliably detect the open blocks around reference pages, we must + # first detect the '[open,refpage=...]' markup delimiting the block; + # skip past the '--' block delimiter on the next line; and identify the + # '--' block delimiter closing the page. + # This can't be done solely with pattern matching, and requires state to + # track 'inside/outside block'. + # When looking for open blocks, possible states are: + # 'outside' - outside a block + # 'start' - have found the '[open...]' line + # 'inside' - have found the following '--' line + openBlockState = 'outside' + + # Dictionary of interesting line numbers and strings related to an API + # name + pageMap = {} + + numLines = len(file) + line = 0 + + # Track the pageInfo object corresponding to the current open block + pi = None + incsvar = None + + while (line < numLines): + setLogLine(line) + + # Look for a file-wide definition + matches = INCSVAR_DEF.match(file[line]) + if matches: + incsvar = matches.group('value') + logDiag('Matched INCS-VAR definition:', incsvar) + + line = line + 1 + continue + + # Perform INCS-VAR substitution immediately. + if incsvar and '{INCS-VAR}' in file[line]: + newLine = file[line].replace('{INCS-VAR}', incsvar) + logDiag('PERFORMING SUBSTITUTION', file[line], '->', newLine) + file[line] = newLine + + # Only one of the patterns can possibly match. Add it to + # the dictionary for that name. + + # [open,refpage=...] starting a refpage block + matches = beginPat.search(file[line]) + if matches is not None: + logDiag('Matched open block pattern') + attribs = matches.group('attribs') + + # If the previous open block wasn't closed, raise an error + if openBlockState != 'outside': + logErr('Nested open block starting at line', line, 'of', + filename) + + openBlockState = 'start' + + # Parse the block attributes + matches = attribPat.findall(attribs) + + # Extract each attribute + name = None + desc = None + refpage_type = None + spec_type = None + anchor = None + alias = None + xrefs = None + + for (key,value) in matches: + logDiag('got attribute', key, '=', value) + if key == 'refpage': + name = value + elif key == 'desc': + desc = unescapeQuotes(value) + elif key == 'type': + refpage_type = value + elif key == 'spec': + spec_type = value + elif key == 'anchor': + anchor = value + elif key == 'alias': + alias = value + elif key == 'xrefs': + xrefs = value + else: + logWarn('unknown open block attribute:', key) + + if name is None or desc is None or refpage_type is None: + logWarn('missing one or more required open block attributes:' + 'refpage, desc, or type') + # Leave pi is None so open block delimiters are ignored + else: + pi = lookupPage(pageMap, name) + pi.desc = desc + # Must match later type definitions in interface/validity includes + pi.type = refpage_type + pi.spec = spec_type + pi.anchor = anchor + if alias: + pi.alias = alias + if xrefs: + pi.refs = xrefs + logDiag('open block for', name, 'added DESC =', desc, + 'TYPE =', refpage_type, 'ALIAS =', alias, + 'XREFS =', xrefs, 'SPEC =', spec_type, + 'ANCHOR =', anchor) + + line = line + 1 + continue + + # '--' starting or ending and open block + if file[line].rstrip() == '--': + if openBlockState == 'outside': + # Only refpage open blocks should use -- delimiters + logWarn('Unexpected double-dash block delimiters') + elif openBlockState == 'start': + # -- delimiter following [open,refpage=...] + openBlockState = 'inside' + + if pi is None: + logWarn('no pageInfo available for opening -- delimiter') + else: + pi.begin = line + 1 + logDiag('opening -- delimiter: added BEGIN =', pi.begin) + elif openBlockState == 'inside': + # -- delimiter ending an open block + if pi is None: + logWarn('no pageInfo available for closing -- delimiter') + else: + pi.end = line - 1 + logDiag('closing -- delimiter: added END =', pi.end) + + openBlockState = 'outside' + pi = None + else: + logWarn('unknown openBlockState:', openBlockState) + + line = line + 1 + continue + + matches = INCLUDE.search(file[line]) + if matches is not None: + # Something got included, not sure what yet. + gen_type = matches.group('generated_type') + refpage_type = matches.group('category') + name = matches.group('entity_name') + + # This will never match in OpenCL + if gen_type == 'validity': + logDiag('Matched validity pattern') + if pi is not None: + if pi.type and refpage_type != pi.type: + logWarn('ERROR: pageMap[' + name + '] type:', + pi.type, 'does not match type:', refpage_type) + pi.type = refpage_type + pi.validity = line + logDiag('added TYPE =', pi.type, 'VALIDITY =', pi.validity) + else: + logWarn('validity include:: line NOT inside block') + + line = line + 1 + continue + + if gen_type == 'api': + logDiag('Matched include pattern') + if pi is not None: + if pi.include is not None: + logDiag('found multiple includes for this block') + if pi.type and refpage_type != pi.type: + logWarn('ERROR: pageMap[' + name + '] type:', + pi.type, 'does not match type:', refpage_type) + pi.type = refpage_type + pi.include = line + logDiag('added TYPE =', pi.type, 'INCLUDE =', pi.include) + else: + logWarn('interface include:: line NOT inside block') + + line = line + 1 + continue + + logDiag('ignoring unrecognized include line ', matches.group()) + + # Vulkan 1.1 markup allows the last API include construct to be + # followed by an asciidoctor endif:: construct (and also preceded, + # at some distance). + # This looks for endif:: immediately following an include:: line + # and, if found, moves the include boundary to this line. + matches = endifPat.search(file[line]) + if matches is not None and pi is not None: + if pi.include == line - 1: + logDiag('Matched endif pattern following include; moving include') + pi.include = line + else: + logDiag('Matched endif pattern (not following include)') + + line = line + 1 + continue + + matches = bodyPat.search(file[line]) + if matches is not None: + logDiag('Matched // refBody pattern') + if pi is not None: + pi.body = line + logDiag('added BODY =', pi.body) + else: + logWarn('// refBody line NOT inside block') + + line = line + 1 + continue + + # OpenCL spec uses // refError to tag "validity" (Errors) language, + # instead of /validity/ includes. + matches = errorPat.search(file[line]) + if matches is not None: + logDiag('Matched // refError pattern') + if pi is not None: + pi.validity = line + logDiag('added VALIDITY (refError) =', pi.validity) + else: + logWarn('// refError line NOT inside block') + + line = line + 1 + continue + + line = line + 1 + continue + + if pi is not None: + logErr('Unclosed open block at EOF!') + + setLogSourcefile(None) + setLogProcname(None) + setLogLine(None) + + return pageMap + + +def getBranch(): + """Determine current git branch + + Returns (branch name, ''), or (None, stderr output) if the branch name + can't be determined""" + + command = [ 'git', 'symbolic-ref', '--short', 'HEAD' ] + results = subprocess.run(command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + # git command failed + if len(results.stderr) > 0: + return (None, results.stderr) + + # Remove newline from output and convert to a string + branch = results.stdout.rstrip().decode() + if len(branch) > 0: + # Strip trailing newline + branch = results.stdout.decode()[0:-1] + + return (branch, '') diff --git a/xml/reg.py b/xml/reg.py new file mode 100644 index 000000000..d78ecde89 --- /dev/null +++ b/xml/reg.py @@ -0,0 +1,1397 @@ +#!/usr/bin/python3 -i +# +# Copyright 2013-2023 The Khronos Group Inc. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Types and classes for manipulating an API registry.""" + +import copy +import re +import sys +import xml.etree.ElementTree as etree +from collections import defaultdict, namedtuple +from generator import OutputGenerator, GeneratorOptions, write +import pdb + +def apiNameMatch(str, supported): + """Return whether a required api name matches a pattern specified for an + XML 'api' attribute or 'supported' attribute. + + - str - api name such as 'vulkan' or 'openxr' + - supported - comma-separated list of XML API names""" + + return (str is not None and str in supported.split(',')) + + +def matchAPIProfile(api, profile, elem): + """Return whether an API and profile + being generated matches an element's profile + + - api - string naming the API to match + - profile - string naming the profile to match + - elem - Element which (may) have 'api' and 'profile' + attributes to match to. + + If a tag is not present in the Element, the corresponding API + or profile always matches. + + Otherwise, the tag must exactly match the API or profile. + + Thus, if 'profile' = core: + + - `` with no attribute will match + - `` will match + - `` will not match + + Possible match conditions: + + ``` + Requested Element + Profile Profile + --------- -------- + None None Always matches + 'string' None Always matches + None 'string' Does not match. Can't generate multiple APIs + or profiles, so if an API/profile constraint + is present, it must be asked for explicitly. + 'string' 'string' Strings must match + ``` + + ** In the future, we will allow regexes for the attributes, + not just strings, so that `api="^(gl|gles2)"` will match. Even + this isn't really quite enough, we might prefer something + like `"gl(core)|gles1(common-lite)"`.""" + # Match 'api', if present + elem_api = elem.get('api') + if elem_api: + if api is None: + raise UserWarning("No API requested, but 'api' attribute is present with value '" + + elem_api + "'") + elif api != elem_api: + # Requested API doesn't match attribute + return False + elem_profile = elem.get('profile') + if elem_profile: + if profile is None: + raise UserWarning("No profile requested, but 'profile' attribute is present with value '" + + elem_profile + "'") + elif profile != elem_profile: + # Requested profile doesn't match attribute + return False + return True + + +class BaseInfo: + """Base class for information about a registry feature + (type/group/enum/command/API/extension). + + Represents the state of a registry feature, used during API generation. + """ + + def __init__(self, elem): + self.required = False + """should this feature be defined during header generation + (has it been removed by a profile or version)?""" + + self.declared = False + "has this feature been defined already?" + + self.elem = elem + "etree Element for this feature" + + def resetState(self): + """Reset required/declared to initial values. Used + prior to generating a new API interface.""" + self.required = False + self.declared = False + + def compareKeys(self, info, key, required = False): + """Return True if self.elem and info.elem have the same attribute + value for key. + If 'required' is not True, also returns True if neither element + has an attribute value for key.""" + + if required and key not in self.elem.keys(): + return False + return self.elem.get(key) == info.elem.get(key) + + def compareElem(self, info, infoName): + """Return True if self.elem and info.elem have the same definition. + info - the other object + infoName - 'type' / 'group' / 'enum' / 'command' / 'feature' / + 'extension'""" + + if infoName == 'enum': + if self.compareKeys(info, 'extends'): + # Either both extend the same type, or no type + if (self.compareKeys(info, 'value', required = True) or + self.compareKeys(info, 'bitpos', required = True)): + # If both specify the same value or bit position, + # they're equal + return True + elif (self.compareKeys(info, 'extnumber') and + self.compareKeys(info, 'offset') and + self.compareKeys(info, 'dir')): + # If both specify the same relative offset, they're equal + return True + elif (self.compareKeys(info, 'alias')): + # If both are aliases of the same value + return True + else: + return False + else: + # The same enum can't extend two different types + return False + else: + # Non-s should never be redefined + return False + + +class TypeInfo(BaseInfo): + """Registry information about a type. No additional state + beyond BaseInfo is required.""" + + def __init__(self, elem): + BaseInfo.__init__(self, elem) + self.additionalValidity = [] + self.removedValidity = [] + + def getMembers(self): + """Get a collection of all member elements for this type, if any.""" + return self.elem.findall('member') + + def resetState(self): + BaseInfo.resetState(self) + self.additionalValidity = [] + self.removedValidity = [] + + +class GroupInfo(BaseInfo): + """Registry information about a group of related enums + in an block, generally corresponding to a C "enum" type.""" + + def __init__(self, elem): + BaseInfo.__init__(self, elem) + + +class EnumInfo(BaseInfo): + """Registry information about an enum""" + + def __init__(self, elem): + BaseInfo.__init__(self, elem) + self.type = elem.get('type') + """numeric type of the value of the tag + ( '' for GLint, 'u' for GLuint, 'ull' for GLuint64 )""" + if self.type is None: + self.type = '' + + +class CmdInfo(BaseInfo): + """Registry information about a command""" + + def __init__(self, elem): + BaseInfo.__init__(self, elem) + self.additionalValidity = [] + self.removedValidity = [] + + def getParams(self): + """Get a collection of all param elements for this command, if any.""" + return self.elem.findall('param') + + def resetState(self): + BaseInfo.resetState(self) + self.additionalValidity = [] + self.removedValidity = [] + + +class FeatureInfo(BaseInfo): + """Registry information about an API + or .""" + + def __init__(self, elem): + BaseInfo.__init__(self, elem) + self.name = elem.get('name') + "feature name string (e.g. 'VK_KHR_surface')" + + self.emit = False + "has this feature been defined already?" + + self.sortorder = int(elem.get('sortorder', 0)) + """explicit numeric sort key within feature and extension groups. + Defaults to 0.""" + + # Determine element category (vendor). Only works + # for elements. + if elem.tag == 'feature': + # Element category (vendor) is meaningless for + self.category = 'VERSION' + """category, e.g. VERSION or khr/vendor tag""" + + self.version = elem.get('name') + """feature name string""" + + self.versionNumber = elem.get('number') + """versionNumber - API version number, taken from the 'number' + attribute of . Extensions do not have API version + numbers and are assigned number 0.""" + + self.number = "0" + self.supported = None + else: + # Extract vendor portion of __ + self.category = self.name.split('_', 2)[1] + self.version = "0" + self.versionNumber = "0" + self.number = elem.get('number') + """extension number, used for ordering and for assigning + enumerant offsets. features do not have extension + numbers and are assigned number 0.""" + + # If there's no 'number' attribute, use 0, so sorting works + if self.number is None: + self.number = 0 + self.supported = elem.get('supported') + +class SpirvInfo(BaseInfo): + """Registry information about an API + or .""" + + def __init__(self, elem): + BaseInfo.__init__(self, elem) + +class Registry: + """Object representing an API registry, loaded from an XML file.""" + + def __init__(self, gen=None, genOpts=None): + if gen is None: + # If not specified, give a default object so messaging will work + self.gen = OutputGenerator() + else: + self.gen = gen + "Output generator used to write headers / messages" + + if genOpts is None: + self.genOpts = GeneratorOptions() + else: + self.genOpts = genOpts + "Options controlling features to write and how to format them" + + self.gen.registry = self + self.gen.genOpts = self.genOpts + self.gen.genOpts.registry = self + + self.tree = None + "ElementTree containing the root ``" + + self.typedict = {} + "dictionary of TypeInfo objects keyed by type name" + + self.groupdict = {} + "dictionary of GroupInfo objects keyed by group name" + + self.enumdict = {} + "dictionary of EnumInfo objects keyed by enum name" + + self.cmddict = {} + "dictionary of CmdInfo objects keyed by command name" + + self.apidict = {} + "dictionary of FeatureInfo objects for `` elements keyed by API name" + + self.extensions = [] + "list of `` Elements" + + self.extdict = {} + "dictionary of FeatureInfo objects for `` elements keyed by extension name" + + self.spirvextdict = {} + "dictionary of FeatureInfo objects for `` elements keyed by spirv extension name" + + self.spirvcapdict = {} + "dictionary of FeatureInfo objects for `` elements keyed by spirv capability name" + + self.emitFeatures = False + """True to actually emit features for a version / extension, + or False to just treat them as emitted""" + + self.breakPat = None + "regexp pattern to break on when generating names" + # self.breakPat = re.compile('VkFenceImportFlagBits.*') + + self.requiredextensions = [] # Hack - can remove it after validity generator goes away + + # ** Global types for automatic source generation ** + # Length Member data + self.commandextensiontuple = namedtuple('commandextensiontuple', + ['command', # The name of the command being modified + 'value', # The value to append to the command + 'extension']) # The name of the extension that added it + self.validextensionstructs = defaultdict(list) + self.commandextensionsuccesses = [] + self.commandextensionerrors = [] + + self.filename = None + + def loadElementTree(self, tree): + """Load ElementTree into a Registry object and parse it.""" + self.tree = tree + self.parseTree() + + def loadFile(self, file): + """Load an API registry XML file into a Registry object and parse it""" + self.filename = file + self.tree = etree.parse(file) + self.parseTree() + + def setGenerator(self, gen): + """Specify output generator object. + + `None` restores the default generator.""" + self.gen = gen + self.gen.setRegistry(self) + + def addElementInfo(self, elem, info, infoName, dictionary): + """Add information about an element to the corresponding dictionary. + + Intended for internal use only. + + - elem - ``/``/``/``/``/``/``/`` Element + - info - corresponding {Type|Group|Enum|Cmd|Feature|Spirv}Info object + - infoName - 'type' / 'group' / 'enum' / 'command' / 'feature' / 'extension' / 'spirvextension' / 'spirvcapability' + - dictionary - self.{type|group|enum|cmd|api|ext|spirvext|spirvcap}dict + + If the Element has an 'api' attribute, the dictionary key is the + tuple (name,api). If not, the key is the name. 'name' is an + attribute of the Element""" + # self.gen.logMsg('diag', 'Adding ElementInfo.required =', + # info.required, 'name =', elem.get('name')) + api = elem.get('api') + if api: + key = (elem.get('name'), api) + else: + key = elem.get('name') + if key in dictionary: + if not dictionary[key].compareElem(info, infoName): + self.gen.logMsg('warn', 'Attempt to redefine', key, + '(this should not happen)') + else: + True + else: + dictionary[key] = info + + def lookupElementInfo(self, fname, dictionary): + """Find a {Type|Enum|Cmd}Info object by name. + + Intended for internal use only. + + If an object qualified by API name exists, use that. + + - fname - name of type / enum / command + - dictionary - self.{type|enum|cmd}dict""" + key = (fname, self.genOpts.apiname) + if key in dictionary: + # self.gen.logMsg('diag', 'Found API-specific element for feature', fname) + return dictionary[key] + if fname in dictionary: + # self.gen.logMsg('diag', 'Found generic element for feature', fname) + return dictionary[fname] + + return None + + def breakOnName(self, regexp): + """Specify a feature name regexp to break on when generating features.""" + self.breakPat = re.compile(regexp) + + def parseTree(self): + """Parse the registry Element, once created""" + # This must be the Element for the root + self.reg = self.tree.getroot() + + # Create dictionary of registry types from toplevel tags + # and add 'name' attribute to each tag (where missing) + # based on its element. + # + # There's usually one block; more are OK + # Required attributes: 'name' or nested tag contents + self.typedict = {} + for type_elem in self.reg.findall('types/type'): + # If the doesn't already have a 'name' attribute, set + # it from contents of its tag. + if type_elem.get('name') is None: + type_elem.set('name', type_elem.find('name').text) + self.addElementInfo(type_elem, TypeInfo(type_elem), 'type', self.typedict) + + # Create dictionary of registry enum groups from tags. + # + # Required attributes: 'name'. If no name is given, one is + # generated, but that group can't be identified and turned into an + # enum type definition - it's just a container for tags. + self.groupdict = {} + for group in self.reg.findall('enums'): + self.addElementInfo(group, GroupInfo(group), 'group', self.groupdict) + + # Create dictionary of registry enums from tags + # + # tags usually define different namespaces for the values + # defined in those tags, but the actual names all share the + # same dictionary. + # Required attributes: 'name', 'value' + # For containing which have type="enum" or type="bitmask", + # tag all contained s are required. This is a stopgap until + # a better scheme for tagging core and extension enums is created. + self.enumdict = {} + for enums in self.reg.findall('enums'): + required = (enums.get('type') is not None) + for enum in enums.findall('enum'): + enumInfo = EnumInfo(enum) + enumInfo.required = required + self.addElementInfo(enum, enumInfo, 'enum', self.enumdict) + + # Create dictionary of registry commands from tags + # and add 'name' attribute to each tag (where missing) + # based on its element. + # + # There's usually only one block; more are OK. + # Required attributes: 'name' or tag contents + self.cmddict = {} + # List of commands which alias others. Contains + # [ aliasName, element ] + # for each alias + cmdAlias = [] + for cmd in self.reg.findall('commands/command'): + # If the doesn't already have a 'name' attribute, set + # it from contents of its tag. + name = cmd.get('name') + if name is None: + name = cmd.set('name', cmd.find('proto/name').text) + ci = CmdInfo(cmd) + self.addElementInfo(cmd, ci, 'command', self.cmddict) + alias = cmd.get('alias') + if alias: + cmdAlias.append([name, alias, cmd]) + + # Now loop over aliases, injecting a copy of the aliased command's + # Element with the aliased prototype name replaced with the command + # name - if it exists. + for (name, alias, cmd) in cmdAlias: + if alias in self.cmddict: + aliasInfo = self.cmddict[alias] + cmdElem = copy.deepcopy(aliasInfo.elem) + cmdElem.find('proto/name').text = name + cmdElem.set('name', name) + cmdElem.set('alias', alias) + ci = CmdInfo(cmdElem) + # Replace the dictionary entry for the CmdInfo element + self.cmddict[name] = ci + + # @ newString = etree.tostring(base, encoding="unicode").replace(aliasValue, aliasName) + # @elem.append(etree.fromstring(replacement)) + else: + self.gen.logMsg('warn', 'No matching found for command', + cmd.get('name'), 'alias', alias) + + # Create dictionaries of API and extension interfaces + # from toplevel and tags. + self.apidict = {} + for feature in self.reg.findall('feature'): + featureInfo = FeatureInfo(feature) + self.addElementInfo(feature, featureInfo, 'feature', self.apidict) + + # Add additional enums defined only in tags + # to the corresponding enumerated type. + # When seen here, the element, processed to contain the + # numeric enum value, is added to the corresponding + # element, as well as adding to the enum dictionary. It is no + # longer removed from the element it is introduced in. + # Instead, generateRequiredInterface ignores elements + # that extend enumerated types. + # + # For tags which are actually just constants, if there's + # no 'extends' tag but there is a 'value' or 'bitpos' tag, just + # add an EnumInfo record to the dictionary. That works because + # output generation of constants is purely dependency-based, and + # doesn't need to iterate through the XML tags. + for elem in feature.findall('require'): + for enum in elem.findall('enum'): + addEnumInfo = False + groupName = enum.get('extends') + if groupName is not None: + # self.gen.logMsg('diag', 'Found extension enum', + # enum.get('name')) + # Add version number attribute to the element + enum.set('version', featureInfo.version) + # Look up the GroupInfo with matching groupName + if groupName in self.groupdict: + # self.gen.logMsg('diag', 'Matching group', + # groupName, 'found, adding element...') + gi = self.groupdict[groupName] + gi.elem.append(copy.deepcopy(enum)) + else: + self.gen.logMsg('warn', 'NO matching group', + groupName, 'for enum', enum.get('name'), 'found.') + addEnumInfo = True + elif enum.get('value') or enum.get('bitpos') or enum.get('alias'): + # self.gen.logMsg('diag', 'Adding extension constant "enum"', + # enum.get('name')) + addEnumInfo = True + if addEnumInfo: + enumInfo = EnumInfo(enum) + self.addElementInfo(enum, enumInfo, 'enum', self.enumdict) + + self.extensions = self.reg.findall('extensions/extension') + self.extdict = {} + for feature in self.extensions: + featureInfo = FeatureInfo(feature) + self.addElementInfo(feature, featureInfo, 'extension', self.extdict) + + # Add additional enums defined only in tags + # to the corresponding core type. + # Algorithm matches that of enums in a "feature" tag as above. + # + # This code also adds a 'extnumber' attribute containing the + # extension number, used for enumerant value calculation. + for elem in feature.findall('require'): + for enum in elem.findall('enum'): + addEnumInfo = False + groupName = enum.get('extends') + if groupName is not None: + # self.gen.logMsg('diag', 'Found extension enum', + # enum.get('name')) + + # Add block's extension number attribute to + # the element unless specified explicitly, such + # as when redefining an enum in another extension. + extnumber = enum.get('extnumber') + if not extnumber: + enum.set('extnumber', featureInfo.number) + + enum.set('extname', featureInfo.name) + enum.set('supported', featureInfo.supported) + # Look up the GroupInfo with matching groupName + if groupName in self.groupdict: + # self.gen.logMsg('diag', 'Matching group', + # groupName, 'found, adding element...') + gi = self.groupdict[groupName] + gi.elem.append(copy.deepcopy(enum)) + else: + self.gen.logMsg('warn', 'NO matching group', + groupName, 'for enum', enum.get('name'), 'found.') + addEnumInfo = True + elif enum.get('value') or enum.get('bitpos') or enum.get('alias'): + # self.gen.logMsg('diag', 'Adding extension constant "enum"', + # enum.get('name')) + addEnumInfo = True + if addEnumInfo: + enumInfo = EnumInfo(enum) + self.addElementInfo(enum, enumInfo, 'enum', self.enumdict) + + # Construct a "validextensionstructs" list for parent structures + # based on "structextends" tags in child structures + disabled_types = [] + for disabled_ext in self.reg.findall('extensions/extension[@supported="disabled"]'): + for type_elem in disabled_ext.findall("*/type"): + disabled_types.append(type_elem.get('name')) + for type_elem in self.reg.findall('types/type'): + if type_elem.get('name') not in disabled_types: + parentStructs = type_elem.get('structextends') + if parentStructs is not None: + for parent in parentStructs.split(','): + # self.gen.logMsg('diag', type.get('name'), 'extends', parent) + self.validextensionstructs[parent].append(type_elem.get('name')) + # Sort the lists so they don't depend on the XML order + for parent in self.validextensionstructs: + self.validextensionstructs[parent].sort() + + # Parse out all spirv tags in dictionaries + # Use addElementInfo to catch duplicates + for spirv in self.reg.findall('spirvextensions/spirvextension'): + spirvInfo = SpirvInfo(spirv) + self.addElementInfo(spirv, spirvInfo, 'spirvextension', self.spirvextdict) + for spirv in self.reg.findall('spirvcapabilities/spirvcapability'): + spirvInfo = SpirvInfo(spirv) + self.addElementInfo(spirv, spirvInfo, 'spirvcapability', self.spirvcapdict) + + def dumpReg(self, maxlen=120, filehandle=sys.stdout): + """Dump all the dictionaries constructed from the Registry object. + + Diagnostic to dump the dictionaries to specified file handle (default stdout). + Truncates type / enum / command elements to maxlen characters (default 120)""" + write('***************************************', file=filehandle) + write(' ** Dumping Registry contents **', file=filehandle) + write('***************************************', file=filehandle) + write('// Types', file=filehandle) + for name in self.typedict: + tobj = self.typedict[name] + write(' Type', name, '->', etree.tostring(tobj.elem)[0:maxlen], file=filehandle) + write('// Groups', file=filehandle) + for name in self.groupdict: + gobj = self.groupdict[name] + write(' Group', name, '->', etree.tostring(gobj.elem)[0:maxlen], file=filehandle) + write('// Enums', file=filehandle) + for name in self.enumdict: + eobj = self.enumdict[name] + write(' Enum', name, '->', etree.tostring(eobj.elem)[0:maxlen], file=filehandle) + write('// Commands', file=filehandle) + for name in self.cmddict: + cobj = self.cmddict[name] + write(' Command', name, '->', etree.tostring(cobj.elem)[0:maxlen], file=filehandle) + write('// APIs', file=filehandle) + for key in self.apidict: + write(' API Version ', key, '->', + etree.tostring(self.apidict[key].elem)[0:maxlen], file=filehandle) + write('// Extensions', file=filehandle) + for key in self.extdict: + write(' Extension', key, '->', + etree.tostring(self.extdict[key].elem)[0:maxlen], file=filehandle) + write('// SPIR-V', file=filehandle) + for key in self.spirvextdict: + write(' SPIR-V Extension', key, '->', + etree.tostring(self.spirvextdict[key].elem)[0:maxlen], file=filehandle) + for key in self.spirvcapdict: + write(' SPIR-V Capability', key, '->', + etree.tostring(self.spirvcapdict[key].elem)[0:maxlen], file=filehandle) + + def markTypeRequired(self, typename, required): + """Require (along with its dependencies) or remove (but not its dependencies) a type. + + - typename - name of type + - required - boolean (to tag features as required or not) + """ + self.gen.logMsg('diag', 'tagging type:', typename, '-> required =', required) + # Get TypeInfo object for tag corresponding to typename + typeinfo = self.lookupElementInfo(typename, self.typedict) + if typeinfo is not None: + if required: + # Tag type dependencies in 'alias' and 'required' attributes as + # required. This does not un-tag dependencies in a + # tag. See comments in markRequired() below for the reason. + for attrib_name in ['requires', 'alias']: + depname = typeinfo.elem.get(attrib_name) + if depname: + self.gen.logMsg('diag', 'Generating dependent type', + depname, 'for', attrib_name, 'type', typename) + # Don't recurse on self-referential structures. + if typename != depname: + self.markTypeRequired(depname, required) + else: + self.gen.logMsg('diag', 'type', typename, 'is self-referential') + # Tag types used in defining this type (e.g. in nested + # tags) + # Look for in entire tree, + # not just immediate children + for subtype in typeinfo.elem.findall('.//type'): + self.gen.logMsg('diag', 'markRequired: type requires dependent ', subtype.text) + if typename != subtype.text: + self.markTypeRequired(subtype.text, required) + else: + self.gen.logMsg('diag', 'type', typename, 'is self-referential') + # Tag enums used in defining this type, for example in + # member[MEMBER_SIZE] + for subenum in typeinfo.elem.findall('.//enum'): + self.gen.logMsg('diag', 'markRequired: type requires dependent ', subenum.text) + self.markEnumRequired(subenum.text, required) + # Tag type dependency in 'bitvalues' attributes as + # required. This ensures that the bit values for a flag + # are emitted + depType = typeinfo.elem.get('bitvalues') + if depType: + self.gen.logMsg('diag', 'Generating bitflag type', + depType, 'for type', typename) + self.markTypeRequired(depType, required) + group = self.lookupElementInfo(depType, self.groupdict) + if group is not None: + group.flagType = typeinfo + + typeinfo.required = required + elif '.h' not in typename: + self.gen.logMsg('warn', 'type:', typename, 'IS NOT DEFINED') + + def markEnumRequired(self, enumname, required): + """Mark an enum as required or not. + + - enumname - name of enum + - required - boolean (to tag features as required or not)""" + + self.gen.logMsg('diag', 'tagging enum:', enumname, '-> required =', required) + enum = self.lookupElementInfo(enumname, self.enumdict) + if enum is not None: + # If the enum is part of a group, and is being removed, then + # look it up in that tag and remove it there, so that it + # isn't visible to generators (which traverse the tag + # elements themselves). + # This isn't the most robust way of doing this, since a removed + # enum that's later required again will no longer have a group + # element, but it makes the change non-intrusive on generator + # code. + if required is False: + groupName = enum.elem.get('extends') + if groupName is not None: + # Look up the Info with matching groupName + if groupName in self.groupdict: + gi = self.groupdict[groupName] + gienum = gi.elem.find("enum[@name='" + enumname + "']") + if gienum is not None: + # Remove copy of this enum from the group + gi.elem.remove(gienum) + else: + self.gen.logMsg('warn', 'Cannot remove enum', + enumname, 'not found in group', + groupName) + else: + self.gen.logMsg('warn', 'Cannot remove enum', + enumname, 'from nonexistent group', + groupName) + + enum.required = required + # Tag enum dependencies in 'alias' attribute as required + depname = enum.elem.get('alias') + if depname: + self.gen.logMsg('diag', 'Generating dependent enum', + depname, 'for alias', enumname, 'required =', enum.required) + self.markEnumRequired(depname, required) + else: + self.gen.logMsg('warn', 'enum:', enumname, 'IS NOT DEFINED') + + def markCmdRequired(self, cmdname, required): + """Mark a command as required or not. + + - cmdname - name of command + - required - boolean (to tag features as required or not)""" + self.gen.logMsg('diag', 'tagging command:', cmdname, '-> required =', required) + cmd = self.lookupElementInfo(cmdname, self.cmddict) + if cmd is not None: + cmd.required = required + # Tag command dependencies in 'alias' attribute as required + depname = cmd.elem.get('alias') + if depname: + self.gen.logMsg('diag', 'Generating dependent command', + depname, 'for alias', cmdname) + self.markCmdRequired(depname, required) + # Tag all parameter types of this command as required. + # This DOES NOT remove types of commands in a + # tag, because many other commands may use the same type. + # We could be more clever and reference count types, + # instead of using a boolean. + if required: + # Look for in entire tree, + # not just immediate children + for type_elem in cmd.elem.findall('.//type'): + self.gen.logMsg('diag', 'markRequired: command implicitly requires dependent type', type_elem.text) + self.markTypeRequired(type_elem.text, required) + else: + self.gen.logMsg('warn', 'command:', cmdname, 'IS NOT DEFINED') + + def markRequired(self, featurename, feature, required): + """Require or remove features specified in the Element. + + - featurename - name of the feature + - feature - Element for `` or `` tag + - required - boolean (to tag features as required or not)""" + self.gen.logMsg('diag', 'markRequired (feature = , required =', required, ')') + + # Loop over types, enums, and commands in the tag + # @@ It would be possible to respect 'api' and 'profile' attributes + # in individual features, but that's not done yet. + for typeElem in feature.findall('type'): + self.markTypeRequired(typeElem.get('name'), required) + for enumElem in feature.findall('enum'): + self.markEnumRequired(enumElem.get('name'), required) + for cmdElem in feature.findall('command'): + self.markCmdRequired(cmdElem.get('name'), required) + + # Extensions may need to extend existing commands or other items in the future. + # So, look for extend tags. + for extendElem in feature.findall('extend'): + extendType = extendElem.get('type') + if extendType == 'command': + commandName = extendElem.get('name') + successExtends = extendElem.get('successcodes') + if successExtends is not None: + for success in successExtends.split(','): + self.commandextensionsuccesses.append(self.commandextensiontuple(command=commandName, + value=success, + extension=featurename)) + errorExtends = extendElem.get('errorcodes') + if errorExtends is not None: + for error in errorExtends.split(','): + self.commandextensionerrors.append(self.commandextensiontuple(command=commandName, + value=error, + extension=featurename)) + else: + self.gen.logMsg('warn', 'extend type:', extendType, 'IS NOT SUPPORTED') + + def getAlias(self, elem, dict): + """Check for an alias in the same require block. + + - elem - Element to check for an alias""" + + # Try to find an alias + alias = elem.get('alias') + if alias is None: + name = elem.get('name') + typeinfo = self.lookupElementInfo(name, dict) + alias = typeinfo.elem.get('alias') + + return alias + + def checkForCorrectionAliases(self, alias, require, tag): + """Check for an alias in the same require block. + + - alias - String name of the alias + - require - `` block from the registry + - tag - tag to look for in the require block""" + + if alias and require.findall(tag + "[@name='" + alias + "']"): + return True + + return False + + def fillFeatureDictionary(self, interface, featurename, api, profile): + """Capture added interfaces for a `` or ``. + + - interface - Element for `` or ``, containing + `` and `` tags + - featurename - name of the feature + - api - string specifying API name being generated + - profile - string specifying API profile being generated""" + + # Explicitly initialize known types - errors for unhandled categories + self.gen.featureDictionary[featurename] = { + "enumconstant": {}, + "command": {}, + "enum": {}, + "struct": {}, + "handle": {}, + "basetype": {}, + "include": {}, + "define": {}, + "bitmask": {}, + "union": {}, + "funcpointer": {}, + } + + # marks things that are required by this version/profile + for require in interface.findall('require'): + if matchAPIProfile(api, profile, require): + + # Determine the required extension or version needed for a require block + # Assumes that only one of these is specified + required_key = require.get('feature') + if required_key is None: + required_key = require.get('extension') + + # Loop over types, enums, and commands in the tag + for typeElem in require.findall('type'): + typename = typeElem.get('name') + typeinfo = self.lookupElementInfo(typename, self.typedict) + + if typeinfo: + # Remove aliases in the same extension/feature; these are always added as a correction. Don't need the original to be visible. + alias = self.getAlias(typeElem, self.typedict) + if not self.checkForCorrectionAliases(alias, require, 'type'): + # Resolve the type info to the actual type, so we get an accurate read for 'structextends' + while alias: + typeinfo = self.lookupElementInfo(alias, self.typedict) + alias = typeinfo.elem.get('alias') + + typecat = typeinfo.elem.get('category') + typeextends = typeinfo.elem.get('structextends') + if not required_key in self.gen.featureDictionary[featurename][typecat]: + self.gen.featureDictionary[featurename][typecat][required_key] = {} + if not typeextends in self.gen.featureDictionary[featurename][typecat][required_key]: + self.gen.featureDictionary[featurename][typecat][required_key][typeextends] = [] + self.gen.featureDictionary[featurename][typecat][required_key][typeextends].append(typename) + + for enumElem in require.findall('enum'): + enumname = enumElem.get('name') + typeinfo = self.lookupElementInfo(enumname, self.enumdict) + + # Remove aliases in the same extension/feature; these are always added as a correction. Don't need the original to be visible. + alias = self.getAlias(enumElem, self.enumdict) + if not self.checkForCorrectionAliases(alias, require, 'enum'): + enumextends = enumElem.get('extends') + if not required_key in self.gen.featureDictionary[featurename]['enumconstant']: + self.gen.featureDictionary[featurename]['enumconstant'][required_key] = {} + if not enumextends in self.gen.featureDictionary[featurename]['enumconstant'][required_key]: + self.gen.featureDictionary[featurename]['enumconstant'][required_key][enumextends] = [] + self.gen.featureDictionary[featurename]['enumconstant'][required_key][enumextends].append(enumname) + + for cmdElem in require.findall('command'): + + # Remove aliases in the same extension/feature; these are always added as a correction. Don't need the original to be visible. + alias = self.getAlias(cmdElem, self.cmddict) + if not self.checkForCorrectionAliases(alias, require, 'command'): + if not required_key in self.gen.featureDictionary[featurename]['command']: + self.gen.featureDictionary[featurename]['command'][required_key] = [] + self.gen.featureDictionary[featurename]['command'][required_key].append(cmdElem.get('name')) + + + def requireAndRemoveFeatures(self, interface, featurename, api, profile): + """Process `` and `` tags for a `` or ``. + + - interface - Element for `` or ``, containing + `` and `` tags + - featurename - name of the feature + - api - string specifying API name being generated + - profile - string specifying API profile being generated""" + # marks things that are required by this version/profile + for feature in interface.findall('require'): + if matchAPIProfile(api, profile, feature): + self.markRequired(featurename, feature, True) + # marks things that are removed by this version/profile + for feature in interface.findall('remove'): + if matchAPIProfile(api, profile, feature): + self.markRequired(featurename, feature, False) + + def assignAdditionalValidity(self, interface, api, profile): + # Loop over all usage inside all tags. + for feature in interface.findall('require'): + if matchAPIProfile(api, profile, feature): + for v in feature.findall('usage'): + if v.get('command'): + self.cmddict[v.get('command')].additionalValidity.append(copy.deepcopy(v)) + if v.get('struct'): + self.typedict[v.get('struct')].additionalValidity.append(copy.deepcopy(v)) + + # Loop over all usage inside all tags. + for feature in interface.findall('remove'): + if matchAPIProfile(api, profile, feature): + for v in feature.findall('usage'): + if v.get('command'): + self.cmddict[v.get('command')].removedValidity.append(copy.deepcopy(v)) + if v.get('struct'): + self.typedict[v.get('struct')].removedValidity.append(copy.deepcopy(v)) + + def generateFeature(self, fname, ftype, dictionary): + """Generate a single type / enum group / enum / command, + and all its dependencies as needed. + + - fname - name of feature (``/``/``) + - ftype - type of feature, 'type' | 'enum' | 'command' + - dictionary - of *Info objects - self.{type|enum|cmd}dict""" + + self.gen.logMsg('diag', 'generateFeature: generating', ftype, fname) + f = self.lookupElementInfo(fname, dictionary) + if f is None: + # No such feature. This is an error, but reported earlier + self.gen.logMsg('diag', 'No entry found for feature', fname, + 'returning!') + return + + # If feature isn't required, or has already been declared, return + if not f.required: + self.gen.logMsg('diag', 'Skipping', ftype, fname, '(not required)') + return + if f.declared: + self.gen.logMsg('diag', 'Skipping', ftype, fname, '(already declared)') + return + # Always mark feature declared, as though actually emitted + f.declared = True + + # Determine if this is an alias, and of what, if so + alias = f.elem.get('alias') + if alias: + self.gen.logMsg('diag', fname, 'is an alias of', alias) + + # Pull in dependent declaration(s) of the feature. + # For types, there may be one type in the 'requires' attribute of + # the element, one in the 'alias' attribute, and many in + # embedded and tags within the element. + # For commands, there may be many in tags within the element. + # For enums, no dependencies are allowed (though perhaps if you + # have a uint64 enum, it should require that type). + genProc = None + followupFeature = None + if ftype == 'type': + genProc = self.gen.genType + + # Generate type dependencies in 'alias' and 'requires' attributes + if alias: + self.generateFeature(alias, 'type', self.typedict) + requires = f.elem.get('requires') + if requires: + self.gen.logMsg('diag', 'Generating required dependent type', + requires) + self.generateFeature(requires, 'type', self.typedict) + + # Generate types used in defining this type (e.g. in nested + # tags) + # Look for in entire tree, + # not just immediate children + for subtype in f.elem.findall('.//type'): + self.gen.logMsg('diag', 'Generating required dependent ', + subtype.text) + self.generateFeature(subtype.text, 'type', self.typedict) + + # Generate enums used in defining this type, for example in + # member[MEMBER_SIZE] + for subtype in f.elem.findall('.//enum'): + self.gen.logMsg('diag', 'Generating required dependent ', + subtype.text) + self.generateFeature(subtype.text, 'enum', self.enumdict) + + # If the type is an enum group, look up the corresponding + # group in the group dictionary and generate that instead. + if f.elem.get('category') == 'enum': + self.gen.logMsg('diag', 'Type', fname, 'is an enum group, so generate that instead') + group = self.lookupElementInfo(fname, self.groupdict) + if alias is not None: + # An alias of another group name. + # Pass to genGroup with 'alias' parameter = aliased name + self.gen.logMsg('diag', 'Generating alias', fname, + 'for enumerated type', alias) + # Now, pass the *aliased* GroupInfo to the genGroup, but + # with an additional parameter which is the alias name. + genProc = self.gen.genGroup + f = self.lookupElementInfo(alias, self.groupdict) + elif group is None: + self.gen.logMsg('warn', 'Skipping enum type', fname, + ': No matching enumerant group') + return + else: + genProc = self.gen.genGroup + f = group + + # @ The enum group is not ready for generation. At this + # @ point, it contains all tags injected by + # @ tags without any verification of whether + # @ they're required or not. It may also contain + # @ duplicates injected by multiple consistent + # @ definitions of an . + + # @ Pass over each enum, marking its enumdict[] entry as + # @ required or not. Mark aliases of enums as required, + # @ too. + + enums = group.elem.findall('enum') + + self.gen.logMsg('diag', 'generateFeature: checking enums for group', fname) + + # Check for required enums, including aliases + # LATER - Check for, report, and remove duplicates? + enumAliases = [] + for elem in enums: + name = elem.get('name') + + required = False + + extname = elem.get('extname') + version = elem.get('version') + if extname is not None: + # 'supported' attribute was injected when the element was + # moved into the group in Registry.parseTree() + if self.genOpts.defaultExtensions == elem.get('supported'): + required = True + elif re.match(self.genOpts.addExtensions, extname) is not None: + required = True + elif version is not None: + required = re.match(self.genOpts.emitversions, version) is not None + else: + required = True + + self.gen.logMsg('diag', '* required =', required, 'for', name) + if required: + # Mark this element as required (in the element, not the EnumInfo) + elem.set('required', 'true') + # If it's an alias, track that for later use + enumAlias = elem.get('alias') + if enumAlias: + enumAliases.append(enumAlias) + for elem in enums: + name = elem.get('name') + if name in enumAliases: + elem.set('required', 'true') + self.gen.logMsg('diag', '* also need to require alias', name) + if f.elem.get('category') == 'bitmask': + followupFeature = f.elem.get('bitvalues') + elif ftype == 'command': + # Generate command dependencies in 'alias' attribute + if alias: + self.generateFeature(alias, 'command', self.cmddict) + + genProc = self.gen.genCmd + for type_elem in f.elem.findall('.//type'): + depname = type_elem.text + self.gen.logMsg('diag', 'Generating required parameter type', + depname) + self.generateFeature(depname, 'type', self.typedict) + elif ftype == 'enum': + # Generate enum dependencies in 'alias' attribute + if alias: + self.generateFeature(alias, 'enum', self.enumdict) + genProc = self.gen.genEnum + + # Actually generate the type only if emitting declarations + if self.emitFeatures: + self.gen.logMsg('diag', 'Emitting', ftype, 'decl for', fname) + genProc(f, fname, alias) + else: + self.gen.logMsg('diag', 'Skipping', ftype, fname, + '(should not be emitted)') + + if followupFeature: + self.gen.logMsg('diag', 'Generating required bitvalues ', + followupFeature) + self.generateFeature(followupFeature, "type", self.typedict) + + def generateRequiredInterface(self, interface): + """Generate all interfaces required by an API version or extension. + + - interface - Element for `` or ``""" + + # Loop over all features inside all tags. + for features in interface.findall('require'): + for t in features.findall('type'): + self.generateFeature(t.get('name'), 'type', self.typedict) + for e in features.findall('enum'): + # If this is an enum extending an enumerated type, don't + # generate it - this has already been done in reg.parseTree, + # by copying this element into the enumerated type. + enumextends = e.get('extends') + if not enumextends: + self.generateFeature(e.get('name'), 'enum', self.enumdict) + for c in features.findall('command'): + self.generateFeature(c.get('name'), 'command', self.cmddict) + + def generateSpirv(self, spirv, dictionary): + if spirv is None: + self.gen.logMsg('diag', 'No entry found for element', name, + 'returning!') + return + + name = spirv.elem.get('name') + # No known alias for spirv elements + alias = None + if spirv.emit: + genProc = self.gen.genSpirv + genProc(spirv, name, alias) + + def apiGen(self): + """Generate interface for specified versions using the current + generator and generator options""" + + self.gen.logMsg('diag', '*******************************************') + self.gen.logMsg('diag', ' Registry.apiGen file:', self.genOpts.filename, + 'api:', self.genOpts.apiname, + 'profile:', self.genOpts.profile) + self.gen.logMsg('diag', '*******************************************') + + # Reset required/declared flags for all features + self.apiReset() + + # Compile regexps used to select versions & extensions + regVersions = re.compile(self.genOpts.versions) + regEmitVersions = re.compile(self.genOpts.emitversions) + regAddExtensions = re.compile(self.genOpts.addExtensions) + regRemoveExtensions = re.compile(self.genOpts.removeExtensions) + regEmitExtensions = re.compile(self.genOpts.emitExtensions) + regEmitSpirv = re.compile(self.genOpts.emitSpirv) + + # Get all matching API feature names & add to list of FeatureInfo + # Note we used to select on feature version attributes, not names. + features = [] + apiMatch = False + for key in self.apidict: + fi = self.apidict[key] + api = fi.elem.get('api') + if apiNameMatch(self.genOpts.apiname, api): + apiMatch = True + if regVersions.match(fi.name): + # Matches API & version #s being generated. Mark for + # emission and add to the features[] list . + # @@ Could use 'declared' instead of 'emit'? + fi.emit = (regEmitVersions.match(fi.name) is not None) + features.append(fi) + if not fi.emit: + self.gen.logMsg('diag', 'NOT tagging feature api =', api, + 'name =', fi.name, 'version =', fi.version, + 'for emission (does not match emitversions pattern)') + else: + self.gen.logMsg('diag', 'Including feature api =', api, + 'name =', fi.name, 'version =', fi.version, + 'for emission (matches emitversions pattern)') + else: + self.gen.logMsg('diag', 'NOT including feature api =', api, + 'name =', fi.name, 'version =', fi.version, + '(does not match requested versions)') + else: + self.gen.logMsg('diag', 'NOT including feature api =', api, + 'name =', fi.name, + '(does not match requested API)') + if not apiMatch: + self.gen.logMsg('warn', 'No matching API versions found!') + + # Get all matching extensions, in order by their extension number, + # and add to the list of features. + # Start with extensions tagged with 'api' pattern matching the API + # being generated. Add extensions matching the pattern specified in + # regExtensions, then remove extensions matching the pattern + # specified in regRemoveExtensions + for (extName, ei) in sorted(self.extdict.items(), key=lambda x: x[1].number if x[1].number is not None else '0'): + extName = ei.name + include = False + + # Include extension if defaultExtensions is not None and is + # exactly matched by the 'supported' attribute. + if apiNameMatch(self.genOpts.defaultExtensions, + ei.elem.get('supported')): + self.gen.logMsg('diag', 'Including extension', + extName, "(defaultExtensions matches the 'supported' attribute)") + include = True + + # Include additional extensions if the extension name matches + # the regexp specified in the generator options. This allows + # forcing extensions into an interface even if they're not + # tagged appropriately in the registry. + # However we still respect the 'supported' attribute. + if regAddExtensions.match(extName) is not None: + if not apiNameMatch(self.genOpts.apiname, ei.elem.get('supported')): + self.gen.logMsg('diag', 'NOT including extension', + extName, '(matches explicitly requested, but does not match the \'supported\' attribute)') + include = False + else: + self.gen.logMsg('diag', 'Including extension', + extName, '(matches explicitly requested extensions to add)') + include = True + # Remove extensions if the name matches the regexp specified + # in generator options. This allows forcing removal of + # extensions from an interface even if they're tagged that + # way in the registry. + if regRemoveExtensions.match(extName) is not None: + self.gen.logMsg('diag', 'Removing extension', + extName, '(matches explicitly requested extensions to remove)') + include = False + + # If the extension is to be included, add it to the + # extension features list. + if include: + ei.emit = (regEmitExtensions.match(extName) is not None) + features.append(ei) + if not ei.emit: + self.gen.logMsg('diag', 'NOT tagging extension', + extName, + 'for emission (does not match emitextensions pattern)') + + # Hack - can be removed when validity generator goes away + # (Jon) I'm not sure what this does, or if it should respect + # the ei.emit flag above. + self.requiredextensions.append(extName) + else: + self.gen.logMsg('diag', 'NOT including extension', + extName, '(does not match api attribute or explicitly requested extensions)') + + # Add all spirv elements to list + # generators decide to emit them all or not + # Currently no filtering as no client of these elements needs filtering + spirvexts = [] + for key in self.spirvextdict: + si = self.spirvextdict[key] + si.emit = (regEmitSpirv.match(key) is not None) + spirvexts.append(si) + spirvcaps = [] + for key in self.spirvcapdict: + si = self.spirvcapdict[key] + si.emit = (regEmitSpirv.match(key) is not None) + spirvcaps.append(si) + + # Sort the features list, if a sort procedure is defined + if self.genOpts.sortProcedure: + self.genOpts.sortProcedure(features) + # print('sortProcedure ->', [f.name for f in features]) + + # Pass 1: loop over requested API versions and extensions tagging + # types/commands/features as required (in an block) or no + # longer required (in an block). It is possible to remove + # a feature in one version and restore it later by requiring it in + # a later version. + # If a profile other than 'None' is being generated, it must + # match the profile attribute (if any) of the and + # tags. + self.gen.logMsg('diag', 'PASS 1: TAG FEATURES') + for f in features: + self.gen.logMsg('diag', 'PASS 1: Tagging required and removed features for', + f.name) + self.fillFeatureDictionary(f.elem, f.name, self.genOpts.apiname, self.genOpts.profile) + self.requireAndRemoveFeatures(f.elem, f.name, self.genOpts.apiname, self.genOpts.profile) + self.assignAdditionalValidity(f.elem, self.genOpts.apiname, self.genOpts.profile) + + # Pass 2: loop over specified API versions and extensions printing + # declarations for required things which haven't already been + # generated. + self.gen.logMsg('diag', 'PASS 2: GENERATE INTERFACES FOR FEATURES') + self.gen.beginFile(self.genOpts) + for f in features: + self.gen.logMsg('diag', 'PASS 2: Generating interface for', + f.name) + emit = self.emitFeatures = f.emit + if not emit: + self.gen.logMsg('diag', 'PASS 2: NOT declaring feature', + f.elem.get('name'), 'because it is not tagged for emission') + # Generate the interface (or just tag its elements as having been + # emitted, if they haven't been). + self.gen.beginFeature(f.elem, emit) + self.generateRequiredInterface(f.elem) + self.gen.endFeature() + # Generate spirv elements + for s in spirvexts: + self.generateSpirv(s, self.spirvextdict) + for s in spirvcaps: + self.generateSpirv(s, self.spirvcapdict) + self.gen.endFile() + + def apiReset(self): + """Reset type/enum/command dictionaries before generating another API. + + Use between apiGen() calls to reset internal state.""" + for datatype in self.typedict: + self.typedict[datatype].resetState() + for enum in self.enumdict: + self.enumdict[enum].resetState() + for cmd in self.cmddict: + self.cmddict[cmd].resetState() + for cmd in self.apidict: + self.apidict[cmd].resetState() + + def __validateStructLimittypes(self, struct): + """Validate 'limittype' attributes for a single struct.""" + limittypeDiags = namedtuple('limittypeDiags', ['missing', 'invalid']) + badFields = defaultdict(lambda : limittypeDiags(missing=[], invalid=[])) + validLimittypes = { 'min', 'max', 'bitmask', 'range', 'struct', 'noauto' } + for member in struct.getMembers(): + memberName = member.findtext('name') + if memberName in ['sType', 'pNext']: + continue + limittype = member.get('limittype') + if not limittype: + badFields[struct.elem.get('name')].missing.append(memberName) + elif limittype == 'struct': + typeName = member.findtext('type') + memberType = self.typedict[typeName] + badFields.update(self.__validateStructLimittypes(memberType)) + elif limittype not in validLimittypes: + badFields[struct.elem.get('name')].invalid.append(memberName) + return badFields + + def __validateLimittype(self): + """Validate 'limittype' attributes.""" + self.gen.logMsg('diag', 'VALIDATING LIMITTYPE ATTRIBUTES') + badFields = self.__validateStructLimittypes(self.typedict['VkPhysicalDeviceProperties2']) + for featStructName in self.validextensionstructs['VkPhysicalDeviceProperties2']: + featStruct = self.typedict[featStructName] + badFields.update(self.__validateStructLimittypes(featStruct)) + + if badFields: + self.gen.logMsg('diag', 'SUMMARY OF FIELDS WITH INCORRECT LIMITTYPES') + for key in sorted(badFields.keys()): + diags = badFields[key] + if diags.missing: + self.gen.logMsg('diag', ' ', key, 'missing limittype:', ', '.join(badFields[key].missing)) + if diags.invalid: + self.gen.logMsg('diag', ' ', key, 'invalid limittype:', ', '.join(badFields[key].invalid)) + return False + return True + + def validateRegistry(self): + """Validate properties of the registry.""" + return self.__validateLimittype() From 38080d39e4f00aa6f940d85d440290912939912a Mon Sep 17 00:00:00 2001 From: Aharon Abramson Date: Mon, 13 Nov 2023 13:39:44 +0200 Subject: [PATCH 007/202] Update xml/cl.xml Co-authored-by: Ewan Crawford --- xml/cl.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/xml/cl.xml b/xml/cl.xml index 283b068d5..4f9c1c3aa 100644 --- a/xml/cl.xml +++ b/xml/cl.xml @@ -7319,6 +7319,9 @@ server's OpenCL/api-docs repository. + + + From cbf85b621d3a07c9d99055406800119b173d1851 Mon Sep 17 00:00:00 2001 From: Aharon Abramson Date: Tue, 21 Nov 2023 10:39:09 +0200 Subject: [PATCH 008/202] Update xml/cl.xml Co-authored-by: Sun Serega --- xml/cl.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xml/cl.xml b/xml/cl.xml index 4f9c1c3aa..3e042e58f 100644 --- a/xml/cl.xml +++ b/xml/cl.xml @@ -1345,7 +1345,7 @@ server's OpenCL/api-docs repository. - + From 711fcda065ae8f0dbb337d625fe8dc72927a8b93 Mon Sep 17 00:00:00 2001 From: Aharon Abramson Date: Tue, 21 Nov 2023 10:55:28 +0200 Subject: [PATCH 009/202] Update xml/cl.xml Co-authored-by: Sun Serega --- xml/cl.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xml/cl.xml b/xml/cl.xml index 3e042e58f..d0982f205 100644 --- a/xml/cl.xml +++ b/xml/cl.xml @@ -7323,7 +7323,7 @@ server's OpenCL/api-docs repository. - + From 2264ad6b61ba8e472bdf6b6cc2ed7bb5aa3b7a6b Mon Sep 17 00:00:00 2001 From: aharon-abramson Date: Wed, 6 Dec 2023 11:00:44 +0200 Subject: [PATCH 010/202] remove changes to files made by mistake --- .asciidoctorconfig.adoc | 12 - .project | 11 - xml/cgenerator.py | 420 --------- xml/checklinks.py | 71 -- xml/cl.xml | 6 +- xml/clconventions.py | 241 ------ xml/conventions.py | 358 -------- xml/docgenerator.py | 454 ---------- xml/extensionmetadocgenerator.py | 659 -------------- xml/genRef.py | 1019 ---------------------- xml/gen_dictionaries.py | 258 ------ xml/gen_version_notes.py | 127 --- xml/gencl.py | 464 ---------- xml/generator.py | 1186 ------------------------- xml/pygenerator.py | 365 -------- xml/realign.py | 47 - xml/reflib.py | 663 -------------- xml/reg.py | 1397 ------------------------------ 18 files changed, 3 insertions(+), 7755 deletions(-) delete mode 100644 .asciidoctorconfig.adoc delete mode 100644 .project delete mode 100644 xml/cgenerator.py delete mode 100644 xml/checklinks.py delete mode 100644 xml/clconventions.py delete mode 100644 xml/conventions.py delete mode 100644 xml/docgenerator.py delete mode 100644 xml/extensionmetadocgenerator.py delete mode 100644 xml/genRef.py delete mode 100644 xml/gen_dictionaries.py delete mode 100644 xml/gen_version_notes.py delete mode 100644 xml/gencl.py delete mode 100644 xml/generator.py delete mode 100644 xml/pygenerator.py delete mode 100644 xml/realign.py delete mode 100644 xml/reflib.py delete mode 100644 xml/reg.py diff --git a/.asciidoctorconfig.adoc b/.asciidoctorconfig.adoc deleted file mode 100644 index 53b403dbc..000000000 --- a/.asciidoctorconfig.adoc +++ /dev/null @@ -1,12 +0,0 @@ -// +++++++++++++++++++++++++++++++++++++++++++++++++++++++ -// + Initial AsciiDoc editor configuration file - V1.0 + -// ++++++++++++++++++++++++++++++++++++++++++++++++++++++ -// -// Did not found any configuration files, so create this at project root level. -// If you do not like those files to be generated - you can turn it off inside Asciidoctor Editor preferences. -// -// You can define editor specific parts here. -// For example: with next line you could set imagesdir attribute to subfolder "images" relative to the folder where this config file is located. -// :imagesdir: {asciidoctorconfigdir}/images -// -// For more information please take a look at https://github.com/de-jcup/eclipse-asciidoctor-editor/wiki/Asciidoctor-configfiles diff --git a/.project b/.project deleted file mode 100644 index 9877aec6c..000000000 --- a/.project +++ /dev/null @@ -1,11 +0,0 @@ - - - OpenCL-Docs - - - - - - - - diff --git a/xml/cgenerator.py b/xml/cgenerator.py deleted file mode 100644 index 4b2a8f1e2..000000000 --- a/xml/cgenerator.py +++ /dev/null @@ -1,420 +0,0 @@ -#!/usr/bin/python3 -i -# -# Copyright 2013-2023 The Khronos Group Inc. -# -# SPDX-License-Identifier: Apache-2.0 - -import os -import re -from generator import (GeneratorOptions, OutputGenerator, noneStr, - regSortFeatures, write) - - -class CGeneratorOptions(GeneratorOptions): - """CGeneratorOptions - subclass of GeneratorOptions. - - Adds options used by COutputGenerator objects during C language header - generation.""" - - def __init__(self, - prefixText="", - genFuncPointers=True, - protectFile=True, - protectFeature=True, - protectProto=None, - protectProtoStr=None, - apicall='', - apientry='', - apientryp='', - indentFuncProto=True, - indentFuncPointer=False, - alignFuncParam=0, - genEnumBeginEndRange=False, - genAliasMacro=False, - aliasMacro='', - misracstyle=False, - misracppstyle=False, - **kwargs - ): - """Constructor. - Additional parameters beyond parent class: - - - prefixText - list of strings to prefix generated header with - (usually a copyright statement + calling convention macros). - - protectFile - True if multiple inclusion protection should be - generated (based on the filename) around the entire header. - - protectFeature - True if #ifndef..#endif protection should be - generated around a feature interface in the header file. - - genFuncPointers - True if function pointer typedefs should be - generated - - protectProto - If conditional protection should be generated - around prototype declarations, set to either '#ifdef' - to require opt-in (#ifdef protectProtoStr) or '#ifndef' - to require opt-out (#ifndef protectProtoStr). Otherwise - set to None. - - protectProtoStr - #ifdef/#ifndef symbol to use around prototype - declarations, if protectProto is set - - apicall - string to use for the function declaration prefix, - such as APICALL on Windows. - - apientry - string to use for the calling convention macro, - in typedefs, such as APIENTRY. - - apientryp - string to use for the calling convention macro - in function pointer typedefs, such as APIENTRYP. - - indentFuncProto - True if prototype declarations should put each - parameter on a separate line - - indentFuncPointer - True if typedefed function pointers should put each - parameter on a separate line - - alignFuncParam - if nonzero and parameters are being put on a - separate line, align parameter names at the specified column - - genEnumBeginEndRange - True if BEGIN_RANGE / END_RANGE macros should - be generated for enumerated types - - genAliasMacro - True if the OpenXR alias macro should be generated - for aliased types (unclear what other circumstances this is useful) - - aliasMacro - alias macro to inject when genAliasMacro is True - - misracstyle - generate MISRA C-friendly headers - - misracppstyle - generate MISRA C++-friendly headers""" - - GeneratorOptions.__init__(self, **kwargs) - - self.prefixText = prefixText - """list of strings to prefix generated header with (usually a copyright statement + calling convention macros).""" - - self.genFuncPointers = genFuncPointers - """True if function pointer typedefs should be generated""" - - self.protectFile = protectFile - """True if multiple inclusion protection should be generated (based on the filename) around the entire header.""" - - self.protectFeature = protectFeature - """True if #ifndef..#endif protection should be generated around a feature interface in the header file.""" - - self.protectProto = protectProto - """If conditional protection should be generated around prototype declarations, set to either '#ifdef' to require opt-in (#ifdef protectProtoStr) or '#ifndef' to require opt-out (#ifndef protectProtoStr). Otherwise set to None.""" - - self.protectProtoStr = protectProtoStr - """#ifdef/#ifndef symbol to use around prototype declarations, if protectProto is set""" - - self.apicall = apicall - """string to use for the function declaration prefix, such as APICALL on Windows.""" - - self.apientry = apientry - """string to use for the calling convention macro, in typedefs, such as APIENTRY.""" - - self.apientryp = apientryp - """string to use for the calling convention macro in function pointer typedefs, such as APIENTRYP.""" - - self.indentFuncProto = indentFuncProto - """True if prototype declarations should put each parameter on a separate line""" - - self.indentFuncPointer = indentFuncPointer - """True if typedefed function pointers should put each parameter on a separate line""" - - self.alignFuncParam = alignFuncParam - """if nonzero and parameters are being put on a separate line, align parameter names at the specified column""" - - self.genEnumBeginEndRange = genEnumBeginEndRange - """True if BEGIN_RANGE / END_RANGE macros should be generated for enumerated types""" - - self.genAliasMacro = genAliasMacro - """True if the OpenXR alias macro should be generated for aliased types (unclear what other circumstances this is useful)""" - - self.aliasMacro = aliasMacro - """alias macro to inject when genAliasMacro is True""" - - self.misracstyle = misracstyle - """generate MISRA C-friendly headers""" - - self.misracppstyle = misracppstyle - """generate MISRA C++-friendly headers""" - - self.codeGenerator = True - """True if this generator makes compilable code""" - - -class COutputGenerator(OutputGenerator): - """Generates C-language API interfaces.""" - - # This is an ordered list of sections in the header file. - TYPE_SECTIONS = ['include', 'define', 'basetype', 'handle', 'enum', - 'group', 'bitmask', 'funcpointer', 'struct'] - ALL_SECTIONS = TYPE_SECTIONS + ['commandPointer', 'command'] - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - # Internal state - accumulators for different inner block text - self.sections = {section: [] for section in self.ALL_SECTIONS} - self.feature_not_empty = False - self.may_alias = None - - def beginFile(self, genOpts): - OutputGenerator.beginFile(self, genOpts) - # C-specific - # - # Multiple inclusion protection & C++ wrappers. - if genOpts.protectFile and self.genOpts.filename: - headerSym = re.sub(r'\.h', '_h_', - os.path.basename(self.genOpts.filename)).upper() - write('#ifndef', headerSym, file=self.outFile) - write('#define', headerSym, '1', file=self.outFile) - self.newline() - - # User-supplied prefix text, if any (list of strings) - if genOpts.prefixText: - for s in genOpts.prefixText: - write(s, file=self.outFile) - - # C++ extern wrapper - after prefix lines so they can add includes. - self.newline() - write('#ifdef __cplusplus', file=self.outFile) - write('extern "C" {', file=self.outFile) - write('#endif', file=self.outFile) - self.newline() - - def endFile(self): - # C-specific - # Finish C++ wrapper and multiple inclusion protection - self.newline() - write('#ifdef __cplusplus', file=self.outFile) - write('}', file=self.outFile) - write('#endif', file=self.outFile) - if self.genOpts.protectFile and self.genOpts.filename: - self.newline() - write('#endif', file=self.outFile) - # Finish processing in superclass - OutputGenerator.endFile(self) - - def beginFeature(self, interface, emit): - # Start processing in superclass - OutputGenerator.beginFeature(self, interface, emit) - # C-specific - # Accumulate includes, defines, types, enums, function pointer typedefs, - # end function prototypes separately for this feature. They're only - # printed in endFeature(). - self.sections = {section: [] for section in self.ALL_SECTIONS} - self.feature_not_empty = False - - def endFeature(self): - "Actually write the interface to the output file." - # C-specific - if self.emit: - if self.feature_not_empty: - if self.genOpts.conventions.writeFeature(self.featureExtraProtect, self.genOpts.filename): - self.newline() - if self.genOpts.protectFeature: - write('#ifndef', self.featureName, file=self.outFile) - # If type declarations are needed by other features based on - # this one, it may be necessary to suppress the ExtraProtect, - # or move it below the 'for section...' loop. - if self.featureExtraProtect is not None: - write('#ifdef', self.featureExtraProtect, file=self.outFile) - self.newline() - write('#define', self.featureName, '1', file=self.outFile) - for section in self.TYPE_SECTIONS: - contents = self.sections[section] - if contents: - write('\n'.join(contents), file=self.outFile) - if self.genOpts.genFuncPointers and self.sections['commandPointer']: - write('\n'.join(self.sections['commandPointer']), file=self.outFile) - self.newline() - if self.sections['command']: - if self.genOpts.protectProto: - write(self.genOpts.protectProto, - self.genOpts.protectProtoStr, file=self.outFile) - write('\n'.join(self.sections['command']), end='', file=self.outFile) - if self.genOpts.protectProto: - write('#endif', file=self.outFile) - else: - self.newline() - if self.featureExtraProtect is not None: - write('#endif /*', self.featureExtraProtect, '*/', file=self.outFile) - if self.genOpts.protectFeature: - write('#endif /*', self.featureName, '*/', file=self.outFile) - # Finish processing in superclass - OutputGenerator.endFeature(self) - - def appendSection(self, section, text): - "Append a definition to the specified section" - # self.sections[section].append('SECTION: ' + section + '\n') - self.sections[section].append(text) - self.feature_not_empty = True - - def genType(self, typeinfo, name, alias): - "Generate type." - OutputGenerator.genType(self, typeinfo, name, alias) - typeElem = typeinfo.elem - - # Vulkan: - # Determine the category of the type, and the type section to add - # its definition to. - # 'funcpointer' is added to the 'struct' section as a workaround for - # internal issue #877, since structures and function pointer types - # can have cross-dependencies. - category = typeElem.get('category') - if category == 'funcpointer': - section = 'struct' - else: - section = category - - if category in ('struct', 'union'): - # If the type is a struct type, generate it using the - # special-purpose generator. - self.genStruct(typeinfo, name, alias) - else: - # OpenXR: this section was not under 'else:' previously, just fell through - if alias: - # If the type is an alias, just emit a typedef declaration - body = 'typedef ' + alias + ' ' + name + ';\n' - else: - # Replace tags with an APIENTRY-style string - # (from self.genOpts). Copy other text through unchanged. - # If the resulting text is an empty string, don't emit it. - body = noneStr(typeElem.text) - for elem in typeElem: - if elem.tag == 'apientry': - body += self.genOpts.apientry + noneStr(elem.tail) - else: - body += noneStr(elem.text) + noneStr(elem.tail) - if body: - # Add extra newline after multi-line entries. - if '\n' in body[0:-1]: - body += '\n' - self.appendSection(section, body) - - def genProtectString(self, protect_str): - """Generate protection string. - - Protection strings are the strings defining the OS/Platform/Graphics - requirements for a given OpenXR command. When generating the - language header files, we need to make sure the items specific to a - graphics API or OS platform are properly wrapped in #ifs.""" - protect_if_str = '' - protect_end_str = '' - if not protect_str: - return (protect_if_str, protect_end_str) - - if ',' in protect_str: - protect_list = protect_str.split(",") - protect_defs = ('defined(%s)' % d for d in protect_list) - protect_def_str = ' && '.join(protect_defs) - protect_if_str = '#if %s\n' % protect_def_str - protect_end_str = '#endif // %s\n' % protect_def_str - else: - protect_if_str = '#ifdef %s\n' % protect_str - protect_end_str = '#endif // %s\n' % protect_str - - return (protect_if_str, protect_end_str) - - def typeMayAlias(self, typeName): - if not self.may_alias: - # First time we've asked if a type may alias. - # So, let's populate the set of all names of types that may. - - # Everyone with an explicit mayalias="true" - self.may_alias = set(typeName - for typeName, data in self.registry.typedict.items() - if data.elem.get('mayalias') == 'true') - - # Every type mentioned in some other type's parentstruct attribute. - parent_structs = (otherType.elem.get('parentstruct') - for otherType in self.registry.typedict.values()) - self.may_alias.update(set(x for x in parent_structs - if x is not None)) - return typeName in self.may_alias - - def genStruct(self, typeinfo, typeName, alias): - """Generate struct (e.g. C "struct" type). - - This is a special case of the tag where the contents are - interpreted as a set of tags instead of freeform C - C type declarations. The tags are just like - tags - they are a declaration of a struct or union member. - Only simple member declarations are supported (no nested - structs etc.) - - If alias is not None, then this struct aliases another; just - generate a typedef of that alias.""" - OutputGenerator.genStruct(self, typeinfo, typeName, alias) - - typeElem = typeinfo.elem - - if alias: - body = 'typedef ' + alias + ' ' + typeName + ';\n' - else: - body = '' - (protect_begin, protect_end) = self.genProtectString(typeElem.get('protect')) - if protect_begin: - body += protect_begin - body += 'typedef ' + typeElem.get('category') - - # This is an OpenXR-specific alternative where aliasing refers - # to an inheritance hierarchy of types rather than C-level type - # aliases. - if self.genOpts.genAliasMacro and self.typeMayAlias(typeName): - body += ' ' + self.genOpts.aliasMacro - - body += ' ' + typeName + ' {\n' - - targetLen = self.getMaxCParamTypeLength(typeinfo) - for member in typeElem.findall('.//member'): - body += self.makeCParamDecl(member, targetLen + 4) - body += ';\n' - body += '} ' + typeName + ';\n' - if protect_end: - body += protect_end - - self.appendSection('struct', body) - - def genGroup(self, groupinfo, groupName, alias=None): - """Generate groups (e.g. C "enum" type). - - These are concatenated together with other types. - - If alias is not None, it is the name of another group type - which aliases this type; just generate that alias.""" - OutputGenerator.genGroup(self, groupinfo, groupName, alias) - groupElem = groupinfo.elem - - # After either enumerated type or alias paths, add the declaration - # to the appropriate section for the group being defined. - if groupElem.get('type') == 'bitmask': - section = 'bitmask' - else: - section = 'group' - - if alias: - # If the group name is aliased, just emit a typedef declaration - # for the alias. - body = 'typedef ' + alias + ' ' + groupName + ';\n' - self.appendSection(section, body) - else: - (section, body) = self.buildEnumCDecl(self.genOpts.genEnumBeginEndRange, groupinfo, groupName) - self.appendSection(section, "\n" + body) - - def genEnum(self, enuminfo, name, alias): - """Generate the C declaration for a constant (a single value).""" - - OutputGenerator.genEnum(self, enuminfo, name, alias) - - body = self.buildConstantCDecl(enuminfo, name, alias) - self.appendSection('enum', body) - - def genCmd(self, cmdinfo, name, alias): - "Command generation" - OutputGenerator.genCmd(self, cmdinfo, name, alias) - - # if alias: - # prefix = '// ' + name + ' is an alias of command ' + alias + '\n' - # else: - # prefix = '' - - prefix = '' - decls = self.makeCDecls(cmdinfo.elem) - self.appendSection('command', prefix + decls[0] + '\n') - if self.genOpts.genFuncPointers: - self.appendSection('commandPointer', decls[1]) - - def misracstyle(self): - return self.genOpts.misracstyle; - - def misracppstyle(self): - return self.genOpts.misracppstyle; diff --git a/xml/checklinks.py b/xml/checklinks.py deleted file mode 100644 index 94b650dad..000000000 --- a/xml/checklinks.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/python3 -# -# Copyright 2013-2023 The Khronos Group Inc. -# SPDX-License-Identifier: Apache-2.0 - -import argparse -import os -import re - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - - parser.add_argument('-d', action='store', dest='directory', - default='../api', - help='Directory containing files to check') - parser.add_argument('--unlinked', action='store_true', - help='Check for unlinked APIs and enums (may have false positives!)') - - args = parser.parse_args() - - links = set() - anchors = set() - - for filename in os.listdir(args.directory): - filename = args.directory + '/' + filename - sourcefile = open(filename, 'r') - sourcetext = sourcefile.read() - sourcefile.close() - - # We're not going to check API links. - #filelinks = re.findall(r"{((cl\w+)|(CL\w+))}", sourcetext) - filelinks = re.findall(r"{((CL\w+))}", sourcetext) - fileanchors = re.findall(r"{((cl\w+)|(CL\w+))_anchor}", sourcetext) - - filelinks = [re.sub(r"_anchor\b", "", link[0]) for link in filelinks] - fileanchors = [anchor[0] for anchor in fileanchors] - - links = links.union(set(filelinks) - set(fileanchors)) - anchors = anchors.union(set(fileanchors)) - - #print("=== " + filename) - #print("links:") - #print(' '.join(filelinks)) - #print("anchors:") - #print(' '.join(fileanchors)) - - if args.unlinked: - # Look for APIs and enums that do not begin with: - # { = asciidoctor attribute link - # character = middle of word - # < = asciidoctor link - # ' = refpage description - # / = proto include - fileunlinkedapi = sorted(list(set(re.findall(r"[^{\w<'/](cl[A-Z]\w+)\b[^'](?!.')", sourcetext)))) - fileunlinkedenums = sorted(list(set(re.findall("r[^{\w<](CL_\w+)", sourcetext)))) - fileunlinkedtypes = sorted(list(set(re.findall("r[^{\w<](cl_\w+)", sourcetext)))) - - if len(fileunlinkedapi) != 0: - print("unlinked APIs in " + filename + ":\n\t" + '\n\t'.join(fileunlinkedapi)) - - if len(fileunlinkedenums) != 0: - print("unlinked enums in " + filename + ":\n\t" + '\n\t'.join(fileunlinkedenums)) - - if len(fileunlinkedtypes) != 0: - print("unlinked types in " + filename + ":\n\t" + '\n\t'.join(fileunlinkedtypes)) - - linkswithoutanchors = sorted(list(links - anchors)) - anchorswithoutlinks = sorted(list(anchors - links)) - - print("links without anchors:\n\t" + '\n\t'.join(linkswithoutanchors)) - #print("anchors without links:\n\t" + '\n\t'.join(anchorswithoutlinks)) diff --git a/xml/cl.xml b/xml/cl.xml index d0982f205..1f4a450fd 100644 --- a/xml/cl.xml +++ b/xml/cl.xml @@ -252,7 +252,7 @@ server's OpenCL/api-docs repository. typedef cl_bitfield cl_device_fp_atomic_capabilities_ext; typedef cl_uint cl_image_requirements_info_ext; typedef cl_bitfield cl_platform_command_buffer_capabilities_khr; - typedef cl_bitfield cl_mutable_dispatch_asserts_khr + typedef cl_bitfield cl_mutable_dispatch_promises_khr Structure types @@ -1783,7 +1783,7 @@ server's OpenCL/api-docs repository. - + @@ -7282,7 +7282,7 @@ server's OpenCL/api-docs repository. - +     diff --git a/xml/clconventions.py b/xml/clconventions.py deleted file mode 100644 index f4df49d2d..000000000 --- a/xml/clconventions.py +++ /dev/null @@ -1,241 +0,0 @@ -#!/usr/bin/python3 -i -# -# Copyright 2013-2023 The Khronos Group Inc. -# SPDX-License-Identifier: Apache-2.0 - -# Working-group-specific style conventions, -# used in generation. - -import re - -from conventions import ConventionsBase - - -class OpenCLConventions(ConventionsBase): - def formatExtension(self, name): - """Mark up a name as an extension for the spec.""" - return '`<<{}>>`'.format(name) - - @property - def null(self): - """Preferred spelling of NULL.""" - return '`NULL`' - - @property - def constFlagBits(self): - """Returns True if static const flag bits should be generated, False if an enumerated type should be generated.""" - return False - - @property - def struct_macro(self): - return 'sname:' - - @property - def external_macro(self): - return 'code:' - - @property - def structtype_member_name(self): - """Return name of the structure type member""" - return 'sType' - - @property - def nextpointer_member_name(self): - """Return name of the structure pointer chain member""" - return 'pNext' - - @property - def valid_pointer_prefix(self): - """Return prefix to pointers which must themselves be valid""" - return 'valid' - - def is_structure_type_member(self, paramtype, paramname): - """Determine if member type and name match the structure type member.""" - return False - - def is_nextpointer_member(self, paramtype, paramname): - """Determine if member type and name match the next pointer chain member.""" - return paramtype == 'void' and paramname == self.nextpointer_member_name - - def generate_structure_type_from_name(self, structname): - """Generate a structure type name token from a structure name. - This should never be called for OpenCL, just other APIs.""" - return '' - - @property - def warning_comment(self): - """Return warning comment to be placed in header of generated Asciidoctor files""" - return '// WARNING: DO NOT MODIFY! This file is automatically generated from the cl.xml registry' - - @property - def file_suffix(self): - """Return suffix of generated Asciidoctor files""" - return '.txt' - - def api_name(self, spectype='api'): - """Return API or specification name for citations in ref pages.ref - pages should link to for - - spectype is the spec this refpage is for: 'api' is the OpenCL API - Specification, 'clang' is the OpenCL C Language specification. - Defaults to 'api'. If an unrecognized spectype is given, returns - None. - """ - if spectype == 'api' or spectype is None: - return 'OpenCL' - elif spectype == 'clang': - return 'OpenCL C' - else: - return None - - @property - def xml_supported_name_of_api(self): - """Return the supported= attribute used in API XML""" - return 'opencl' - - @property - def api_prefix(self): - """Return API token prefix""" - return 'CL_' - - @property - def api_version_prefix(self): - """Return API core version token prefix""" - return 'CL_VERSION_' - - @property - def KHR_prefix(self): - """Return extension name prefix for KHR extensions""" - return 'cl_khr_' - - @property - def EXT_prefix(self): - """Return extension name prefix for EXT extensions""" - return 'cl_ext_' - - @property - def write_contacts(self): - """Return whether contact list should be written to extension appendices""" - return True - - @property - def write_refpage_include(self): - """Return whether refpage include should be written to extension appendices""" - return False - - def writeFeature(self, featureExtraProtect, filename): - """Returns True if OutputGenerator.endFeature should write this feature. - Used in COutputGenerator - """ - return True - - def requires_error_validation(self, return_type): - """Returns True if the return_type element is an API result code - requiring error validation. - """ - return False - - @property - def required_errors(self): - """Return a list of required error codes for validation.""" - return [] - - def is_externsync_command(self, protoname): - """Returns True if the protoname element is an API command requiring - external synchronization - """ - return False - - def is_api_name(self, name): - """Returns True if name is in the reserved API namespace. - For OpenCL, these are names with a case-insensitive 'cl' prefix. - """ - return name[0:2].lower() == 'cl' - - def is_voidpointer_alias(self, tag, text, tail): - """Return True if the declaration components (tag,text,tail) of an - element represents a void * type - """ - return tag == 'type' and text == 'void' and tail.startswith('*') - - def make_voidpointer_alias(self, tail): - """Reformat a void * declaration to include the API alias macro. - Vulkan doesn't have an API alias macro, so do nothing. - """ - return tail - - def specURL(self, spectype = 'api'): - """Return public registry URL which ref pages should link to for - full Specification, so xrefs in the asciidoc source that aren't - to ref pages can link into it instead. - - spectype is the spec this refpage is for: 'api' is the OpenCL API - Specification, 'clang' is the OpenCL C Language specification. - Defaults to 'api'. If an unrecognized spectype is given, returns - None. - """ - if spectype == 'api' or spectype is None: - return 'https://www.khronos.org/registry/OpenCL/specs/3.0-unified/html/OpenCL_API.html' - elif spectype == 'clang': - return 'https://www.khronos.org/registry/OpenCL/specs/3.0-unified/html/OpenCL_C.html' - else: - return None - - @property - def xml_api_name(self): - """Return the name used in the default API XML registry for the default API""" - return 'opencl' - - @property - def registry_path(self): - """Return relpath to the default API XML registry in this project.""" - return 'xml/cl.xml' - - @property - def specification_path(self): - """Return relpath to the Asciidoctor specification sources in this project.""" - return '../appendices/meta' - - @property - def extra_refpage_headers(self): - """Return any extra text to add to refpage headers.""" - return 'include::{config}/attribs.txt[]\n' + \ - 'include::{config}/opencl.asciidoc[]\n' + \ - 'include::{apispec}/footnotes.asciidoc[]\n' + \ - 'include::{cspec}/footnotes.asciidoc[]\n' + \ - 'include::{cspec}/feature-dictionary.asciidoc[]\n' + \ - 'include::{generated}/api/api-dictionary-no-links.asciidoc[]' - - @property - def extension_index_prefixes(self): - """Return a list of extension prefixes used to group extension refpages.""" - return ['cl_khr', 'cl_ext', 'cl'] - - @property - def unified_flag_refpages(self): - """Return True if Flags/FlagBits refpages are unified, False if - they're separate. - """ - return False - - @property - def spec_reflow_path(self): - """Return the relative path to the spec source folder to reflow""" - return '.' - - @property - def spec_no_reflow_dirs(self): - """Return a set of directories not to automatically descend into - when reflowing spec text - """ - return ('scripts', 'style') - - @property - def should_skip_checking_codes(self): - """Return True if more than the basic validation of return codes should - be skipped for a command. - - OpenCL has a different style of error handling than OpenXR or - Vulkan, so these checks are not appropriate.""" - - return True diff --git a/xml/conventions.py b/xml/conventions.py deleted file mode 100644 index 6b6b23d14..000000000 --- a/xml/conventions.py +++ /dev/null @@ -1,358 +0,0 @@ -#!/usr/bin/python3 -i -# -# Copyright 2013-2023 The Khronos Group Inc. -# -# SPDX-License-Identifier: Apache-2.0 - -# Base class for working-group-specific style conventions, -# used in generation. - -from enum import Enum - -# Type categories that respond "False" to isStructAlwaysValid -# basetype is home to typedefs like ..Bool32 -CATEGORIES_REQUIRING_VALIDATION = set(('handle', - 'enum', - 'bitmask', - 'basetype', - None)) - -# These are basic C types pulled in via openxr_platform_defines.h -TYPES_KNOWN_ALWAYS_VALID = set(('char', - 'float', - 'int8_t', 'uint8_t', - 'int32_t', 'uint32_t', - 'int64_t', 'uint64_t', - 'size_t', - 'uintptr_t', - 'int', - )) - - -class ProseListFormats(Enum): - """A connective, possibly with a quantifier.""" - AND = 0 - EACH_AND = 1 - OR = 2 - ANY_OR = 3 - - @classmethod - def from_string(cls, s): - if s == 'or': - return cls.OR - if s == 'and': - return cls.AND - return None - - @property - def connective(self): - if self in (ProseListFormats.OR, ProseListFormats.ANY_OR): - return 'or' - return 'and' - - def quantifier(self, n): - """Return the desired quantifier for a list of a given length.""" - if self == ProseListFormats.ANY_OR: - if n > 1: - return 'any of ' - elif self == ProseListFormats.EACH_AND: - if n > 2: - return 'each of ' - if n == 2: - return 'both of ' - return '' - - -class ConventionsBase: - """WG-specific conventions.""" - - def __init__(self): - self._command_prefix = None - self._type_prefix = None - - def formatExtension(self, name): - """Mark up an extension name as a link the spec.""" - return '`apiext:{}`'.format(name) - - @property - def null(self): - """Preferred spelling of NULL.""" - raise NotImplementedError - - def makeProseList(self, elements, fmt=ProseListFormats.AND, with_verb=False, *args, **kwargs): - """Make a (comma-separated) list for use in prose. - - Adds a connective (by default, 'and') - before the last element if there are more than 1. - - Adds the right one of "is" or "are" to the end if with_verb is true. - - Optionally adds a quantifier (like 'any') before a list of 2 or more, - if specified by fmt. - - Override with a different method or different call to - _implMakeProseList if you want to add a comma for two elements, - or not use a serial comma. - """ - return self._implMakeProseList(elements, fmt, with_verb, *args, **kwargs) - - @property - def struct_macro(self): - """Get the appropriate format macro for a structure. - - May override. - """ - return 'slink:' - - @property - def external_macro(self): - """Get the appropriate format macro for an external type like uint32_t. - - May override. - """ - return 'code:' - - def makeStructName(self, name): - """Prepend the appropriate format macro for a structure to a structure type name. - - Uses struct_macro, so just override that if you want to change behavior. - """ - return self.struct_macro + name - - def makeExternalTypeName(self, name): - """Prepend the appropriate format macro for an external type like uint32_t to a type name. - - Uses external_macro, so just override that if you want to change behavior. - """ - return self.external_macro + name - - def _implMakeProseList(self, elements, fmt, with_verb, comma_for_two_elts=False, serial_comma=True): - """Internal-use implementation to make a (comma-separated) list for use in prose. - - Adds a connective (by default, 'and') - before the last element if there are more than 1, - and only includes commas if there are more than 2 - (if comma_for_two_elts is False). - - Adds the right one of "is" or "are" to the end if with_verb is true. - - Optionally adds a quantifier (like 'any') before a list of 2 or more, - if specified by fmt. - - Don't edit these defaults, override self.makeProseList(). - """ - assert(serial_comma) # didn't implement what we didn't need - if isinstance(fmt, str): - fmt = ProseListFormats.from_string(fmt) - - my_elts = list(elements) - if len(my_elts) > 1: - my_elts[-1] = '{} {}'.format(fmt.connective, my_elts[-1]) - - if not comma_for_two_elts and len(my_elts) <= 2: - prose = ' '.join(my_elts) - else: - prose = ', '.join(my_elts) - - quantifier = fmt.quantifier(len(my_elts)) - - parts = [quantifier, prose] - - if with_verb: - if len(my_elts) > 1: - parts.append(' are') - else: - parts.append(' is') - return ''.join(parts) - - @property - def file_suffix(self): - """Return suffix of generated Asciidoctor files""" - raise NotImplementedError - - def api_name(self, spectype=None): - """Return API or specification name for citations in ref pages. - - spectype is the spec this refpage is for. - 'api' (the default value) is the main API Specification. - If an unrecognized spectype is given, returns None. - - Must implement.""" - raise NotImplementedError - - def should_insert_may_alias_macro(self, genOpts): - """Return true if we should insert a "may alias" macro in this file. - - Only used by OpenXR right now.""" - return False - - @property - def command_prefix(self): - """Return the expected prefix of commands/functions. - - Implemented in terms of api_prefix.""" - if not self._command_prefix: - self._command_prefix = self.api_prefix[:].replace('_', '').lower() - return self._command_prefix - - @property - def type_prefix(self): - """Return the expected prefix of type names. - - Implemented in terms of command_prefix (and in turn, api_prefix).""" - if not self._type_prefix: - self._type_prefix = ''.join( - (self.command_prefix[0:1].upper(), self.command_prefix[1:])) - return self._type_prefix - - @property - def api_prefix(self): - """Return API token prefix. - - Typically two uppercase letters followed by an underscore. - - Must implement.""" - raise NotImplementedError - - @property - def api_version_prefix(self): - """Return API core version token prefix. - - Implemented in terms of api_prefix. - - May override.""" - return self.api_prefix + 'VERSION_' - - @property - def KHR_prefix(self): - """Return extension name prefix for KHR extensions. - - Implemented in terms of api_prefix. - - May override.""" - return self.api_prefix + 'KHR_' - - @property - def EXT_prefix(self): - """Return extension name prefix for EXT extensions. - - Implemented in terms of api_prefix. - - May override.""" - return self.api_prefix + 'EXT_' - - def writeFeature(self, featureExtraProtect, filename): - """Return True if OutputGenerator.endFeature should write this feature. - - Defaults to always True. - Used in COutputGenerator. - - May override.""" - return True - - def requires_error_validation(self, return_type): - """Return True if the return_type element is an API result code - requiring error validation. - - Defaults to always False. - - May override.""" - return False - - @property - def required_errors(self): - """Return a list of required error codes for validation. - - Defaults to an empty list. - - May override.""" - return [] - - def is_voidpointer_alias(self, tag, text, tail): - """Return True if the declaration components (tag,text,tail) of an - element represents a void * type. - - Defaults to a reasonable implementation. - - May override.""" - return tag == 'type' and text == 'void' and tail.startswith('*') - - def make_voidpointer_alias(self, tail): - """Reformat a void * declaration to include the API alias macro. - - Defaults to a no-op. - - Must override if you actually want to use this feature in your project.""" - return tail - - def category_requires_validation(self, category): - """Return True if the given type 'category' always requires validation. - - Defaults to a reasonable implementation. - - May override.""" - return category in CATEGORIES_REQUIRING_VALIDATION - - def type_always_valid(self, typename): - """Return True if the given type name is always valid (never requires validation). - - This is for things like integers. - - Defaults to a reasonable implementation. - - May override.""" - return typename in TYPES_KNOWN_ALWAYS_VALID - - @property - def should_skip_checking_codes(self): - """Return True if more than the basic validation of return codes should - be skipped for a command.""" - - return False - - @property - def generate_index_terms(self): - """Return True if asiidoctor index terms should be generated as part - of an API interface from the docgenerator.""" - - return False - - @property - def generate_enum_table(self): - """Return True if asciidoctor tables describing enumerants in a - group should be generated as part of group generation.""" - return False - - @property - def generate_max_enum_in_docs(self): - """Return True if MAX_ENUM tokens should be generated in - documentation includes.""" - return False - - - def extension_include_string(self, ext): - """Return format string for include:: line for an extension appendix - file. ext is an object with the following members: - - name - extension string string - - vendor - vendor portion of name - - barename - remainder of name - - Must implement.""" - raise NotImplementedError - - @property - def refpage_generated_include_path(self): - """Return path relative to the generated reference pages, to the - generated API include files. - - Must implement.""" - raise NotImplementedError - - def valid_flag_bit(self, bitpos): - """Return True if bitpos is an allowed numeric bit position for - an API flag. - - Behavior depends on the data type used for flags (which may be 32 - or 64 bits), and may depend on assumptions about compiler - handling of sign bits in enumerated types, as well.""" - return True diff --git a/xml/docgenerator.py b/xml/docgenerator.py deleted file mode 100644 index 073552534..000000000 --- a/xml/docgenerator.py +++ /dev/null @@ -1,454 +0,0 @@ -#!/usr/bin/python3 -i -# -# Copyright 2013-2023 The Khronos Group Inc. -# -# SPDX-License-Identifier: Apache-2.0 - -from pathlib import Path - -from generator import GeneratorOptions, OutputGenerator, noneStr, write - -ENUM_TABLE_PREFIX = """ -[cols=",",options="header",] -|======================================================================= -|Enum |Description""" - -ENUM_TABLE_SUFFIX = """|=======================================================================""" - -FLAG_BLOCK_PREFIX = """.Flag Descriptions -****""" - -FLAG_BLOCK_SUFFIX = """****""" - - -class DocGeneratorOptions(GeneratorOptions): - """DocGeneratorOptions - subclass of GeneratorOptions for - generating declaration snippets for the spec. - - Shares many members with CGeneratorOptions, since - both are writing C-style declarations.""" - - def __init__(self, - prefixText="", - apicall='', - apientry='', - apientryp='', - indentFuncProto=True, - indentFuncPointer=False, - alignFuncParam=0, - secondaryInclude=False, - expandEnumerants=True, - extEnumerantAdditions=False, - extEnumerantFormatString=" (Added by the {} extension)", - **kwargs): - """Constructor. - - Since this generator outputs multiple files at once, - the filename is just a "stamp" to indicate last generation time. - - Shares many parameters/members with CGeneratorOptions, since - both are writing C-style declarations: - - - prefixText - list of strings to prefix generated header with - (usually a copyright statement + calling convention macros). - - apicall - string to use for the function declaration prefix, - such as APICALL on Windows. - - apientry - string to use for the calling convention macro, - in typedefs, such as APIENTRY. - - apientryp - string to use for the calling convention macro - in function pointer typedefs, such as APIENTRYP. - - indentFuncProto - True if prototype declarations should put each - parameter on a separate line - - indentFuncPointer - True if typedefed function pointers should put each - parameter on a separate line - - alignFuncParam - if nonzero and parameters are being put on a - separate line, align parameter names at the specified column - - Additional parameters/members: - - - expandEnumerants - if True, add BEGIN/END_RANGE macros in enumerated - type declarations - - secondaryInclude - if True, add secondary (no xref anchor) versions - of generated files - - extEnumerantAdditions - if True, include enumerants added by extensions - in comment tables for core enumeration types. - - extEnumerantFormatString - A format string for any additional message for - enumerants from extensions if extEnumerantAdditions is True. The correctly- - marked-up extension name will be passed. - """ - GeneratorOptions.__init__(self, **kwargs) - self.prefixText = prefixText - """list of strings to prefix generated header with (usually a copyright statement + calling convention macros).""" - - self.apicall = apicall - """string to use for the function declaration prefix, such as APICALL on Windows.""" - - self.apientry = apientry - """string to use for the calling convention macro, in typedefs, such as APIENTRY.""" - - self.apientryp = apientryp - """string to use for the calling convention macro in function pointer typedefs, such as APIENTRYP.""" - - self.indentFuncProto = indentFuncProto - """True if prototype declarations should put each parameter on a separate line""" - - self.indentFuncPointer = indentFuncPointer - """True if typedefed function pointers should put each parameter on a separate line""" - - self.alignFuncParam = alignFuncParam - """if nonzero and parameters are being put on a separate line, align parameter names at the specified column""" - - self.secondaryInclude = secondaryInclude - """if True, add secondary (no xref anchor) versions of generated files""" - - self.expandEnumerants = expandEnumerants - """if True, add BEGIN/END_RANGE macros in enumerated type declarations""" - - self.extEnumerantAdditions = extEnumerantAdditions - """if True, include enumerants added by extensions in comment tables for core enumeration types.""" - - self.extEnumerantFormatString = extEnumerantFormatString - """A format string for any additional message for - enumerants from extensions if extEnumerantAdditions is True. The correctly- - marked-up extension name will be passed.""" - - -class DocOutputGenerator(OutputGenerator): - """DocOutputGenerator - subclass of OutputGenerator. - - Generates AsciiDoc includes with C-language API interfaces, for reference - pages and the corresponding specification. Similar to COutputGenerator, - but each interface is written into a different file as determined by the - options, only actual C types are emitted, and none of the boilerplate - preprocessor code is emitted.""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - # Keep track of all extension numbers - self.extension_numbers = set() - - def beginFile(self, genOpts): - OutputGenerator.beginFile(self, genOpts) - - # This should be a separate conventions property rather than an - # inferred type name pattern for different APIs. - self.result_type = genOpts.conventions.type_prefix + "Result" - - def endFile(self): - OutputGenerator.endFile(self) - - def beginFeature(self, interface, emit): - # Start processing in superclass - OutputGenerator.beginFeature(self, interface, emit) - - # Decide if we're in a core or an - self.in_core = (interface.tag == 'feature') - - # Verify that each has a unique number during doc - # generation - # TODO move this to consistency_tools - if not self.in_core: - extension_number = interface.get('number') - if extension_number is not None and extension_number != "0": - if extension_number in self.extension_numbers: - self.logMsg('error', 'Duplicate extension number ', extension_number, ' detected in feature ', interface.get('name'), '\n') - exit(1) - else: - self.extension_numbers.add(extension_number) - - def endFeature(self): - # Finish processing in superclass - OutputGenerator.endFeature(self) - - def genRequirements(self, name, mustBeFound = True): - """Generate text showing what core versions and extensions introduce - an API. This relies on the map in api.py, which may be loaded at - runtime into self.apidict. If not present, no message is - generated. - - - name - name of the API - - mustBeFound - If True, when requirements for 'name' cannot be - determined, a warning comment is generated. - """ - - if self.apidict: - if name in self.apidict.requiredBy: - features = [] - for (base,dependency) in self.apidict.requiredBy[name]: - if dependency is not None: - features.append('{} with {}'.format(base, dependency)) - else: - features.append(base) - return '// Provided by {}\n'.format(', '.join(features)) - else: - if mustBeFound: - self.logMsg('warn', 'genRequirements: API {} not found'.format(name)) - return '' - else: - # No API dictionary available, return nothing - return '' - - def writeInclude(self, directory, basename, contents): - """Generate an include file. - - - directory - subdirectory to put file in - - basename - base name of the file - - contents - contents of the file (Asciidoc boilerplate aside)""" - # Create subdirectory, if needed - directory = self.genOpts.directory + '/' + directory - self.makeDir(directory) - - # Create file - filename = directory + '/' + basename + '.txt' - self.logMsg('diag', '# Generating include file:', filename) - fp = open(filename, 'w', encoding='utf-8') - - # Asciidoc anchor - write(self.genOpts.conventions.warning_comment, file=fp) - write('[[{0},{0}]]'.format(basename), file=fp) - - if self.genOpts.conventions.generate_index_terms: - index_terms = [] - if basename.startswith(self.conventions.command_prefix): - index_terms.append(basename[2:] + " (function)") - elif basename.startswith(self.conventions.type_prefix): - index_terms.append(basename[2:] + " (type)") - elif basename.startswith(self.conventions.api_prefix): - index_terms.append(basename[len(self.conventions.api_prefix):] + " (define)") - index_terms.append(basename) - write('indexterm:[{}]'.format(','.join(index_terms)), file=fp) - - write('[source,opencl]', file=fp) - write('----', file=fp) - write(contents, file=fp) - write('----', file=fp) - fp.close() - - if self.genOpts.secondaryInclude: - # Create secondary no cross-reference include file - filename = directory + '/' + basename + '.no-xref.txt' - self.logMsg('diag', '# Generating include file:', filename) - fp = open(filename, 'w', encoding='utf-8') - - # Asciidoc anchor - write(self.genOpts.conventions.warning_comment, file=fp) - write('// Include this no-xref version without cross reference id for multiple includes of same file', file=fp) - write('[source,opencl]', file=fp) - write('----', file=fp) - write(contents, file=fp) - write('----', file=fp) - fp.close() - - def writeTable(self, basename, values): - """Output a table of enumerants.""" - directory = Path(self.genOpts.directory) / 'enums' - self.makeDir(str(directory)) - - filename = str(directory / '{}.comments.txt'.format(basename)) - self.logMsg('diag', '# Generating include file:', filename) - - with open(filename, 'w', encoding='utf-8') as fp: - write(self.conventions.warning_comment, file=fp) - write(ENUM_TABLE_PREFIX, file=fp) - - for data in values: - write("|ename:{}".format(data['name']), file=fp) - write("|{}".format(data['comment']), file=fp) - - write(ENUM_TABLE_SUFFIX, file=fp) - - def writeFlagBox(self, basename, values): - """Output a box of flag bit comments.""" - directory = Path(self.genOpts.directory) / 'enums' - self.makeDir(str(directory)) - - filename = str(directory / '{}.comments.txt'.format(basename)) - self.logMsg('diag', '# Generating include file:', filename) - - with open(filename, 'w', encoding='utf-8') as fp: - write(self.conventions.warning_comment, file=fp) - write(FLAG_BLOCK_PREFIX, file=fp) - - for data in values: - write("* ename:{} -- {}".format(data['name'], - data['comment']), - file=fp) - - write(FLAG_BLOCK_SUFFIX, file=fp) - - def genType(self, typeinfo, name, alias): - """Generate type.""" - OutputGenerator.genType(self, typeinfo, name, alias) - typeElem = typeinfo.elem - # If the type is a struct type, traverse the embedded tags - # generating a structure. Otherwise, emit the tag text. - category = typeElem.get('category') - - if category in ('struct', 'union'): - # If the type is a struct type, generate it using the - # special-purpose generator. - self.genStruct(typeinfo, name, alias) - else: - body = self.genRequirements(name) - if alias: - # If the type is an alias, just emit a typedef declaration - body += 'typedef ' + alias + ' ' + name + ';\n' - self.writeInclude(OutputGenerator.categoryToPath[category], - name, body) - else: - # Replace tags with an APIENTRY-style string - # (from self.genOpts). Copy other text through unchanged. - # If the resulting text is an empty string, don't emit it. - body += noneStr(typeElem.text) - for elem in typeElem: - if elem.tag == 'apientry': - body += self.genOpts.apientry + noneStr(elem.tail) - else: - body += noneStr(elem.text) + noneStr(elem.tail) - - if body: - if category in OutputGenerator.categoryToPath: - self.writeInclude(OutputGenerator.categoryToPath[category], - name, body + '\n') - else: - self.logMsg('diag', '# NOT writing include file for type:', - name, '- bad category: ', category) - else: - self.logMsg('diag', '# NOT writing empty include file for type', name) - - def genStruct(self, typeinfo, typeName, alias): - """Generate struct.""" - OutputGenerator.genStruct(self, typeinfo, typeName, alias) - - typeElem = typeinfo.elem - - body = self.genRequirements(typeName) - if alias: - body += 'typedef ' + alias + ' ' + typeName + ';\n' - else: - body += 'typedef ' + typeElem.get('category') + ' ' + typeName + ' {\n' - - targetLen = self.getMaxCParamTypeLength(typeinfo) - for member in typeElem.findall('.//member'): - body += self.makeCParamDecl(member, targetLen + 4) - body += ';\n' - body += '} ' + typeName + ';' - - self.writeInclude('structs', typeName, body) - - def genEnumTable(self, groupinfo, groupName): - """Generate tables of enumerant values and short descriptions from - the XML.""" - - values = [] - got_comment = False - missing_comments = [] - for elem in groupinfo.elem.findall('enum'): - if not elem.get('required'): - continue - name = elem.get('name') - - data = { - 'name': name, - } - - (numVal, strVal) = self.enumToValue(elem, True) - data['value'] = numVal - - extname = elem.get('extname') - - added_by_extension_to_core = (extname is not None and self.in_core) - if added_by_extension_to_core and not self.genOpts.extEnumerantAdditions: - # We're skipping such values - continue - - comment = elem.get('comment') - if comment: - got_comment = True - elif name.endswith('_UNKNOWN') and numVal == 0: - # This is a placeholder for 0-initialization to be clearly invalid. - # Just skip this silently - continue - else: - # Skip but record this in case it's an odd-one-out missing a comment. - missing_comments.append(name) - continue - - if added_by_extension_to_core and self.genOpts.extEnumerantFormatString: - # Add a note to the comment - comment += self.genOpts.extEnumerantFormatString.format( - self.conventions.formatExtension(extname)) - - data['comment'] = comment - values.append(data) - - if got_comment: - # If any had a comment, output it. - - if missing_comments: - self.logMsg('warn', 'The following values for', groupName, - 'were omitted from the table due to missing comment attributes:', - ', '.join(missing_comments)) - - group_type = groupinfo.elem.get('type') - if groupName == self.result_type: - # Split this into success and failure - self.writeTable(groupName + '.success', - (data for data in values - if data['value'] >= 0)) - self.writeTable(groupName + '.error', - (data for data in values - if data['value'] < 0)) - elif group_type == 'bitmask': - self.writeFlagBox(groupName, values) - elif group_type == 'enum': - self.writeTable(groupName, values) - else: - raise RuntimeError("Unrecognized enums type: " + str(group_type)) - - def genGroup(self, groupinfo, groupName, alias): - """Generate group (e.g. C "enum" type).""" - OutputGenerator.genGroup(self, groupinfo, groupName, alias) - - body = self.genRequirements(groupName) - if alias: - # If the group name is aliased, just emit a typedef declaration - # for the alias. - body += 'typedef ' + alias + ' ' + groupName + ';\n' - else: - expand = self.genOpts.expandEnumerants - (_, enumbody) = self.buildEnumCDecl(expand, groupinfo, groupName) - body += enumbody - if self.genOpts.conventions.generate_enum_table: - self.genEnumTable(groupinfo, groupName) - - self.writeInclude('enums', groupName, body) - - def genEnum(self, enuminfo, name, alias): - """Generate the C declaration for a constant (a single value).""" - - OutputGenerator.genEnum(self, enuminfo, name, alias) - - body = self.buildConstantCDecl(enuminfo, name, alias) - - self.writeInclude('enums', name, body) - - def genCmd(self, cmdinfo, name, alias): - "Generate command." - OutputGenerator.genCmd(self, cmdinfo, name, alias) - - return_type = cmdinfo.elem.find('proto/type') - if self.genOpts.conventions.requires_error_validation(return_type): - # This command returns an API result code, so check that it - # returns at least the required errors. - # TODO move this to consistency_tools - required_errors = set(self.genOpts.conventions.required_errors) - errorcodes = cmdinfo.elem.get('errorcodes').split(',') - if not required_errors.issubset(set(errorcodes)): - self.logMsg('error', 'Missing required error code for command: ', name, '\n') - exit(1) - - body = self.genRequirements(name) - decls = self.makeCDecls(cmdinfo.elem) - body += decls[0] - self.writeInclude('protos', name, body) diff --git a/xml/extensionmetadocgenerator.py b/xml/extensionmetadocgenerator.py deleted file mode 100644 index d6243889d..000000000 --- a/xml/extensionmetadocgenerator.py +++ /dev/null @@ -1,659 +0,0 @@ -#!/usr/bin/python3 -i -# -# Copyright 2013-2023 The Khronos Group Inc. -# -# SPDX-License-Identifier: Apache-2.0 - -import os -import re -import sys -from functools import total_ordering -from generator import GeneratorOptions, OutputGenerator, regSortFeatures, write - -class ExtensionMetaDocGeneratorOptions(GeneratorOptions): - """ExtensionMetaDocGeneratorOptions - subclass of GeneratorOptions. - - Represents options during extension metainformation generation for Asciidoc""" - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - -EXT_NAME_DECOMPOSE_RE = re.compile(r'[A-Z]+_(?P[A-Z]+)_(?P[\w_]+)') - - -@total_ordering -class Extension: - def __init__(self, - generator, # needed for logging and API conventions - filename, - name, - number, - ext_type, - requires, - requiresCore, - contact, - promotedTo, - deprecatedBy, - obsoletedBy, - provisional, - revision, - specialuse ): - self.generator = generator - self.conventions = generator.genOpts.conventions - self.filename = filename - self.name = name - self.number = number - self.ext_type = ext_type - self.requires = requires - self.requiresCore = requiresCore - self.contact = contact - self.promotedTo = promotedTo - self.deprecatedBy = deprecatedBy - self.obsoletedBy = obsoletedBy - self.provisional = provisional - self.revision = revision - self.specialuse = specialuse - - self.deprecationType = None - self.supercedingAPIVersion = None - self.supercedingExtension = None - - if self.promotedTo is not None and self.deprecatedBy is not None and self.obsoletedBy is not None: - self.generator.logMsg('warn', 'All \'promotedto\', \'deprecatedby\' and \'obsoletedby\' attributes used on extension ' + self.name + '! Ignoring \'promotedto\' and \'deprecatedby\'.') - elif self.promotedTo is not None and self.deprecatedBy is not None: - self.generator.logMsg('warn', 'Both \'promotedto\' and \'deprecatedby\' attributes used on extension ' + self.name + '! Ignoring \'deprecatedby\'.') - elif self.promotedTo is not None and self.obsoletedBy is not None: - self.generator.logMsg('warn', 'Both \'promotedto\' and \'obsoletedby\' attributes used on extension ' + self.name + '! Ignoring \'promotedto\'.') - elif self.deprecatedBy is not None and self.obsoletedBy is not None: - self.generator.logMsg('warn', 'Both \'deprecatedby\' and \'obsoletedby\' attributes used on extension ' + self.name + '! Ignoring \'deprecatedby\'.') - - supercededBy = None - if self.promotedTo is not None: - self.deprecationType = 'promotion' - supercededBy = promotedTo - elif self.deprecatedBy is not None: - self.deprecationType = 'deprecation' - supercededBy = deprecatedBy - elif self.obsoletedBy is not None: - self.deprecationType = 'obsoletion' - supercededBy = obsoletedBy - - if supercededBy is not None: - if supercededBy == '' and not self.deprecationType == 'promotion': - pass # supercedingAPIVersion, supercedingExtension is None - elif supercededBy.startswith(self.conventions.api_version_prefix): - self.supercedingAPIVersion = supercededBy - elif supercededBy.startswith(self.conventions.api_prefix): - self.supercedingExtension = supercededBy - else: - self.generator.logMsg('error', 'Unrecognized ' + self.deprecationType + ' attribute value \'' + supercededBy + '\'!') - - match = EXT_NAME_DECOMPOSE_RE.match(self.name) - self.vendor = match.group('tag') - self.bare_name = match.group('name') - - def __str__(self): - return self.name - def __eq__(self, other): - return self.name == other.name - def __ne__(self, other): - return self.name != other.name - - def __lt__(self, other): - self_is_KHR = self.name.startswith(self.conventions.KHR_prefix) - self_is_EXT = self.name.startswith(self.conventions.EXT_prefix) - other_is_KHR = other.name.startswith(self.conventions.KHR_prefix) - other_is_EXT = other.name.startswith(self.conventions.EXT_prefix) - - swap = False - if self_is_KHR and not other_is_KHR: - return not swap - if other_is_KHR and not self_is_KHR: - return swap - if self_is_EXT and not other_is_EXT: - return not swap - if other_is_EXT and not self_is_EXT: - return swap - - return self.name < other.name - - def typeToStr(self): - if self.ext_type == 'instance': - return 'Instance extension' - if self.ext_type == 'device': - return 'Device extension' - - if self.ext_type is not None: - self.generator.logMsg('warn', 'The type attribute of ' + self.name + ' extension is neither \'instance\' nor \'device\'. That is invalid (at the time this script was written).') - else: # should be unreachable - self.generator.logMsg('error', 'Logic error in typeToStr(): Missing type attribute!') - return None - - def specLink(self, xrefName, xrefText, isRefpage = False): - """Generate a string containing a link to a specification anchor in - asciidoctor markup form. - - - xrefName - anchor name in the spec - - xrefText - text to show for the link, or None - - isRefpage = True if generating a refpage include, False if - generating a specification extension appendix include""" - - if isRefpage: - # Always link into API spec - specURL = self.conventions.specURL('api') - return 'link:{}#{}[{}^]'.format(specURL, xrefName, xrefText) - else: - return '<<' + xrefName + ', ' + xrefText + '>>' - - def conditionalLinkCoreAPI(self, apiVersion, linkSuffix, isRefpage): - versionMatch = re.match(self.conventions.api_version_prefix + r'(\d+)_(\d+)', apiVersion) - major = versionMatch.group(1) - minor = versionMatch.group(2) - - dottedVersion = major + '.' + minor - - xrefName = 'versions-' + dottedVersion + linkSuffix - xrefText = self.conventions.api_name() + ' ' + dottedVersion - - doc = 'ifdef::' + apiVersion + '[]\n' - doc += ' ' + self.specLink(xrefName, xrefText, isRefpage) + '\n' - doc += 'endif::' + apiVersion + '[]\n' - doc += 'ifndef::' + apiVersion + '[]\n' - doc += ' ' + self.conventions.api_name() + ' ' + dottedVersion + '\n' - doc += 'endif::' + apiVersion + '[]\n' - - return doc - - def conditionalLinkExt(self, extName, indent = ' '): - doc = 'ifdef::' + extName + '[]\n' - doc += indent + self.conventions.formatExtension(extName) + '\n' - doc += 'endif::' + extName + '[]\n' - doc += 'ifndef::' + extName + '[]\n' - doc += indent + '`' + extName + '`\n' - doc += 'endif::' + extName + '[]\n' - - return doc - - def resolveDeprecationChain(self, extensionsList, succeededBy, isRefpage, file): - ext = next(x for x in extensionsList if x.name == succeededBy) - - if ext.deprecationType: - if ext.deprecationType == 'promotion': - if ext.supercedingAPIVersion: - write(' ** Which in turn was _promoted_ to\n' + ext.conditionalLinkCoreAPI(ext.supercedingAPIVersion, '-promotions', isRefpage), file=file) - else: # ext.supercedingExtension - write(' ** Which in turn was _promoted_ to extension\n' + ext.conditionalLinkExt(ext.supercedingExtension), file=file) - ext.resolveDeprecationChain(extensionsList, ext.supercedingExtension, file) - elif ext.deprecationType == 'deprecation': - if ext.supercedingAPIVersion: - write(' ** Which in turn was _deprecated_ by\n' + ext.conditionalLinkCoreAPI(ext.supercedingAPIVersion, '-new-feature', isRefpage), file=file) - elif ext.supercedingExtension: - write(' ** Which in turn was _deprecated_ by\n' + ext.conditionalLinkExt(ext.supercedingExtension) + ' extension', file=file) - ext.resolveDeprecationChain(extensionsList, ext.supercedingExtension, file) - else: - write(' ** Which in turn was _deprecated_ without replacement', file=file) - elif ext.deprecationType == 'obsoletion': - if ext.supercedingAPIVersion: - write(' ** Which in turn was _obsoleted_ by\n' + ext.conditionalLinkCoreAPI(ext.supercedingAPIVersion, '-new-feature', isRefpage), file=file) - elif ext.supercedingExtension: - write(' ** Which in turn was _obsoleted_ by\n' + ext.conditionalLinkExt(ext.supercedingExtension) + ' extension', file=file) - ext.resolveDeprecationChain(extensionsList, ext.supercedingExtension, file) - else: - write(' ** Which in turn was _obsoleted_ without replacement', file=file) - else: # should be unreachable - self.generator.logMsg('error', 'Logic error in resolveDeprecationChain(): deprecationType is neither \'promotion\', \'deprecation\' nor \'obsoletion\'!') - - - def writeTag(self, tag, value, isRefpage, fp): - """Write a tag and (if non-None) a tag value to a file. - - - tag - string tag name - - value - tag value, or None - - isRefpage - controls style in which the tag is marked up - - fp - open file pointer to write to""" - - if isRefpage: - # Use subsection headers for the tag name - tagPrefix = '== ' - tagSuffix = '' - else: - # Use an bolded item list for the tag name - tagPrefix = '*' - tagSuffix = '*::' - - write(tagPrefix + tag + tagSuffix, file=fp) - if value is not None: - write(value, file=fp) - - if isRefpage: - write('', file=fp) - - def makeMetafile(self, extensionsList, isRefpage = False): - """Generate a file containing extension metainformation in - asciidoctor markup form. - - - extensionsList - list of extensions spec is being generated against - - isRefpage - True if generating a refpage include, False if - generating a specification extension appendix include""" - - if isRefpage: - filename = self.filename.replace('meta/', 'meta/refpage.') - else: - filename = self.filename - - fp = self.generator.newFile(filename) - - if not isRefpage: - write('[[' + self.name + ']]', file=fp) - write('=== ' + self.name, file=fp) - write('', file=fp) - - self.writeTag('Name String', '`' + self.name + '`', isRefpage, fp) - self.writeTag('Extension Type', self.typeToStr(), isRefpage, fp) - - self.writeTag('Registered Extension Number', self.number, isRefpage, fp) - self.writeTag('Revision', self.revision, isRefpage, fp) - - # Only API extension dependencies are coded in XML, others are explicit - self.writeTag('Extension and Version Dependencies', None, isRefpage, fp) - - write(' * Requires ' + self.conventions.api_name() + ' ' + self.requiresCore, file=fp) - if self.requires: - for dep in self.requires.split(','): - write(' * Requires', self.conventions.formatExtension(dep), - file=fp) - if self.provisional == 'true': - write(' * *This is a _provisional_ extension and must: be used with caution.', file=fp) - write(' See the ' + - self.specLink(xrefName = 'boilerplate-provisional-header', - xrefText = 'description', - isRefpage = isRefpage) + - ' of provisional header files for enablement and stability details.*', file=fp) - write('', file=fp) - - if self.deprecationType: - self.writeTag('Deprecation state', None, isRefpage, fp) - - if self.deprecationType == 'promotion': - if self.supercedingAPIVersion: - write(' * _Promoted_ to\n' + self.conditionalLinkCoreAPI(self.supercedingAPIVersion, '-promotions', isRefpage), file=fp) - else: # ext.supercedingExtension - write(' * _Promoted_ to\n' + self.conditionalLinkExt(self.supercedingExtension) + ' extension', file=fp) - self.resolveDeprecationChain(extensionsList, self.supercedingExtension, isRefpage, fp) - elif self.deprecationType == 'deprecation': - if self.supercedingAPIVersion: - write(' * _Deprecated_ by\n' + self.conditionalLinkCoreAPI(self.supercedingAPIVersion, '-new-features', isRefpage), file=fp) - elif self.supercedingExtension: - write(' * _Deprecated_ by\n' + self.conditionalLinkExt(self.supercedingExtension) + ' extension' , file=fp) - self.resolveDeprecationChain(extensionsList, self.supercedingExtension, isRefpage, fp) - else: - write(' * _Deprecated_ without replacement' , file=fp) - elif self.deprecationType == 'obsoletion': - if self.supercedingAPIVersion: - write(' * _Obsoleted_ by\n' + self.conditionalLinkCoreAPI(self.supercedingAPIVersion, '-new-features', isRefpage), file=fp) - elif self.supercedingExtension: - write(' * _Obsoleted_ by\n' + self.conditionalLinkExt(self.supercedingExtension) + ' extension' , file=fp) - self.resolveDeprecationChain(extensionsList, self.supercedingExtension, isRefpage, fp) - else: - # TODO: Does not make sense to retroactively ban use of extensions from 1.0. - # Needs some tweaks to the semantics and this message, when such extension(s) occur. - write(' * _Obsoleted_ without replacement' , file=fp) - else: # should be unreachable - self.generator.logMsg('error', 'Logic error in makeMetafile(): deprecationType is neither \'promotion\', \'deprecation\' nor \'obsoletion\'!') - write('', file=fp) - - if self.specialuse is not None: - specialuses = self.specialuse.split(',') - if len(specialuses) > 1: - header = 'Special Uses' - else: - header = 'Special Use' - self.writeTag(header, None, isRefpage, fp) - - for use in specialuses: - # Each specialuse attribute value expands an asciidoctor - # attribute of the same name, instead of using the shorter, - # and harder to understand attribute - write('* {}'.format( - self.specLink( - xrefName = self.conventions.special_use_section_anchor, - xrefText = '{' + use + '}', - isRefpage = isRefpage)), file=fp) - write('', file=fp) - - if self.conventions.write_contacts: - self.writeTag('Contact', None, isRefpage, fp) - - contacts = self.contact.split(',') - for contact in contacts: - contactWords = contact.strip().split() - name = ' '.join(contactWords[:-1]) - handle = contactWords[-1] - if handle.startswith('gitlab:'): - prettyHandle = 'icon:gitlab[alt=GitLab, role="red"]' + handle.replace('gitlab:@', '') - elif handle.startswith('@'): - issuePlaceholderText = '[' + self.name + '] ' + handle - issuePlaceholderText += '%0A<>' - trackerLink = 'link:++https://github.com/KhronosGroup/Vulkan-Docs/issues/new?body=' + issuePlaceholderText + '++' - prettyHandle = trackerLink + '[icon:github[alt=GitHub,role="black"]' + handle[1:] + ', window=_blank]' - else: - prettyHandle = handle - - write(' * ' + name + ' ' + prettyHandle, file=fp) - write('', file=fp) - - # Check if a proposal document for this extension exists in the - # current repository, and link to the same document (parameterized - # by a URL prefix attribute) if it does. - # The assumption is that a proposal document for an extension - # VK_name will be located in 'proposals/VK_name.asciidoc' relative - # to the repository root, and that this script will be invoked from - # the repository root. - path = 'proposals/{}.asciidoc'.format(self.name) - if os.path.exists(path) and os.access(path, os.R_OK): - self.writeTag('Extension Proposal', - 'link:{{specRepositoryURL}}/{}[{}]'.format(path, self.name), isRefpage, fp) - - fp.close() - -class ExtensionMetaDocOutputGenerator(OutputGenerator): - """ExtensionMetaDocOutputGenerator - subclass of OutputGenerator. - - Generates AsciiDoc includes with metainformation for the API extension - appendices. The fields used from tags in the API XML are: - - - name extension name string - - number extension number (optional) - - contact name and GitHub login or email address (optional) - - type 'instance' | 'device' (optional) - - requires list of comma-separated required API extensions (optional) - - requiresCore required core version of API (optional) - - promotedTo extension or API version it was promoted to - - deprecatedBy extension or API version which deprecated this extension, - or empty string if deprecated without replacement - - obsoletedBy extension or API version which obsoleted this extension, - or empty string if obsoleted without replacement - - provisional 'true' if this extension is released provisionally""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.extensions = [] - # List of strings containing all vendor tags - self.vendor_tags = [] - self.file_suffix = '' - - def newFile(self, filename): - self.logMsg('diag', '# Generating include file:', filename) - fp = open(filename, 'w', encoding='utf-8') - write(self.genOpts.conventions.warning_comment, file=fp) - return fp - - def beginFile(self, genOpts): - OutputGenerator.beginFile(self, genOpts) - - self.directory = self.genOpts.directory - self.file_suffix = self.genOpts.conventions.file_suffix - - # Iterate over all 'tag' Elements and add the names of all the valid vendor - # tags to the list - root = self.registry.tree.getroot() - for tag in root.findall('tags/tag'): - self.vendor_tags.append(tag.get('name')) - - # Create subdirectory, if needed - self.makeDir(self.directory) - - def conditionalExt(self, extName, content, ifdef = None, condition = None): - doc = '' - - innerdoc = 'ifdef::' + extName + '[]\n' - innerdoc += content + '\n' - innerdoc += 'endif::' + extName + '[]\n' - - if ifdef: - if ifdef == 'ifndef': - if condition: - doc += 'ifndef::' + condition + '[]\n' - doc += innerdoc - doc += 'endif::' + condition + '[]\n' - else: # no condition is as if condition is defined; "nothing" is always defined :p - pass # so no output - elif ifdef == 'ifdef': - if condition: - doc += 'ifdef::' + condition + '+' + extName + '[]\n' - doc += content + '\n' # does not include innerdoc; the ifdef was merged with the one above - doc += 'endif::' + condition + '+' + extName + '[]\n' - else: # no condition is as if condition is defined; "nothing" is always defined :p - doc += innerdoc - else: # should be unreachable - raise RuntimeError('Should be unreachable: ifdef is neither \'ifdef \' nor \'ifndef\'!') - else: - doc += innerdoc - - return doc - - def makeExtensionInclude(self, ext): - return self.conventions.extension_include_string(ext) - - def endFile(self): - self.extensions.sort() - - # Generate metadoc extension files, in refpage and non-refpage form - for ext in self.extensions: - ext.makeMetafile(self.extensions, isRefpage = False) - if self.conventions.write_refpage_include: - ext.makeMetafile(self.extensions, isRefpage = True) - - # Generate list of promoted extensions - promotedExtensions = {} - for ext in self.extensions: - if ext.deprecationType == 'promotion' and ext.supercedingAPIVersion: - promotedExtensions.setdefault(ext.supercedingAPIVersion, []).append(ext) - - for coreVersion, extensions in promotedExtensions.items(): - promoted_extensions_fp = self.newFile(self.directory + '/promoted_extensions_' + coreVersion + self.file_suffix) - - for ext in extensions: - indent = '' - write(' * {blank}\n+\n' + ext.conditionalLinkExt(ext.name, indent), file=promoted_extensions_fp) - - promoted_extensions_fp.close() - - # Re-sort to match earlier behavior - # TODO: Remove this extra sort when re-arranging section order OK. - - def makeSortKey(ext): - name = ext.name.lower() - prefixes = self.conventions.extension_index_prefixes - for i, prefix in enumerate(prefixes): - if ext.name.startswith(prefix): - return (i, name) - return (len(prefixes), name) - - self.extensions.sort(key=makeSortKey) - - # Generate include directives for the extensions appendix, grouping - # extensions by status (current, deprecated, provisional, etc.) - with self.newFile(self.directory + '/current_extensions_appendix' + self.file_suffix) as current_extensions_appendix_fp, \ - self.newFile(self.directory + '/deprecated_extensions_appendix' + self.file_suffix) as deprecated_extensions_appendix_fp, \ - self.newFile(self.directory + '/current_extension_appendices' + self.file_suffix) as current_extension_appendices_fp, \ - self.newFile(self.directory + '/current_extension_appendices_toc' + self.file_suffix) as current_extension_appendices_toc_fp, \ - self.newFile(self.directory + '/deprecated_extension_appendices' + self.file_suffix) as deprecated_extension_appendices_fp, \ - self.newFile(self.directory + '/deprecated_extension_appendices_toc' + self.file_suffix) as deprecated_extension_appendices_toc_fp, \ - self.newFile(self.directory + '/deprecated_extensions_guard_macro' + self.file_suffix) as deprecated_extensions_guard_macro_fp, \ - self.newFile(self.directory + '/provisional_extensions_appendix' + self.file_suffix) as provisional_extensions_appendix_fp, \ - self.newFile(self.directory + '/provisional_extension_appendices' + self.file_suffix) as provisional_extension_appendices_fp, \ - self.newFile(self.directory + '/provisional_extension_appendices_toc' + self.file_suffix) as provisional_extension_appendices_toc_fp, \ - self.newFile(self.directory + '/provisional_extensions_guard_macro' + self.file_suffix) as provisional_extensions_guard_macro_fp: - - write('', file=current_extensions_appendix_fp) - write('include::deprecated_extensions_guard_macro' + self.file_suffix + '[]', file=current_extensions_appendix_fp) - write('', file=current_extensions_appendix_fp) - write('ifndef::HAS_DEPRECATED_EXTENSIONS[]', file=current_extensions_appendix_fp) - write('[[extension-appendices-list]]', file=current_extensions_appendix_fp) - write('== List of Extensions', file=current_extensions_appendix_fp) - write('endif::HAS_DEPRECATED_EXTENSIONS[]', file=current_extensions_appendix_fp) - write('ifdef::HAS_DEPRECATED_EXTENSIONS[]', file=current_extensions_appendix_fp) - write('[[extension-appendices-list]]', file=current_extensions_appendix_fp) - write('== List of Current Extensions', file=current_extensions_appendix_fp) - write('endif::HAS_DEPRECATED_EXTENSIONS[]', file=current_extensions_appendix_fp) - write('', file=current_extensions_appendix_fp) - write('include::current_extension_appendices_toc' + self.file_suffix + '[]', file=current_extensions_appendix_fp) - write('\n<<<\n', file=current_extensions_appendix_fp) - write('include::current_extension_appendices' + self.file_suffix + '[]', file=current_extensions_appendix_fp) - - write('', file=deprecated_extensions_appendix_fp) - write('include::deprecated_extensions_guard_macro' + self.file_suffix + '[]', file=deprecated_extensions_appendix_fp) - write('', file=deprecated_extensions_appendix_fp) - write('ifdef::HAS_DEPRECATED_EXTENSIONS[]', file=deprecated_extensions_appendix_fp) - write('[[deprecated-extension-appendices-list]]', file=deprecated_extensions_appendix_fp) - write('== List of Deprecated Extensions', file=deprecated_extensions_appendix_fp) - write('include::deprecated_extension_appendices_toc' + self.file_suffix + '[]', file=deprecated_extensions_appendix_fp) - write('\n<<<\n', file=deprecated_extensions_appendix_fp) - write('include::deprecated_extension_appendices' + self.file_suffix + '[]', file=deprecated_extensions_appendix_fp) - write('endif::HAS_DEPRECATED_EXTENSIONS[]', file=deprecated_extensions_appendix_fp) - - # add include guards to allow multiple includes - write('ifndef::DEPRECATED_EXTENSIONS_GUARD_MACRO_INCLUDE_GUARD[]', file=deprecated_extensions_guard_macro_fp) - write(':DEPRECATED_EXTENSIONS_GUARD_MACRO_INCLUDE_GUARD:\n', file=deprecated_extensions_guard_macro_fp) - write('ifndef::PROVISIONAL_EXTENSIONS_GUARD_MACRO_INCLUDE_GUARD[]', file=provisional_extensions_guard_macro_fp) - write(':PROVISIONAL_EXTENSIONS_GUARD_MACRO_INCLUDE_GUARD:\n', file=provisional_extensions_guard_macro_fp) - - write('', file=provisional_extensions_appendix_fp) - write('include::provisional_extensions_guard_macro' + self.file_suffix + '[]', file=provisional_extensions_appendix_fp) - write('', file=provisional_extensions_appendix_fp) - write('ifdef::HAS_PROVISIONAL_EXTENSIONS[]', file=provisional_extensions_appendix_fp) - write('[[provisional-extension-appendices-list]]', file=provisional_extensions_appendix_fp) - write('== List of Provisional Extensions', file=provisional_extensions_appendix_fp) - write('include::provisional_extension_appendices_toc' + self.file_suffix + '[]', file=provisional_extensions_appendix_fp) - write('\n<<<\n', file=provisional_extensions_appendix_fp) - write('include::provisional_extension_appendices' + self.file_suffix + '[]', file=provisional_extensions_appendix_fp) - write('endif::HAS_PROVISIONAL_EXTENSIONS[]', file=provisional_extensions_appendix_fp) - - for ext in self.extensions: - include = self.makeExtensionInclude(ext) - link = ' * ' + self.conventions.formatExtension(ext.name) - if ext.provisional == 'true': - write(self.conditionalExt(ext.name, include), file=provisional_extension_appendices_fp) - write(self.conditionalExt(ext.name, link), file=provisional_extension_appendices_toc_fp) - write(self.conditionalExt(ext.name, ':HAS_PROVISIONAL_EXTENSIONS:'), file=provisional_extensions_guard_macro_fp) - elif ext.deprecationType is None: - write(self.conditionalExt(ext.name, include), file=current_extension_appendices_fp) - write(self.conditionalExt(ext.name, link), file=current_extension_appendices_toc_fp) - else: - condition = ext.supercedingAPIVersion if ext.supercedingAPIVersion else ext.supercedingExtension # potentially None too - - write(self.conditionalExt(ext.name, include, 'ifndef', condition), file=current_extension_appendices_fp) - write(self.conditionalExt(ext.name, link, 'ifndef', condition), file=current_extension_appendices_toc_fp) - - write(self.conditionalExt(ext.name, include, 'ifdef', condition), file=deprecated_extension_appendices_fp) - write(self.conditionalExt(ext.name, link, 'ifdef', condition), file=deprecated_extension_appendices_toc_fp) - - write(self.conditionalExt(ext.name, ':HAS_DEPRECATED_EXTENSIONS:', 'ifdef', condition), file=deprecated_extensions_guard_macro_fp) - - write('endif::DEPRECATED_EXTENSIONS_GUARD_MACRO_INCLUDE_GUARD[]', file=deprecated_extensions_guard_macro_fp) - write('endif::PROVISIONAL_EXTENSIONS_GUARD_MACRO_INCLUDE_GUARD[]', file=provisional_extensions_guard_macro_fp) - - OutputGenerator.endFile(self) - - def beginFeature(self, interface, emit): - # Start processing in superclass - OutputGenerator.beginFeature(self, interface, emit) - - if interface.tag != 'extension': - self.logMsg('diag', 'beginFeature: ignoring non-extension feature', self.featureName) - return - - # These attributes must exist - name = self.featureName - number = self.getAttrib(interface, 'number') - ext_type = self.getAttrib(interface, 'type') - revision = self.getSpecVersion(interface, name) - - # These attributes are optional - OPTIONAL = False - requires = self.getAttrib(interface, 'requires', OPTIONAL) - requiresCore = self.getAttrib(interface, 'requiresCore', OPTIONAL, '1.0') # TODO update this line with update_version.py - contact = self.getAttrib(interface, 'contact', OPTIONAL) - promotedTo = self.getAttrib(interface, 'promotedto', OPTIONAL) - deprecatedBy = self.getAttrib(interface, 'deprecatedby', OPTIONAL) - obsoletedBy = self.getAttrib(interface, 'obsoletedby', OPTIONAL) - provisional = self.getAttrib(interface, 'provisional', OPTIONAL, 'false') - specialuse = self.getAttrib(interface, 'specialuse', OPTIONAL) - - filename = self.directory + '/' + name + self.file_suffix - - extdata = Extension( - generator = self, - filename = filename, - name = name, - number = number, - ext_type = ext_type, - requires = requires, - requiresCore = requiresCore, - contact = contact, - promotedTo = promotedTo, - deprecatedBy = deprecatedBy, - obsoletedBy = obsoletedBy, - provisional = provisional, - revision = revision, - specialuse = specialuse) - self.extensions.append(extdata) - - - def endFeature(self): - # Finish processing in superclass - OutputGenerator.endFeature(self) - - def getAttrib(self, elem, attribute, required=True, default=None): - """Query an attribute from an element, or return a default value - - - elem - element to query - - attribute - attribute name - - required - whether attribute must exist - - default - default value if attribute not present""" - attrib = elem.get(attribute, default) - if required and (attrib is None): - name = elem.get('name', 'UNKNOWN') - self.logMsg('error', 'While processing \'' + self.featureName + ', <' + elem.tag + '> \'' + name + '\' does not contain required attribute \'' + attribute + '\'') - return attrib - - def numbersToWords(self, name): - allowlist = ['WIN32', 'INT16', 'D3D1'] - - # temporarily replace allowlist items - for i, w in enumerate(allowlist): - name = re.sub(w, '{' + str(i) + '}', name) - - name = re.sub(r'(?<=[A-Z])(\d+)(?![A-Z])', r'_\g<1>', name) - - # undo allowlist substitution - for i, w in enumerate(allowlist): - name = re.sub('\\{' + str(i) + '}', w, name) - - return name - - def getSpecVersion(self, elem, extname, default=None): - """Determine the extension revision from the EXTENSION_NAME_SPEC_VERSION - enumerant. - - - elem - element to query - - extname - extension name from the 'name' attribute - - default - default value if SPEC_VERSION token not present""" - # The literal enumerant name to match - versioningEnumName = self.numbersToWords(extname.upper()) + '_SPEC_VERSION' - - for enum in elem.findall('./require/enum'): - enumName = self.getAttrib(enum, 'name') - if enumName == versioningEnumName: - return self.getAttrib(enum, 'value') - - #if not found: - for enum in elem.findall('./require/enum'): - enumName = self.getAttrib(enum, 'name') - if enumName.find('SPEC_VERSION') != -1: - self.logMsg('diag', 'Missing ' + versioningEnumName + '! Potential misnamed candidate ' + enumName + '.') - return self.getAttrib(enum, 'value') - - self.logMsg('error', 'Missing ' + versioningEnumName + '!') - return default diff --git a/xml/genRef.py b/xml/genRef.py deleted file mode 100644 index 87d8d7a60..000000000 --- a/xml/genRef.py +++ /dev/null @@ -1,1019 +0,0 @@ -#!/usr/bin/python3 -# -# Copyright 2016-2023 The Khronos Group Inc. -# -# SPDX-License-Identifier: Apache-2.0 - -# genRef.py - create API ref pages from spec source files -# -# Usage: genRef.py files - -import argparse -import io -import os -import re -import sys -from collections import OrderedDict -from reflib import (findRefs, fixupRefs, loadFile, logDiag, logWarn, - printPageInfo, setLogFile) -from reg import Registry -from clconventions import OpenCLConventions as APIConventions - - -def makeExtensionInclude(name): - """Return an include command, given an extension name.""" - return 'include::{}/refpage.{}{}[]'.format( - conventions.specification_path, - name, - conventions.file_suffix) - - -def makeAPIInclude(type, name): - """Return an include command for a generated API interface - - type - type of the API, e.g. 'flags', 'handles', etc - - name - name of the API""" - - return 'include::{}/api/{}/{}{}\n'.format( - conventions.refpage_generated_include_path, - type, name, conventions.file_suffix) - - -def isextension(name): - """Return True if name is an API extension name (ends with an upper-case - author ID). - - This assumes that author IDs are at least two characters.""" - return name[-2:].isalpha() and name[-2:].isupper() - - -def printCopyrightSourceComments(fp): - """Print Khronos CC-BY copyright notice on open file fp. - - Writes an asciidoc comment block, which copyrights the source - file.""" - print('// Copyright 2014-2023 The Khronos Group, Inc.', file=fp) - print('//', file=fp) - # This works around constraints of the 'reuse' tool - print('// SPDX' + '-License-Identifier: CC-BY-4.0', file=fp) - print('', file=fp) - - -def printFooter(fp): - """Print footer material at the end of each refpage on open file fp. - - If generating separate refpages, adds the copyright. - If generating the single combined refpage, just add a separator.""" - - print('ifdef::doctype-manpage[]', - '== Copyright', - '', - 'include::{config}/copyright-ccby.txt[]', - 'endif::doctype-manpage[]', - '', - 'ifndef::doctype-manpage[]', - '<<<', - 'endif::doctype-manpage[]', - '', - sep='\n', file=fp) - - -def macroPrefix(name): - """Add a spec asciidoc macro prefix to an API name, depending on its type - (protos, structs, enums, etc.). - - If the name is not recognized, use the generic link macro 'reflink:'.""" - if name in api.basetypes: - return 'basetype:' + name - if name in api.defines: - return 'dlink:' + name - if name in api.enums: - return 'elink:' + name - if name in api.flags: - return 'elink:' + name - if name in api.funcpointers: - return 'tlink:' + name - if name in api.handles: - return 'slink:' + name - if name in api.protos: - return 'flink:' + name - if name in api.structs: - return 'slink:' + name - if name == 'TBD': - return 'No cross-references are available' - return 'reflink:' + name - - -def seeAlsoList(apiName, explicitRefs=None, apiAliases=[]): - """Return an asciidoc string with a list of 'See Also' references for the - API entity 'apiName', based on the relationship mapping in the api module. - - 'explicitRefs' is a list of additional cross-references. - - If apiAliases is not None, it is a list of aliases of apiName whose - cross-references will also be included. - - If no relationships are available, return None.""" - - refs = set(()) - - # apiName and its aliases are treated equally - allApis = apiAliases.copy() - allApis.append(apiName) - - # Add all the implicit references to refs - for name in allApis: - if name in api.mapDict: - refs.update(api.mapDict[name]) - - # Add all the explicit references - if explicitRefs is not None: - if isinstance(explicitRefs, str): - explicitRefs = explicitRefs.split() - refs.update(name for name in explicitRefs) - - # Add extensions / core versions based on dependencies - for name in allApis: - if name in api.requiredBy: - for (base,dependency) in api.requiredBy[name]: - refs.add(base) - if dependency is not None: - refs.add(dependency) - - if len(refs) == 0: - return None - else: - return ', '.join(macroPrefix(name) for name in sorted(refs)) + '\n' - - -def remapIncludes(lines, baseDir, specDir): - """Remap include directives in a list of lines so they can be extracted to a - different directory. - - Returns remapped lines. - - - lines - text to remap - - baseDir - target directory - - specDir - source directory""" - # This should be compiled only once - includePat = re.compile(r'^include::(?P.*)\[\]') - - newLines = [] - for line in lines: - matches = includePat.search(line) - if matches is not None: - path = matches.group('path') - - if path[0] != '{': - # Relative path to include file from here - incPath = specDir + '/' + path - # Remap to be relative to baseDir - newPath = os.path.relpath(incPath, baseDir) - newLine = 'include::' + newPath + '[]\n' - logDiag('remapIncludes: remapping', line, '->', newLine) - newLines.append(newLine) - else: - # An asciidoctor variable starts the path. - # This must be an absolute path, not needing to be rewritten. - newLines.append(line) - else: - newLines.append(line) - return newLines - - -def refPageShell(pageName, pageDesc, fp, head_content = None, sections=None, tail_content=None, man_section=3): - """Generate body of a reference page. - - - pageName - string name of the page - - pageDesc - string short description of the page - - fp - file to write to - - head_content - text to include before the sections - - sections - iterable returning (title,body) for each section. - - tail_content - text to include after the sections - - man_section - Unix man page section""" - - printCopyrightSourceComments(fp) - - print(':data-uri:', - ':icons: font', - conventions.extra_refpage_headers, - '', - sep='\n', file=fp) - - s = '{}({})'.format(pageName, man_section) - print('= ' + s, - '', - sep='\n', file=fp) - if pageDesc.strip() == '': - pageDesc = 'NO SHORT DESCRIPTION PROVIDED' - logWarn('refPageHead: no short description provided for', pageName) - - print('== Name', - '{} - {}'.format(pageName, pageDesc), - '', - sep='\n', file=fp) - - if head_content is not None: - print(head_content, - '', - sep='\n', file=fp) - - if sections is not None: - for title, content in sections.items(): - print('== {}'.format(title), - '', - content, - '', - sep='\n', file=fp) - - if tail_content is not None: - print(tail_content, - '', - sep='\n', file=fp) - - -def refPageHead(pageName, pageDesc, specText, fieldName, fieldText, descText, fp): - """Generate header of a reference page. - - - pageName - string name of the page - - pageDesc - string short description of the page - - specType - string containing 'spec' field from refpage open block, or None. - Used to determine containing spec name and URL. - - specText - string that goes in the "C Specification" section - - fieldName - string heading an additional section following specText, if not None - - fieldText - string that goes in the additional section - - descText - string that goes in the "Description" section - - fp - file to write to""" - sections = OrderedDict() - - if specText is not None: - sections['C Specification'] = specText - - if fieldName is not None: - sections[fieldName] = fieldText - - if descText is None or descText.strip() == '': - logWarn('refPageHead: no description provided for', pageName) - - if descText is not None: - sections['Description'] = descText - - refPageShell(pageName, pageDesc, fp, head_content=None, sections=sections) - - -def refPageTail(pageName, - specType=None, - specAnchor=None, - seeAlso=None, - fp=None, - auto=False): - """Generate end boilerplate of a reference page. - - - pageName - name of the page - - specType - None or the 'spec' attribute from the refpage block, - identifying the specification name and URL this refpage links to. - - specAnchor - None or the 'anchor' attribute from the refpage block, - identifying the anchor in the specification this refpage links to. If - None, the pageName is assumed to be a valid anchor.""" - - specName = conventions.api_name(specType) - specURL = conventions.specURL(specType) - if specAnchor is None: - specAnchor = pageName - - if seeAlso is None: - seeAlso = 'No cross-references are available\n' - - notes = [ - 'For more information, see the {}#{}[{} Specification^]'.format( - specURL, specAnchor, specName), - '', - ] - - if auto: - notes.extend(( - 'This page is a generated document.', - 'Fixes and changes should be made to the generator scripts, ' - 'not directly.', - )) - else: - notes.extend(( - 'This page is extracted from the ' + specName + ' Specification. ', - 'Fixes and changes should be made to the Specification, ' - 'not directly.', - )) - - print('== See Also', - '', - seeAlso, - '', - sep='\n', file=fp) - - print('== Document Notes', - '', - '\n'.join(notes), - '', - sep='\n', file=fp) - - printFooter(fp) - - -def xrefRewriteInitialize(): - """Initialize substitution patterns for asciidoctor xrefs.""" - - global refLinkPattern, refLinkSubstitute - global refLinkTextPattern, refLinkTextSubstitute - global specLinkPattern, specLinkSubstitute - - # These are xrefs to Vulkan API entities, rewritten to link to refpages - # The refLink variants are for xrefs with only an anchor and no text. - # The refLinkText variants are for xrefs with both anchor and text - refLinkPattern = re.compile(r'<<([Vv][Kk][^>,]+)>>') - refLinkSubstitute = r'link:\1.html[\1^]' - - refLinkTextPattern = re.compile(r'<<([Vv][Kk][^>,]+)[,]?[ \t\n]*([^>,]*)>>') - refLinkTextSubstitute = r'link:\1.html[\2^]' - - # These are xrefs to other anchors, rewritten to link to the spec - specLinkPattern = re.compile(r'<<([^>,]+)[,]?[ \t\n]*([^>,]*)>>') - - # Unfortunately, specLinkSubstitute depends on the link target, - # so can't be constructed in advance. - specLinkSubstitute = None - - -def xrefRewrite(text, specURL): - """Rewrite asciidoctor xrefs in text to resolve properly in refpages. - Xrefs which are to Vulkan refpages are rewritten to link to those - refpages. The remainder are rewritten to generate external links into - the supplied specification document URL. - - - text - string to rewrite, or None - - specURL - URL to target - - Returns rewritten text, or None, respectively""" - - global refLinkPattern, refLinkSubstitute - global refLinkTextPattern, refLinkTextSubstitute - global specLinkPattern, specLinkSubstitute - - specLinkSubstitute = r'link:{}#\1[\2^]'.format(specURL) - - if text is not None: - text, _ = refLinkPattern.subn(refLinkSubstitute, text) - text, _ = refLinkTextPattern.subn(refLinkTextSubstitute, text) - text, _ = specLinkPattern.subn(specLinkSubstitute, text) - - return text - -def emitPage(baseDir, specDir, pi, file): - """Extract a single reference page into baseDir. - - - baseDir - base directory to emit page into - - specDir - directory extracted page source came from - - pi - pageInfo for this page relative to file - - file - list of strings making up the file, indexed by pi""" - pageName = baseDir + '/' + pi.name + '.txt' - - # Add a dictionary entry for this page - global genDict - genDict[pi.name] = None - logDiag('emitPage:', pageName) - - # Short description - if pi.desc is None: - pi.desc = '(no short description available)' - - # Member/parameter section label and text, if there is one - field = None - fieldText = None - - if pi.type != 'freeform' and pi.type != 'spirv': - if pi.include is None: - # Not sure how this happens yet - logWarn('emitPage:', pageName, 'INCLUDE is None, no page generated') - return - - # Specification text from beginning to just before the parameter - # section. This covers the description, the prototype, the version - # note, and any additional version note text. If a parameter section - # is absent then go a line beyond the include. - remap_end = pi.include + 1 if pi.param is None else pi.param - lines = remapIncludes(file[pi.begin:remap_end], baseDir, specDir) - specText = ''.join(lines) - - if pi.param is not None: - if pi.type == 'structs': - field = 'Members' - elif pi.type in ['protos', 'funcpointers']: - field = 'Parameters' - else: - logWarn('emitPage: unknown field type:', pi.type, - 'for', pi.name) - lines = remapIncludes(file[pi.param:pi.body], baseDir, specDir) - fieldText = ''.join(lines) - - # Description text - if pi.body != pi.include: - lines = remapIncludes(file[pi.body:pi.end + 1], baseDir, specDir) - descText = ''.join(lines) - else: - descText = None - logWarn('emitPage: INCLUDE == BODY, so description will be empty for', pi.name) - if pi.begin != pi.include: - logWarn('emitPage: Note: BEGIN != INCLUDE, so the description might be incorrectly located before the API include!') - else: - specText = None - descText = ''.join(file[pi.begin:pi.end + 1]) - - # Rewrite asciidoctor xrefs to resolve properly in refpages - specURL = conventions.specURL(pi.spec) - - specText = xrefRewrite(specText, specURL) - fieldText = xrefRewrite(fieldText, specURL) - descText = xrefRewrite(descText, specURL) - - fp = open(pageName, 'w', encoding='utf-8') - refPageHead(pi.name, - pi.desc, - specText, - field, fieldText, - descText, - fp) - refPageTail(pageName=pi.name, - specType=pi.spec, - specAnchor=pi.anchor, - seeAlso=seeAlsoList(pi.name, pi.refs, pi.alias.split()), - fp=fp, - auto=False) - fp.close() - - -def autoGenEnumsPage(baseDir, pi, file): - """Autogenerate a single reference page in baseDir. - - Script only knows how to do this for /enums/ pages, at present. - - - baseDir - base directory to emit page into - - pi - pageInfo for this page relative to file - - file - list of strings making up the file, indexed by pi""" - pageName = baseDir + '/' + pi.name + '.txt' - fp = open(pageName, 'w', encoding='utf-8') - - # Add a dictionary entry for this page - global genDict - genDict[pi.name] = None - logDiag('autoGenEnumsPage:', pageName) - - # Short description - if pi.desc is None: - pi.desc = '(no short description available)' - - # Description text. Allow for the case where an enum definition - # is not embedded. - if not pi.embed: - embedRef = '' - else: - embedRef = ''.join(( - ' * The reference page for ', - macroPrefix(pi.embed), - ', where this interface is defined.\n')) - - txt = ''.join(( - 'For more information, see:\n\n', - embedRef, - ' * The See Also section for other reference pages using this type.\n', - ' * The ' + apiName + ' Specification.\n')) - - refPageHead(pi.name, - pi.desc, - ''.join(file[pi.begin:pi.include + 1]), - None, None, - txt, - fp) - refPageTail(pageName=pi.name, - specType=pi.spec, - specAnchor=pi.anchor, - seeAlso=seeAlsoList(pi.name, pi.refs, pi.alias.split()), - fp=fp, - auto=True) - fp.close() - - -# Pattern to break apart an API *Flags{authorID} name, used in -# autoGenFlagsPage. -flagNamePat = re.compile(r'(?P\w+)Flags(?P[A-Z]*)') - - -def autoGenFlagsPage(baseDir, flagName): - """Autogenerate a single reference page in baseDir for an API *Flags type. - - - baseDir - base directory to emit page into - - flagName - API *Flags name""" - pageName = baseDir + '/' + flagName + '.txt' - fp = open(pageName, 'w', encoding='utf-8') - - # Add a dictionary entry for this page - global genDict - genDict[flagName] = None - logDiag('autoGenFlagsPage:', pageName) - - # Short description - matches = flagNamePat.search(flagName) - if matches is not None: - name = matches.group('name') - author = matches.group('author') - logDiag('autoGenFlagsPage: split name into', name, 'Flags', author) - flagBits = name + 'FlagBits' + author - desc = 'Bitmask of ' + flagBits - else: - logWarn('autoGenFlagsPage:', pageName, 'does not end in "Flags{author ID}". Cannot infer FlagBits type.') - flagBits = None - desc = 'Unknown ' + apiName + ' flags type' - - # Description text - if flagBits is not None: - txt = ''.join(( - 'etext:' + flagName, - ' is a mask of zero or more elink:' + flagBits + '.\n', - 'It is used as a member and/or parameter of the structures and commands\n', - 'in the See Also section below.\n')) - else: - txt = ''.join(( - 'etext:' + flagName, - ' is an unknown ' + apiName + ' type, assumed to be a bitmask.\n')) - - refPageHead(flagName, - desc, - makeAPIInclude('flags', flagName), - None, None, - txt, - fp) - refPageTail(pageName=flagName, - specType=pi.spec, - specAnchor=pi.anchor, - seeAlso=seeAlsoList(flagName, None), - fp=fp, - auto=True) - fp.close() - - -def autoGenHandlePage(baseDir, handleName): - """Autogenerate a single handle page in baseDir for an API handle type. - - - baseDir - base directory to emit page into - - handleName - API handle name""" - # @@ Need to determine creation function & add handles/ include for the - # @@ interface in generator.py. - pageName = baseDir + '/' + handleName + '.txt' - fp = open(pageName, 'w', encoding='utf-8') - - # Add a dictionary entry for this page - global genDict - genDict[handleName] = None - logDiag('autoGenHandlePage:', pageName) - - # Short description - desc = apiName + ' object handle' - - descText = ''.join(( - 'sname:' + handleName, - ' is an object handle type, referring to an object used\n', - 'by the ' + apiName + ' implementation. These handles are created or allocated\n', - 'by the @@ TBD @@ function, and used by other ' + apiName + ' structures\n', - 'and commands in the See Also section below.\n')) - - refPageHead(handleName, - desc, - makeAPIInclude('handles', handleName), - None, None, - descText, - fp) - refPageTail(pageName=handleName, - specType=pi.spec, - specAnchor=pi.anchor, - seeAlso=seeAlsoList(handleName, None), - fp=fp, - auto=True) - fp.close() - - -def genRef(specFile, baseDir): - """Extract reference pages from a spec asciidoc source file. - - - specFile - filename to extract from - - baseDir - output directory to generate page in""" - file = loadFile(specFile) - if file is None: - return - - # Save the path to this file for later use in rewriting relative includes - specDir = os.path.dirname(os.path.abspath(specFile)) - - pageMap = findRefs(file, specFile) - logDiag(specFile + ': found', len(pageMap.keys()), 'potential pages') - - sys.stderr.flush() - - # Fix up references in pageMap - fixupRefs(pageMap, specFile, file) - - # Create each page, if possible - pages = {} - - for name in sorted(pageMap): - pi = pageMap[name] - - printPageInfo(pi, file) - - if pi.Warning: - logDiag('genRef:', pi.name + ':', pi.Warning) - - if pi.extractPage: - emitPage(baseDir, specDir, pi, file) - elif pi.type == 'enums': - autoGenEnumsPage(baseDir, pi, file) - elif pi.type == 'flags': - autoGenFlagsPage(baseDir, pi.name) - else: - # Don't extract this page - logWarn('genRef: Cannot extract or autogenerate:', pi.name) - - pages[pi.name] = pi - for alias in pi.alias.split(): - pages[alias] = pi - - return pages - - -def genSinglePageRef(baseDir): - """Generate baseDir/apispec.txt, the single-page version of the ref pages. - - This assumes there's a page for everything in the api module dictionaries. - Extensions (KHR, EXT, etc.) are currently skipped""" - # Accumulate head of page - head = io.StringIO() - - printCopyrightSourceComments(head) - - print('= ' + apiName + ' API Reference Pages', - ':data-uri:', - ':icons: font', - ':doctype: book', - ':numbered!:', - ':max-width: 200', - ':data-uri:', - ':toc2:', - ':toclevels: 2', - '', - sep='\n', file=head) - - print('== Copyright', file=head) - print('', file=head) - print('include::{config}/copyright-ccby.txt[]', file=head) - print('', file=head) - # Inject the table of contents. Asciidoc really ought to be generating - # this for us. - - sections = [ - [api.protos, 'protos', apiName + ' Commands'], - [api.handles, 'handles', 'Object Handles'], - [api.structs, 'structs', 'Structures'], - [api.enums, 'enums', 'Enumerations'], - [api.flags, 'flags', 'Flags'], - [api.funcpointers, 'funcpointers', 'Function Pointer Types'], - [api.basetypes, 'basetypes', apiName + ' Scalar types'], - [api.defines, 'defines', 'C Macro Definitions'], - [extensions, 'extensions', apiName + ' Extensions'] - ] - - # Accumulate body of page - body = io.StringIO() - - for (apiDict, label, title) in sections: - # Add section title/anchor header to body - anchor = '[[' + label + ',' + title + ']]' - print(anchor, - '== ' + title, - '', - ':leveloffset: 2', - '', - sep='\n', file=body) - - if label == 'extensions': - # preserve order of extensions since we already sorted the way we want. - keys = apiDict.keys() - else: - keys = sorted(apiDict.keys()) - - for refPage in keys: - # Don't generate links for aliases, which are included with the - # aliased page - if refPage not in api.alias: - # Add page to body - if 'FlagBits' in refPage and conventions.unified_flag_refpages: - # OpenXR does not create separate ref pages for FlagBits: - # the FlagBits includes go in the Flags refpage. - # Previously the Vulkan script would only emit non-empty - # Vk*Flags pages, via the logic - # if refPage not in api.flags or api.flags[refPage] is not None - # emit page - # Now, all are emitted. - continue - else: - print('include::' + refPage + '.txt[]', file=body) - else: - # Alternatively, we could (probably should) link to the - # aliased refpage - logWarn('(Benign) Not including', refPage, - 'in single-page reference', - 'because it is an alias of', api.alias[refPage]) - - print('\n' + ':leveloffset: 0' + '\n', file=body) - - # Write head and body to the output file - pageName = baseDir + '/apispec.txt' - fp = open(pageName, 'w', encoding='utf-8') - - print(head.getvalue(), file=fp, end='') - print(body.getvalue(), file=fp, end='') - - head.close() - body.close() - fp.close() - - -def genExtension(baseDir, extpath, name, info): - """Generate refpage, and add dictionary entry for an extension - - - baseDir - output directory to generate page in - - extpath - None, or path to per-extension specification sources if - those are to be included in extension refpages - - name - extension name - - info - Element from XML""" - - # Add a dictionary entry for this page - global genDict - genDict[name] = None - declares = [] - elem = info.elem - - # Type of extension (instance, device, etc.) - ext_type = elem.get('type') - - # Autogenerate interfaces from entry - for required in elem.find('require'): - req_name = required.get('name') - if not req_name: - # This isn't what we're looking for - continue - if req_name.endswith('_SPEC_VERSION') or req_name.endswith('_EXTENSION_NAME'): - # Don't link to spec version or extension name - those ref pages aren't created. - continue - - if required.get('extends'): - # These are either extensions of enumerated types, or const enum - # values: neither of which get a ref page - although we could - # include the enumerated types in the See Also list. - continue - - if req_name not in genDict: - logWarn('ERROR: {} (in extension {}) does not have a ref page.'.format(req_name, name)) - - declares.append(req_name) - - # import pdb - # pdb.set_trace() - - appbody = None - if extpath is not None: - appfp = open('{}/{}.txt'.format(extpath, name), 'r', encoding='utf-8') - if appfp is not None: - appbody = appfp.read() - - # Transform internal links to crosslinks - specURL = conventions.specURL() - appbody = xrefRewrite(appbody, specURL) - else: - logWarn('Cannot find extension appendix for', name) - - # Fall through to autogenerated page - extpath = None - appbody = None - appfp.close() - - # Include the extension appendix without an extra title - # head_content = 'include::{{appendices}}/{}.txt[]'.format(name) - - # Write the extension refpage - pageName = baseDir + '/' + name + '.txt' - logDiag('genExtension:', pageName) - fp = open(pageName, 'w', encoding='utf-8') - - # There are no generated titled sections - sections = None - - # 'See link:{html_spec_relative}#%s[ %s] in the main specification for complete information.' % ( - # name, name) - refPageShell(name, - "{} extension".format(ext_type), - fp, - appbody, - sections=sections) - refPageTail(pageName=name, - specType=None, - specAnchor=name, - seeAlso=seeAlsoList(name, declares), - fp=fp, - auto=True) - fp.close() - - -if __name__ == '__main__': - global genDict, extensions, conventions, apiName - genDict = {} - extensions = OrderedDict() - conventions = APIConventions() - apiName = conventions.api_name('api') - - parser = argparse.ArgumentParser() - - parser.add_argument('-diag', action='store', dest='diagFile', - help='Set the diagnostic file') - parser.add_argument('-warn', action='store', dest='warnFile', - help='Set the warning file') - parser.add_argument('-log', action='store', dest='logFile', - help='Set the log file for both diagnostics and warnings') - parser.add_argument('-genpath', action='store', - default='gen', - help='Path to directory containing generated files') - parser.add_argument('-basedir', action='store', dest='baseDir', - default=None, - help='Set the base directory in which pages are generated') - parser.add_argument('-noauto', action='store_true', - help='Don\'t generate inferred ref pages automatically') - parser.add_argument('files', metavar='filename', nargs='*', - help='a filename to extract ref pages from') - parser.add_argument('--version', action='version', version='%(prog)s 1.0') - parser.add_argument('-extension', action='append', - default=[], - help='Specify an extension or extensions to add to targets') - parser.add_argument('-rewrite', action='store', - default=None, - help='Name of output file to write Apache mod_rewrite directives to') - parser.add_argument('-toc', action='store', - default=None, - help='Name of output file to write an alphabetical TOC to') - parser.add_argument('-registry', action='store', - default=conventions.registry_path, - help='Use specified registry file instead of default') - parser.add_argument('-extpath', action='store', - default=None, - help='Use extension descriptions from this directory instead of autogenerating extension refpages') - - results = parser.parse_args() - - # Look for api.py in the specified directory - if results.genpath is not None: - sys.path.insert(0, results.genpath) - import api - - setLogFile(True, True, results.logFile) - setLogFile(True, False, results.diagFile) - setLogFile(False, True, results.warnFile) - - # Initialize static rewrite patterns for spec xrefs - xrefRewriteInitialize() - - if results.baseDir is None: - baseDir = results.genpath + '/ref' - else: - baseDir = results.baseDir - - # Dictionary of pages & aliases - pages = {} - - for file in results.files: - d = genRef(file, baseDir) - pages.update(d) - - # Now figure out which pages *weren't* generated from the spec. - # This relies on the dictionaries of API constructs in the api module. - - if not results.noauto: - registry = Registry() - registry.loadFile(results.registry) - - if conventions.write_refpage_include: - # Only extensions with a supported="..." attribute in this set - # will be considered for extraction/generation. - supported_strings = set((conventions.xml_api_name,)) - ext_names = set(k for k, v in registry.extdict.items() - if v.supported in supported_strings) - - desired_extensions = ext_names.intersection(set(results.extension)) - for prefix in conventions.extension_index_prefixes: - # Splits up into chunks, sorted within each chunk. - filtered_extensions = sorted( - [name for name in desired_extensions - if name.startswith(prefix) and name not in extensions]) - for name in filtered_extensions: - # logWarn('NOT autogenerating extension refpage for', name) - extensions[name] = None - genExtension(baseDir, results.extpath, name, registry.extdict[name]) - - # autoGenFlagsPage is no longer needed because they are added to - # the spec sources now. - # for page in api.flags: - # if page not in genDict: - # autoGenFlagsPage(baseDir, page) - - # autoGenHandlePage is no longer needed because they are added to - # the spec sources now. - # for page in api.structs: - # if typeCategory[page] == 'handle': - # autoGenHandlePage(baseDir, page) - - sections = [ - (api.flags, 'Flag Types'), - (api.enums, 'Enumerated Types'), - (api.structs, 'Structures'), - (api.protos, 'Prototypes'), - (api.funcpointers, 'Function Pointers'), - (api.basetypes, apiName + ' Scalar Types'), - (extensions, apiName + ' Extensions'), - ] - - # Summarize pages that weren't generated, for good or bad reasons - - for (apiDict, title) in sections: - # OpenXR was keeping a 'flagged' state which only printed out a - # warning for the first non-generated page, but was otherwise - # unused. This doesn't seem helpful. - for page in apiDict: - if page not in genDict: - # Page was not generated - why not? - if page in api.alias: - logWarn('(Benign, is an alias) Ref page for', title, page, 'is aliased into', api.alias[page]) - elif page in api.flags and api.flags[page] is None: - logWarn('(Benign, no FlagBits defined) No ref page generated for ', title, - page) - else: - # Could introduce additional logic to detect - # external types and not emit them. - logWarn('No ref page generated for ', title, page) - - genSinglePageRef(baseDir) - - if results.rewrite: - # Generate Apache rewrite directives for refpage aliases - fp = open(results.rewrite, 'w', encoding='utf-8') - - for page in sorted(pages): - p = pages[page] - rewrite = p.name - - if page != rewrite: - print('RewriteRule ^', page, '.html$ ', rewrite, '.html', - sep='', file=fp) - fp.close() - - if results.toc: - # Generate dynamic portion of refpage TOC - fp = open(results.toc, 'w', encoding='utf-8') - - # Run through dictionary of pages generating an TOC - print(12 * ' ', '
  • Alphabetic Contents', sep='', file=fp) - print(16 * ' ', '
      ', sep='', file=fp) - lastLetter = None - - for page in sorted(pages, key=str.upper): - p = pages[page] - letter = page[0:1].upper() - - if letter != lastLetter: - if lastLetter: - # End previous block - print(24 * ' ', '
    ', sep='', file=fp) - print(20 * ' ', '
    • ', sep='', file=fp) - # Start new block - print(20 * ' ', '
  • ', letter, sep='', file=fp) - print(24 * ' ', '
      ', sep='', file=fp) - lastLetter = letter - - # Add this page to the list - print(28 * ' ', '
    • ', page, '
      • ', - sep='', file=fp) - - if lastLetter: - # Close the final letter block - print(24 * ' ', '
    ', sep='', file=fp) - print(20 * ' ', '
    • ', sep='', file=fp) - - # Close the list - print(16 * ' ', '', sep='', file=fp) - print(12 * ' ', '', sep='', file=fp) - - # print('name {} -> page {}'.format(page, pages[page].name)) - - fp.close() diff --git a/xml/gen_dictionaries.py b/xml/gen_dictionaries.py deleted file mode 100644 index 069800dbb..000000000 --- a/xml/gen_dictionaries.py +++ /dev/null @@ -1,258 +0,0 @@ -#!/usr/bin/python3 - -# Copyright 2019-2023 The Khronos Group Inc. -# SPDX-License-Identifier: Apache-2.0 - -from collections import OrderedDict - -import argparse -import sys -import urllib -import xml.etree.ElementTree as etree -import urllib.request - -def parse_xml(path): - file = urllib.request.urlopen(path) if path.startswith("http") else open(path, 'r') - with file: - tree = etree.parse(file) - return tree - -# File Header: -def GetHeader(): - return """// Copyright 2017-2023 The Khronos Group. This work is licensed under a -// Creative Commons Attribution 4.0 International License; see -// http://creativecommons.org/licenses/by/4.0/ - -""" - -# File Footer: -def GetFooter(): - return """ -""" - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - - parser.add_argument('-registry', action='store', - default='cl.xml', - help='Use specified registry file instead of cl.xml') - parser.add_argument('-o', action='store', dest='directory', - default='.', - help='Create target and related files in specified directory') - - args = parser.parse_args() - - linkFileName = args.directory + '/api-dictionary.asciidoc' - nolinkFileName = args.directory + '/api-dictionary-no-links.asciidoc' - typeFileName = args.directory + '/api-types.txt' - - specpath = args.registry - #specpath = "https://raw.githubusercontent.com/KhronosGroup/OpenCL-Registry/main/xml/cl.xml" - - print('Generating dictionaries from: ' + specpath) - - spec = parse_xml(specpath) - - linkFile = open(linkFileName, 'w') - nolinkFile = open(nolinkFileName, 'w') - linkFile.write( GetHeader() ) - nolinkFile.write( GetHeader() ) - typeFile = open(typeFileName, 'w') - - # Generate the API functions dictionaries: - - numberOfFuncs = 0 - - # Add core API functions with and without links: - for feature in spec.findall('feature/require'): - for api in feature.findall('command'): - name = api.get('name') - #print('found api: ' + name) - - # Example with link: - # - # // clEnqueueNDRangeKernel - # :clEnqueueNDRangeKernel_label: pass:q[*clEnqueueNDRangeKernel*] - # :clEnqueueNDRangeKernel: <> - linkFile.write('// ' + name + '\n') - linkFile.write(':' + name + '_label: pass:q[*' + name + '*]\n') - linkFile.write(':' + name + ': <<' + name + ',{' + name + '_label}>>\n') - linkFile.write('\n') - - # Example without link: - # - # // clEnqueueNDRangeKernel - # :clEnqueueNDRangeKernel: pass:q[*clEnqueueNDRangeKernel*] - nolinkFile.write('// ' + name + '\n') - nolinkFile.write(':' + name + ': pass:q[*' + name + '*]\n') - nolinkFile.write('\n') - - numberOfFuncs = numberOfFuncs + 1 - - # Add extension API functions without links: - for extension in spec.findall('extensions/extension/require'): - for api in extension.findall('command'): - name = api.get('name') - #print('found extension api: ' +name) - - # Example without link: - # - # // clGetGLObjectInfo - # :clGetGLObjectInfo: pass:q[*clGetGLObjectInfo*] - linkFile.write('// ' + name + '\n') - linkFile.write(':' + name + ': pass:q[*' + name + '*]\n') - linkFile.write('\n') - - nolinkFile.write('// ' + name + '\n') - nolinkFile.write(':' + name + ': pass:q[*' + name + '*]\n') - nolinkFile.write('\n') - - numberOfFuncs = numberOfFuncs + 1 - - print('Found ' + str(numberOfFuncs) + ' API functions.') - - # Generate the API enums dictionaries: - - numberOfEnums = 0 - - for enums in spec.findall('enums'): - name = enums.get('name') - for enum in enums.findall('enum'): - name = enum.get('name') - #print('found enum: ' + name) - - # Create a variant of the name that precedes underscores with - # "zero width" spaces. This causes some long names to be - # broken at more intuitive places. - htmlName = name[:3] + name[3:].replace("_", "_") - otherName = name[:3] + name[3:].replace("_", "_​") - - # Example with link: - # - # // CL_MEM_READ_ONLY - #:CL_MEM_READ_ONLY_label: pass:q[`CL_MEM_READ_ONLY`] - #:CL_MEM_READ_ONLY: <> - #:CL_MEM_READ_ONLY_anchor: [[CL_MEM_READ_ONLY]]{CL_MEM_READ_ONLY} - linkFile.write('// ' + name + '\n') - linkFile.write('ifdef::backend-html5[]\n') - linkFile.write(':' + name + '_label: pass:q[`' + htmlName + '`]\n') - linkFile.write('endif::[]\n') - linkFile.write('ifndef::backend-html5[]\n') - linkFile.write(':' + name + '_label: pass:q[`' + otherName + '`]\n') - linkFile.write('endif::[]\n') - linkFile.write(':' + name + ': <<' + name + ',{' + name + '_label}>>\n') - linkFile.write(':' + name + '_anchor: [[' + name + ']]{' + name + '}\n') - linkFile.write('\n') - - # Example without link: - # - # // CL_MEM_READ_ONLY - #:CL_MEM_READ_ONLY: pass:q[`CL_MEM_READ_ONLY`] - #:CL_MEM_READ_ONLY_anchor: {CL_MEM_READ_ONLY} - nolinkFile.write('// ' + name + '\n') - nolinkFile.write('ifdef::backend-html5[]\n') - nolinkFile.write(':' + name + ': pass:q[`' + htmlName + '`]\n') - nolinkFile.write('endif::[]\n') - nolinkFile.write('ifndef::backend-html5[]\n') - nolinkFile.write(':' + name + ': pass:q[`' + otherName + '`]\n') - nolinkFile.write('endif::[]\n') - nolinkFile.write(':' + name + '_anchor: {' + name + '}\n') - nolinkFile.write('\n') - - numberOfEnums = numberOfEnums + 1 - - print('Found ' + str(numberOfEnums) + ' API enumerations.') - - # Generate the API types dictionaries: - - numberOfTypes = 0 - - for types in spec.findall('types'): - for type in types.findall('type'): - addLink = False - name = "" - category = type.get('category') - if category == 'basetype': - name = type.get('name') - elif category == 'struct': - addLink = True - name = type.get('name') - elif category == 'define': - name = type.find('name').text - else: - continue - - #print('found type: ' +name) - - # Create a variant of the name that precedes underscores with - # "zero width" spaces. This causes some long names to be - # broken at more intuitive places. - if name.endswith('_t'): - htmlName = name - otherName = name - else: - htmlName = name[:3] + name[3:].replace("_", "_") - otherName = name[:3] + name[3:].replace("_", "_​") - - # Some types can have spaces in the name (such as unsigned char), - # but Asciidoctor attributes cannot. So, replace spaces with - # underscores for the attribute name. - attribName = name.replace(" ", "_") - - # Append the type suffix for disambiguation, since asciidoctor - # attributes are not case-sensitive (currently). - attribName = attribName + "_TYPE" - - # Example with link: - # - # // cl_image_desc - # :cl_image_desc_TYPE_label: pass:q[`cl_image_desc`] - # :cl_image_desc_TYPE: <> - linkFile.write('// ' + name + '\n') - if addLink: - linkFile.write('ifdef::backend-html5[]\n') - linkFile.write(':' + attribName + '_label: pass:q[`' + htmlName + '`]\n') - linkFile.write('endif::[]\n') - linkFile.write('ifndef::backend-html5[]\n') - linkFile.write(':' + attribName + '_label: pass:q[`' + otherName + '`]\n') - linkFile.write('endif::[]\n') - linkFile.write(':' + attribName + ': <<' + name + ',{' + attribName + '_label}>>\n') - else: - linkFile.write('ifdef::backend-html5[]\n') - linkFile.write(':' + attribName + ': pass:q[`' + htmlName + '`]\n') - linkFile.write('endif::[]\n') - linkFile.write('ifndef::backend-html5[]\n') - linkFile.write(':' + attribName + ': pass:q[`' + otherName + '`]\n') - linkFile.write('endif::[]\n') - linkFile.write('\n') - - # // cl_image_desc - # :cl_image_desc_TYPE: pass:q[`cl_image_desc`] - nolinkFile.write('// ' + name + '\n') - nolinkFile.write('ifdef::backend-html5[]\n') - nolinkFile.write(':' + attribName + ': pass:q[`' + htmlName + '`]\n') - nolinkFile.write('endif::[]\n') - nolinkFile.write('ifndef::backend-html5[]\n') - nolinkFile.write(':' + attribName + ': pass:q[`' + otherName + '`]\n') - nolinkFile.write('endif::[]\n') - nolinkFile.write('\n') - - # Print the type list to a file for custom syntax highlighting. - # For this we only care about CL types, not base types. - if category != 'basetype': - typeFile.write(' ' + name + '\n') - - numberOfTypes = numberOfTypes + 1 - - print('Found ' + str(numberOfTypes) + ' API types.') - - linkFile.write( GetFooter() ) - linkFile.close() - nolinkFile.write( GetFooter() ) - nolinkFile.close() - typeFile.close() - - print('Successfully generated file: ' + linkFileName) - print('Successfully generated file: ' + nolinkFileName) - print('Successfully generated file: ' + typeFileName) - diff --git a/xml/gen_version_notes.py b/xml/gen_version_notes.py deleted file mode 100644 index 9fed05178..000000000 --- a/xml/gen_version_notes.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/python3 - -# Copyright 2019-2023 The Khronos Group Inc. -# SPDX-License-Identifier: Apache-2.0 - -from collections import OrderedDict - -import argparse -import sys -import os -import urllib -import xml.etree.ElementTree as etree -import urllib.request - - -def parse_xml(path): - file = urllib.request.urlopen(path) if path.startswith("http") else open( - path, 'r') - with file: - tree = etree.parse(file) - return tree - - -# File Header: -def GetHeader(): - return """// Copyright 2017-2023 The Khronos Group. This work is licensed under a -// Creative Commons Attribution 4.0 International License; see -// http://creativecommons.org/licenses/by/4.0/ -""" - - -# File Footer: -def GetFooter(): - return """ -""" - -def FullNote(name, added_in, deprecated_by): - # Four patterns: (1) always present in OpenCL, (2) added after 1.0, (3) in - # 1.0 but now deprecated, and (4) added after 1.0 but now deprecated. - if added_in == "1.0" and deprecated_by == None: - return "\n// Intentionally empty, %s has always been present." % name - if added_in != "1.0" and deprecated_by == None: - return "\nIMPORTANT: {%s} is <> version %s." % (name, added_in) - if added_in == "1.0" and deprecated_by != None: - return "\nIMPORTANT: {%s} is <> version %s." % (name, deprecated_by) - if added_in != "1.0" and deprecated_by != None: - return "\nIMPORTANT: {%s} is <> version %s and <> version %s." % (name, added_in, deprecated_by) - -def ShortNote(name, added_in, deprecated_by): - # Four patterns: (1) always present in OpenCL, (2) added after 1.0, (3) in - # 1.0 but now deprecated, and (4) added after 1.0 but now deprecated. - if added_in == "1.0" and deprecated_by == None: - return "// Intentionally empty, %s has always been present." % name - if added_in != "1.0" and deprecated_by == None: - return "<> version %s." % added_in - if added_in == "1.0" and deprecated_by != None: - return "<> version %s." % deprecated_by - if added_in != "1.0" and deprecated_by != None: - return "<> version %s and <> version %s." % (added_in, deprecated_by) - -# Find feature groups that are parents of a feature/require/${entry_type} -# hierarchy, and then find all the ${entry_type} within each hierarchy: -def process_xml(spec, entry_type, note_printer): - numberOfEntries = 0 - numberOfNewEntries = 0 - numberOfDeprecatedEntries = 0 - - for feature in spec.findall('.//feature/require/%s/../..' % entry_type): - for entry in feature.findall('.//%s' % entry_type): - name = entry.get('name') - - numberOfEntries += 1 - added_in = feature.get('number') - deprecated_by = None - - # All the groups that this specific API ${entry_type} belongs. - categories = spec.findall( - './/require[@comment]/%s[@name="%s"]/..' % (entry_type, name)) - for category in categories: - comment = category.get('comment') - if "deprecated in OpenCL" in comment: - words = comment.split(" ") - assert " ".join(words[-4:-1]) == "deprecated in OpenCL" - assert deprecated_by == None # Can't deprecate something twice. - deprecated_by = words[-1] - - versionFileName = os.path.join(args.directory, name + ".asciidoc") - with open(versionFileName, 'w') as versionFile: - versionFile.write(GetHeader()) - versionFile.write(note_printer(name, added_in, deprecated_by)) - versionFile.write(GetFooter()) - - numberOfNewEntries += 0 if added_in == "1.0" else 1 - numberOfDeprecatedEntries += 0 if deprecated_by == None else 1 - - print('Found ' + str(numberOfEntries) + ' API ' + entry_type + 's, ' - + str(numberOfNewEntries) + " newer than 1.0, " - + str(numberOfDeprecatedEntries) + " are deprecated.") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - - parser.add_argument( - '-registry', - action='store', - default='cl.xml', - help='Use specified registry file instead of cl.xml') - parser.add_argument( - '-o', - action='store', - dest='directory', - default='.', - help='Create target and related files in specified directory') - - args = parser.parse_args() - - specpath = args.registry - - print('Generating version notes from: ' + specpath) - - spec = parse_xml(specpath) - - # Generate the API functions dictionaries: - - process_xml(spec, "command", FullNote) - process_xml(spec, "enum", ShortNote) diff --git a/xml/gencl.py b/xml/gencl.py deleted file mode 100644 index eb77a8cdf..000000000 --- a/xml/gencl.py +++ /dev/null @@ -1,464 +0,0 @@ -#!/usr/bin/python3 -# -# Copyright 2013-2023 The Khronos Group Inc. -# -# SPDX-License-Identifier: Apache-2.0 - -import argparse -import pdb -import re -import sys -import time -import xml.etree.ElementTree as etree - -from cgenerator import CGeneratorOptions, COutputGenerator -from docgenerator import DocGeneratorOptions, DocOutputGenerator -from extensionmetadocgenerator import (ExtensionMetaDocGeneratorOptions, - ExtensionMetaDocOutputGenerator) - -from generator import write - - -from pygenerator import PyOutputGenerator -from reflib import logDiag, logWarn, setLogFile -from reg import Registry - -from clconventions import OpenCLConventions as APIConventions - - -# Simple timer functions -startTime = None - - -def startTimer(timeit): - global startTime - if timeit: - startTime = time.process_time() - - -def endTimer(timeit, msg): - global startTime - if timeit: - endTime = time.process_time() - logDiag(msg, endTime - startTime) - startTime = None - - -def makeREstring(strings, default=None, strings_are_regex=False): - """Turn a list of strings into a regexp string matching exactly those strings.""" - if strings or default is None: - if not strings_are_regex: - strings = (re.escape(s) for s in strings) - return '^(' + '|'.join(strings) + ')$' - return default - -def makeGenOpts(args): - """Returns a directory of [ generator function, generator options ] indexed - by specified short names. The generator options incorporate the following - parameters: - - args is an parsed argument object; see below for the fields that are used.""" - global genOpts - genOpts = {} - - # Default class of extensions to include, or None - defaultExtensions = args.defaultExtensions - - # Additional extensions to include (list of extensions) - extensions = args.extension - - # Extensions to remove (list of extensions) - removeExtensions = args.removeExtensions - - # Extensions to emit (list of extensions) - emitExtensions = args.emitExtensions - - # SPIR-V capabilities / features to emit (list of extensions & capabilities) - # emitSpirv = args.emitSpirv - - # Features to include (list of features) - features = args.feature - - # Whether to disable inclusion protect in headers - protect = args.protect - - # Output target directory - directory = args.directory - - # Path to generated files, particularly api.py - genpath = args.genpath - - # Generate MISRA C-friendly headers - misracstyle = args.misracstyle; - - # Generate MISRA C++-friendly headers - misracppstyle = args.misracppstyle; - - # Descriptive names for various regexp patterns used to select - # versions and extensions - allSpirv = allFeatures = allExtensions = r'.*' - - # Turn lists of names/patterns into matching regular expressions - addExtensionsPat = makeREstring(extensions, None) - removeExtensionsPat = makeREstring(removeExtensions, None) - emitExtensionsPat = makeREstring(emitExtensions, allExtensions) - # emitSpirvPat = makeREstring(emitSpirv, allSpirv) - featuresPat = makeREstring(features, allFeatures) - - # Copyright text prefixing all headers (list of strings). - # The SPDX formatting below works around constraints of the 'reuse' tool - prefixStrings = [ - '/*', - '** Copyright 2015-2023 The Khronos Group Inc.', - '**', - '** SPDX' + '-License-Identifier: Apache-2.0', - '*/', - '' - ] - - # Text specific to OpenCL headers - clPrefixStrings = [ - '/*', - '** This header is generated from the Khronos OpenCL XML API Registry.', - '**', - '*/', - '' - ] - - # Defaults for generating re-inclusion protection wrappers (or not) - protectFile = protect - - # An API style conventions object - conventions = APIConventions() - - # API include files for spec and ref pages - # Overwrites include subdirectories in spec source tree - # The generated include files do not include the calling convention - # macros (apientry etc.), unlike the header files. - # Because the 1.0 core branch includes ref pages for extensions, - # all the extension interfaces need to be generated, even though - # none are used by the core spec itself. - genOpts['apiinc'] = [ - DocOutputGenerator, - DocGeneratorOptions( - conventions = conventions, - filename = 'timeMarker', - directory = directory, - genpath = genpath, - apiname = 'opencl', - profile = None, - versions = featuresPat, - emitversions = featuresPat, - defaultExtensions = defaultExtensions, - addExtensions = addExtensionsPat, - removeExtensions = removeExtensionsPat, - emitExtensions = emitExtensionsPat, - prefixText = prefixStrings + clPrefixStrings, - apicall = '', - apientry = '', - apientryp = '*', - alignFuncParam = 0, - expandEnumerants = False) - ] - - # Python representation of API information, used by scripts that - # don't need to load the full XML. - genOpts['api.py'] = [ - PyOutputGenerator, - DocGeneratorOptions( - conventions = conventions, - filename = 'api.py', - directory = directory, - genpath = genpath, - apiname = 'opencl', - profile = None, - versions = featuresPat, - emitversions = featuresPat, - defaultExtensions = None, - addExtensions = addExtensionsPat, - removeExtensions = removeExtensionsPat, - emitExtensions = emitExtensionsPat, - reparentEnums = False) - ] - - # Extension metainformation for spec extension appendices - # Includes all extensions by default, but only so that the generated - # 'promoted_extensions_*' files refer to all extensions that were - # promoted to a core version. - genOpts['extinc'] = [ - ExtensionMetaDocOutputGenerator, - ExtensionMetaDocGeneratorOptions( - conventions = conventions, - filename = 'timeMarker', - directory = directory, - genpath = None, - apiname = 'opencl', - profile = None, - versions = featuresPat, - emitversions = None, - defaultExtensions = defaultExtensions, - addExtensions = addExtensionsPat, - removeExtensions = None, - emitExtensions = emitExtensionsPat) - ] - - # Platform extensions, in their own header files - # Each element of the platforms[] array defines information for - # generating a single platform: - # [0] is the generated header file name - # [1] is the set of platform extensions to generate - # [2] is additional extensions whose interfaces should be considered, - # but suppressed in the output, to avoid duplicate definitions of - # dependent types like VkDisplayKHR and VkSurfaceKHR which come from - # non-platform extensions. - - # Track all platform extensions, for exclusion from vulkan_core.h - allPlatformExtensions = [] - - # # Extensions suppressed for all platforms. - # # Covers common WSI extension types. - # commonSuppressExtensions = [ 'VK_KHR_display', 'VK_KHR_swapchain' ] - # - # platforms = [ - # [ 'vulkan_android.h', [ 'VK_KHR_android_surface', - # 'VK_ANDROID_external_memory_android_hardware_buffer' - # ], commonSuppressExtensions ], - # [ 'vulkan_fuchsia.h', [ 'VK_FUCHSIA_imagepipe_surface'], commonSuppressExtensions ], - # [ 'vulkan_ios.h', [ 'VK_MVK_ios_surface' ], commonSuppressExtensions ], - # [ 'vulkan_macos.h', [ 'VK_MVK_macos_surface' ], commonSuppressExtensions ], - # [ 'vulkan_vi.h', [ 'VK_NN_vi_surface' ], commonSuppressExtensions ], - # [ 'vulkan_wayland.h', [ 'VK_KHR_wayland_surface' ], commonSuppressExtensions ], - # [ 'vulkan_win32.h', [ 'VK_.*_win32(|_.*)' ], commonSuppressExtensions + [ 'VK_KHR_external_semaphore', 'VK_KHR_external_memory_capabilities', 'VK_KHR_external_fence', 'VK_KHR_external_fence_capabilities', 'VK_NV_external_memory_capabilities' ] ], - # [ 'vulkan_xcb.h', [ 'VK_KHR_xcb_surface' ], commonSuppressExtensions ], - # [ 'vulkan_xlib.h', [ 'VK_KHR_xlib_surface' ], commonSuppressExtensions ], - # [ 'vulkan_xlib_xrandr.h', [ 'VK_EXT_acquire_xlib_display' ], commonSuppressExtensions ], - # ] - # - # for platform in platforms: - # headername = platform[0] - # - # allPlatformExtensions += platform[1] - # - # addPlatformExtensionsRE = makeREstring(platform[1] + platform[2]) - # emitPlatformExtensionsRE = makeREstring(platform[1]) - # - # opts = CGeneratorOptions( - # filename = headername, - # directory = directory, - # apiname = 'vulkan', - # profile = None, - # versions = featuresPat, - # emitversions = None, - # defaultExtensions = None, - # addExtensions = addPlatformExtensionsRE, - # removeExtensions = None, - # emitExtensions = emitPlatformExtensionsRE, - # prefixText = prefixStrings + clPrefixStrings, - # genFuncPointers = True, - # protectFile = protectFile, - # protectFeature = False, - # protectProto = '#ifndef', - # protectProtoStr = 'VK_NO_PROTOTYPES', - # apicall = 'VKAPI_ATTR ', - # apientry = 'VKAPI_CALL ', - # apientryp = 'VKAPI_PTR *', - # alignFuncParam = 0) - # - # genOpts[headername] = [ COutputGenerator, opts ] - - # Header for core API + extensions. - # To generate just the core API, - # change to 'defaultExtensions = None' below. - # - # By default this adds all enabled, non-platform extensions. - # It removes all platform extensions (from the platform headers options - # constructed above) as well as any explicitly specified removals. - - removeExtensionsPat = makeREstring( - allPlatformExtensions + removeExtensions, None, strings_are_regex=True) - - genOpts['cl.h'] = [ - COutputGenerator, - CGeneratorOptions( - conventions = conventions, - filename = 'cl.h', - directory = directory, - genpath = None, - apiname = 'opencl', - profile = None, - versions = featuresPat, - emitversions = featuresPat, - defaultExtensions = defaultExtensions, - addExtensions = None, - removeExtensions = removeExtensionsPat, - emitExtensions = emitExtensionsPat, - prefixText = prefixStrings + clPrefixStrings, - genFuncPointers = False, - protectFile = protectFile, - protectFeature = False, - protectProto = '#ifndef', - protectProtoStr = 'CL_NO_PROTOTYPES', - apicall = 'CL_API_ENTRY ', - apientry = 'CL_API_CALL ', - apientryp = 'CL_API_CALL *', - alignFuncParam = 0, - misracstyle = misracstyle, - misracppstyle = misracppstyle) - ] - -def genTarget(args): - """Create an API generator and corresponding generator options based on - the requested target and command line options. - - This is encapsulated in a function so it can be profiled and/or timed. - The args parameter is an parsed argument object containing the following - fields that are used: - - - target - target to generate - - directory - directory to generate it in - - protect - True if re-inclusion wrappers should be created - - extensions - list of additional extensions to include in generated interfaces""" - - # Create generator options with parameters specified on command line - makeGenOpts(args) - - # pdb.set_trace() - - # Select a generator matching the requested target - if args.target in genOpts: - createGenerator = genOpts[args.target][0] - options = genOpts[args.target][1] - - logDiag('* Building', options.filename) - logDiag('* options.versions =', options.versions) - logDiag('* options.emitversions =', options.emitversions) - logDiag('* options.defaultExtensions =', options.defaultExtensions) - logDiag('* options.addExtensions =', options.addExtensions) - logDiag('* options.removeExtensions =', options.removeExtensions) - logDiag('* options.emitExtensions =', options.emitExtensions) - - gen = createGenerator(errFile=errWarn, - warnFile=errWarn, - diagFile=diag) - return (gen, options) - else: - logErr('No generator options for unknown target:', args.target) - return None - - -# -feature name -# -extension name -# For both, "name" may be a single name, or a space-separated list -# of names, or a regular expression. -if __name__ == '__main__': - parser = argparse.ArgumentParser() - - parser.add_argument('-defaultExtensions', action='store', - default='opencl', - help='Specify a single class of extensions to add to targets') - parser.add_argument('-extension', action='append', - default=[], - help='Specify an extension or extensions to add to targets') - parser.add_argument('-removeExtensions', action='append', - default=[], - help='Specify an extension or extensions to remove from targets') - parser.add_argument('-emitExtensions', action='append', - default=[], - help='Specify an extension or extensions to emit in targets') - - - - parser.add_argument('-feature', action='append', - default=[], - help='Specify a core API feature name or names to add to targets') - parser.add_argument('-debug', action='store_true', - help='Enable debugging') - parser.add_argument('-dump', action='store_true', - help='Enable dump to stderr') - parser.add_argument('-diagfile', action='store', - default=None, - help='Write diagnostics to specified file') - parser.add_argument('-errfile', action='store', - default=None, - help='Write errors and warnings to specified file instead of stderr') - parser.add_argument('-noprotect', dest='protect', action='store_false', - help='Disable inclusion protection in output headers') - parser.add_argument('-profile', action='store_true', - help='Enable profiling') - parser.add_argument('-registry', action='store', - default='cl.xml', - help='Use specified registry file instead of cl.xml') - parser.add_argument('-time', action='store_true', - help='Enable timing') - parser.add_argument('-validate', action='store_true', - help='Validate the registry properties and exit') - parser.add_argument('-genpath', action='store', default='gen', - help='Path to generated files') - parser.add_argument('-o', action='store', dest='directory', - default='.', - help='Create target and related files in specified directory') - parser.add_argument('target', metavar='target', nargs='?', - help='Specify target') - parser.add_argument('-quiet', action='store_true', default=True, - help='Suppress script output during normal execution.') - parser.add_argument('-verbose', action='store_false', dest='quiet', default=True, - help='Enable script output during normal execution.') - parser.add_argument('-misracstyle', dest='misracstyle', action='store_true', - help='generate MISRA C-friendly headers') - parser.add_argument('-misracppstyle', dest='misracppstyle', action='store_true', - help='generate MISRA C++-friendly headers') - - args = parser.parse_args() - - # This splits arguments which are space-separated lists - args.feature = [name for arg in args.feature for name in arg.split()] - args.extension = [name for arg in args.extension for name in arg.split()] - - # create error/warning & diagnostic files - if args.errfile: - errWarn = open(args.errfile, 'w', encoding='utf-8') - else: - errWarn = sys.stderr - - if args.diagfile: - diag = open(args.diagfile, 'w', encoding='utf-8') - else: - diag = None - - (gen, options) = (None, None) - if not args.validate: - # Create the API generator & generator options - (gen, options) = genTarget(args) - - # Create the registry object with the specified generator and generator - # options. The options are set before XML loading as they may affect it. - reg = Registry(gen, options) - - # Parse the specified registry XML into an ElementTree object - startTimer(args.time) - tree = etree.parse(args.registry) - endTimer(args.time, '* Time to make ElementTree =') - - # Load the XML tree into the registry object - startTimer(args.time) - reg.loadElementTree(tree) - endTimer(args.time, '* Time to parse ElementTree =') - - if args.validate: - success = reg.validateRegistry() - sys.exit(0 if success else 1) - - if args.dump: - logDiag('* Dumping registry to regdump.txt') - reg.dumpReg(filehandle=open('regdump.txt', 'w', encoding='utf-8')) - - # Finally, use the output generator to create the requested target - if args.debug: - pdb.run('reg.apiGen()') - else: - startTimer(args.time) - reg.apiGen() - endTimer(args.time, '* Time to generate ' + options.filename + ' =') - - if not args.quiet: - logDiag('* Generated', options.filename) diff --git a/xml/generator.py b/xml/generator.py deleted file mode 100644 index c7c460d95..000000000 --- a/xml/generator.py +++ /dev/null @@ -1,1186 +0,0 @@ -#!/usr/bin/python3 -i -# -# Copyright 2013-2023 The Khronos Group Inc. -# -# SPDX-License-Identifier: Apache-2.0 -"""Base class for source/header/doc generators, as well as some utility functions.""" - -from __future__ import unicode_literals - -import io -import os -import pdb -import re -import shutil -import sys -import tempfile -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path - -from spec_tools.util import getElemName, getElemType - - -def write(*args, **kwargs): - file = kwargs.pop('file', sys.stdout) - end = kwargs.pop('end', '\n') - file.write(' '.join(str(arg) for arg in args)) - file.write(end) - - -def noneStr(s): - """Return string argument, or "" if argument is None. - - Used in converting etree Elements into text. - s - string to convert""" - if s: - return s - return "" - - -def enquote(s): - """Return string argument with surrounding quotes, - for serialization into Python code.""" - if s: - return "'{}'".format(s) - return None - - -def regSortCategoryKey(feature): - """Sort key for regSortFeatures. - Sorts by category of the feature name string: - - - Core API features (those defined with a `` tag) - - ARB/KHR/OES (Khronos extensions) - - other (EXT/vendor extensions)""" - - if feature.elem.tag == 'feature': - return 0 - if (feature.category == 'ARB' - or feature.category == 'KHR' - or feature.category == 'OES'): - return 1 - - return 2 - - -def regSortOrderKey(feature): - """Sort key for regSortFeatures - key is the sortorder attribute.""" - - # print("regSortOrderKey {} -> {}".format(feature.name, feature.sortorder)) - return feature.sortorder - - -def regSortFeatureVersionKey(feature): - """Sort key for regSortFeatures - key is the feature version. - `` elements all have version number 0.""" - - return float(feature.versionNumber) - - -def regSortExtensionNumberKey(feature): - """Sort key for regSortFeatures - key is the extension number. - `` elements all have extension number 0.""" - - return int(feature.number) - - -def regSortFeatures(featureList): - """Default sort procedure for features. - - - Sorts by explicit sort order (default 0) relative to other features - - then by feature category ('feature' or 'extension'), - - then by version number (for features) - - then by extension number (for extensions)""" - featureList.sort(key=regSortExtensionNumberKey) - featureList.sort(key=regSortFeatureVersionKey) - featureList.sort(key=regSortCategoryKey) - featureList.sort(key=regSortOrderKey) - - -class GeneratorOptions: - """Base class for options used during header/documentation production. - - These options are target language independent, and used by - Registry.apiGen() and by base OutputGenerator objects.""" - - def __init__(self, - conventions=None, - filename=None, - directory='.', - genpath=None, - apiname=None, - profile=None, - versions='.*', - emitversions='.*', - defaultExtensions=None, - addExtensions=None, - removeExtensions=None, - emitExtensions=None, - emitSpirv=None, - reparentEnums=True, - sortProcedure=regSortFeatures): - """Constructor. - - Arguments: - - - conventions - may be mandatory for some generators: - an object that implements ConventionsBase - - filename - basename of file to generate, or None to write to stdout. - - directory - directory in which to generate files - - genpath - path to previously generated files, such as api.py - - apiname - string matching `` 'apiname' attribute, e.g. 'gl'. - - profile - string specifying API profile , e.g. 'core', or None. - - versions - regex matching API versions to process interfaces for. - Normally `'.*'` or `'[0-9][.][0-9]'` to match all defined versions. - - emitversions - regex matching API versions to actually emit - interfaces for (though all requested versions are considered - when deciding which interfaces to generate). For GL 4.3 glext.h, - this might be `'1[.][2-5]|[2-4][.][0-9]'`. - - defaultExtensions - If not None, a string which must in its - entirety match the pattern in the "supported" attribute of - the ``. Defaults to None. Usually the same as apiname. - - addExtensions - regex matching names of additional extensions - to include. Defaults to None. - - removeExtensions - regex matching names of extensions to - remove (after defaultExtensions and addExtensions). Defaults - to None. - - emitExtensions - regex matching names of extensions to actually emit - interfaces for (though all requested versions are considered when - deciding which interfaces to generate). - to None. - - emitSpirv - regex matching names of extensions and capabilities - to actually emit interfaces for. - - reparentEnums - move elements which extend an enumerated - type from or elements to the target - element. This is required for almost all purposes, but the - InterfaceGenerator relies on the list of interfaces in the - or being complete. Defaults to True. - - sortProcedure - takes a list of FeatureInfo objects and sorts - them in place to a preferred order in the generated output. - Default is core API versions, ARB/KHR/OES extensions, all other - extensions, by core API version number or extension number in each - group. - - The regex patterns can be None or empty, in which case they match - nothing.""" - self.conventions = conventions - """may be mandatory for some generators: - an object that implements ConventionsBase""" - - self.filename = filename - "basename of file to generate, or None to write to stdout." - - self.genpath = genpath - """path to previously generated files, such as api.py""" - - self.directory = directory - "directory in which to generate filename" - - self.apiname = apiname - "string matching `` 'apiname' attribute, e.g. 'gl'." - - self.profile = profile - "string specifying API profile , e.g. 'core', or None." - - self.versions = self.emptyRegex(versions) - """regex matching API versions to process interfaces for. - Normally `'.*'` or `'[0-9][.][0-9]'` to match all defined versions.""" - - self.emitversions = self.emptyRegex(emitversions) - """regex matching API versions to actually emit - interfaces for (though all requested versions are considered - when deciding which interfaces to generate). For GL 4.3 glext.h, - this might be `'1[.][2-5]|[2-4][.][0-9]'`.""" - - self.defaultExtensions = defaultExtensions - """If not None, a string which must in its - entirety match the pattern in the "supported" attribute of - the ``. Defaults to None. Usually the same as apiname.""" - - self.addExtensions = self.emptyRegex(addExtensions) - """regex matching names of additional extensions - to include. Defaults to None.""" - - self.removeExtensions = self.emptyRegex(removeExtensions) - """regex matching names of extensions to - remove (after defaultExtensions and addExtensions). Defaults - to None.""" - - self.emitExtensions = self.emptyRegex(emitExtensions) - """regex matching names of extensions to actually emit - interfaces for (though all requested versions are considered when - deciding which interfaces to generate).""" - - self.emitSpirv = self.emptyRegex(emitSpirv) - """regex matching names of extensions and capabilities - to actually emit interfaces for.""" - - self.reparentEnums = reparentEnums - """boolean specifying whether to remove elements from - or when extending an type.""" - - self.sortProcedure = sortProcedure - """takes a list of FeatureInfo objects and sorts - them in place to a preferred order in the generated output. - Default is core API versions, ARB/KHR/OES extensions, all - other extensions, alphabetically within each group.""" - - self.codeGenerator = False - """True if this generator makes compilable code""" - - def emptyRegex(self, pat): - """Substitute a regular expression which matches no version - or extension names for None or the empty string.""" - if not pat: - return '_nomatch_^' - - return pat - - -class OutputGenerator: - """Generate specified API interfaces in a specific style, such as a C header. - - Base class for generating API interfaces. - Manages basic logic, logging, and output file control. - Derived classes actually generate formatted output. - """ - - # categoryToPath - map XML 'category' to include file directory name - categoryToPath = { - 'bitmask': 'flags', - 'enum': 'enums', - 'funcpointer': 'funcpointers', - 'handle': 'handles', - 'define': 'defines', - 'basetype': 'basetypes', - } - - def __init__(self, errFile=sys.stderr, warnFile=sys.stderr, diagFile=sys.stdout): - """Constructor - - - errFile, warnFile, diagFile - file handles to write errors, - warnings, diagnostics to. May be None to not write.""" - self.outFile = None - self.errFile = errFile - self.warnFile = warnFile - self.diagFile = diagFile - # Internal state - self.featureName = None - self.genOpts = None - self.registry = None - self.featureDictionary = {} - # Used for extension enum value generation - self.extBase = 1000000000 - self.extBlockSize = 1000 - self.madeDirs = {} - - # API dictionary, which may be loaded by the beginFile method of - # derived generators. - self.apidict = None - - def logMsg(self, level, *args): - """Write a message of different categories to different - destinations. - - - `level` - - 'diag' (diagnostic, voluminous) - - 'warn' (warning) - - 'error' (fatal error - raises exception after logging) - - - `*args` - print()-style arguments to direct to corresponding log""" - if level == 'error': - strfile = io.StringIO() - write('ERROR:', *args, file=strfile) - if self.errFile is not None: - write(strfile.getvalue(), file=self.errFile) - raise UserWarning(strfile.getvalue()) - elif level == 'warn': - if self.warnFile is not None: - write('WARNING:', *args, file=self.warnFile) - elif level == 'diag': - if self.diagFile is not None: - write('DIAG:', *args, file=self.diagFile) - else: - raise UserWarning( - '*** FATAL ERROR in Generator.logMsg: unknown level:' + level) - - def enumToValue(self, elem, needsNum, bitwidth = 32, forceSuffix = False): - """Parse and convert an `` tag into a value. - - Returns a list: - - - first element - integer representation of the value, or None - if needsNum is False. The value must be a legal number - if needsNum is True. - - second element - string representation of the value - - There are several possible representations of values. - - - A 'value' attribute simply contains the value. - - A 'bitpos' attribute defines a value by specifying the bit - position which is set in that value. - - An 'offset','extbase','extends' triplet specifies a value - as an offset to a base value defined by the specified - 'extbase' extension name, which is then cast to the - typename specified by 'extends'. This requires probing - the registry database, and imbeds knowledge of the - API extension enum scheme in this function. - - An 'alias' attribute contains the name of another enum - which this is an alias of. The other enum must be - declared first when emitting this enum.""" - name = elem.get('name') - numVal = None - if 'value' in elem.keys(): - value = elem.get('value') - # print('About to translate value =', value, 'type =', type(value)) - if needsNum: - numVal = int(value, 0) - # If there's a non-integer, numeric 'type' attribute (e.g. 'u' or - # 'ull'), append it to the string value. - # t = enuminfo.elem.get('type') - # if t is not None and t != '' and t != 'i' and t != 's': - # value += enuminfo.type - if forceSuffix: - if bitwidth == 64: - value = value + 'ULL' - else: - value = value + 'U' - self.logMsg('diag', 'Enum', name, '-> value [', numVal, ',', value, ']') - return [numVal, value] - if 'bitpos' in elem.keys(): - value = elem.get('bitpos') - bitpos = int(value, 0) - numVal = 1 << bitpos - value = '0x%08x' % numVal - if bitwidth == 64: - value = value + 'ULL' - elif forceSuffix: - value = value + 'U' - self.logMsg('diag', 'Enum', name, '-> bitpos [', numVal, ',', value, ']') - return [numVal, value] - if 'offset' in elem.keys(): - # Obtain values in the mapping from the attributes - enumNegative = False - offset = int(elem.get('offset'), 0) - extnumber = int(elem.get('extnumber'), 0) - extends = elem.get('extends') - if 'dir' in elem.keys(): - enumNegative = True - self.logMsg('diag', 'Enum', name, 'offset =', offset, - 'extnumber =', extnumber, 'extends =', extends, - 'enumNegative =', enumNegative) - # Now determine the actual enumerant value, as defined - # in the "Layers and Extensions" appendix of the spec. - numVal = self.extBase + (extnumber - 1) * self.extBlockSize + offset - if enumNegative: - numVal *= -1 - value = '%d' % numVal - # More logic needed! - self.logMsg('diag', 'Enum', name, '-> offset [', numVal, ',', value, ']') - return [numVal, value] - if 'alias' in elem.keys(): - return [None, elem.get('alias')] - return [None, None] - - def checkDuplicateEnums(self, enums): - """Check enumerated values for duplicates. - - - enums - list of `` Elements - - returns the list with duplicates stripped""" - # Dictionaries indexed by name and numeric value. - # Entries are [ Element, numVal, strVal ] matching name or value - - nameMap = {} - valueMap = {} - - stripped = [] - for elem in enums: - name = elem.get('name') - (numVal, strVal) = self.enumToValue(elem, True) - - if name in nameMap: - # Duplicate name found; check values - (name2, numVal2, strVal2) = nameMap[name] - - # Duplicate enum values for the same name are benign. This - # happens when defining the same enum conditionally in - # several extension blocks. - if (strVal2 == strVal or (numVal is not None - and numVal == numVal2)): - True - # self.logMsg('info', 'checkDuplicateEnums: Duplicate enum (' + name + - # ') found with the same value:' + strVal) - else: - self.logMsg('warn', 'checkDuplicateEnums: Duplicate enum (' + name - + ') found with different values:' + strVal - + ' and ' + strVal2) - - # Don't add the duplicate to the returned list - continue - elif numVal in valueMap: - # Duplicate value found (such as an alias); report it, but - # still add this enum to the list. - (name2, numVal2, strVal2) = valueMap[numVal] - - msg = 'Two enums found with the same value: {} = {} = {}'.format( - name, name2.get('name'), strVal) - self.logMsg('error', msg) - - # Track this enum to detect followon duplicates - nameMap[name] = [elem, numVal, strVal] - if numVal is not None: - valueMap[numVal] = [elem, numVal, strVal] - - # Add this enum to the list - stripped.append(elem) - - # Return the list - return stripped - - def misracstyle(self): - return False; - - def misracppstyle(self): - return False; - - def buildEnumCDecl(self, expand, groupinfo, groupName): - """Generate the C declaration for an enum""" - groupElem = groupinfo.elem - - # Determine the required bit width for the enum group. - # 32 is the default, which generates C enum types for the values. - bitwidth = 32 - - # If the constFlagBits preference is set, 64 is the default for bitmasks - if self.genOpts.conventions.constFlagBits and groupElem.get('type') == 'bitmask': - bitwidth = 64 - - # Check for an explicitly defined bitwidth, which will override any defaults. - if groupElem.get('bitwidth'): - try: - bitwidth = int(groupElem.get('bitwidth')) - except ValueError as ve: - self.logMsg('error', 'Invalid value for bitwidth attribute (', groupElem.get('bitwidth'), ') for ', groupName, ' - must be an integer value\n') - exit(1) - - usebitmask = False - usedefine = False - - # Bitmask flags can be generated as either "static const uint{32,64}_t" values, - # or as 32-bit C enums. 64-bit types must use uint64_t values. - if groupElem.get('type') == 'bitmask': - if bitwidth > 32 or self.misracppstyle(): - usebitmask = True - if self.misracstyle(): - usedefine = True - - if usedefine or usebitmask: - # Validate the bitwidth and generate values appropriately - if bitwidth > 64: - self.logMsg('error', 'Invalid value for bitwidth attribute (', groupElem.get('bitwidth'), ') for bitmask type ', groupName, ' - must be less than or equal to 64\n') - exit(1) - else: - return self.buildEnumCDecl_BitmaskOrDefine(groupinfo, groupName, bitwidth, usedefine) - else: - # Validate the bitwidth and generate values appropriately - if bitwidth > 32: - self.logMsg('error', 'Invalid value for bitwidth attribute (', groupElem.get('bitwidth'), ') for enum type ', groupName, ' - must be less than or equal to 32\n') - exit(1) - else: - return self.buildEnumCDecl_Enum(expand, groupinfo, groupName) - - def buildEnumCDecl_BitmaskOrDefine(self, groupinfo, groupName, bitwidth, usedefine): - """Generate the C declaration for an "enum" that is actually a - set of flag bits""" - groupElem = groupinfo.elem - flagTypeName = groupElem.get('name') - - # Prefix - body = "// Flag bits for " + flagTypeName + "\n" - - if bitwidth == 64: - body += "typedef VkFlags64 %s;\n" % flagTypeName; - else: - body += "typedef VkFlags %s;\n" % flagTypeName; - - # Maximum allowable value for a flag (unsigned 64-bit integer) - maxValidValue = 2**(64) - 1 - minValidValue = 0 - - # Get a list of nested 'enum' tags. - enums = groupElem.findall('enum') - - # Check for and report duplicates, and return a list with them - # removed. - enums = self.checkDuplicateEnums(enums) - - # Accumulate non-numeric enumerant values separately and append - # them following the numeric values, to allow for aliases. - # NOTE: this doesn't do a topological sort yet, so aliases of - # aliases can still get in the wrong order. - aliasText = '' - - # Loop over the nested 'enum' tags. - for elem in enums: - # Convert the value to an integer and use that to track min/max. - # Values of form -(number) are accepted but nothing more complex. - # Should catch exceptions here for more complex constructs. Not yet. - (numVal, strVal) = self.enumToValue(elem, True, bitwidth, True) - name = elem.get('name') - - # Range check for the enum value - if numVal is not None and (numVal > maxValidValue or numVal < minValidValue): - self.logMsg('error', 'Allowable range for flag types in C is [', minValidValue, ',', maxValidValue, '], but', name, 'flag has a value outside of this (', strVal, ')\n') - exit(1) - - decl = self.genRequirements(name, mustBeFound = False) - - if self.isEnumRequired(elem): - protect = elem.get('protect') - if protect is not None: - body += '#ifdef {}\n'.format(protect) - - if usedefine: - decl += "#define {} {}\n".format(name, strVal) - elif self.misracppstyle(): - decl += "static constexpr {} {} {{{}}};\n".format(flagTypeName, name, strVal) - else: - # Some C compilers only allow initializing a 'static const' variable with a literal value. - # So initializing an alias from another 'static const' value would fail to compile. - # Work around this by chasing the aliases to get the actual value. - while numVal is None: - alias = self.registry.tree.find("enums/enum[@name='" + strVal + "']") - (numVal, strVal) = self.enumToValue(alias, True, bitwidth, True) - decl += "static const {} {} = {};\n".format(flagTypeName, name, strVal) - - if numVal is not None: - body += decl - else: - aliasText += decl - - if protect is not None: - body += '#endif\n' - - # Now append the non-numeric enumerant values - body += aliasText - - # Postfix - - return ("bitmask", body) - - def buildEnumCDecl_Enum(self, expand, groupinfo, groupName): - """Generate the C declaration for an enumerated type""" - groupElem = groupinfo.elem - - # Break the group name into prefix and suffix portions for range - # enum generation - expandName = re.sub(r'([0-9]+|[a-z_])([A-Z0-9])', r'\1_\2', groupName).upper() - expandPrefix = expandName - expandSuffix = '' - expandSuffixMatch = re.search(r'[A-Z][A-Z]+$', groupName) - if expandSuffixMatch: - expandSuffix = '_' + expandSuffixMatch.group() - # Strip off the suffix from the prefix - expandPrefix = expandName.rsplit(expandSuffix, 1)[0] - - # Prefix - body = ["typedef enum %s {" % groupName] - - # @@ Should use the type="bitmask" attribute instead - isEnum = ('FLAG_BITS' not in expandPrefix) - - # Allowable range for a C enum - which is that of a signed 32-bit integer - maxValidValue = 2**(32 - 1) - 1 - minValidValue = (maxValidValue * -1) - 1 - - - # Get a list of nested 'enum' tags. - enums = groupElem.findall('enum') - - # Check for and report duplicates, and return a list with them - # removed. - enums = self.checkDuplicateEnums(enums) - - # Loop over the nested 'enum' tags. Keep track of the minimum and - # maximum numeric values, if they can be determined; but only for - # core API enumerants, not extension enumerants. This is inferred - # by looking for 'extends' attributes. - minName = None - - # Accumulate non-numeric enumerant values separately and append - # them following the numeric values, to allow for aliases. - # NOTE: this doesn't do a topological sort yet, so aliases of - # aliases can still get in the wrong order. - aliasText = [] - - for elem in enums: - # Convert the value to an integer and use that to track min/max. - # Values of form -(number) are accepted but nothing more complex. - # Should catch exceptions here for more complex constructs. Not yet. - (numVal, strVal) = self.enumToValue(elem, True) - name = elem.get('name') - - # Extension enumerants are only included if they are required - if self.isEnumRequired(elem): - decl = '' - - protect = elem.get('protect') - if protect is not None: - decl += '#ifdef {}\n'.format(protect) - - # Indent requirements comment, if there is one - requirements = self.genRequirements(name, mustBeFound = False) - if requirements != '': - requirements = ' ' + requirements - decl += requirements - decl += ' {} = {},'.format(name, strVal) - - if protect is not None: - decl += '\n#endif' - - if numVal is not None: - body.append(decl) - else: - aliasText.append(decl) - - # Range check for the enum value - if numVal is not None and (numVal > maxValidValue or numVal < minValidValue): - self.logMsg('error', 'Allowable range for C enum types is [', minValidValue, ',', maxValidValue, '], but', name, 'has a value outside of this (', strVal, ')\n') - exit(1) - - # Don't track min/max for non-numbers (numVal is None) - if isEnum and numVal is not None and elem.get('extends') is None: - if minName is None: - minName = maxName = name - minValue = maxValue = numVal - elif numVal < minValue: - minName = name - minValue = numVal - elif numVal > maxValue: - maxName = name - maxValue = numVal - - # Now append the non-numeric enumerant values - body.extend(aliasText) - - # Generate min/max value tokens - legacy use case. - if isEnum and expand: - body.extend((" {}_BEGIN_RANGE{} = {},".format(expandPrefix, expandSuffix, minName), - " {}_END_RANGE{} = {},".format( - expandPrefix, expandSuffix, maxName), - " {}_RANGE_SIZE{} = ({} - {} + 1),".format(expandPrefix, expandSuffix, maxName, minName))) - - # Generate a range-padding value to ensure the enum is 32 bits, but - # only in code generators, so it doesn't appear in documentation - if (self.genOpts.codeGenerator or - self.conventions.generate_max_enum_in_docs): - body.append(" {}_MAX_ENUM{} = 0x7FFFFFFF".format( - expandPrefix, expandSuffix)) - - # Postfix - body.append("} %s;" % groupName) - - # Determine appropriate section for this declaration - if groupElem.get('type') == 'bitmask': - section = 'bitmask' - else: - section = 'group' - - return (section, '\n'.join(body)) - - def buildConstantCDecl(self, enuminfo, name, alias): - """Generate the C declaration for a constant (a single - value). - - tags may specify their values in several ways, but are - usually just integers or floating-point numbers.""" - - (_, strVal) = self.enumToValue(enuminfo.elem, False) - - if self.misracppstyle() and enuminfo.elem.get('type') and not alias: - # Generate e.g.: static constexpr uint32_t x = ~static_cast(1U); - # This appeases MISRA "underlying type" rules. - typeStr = enuminfo.elem.get('type'); - invert = '~' in strVal - number = strVal.strip("()~UL") - if typeStr != "float": - number += 'U' - strVal = "~" if invert else "" - strVal += "static_cast<" + typeStr + ">(" + number + ")" - body = 'static constexpr ' + typeStr.ljust(9) + name.ljust(33) + ' {' + strVal + '};' - elif enuminfo.elem.get('type') and not alias: - # Generate e.g.: #define x (~0ULL) - typeStr = enuminfo.elem.get('type'); - invert = '~' in strVal - paren = '(' in strVal - number = strVal.strip("()~UL") - if typeStr != "float": - if typeStr == "uint64_t": - number += 'ULL' - else: - number += 'U' - strVal = "~" if invert else "" - strVal += number - if paren: - strVal = "(" + strVal + ")"; - body = '#define ' + name.ljust(33) + ' ' + strVal; - else: - body = '#define ' + name.ljust(33) + ' ' + strVal - - return body - - def makeDir(self, path): - """Create a directory, if not already done. - - Generally called from derived generators creating hierarchies.""" - self.logMsg('diag', 'OutputGenerator::makeDir(' + path + ')') - if path not in self.madeDirs: - # This can get race conditions with multiple writers, see - # https://stackoverflow.com/questions/273192/ - if not os.path.exists(path): - os.makedirs(path) - self.madeDirs[path] = None - - def beginFile(self, genOpts): - """Start a new interface file - - - genOpts - GeneratorOptions controlling what's generated and how""" - self.genOpts = genOpts - self.should_insert_may_alias_macro = \ - self.genOpts.conventions.should_insert_may_alias_macro(self.genOpts) - - # Try to import the API dictionary, api.py, if it exists. Nothing in - # api.py cannot be extracted directly from the XML, and in the - # future we should do that. - if self.genOpts.genpath is not None: - try: - sys.path.insert(0, self.genOpts.genpath) - import api - self.apidict = api - except ImportError: - self.apidict = None - - self.conventions = genOpts.conventions - - # Open a temporary file for accumulating output. - if self.genOpts.filename is not None: - self.outFile = tempfile.NamedTemporaryFile(mode='w', encoding='utf-8', newline='\n', delete=False) - else: - self.outFile = sys.stdout - - def endFile(self): - if self.errFile: - self.errFile.flush() - if self.warnFile: - self.warnFile.flush() - if self.diagFile: - self.diagFile.flush() - self.outFile.flush() - if self.outFile != sys.stdout and self.outFile != sys.stderr: - self.outFile.close() - - # On successfully generating output, move the temporary file to the - # target file. - if self.genOpts.filename is not None: - if sys.platform == 'win32': - directory = Path(self.genOpts.directory) - if not Path.exists(directory): - os.makedirs(directory) - shutil.copy(self.outFile.name, self.genOpts.directory + '/' + self.genOpts.filename) - os.remove(self.outFile.name) - self.genOpts = None - - def beginFeature(self, interface, emit): - """Write interface for a feature and tag generated features as having been done. - - - interface - element for the `` / `` to generate - - emit - actually write to the header only when True""" - self.emit = emit - self.featureName = interface.get('name') - # If there's an additional 'protect' attribute in the feature, save it - self.featureExtraProtect = interface.get('protect') - - def endFeature(self): - """Finish an interface file, closing it when done. - - Derived classes responsible for emitting feature""" - self.featureName = None - self.featureExtraProtect = None - - def genRequirements(self, name, mustBeFound = True): - """Generate text showing what core versions and extensions introduce - an API. This exists in the base Generator class because it's used by - the shared enumerant-generating interfaces (buildEnumCDecl, etc.). - Here it returns an empty string for most generators, but can be - overridden by e.g. DocGenerator. - - - name - name of the API - - mustBeFound - If True, when requirements for 'name' cannot be - determined, a warning comment is generated. - """ - - return '' - - def validateFeature(self, featureType, featureName): - """Validate we're generating something only inside a `` tag""" - if self.featureName is None: - raise UserWarning('Attempt to generate', featureType, - featureName, 'when not in feature') - - def genType(self, typeinfo, name, alias): - """Generate interface for a type - - - typeinfo - TypeInfo for a type - - Extend to generate as desired in your derived class.""" - self.validateFeature('type', name) - - def genStruct(self, typeinfo, typeName, alias): - """Generate interface for a C "struct" type. - - - typeinfo - TypeInfo for a type interpreted as a struct - - Extend to generate as desired in your derived class.""" - self.validateFeature('struct', typeName) - - # The mixed-mode tags may contain no-op tags. - # It is convenient to remove them here where all output generators - # will benefit. - for member in typeinfo.elem.findall('.//member'): - for comment in member.findall('comment'): - member.remove(comment) - - def genGroup(self, groupinfo, groupName, alias): - """Generate interface for a group of enums (C "enum") - - - groupinfo - GroupInfo for a group. - - Extend to generate as desired in your derived class.""" - - self.validateFeature('group', groupName) - - def genEnum(self, enuminfo, typeName, alias): - """Generate interface for an enum (constant). - - - enuminfo - EnumInfo for an enum - - name - enum name - - Extend to generate as desired in your derived class.""" - self.validateFeature('enum', typeName) - - def genCmd(self, cmd, cmdinfo, alias): - """Generate interface for a command. - - - cmdinfo - CmdInfo for a command - - Extend to generate as desired in your derived class.""" - self.validateFeature('command', cmdinfo) - - def genSpirv(self, spirv, spirvinfo, alias): - """Generate interface for a spirv element. - - - spirvinfo - SpirvInfo for a command - - Extend to generate as desired in your derived class.""" - return - - def makeProtoName(self, name, tail): - """Turn a `` `` into C-language prototype - and typedef declarations for that name. - - - name - contents of `` tag - - tail - whatever text follows that tag in the Element""" - return self.genOpts.apientry + name + tail - - def makeTypedefName(self, name, tail): - """Make the function-pointer typedef name for a command.""" - return '(' + self.genOpts.apientryp + 'PFN_' + name + tail + ')' - - def makeCParamDecl(self, param, aligncol): - """Return a string which is an indented, formatted - declaration for a `` or `` block (e.g. function parameter - or structure/union member). - - - param - Element (`` or ``) to format - - aligncol - if non-zero, attempt to align the nested `` element - at this column""" - indent = ' ' - paramdecl = indent - prefix = noneStr(param.text) - - for elem in param: - text = noneStr(elem.text) - tail = noneStr(elem.tail) - - if self.should_insert_may_alias_macro and self.genOpts.conventions.is_voidpointer_alias(elem.tag, text, tail): - # OpenXR-specific macro insertion - but not in apiinc for the spec - tail = self.genOpts.conventions.make_voidpointer_alias(tail) - if elem.tag == 'name' and aligncol > 0: - self.logMsg('diag', 'Aligning parameter', elem.text, 'to column', self.genOpts.alignFuncParam) - # Align at specified column, if possible - paramdecl = paramdecl.rstrip() - oldLen = len(paramdecl) - # This works around a problem where very long type names - - # longer than the alignment column - would run into the tail - # text. - paramdecl = paramdecl.ljust(aligncol - 1) + ' ' - newLen = len(paramdecl) - self.logMsg('diag', 'Adjust length of parameter decl from', oldLen, 'to', newLen, ':', paramdecl) - - if (self.misracppstyle() and prefix.find('const ') != -1): - # Change pointer type order from e.g. "const void *" to "void const *". - # If the string starts with 'const', reorder it to be after the first type. - paramdecl += prefix.replace('const ', '') + text + ' const' + tail - else: - paramdecl += prefix + text + tail - - # Clear prefix for subsequent iterations - prefix = '' - - # If prefix was originally non-empty and the param has no elements - # (e.g. is nothing but text), preserve it. - paramdecl = paramdecl + prefix - - if aligncol == 0: - # Squeeze out multiple spaces other than the indentation - paramdecl = indent + ' '.join(paramdecl.split()) - return paramdecl - - def getCParamTypeLength(self, param): - """Return the length of the type field is an indented, formatted - declaration for a `` or `` block (e.g. function parameter - or structure/union member). - - - param - Element (`` or ``) to identify""" - - # Allow for missing tag - newLen = 0 - paramdecl = ' ' + noneStr(param.text) - for elem in param: - text = noneStr(elem.text) - tail = noneStr(elem.tail) - - if self.should_insert_may_alias_macro and self.genOpts.conventions.is_voidpointer_alias(elem.tag, text, tail): - # OpenXR-specific macro insertion - tail = self.genOpts.conventions.make_voidpointer_alias(tail) - if elem.tag == 'name': - # Align at specified column, if possible - newLen = len(paramdecl.rstrip()) - self.logMsg('diag', 'Identifying length of', elem.text, 'as', newLen) - paramdecl += text + tail - - return newLen - - def getMaxCParamTypeLength(self, info): - """Return the length of the longest type field for a member/parameter. - - - info - TypeInfo or CommandInfo. - """ - lengths = (self.getCParamTypeLength(member) - for member in info.getMembers()) - return max(lengths) - - def getHandleParent(self, typename): - """Get the parent of a handle object.""" - info = self.registry.typedict.get(typename) - if info is None: - return None - - elem = info.elem - if elem is not None: - return elem.get('parent') - - return None - - def iterateHandleAncestors(self, typename): - """Iterate through the ancestors of a handle type.""" - current = self.getHandleParent(typename) - while current is not None: - yield current - current = self.getHandleParent(current) - - def getHandleAncestors(self, typename): - """Get the ancestors of a handle object.""" - return list(self.iterateHandleAncestors(typename)) - - def getTypeCategory(self, typename): - """Get the category of a type.""" - info = self.registry.typedict.get(typename) - if info is None: - return None - - elem = info.elem - if elem is not None: - return elem.get('category') - return None - - def isStructAlwaysValid(self, structname): - """Try to do check if a structure is always considered valid (i.e. there's no rules to its acceptance).""" - # A conventions object is required for this call. - if not self.conventions: - raise RuntimeError("To use isStructAlwaysValid, be sure your options include a Conventions object.") - - if self.conventions.type_always_valid(structname): - return True - - category = self.getTypeCategory(structname) - if self.conventions.category_requires_validation(category): - return False - - info = self.registry.typedict.get(structname) - assert(info is not None) - - members = info.getMembers() - - for member in members: - member_name = getElemName(member) - if member_name in (self.conventions.structtype_member_name, - self.conventions.nextpointer_member_name): - return False - - if member.get('noautovalidity'): - return False - - member_type = getElemType(member) - - if member_type in ('void', 'char') or self.paramIsArray(member) or self.paramIsPointer(member): - return False - - if self.conventions.type_always_valid(member_type): - continue - - member_category = self.getTypeCategory(member_type) - - if self.conventions.category_requires_validation(member_category): - return False - - if member_category in ('struct', 'union'): - if self.isStructAlwaysValid(member_type) is False: - return False - - return True - - def isEnumRequired(self, elem): - """Return True if this `` element is - required, False otherwise - - - elem - `` element to test""" - required = elem.get('required') is not None - self.logMsg('diag', 'isEnumRequired:', elem.get('name'), - '->', required) - return required - - # @@@ This code is overridden by equivalent code now run in - # @@@ Registry.generateFeature - - required = False - - extname = elem.get('extname') - if extname is not None: - # 'supported' attribute was injected when the element was - # moved into the group in Registry.parseTree() - if self.genOpts.defaultExtensions == elem.get('supported'): - required = True - elif re.match(self.genOpts.addExtensions, extname) is not None: - required = True - elif elem.get('version') is not None: - required = re.match(self.genOpts.emitversions, elem.get('version')) is not None - else: - required = True - - return required - - def makeCDecls(self, cmd): - """Return C prototype and function pointer typedef for a - `` Element, as a two-element list of strings. - - - cmd - Element containing a `` tag""" - proto = cmd.find('proto') - params = cmd.findall('param') - # Begin accumulating prototype and typedef strings - pdecl = self.genOpts.apicall - tdecl = 'typedef ' - - # Insert the function return type/name. - # For prototypes, add APIENTRY macro before the name - # For typedefs, add (APIENTRY *) around the name and - # use the PFN_cmdnameproc naming convention. - # Done by walking the tree for element by element. - # etree has elem.text followed by (elem[i], elem[i].tail) - # for each child element and any following text - # Leading text - pdecl += noneStr(proto.text) - tdecl += noneStr(proto.text) - # For each child element, if it's a wrap in appropriate - # declaration. Otherwise append its contents and tail contents. - for elem in proto: - text = noneStr(elem.text) - tail = noneStr(elem.tail) - if elem.tag == 'name': - pdecl += self.makeProtoName(text, tail) - tdecl += self.makeTypedefName(text, tail) - else: - pdecl += text + tail - tdecl += text + tail - - if self.genOpts.alignFuncParam == 0: - # Squeeze out multiple spaces - there is no indentation - pdecl = ' '.join(pdecl.split()) - tdecl = ' '.join(tdecl.split()) - - # Now add the parameter declaration list, which is identical - # for prototypes and typedefs. Concatenate all the text from - # a node without the tags. No tree walking required - # since all tags are ignored. - # Uses: self.indentFuncProto - # self.indentFuncPointer - # self.alignFuncParam - n = len(params) - # Indented parameters - if n > 0: - indentdecl = '(\n' - indentdecl += ',\n'.join(self.makeCParamDecl(p, self.genOpts.alignFuncParam) - for p in params) - indentdecl += ');' - else: - indentdecl = '(void);' - # Non-indented parameters - paramdecl = '(' - if n > 0: - paramnames = [] - if self.misracppstyle(): - for p in params: - param = '' - firstIter = True; - for t in p.itertext(): - if (firstIter): - prefix = t - firstIter = False - else: - # Change pointer type order from e.g. "const void *" to "void const *". - # If the string starts with 'const', reorder it to be after the first type. - if (prefix.find('const ') != -1): - param += prefix.replace('const ', '') + t + ' const ' - else: - param += prefix + t - # Clear prefix for subsequent iterations - prefix = '' - paramnames.append(param); - else: - paramnames = (''.join(t for t in p.itertext()) - for p in params) - paramdecl += ', '.join(paramnames) - else: - paramdecl += 'void' - paramdecl += ");" - return [pdecl + indentdecl, tdecl + paramdecl] - - def newline(self): - """Print a newline to the output file (utility function)""" - write('', file=self.outFile) - - def setRegistry(self, registry): - self.registry = registry diff --git a/xml/pygenerator.py b/xml/pygenerator.py deleted file mode 100644 index b2e76e66d..000000000 --- a/xml/pygenerator.py +++ /dev/null @@ -1,365 +0,0 @@ -#!/usr/bin/python3 -i -# -# Copyright 2013-2023 The Khronos Group Inc. -# -# SPDX-License-Identifier: Apache-2.0 - -import sys -from generator import OutputGenerator, enquote, noneStr, write -import pprint - -class PyOutputGenerator(OutputGenerator): - """PyOutputGenerator - subclass of OutputGenerator. - Generates Python data structures describing API names and relationships. - Similar to DocOutputGenerator, but writes a single file.""" - - def apiName(self, name): - """Return True if name is in the reserved API namespace. - - Delegates to the conventions object. """ - return self.genOpts.conventions.is_api_name(name) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # Track features being generated - self.features = [] - - # Reverse map from interface names to features requiring them - self.apimap = {} - - def beginFile(self, genOpts): - OutputGenerator.beginFile(self, genOpts) - # - # Dictionaries are keyed by the name of the entity (e.g. - # self.structs is keyed by structure names). Values are - # the names of related entities (e.g. structs contain - # a list of type names of members, enums contain a list - # of enumerants belong to the enumerated type, etc.), or - # just None if there are no directly related entities. - # - # Collect the mappings, then emit the Python script in endFile - self.basetypes = {} - self.consts = {} - self.enums = {} - self.flags = {} - self.funcpointers = {} - self.protos = {} - self.structs = {} - self.handles = {} - self.defines = {} - self.alias = {} - # Dictionary containing the type of a type name - # (e.g. the string name of the dictionary with its contents). - self.typeCategory = {} - self.mapDict = {} - - def addInterfaceMapping(self, api, feature, required): - """Add a reverse mapping in self.apimap from an API to a feature - requiring that API. - - - api - name of the API - - feature - name of the feature requiring it - - required - None, or an additional feature dependency within - 'feature' """ - - # Each entry in self.apimap contains one or more - # ( feature, required ) tuples. - deps = ( feature, required ) - - if api in self.apimap: - self.apimap[api].append(deps) - else: - self.apimap[api] = [ deps ] - - def mapInterfaceKeys(self, feature, key): - """Construct reverse mapping of APIs to features requiring them in - self.apimap. - - - feature - name of the feature being generated - - key - API category - 'define', 'basetype', etc.""" - - dict = self.featureDictionary[feature][key] - - if dict: - # Not clear why handling of command vs. type APIs is different - - # see interfacedocgenerator.py, which this was based on. - if key == 'command': - for required in dict: - for api in dict[required]: - self.addInterfaceMapping(api, feature, required) - else: - for required in dict: - for parent in dict[required]: - for api in dict[required][parent]: - self.addInterfaceMapping(api, feature, required) - - def mapInterfaces(self, feature): - """Construct reverse mapping of APIs to features requiring them in - self.apimap. - - - feature - name of the feature being generated""" - - # Map each category of interface - self.mapInterfaceKeys(feature, 'basetype') - self.mapInterfaceKeys(feature, 'bitmask') - self.mapInterfaceKeys(feature, 'command') - self.mapInterfaceKeys(feature, 'define') - self.mapInterfaceKeys(feature, 'enum') - self.mapInterfaceKeys(feature, 'enumconstant') - self.mapInterfaceKeys(feature, 'funcpointer') - self.mapInterfaceKeys(feature, 'handle') - self.mapInterfaceKeys(feature, 'include') - self.mapInterfaceKeys(feature, 'struct') - self.mapInterfaceKeys(feature, 'union') - - def endFile(self): - # Print out all the dictionaries as Python strings. - # Could just print(dict) but that's not human-readable - dicts = ( [ self.basetypes, 'basetypes' ], - [ self.consts, 'consts' ], - [ self.enums, 'enums' ], - [ self.flags, 'flags' ], - [ self.funcpointers, 'funcpointers' ], - [ self.protos, 'protos' ], - [ self.structs, 'structs' ], - [ self.handles, 'handles' ], - [ self.defines, 'defines' ], - [ self.typeCategory, 'typeCategory' ], - [ self.alias, 'alias' ] ) - for (entry_dict, name) in dicts: - write(name + ' = {}', file=self.outFile) - for key in sorted(entry_dict.keys()): - write(name + '[' + enquote(key) + '] = ', entry_dict[key], - file=self.outFile) - - # Dictionary containing the relationships of a type - # (e.g. a dictionary with each related type as keys). - write('mapDict = {}', file=self.outFile) - - # Could just print(self.mapDict), but prefer something - # human-readable and stable-ordered - for baseType in sorted(self.mapDict.keys()): - write('mapDict[' + enquote(baseType) + '] = ', file=self.outFile, end='') - pprint.pprint(self.mapDict[baseType], self.outFile) - - # Generate feature <-> interface mappings - for feature in self.features: - self.mapInterfaces(feature) - - # Write out the reverse map from APIs to requiring features - write('requiredBy = {}', file=self.outFile) - - for api in sorted(self.apimap): - # Construct list of requirements as Python list arguments - ##reqs = ', '.join('({}, {})'.format(enquote(dep[0]), enquote(dep[1])) for dep in self.apimap[api]) - ##write('requiredBy[{}] = ( {} )'.format(enquote(api), reqs), file=self.outFile) - - # Ideally these would be sorted by dep[0] as well - reqs = ', '.join('({}, {})'.format(enquote(dep[0]), enquote(dep[1])) for dep in self.apimap[api]) - write('requiredBy[{}] = {}'.format(enquote(api), pprint.saferepr(self.apimap[api])), file=self.outFile) - - OutputGenerator.endFile(self) - - def beginFeature(self, interface, emit): - # Start processing in superclass - OutputGenerator.beginFeature(self, interface, emit) - - # Add this feature to the list being tracked - self.features.append( self.featureName ) - - def endFeature(self): - # Finish processing in superclass - OutputGenerator.endFeature(self) - - def addName(self, entry_dict, name, value): - """Add a string entry to the dictionary, quoting it so it gets printed - out correctly in self.endFile().""" - entry_dict[name] = enquote(value) - - def addMapping(self, baseType, refType): - """Add a mapping between types to mapDict. - - Only include API types, so we don't end up with a lot of useless uint32_t and void types.""" - if not self.apiName(baseType) or not self.apiName(refType): - self.logMsg('diag', 'PyOutputGenerator::addMapping: IGNORE map from', baseType, '<->', refType) - return - - self.logMsg('diag', 'PyOutputGenerator::addMapping: map from', - baseType, '<->', refType) - - if baseType not in self.mapDict: - baseDict = {} - self.mapDict[baseType] = baseDict - else: - baseDict = self.mapDict[baseType] - if refType not in self.mapDict: - refDict = {} - self.mapDict[refType] = refDict - else: - refDict = self.mapDict[refType] - - baseDict[refType] = None - refDict[baseType] = None - - def genType(self, typeinfo, name, alias): - """Generate type. - - - For 'struct' or 'union' types, defer to genStruct() to - add to the dictionary. - - For 'bitmask' types, add the type name to the 'flags' dictionary, - with the value being the corresponding 'enums' name defining - the acceptable flag bits. - - For 'enum' types, add the type name to the 'enums' dictionary, - with the value being '@STOPHERE@' (because this case seems - never to happen). - - For 'funcpointer' types, add the type name to the 'funcpointers' - dictionary. - - For 'handle' and 'define' types, add the handle or #define name - to the 'struct' dictionary, because that's how the spec sources - tag these types even though they aren't structs.""" - OutputGenerator.genType(self, typeinfo, name, alias) - typeElem = typeinfo.elem - # If the type is a struct type, traverse the embedded tags - # generating a structure. Otherwise, emit the tag text. - category = typeElem.get('category') - - # Add a typeCategory{} entry for the category of this type. - self.addName(self.typeCategory, name, category) - - if category in ('struct', 'union'): - self.genStruct(typeinfo, name, alias) - else: - if alias: - # Add name -> alias mapping - self.addName(self.alias, name, alias) - - # Always emit an alias (?!) - count = 1 - - # May want to only emit full type definition when not an alias? - else: - # Extract the type name - # (from self.genOpts). Copy other text through unchanged. - # If the resulting text is an empty string, don't emit it. - count = len(noneStr(typeElem.text)) - for elem in typeElem: - count += len(noneStr(elem.text)) + len(noneStr(elem.tail)) - - if count > 0: - if category == 'bitmask': - requiredEnum = typeElem.get('requires') - self.addName(self.flags, name, requiredEnum) - - # This happens when the Flags type is defined, but no - # FlagBits are defined yet. - if requiredEnum is not None: - self.addMapping(name, requiredEnum) - elif category == 'enum': - # This case does not seem to come up. It nominally would - # result from - # , - # but the output generator doesn't emit them directly. - self.logMsg('warn', 'PyOutputGenerator::genType: invalid \'enum\' category for name:', name) - elif category == 'funcpointer': - self.funcpointers[name] = None - elif category == 'handle': - self.handles[name] = None - elif category == 'define': - self.defines[name] = None - elif category == 'basetype': - # Don't add an entry for base types that are not API types - # e.g. an API Bool type gets an entry, uint32_t does not - if self.apiName(name): - self.basetypes[name] = None - self.addName(self.typeCategory, name, 'basetype') - else: - self.logMsg('diag', 'PyOutputGenerator::genType: unprocessed type:', name, 'category:', category) - else: - self.logMsg('diag', 'PyOutputGenerator::genType: unprocessed type:', name) - - def genStruct(self, typeinfo, typeName, alias): - """Generate struct (e.g. C "struct" type). - - Add the struct name to the 'structs' dictionary, with the - value being an ordered list of the struct member names.""" - OutputGenerator.genStruct(self, typeinfo, typeName, alias) - - if alias: - # Add name -> alias mapping - self.addName(self.alias, typeName, alias) - else: - # May want to only emit definition on this branch - True - - members = [member.text for member in typeinfo.elem.findall('.//member/name')] - self.structs[typeName] = members - memberTypes = [member.text for member in typeinfo.elem.findall('.//member/type')] - for member_type in memberTypes: - self.addMapping(typeName, member_type) - - def genGroup(self, groupinfo, groupName, alias): - """Generate group (e.g. C "enum" type). - - These are concatenated together with other types. - - - Add the enum type name to the 'enums' dictionary, with - the value being an ordered list of the enumerant names. - - Add each enumerant name to the 'consts' dictionary, with - the value being the enum type the enumerant is part of.""" - OutputGenerator.genGroup(self, groupinfo, groupName, alias) - groupElem = groupinfo.elem - - if alias: - # Add name -> alias mapping - self.addName(self.alias, groupName, alias) - else: - # May want to only emit definition on this branch - True - - # Loop over the nested 'enum' tags. - enumerants = [elem.get('name') for elem in groupElem.findall('enum')] - for name in enumerants: - self.addName(self.consts, name, groupName) - self.enums[groupName] = enumerants - - def genEnum(self, enuminfo, name, alias): - """Generate enumerant (compile-time constants). - - - Add the constant name to the 'consts' dictionary, with the - value being None to indicate that the constant isn't - an enumeration value.""" - OutputGenerator.genEnum(self, enuminfo, name, alias) - - if name not in self.consts: - # Add a typeCategory{} entry for the category of this type. - self.addName(self.typeCategory, name, 'consts') - self.consts[name] = None - # Otherwise, don't add it to the consts dictionary because it's - # already present. This happens due to the generator 'reparentEnums' - # parameter being False, so each extension enum appears in both the - # type and in the or it originally - # came from. - - def genCmd(self, cmdinfo, name, alias): - """Generate command. - - - Add the command name to the 'protos' dictionary, with the - value being an ordered list of the parameter names.""" - OutputGenerator.genCmd(self, cmdinfo, name, alias) - - if alias: - # Add name -> alias mapping - self.addName(self.alias, name, alias) - else: - # May want to only emit definition on this branch - True - - # Add a typeCategory{} entry for the category of this type. - self.addName(self.typeCategory, name, 'protos') - - params = [param.text for param in cmdinfo.elem.findall('param/name')] - self.protos[name] = params - paramTypes = [param.text for param in cmdinfo.elem.findall('param/type')] - for param_type in paramTypes: - self.addMapping(name, param_type) diff --git a/xml/realign.py b/xml/realign.py deleted file mode 100644 index b59865b3d..000000000 --- a/xml/realign.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/python3 -# -# Copyright 2013-2023 The Khronos Group Inc. -# SPDX-License-Identifier: Apache-2.0 - -# Usage: realign [infile] > outfile -# Used to realign XML tags in the Vulkan registry after it's operated on by -# some other filter, since whitespace inside a tag isn't part of the -# internal representation. - -import copy, sys, string, re - -def realignXML(fp): - patterns = [ - [ '(^ *\ 1): - realignXML(open(sys.argv[1], 'r', encoding='utf-8')) - else: - realignXML(sys.stdin) diff --git a/xml/reflib.py b/xml/reflib.py deleted file mode 100644 index 426a1811b..000000000 --- a/xml/reflib.py +++ /dev/null @@ -1,663 +0,0 @@ -#!/usr/bin/python3 -# -# Copyright 2016-2023 The Khronos Group Inc. -# -# SPDX-License-Identifier: Apache-2.0 - -# Utility functions for automatic ref page generation and other script stuff - -import io -import re -import sys -import subprocess - -# global errFile, warnFile, diagFile - -errFile = sys.stderr -warnFile = sys.stdout -diagFile = None -logSourcefile = None -logProcname = None -logLine = None - -def unescapeQuotes(s): - """Remove \' escape sequences in a string (refpage description)""" - return s.replace('\\\'', '\'') - -def write(*args, **kwargs ): - file = kwargs.pop('file',sys.stdout) - end = kwargs.pop('end','\n') - file.write(' '.join(str(arg) for arg in args)) - file.write(end) - -def setLogSourcefile(filename): - """Metadata which may be printed (if not None) for diagnostic messages""" - global logSourcefile - logSourcefile = filename - -def setLogProcname(procname): - global logProcname - logProcname = procname - -def setLogLine(line): - global logLine - logLine = line - -def logHeader(severity): - """Generate prefix for a diagnostic line using metadata and severity""" - global logSourcefile, logProcname, logLine - - msg = severity + ': ' - if logProcname: - msg = msg + ' in ' + logProcname - if logSourcefile: - msg = msg + ' for ' + logSourcefile - if logLine: - msg = msg + ' line ' + str(logLine) - return msg + ' ' - -def setLogFile(setDiag, setWarn, filename): - """Set the file handle to log either or both warnings and diagnostics to. - - - setDiag and setWarn are True if the corresponding handle is to be set. - - filename is None for no logging, '-' for stdout, or a pathname.""" - global diagFile, warnFile - - if filename is None: - return - - if filename == '-': - fp = sys.stdout - else: - fp = open(filename, 'w', encoding='utf-8') - - if setDiag: - diagFile = fp - if setWarn: - warnFile = fp - -def logDiag(*args, **kwargs): - file = kwargs.pop('file', diagFile) - end = kwargs.pop('end','\n') - if file is not None: - file.write(logHeader('DIAG') + ' '.join(str(arg) for arg in args)) - file.write(end) - -def logWarn(*args, **kwargs): - file = kwargs.pop('file', warnFile) - end = kwargs.pop('end','\n') - if file is not None: - file.write(logHeader('WARN') + ' '.join(str(arg) for arg in args)) - file.write(end) - -def logErr(*args, **kwargs): - file = kwargs.pop('file', errFile) - end = kwargs.pop('end','\n') - - strfile = io.StringIO() - strfile.write(logHeader('ERROR') + ' '.join(str(arg) for arg in args)) - strfile.write(end) - - if file is not None: - file.write(strfile.getvalue()) - sys.exit(1) - -def isempty(s): - """Return True if s is nothing but white space, False otherwise""" - return len(''.join(s.split())) == 0 - -class pageInfo: - """Information about a ref page relative to the file it's extracted from.""" - def __init__(self): - self.extractPage = True - """True if page should be extracted""" - - self.Warning = None - """string warning if page is suboptimal or can't be generated""" - - self.embed = False - """False or the name of the ref page this include is embedded within""" - - self.type = None - """'structs', 'protos', 'funcpointers', 'flags', 'enums'""" - - self.name = None - """struct/proto/enumerant/etc. name""" - - self.desc = None - """short description of ref page""" - - self.begin = None - """index of first line of the page (heuristic or // refBegin)""" - - self.include = None - """index of include:: line defining the page""" - - self.param = None - """index of first line of parameter/member definitions""" - - self.body = None - """index of first line of body text""" - - self.validity = None - """index of validity include""" - - self.end = None - """index of last line of the page (heuristic validity include, or // refEnd)""" - - self.alias = '' - """aliases of this name, if supplied, or ''""" - - self.refs = '' - """cross-references on // refEnd line, if supplied""" - - self.spec = None - """'spec' attribute in refpage open block, if supplied, or None for the default ('api') type""" - - self.anchor = None - """'anchor' attribute in refpage open block, if supplied, or inferred to be the same as the 'name'""" - -def printPageInfoField(desc, line, file): - """Print a single field of a pageInfo struct, possibly None. - - - desc - string description of field - - line - field value or None - - file - indexed by line""" - if line is not None: - logDiag(desc + ':', line + 1, '\t-> ', file[line], end='') - else: - logDiag(desc + ':', line) - -def printPageInfo(pi, file): - """Print out fields of a pageInfo struct - - - pi - pageInfo - - file - indexed by pageInfo""" - logDiag('TYPE: ', pi.type) - logDiag('NAME: ', pi.name) - logDiag('WARNING:', pi.Warning) - logDiag('EXTRACT:', pi.extractPage) - logDiag('EMBED: ', pi.embed) - logDiag('DESC: ', pi.desc) - printPageInfoField('BEGIN ', pi.begin, file) - printPageInfoField('INCLUDE ', pi.include, file) - printPageInfoField('PARAM ', pi.param, file) - printPageInfoField('BODY ', pi.body, file) - printPageInfoField('VALIDITY', pi.validity, file) - printPageInfoField('END ', pi.end, file) - logDiag('REFS: "' + pi.refs + '"') - -def prevPara(file, line): - """Go back one paragraph from the specified line and return the line number - of the first line of that paragraph. - - Paragraphs are delimited by blank lines. It is assumed that the - current line is the first line of a paragraph. - - - file is an array of strings - - line is the starting point (zero-based)""" - # Skip over current paragraph - while (line >= 0 and not isempty(file[line])): - line = line - 1 - # Skip over white space - while (line >= 0 and isempty(file[line])): - line = line - 1 - # Skip to first line of previous paragraph - while (line >= 1 and not isempty(file[line-1])): - line = line - 1 - return line - -def nextPara(file, line): - """Go forward one paragraph from the specified line and return the line - number of the first line of that paragraph. - - Paragraphs are delimited by blank lines. It is assumed that the - current line is standalone (which is bogus). - - - file is an array of strings - - line is the starting point (zero-based)""" - maxLine = len(file) - 1 - # Skip over current paragraph - while (line != maxLine and not isempty(file[line])): - line = line + 1 - # Skip over white space - while (line != maxLine and isempty(file[line])): - line = line + 1 - return line - -def lookupPage(pageMap, name): - """Return (creating if needed) the pageInfo entry in pageMap for name""" - if name not in pageMap: - pi = pageInfo() - pi.name = name - pageMap[name] = pi - else: - pi = pageMap[name] - return pi - -def loadFile(filename): - """Load a file into a list of strings. Return the list or None on failure""" - try: - fp = open(filename, 'r', encoding='utf-8') - except: - logWarn('Cannot open file', filename, ':', sys.exc_info()[0]) - return None - - file = fp.readlines() - fp.close() - - return file - -def clampToBlock(line, minline, maxline): - """Clamp a line number to be in the range [minline,maxline]. - - If the line number is None, just return it. - If minline is None, don't clamp to that value.""" - if line is None: - return line - if minline and line < minline: - return minline - if line > maxline: - return maxline - - return line - -def fixupRefs(pageMap, specFile, file): - """Fill in missing fields in pageInfo structures, to the extent they can be - inferred. - - - pageMap - dictionary of pageInfo structures - - specFile - filename - - file - list of strings making up the file, indexed by pageInfo""" - # All potential ref pages are now in pageMap. Process them to - # identify actual page start/end/description boundaries, if - # not already determined from the text. - for name in sorted(pageMap.keys()): - pi = pageMap[name] - - # # If nothing is found but an include line with no begin, validity, - # # or end, this is not intended as a ref page (yet). Set the begin - # # line to the include line, so autogeneration can at least - # # pull the include out, but mark it not to be extracted. - # # Examples include the host sync table includes in - # # chapters/fundamentals.txt and the table of Vk*Flag types in - # # appendices/boilerplate.txt. - # if pi.begin is None and pi.validity is None and pi.end is None: - # pi.begin = pi.include - # pi.extractPage = False - # pi.Warning = 'No begin, validity, or end lines identified' - # continue - - # Using open block delimiters, ref pages must *always* have a - # defined begin and end. If either is undefined, that's fatal. - if pi.begin is None: - pi.extractPage = False - pi.Warning = 'Can\'t identify begin of ref page open block' - continue - - if pi.end is None: - pi.extractPage = False - pi.Warning = 'Can\'t identify end of ref page open block' - continue - - # If there's no description of the page, infer one from the type - if pi.desc is None: - if pi.type is not None: - # pi.desc = pi.type[0:len(pi.type)-1] + ' (no short description available)' - pi.Warning = 'No short description available; could infer from the type and name' - else: - pi.extractPage = False - pi.Warning = 'No short description available, cannot infer from the type' - continue - - # Try to determine where the parameter and body sections of the page - # begin. funcpointer, proto, and struct pages infer the location of - # the parameter and body sections. Other pages infer the location of - # the body, but have no parameter sections. - if pi.include is not None: - if pi.type in ['funcpointers', 'protos', 'structs']: - pi.param = nextPara(file, pi.include) - if pi.body is None: - pi.body = nextPara(file, pi.param) - else: - if pi.body is None: - pi.body = nextPara(file, pi.include) - else: - pi.Warning = 'Page does not have an API definition include::' - - # It's possible for the inferred param and body lines to run past - # the end of block, if, for example, there is no parameter section. - pi.param = clampToBlock(pi.param, pi.include, pi.end) - pi.body = clampToBlock(pi.body, pi.param, pi.end) - - # We can get to this point with .include, .param, and .validity - # all being None, indicating those sections weren't found. - - logDiag('fixupRefs: after processing,', pi.name, 'looks like:') - printPageInfo(pi, file) - - # Now that all the valid pages have been found, try to make some - # inferences about invalid pages. - # - # If a reference without a .end is entirely inside a valid reference, - # then it's intentionally embedded - may want to create an indirect - # page that links into the embedding page. This is done by a very - # inefficient double loop, but the loop depth is small. - for name in sorted(pageMap.keys()): - pi = pageMap[name] - - if pi.end is None: - for embedName in sorted(pageMap.keys()): - logDiag('fixupRefs: comparing', pi.name, 'to', embedName) - embed = pageMap[embedName] - # Don't check embeddings which are themselves invalid - if not embed.extractPage: - logDiag('Skipping check for embedding in:', embed.name) - continue - if embed.begin is None or embed.end is None: - logDiag('fixupRefs:', name + ':', - 'can\'t compare to unanchored ref:', embed.name, - 'in', specFile, 'at line', pi.include ) - printPageInfo(pi, file) - printPageInfo(embed, file) - # If an embed is found, change the error to a warning - elif (pi.include is not None and pi.include >= embed.begin and - pi.include <= embed.end): - logDiag('fixupRefs: Found embed for:', name, - 'inside:', embedName, - 'in', specFile, 'at line', pi.include ) - pi.embed = embed.name - pi.Warning = 'Embedded in definition for ' + embed.name - break - else: - logDiag('fixupRefs: No embed match for:', name, - 'inside:', embedName, 'in', specFile, - 'at line', pi.include) - - -# Patterns used to recognize interesting lines in an asciidoc source file. -# These patterns are only compiled once. -INCSVAR_DEF = re.compile(r':INCS-VAR: (?P.*)') -endifPat = re.compile(r'^endif::(?P[\w_+,]+)\[\]') -beginPat = re.compile(r'^\[open,(?Prefpage=.*)\]') -# attribute key/value pairs of an open block -attribStr = r"([a-z]+)='([^'\\]*(?:\\.[^'\\]*)*)'" -attribPat = re.compile(attribStr) -bodyPat = re.compile(r'^// *refBody') -errorPat = re.compile(r'^// *refError') - -# This regex transplanted from check_spec_links -# It looks for either OpenXR or Vulkan generated file conventions, and for -# the api/validity include (generated_type), protos/struct/etc path -# (category), and API name (entity_name). It could be put into the API -# conventions object. -INCLUDE = re.compile( - r'include::(?P((../){1,4}|\{INCS-VAR\}/|\{generated\}/)(generated/)?)(?P[\w]+)/(?P\w+)/(?P[^./]+).txt[\[][\]]') - - -def findRefs(file, filename): - """Identify reference pages in a list of strings, returning a dictionary of - pageInfo entries for each one found, or None on failure.""" - setLogSourcefile(filename) - setLogProcname('findRefs') - - # To reliably detect the open blocks around reference pages, we must - # first detect the '[open,refpage=...]' markup delimiting the block; - # skip past the '--' block delimiter on the next line; and identify the - # '--' block delimiter closing the page. - # This can't be done solely with pattern matching, and requires state to - # track 'inside/outside block'. - # When looking for open blocks, possible states are: - # 'outside' - outside a block - # 'start' - have found the '[open...]' line - # 'inside' - have found the following '--' line - openBlockState = 'outside' - - # Dictionary of interesting line numbers and strings related to an API - # name - pageMap = {} - - numLines = len(file) - line = 0 - - # Track the pageInfo object corresponding to the current open block - pi = None - incsvar = None - - while (line < numLines): - setLogLine(line) - - # Look for a file-wide definition - matches = INCSVAR_DEF.match(file[line]) - if matches: - incsvar = matches.group('value') - logDiag('Matched INCS-VAR definition:', incsvar) - - line = line + 1 - continue - - # Perform INCS-VAR substitution immediately. - if incsvar and '{INCS-VAR}' in file[line]: - newLine = file[line].replace('{INCS-VAR}', incsvar) - logDiag('PERFORMING SUBSTITUTION', file[line], '->', newLine) - file[line] = newLine - - # Only one of the patterns can possibly match. Add it to - # the dictionary for that name. - - # [open,refpage=...] starting a refpage block - matches = beginPat.search(file[line]) - if matches is not None: - logDiag('Matched open block pattern') - attribs = matches.group('attribs') - - # If the previous open block wasn't closed, raise an error - if openBlockState != 'outside': - logErr('Nested open block starting at line', line, 'of', - filename) - - openBlockState = 'start' - - # Parse the block attributes - matches = attribPat.findall(attribs) - - # Extract each attribute - name = None - desc = None - refpage_type = None - spec_type = None - anchor = None - alias = None - xrefs = None - - for (key,value) in matches: - logDiag('got attribute', key, '=', value) - if key == 'refpage': - name = value - elif key == 'desc': - desc = unescapeQuotes(value) - elif key == 'type': - refpage_type = value - elif key == 'spec': - spec_type = value - elif key == 'anchor': - anchor = value - elif key == 'alias': - alias = value - elif key == 'xrefs': - xrefs = value - else: - logWarn('unknown open block attribute:', key) - - if name is None or desc is None or refpage_type is None: - logWarn('missing one or more required open block attributes:' - 'refpage, desc, or type') - # Leave pi is None so open block delimiters are ignored - else: - pi = lookupPage(pageMap, name) - pi.desc = desc - # Must match later type definitions in interface/validity includes - pi.type = refpage_type - pi.spec = spec_type - pi.anchor = anchor - if alias: - pi.alias = alias - if xrefs: - pi.refs = xrefs - logDiag('open block for', name, 'added DESC =', desc, - 'TYPE =', refpage_type, 'ALIAS =', alias, - 'XREFS =', xrefs, 'SPEC =', spec_type, - 'ANCHOR =', anchor) - - line = line + 1 - continue - - # '--' starting or ending and open block - if file[line].rstrip() == '--': - if openBlockState == 'outside': - # Only refpage open blocks should use -- delimiters - logWarn('Unexpected double-dash block delimiters') - elif openBlockState == 'start': - # -- delimiter following [open,refpage=...] - openBlockState = 'inside' - - if pi is None: - logWarn('no pageInfo available for opening -- delimiter') - else: - pi.begin = line + 1 - logDiag('opening -- delimiter: added BEGIN =', pi.begin) - elif openBlockState == 'inside': - # -- delimiter ending an open block - if pi is None: - logWarn('no pageInfo available for closing -- delimiter') - else: - pi.end = line - 1 - logDiag('closing -- delimiter: added END =', pi.end) - - openBlockState = 'outside' - pi = None - else: - logWarn('unknown openBlockState:', openBlockState) - - line = line + 1 - continue - - matches = INCLUDE.search(file[line]) - if matches is not None: - # Something got included, not sure what yet. - gen_type = matches.group('generated_type') - refpage_type = matches.group('category') - name = matches.group('entity_name') - - # This will never match in OpenCL - if gen_type == 'validity': - logDiag('Matched validity pattern') - if pi is not None: - if pi.type and refpage_type != pi.type: - logWarn('ERROR: pageMap[' + name + '] type:', - pi.type, 'does not match type:', refpage_type) - pi.type = refpage_type - pi.validity = line - logDiag('added TYPE =', pi.type, 'VALIDITY =', pi.validity) - else: - logWarn('validity include:: line NOT inside block') - - line = line + 1 - continue - - if gen_type == 'api': - logDiag('Matched include pattern') - if pi is not None: - if pi.include is not None: - logDiag('found multiple includes for this block') - if pi.type and refpage_type != pi.type: - logWarn('ERROR: pageMap[' + name + '] type:', - pi.type, 'does not match type:', refpage_type) - pi.type = refpage_type - pi.include = line - logDiag('added TYPE =', pi.type, 'INCLUDE =', pi.include) - else: - logWarn('interface include:: line NOT inside block') - - line = line + 1 - continue - - logDiag('ignoring unrecognized include line ', matches.group()) - - # Vulkan 1.1 markup allows the last API include construct to be - # followed by an asciidoctor endif:: construct (and also preceded, - # at some distance). - # This looks for endif:: immediately following an include:: line - # and, if found, moves the include boundary to this line. - matches = endifPat.search(file[line]) - if matches is not None and pi is not None: - if pi.include == line - 1: - logDiag('Matched endif pattern following include; moving include') - pi.include = line - else: - logDiag('Matched endif pattern (not following include)') - - line = line + 1 - continue - - matches = bodyPat.search(file[line]) - if matches is not None: - logDiag('Matched // refBody pattern') - if pi is not None: - pi.body = line - logDiag('added BODY =', pi.body) - else: - logWarn('// refBody line NOT inside block') - - line = line + 1 - continue - - # OpenCL spec uses // refError to tag "validity" (Errors) language, - # instead of /validity/ includes. - matches = errorPat.search(file[line]) - if matches is not None: - logDiag('Matched // refError pattern') - if pi is not None: - pi.validity = line - logDiag('added VALIDITY (refError) =', pi.validity) - else: - logWarn('// refError line NOT inside block') - - line = line + 1 - continue - - line = line + 1 - continue - - if pi is not None: - logErr('Unclosed open block at EOF!') - - setLogSourcefile(None) - setLogProcname(None) - setLogLine(None) - - return pageMap - - -def getBranch(): - """Determine current git branch - - Returns (branch name, ''), or (None, stderr output) if the branch name - can't be determined""" - - command = [ 'git', 'symbolic-ref', '--short', 'HEAD' ] - results = subprocess.run(command, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - # git command failed - if len(results.stderr) > 0: - return (None, results.stderr) - - # Remove newline from output and convert to a string - branch = results.stdout.rstrip().decode() - if len(branch) > 0: - # Strip trailing newline - branch = results.stdout.decode()[0:-1] - - return (branch, '') diff --git a/xml/reg.py b/xml/reg.py deleted file mode 100644 index d78ecde89..000000000 --- a/xml/reg.py +++ /dev/null @@ -1,1397 +0,0 @@ -#!/usr/bin/python3 -i -# -# Copyright 2013-2023 The Khronos Group Inc. -# -# SPDX-License-Identifier: Apache-2.0 - -"""Types and classes for manipulating an API registry.""" - -import copy -import re -import sys -import xml.etree.ElementTree as etree -from collections import defaultdict, namedtuple -from generator import OutputGenerator, GeneratorOptions, write -import pdb - -def apiNameMatch(str, supported): - """Return whether a required api name matches a pattern specified for an - XML 'api' attribute or 'supported' attribute. - - - str - api name such as 'vulkan' or 'openxr' - - supported - comma-separated list of XML API names""" - - return (str is not None and str in supported.split(',')) - - -def matchAPIProfile(api, profile, elem): - """Return whether an API and profile - being generated matches an element's profile - - - api - string naming the API to match - - profile - string naming the profile to match - - elem - Element which (may) have 'api' and 'profile' - attributes to match to. - - If a tag is not present in the Element, the corresponding API - or profile always matches. - - Otherwise, the tag must exactly match the API or profile. - - Thus, if 'profile' = core: - - - `` with no attribute will match - - `` will match - - `` will not match - - Possible match conditions: - - ``` - Requested Element - Profile Profile - --------- -------- - None None Always matches - 'string' None Always matches - None 'string' Does not match. Can't generate multiple APIs - or profiles, so if an API/profile constraint - is present, it must be asked for explicitly. - 'string' 'string' Strings must match - ``` - - ** In the future, we will allow regexes for the attributes, - not just strings, so that `api="^(gl|gles2)"` will match. Even - this isn't really quite enough, we might prefer something - like `"gl(core)|gles1(common-lite)"`.""" - # Match 'api', if present - elem_api = elem.get('api') - if elem_api: - if api is None: - raise UserWarning("No API requested, but 'api' attribute is present with value '" - + elem_api + "'") - elif api != elem_api: - # Requested API doesn't match attribute - return False - elem_profile = elem.get('profile') - if elem_profile: - if profile is None: - raise UserWarning("No profile requested, but 'profile' attribute is present with value '" - + elem_profile + "'") - elif profile != elem_profile: - # Requested profile doesn't match attribute - return False - return True - - -class BaseInfo: - """Base class for information about a registry feature - (type/group/enum/command/API/extension). - - Represents the state of a registry feature, used during API generation. - """ - - def __init__(self, elem): - self.required = False - """should this feature be defined during header generation - (has it been removed by a profile or version)?""" - - self.declared = False - "has this feature been defined already?" - - self.elem = elem - "etree Element for this feature" - - def resetState(self): - """Reset required/declared to initial values. Used - prior to generating a new API interface.""" - self.required = False - self.declared = False - - def compareKeys(self, info, key, required = False): - """Return True if self.elem and info.elem have the same attribute - value for key. - If 'required' is not True, also returns True if neither element - has an attribute value for key.""" - - if required and key not in self.elem.keys(): - return False - return self.elem.get(key) == info.elem.get(key) - - def compareElem(self, info, infoName): - """Return True if self.elem and info.elem have the same definition. - info - the other object - infoName - 'type' / 'group' / 'enum' / 'command' / 'feature' / - 'extension'""" - - if infoName == 'enum': - if self.compareKeys(info, 'extends'): - # Either both extend the same type, or no type - if (self.compareKeys(info, 'value', required = True) or - self.compareKeys(info, 'bitpos', required = True)): - # If both specify the same value or bit position, - # they're equal - return True - elif (self.compareKeys(info, 'extnumber') and - self.compareKeys(info, 'offset') and - self.compareKeys(info, 'dir')): - # If both specify the same relative offset, they're equal - return True - elif (self.compareKeys(info, 'alias')): - # If both are aliases of the same value - return True - else: - return False - else: - # The same enum can't extend two different types - return False - else: - # Non-s should never be redefined - return False - - -class TypeInfo(BaseInfo): - """Registry information about a type. No additional state - beyond BaseInfo is required.""" - - def __init__(self, elem): - BaseInfo.__init__(self, elem) - self.additionalValidity = [] - self.removedValidity = [] - - def getMembers(self): - """Get a collection of all member elements for this type, if any.""" - return self.elem.findall('member') - - def resetState(self): - BaseInfo.resetState(self) - self.additionalValidity = [] - self.removedValidity = [] - - -class GroupInfo(BaseInfo): - """Registry information about a group of related enums - in an block, generally corresponding to a C "enum" type.""" - - def __init__(self, elem): - BaseInfo.__init__(self, elem) - - -class EnumInfo(BaseInfo): - """Registry information about an enum""" - - def __init__(self, elem): - BaseInfo.__init__(self, elem) - self.type = elem.get('type') - """numeric type of the value of the tag - ( '' for GLint, 'u' for GLuint, 'ull' for GLuint64 )""" - if self.type is None: - self.type = '' - - -class CmdInfo(BaseInfo): - """Registry information about a command""" - - def __init__(self, elem): - BaseInfo.__init__(self, elem) - self.additionalValidity = [] - self.removedValidity = [] - - def getParams(self): - """Get a collection of all param elements for this command, if any.""" - return self.elem.findall('param') - - def resetState(self): - BaseInfo.resetState(self) - self.additionalValidity = [] - self.removedValidity = [] - - -class FeatureInfo(BaseInfo): - """Registry information about an API - or .""" - - def __init__(self, elem): - BaseInfo.__init__(self, elem) - self.name = elem.get('name') - "feature name string (e.g. 'VK_KHR_surface')" - - self.emit = False - "has this feature been defined already?" - - self.sortorder = int(elem.get('sortorder', 0)) - """explicit numeric sort key within feature and extension groups. - Defaults to 0.""" - - # Determine element category (vendor). Only works - # for elements. - if elem.tag == 'feature': - # Element category (vendor) is meaningless for - self.category = 'VERSION' - """category, e.g. VERSION or khr/vendor tag""" - - self.version = elem.get('name') - """feature name string""" - - self.versionNumber = elem.get('number') - """versionNumber - API version number, taken from the 'number' - attribute of . Extensions do not have API version - numbers and are assigned number 0.""" - - self.number = "0" - self.supported = None - else: - # Extract vendor portion of __ - self.category = self.name.split('_', 2)[1] - self.version = "0" - self.versionNumber = "0" - self.number = elem.get('number') - """extension number, used for ordering and for assigning - enumerant offsets. features do not have extension - numbers and are assigned number 0.""" - - # If there's no 'number' attribute, use 0, so sorting works - if self.number is None: - self.number = 0 - self.supported = elem.get('supported') - -class SpirvInfo(BaseInfo): - """Registry information about an API - or .""" - - def __init__(self, elem): - BaseInfo.__init__(self, elem) - -class Registry: - """Object representing an API registry, loaded from an XML file.""" - - def __init__(self, gen=None, genOpts=None): - if gen is None: - # If not specified, give a default object so messaging will work - self.gen = OutputGenerator() - else: - self.gen = gen - "Output generator used to write headers / messages" - - if genOpts is None: - self.genOpts = GeneratorOptions() - else: - self.genOpts = genOpts - "Options controlling features to write and how to format them" - - self.gen.registry = self - self.gen.genOpts = self.genOpts - self.gen.genOpts.registry = self - - self.tree = None - "ElementTree containing the root ``" - - self.typedict = {} - "dictionary of TypeInfo objects keyed by type name" - - self.groupdict = {} - "dictionary of GroupInfo objects keyed by group name" - - self.enumdict = {} - "dictionary of EnumInfo objects keyed by enum name" - - self.cmddict = {} - "dictionary of CmdInfo objects keyed by command name" - - self.apidict = {} - "dictionary of FeatureInfo objects for `` elements keyed by API name" - - self.extensions = [] - "list of `` Elements" - - self.extdict = {} - "dictionary of FeatureInfo objects for `` elements keyed by extension name" - - self.spirvextdict = {} - "dictionary of FeatureInfo objects for `` elements keyed by spirv extension name" - - self.spirvcapdict = {} - "dictionary of FeatureInfo objects for `` elements keyed by spirv capability name" - - self.emitFeatures = False - """True to actually emit features for a version / extension, - or False to just treat them as emitted""" - - self.breakPat = None - "regexp pattern to break on when generating names" - # self.breakPat = re.compile('VkFenceImportFlagBits.*') - - self.requiredextensions = [] # Hack - can remove it after validity generator goes away - - # ** Global types for automatic source generation ** - # Length Member data - self.commandextensiontuple = namedtuple('commandextensiontuple', - ['command', # The name of the command being modified - 'value', # The value to append to the command - 'extension']) # The name of the extension that added it - self.validextensionstructs = defaultdict(list) - self.commandextensionsuccesses = [] - self.commandextensionerrors = [] - - self.filename = None - - def loadElementTree(self, tree): - """Load ElementTree into a Registry object and parse it.""" - self.tree = tree - self.parseTree() - - def loadFile(self, file): - """Load an API registry XML file into a Registry object and parse it""" - self.filename = file - self.tree = etree.parse(file) - self.parseTree() - - def setGenerator(self, gen): - """Specify output generator object. - - `None` restores the default generator.""" - self.gen = gen - self.gen.setRegistry(self) - - def addElementInfo(self, elem, info, infoName, dictionary): - """Add information about an element to the corresponding dictionary. - - Intended for internal use only. - - - elem - ``/``/``/``/``/``/``/`` Element - - info - corresponding {Type|Group|Enum|Cmd|Feature|Spirv}Info object - - infoName - 'type' / 'group' / 'enum' / 'command' / 'feature' / 'extension' / 'spirvextension' / 'spirvcapability' - - dictionary - self.{type|group|enum|cmd|api|ext|spirvext|spirvcap}dict - - If the Element has an 'api' attribute, the dictionary key is the - tuple (name,api). If not, the key is the name. 'name' is an - attribute of the Element""" - # self.gen.logMsg('diag', 'Adding ElementInfo.required =', - # info.required, 'name =', elem.get('name')) - api = elem.get('api') - if api: - key = (elem.get('name'), api) - else: - key = elem.get('name') - if key in dictionary: - if not dictionary[key].compareElem(info, infoName): - self.gen.logMsg('warn', 'Attempt to redefine', key, - '(this should not happen)') - else: - True - else: - dictionary[key] = info - - def lookupElementInfo(self, fname, dictionary): - """Find a {Type|Enum|Cmd}Info object by name. - - Intended for internal use only. - - If an object qualified by API name exists, use that. - - - fname - name of type / enum / command - - dictionary - self.{type|enum|cmd}dict""" - key = (fname, self.genOpts.apiname) - if key in dictionary: - # self.gen.logMsg('diag', 'Found API-specific element for feature', fname) - return dictionary[key] - if fname in dictionary: - # self.gen.logMsg('diag', 'Found generic element for feature', fname) - return dictionary[fname] - - return None - - def breakOnName(self, regexp): - """Specify a feature name regexp to break on when generating features.""" - self.breakPat = re.compile(regexp) - - def parseTree(self): - """Parse the registry Element, once created""" - # This must be the Element for the root - self.reg = self.tree.getroot() - - # Create dictionary of registry types from toplevel tags - # and add 'name' attribute to each tag (where missing) - # based on its element. - # - # There's usually one block; more are OK - # Required attributes: 'name' or nested tag contents - self.typedict = {} - for type_elem in self.reg.findall('types/type'): - # If the doesn't already have a 'name' attribute, set - # it from contents of its tag. - if type_elem.get('name') is None: - type_elem.set('name', type_elem.find('name').text) - self.addElementInfo(type_elem, TypeInfo(type_elem), 'type', self.typedict) - - # Create dictionary of registry enum groups from tags. - # - # Required attributes: 'name'. If no name is given, one is - # generated, but that group can't be identified and turned into an - # enum type definition - it's just a container for tags. - self.groupdict = {} - for group in self.reg.findall('enums'): - self.addElementInfo(group, GroupInfo(group), 'group', self.groupdict) - - # Create dictionary of registry enums from tags - # - # tags usually define different namespaces for the values - # defined in those tags, but the actual names all share the - # same dictionary. - # Required attributes: 'name', 'value' - # For containing which have type="enum" or type="bitmask", - # tag all contained s are required. This is a stopgap until - # a better scheme for tagging core and extension enums is created. - self.enumdict = {} - for enums in self.reg.findall('enums'): - required = (enums.get('type') is not None) - for enum in enums.findall('enum'): - enumInfo = EnumInfo(enum) - enumInfo.required = required - self.addElementInfo(enum, enumInfo, 'enum', self.enumdict) - - # Create dictionary of registry commands from tags - # and add 'name' attribute to each tag (where missing) - # based on its element. - # - # There's usually only one block; more are OK. - # Required attributes: 'name' or tag contents - self.cmddict = {} - # List of commands which alias others. Contains - # [ aliasName, element ] - # for each alias - cmdAlias = [] - for cmd in self.reg.findall('commands/command'): - # If the doesn't already have a 'name' attribute, set - # it from contents of its tag. - name = cmd.get('name') - if name is None: - name = cmd.set('name', cmd.find('proto/name').text) - ci = CmdInfo(cmd) - self.addElementInfo(cmd, ci, 'command', self.cmddict) - alias = cmd.get('alias') - if alias: - cmdAlias.append([name, alias, cmd]) - - # Now loop over aliases, injecting a copy of the aliased command's - # Element with the aliased prototype name replaced with the command - # name - if it exists. - for (name, alias, cmd) in cmdAlias: - if alias in self.cmddict: - aliasInfo = self.cmddict[alias] - cmdElem = copy.deepcopy(aliasInfo.elem) - cmdElem.find('proto/name').text = name - cmdElem.set('name', name) - cmdElem.set('alias', alias) - ci = CmdInfo(cmdElem) - # Replace the dictionary entry for the CmdInfo element - self.cmddict[name] = ci - - # @ newString = etree.tostring(base, encoding="unicode").replace(aliasValue, aliasName) - # @elem.append(etree.fromstring(replacement)) - else: - self.gen.logMsg('warn', 'No matching found for command', - cmd.get('name'), 'alias', alias) - - # Create dictionaries of API and extension interfaces - # from toplevel and tags. - self.apidict = {} - for feature in self.reg.findall('feature'): - featureInfo = FeatureInfo(feature) - self.addElementInfo(feature, featureInfo, 'feature', self.apidict) - - # Add additional enums defined only in tags - # to the corresponding enumerated type. - # When seen here, the element, processed to contain the - # numeric enum value, is added to the corresponding - # element, as well as adding to the enum dictionary. It is no - # longer removed from the element it is introduced in. - # Instead, generateRequiredInterface ignores elements - # that extend enumerated types. - # - # For tags which are actually just constants, if there's - # no 'extends' tag but there is a 'value' or 'bitpos' tag, just - # add an EnumInfo record to the dictionary. That works because - # output generation of constants is purely dependency-based, and - # doesn't need to iterate through the XML tags. - for elem in feature.findall('require'): - for enum in elem.findall('enum'): - addEnumInfo = False - groupName = enum.get('extends') - if groupName is not None: - # self.gen.logMsg('diag', 'Found extension enum', - # enum.get('name')) - # Add version number attribute to the element - enum.set('version', featureInfo.version) - # Look up the GroupInfo with matching groupName - if groupName in self.groupdict: - # self.gen.logMsg('diag', 'Matching group', - # groupName, 'found, adding element...') - gi = self.groupdict[groupName] - gi.elem.append(copy.deepcopy(enum)) - else: - self.gen.logMsg('warn', 'NO matching group', - groupName, 'for enum', enum.get('name'), 'found.') - addEnumInfo = True - elif enum.get('value') or enum.get('bitpos') or enum.get('alias'): - # self.gen.logMsg('diag', 'Adding extension constant "enum"', - # enum.get('name')) - addEnumInfo = True - if addEnumInfo: - enumInfo = EnumInfo(enum) - self.addElementInfo(enum, enumInfo, 'enum', self.enumdict) - - self.extensions = self.reg.findall('extensions/extension') - self.extdict = {} - for feature in self.extensions: - featureInfo = FeatureInfo(feature) - self.addElementInfo(feature, featureInfo, 'extension', self.extdict) - - # Add additional enums defined only in tags - # to the corresponding core type. - # Algorithm matches that of enums in a "feature" tag as above. - # - # This code also adds a 'extnumber' attribute containing the - # extension number, used for enumerant value calculation. - for elem in feature.findall('require'): - for enum in elem.findall('enum'): - addEnumInfo = False - groupName = enum.get('extends') - if groupName is not None: - # self.gen.logMsg('diag', 'Found extension enum', - # enum.get('name')) - - # Add block's extension number attribute to - # the element unless specified explicitly, such - # as when redefining an enum in another extension. - extnumber = enum.get('extnumber') - if not extnumber: - enum.set('extnumber', featureInfo.number) - - enum.set('extname', featureInfo.name) - enum.set('supported', featureInfo.supported) - # Look up the GroupInfo with matching groupName - if groupName in self.groupdict: - # self.gen.logMsg('diag', 'Matching group', - # groupName, 'found, adding element...') - gi = self.groupdict[groupName] - gi.elem.append(copy.deepcopy(enum)) - else: - self.gen.logMsg('warn', 'NO matching group', - groupName, 'for enum', enum.get('name'), 'found.') - addEnumInfo = True - elif enum.get('value') or enum.get('bitpos') or enum.get('alias'): - # self.gen.logMsg('diag', 'Adding extension constant "enum"', - # enum.get('name')) - addEnumInfo = True - if addEnumInfo: - enumInfo = EnumInfo(enum) - self.addElementInfo(enum, enumInfo, 'enum', self.enumdict) - - # Construct a "validextensionstructs" list for parent structures - # based on "structextends" tags in child structures - disabled_types = [] - for disabled_ext in self.reg.findall('extensions/extension[@supported="disabled"]'): - for type_elem in disabled_ext.findall("*/type"): - disabled_types.append(type_elem.get('name')) - for type_elem in self.reg.findall('types/type'): - if type_elem.get('name') not in disabled_types: - parentStructs = type_elem.get('structextends') - if parentStructs is not None: - for parent in parentStructs.split(','): - # self.gen.logMsg('diag', type.get('name'), 'extends', parent) - self.validextensionstructs[parent].append(type_elem.get('name')) - # Sort the lists so they don't depend on the XML order - for parent in self.validextensionstructs: - self.validextensionstructs[parent].sort() - - # Parse out all spirv tags in dictionaries - # Use addElementInfo to catch duplicates - for spirv in self.reg.findall('spirvextensions/spirvextension'): - spirvInfo = SpirvInfo(spirv) - self.addElementInfo(spirv, spirvInfo, 'spirvextension', self.spirvextdict) - for spirv in self.reg.findall('spirvcapabilities/spirvcapability'): - spirvInfo = SpirvInfo(spirv) - self.addElementInfo(spirv, spirvInfo, 'spirvcapability', self.spirvcapdict) - - def dumpReg(self, maxlen=120, filehandle=sys.stdout): - """Dump all the dictionaries constructed from the Registry object. - - Diagnostic to dump the dictionaries to specified file handle (default stdout). - Truncates type / enum / command elements to maxlen characters (default 120)""" - write('***************************************', file=filehandle) - write(' ** Dumping Registry contents **', file=filehandle) - write('***************************************', file=filehandle) - write('// Types', file=filehandle) - for name in self.typedict: - tobj = self.typedict[name] - write(' Type', name, '->', etree.tostring(tobj.elem)[0:maxlen], file=filehandle) - write('// Groups', file=filehandle) - for name in self.groupdict: - gobj = self.groupdict[name] - write(' Group', name, '->', etree.tostring(gobj.elem)[0:maxlen], file=filehandle) - write('// Enums', file=filehandle) - for name in self.enumdict: - eobj = self.enumdict[name] - write(' Enum', name, '->', etree.tostring(eobj.elem)[0:maxlen], file=filehandle) - write('// Commands', file=filehandle) - for name in self.cmddict: - cobj = self.cmddict[name] - write(' Command', name, '->', etree.tostring(cobj.elem)[0:maxlen], file=filehandle) - write('// APIs', file=filehandle) - for key in self.apidict: - write(' API Version ', key, '->', - etree.tostring(self.apidict[key].elem)[0:maxlen], file=filehandle) - write('// Extensions', file=filehandle) - for key in self.extdict: - write(' Extension', key, '->', - etree.tostring(self.extdict[key].elem)[0:maxlen], file=filehandle) - write('// SPIR-V', file=filehandle) - for key in self.spirvextdict: - write(' SPIR-V Extension', key, '->', - etree.tostring(self.spirvextdict[key].elem)[0:maxlen], file=filehandle) - for key in self.spirvcapdict: - write(' SPIR-V Capability', key, '->', - etree.tostring(self.spirvcapdict[key].elem)[0:maxlen], file=filehandle) - - def markTypeRequired(self, typename, required): - """Require (along with its dependencies) or remove (but not its dependencies) a type. - - - typename - name of type - - required - boolean (to tag features as required or not) - """ - self.gen.logMsg('diag', 'tagging type:', typename, '-> required =', required) - # Get TypeInfo object for tag corresponding to typename - typeinfo = self.lookupElementInfo(typename, self.typedict) - if typeinfo is not None: - if required: - # Tag type dependencies in 'alias' and 'required' attributes as - # required. This does not un-tag dependencies in a - # tag. See comments in markRequired() below for the reason. - for attrib_name in ['requires', 'alias']: - depname = typeinfo.elem.get(attrib_name) - if depname: - self.gen.logMsg('diag', 'Generating dependent type', - depname, 'for', attrib_name, 'type', typename) - # Don't recurse on self-referential structures. - if typename != depname: - self.markTypeRequired(depname, required) - else: - self.gen.logMsg('diag', 'type', typename, 'is self-referential') - # Tag types used in defining this type (e.g. in nested - # tags) - # Look for in entire tree, - # not just immediate children - for subtype in typeinfo.elem.findall('.//type'): - self.gen.logMsg('diag', 'markRequired: type requires dependent ', subtype.text) - if typename != subtype.text: - self.markTypeRequired(subtype.text, required) - else: - self.gen.logMsg('diag', 'type', typename, 'is self-referential') - # Tag enums used in defining this type, for example in - # member[MEMBER_SIZE] - for subenum in typeinfo.elem.findall('.//enum'): - self.gen.logMsg('diag', 'markRequired: type requires dependent ', subenum.text) - self.markEnumRequired(subenum.text, required) - # Tag type dependency in 'bitvalues' attributes as - # required. This ensures that the bit values for a flag - # are emitted - depType = typeinfo.elem.get('bitvalues') - if depType: - self.gen.logMsg('diag', 'Generating bitflag type', - depType, 'for type', typename) - self.markTypeRequired(depType, required) - group = self.lookupElementInfo(depType, self.groupdict) - if group is not None: - group.flagType = typeinfo - - typeinfo.required = required - elif '.h' not in typename: - self.gen.logMsg('warn', 'type:', typename, 'IS NOT DEFINED') - - def markEnumRequired(self, enumname, required): - """Mark an enum as required or not. - - - enumname - name of enum - - required - boolean (to tag features as required or not)""" - - self.gen.logMsg('diag', 'tagging enum:', enumname, '-> required =', required) - enum = self.lookupElementInfo(enumname, self.enumdict) - if enum is not None: - # If the enum is part of a group, and is being removed, then - # look it up in that tag and remove it there, so that it - # isn't visible to generators (which traverse the tag - # elements themselves). - # This isn't the most robust way of doing this, since a removed - # enum that's later required again will no longer have a group - # element, but it makes the change non-intrusive on generator - # code. - if required is False: - groupName = enum.elem.get('extends') - if groupName is not None: - # Look up the Info with matching groupName - if groupName in self.groupdict: - gi = self.groupdict[groupName] - gienum = gi.elem.find("enum[@name='" + enumname + "']") - if gienum is not None: - # Remove copy of this enum from the group - gi.elem.remove(gienum) - else: - self.gen.logMsg('warn', 'Cannot remove enum', - enumname, 'not found in group', - groupName) - else: - self.gen.logMsg('warn', 'Cannot remove enum', - enumname, 'from nonexistent group', - groupName) - - enum.required = required - # Tag enum dependencies in 'alias' attribute as required - depname = enum.elem.get('alias') - if depname: - self.gen.logMsg('diag', 'Generating dependent enum', - depname, 'for alias', enumname, 'required =', enum.required) - self.markEnumRequired(depname, required) - else: - self.gen.logMsg('warn', 'enum:', enumname, 'IS NOT DEFINED') - - def markCmdRequired(self, cmdname, required): - """Mark a command as required or not. - - - cmdname - name of command - - required - boolean (to tag features as required or not)""" - self.gen.logMsg('diag', 'tagging command:', cmdname, '-> required =', required) - cmd = self.lookupElementInfo(cmdname, self.cmddict) - if cmd is not None: - cmd.required = required - # Tag command dependencies in 'alias' attribute as required - depname = cmd.elem.get('alias') - if depname: - self.gen.logMsg('diag', 'Generating dependent command', - depname, 'for alias', cmdname) - self.markCmdRequired(depname, required) - # Tag all parameter types of this command as required. - # This DOES NOT remove types of commands in a - # tag, because many other commands may use the same type. - # We could be more clever and reference count types, - # instead of using a boolean. - if required: - # Look for in entire tree, - # not just immediate children - for type_elem in cmd.elem.findall('.//type'): - self.gen.logMsg('diag', 'markRequired: command implicitly requires dependent type', type_elem.text) - self.markTypeRequired(type_elem.text, required) - else: - self.gen.logMsg('warn', 'command:', cmdname, 'IS NOT DEFINED') - - def markRequired(self, featurename, feature, required): - """Require or remove features specified in the Element. - - - featurename - name of the feature - - feature - Element for `` or `` tag - - required - boolean (to tag features as required or not)""" - self.gen.logMsg('diag', 'markRequired (feature = , required =', required, ')') - - # Loop over types, enums, and commands in the tag - # @@ It would be possible to respect 'api' and 'profile' attributes - # in individual features, but that's not done yet. - for typeElem in feature.findall('type'): - self.markTypeRequired(typeElem.get('name'), required) - for enumElem in feature.findall('enum'): - self.markEnumRequired(enumElem.get('name'), required) - for cmdElem in feature.findall('command'): - self.markCmdRequired(cmdElem.get('name'), required) - - # Extensions may need to extend existing commands or other items in the future. - # So, look for extend tags. - for extendElem in feature.findall('extend'): - extendType = extendElem.get('type') - if extendType == 'command': - commandName = extendElem.get('name') - successExtends = extendElem.get('successcodes') - if successExtends is not None: - for success in successExtends.split(','): - self.commandextensionsuccesses.append(self.commandextensiontuple(command=commandName, - value=success, - extension=featurename)) - errorExtends = extendElem.get('errorcodes') - if errorExtends is not None: - for error in errorExtends.split(','): - self.commandextensionerrors.append(self.commandextensiontuple(command=commandName, - value=error, - extension=featurename)) - else: - self.gen.logMsg('warn', 'extend type:', extendType, 'IS NOT SUPPORTED') - - def getAlias(self, elem, dict): - """Check for an alias in the same require block. - - - elem - Element to check for an alias""" - - # Try to find an alias - alias = elem.get('alias') - if alias is None: - name = elem.get('name') - typeinfo = self.lookupElementInfo(name, dict) - alias = typeinfo.elem.get('alias') - - return alias - - def checkForCorrectionAliases(self, alias, require, tag): - """Check for an alias in the same require block. - - - alias - String name of the alias - - require - `` block from the registry - - tag - tag to look for in the require block""" - - if alias and require.findall(tag + "[@name='" + alias + "']"): - return True - - return False - - def fillFeatureDictionary(self, interface, featurename, api, profile): - """Capture added interfaces for a `` or ``. - - - interface - Element for `` or ``, containing - `` and `` tags - - featurename - name of the feature - - api - string specifying API name being generated - - profile - string specifying API profile being generated""" - - # Explicitly initialize known types - errors for unhandled categories - self.gen.featureDictionary[featurename] = { - "enumconstant": {}, - "command": {}, - "enum": {}, - "struct": {}, - "handle": {}, - "basetype": {}, - "include": {}, - "define": {}, - "bitmask": {}, - "union": {}, - "funcpointer": {}, - } - - # marks things that are required by this version/profile - for require in interface.findall('require'): - if matchAPIProfile(api, profile, require): - - # Determine the required extension or version needed for a require block - # Assumes that only one of these is specified - required_key = require.get('feature') - if required_key is None: - required_key = require.get('extension') - - # Loop over types, enums, and commands in the tag - for typeElem in require.findall('type'): - typename = typeElem.get('name') - typeinfo = self.lookupElementInfo(typename, self.typedict) - - if typeinfo: - # Remove aliases in the same extension/feature; these are always added as a correction. Don't need the original to be visible. - alias = self.getAlias(typeElem, self.typedict) - if not self.checkForCorrectionAliases(alias, require, 'type'): - # Resolve the type info to the actual type, so we get an accurate read for 'structextends' - while alias: - typeinfo = self.lookupElementInfo(alias, self.typedict) - alias = typeinfo.elem.get('alias') - - typecat = typeinfo.elem.get('category') - typeextends = typeinfo.elem.get('structextends') - if not required_key in self.gen.featureDictionary[featurename][typecat]: - self.gen.featureDictionary[featurename][typecat][required_key] = {} - if not typeextends in self.gen.featureDictionary[featurename][typecat][required_key]: - self.gen.featureDictionary[featurename][typecat][required_key][typeextends] = [] - self.gen.featureDictionary[featurename][typecat][required_key][typeextends].append(typename) - - for enumElem in require.findall('enum'): - enumname = enumElem.get('name') - typeinfo = self.lookupElementInfo(enumname, self.enumdict) - - # Remove aliases in the same extension/feature; these are always added as a correction. Don't need the original to be visible. - alias = self.getAlias(enumElem, self.enumdict) - if not self.checkForCorrectionAliases(alias, require, 'enum'): - enumextends = enumElem.get('extends') - if not required_key in self.gen.featureDictionary[featurename]['enumconstant']: - self.gen.featureDictionary[featurename]['enumconstant'][required_key] = {} - if not enumextends in self.gen.featureDictionary[featurename]['enumconstant'][required_key]: - self.gen.featureDictionary[featurename]['enumconstant'][required_key][enumextends] = [] - self.gen.featureDictionary[featurename]['enumconstant'][required_key][enumextends].append(enumname) - - for cmdElem in require.findall('command'): - - # Remove aliases in the same extension/feature; these are always added as a correction. Don't need the original to be visible. - alias = self.getAlias(cmdElem, self.cmddict) - if not self.checkForCorrectionAliases(alias, require, 'command'): - if not required_key in self.gen.featureDictionary[featurename]['command']: - self.gen.featureDictionary[featurename]['command'][required_key] = [] - self.gen.featureDictionary[featurename]['command'][required_key].append(cmdElem.get('name')) - - - def requireAndRemoveFeatures(self, interface, featurename, api, profile): - """Process `` and `` tags for a `` or ``. - - - interface - Element for `` or ``, containing - `` and `` tags - - featurename - name of the feature - - api - string specifying API name being generated - - profile - string specifying API profile being generated""" - # marks things that are required by this version/profile - for feature in interface.findall('require'): - if matchAPIProfile(api, profile, feature): - self.markRequired(featurename, feature, True) - # marks things that are removed by this version/profile - for feature in interface.findall('remove'): - if matchAPIProfile(api, profile, feature): - self.markRequired(featurename, feature, False) - - def assignAdditionalValidity(self, interface, api, profile): - # Loop over all usage inside all tags. - for feature in interface.findall('require'): - if matchAPIProfile(api, profile, feature): - for v in feature.findall('usage'): - if v.get('command'): - self.cmddict[v.get('command')].additionalValidity.append(copy.deepcopy(v)) - if v.get('struct'): - self.typedict[v.get('struct')].additionalValidity.append(copy.deepcopy(v)) - - # Loop over all usage inside all tags. - for feature in interface.findall('remove'): - if matchAPIProfile(api, profile, feature): - for v in feature.findall('usage'): - if v.get('command'): - self.cmddict[v.get('command')].removedValidity.append(copy.deepcopy(v)) - if v.get('struct'): - self.typedict[v.get('struct')].removedValidity.append(copy.deepcopy(v)) - - def generateFeature(self, fname, ftype, dictionary): - """Generate a single type / enum group / enum / command, - and all its dependencies as needed. - - - fname - name of feature (``/``/``) - - ftype - type of feature, 'type' | 'enum' | 'command' - - dictionary - of *Info objects - self.{type|enum|cmd}dict""" - - self.gen.logMsg('diag', 'generateFeature: generating', ftype, fname) - f = self.lookupElementInfo(fname, dictionary) - if f is None: - # No such feature. This is an error, but reported earlier - self.gen.logMsg('diag', 'No entry found for feature', fname, - 'returning!') - return - - # If feature isn't required, or has already been declared, return - if not f.required: - self.gen.logMsg('diag', 'Skipping', ftype, fname, '(not required)') - return - if f.declared: - self.gen.logMsg('diag', 'Skipping', ftype, fname, '(already declared)') - return - # Always mark feature declared, as though actually emitted - f.declared = True - - # Determine if this is an alias, and of what, if so - alias = f.elem.get('alias') - if alias: - self.gen.logMsg('diag', fname, 'is an alias of', alias) - - # Pull in dependent declaration(s) of the feature. - # For types, there may be one type in the 'requires' attribute of - # the element, one in the 'alias' attribute, and many in - # embedded and tags within the element. - # For commands, there may be many in tags within the element. - # For enums, no dependencies are allowed (though perhaps if you - # have a uint64 enum, it should require that type). - genProc = None - followupFeature = None - if ftype == 'type': - genProc = self.gen.genType - - # Generate type dependencies in 'alias' and 'requires' attributes - if alias: - self.generateFeature(alias, 'type', self.typedict) - requires = f.elem.get('requires') - if requires: - self.gen.logMsg('diag', 'Generating required dependent type', - requires) - self.generateFeature(requires, 'type', self.typedict) - - # Generate types used in defining this type (e.g. in nested - # tags) - # Look for in entire tree, - # not just immediate children - for subtype in f.elem.findall('.//type'): - self.gen.logMsg('diag', 'Generating required dependent ', - subtype.text) - self.generateFeature(subtype.text, 'type', self.typedict) - - # Generate enums used in defining this type, for example in - # member[MEMBER_SIZE] - for subtype in f.elem.findall('.//enum'): - self.gen.logMsg('diag', 'Generating required dependent ', - subtype.text) - self.generateFeature(subtype.text, 'enum', self.enumdict) - - # If the type is an enum group, look up the corresponding - # group in the group dictionary and generate that instead. - if f.elem.get('category') == 'enum': - self.gen.logMsg('diag', 'Type', fname, 'is an enum group, so generate that instead') - group = self.lookupElementInfo(fname, self.groupdict) - if alias is not None: - # An alias of another group name. - # Pass to genGroup with 'alias' parameter = aliased name - self.gen.logMsg('diag', 'Generating alias', fname, - 'for enumerated type', alias) - # Now, pass the *aliased* GroupInfo to the genGroup, but - # with an additional parameter which is the alias name. - genProc = self.gen.genGroup - f = self.lookupElementInfo(alias, self.groupdict) - elif group is None: - self.gen.logMsg('warn', 'Skipping enum type', fname, - ': No matching enumerant group') - return - else: - genProc = self.gen.genGroup - f = group - - # @ The enum group is not ready for generation. At this - # @ point, it contains all tags injected by - # @ tags without any verification of whether - # @ they're required or not. It may also contain - # @ duplicates injected by multiple consistent - # @ definitions of an . - - # @ Pass over each enum, marking its enumdict[] entry as - # @ required or not. Mark aliases of enums as required, - # @ too. - - enums = group.elem.findall('enum') - - self.gen.logMsg('diag', 'generateFeature: checking enums for group', fname) - - # Check for required enums, including aliases - # LATER - Check for, report, and remove duplicates? - enumAliases = [] - for elem in enums: - name = elem.get('name') - - required = False - - extname = elem.get('extname') - version = elem.get('version') - if extname is not None: - # 'supported' attribute was injected when the element was - # moved into the group in Registry.parseTree() - if self.genOpts.defaultExtensions == elem.get('supported'): - required = True - elif re.match(self.genOpts.addExtensions, extname) is not None: - required = True - elif version is not None: - required = re.match(self.genOpts.emitversions, version) is not None - else: - required = True - - self.gen.logMsg('diag', '* required =', required, 'for', name) - if required: - # Mark this element as required (in the element, not the EnumInfo) - elem.set('required', 'true') - # If it's an alias, track that for later use - enumAlias = elem.get('alias') - if enumAlias: - enumAliases.append(enumAlias) - for elem in enums: - name = elem.get('name') - if name in enumAliases: - elem.set('required', 'true') - self.gen.logMsg('diag', '* also need to require alias', name) - if f.elem.get('category') == 'bitmask': - followupFeature = f.elem.get('bitvalues') - elif ftype == 'command': - # Generate command dependencies in 'alias' attribute - if alias: - self.generateFeature(alias, 'command', self.cmddict) - - genProc = self.gen.genCmd - for type_elem in f.elem.findall('.//type'): - depname = type_elem.text - self.gen.logMsg('diag', 'Generating required parameter type', - depname) - self.generateFeature(depname, 'type', self.typedict) - elif ftype == 'enum': - # Generate enum dependencies in 'alias' attribute - if alias: - self.generateFeature(alias, 'enum', self.enumdict) - genProc = self.gen.genEnum - - # Actually generate the type only if emitting declarations - if self.emitFeatures: - self.gen.logMsg('diag', 'Emitting', ftype, 'decl for', fname) - genProc(f, fname, alias) - else: - self.gen.logMsg('diag', 'Skipping', ftype, fname, - '(should not be emitted)') - - if followupFeature: - self.gen.logMsg('diag', 'Generating required bitvalues ', - followupFeature) - self.generateFeature(followupFeature, "type", self.typedict) - - def generateRequiredInterface(self, interface): - """Generate all interfaces required by an API version or extension. - - - interface - Element for `` or ``""" - - # Loop over all features inside all tags. - for features in interface.findall('require'): - for t in features.findall('type'): - self.generateFeature(t.get('name'), 'type', self.typedict) - for e in features.findall('enum'): - # If this is an enum extending an enumerated type, don't - # generate it - this has already been done in reg.parseTree, - # by copying this element into the enumerated type. - enumextends = e.get('extends') - if not enumextends: - self.generateFeature(e.get('name'), 'enum', self.enumdict) - for c in features.findall('command'): - self.generateFeature(c.get('name'), 'command', self.cmddict) - - def generateSpirv(self, spirv, dictionary): - if spirv is None: - self.gen.logMsg('diag', 'No entry found for element', name, - 'returning!') - return - - name = spirv.elem.get('name') - # No known alias for spirv elements - alias = None - if spirv.emit: - genProc = self.gen.genSpirv - genProc(spirv, name, alias) - - def apiGen(self): - """Generate interface for specified versions using the current - generator and generator options""" - - self.gen.logMsg('diag', '*******************************************') - self.gen.logMsg('diag', ' Registry.apiGen file:', self.genOpts.filename, - 'api:', self.genOpts.apiname, - 'profile:', self.genOpts.profile) - self.gen.logMsg('diag', '*******************************************') - - # Reset required/declared flags for all features - self.apiReset() - - # Compile regexps used to select versions & extensions - regVersions = re.compile(self.genOpts.versions) - regEmitVersions = re.compile(self.genOpts.emitversions) - regAddExtensions = re.compile(self.genOpts.addExtensions) - regRemoveExtensions = re.compile(self.genOpts.removeExtensions) - regEmitExtensions = re.compile(self.genOpts.emitExtensions) - regEmitSpirv = re.compile(self.genOpts.emitSpirv) - - # Get all matching API feature names & add to list of FeatureInfo - # Note we used to select on feature version attributes, not names. - features = [] - apiMatch = False - for key in self.apidict: - fi = self.apidict[key] - api = fi.elem.get('api') - if apiNameMatch(self.genOpts.apiname, api): - apiMatch = True - if regVersions.match(fi.name): - # Matches API & version #s being generated. Mark for - # emission and add to the features[] list . - # @@ Could use 'declared' instead of 'emit'? - fi.emit = (regEmitVersions.match(fi.name) is not None) - features.append(fi) - if not fi.emit: - self.gen.logMsg('diag', 'NOT tagging feature api =', api, - 'name =', fi.name, 'version =', fi.version, - 'for emission (does not match emitversions pattern)') - else: - self.gen.logMsg('diag', 'Including feature api =', api, - 'name =', fi.name, 'version =', fi.version, - 'for emission (matches emitversions pattern)') - else: - self.gen.logMsg('diag', 'NOT including feature api =', api, - 'name =', fi.name, 'version =', fi.version, - '(does not match requested versions)') - else: - self.gen.logMsg('diag', 'NOT including feature api =', api, - 'name =', fi.name, - '(does not match requested API)') - if not apiMatch: - self.gen.logMsg('warn', 'No matching API versions found!') - - # Get all matching extensions, in order by their extension number, - # and add to the list of features. - # Start with extensions tagged with 'api' pattern matching the API - # being generated. Add extensions matching the pattern specified in - # regExtensions, then remove extensions matching the pattern - # specified in regRemoveExtensions - for (extName, ei) in sorted(self.extdict.items(), key=lambda x: x[1].number if x[1].number is not None else '0'): - extName = ei.name - include = False - - # Include extension if defaultExtensions is not None and is - # exactly matched by the 'supported' attribute. - if apiNameMatch(self.genOpts.defaultExtensions, - ei.elem.get('supported')): - self.gen.logMsg('diag', 'Including extension', - extName, "(defaultExtensions matches the 'supported' attribute)") - include = True - - # Include additional extensions if the extension name matches - # the regexp specified in the generator options. This allows - # forcing extensions into an interface even if they're not - # tagged appropriately in the registry. - # However we still respect the 'supported' attribute. - if regAddExtensions.match(extName) is not None: - if not apiNameMatch(self.genOpts.apiname, ei.elem.get('supported')): - self.gen.logMsg('diag', 'NOT including extension', - extName, '(matches explicitly requested, but does not match the \'supported\' attribute)') - include = False - else: - self.gen.logMsg('diag', 'Including extension', - extName, '(matches explicitly requested extensions to add)') - include = True - # Remove extensions if the name matches the regexp specified - # in generator options. This allows forcing removal of - # extensions from an interface even if they're tagged that - # way in the registry. - if regRemoveExtensions.match(extName) is not None: - self.gen.logMsg('diag', 'Removing extension', - extName, '(matches explicitly requested extensions to remove)') - include = False - - # If the extension is to be included, add it to the - # extension features list. - if include: - ei.emit = (regEmitExtensions.match(extName) is not None) - features.append(ei) - if not ei.emit: - self.gen.logMsg('diag', 'NOT tagging extension', - extName, - 'for emission (does not match emitextensions pattern)') - - # Hack - can be removed when validity generator goes away - # (Jon) I'm not sure what this does, or if it should respect - # the ei.emit flag above. - self.requiredextensions.append(extName) - else: - self.gen.logMsg('diag', 'NOT including extension', - extName, '(does not match api attribute or explicitly requested extensions)') - - # Add all spirv elements to list - # generators decide to emit them all or not - # Currently no filtering as no client of these elements needs filtering - spirvexts = [] - for key in self.spirvextdict: - si = self.spirvextdict[key] - si.emit = (regEmitSpirv.match(key) is not None) - spirvexts.append(si) - spirvcaps = [] - for key in self.spirvcapdict: - si = self.spirvcapdict[key] - si.emit = (regEmitSpirv.match(key) is not None) - spirvcaps.append(si) - - # Sort the features list, if a sort procedure is defined - if self.genOpts.sortProcedure: - self.genOpts.sortProcedure(features) - # print('sortProcedure ->', [f.name for f in features]) - - # Pass 1: loop over requested API versions and extensions tagging - # types/commands/features as required (in an block) or no - # longer required (in an block). It is possible to remove - # a feature in one version and restore it later by requiring it in - # a later version. - # If a profile other than 'None' is being generated, it must - # match the profile attribute (if any) of the and - # tags. - self.gen.logMsg('diag', 'PASS 1: TAG FEATURES') - for f in features: - self.gen.logMsg('diag', 'PASS 1: Tagging required and removed features for', - f.name) - self.fillFeatureDictionary(f.elem, f.name, self.genOpts.apiname, self.genOpts.profile) - self.requireAndRemoveFeatures(f.elem, f.name, self.genOpts.apiname, self.genOpts.profile) - self.assignAdditionalValidity(f.elem, self.genOpts.apiname, self.genOpts.profile) - - # Pass 2: loop over specified API versions and extensions printing - # declarations for required things which haven't already been - # generated. - self.gen.logMsg('diag', 'PASS 2: GENERATE INTERFACES FOR FEATURES') - self.gen.beginFile(self.genOpts) - for f in features: - self.gen.logMsg('diag', 'PASS 2: Generating interface for', - f.name) - emit = self.emitFeatures = f.emit - if not emit: - self.gen.logMsg('diag', 'PASS 2: NOT declaring feature', - f.elem.get('name'), 'because it is not tagged for emission') - # Generate the interface (or just tag its elements as having been - # emitted, if they haven't been). - self.gen.beginFeature(f.elem, emit) - self.generateRequiredInterface(f.elem) - self.gen.endFeature() - # Generate spirv elements - for s in spirvexts: - self.generateSpirv(s, self.spirvextdict) - for s in spirvcaps: - self.generateSpirv(s, self.spirvcapdict) - self.gen.endFile() - - def apiReset(self): - """Reset type/enum/command dictionaries before generating another API. - - Use between apiGen() calls to reset internal state.""" - for datatype in self.typedict: - self.typedict[datatype].resetState() - for enum in self.enumdict: - self.enumdict[enum].resetState() - for cmd in self.cmddict: - self.cmddict[cmd].resetState() - for cmd in self.apidict: - self.apidict[cmd].resetState() - - def __validateStructLimittypes(self, struct): - """Validate 'limittype' attributes for a single struct.""" - limittypeDiags = namedtuple('limittypeDiags', ['missing', 'invalid']) - badFields = defaultdict(lambda : limittypeDiags(missing=[], invalid=[])) - validLimittypes = { 'min', 'max', 'bitmask', 'range', 'struct', 'noauto' } - for member in struct.getMembers(): - memberName = member.findtext('name') - if memberName in ['sType', 'pNext']: - continue - limittype = member.get('limittype') - if not limittype: - badFields[struct.elem.get('name')].missing.append(memberName) - elif limittype == 'struct': - typeName = member.findtext('type') - memberType = self.typedict[typeName] - badFields.update(self.__validateStructLimittypes(memberType)) - elif limittype not in validLimittypes: - badFields[struct.elem.get('name')].invalid.append(memberName) - return badFields - - def __validateLimittype(self): - """Validate 'limittype' attributes.""" - self.gen.logMsg('diag', 'VALIDATING LIMITTYPE ATTRIBUTES') - badFields = self.__validateStructLimittypes(self.typedict['VkPhysicalDeviceProperties2']) - for featStructName in self.validextensionstructs['VkPhysicalDeviceProperties2']: - featStruct = self.typedict[featStructName] - badFields.update(self.__validateStructLimittypes(featStruct)) - - if badFields: - self.gen.logMsg('diag', 'SUMMARY OF FIELDS WITH INCORRECT LIMITTYPES') - for key in sorted(badFields.keys()): - diags = badFields[key] - if diags.missing: - self.gen.logMsg('diag', ' ', key, 'missing limittype:', ', '.join(badFields[key].missing)) - if diags.invalid: - self.gen.logMsg('diag', ' ', key, 'invalid limittype:', ', '.join(badFields[key].invalid)) - return False - return True - - def validateRegistry(self): - """Validate properties of the registry.""" - return self.__validateLimittype() From 3855094768a2dce03a5aeb1e36d841bee3607adc Mon Sep 17 00:00:00 2001 From: aharon-abramson Date: Mon, 13 Nov 2023 13:19:23 +0200 Subject: [PATCH 011/202] rename more occurances of "promise" to "assert" --- xml/cl.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xml/cl.xml b/xml/cl.xml index 1f4a450fd..d0982f205 100644 --- a/xml/cl.xml +++ b/xml/cl.xml @@ -252,7 +252,7 @@ server's OpenCL/api-docs repository. typedef cl_bitfield cl_device_fp_atomic_capabilities_ext; typedef cl_uint cl_image_requirements_info_ext; typedef cl_bitfield cl_platform_command_buffer_capabilities_khr; - typedef cl_bitfield cl_mutable_dispatch_promises_khr + typedef cl_bitfield cl_mutable_dispatch_asserts_khr Structure types @@ -1783,7 +1783,7 @@ server's OpenCL/api-docs repository. - + @@ -7282,7 +7282,7 @@ server's OpenCL/api-docs repository. - + From 50bfadb7fe7d7c474afdeee1d8a365a7b48b9596 Mon Sep 17 00:00:00 2001 From: aharon-abramson Date: Tue, 16 Jan 2024 11:27:09 +0200 Subject: [PATCH 012/202] Update cl_khr_command_buffer_mutable_dispatch.asciidoc --- ext/cl_khr_command_buffer_mutable_dispatch.asciidoc | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc index a8b234df7..9aafb3b3d 100644 --- a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc +++ b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc @@ -975,13 +975,3 @@ non-trivial deep copying of the underlying objects contained in the command-buffer. As a result of this new entry-point being an additive change to the specification it is omitted, and if its functionality has demand later, it may be a introduced as a stand alone extension. --- - -. Introduce a `CL_MUTABLE_DISPATCH_ADDITIONAL_WORK_GROUPS_KHR` capability to - allow the number of work-groups in kernel execution to be increased during - update. -+ --- -*Resolved*: Can be included in the final release of the extension if there is -implementation coverage. --- From 2eb8460598643a8eb07952682d9c71c8dfd65ace Mon Sep 17 00:00:00 2001 From: aharon-abramson Date: Tue, 6 Feb 2024 11:32:21 +0200 Subject: [PATCH 013/202] Update cl_khr_command_buffer_mutable_dispatch.asciidoc replace error with undefined behavior --- ext/cl_khr_command_buffer_mutable_dispatch.asciidoc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc index 9aafb3b3d..af9680bb3 100644 --- a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc +++ b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc @@ -337,6 +337,11 @@ description of property values. {CL_MUTABLE_DISPATCH_ASSERT_NO_ADDITIONAL_WORK_GROUPS_KHR} An assertion by the user that the number of work-groups of any ND-range kernel recorded in this command buffer will not be updated beyond the number defined when the ND-range kernel was recorded. + If the user's update to the values of _local_work_size_ and/or _global_work_size_ result in an increase + in the number of work-groups in the ND-range over the number specified when the ND-range kernel was + recorded, the behavior is undefined. + If the user updates _local_work_size_ to be _NULL_, the assertion is ignored. + |==== ==== Modifications to clCommandNDRangeKernelKHR @@ -561,10 +566,6 @@ the array violates the defined conditions: * {CL_INVALID_VALUE} if _type_ is not {CL_STRUCTURE_TYPE_MUTABLE_DISPATCH_CONFIG_KHR}. -* {CL_INVALID_OPERATION} if {CL_MUTABLE_DISPATCH_ASSERT_NO_ADDITIONAL_WORK_GROUPS_KHR} is specified, - and values of _local_work_size_ and/or _global_work_size_ result in an increase to the number of work- - groups in the ND-range over the number specified when the ND-range kernel was recorded. - * {CL_INVALID_OPERATION} if the values of _local_work_size_ and/or _global_work_size_ result in a change to work-group uniformity. From 39bdaeb09a263818ea9ae4a5d6c1880382df2f9a Mon Sep 17 00:00:00 2001 From: aharon-abramson Date: Wed, 7 Feb 2024 11:29:14 +0200 Subject: [PATCH 014/202] add CL_COMMAND_BUFFER_MUTABLE_DISPATCH_ASSERTS_KHR --- ...r_command_buffer_mutable_dispatch.asciidoc | 20 ++++++++++++++++--- xml/cl.xml | 2 +- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc index af9680bb3..0e8cf61ff 100644 --- a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc +++ b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc @@ -235,6 +235,9 @@ CL_INVALID_MUTABLE_COMMAND_KHR -1141 // Accepted values for the param_name parameter to clGetDeviceInfo CL_DEVICE_MUTABLE_DISPATCH_CAPABILITIES_KHR 0x12B0 +/* cl_command_buffer_properties_khr */ +#define CL_COMMAND_BUFFER_MUTABLE_DISPATCH_ASSERTS_KHR 0x12B7 + // Property to cl_ndrange_kernel_command_properties_khr CL_MUTABLE_DISPATCH_UPDATABLE_FIELDS_KHR 0x12B1 CL_MUTABLE_DISPATCH_ASSERTS_KHR 0x12B7 @@ -330,7 +333,18 @@ description of property values. command-buffer, by default command-buffers are immutable. If set, commands in the command-buffer may be updated via {clUpdateMutableCommandsKHR}. -| {CL_MUTABLE_DISPATCH_ASSERTS_KHR} +|==== + +Add a {CL_COMMAND_BUFFER_ASSERTS_KHR} property to the +<> table. + +[cols=",,",options="header",] +|==== +| *Recording Properties* +| *Property Value* +| *Description* + +| {CL_COMMAND_BUFFER_ASSERTS_KHR} | {cl_mutable_dispatch_asserts_khr_TYPE} | This is a bitfield and can be set to a combination of the following values: @@ -340,8 +354,8 @@ description of property values. If the user's update to the values of _local_work_size_ and/or _global_work_size_ result in an increase in the number of work-groups in the ND-range over the number specified when the ND-range kernel was recorded, the behavior is undefined. - If the user updates _local_work_size_ to be _NULL_, the assertion is ignored. - + If the user updates _local_work_size_ to be _NULL_, the behavior is undefined. + |==== ==== Modifications to clCommandNDRangeKernelKHR diff --git a/xml/cl.xml b/xml/cl.xml index d0982f205..35f2faedd 100644 --- a/xml/cl.xml +++ b/xml/cl.xml @@ -1783,7 +1783,7 @@ server's OpenCL/api-docs repository. - + From df6029f5e5d63d2edd177cc84652df61da036355 Mon Sep 17 00:00:00 2001 From: aharon-abramson Date: Wed, 7 Feb 2024 11:49:00 +0200 Subject: [PATCH 015/202] fixing mix-up in enum values --- ext/cl_khr_command_buffer_mutable_dispatch.asciidoc | 2 +- xml/cl.xml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc index 0e8cf61ff..2b7949f35 100644 --- a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc +++ b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc @@ -240,7 +240,7 @@ CL_DEVICE_MUTABLE_DISPATCH_CAPABILITIES_KHR 0x12B0 // Property to cl_ndrange_kernel_command_properties_khr CL_MUTABLE_DISPATCH_UPDATABLE_FIELDS_KHR 0x12B1 -CL_MUTABLE_DISPATCH_ASSERTS_KHR 0x12B7 +CL_MUTABLE_DISPATCH_ASSERTS_KHR 0x12B8 // Bits for cl_mutable_dispatch_fields_khr bitfield CL_MUTABLE_DISPATCH_GLOBAL_OFFSET_KHR (0x1 << 0) diff --git a/xml/cl.xml b/xml/cl.xml index 35f2faedd..4912779d7 100644 --- a/xml/cl.xml +++ b/xml/cl.xml @@ -1784,7 +1784,8 @@ server's OpenCL/api-docs repository. - + + From 65b173edb337317e1c1200f48c64ad952ccc386a Mon Sep 17 00:00:00 2001 From: aharon-abramson Date: Thu, 8 Feb 2024 13:43:27 +0200 Subject: [PATCH 016/202] return errors in non-update APIs --- ...r_command_buffer_mutable_dispatch.asciidoc | 22 ++++++++++++++++--- xml/cl.xml | 1 + 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc index 2b7949f35..8ba601ba3 100644 --- a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc +++ b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc @@ -236,7 +236,7 @@ CL_INVALID_MUTABLE_COMMAND_KHR -1141 CL_DEVICE_MUTABLE_DISPATCH_CAPABILITIES_KHR 0x12B0 /* cl_command_buffer_properties_khr */ -#define CL_COMMAND_BUFFER_MUTABLE_DISPATCH_ASSERTS_KHR 0x12B7 +CL_COMMAND_BUFFER_MUTABLE_DISPATCH_ASSERTS_KHR 0x12B7 // Property to cl_ndrange_kernel_command_properties_khr CL_MUTABLE_DISPATCH_UPDATABLE_FIELDS_KHR 0x12B1 @@ -354,10 +354,14 @@ Add a {CL_COMMAND_BUFFER_ASSERTS_KHR} property to the If the user's update to the values of _local_work_size_ and/or _global_work_size_ result in an increase in the number of work-groups in the ND-range over the number specified when the ND-range kernel was recorded, the behavior is undefined. - If the user updates _local_work_size_ to be _NULL_, the behavior is undefined. |==== +===== Additional Errors + +* {CL_INVALID_VALUE} if _properties_ has a {CL_COMMAND_BUFFER_ASSERTS_KHR} property with + {CL_MUTABLE_DISPATCH_ASSERT_NO_ADDITIONAL_WORK_GROUPS_KHR}, but _local_work_size_ is `NULL`. + ==== Modifications to clCommandNDRangeKernelKHR ===== Properties Parameter @@ -438,7 +442,6 @@ in the table below. defined as the product for each _i_ from _0_ to _work_dim - 1_ of _ceil((global_work_size[i] - global_work_offset[i])/local_work_size[i])_ (if _global_work_offset_ is NULL, _global_work_offset[i]_ should be replaced with _0_). - In case _local_work_size_ is NULL, the effect of this flag is undefined. |==== ===== Mutable Handle Parameter @@ -464,6 +467,11 @@ Is replaced with {CL_DEVICE_MUTABLE_DISPATCH_CAPABILITIES_KHR} for the device associated with _command_queue_. If _command_queue_ is `NULL`, the device associated with _command_buffer_ must report support for these properties. + +The following error condition is added: + +* {CL_INVALID_VALUE} if _properties_ has a {CL_MUTABLE_DISPATCH_UPDATABLE_FIELDS_KHR} property with + {CL_MUTABLE_DISPATCH_ASSERT_NO_ADDITIONAL_WORK_GROUPS_KHR}, but _local_work_size_ is `NULL`. [[mutable-commands]] ==== New Section in the OpenCL API specification 5.X.5 - Mutable Commands: @@ -528,6 +536,14 @@ state of all commands is known, rather than iteratively updating each command individually. ==== +[NOTE] +==== +if the command buffer has been created with {CL_MUTABLE_DISPATCH_ASSERT_NO_ADDITIONAL_WORK_GROUPS_KHR}, or +the updated ND-range command has been recorded with this flag, and the ND-range parameters are updated so +that the new number of work-groups exceeds the number when the ND-range command was recorded, the behavior +is undefined. +==== + _command_buffer_ Refers to a valid command-buffer object. _mutable_config_ Is a pointer to a diff --git a/xml/cl.xml b/xml/cl.xml index 4912779d7..817e4644c 100644 --- a/xml/cl.xml +++ b/xml/cl.xml @@ -7153,6 +7153,7 @@ server's OpenCL/api-docs repository.     + From 980a2dce3b4d0343a1ab9358c37b06a667c13194 Mon Sep 17 00:00:00 2001 From: Aharon Abramson Date: Sun, 11 Feb 2024 13:30:46 +0200 Subject: [PATCH 017/202] spaces Co-authored-by: Sun Serega --- xml/cl.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xml/cl.xml b/xml/cl.xml index 817e4644c..5797c97b5 100644 --- a/xml/cl.xml +++ b/xml/cl.xml @@ -1784,7 +1784,7 @@ server's OpenCL/api-docs repository. - +     From 23777a31772984c36508b46a9882156c8a562861 Mon Sep 17 00:00:00 2001 From: Aharon Abramson Date: Sun, 11 Feb 2024 13:31:18 +0200 Subject: [PATCH 018/202] Update ext/cl_khr_command_buffer_mutable_dispatch.asciidoc Co-authored-by: Ewan Crawford --- ext/cl_khr_command_buffer_mutable_dispatch.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc index 8ba601ba3..d1c76a7b9 100644 --- a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc +++ b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc @@ -538,7 +538,7 @@ individually. [NOTE] ==== -if the command buffer has been created with {CL_MUTABLE_DISPATCH_ASSERT_NO_ADDITIONAL_WORK_GROUPS_KHR}, or +If the command buffer has been created with {CL_MUTABLE_DISPATCH_ASSERT_NO_ADDITIONAL_WORK_GROUPS_KHR}, or the updated ND-range command has been recorded with this flag, and the ND-range parameters are updated so that the new number of work-groups exceeds the number when the ND-range command was recorded, the behavior is undefined. From df7f8490e1d0d73dd00d425fb0c9273649bc6bd3 Mon Sep 17 00:00:00 2001 From: aharon-abramson Date: Sun, 11 Feb 2024 13:37:13 +0200 Subject: [PATCH 019/202] update the formula for number of WGs --- ext/cl_khr_command_buffer_mutable_dispatch.asciidoc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc index d1c76a7b9..0de7cb03a 100644 --- a/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc +++ b/ext/cl_khr_command_buffer_mutable_dispatch.asciidoc @@ -440,8 +440,7 @@ in the table below. An assertion by the user that the number of work-groups of this ND-range kernel will not be updated beyond the number defined when the ND-range kernel was recorded. The number of work-groups is defined as the product for each _i_ from _0_ to _work_dim - 1_ of - _ceil((global_work_size[i] - global_work_offset[i])/local_work_size[i])_ (if - _global_work_offset_ is NULL, _global_work_offset[i]_ should be replaced with _0_). + _ceil(global_work_size[i]/local_work_size[i])_. |==== ===== Mutable Handle Parameter From e89bf7f8410948b05977712f375927dd4aef1621 Mon Sep 17 00:00:00 2001 From: Aharon Abramson Date: Mon, 12 Feb 2024 09:35:58 +0200 Subject: [PATCH 020/202] Update xml/cl.xml Co-authored-by: Sun Serega --- xml/cl.xml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/xml/cl.xml b/xml/cl.xml index 5797c97b5..0a36a34b6 100644 --- a/xml/cl.xml +++ b/xml/cl.xml @@ -7321,7 +7321,10 @@ server's OpenCL/api-docs repository. - + + + + From f6486595d2dfa04aa6f88f3da952eb330eaaf47f Mon Sep 17 00:00:00 2001 From: joshqti <127994991+joshqti@users.noreply.github.com> Date: Tue, 7 Nov 2023 07:22:16 -0800 Subject: [PATCH 021/202] Remove minor TODO comment (#957) (#988) Delete obsolete comment in cl_khr_semaphore. Issue --- ext/cl_khr_semaphore.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/ext/cl_khr_semaphore.asciidoc b/ext/cl_khr_semaphore.asciidoc index 751732420..e0aff5357 100644 --- a/ext/cl_khr_semaphore.asciidoc +++ b/ext/cl_khr_semaphore.asciidoc @@ -155,7 +155,6 @@ CL_SEMAPHORE_PROPERTIES_KHR 0x203B CL_SEMAPHORE_PAYLOAD_KHR 0x203C ---- -// TODO: We don't need an enum assigned for CL_SEMAPHORE_DEVICE_HANDLE_LIST_END_KHR and should just use 0. // TODO: Do we need to define CL_SEMAPHORE_DEVICE_HANDLE_LIST here or should it be in the external semaphore spec instead? New attributes that can be passed as part of {cl_semaphore_info_khr_TYPE} or {cl_semaphore_properties_khr_TYPE}: From e5d39eb1587e400dfddf11880db03468b3ed71bc Mon Sep 17 00:00:00 2001 From: Jon Leech <4693344+oddhack@users.noreply.github.com> Date: Tue, 7 Nov 2023 09:12:05 -0800 Subject: [PATCH 022/202] Use hexapdf instead of ghostscript for PDF optimization (#991) * Use hexapdf instead of ghostscript for PDF optimization Resulting PDFs tend to be considerably smaller, and also runs about 15% faster when doing a full PDF build (2:39 vs. 3:06 on my machine). The hexapdf tool does need to be installed in the build environment - it is in the khronosgroup/docker-images:asciidoctor-spec Docker image. * Add hexapdf to Travis environment. --- .travis.yml | 1 + Makefile | 77 +++++++++-------------------------------------------- README.adoc | 10 ++++--- 3 files changed, 20 insertions(+), 68 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0b0580d64..fc2b142cd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ before_install: - gem install coderay -v 1.1.1 - gem install rouge -v 3.19.0 - gem install ttfunk -v 1.5.1 + - gem install hexapdf -v 0.27.0 - gem install asciidoctor-pdf -v 1.5.0 - gem install asciidoctor-mathematical -v 0.3.5 diff --git a/Makefile b/Makefile index a6243f410..85d2ca239 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,6 @@ RM = rm -f RMRF = rm -rf MKDIR = mkdir -p CP = cp -GS_EXISTS := $(shell command -v gs 2> /dev/null) GITHEAD = ./.git/logs/HEAD # Target directories for output files @@ -163,6 +162,10 @@ icdinst: icdinsthtml icdinstpdf html: apihtml envhtml exthtml extensionshtml cxxhtml chtml icdinsthtml +# PDF optimizer - usage $(OPTIMIZEPDF) in.pdf out.pdf +# OPTIMIZEPDFOPTS=--compress-pages is slightly better, but much slower +OPTIMIZEPDF = hexapdf optimize $(OPTIMIZEPDFOPTS) + pdf: apipdf envpdf extpdf extensionspdf cxxpdf cpdf icdinstpdf # Spec targets. @@ -192,13 +195,7 @@ $(PDFDIR)/$(APISPEC).pdf: $(APISPECSRC) $(QUIET)$(MKDIR) $(PDFDIR) $(QUIET)$(MKDIR) $(PDFMATHDIR) $(QUIET)$(ASCIIDOCTOR) -b pdf $(ADOCOPTS) $(ADOCPDFOPTS) -o $@ $(APISPEC).txt -ifndef GS_EXISTS - $(QUIET) echo "Warning: Ghostscript not installed, skipping pdf optimization" -else - $(QUIET)$(CURDIR)/config/optimize-pdf $@ - $(QUIET)rm $@ - $(QUIET)mv $(PDFDIR)/$(APISPEC)-optimized.pdf $@ -endif + $(QUIET)$(OPTIMIZEPDF) $@ $@.out.pdf && mv $@.out.pdf $@ # Environment spec @@ -218,13 +215,7 @@ $(PDFDIR)/$(ENVSPEC).pdf: $(ENVSPECSRC) $(QUIET)$(MKDIR) $(PDFDIR) $(QUIET)$(MKDIR) $(PDFMATHDIR) $(QUIET)$(ASCIIDOCTOR) -b pdf $(ADOCOPTS) $(ADOCPDFOPTS) -o $@ $(ENVSPEC).txt -ifndef GS_EXISTS - $(QUIET) echo "Warning: Ghostscript not installed, skipping pdf optimization" -else - $(QUIET)$(CURDIR)/config/optimize-pdf $@ - $(QUIET)rm $@ - $(QUIET)mv $(PDFDIR)/$(ENVSPEC)-optimized.pdf $@ -endif + $(QUIET)$(OPTIMIZEPDF) $@ $@.out.pdf && mv $@.out.pdf $@ # Extensions spec EXTSPEC = OpenCL_Ext @@ -242,13 +233,7 @@ $(PDFDIR)/$(EXTSPEC).pdf: $(EXTSPECSRC) $(QUIET)$(MKDIR) $(PDFDIR) $(QUIET)$(MKDIR) $(PDFMATHDIR) $(QUIET)$(ASCIIDOCTOR) -b pdf $(ADOCOPTS) $(ADOCPDFOPTS) -o $@ $(EXTSPEC).txt -ifndef GS_EXISTS - $(QUIET) echo "Warning: Ghostscript not installed, skipping pdf optimization" -else - $(QUIET)$(CURDIR)/config/optimize-pdf $@ - $(QUIET)rm $@ - $(QUIET)mv $(PDFDIR)/$(EXTSPEC)-optimized.pdf $@ -endif + $(QUIET)$(OPTIMIZEPDF) $@ $@.out.pdf && mv $@.out.pdf $@ # Individual extensions spec(s) EXTDIR = extensions @@ -280,13 +265,7 @@ $(PDFDIR)/$(EXTENSIONSSPEC).pdf: $(EXTENSIONSSPECSRC) $(GENDEPENDS) $(QUIET)$(MKDIR) $(PDFDIR) $(QUIET)$(MKDIR) $(PDFMATHDIR) $(QUIET)$(ASCIIDOCTOR) -b pdf $(ADOCOPTS) $(ADOCPDFOPTS) -o $@ $(EXTDIR)/$(EXTENSIONSSPEC).txt -ifndef GS_EXISTS - $(QUIET) echo "Warning: Ghostscript not installed, skipping pdf optimization" -else - $(QUIET)$(CURDIR)/config/optimize-pdf $@ - $(QUIET)rm $@ - $(QUIET)mv $(PDFDIR)/$(EXTENSIONSSPEC)-optimized.pdf $@ -endif + $(QUIET)$(OPTIMIZEPDF) $@ $@.out.pdf && mv $@.out.pdf $@ # Language Extensions spec CEXTDOC = OpenCL_LangExt @@ -304,13 +283,7 @@ $(PDFDIR)/$(CEXTDOC).pdf: $(CEXTDOCSRC) $(QUIET)$(MKDIR) $(PDFDIR) $(QUIET)$(MKDIR) $(PDFMATHDIR) $(QUIET)$(ASCIIDOCTOR) -b pdf $(ADOCOPTS) $(ADOCPDFOPTS) -o $@ $(CEXTDOC).txt -ifndef GS_EXISTS - $(QUIET) echo "Warning: Ghostscript not installed, skipping pdf optimization" -else - $(QUIET)$(CURDIR)/config/optimize-pdf $@ - $(QUIET)rm $@ - $(QUIET)mv $(PDFDIR)/$(CEXTDOC)-optimized.pdf $@ -endif + $(QUIET)$(OPTIMIZEPDF) $@ $@.out.pdf && mv $@.out.pdf $@ # C++ (cxx) spec CXXSPEC = OpenCL_Cxx @@ -328,13 +301,7 @@ $(PDFDIR)/$(CXXSPEC).pdf: $(CXXSPECSRC) $(QUIET)$(MKDIR) $(PDFDIR) $(QUIET)$(MKDIR) $(PDFMATHDIR) $(QUIET)$(ASCIIDOCTOR) -b pdf $(ADOCOPTS) $(ADOCPDFOPTS) -o $@ $(CXXSPEC).txt -ifndef GS_EXISTS - $(QUIET) echo "Warning: Ghostscript not installed, skipping pdf optimization" -else - $(QUIET)$(CURDIR)/config/optimize-pdf $@ - $(QUIET)rm $@ - $(QUIET)mv $(PDFDIR)/$(CXXSPEC)-optimized.pdf $@ -endif + $(QUIET)$(OPTIMIZEPDF) $@ $@.out.pdf && mv $@.out.pdf $@ # C spec CSPEC = OpenCL_C @@ -352,13 +319,7 @@ $(PDFDIR)/$(CSPEC).pdf: $(CSPECSRC) $(QUIET)$(MKDIR) $(PDFDIR) $(QUIET)$(MKDIR) $(PDFMATHDIR) $(QUIET)$(ASCIIDOCTOR) -b pdf $(ADOCOPTS) $(ADOCPDFOPTS) -o $@ $(CSPEC).txt -ifndef GS_EXISTS - $(QUIET) echo "Warning: Ghostscript not installed, skipping pdf optimization" -else - $(QUIET)$(CURDIR)/config/optimize-pdf $@ - $(QUIET)rm $@ - $(QUIET)mv $(PDFDIR)/$(CSPEC)-optimized.pdf $@ -endif + $(QUIET)$(OPTIMIZEPDF) $@ $@.out.pdf && mv $@.out.pdf $@ # C++ for OpenCL doc CXX4OPENCLDOC = CXX_for_OpenCL @@ -376,13 +337,7 @@ $(PDFDIR)/$(CXX4OPENCLDOC).pdf: $(CXX4OPENCLDOCSRC) $(QUIET)$(MKDIR) $(PDFDIR) $(QUIET)$(MKDIR) $(PDFMATHDIR) $(QUIET)$(ASCIIDOCTOR) -b pdf $(CXX4OPENCL_ADOCOPTS) $(ADOCPDFOPTS) -o $@ $(CXX4OPENCLDOC).txt -ifndef GS_EXISTS - $(QUIET) echo "Warning: Ghostscript not installed, skipping pdf optimization" -else - $(QUIET)$(CURDIR)/config/optimize-pdf $@ - $(QUIET)rm $@ - $(QUIET)mv $(PDFDIR)/$(CXX4OPENCLDOC)-optimized.pdf $@ -endif + $(QUIET)$(OPTIMIZEPDF) $@ $@.out.pdf && mv $@.out.pdf $@ # ICD installation guidelines ICDINSTSPEC = OpenCL_ICD_Installation @@ -400,13 +355,7 @@ $(PDFDIR)/$(ICDINSTSPEC).pdf: $(ICDINSTSPECSRC) $(QUIET)$(MKDIR) $(PDFDIR) $(QUIET)$(MKDIR) $(PDFMATHDIR) $(QUIET)$(ASCIIDOCTOR) -b pdf $(ADOCOPTS) $(ADOCPDFOPTS) -o $@ $(ICDINSTSPEC).txt -ifndef GS_EXISTS - $(QUIET) echo "Warning: Ghostscript not installed, skipping pdf optimization" -else - $(QUIET)$(CURDIR)/config/optimize-pdf $@ - $(QUIET)rm $@ - $(QUIET)mv $(PDFDIR)/$(ICDINSTSPEC)-optimized.pdf $@ -endif + $(QUIET)$(OPTIMIZEPDF) $@ $@.out.pdf && mv $@.out.pdf $@ # Clean generated and output files diff --git a/README.adoc b/README.adoc index a71ccdb2f..410abb0bc 100644 --- a/README.adoc +++ b/README.adoc @@ -246,6 +246,8 @@ scheme. This section describes the software components used by the OpenCL spec toolchain. +The specified versions are known to work. +Later compatible versions will probably work as well. Before building the OpenCL specs, you must install the following tools: @@ -260,10 +262,6 @@ Before building the OpenCL specs, you must install the following tools: Any version supporting the following operations should work: ** `git symbolic-ref --short HEAD` ** `git log -1 --format="%H"` - * Ghostscript (ghostscript, version: 9.10). - This is for the PDF build, and it can still progress without it. - Ghostscript is used to optimize the size of the PDF, so will be a lot - smaller if it is included. * ttf Fonts. These are needed the PDF build for latexmath rendering. See https://github.com/asciidoctor/asciidoctor-mathematical/blob/master/README.md#dependencies[Font Dependencies for asciidoctor-mathematical]. @@ -277,6 +275,7 @@ parts you don't use) completely before trying to install. * Asciidoctor (asciidoctor, version: 2.0.16) * Coderay (coderay, version: 1.1.1) + * hexapdf (version: 0.27.0) * rouge (rouge, version 3.19.0) * ttfunk (ttfunk, version: 1.5.1) * Asciidoctor PDF (asciidoctor-pdf, version: 1.5.0) @@ -444,6 +443,7 @@ echo "2.3.3" > ~/.rbenv/version gem install asciidoctor -v 2.0.16 gem install coderay -v 1.1.1 +gem install hexapdf -v 0.27.0 gem install rouge -v 3.19.0 gem install ttfunk -v 1.5.1 gem install asciidoctor-pdf -v 1.5.0 @@ -656,6 +656,7 @@ command, once the platform is set up: ---- gem install asciidoctor -v 2.0.16 gem install coderay -v 1.1.1 +gem install hexapdf -v 0.27.0 gen install rouge -v 3.19.0 gem install ttfunk -v 1.5.1 @@ -687,6 +688,7 @@ by Khronos. [[history]] == Revision History + * 2023-11-05 - Add hexapdf, remove ghostscript * 2020-03-13 - Updated package versions to match Travis build. * 2019-06-20 - Add directions for publishing OpenCL 2.2 reference pages, generated from the spec sources in this repository, in the From ceb87e7511ee73971f132d850cb3eb94bbb37f6e Mon Sep 17 00:00:00 2001 From: joshqti <127994991+joshqti@users.noreply.github.com> Date: Tue, 14 Nov 2023 10:36:06 -0800 Subject: [PATCH 023/202] cl_khr_semaphore: Enforce one device semaphores (#973) (#996) * cl_khr_semaphore: Enforce one device semaphores (#973) Only permit semaphores to be associated with a single device. Add an error code for invalid use. * Changes wording according to review comments * Change error code to CL_INVALID_PROPERTY if a context is multi-device, and no device is specified. --- ext/cl_khr_semaphore.asciidoc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ext/cl_khr_semaphore.asciidoc b/ext/cl_khr_semaphore.asciidoc index e0aff5357..3c6c9da15 100644 --- a/ext/cl_khr_semaphore.asciidoc +++ b/ext/cl_khr_semaphore.asciidoc @@ -257,10 +257,10 @@ Following new properties are added to the list of possible supported properties | Specifies the type of semaphore to create. This property is always required. | {CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR} | {cl_device_id_TYPE}[] - | Specifies the list of OpenCL devices (terminated with {CL_SEMAPHORE_DEVICE_HANDLE_LIST_END_KHR}) to associate with the semaphore. + | Specifies the list of OpenCL devices (terminated with {CL_SEMAPHORE_DEVICE_HANDLE_LIST_END_KHR}) to associate with the semaphore. Only a single device is permitted in the list. |==== -If {CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR} is not specified as part of _sema_props_, the semaphore object created by {clCreateSemaphoreWithPropertiesKHR} is by default accessible to all devices in the _context_. +If {CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR} is not specified as part of _sema_props_, the semaphore object created by {clCreateSemaphoreWithPropertiesKHR} is by default accessible to all devices in the _context_. For a multi-device context {CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR} must be specified in _sema_props_. _errcode_ret_ returns an appropriate error code. If _errcode_ret_ is `NULL`, no error code is returned. @@ -268,8 +268,8 @@ _errcode_ret_ returns an appropriate error code. If _errcode_ret_ is `NULL`, no Otherwise, it returns a `NULL` value with one of the following error values returned in _errcode_ret_: * {CL_INVALID_CONTEXT} if _context_ is not a valid context. -* {CL_INVALID_PROPERTY} if a property name in _sema_props_ is not a supported property name, if the value specified for a supported property name is not valid, or if the same property name is specified more than once. -* {CL_INVALID_DEVICE} if {CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR} is specified as part of _sema_props_, but it does not identify a valid device or if a device identified by {CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR} is not one of the devices within _context_. +* {CL_INVALID_PROPERTY} if a property name in _sema_props_ is not a supported property name, if the value specified for a supported property name is not valid, or if the same property name is specified more than once. Additionally, if _context_ is a multiple device context and _sema_props_ does not specify {CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR}. +* {CL_INVALID_DEVICE} if {CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR} is specified as part of _sema_props_, but it does not identify exactly one valid device or if a device identified by {CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR} is not one of the devices within _context_. * {CL_INVALID_VALUE} ** if _sema_props_ is `NULL`, or ** if _sema_props_ do not specify pairs for minimum set of properties (i.e. {CL_SEMAPHORE_TYPE_KHR}) required for successful creation of a {cl_semaphore_khr_TYPE}, or From 981ba9564a3ec5aa73cb65a4a3297f1f92ccd29e Mon Sep 17 00:00:00 2001 From: Ben Ashbaugh Date: Wed, 15 Nov 2023 15:05:31 -0800 Subject: [PATCH 024/202] move the layers spec to the extensions directory (#999) Since the layers spec is not published in the OpenCL extension spec and is instead published on the OpenCL registry similar to EXT and vendor extensions, it makes more sense to put it in the extensions directory. --- {ext => extensions}/cl_loader_layers.asciidoc | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {ext => extensions}/cl_loader_layers.asciidoc (100%) diff --git a/ext/cl_loader_layers.asciidoc b/extensions/cl_loader_layers.asciidoc similarity index 100% rename from ext/cl_loader_layers.asciidoc rename to extensions/cl_loader_layers.asciidoc From ea1c145d2095f48236ac48445ffababe11dbb41a Mon Sep 17 00:00:00 2001 From: joshqti <127994991+joshqti@users.noreply.github.com> Date: Tue, 21 Nov 2023 21:05:25 -0800 Subject: [PATCH 025/202] cl_semaphore_khr: Query if semaphore is exportable (#997) * cl_semaphore_khr: Query if semaphore is exportable Add query to clGetSemaphoreInfoKHR that returns CL_TRUE if a semaphore is exportable. * Change extension version to 0.9.1 * Add missing brackets around return types. --- ext/cl_khr_external_semaphore.asciidoc | 11 +++++++++++ ext/cl_khr_semaphore.asciidoc | 2 -- xml/cl.xml | 5 ++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/ext/cl_khr_external_semaphore.asciidoc b/ext/cl_khr_external_semaphore.asciidoc index 10cdaa60b..5cc6fe347 100644 --- a/ext/cl_khr_external_semaphore.asciidoc +++ b/ext/cl_khr_external_semaphore.asciidoc @@ -40,6 +40,7 @@ Other related extensions define specific external semaphores that may be importe |==== | *Date* | *Version* | *Description* | 2021-09-10 | 0.9.0 | Initial version (provisional). +| 2023-11-16 | 0.9.1 | Added CL_SEMAPHORE_EXPORTABLE_KHR. |==== NOTE: This is a preview of an OpenCL provisional extension specification that has been Ratified under the Khronos Intellectual Property Framework. It is being made publicly available prior to being uploaded to the Khronos registry to enable review and feedback from the community. If you have feedback please create an issue on https://github.com/KhronosGroup/OpenCL-Docs/ @@ -129,6 +130,13 @@ CL_SEMAPHORE_EXPORT_HANDLE_TYPES_KHR 0x203F CL_SEMAPHORE_EXPORT_HANDLE_TYPES_LIST_END_KHR 0 ---- +The following new attribute that can be passed as part of {cl_semaphore_info_khr_TYPE}: + +[source] +---- +CL_SEMAPHORE_EXPORTABLE_KHR 0x2054 +---- + External semaphore handle type added by `cl_khr_external_semaphore_dx_fence`: [source] @@ -234,6 +242,9 @@ Add to the list of supported _param_names_ by {clGetSemaphoreInfoKHR}: | Returns the list of external semaphore handle types that may be used for exporting. The size of this query may be 0 indicating that this semaphore does not support any handle types for exporting. +| {CL_SEMAPHORE_EXPORTABLE_KHR} + | {cl_bool_TYPE} + | Returns {CL_TRUE} if the semaphore is exportable and {CL_FALSE} otherwise. |==== === Exporting semaphore external handles diff --git a/ext/cl_khr_semaphore.asciidoc b/ext/cl_khr_semaphore.asciidoc index 3c6c9da15..e6cfd4ab7 100644 --- a/ext/cl_khr_semaphore.asciidoc +++ b/ext/cl_khr_semaphore.asciidoc @@ -155,8 +155,6 @@ CL_SEMAPHORE_PROPERTIES_KHR 0x203B CL_SEMAPHORE_PAYLOAD_KHR 0x203C ---- -// TODO: Do we need to define CL_SEMAPHORE_DEVICE_HANDLE_LIST here or should it be in the external semaphore spec instead? - New attributes that can be passed as part of {cl_semaphore_info_khr_TYPE} or {cl_semaphore_properties_khr_TYPE}: [source] diff --git a/xml/cl.xml b/xml/cl.xml index 0a36a34b6..b64d8dff4 100644 --- a/xml/cl.xml +++ b/xml/cl.xml @@ -1854,7 +1854,7 @@ server's OpenCL/api-docs repository. - + @@ -6991,6 +6991,9 @@ server's OpenCL/api-docs repository. + + + From a164c7ff48b51ce5f81ff11912c0767d03a7aa2f Mon Sep 17 00:00:00 2001 From: Ben Ashbaugh Date: Tue, 28 Nov 2023 04:22:16 -0800 Subject: [PATCH 026/202] remove TODO comment in semaphore spec (#1012) The default behavior when the device handle list is not specified is now properly described, so the TODO comment can be removed. --- ext/cl_khr_semaphore.asciidoc | 2 -- 1 file changed, 2 deletions(-) diff --git a/ext/cl_khr_semaphore.asciidoc b/ext/cl_khr_semaphore.asciidoc index e6cfd4ab7..ff4297b32 100644 --- a/ext/cl_khr_semaphore.asciidoc +++ b/ext/cl_khr_semaphore.asciidoc @@ -239,8 +239,6 @@ include::{generated}/api/protos/clCreateSemaphoreWithPropertiesKHR.txt[] _context_ identifies a valid OpenCL context that the created {cl_semaphore_khr_TYPE} will belong to. -// TODO: Do we want the same "all devices in the context" behavior if CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR is not specified? - _sema_props_ specifies additional semaphore properties in the form list of pairs terminated with 0. {CL_SEMAPHORE_TYPE_KHR} must be part of the list of properties specified by _sema_props_. From 92e08baa6a698fa2358540f55e1e73eef5ed583b Mon Sep 17 00:00:00 2001 From: Ben Ashbaugh Date: Tue, 28 Nov 2023 04:37:09 -0800 Subject: [PATCH 027/202] document USM error conditions for clSetKernelExecInfo (#974) --- extensions/cl_intel_unified_shared_memory.asciidoc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/extensions/cl_intel_unified_shared_memory.asciidoc b/extensions/cl_intel_unified_shared_memory.asciidoc index f591bc247..33381e294 100644 --- a/extensions/cl_intel_unified_shared_memory.asciidoc +++ b/extensions/cl_intel_unified_shared_memory.asciidoc @@ -788,6 +788,13 @@ The new _param_name_ values described below may be used with the existing *clSet |==== +The following errors may be returned by *clSetKernelExecInfo* for these new _param_name_ values: + +* `CL_INVALID_OPERATION` if _param_name_ is `CL_KERNEL_EXEC_INFO_USM_PTRS_INTEL` and no devices in the context associated with _kernel_ support Unified Shared Memory. +* `CL_INVALID_OPERATION` if _param_name_ is `CL_KERNEL_EXEC_INFO_INDIRECT_HOST_ACCESS_INTEL` and no devices in the context associated with _kernel_ support host Unified Shared Memory allocations. +* `CL_INVALID_OPERATION` if _param_name_ is `CL_KERNEL_EXEC_INFO_INDIRECT_DEVICE_ACCESS_INTEL` and no devices in the context associated with _kernel_ support device Unified Shared Memory allocations. +* `CL_INVALID_OPERATION` if _param_name_ is `CL_KERNEL_EXEC_INFO_INDIRECT_SHARED_ACCESS_INTEL` and no devices in the context associated with _kernel_ support shared Unified Shared Memory allocations. + ==== Filling and Copying Unified Shared Memory The function @@ -1281,6 +1288,7 @@ Note that there is no similar SVM "rect" memcpy. |S|2020-08-26|Maciej Dziuban|Added initial placement flags for shared allocations. |1.0.0|2021-11-07|Ben Ashbaugh|Added version and other minor updates prior to posting on the OpenCL registry. |1.0.0|2022-11-08|Ben Ashbaugh|Added new issues regarding error behavior for clSetKernelArgMemPointerINTEL and rect copies. +|1.0.1|2023-08-28|Ben Ashbaugh|Documented error conditions for clSetKernelExecInfo. |======================================== //************************************************************************ From 0cd45cb95c98bff247663f78149683a5771c7f10 Mon Sep 17 00:00:00 2001 From: Ben Ashbaugh Date: Tue, 28 Nov 2023 04:47:44 -0800 Subject: [PATCH 028/202] add description metadata (#1000) --- OpenCL_API.txt | 4 ++++ OpenCL_C.txt | 3 +++ OpenCL_Env.txt | 3 +++ OpenCL_Ext.txt | 3 +++ 4 files changed, 13 insertions(+) diff --git a/OpenCL_API.txt b/OpenCL_API.txt index cc0092ee9..db6d507fe 100644 --- a/OpenCL_API.txt +++ b/OpenCL_API.txt @@ -19,6 +19,10 @@ Khronos{R} OpenCL Working Group :docinfo: shared-header :docinfodir: config :title-logo-image: image:images/OpenCL.png[top="25%",width="55%"] +:description: OpenCL(TM) is an open, royalty-free standard for cross-platform \ +parallel programming of diverse accelerators. \ +This document describes the OpenCL API. + // Various special / math symbols. This is easier to edit with than Unicode. include::config/attribs.txt[] diff --git a/OpenCL_C.txt b/OpenCL_C.txt index c1a7e2510..1492dc238 100644 --- a/OpenCL_C.txt +++ b/OpenCL_C.txt @@ -20,6 +20,9 @@ Khronos{R} OpenCL Working Group :docinfo: shared-header :docinfodir: config :title-logo-image: image:images/OpenCL.png[top="25%",width="55%"] +:description: OpenCL(TM) is an open, royalty-free standard for cross-platform \ +parallel programming of diverse accelerators. \ +This document describes the OpenCL C language. // Various special / math symbols. This is easier to edit with than Unicode. include::config/attribs.txt[] diff --git a/OpenCL_Env.txt b/OpenCL_Env.txt index 7d10a347d..debc6b13a 100644 --- a/OpenCL_Env.txt +++ b/OpenCL_Env.txt @@ -18,6 +18,9 @@ Khronos{R} OpenCL Working Group :docinfo: shared-header :docinfodir: config :title-logo-image: image:images/OpenCL.png[top="25%",width="55%"] +:description: OpenCL(TM) is an open, royalty-free standard for cross-platform \ +parallel programming of diverse accelerators. \ +This document describes the OpenCL SPIR-V environment. // Various special / math symbols. This is easier to edit with than Unicode. include::config/attribs.txt[] diff --git a/OpenCL_Ext.txt b/OpenCL_Ext.txt index 0fe4b83c3..cedcd485a 100644 --- a/OpenCL_Ext.txt +++ b/OpenCL_Ext.txt @@ -21,6 +21,9 @@ ifndef::backend-html5[:toclevels: 2] :docinfo: shared-header :docinfodir: config :title-logo-image: image:images/OpenCL.png[top="25%",width="55%"] +:description: OpenCL(TM) is an open, royalty-free standard for cross-platform \ +parallel programming of diverse accelerators. \ +This document describes OpenCL extensions. // Various special / math symbols. This is easier to edit with than Unicode. include::config/attribs.txt[] From 6134e8d8c458f1540972bc499b0ddef6b7ff4bcf Mon Sep 17 00:00:00 2001 From: Ben Ashbaugh Date: Tue, 28 Nov 2023 04:53:18 -0800 Subject: [PATCH 029/202] consistently put newer versions at the bottom of version history tables (#1010) --- ext/cl_khr_external_memory.asciidoc | 6 +++--- ext/cl_khr_integer_dot_product.asciidoc | 2 +- ext/cl_khr_semaphore.asciidoc | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ext/cl_khr_external_memory.asciidoc b/ext/cl_khr_external_memory.asciidoc index 43a780de2..01f7330d1 100644 --- a/ext/cl_khr_external_memory.asciidoc +++ b/ext/cl_khr_external_memory.asciidoc @@ -30,10 +30,10 @@ Other related extensions define specific external memory types that may be impor [cols="1,1,3",options="header",] |==== | *Date* | *Version* | *Description* -| 2023-08-29 | 0.9.3 | Added query for {CL_DEVICE_EXTERNAL_MEMORY_IMPORT_ASSUME_LINEAR_HANDLE_TYPES_KHR} (provisional). -| 2023-08-01 | 0.9.2 | Changed device handle list enum to the memory-specific {CL_MEM_DEVICE_HANDLE_LIST_KHR} (provisional). -| 2023-05-04 | 0.9.1 | Clarified device handle list enum cannot be specified without an external memory handle (provisional). | 2021-09-10 | 0.9.0 | Initial version (provisional). +| 2023-05-04 | 0.9.1 | Clarified device handle list enum cannot be specified without an external memory handle (provisional). +| 2023-08-01 | 0.9.2 | Changed device handle list enum to the memory-specific {CL_MEM_DEVICE_HANDLE_LIST_KHR} (provisional). +| 2023-08-29 | 0.9.3 | Added query for {CL_DEVICE_EXTERNAL_MEMORY_IMPORT_ASSUME_LINEAR_HANDLE_TYPES_KHR} (provisional). |==== NOTE: This is a preview of an OpenCL provisional extension specification that has been Ratified under the Khronos Intellectual Property Framework. It is being made publicly available prior to being uploaded to the Khronos registry to enable review and feedback from the community. If you have feedback please create an issue on https://github.com/KhronosGroup/OpenCL-Docs/ diff --git a/ext/cl_khr_integer_dot_product.asciidoc b/ext/cl_khr_integer_dot_product.asciidoc index c124fec12..d75742f10 100644 --- a/ext/cl_khr_integer_dot_product.asciidoc +++ b/ext/cl_khr_integer_dot_product.asciidoc @@ -19,8 +19,8 @@ functions to compute the dot product of vectors of integers. [cols="1,1,3",options="header",] |==== | *Date* | *Version* | *Description* -| 2021-06-23 | 2.0.0 | All 8-bit support is mandatory, added 8-bit acceleration properties. | 2021-06-17 | 1.0.0 | Initial version. +| 2021-06-23 | 2.0.0 | All 8-bit support is mandatory, added 8-bit acceleration properties. |==== ==== Dependencies diff --git a/ext/cl_khr_semaphore.asciidoc b/ext/cl_khr_semaphore.asciidoc index ff4297b32..9246b9ce6 100644 --- a/ext/cl_khr_semaphore.asciidoc +++ b/ext/cl_khr_semaphore.asciidoc @@ -37,8 +37,8 @@ In particular, this extension defines: [cols="1,1,3",options="header",] |==== | *Date* | *Version* | *Description* -| 2023-08-01 | 0.9.1 | Changed device handle list enum to the semaphore-specific {CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR} (provisional). | 2021-09-10 | 0.9.0 | Initial version (provisional). +| 2023-08-01 | 0.9.1 | Changed device handle list enum to the semaphore-specific {CL_SEMAPHORE_DEVICE_HANDLE_LIST_KHR} (provisional). |==== NOTE: This is a preview of an OpenCL provisional extension specification that has been Ratified under the Khronos Intellectual Property Framework. It is being made publicly available prior to being uploaded to the Khronos registry to enable review and feedback from the community. If you have feedback please create an issue on https://github.com/KhronosGroup/OpenCL-Docs/ From a40434d0f12e6a867d60cc2835881a7d8b6147e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Petit?= Date: Wed, 29 Nov 2023 18:44:43 +0000 Subject: [PATCH 030/202] Reserve enum for cl_ext_yuv_images (#1022) Change-Id: I942c3ce47284e7aea93edc02cf0f327af95e4ed9 --- xml/cl.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xml/cl.xml b/xml/cl.xml index b64d8dff4..404a47e1d 100644 --- a/xml/cl.xml +++ b/xml/cl.xml @@ -1567,8 +1567,7 @@ server's OpenCL/api-docs repository. - - + From 502e3c0c938e33c0b296dd93888dabf05d5f479c Mon Sep 17 00:00:00 2001 From: Ben Ashbaugh Date: Wed, 29 Nov 2023 10:45:05 -0800 Subject: [PATCH 031/202] allow clSetCommandQueueProperty to return an error for non-OpenCL 1.0 devices (#980) This follows the same pattern and text used by clSetProgramReleaseCallback, which may return an error for non-OpenCL 2.2 devices. --- api/opencl_runtime_layer.asciidoc | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/api/opencl_runtime_layer.asciidoc b/api/opencl_runtime_layer.asciidoc index ed59544f0..861b1c0ad 100644 --- a/api/opencl_runtime_layer.asciidoc +++ b/api/opencl_runtime_layer.asciidoc @@ -428,25 +428,28 @@ include::{generated}/api/version-notes/clSetCommandQueueProperty.asciidoc[] changed by {clSetCommandQueueProperty}. If _old_properties_ is `NULL`, it is ignored. -[NOTE] -==== -Changing the {CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE} command-queue property -will cause the OpenCL implementation to block until all previously queued -commands in _command_queue_ have completed. This can be an expensive operation -and therefore changes to this property should only be done when absolutely -necessary. -==== - // refError +{clSetCommandQueueProperty} may unconditionally return an error if no +devices in the context associated with _command_queue_ support modifying +the properties of a command-queue. +Support for modifying the properties of a command-queue is required only for +OpenCL 1.0 devices. + {clSetCommandQueueProperty} returns {CL_SUCCESS} if the function is executed successfully. Otherwise, it returns one of the following errors: * {CL_INVALID_COMMAND_QUEUE} if _command_queue_ is not a valid command-queue. + * {CL_INVALID_OPERATION} if no devices in the context associated with + _command_queue_ support modifying the properties of a command-queue. * {CL_INVALID_VALUE} if values specified in _properties_ are not valid. * {CL_INVALID_QUEUE_PROPERTIES} if values specified in _properties_ are valid but are not supported by the device. + * {CL_OUT_OF_RESOURCES} if there is a failure to allocate resources required + by the OpenCL implementation on the device. + * {CL_OUT_OF_HOST_MEMORY} if there is a failure to allocate resources + required by the OpenCL implementation on the host. -- @@ -5672,14 +5675,14 @@ object is deleted. This provides a mechanism for an application to be notified when destructors for program scope global variables are complete. +// refError + {clSetProgramReleaseCallback} may unconditionally return an error if no devices in the context associated with _program_ support destructors for program scope global variables. Support for constructors and destructors for program scope global variables is required only for OpenCL 2.2 devices. -// refError - {clSetProgramReleaseCallback} returns {CL_SUCCESS} if the function is executed successfully. Otherwise, it returns one of the following errors: From 3300536d54e09ea8f95e5e9d2fdb45fd1d46e323 Mon Sep 17 00:00:00 2001 From: Ben Ashbaugh Date: Wed, 29 Nov 2023 10:46:13 -0800 Subject: [PATCH 032/202] clarify the free function for clEnqueueSVMFree must be thread-safe (#1016) --- api/opencl_platform_layer.asciidoc | 2 +- api/opencl_runtime_layer.asciidoc | 26 +++++++++++++++----------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/api/opencl_platform_layer.asciidoc b/api/opencl_platform_layer.asciidoc index f34098091..abac6b88a 100644 --- a/api/opencl_platform_layer.asciidoc +++ b/api/opencl_platform_layer.asciidoc @@ -1837,7 +1837,7 @@ include::{generated}/api/version-notes/clCreateContext.asciidoc[] at runtime in this context. This callback function may be called asynchronously by the OpenCL implementation. - It is the applications responsibility to ensure that the callback function + It is the application's responsibility to ensure that the callback function is thread-safe. If _pfn_notify_ is `NULL`, no callback function is registered. * _user_data_ will be passed as the _user_data_ argument when _pfn_notify_ is diff --git a/api/opencl_runtime_layer.asciidoc b/api/opencl_runtime_layer.asciidoc index 861b1c0ad..75c6728ae 100644 --- a/api/opencl_runtime_layer.asciidoc +++ b/api/opencl_runtime_layer.asciidoc @@ -4534,6 +4534,10 @@ include::{generated}/api/version-notes/clEnqueueSVMFree.asciidoc[] function returns. * _pfn_free_func_ specifies the callback function to be called to free the SVM pointers. + This callback function may be called asynchronously by the OpenCL + implementation. + It is the application's responsibility to ensure that the callback function + is thread-safe. _pfn_free_func_ takes four arguments: _queue_ which is the command-queue in which {clEnqueueSVMFree} was enqueued, the count and list of SVM pointers to free and _user_data_ which is a pointer to user specified data. @@ -5663,7 +5667,7 @@ include::{generated}/api/version-notes/clSetProgramReleaseCallback.asciidoc[] ** _user_data_ is a pointer to user supplied data. * _user_data_ will be passed as the _user_data_ argument when _pfn_notify_ is called. - user data can be `NULL`. + _user_data_ can be `NULL`. Each call to {clSetProgramReleaseCallback} registers the specified callback function on a callback stack associated with _program_. @@ -5816,10 +5820,10 @@ include::{generated}/api/version-notes/clBuildProgram.asciidoc[] has completed. This callback function may be called asynchronously by the OpenCL implementation. - It is the applications responsibility to ensure that the callback function + It is the application's responsibility to ensure that the callback function is thread-safe. - ** _user_data_ will be passed as an argument when _pfn_notify_ is called. - _user_data_ can be `NULL`. + * _user_data_ will be passed as an argument when _pfn_notify_ is called. + _user_data_ can be `NULL`. The program executable is built from the program source or binary for all the devices, or a specific device(s) in the OpenCL context associated with @@ -5954,10 +5958,10 @@ include::{generated}/api/version-notes/clCompileProgram.asciidoc[] compiler has completed. This callback function may be called asynchronously by the OpenCL implementation. - It is the applications responsibility to ensure that the callback function + It is the application's responsibility to ensure that the callback function is thread-safe. - ** _user_data_ will be passed as an argument when _pfn_notify_ is called. - _user_data_ can be `NULL`. + * _user_data_ will be passed as an argument when _pfn_notify_ is called. + _user_data_ can be `NULL`. The pre-processor runs before the program sources are compiled. The compiled binary is built for all devices associated with _program_ or @@ -6097,8 +6101,8 @@ include::{generated}/api/version-notes/clLinkProgram.asciidoc[] The notification routine is a callback function that an application can register and which will be called when the program executable has been built (successfully or unsuccessfully). - ** _user_data_ will be passed as an argument when _pfn_notify_ is called. - _user_data_ can be `NULL`. + * _user_data_ will be passed as an argument when _pfn_notify_ is called. + _user_data_ can be `NULL`. If _pfn_notify_ is not `NULL`, {clLinkProgram} does not need to wait for the linker to complete, and can return immediately once the linking operation can @@ -6109,7 +6113,7 @@ Any state changes of the program object that result from calling {clLinkProgram} (e.g. link status or log) will be observable from this callback function. This callback function may be called asynchronously by the OpenCL implementation. -It is the applications responsibility to ensure that the callback function +It is the application's responsibility to ensure that the callback function is thread-safe. If _pfn_notify_ is `NULL`, {clLinkProgram} does not return until the linker @@ -8861,7 +8865,7 @@ include::{generated}/api/version-notes/clSetEventCallback.asciidoc[] the application. This callback function may be called asynchronously by the OpenCL implementation. - It is the applications responsibility to ensure that the callback function + It is the application's responsibility to ensure that the callback function is thread-safe. The parameters to this callback function are: ** _event_ is the event object for which the callback function is invoked. From d65ddf8e5effafbea1dee7bc9c2658effaeb59c9 Mon Sep 17 00:00:00 2001 From: Ben Ashbaugh Date: Wed, 29 Nov 2023 10:46:49 -0800 Subject: [PATCH 033/202] use title case for more chapter headings (#1011) --- ext/cl_khr_3d_image_writes.asciidoc | 4 ++-- ext/cl_khr_async_work_group_copy_fence.asciidoc | 4 ++-- ext/cl_khr_byte_addressable_store.asciidoc | 4 ++-- ext/cl_khr_create_command_queue.asciidoc | 4 ++-- ext/cl_khr_d3d10_sharing.asciidoc | 4 ++-- ext/cl_khr_d3d11_sharing.asciidoc | 4 ++-- ext/cl_khr_depth_images.asciidoc | 4 ++-- ext/cl_khr_device_enqueue_local_arg_types.asciidoc | 4 ++-- ext/cl_khr_device_uuid.asciidoc | 4 ++-- ext/cl_khr_dx9_media_sharing.asciidoc | 4 ++-- ext/cl_khr_egl_event.asciidoc | 4 ++-- ext/cl_khr_egl_image.asciidoc | 4 ++-- ext/cl_khr_expect_assume.asciidoc | 2 +- ext/cl_khr_extended_async_copies.asciidoc | 4 ++-- ext/cl_khr_extended_versioning.asciidoc | 4 ++-- ext/cl_khr_fp16.asciidoc | 4 ++-- ext/cl_khr_fp64.asciidoc | 4 ++-- ext/cl_khr_gl_depth_images.asciidoc | 4 ++-- ext/cl_khr_gl_event.asciidoc | 4 ++-- ext/cl_khr_gl_msaa_sharing.asciidoc | 4 ++-- ext/cl_khr_gl_sharing__context.asciidoc | 4 ++-- ext/cl_khr_gl_sharing__memobjs.asciidoc | 4 ++-- ext/cl_khr_icd.asciidoc | 4 ++-- ext/cl_khr_il_program.asciidoc | 4 ++-- ext/cl_khr_image2d_from_buffer.asciidoc | 4 ++-- ext/cl_khr_initialize_memory.asciidoc | 4 ++-- ext/cl_khr_int32_atomics.asciidoc | 4 ++-- ext/cl_khr_int64_atomics.asciidoc | 4 ++-- ext/cl_khr_mipmap_image.asciidoc | 4 ++-- ext/cl_khr_pci_bus_info.asciidoc | 2 +- ext/cl_khr_priority_hints.asciidoc | 4 ++-- ext/cl_khr_select_fprounding_mode.asciidoc | 4 ++-- ext/cl_khr_spir.asciidoc | 4 ++-- ext/cl_khr_srgb_image_writes.asciidoc | 4 ++-- ext/cl_khr_subgroup_extensions.asciidoc | 4 ++-- ext/cl_khr_subgroup_named_barrier.asciidoc | 4 ++-- ext/cl_khr_subgroups.asciidoc | 4 ++-- ext/cl_khr_terminate_context.asciidoc | 4 ++-- ext/cl_khr_throttle_hints.asciidoc | 4 ++-- 39 files changed, 76 insertions(+), 76 deletions(-) diff --git a/ext/cl_khr_3d_image_writes.asciidoc b/ext/cl_khr_3d_image_writes.asciidoc index 211555705..f3a9c07d9 100644 --- a/ext/cl_khr_3d_image_writes.asciidoc +++ b/ext/cl_khr_3d_image_writes.asciidoc @@ -11,9 +11,9 @@ This extension adds built-in functions that allow a kernel to write to 3D image This extension became a core feature in OpenCL 2.0. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_async_work_group_copy_fence.asciidoc b/ext/cl_khr_async_work_group_copy_fence.asciidoc index c2b20af0a..420e4afba 100644 --- a/ext/cl_khr_async_work_group_copy_fence.asciidoc +++ b/ext/cl_khr_async_work_group_copy_fence.asciidoc @@ -8,9 +8,9 @@ This section describes the *cl_khr_async_work_group_copy_fence* extension. The extension adds a new built-in function to OpenCL C to establish a memory synchronization ordering of asynchronous copies. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_byte_addressable_store.asciidoc b/ext/cl_khr_byte_addressable_store.asciidoc index 11b7bd88a..357756527 100644 --- a/ext/cl_khr_byte_addressable_store.asciidoc +++ b/ext/cl_khr_byte_addressable_store.asciidoc @@ -11,9 +11,9 @@ With this extension, applications are able to read from and write to pointers to This extension became a core feature in OpenCL 1.1. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_create_command_queue.asciidoc b/ext/cl_khr_create_command_queue.asciidoc index c8876239f..84b7ca311 100644 --- a/ext/cl_khr_create_command_queue.asciidoc +++ b/ext/cl_khr_create_command_queue.asciidoc @@ -27,9 +27,9 @@ Applications that only target OpenCL 2.x devices should use the core OpenCL 2.x {clCreateCommandQueueWithProperties} API instead of this extension API. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_d3d10_sharing.asciidoc b/ext/cl_khr_d3d10_sharing.asciidoc index 05a8ab0fe..3e8cb1557 100644 --- a/ext/cl_khr_d3d10_sharing.asciidoc +++ b/ext/cl_khr_d3d10_sharing.asciidoc @@ -12,9 +12,9 @@ This section describes the *cl_khr_d3d10_sharing* extension. The goal of this extension is to provide interoperability between OpenCL and Direct3D 10. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_d3d11_sharing.asciidoc b/ext/cl_khr_d3d11_sharing.asciidoc index 200ad0799..db190c244 100644 --- a/ext/cl_khr_d3d11_sharing.asciidoc +++ b/ext/cl_khr_d3d11_sharing.asciidoc @@ -12,9 +12,9 @@ This section describes the *cl_khr_d3d11_sharing* extension. The goal of this extension is to provide interoperability between OpenCL and Direct3D 11. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_depth_images.asciidoc b/ext/cl_khr_depth_images.asciidoc index c066cc6a7..665ba8467 100644 --- a/ext/cl_khr_depth_images.asciidoc +++ b/ext/cl_khr_depth_images.asciidoc @@ -11,9 +11,9 @@ This extension adds support for depth images. This extension became a core feature in OpenCL 2.0. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_device_enqueue_local_arg_types.asciidoc b/ext/cl_khr_device_enqueue_local_arg_types.asciidoc index 88e26e8de..b7775aae0 100644 --- a/ext/cl_khr_device_enqueue_local_arg_types.asciidoc +++ b/ext/cl_khr_device_enqueue_local_arg_types.asciidoc @@ -11,9 +11,9 @@ requiring arguments to blocks to be pointers to void in local memory. The name of this extension is *cl_khr_device_enqueue_local_arg_types*. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_device_uuid.asciidoc b/ext/cl_khr_device_uuid.asciidoc index 6dbb2b1c4..fe98af0db 100644 --- a/ext/cl_khr_device_uuid.asciidoc +++ b/ext/cl_khr_device_uuid.asciidoc @@ -12,9 +12,9 @@ This extension adds the ability to query a universally unique identifier The UUIDs returned by the query may be used to identify drivers and devices across processes or APIs. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_dx9_media_sharing.asciidoc b/ext/cl_khr_dx9_media_sharing.asciidoc index 83827464b..30610072b 100644 --- a/ext/cl_khr_dx9_media_sharing.asciidoc +++ b/ext/cl_khr_dx9_media_sharing.asciidoc @@ -20,9 +20,9 @@ Note that OpenCL memory objects may be created from the adapter media surface if and only if the OpenCL context has been created from that adapter. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_egl_event.asciidoc b/ext/cl_khr_egl_event.asciidoc index d8d6a5630..70dafeb0e 100644 --- a/ext/cl_khr_egl_event.asciidoc +++ b/ext/cl_khr_egl_event.asciidoc @@ -15,9 +15,9 @@ between the two APIs. The companion *EGL_KHR_cl_event* extension provides the complementary functionality of creating an EGL sync object from an OpenCL event object. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_egl_image.asciidoc b/ext/cl_khr_egl_image.asciidoc index 17c32d5cf..2d1a8a75a 100644 --- a/ext/cl_khr_egl_image.asciidoc +++ b/ext/cl_khr_egl_image.asciidoc @@ -12,9 +12,9 @@ This section describes the *cl_khr_egl_image* extension. This extension provides a mechanism to creating OpenCL memory objects from from EGLImages. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_expect_assume.asciidoc b/ext/cl_khr_expect_assume.asciidoc index df8cfdcc8..f4af54bff 100644 --- a/ext/cl_khr_expect_assume.asciidoc +++ b/ext/cl_khr_expect_assume.asciidoc @@ -16,7 +16,7 @@ These functions are not required for functional correctness. The initial version of this extension extends the OpenCL SPIR-V environment to support new instructions for offline compilation tool chains. Similar functionality may be provided by some OpenCL C online compilation tool chains, but formal support in OpenCL C is not required by the initial version of the extension. -=== General information +=== General Information ==== Name Strings diff --git a/ext/cl_khr_extended_async_copies.asciidoc b/ext/cl_khr_extended_async_copies.asciidoc index 28a82538c..6dd262198 100644 --- a/ext/cl_khr_extended_async_copies.asciidoc +++ b/ext/cl_khr_extended_async_copies.asciidoc @@ -12,9 +12,9 @@ to support more patterns: 1. for async copy between 2D source and 2D destination. 2. for async copy between 3D source and 3D destination. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_extended_versioning.asciidoc b/ext/cl_khr_extended_versioning.asciidoc index e62e4c08d..984bc2f38 100644 --- a/ext/cl_khr_extended_versioning.asciidoc +++ b/ext/cl_khr_extended_versioning.asciidoc @@ -17,7 +17,7 @@ Extended versioning was promoted to a core feature in OpenCL 3.0, however note that the query for {CL_DEVICE_OPENCL_C_NUMERIC_VERSION_KHR} was replaced by the query for {CL_DEVICE_OPENCL_C_ALL_VERSIONS}. -=== General information +=== General Information ==== Name Strings @@ -30,7 +30,7 @@ Ben Ashbaugh, Intel + Alastair Murray, Codeplay Software Ltd. + Einar Hov, Arm Ltd. -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_fp16.asciidoc b/ext/cl_khr_fp16.asciidoc index a2fb42cb2..c6233a695 100644 --- a/ext/cl_khr_fp16.asciidoc +++ b/ext/cl_khr_fp16.asciidoc @@ -9,9 +9,9 @@ This section describes the *cl_khr_fp16* extension. This extension adds support for half scalar and vector types as built-in types that can be used for arithmetic operations, conversions etc. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_fp64.asciidoc b/ext/cl_khr_fp64.asciidoc index 4cf9d679d..cb3a45fac 100644 --- a/ext/cl_khr_fp64.asciidoc +++ b/ext/cl_khr_fp64.asciidoc @@ -8,9 +8,9 @@ This section describes the *cl_khr_fp64* extension. This extension became an optional core feature in OpenCL 1.2. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_gl_depth_images.asciidoc b/ext/cl_khr_gl_depth_images.asciidoc index f713577ac..d50e38260 100644 --- a/ext/cl_khr_gl_depth_images.asciidoc +++ b/ext/cl_khr_gl_depth_images.asciidoc @@ -12,9 +12,9 @@ cl_khr_gl_sharing_extension) defined in Objects>> to allow an OpenCL image to be created from an OpenGL depth or depth-stencil texture. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_gl_event.asciidoc b/ext/cl_khr_gl_event.asciidoc index 68ea4e53c..97df4a870 100644 --- a/ext/cl_khr_gl_event.asciidoc +++ b/ext/cl_khr_gl_event.asciidoc @@ -20,9 +20,9 @@ In addition, this extension modifies the behavior of guarantee synchronization with an OpenGL context bound in the same thread as the OpenCL context. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_gl_msaa_sharing.asciidoc b/ext/cl_khr_gl_msaa_sharing.asciidoc index 6c2958591..1418443d7 100644 --- a/ext/cl_khr_gl_msaa_sharing.asciidoc +++ b/ext/cl_khr_gl_msaa_sharing.asciidoc @@ -15,9 +15,9 @@ MSAA) texture (color or depth). This extension name is *cl_khr_gl_msaa_sharing*. This extension requires *cl_khr_gl_depth_images*. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_gl_sharing__context.asciidoc b/ext/cl_khr_gl_sharing__context.asciidoc index 4237fa300..300bd070e 100644 --- a/ext/cl_khr_gl_sharing__context.asciidoc +++ b/ext/cl_khr_gl_sharing__context.asciidoc @@ -18,9 +18,9 @@ may be used to share OpenGL buffer, texture, and renderbuffer objects with the O An OpenGL implementation supporting buffer objects and sharing of texture and buffer object images with OpenCL is required by this extension. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_gl_sharing__memobjs.asciidoc b/ext/cl_khr_gl_sharing__memobjs.asciidoc index f8a919a52..b9b5d1761 100644 --- a/ext/cl_khr_gl_sharing__memobjs.asciidoc +++ b/ext/cl_khr_gl_sharing__memobjs.asciidoc @@ -19,9 +19,9 @@ Any supported OpenGL object defined within the associated OpenGL context or share group object may be shared, with the exception of the default OpenGL objects (i.e. objects named zero), which may not be shared. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_icd.asciidoc b/ext/cl_khr_icd.asciidoc index c50afb124..816a9300e 100644 --- a/ext/cl_khr_icd.asciidoc +++ b/ext/cl_khr_icd.asciidoc @@ -20,9 +20,9 @@ This is a platform extension, so if this extension is supported by an implementation, the string *cl_khr_icd* will be present in the {CL_PLATFORM_EXTENSIONS} string. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_il_program.asciidoc b/ext/cl_khr_il_program.asciidoc index 0742baa0b..721c7eccd 100644 --- a/ext/cl_khr_il_program.asciidoc +++ b/ext/cl_khr_il_program.asciidoc @@ -14,9 +14,9 @@ the OpenCL environment may be found in the OpenCL SPIR-V Environment Specificati This functionality described by this extension is a core feature in OpenCL 2.1. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_image2d_from_buffer.asciidoc b/ext/cl_khr_image2d_from_buffer.asciidoc index f7f13ada2..4c08840c9 100644 --- a/ext/cl_khr_image2d_from_buffer.asciidoc +++ b/ext/cl_khr_image2d_from_buffer.asciidoc @@ -11,9 +11,9 @@ This extension allows a 2D image to be created from an existing OpenCL buffer me This extension became a core feature in OpenCL 2.0. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_initialize_memory.asciidoc b/ext/cl_khr_initialize_memory.asciidoc index dbd4c42ed..b2730b913 100644 --- a/ext/cl_khr_initialize_memory.asciidoc +++ b/ext/cl_khr_initialize_memory.asciidoc @@ -19,9 +19,9 @@ This extension adds support for initializing local and private memory before a kernel begins execution. This extension name is *cl_khr_initialize_memory*. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_int32_atomics.asciidoc b/ext/cl_khr_int32_atomics.asciidoc index 6d2b3de86..cf5657073 100644 --- a/ext/cl_khr_int32_atomics.asciidoc +++ b/ext/cl_khr_int32_atomics.asciidoc @@ -10,9 +10,9 @@ These extensions allow atomic operations to be performed on 32-bit signed and un These extensions became core features in OpenCL 1.1, except the built-in atomic function names are changed to use the **atomic_** prefix instead of **atom_** and the volatile qualifier was added to the pointer parameter _p_. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_int64_atomics.asciidoc b/ext/cl_khr_int64_atomics.asciidoc index 95af963a7..f2875a562 100644 --- a/ext/cl_khr_int64_atomics.asciidoc +++ b/ext/cl_khr_int64_atomics.asciidoc @@ -7,9 +7,9 @@ This section describes the *cl_khr_int64_base_atomics* and *cl_khr_int64_extended_atomics* extensions. These extensions allow atomic operations to be performed on 64-bit signed and unsigned integers in global and local memory. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_mipmap_image.asciidoc b/ext/cl_khr_mipmap_image.asciidoc index b88ef2e2d..d5a270da5 100644 --- a/ext/cl_khr_mipmap_image.asciidoc +++ b/ext/cl_khr_mipmap_image.asciidoc @@ -17,9 +17,9 @@ be used to write a mip-mapped image in an OpenCL C program. If the *cl_khr_mipmap_image_writes* extension is supported by the OpenCL device, the *cl_khr_mipmap_image* extension must also be supported. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_pci_bus_info.asciidoc b/ext/cl_khr_pci_bus_info.asciidoc index d144a8134..3b92c929a 100644 --- a/ext/cl_khr_pci_bus_info.asciidoc +++ b/ext/cl_khr_pci_bus_info.asciidoc @@ -18,7 +18,7 @@ extension string for each individual OpenCL device for which they intend to issue the new query for and should not have any assumptions about the availability of the extension on any given platform. -=== General information +=== General Information ==== Name Strings diff --git a/ext/cl_khr_priority_hints.asciidoc b/ext/cl_khr_priority_hints.asciidoc index fbffaf13c..5da9a971c 100644 --- a/ext/cl_khr_priority_hints.asciidoc +++ b/ext/cl_khr_priority_hints.asciidoc @@ -12,9 +12,9 @@ It is expected that the the user guides associated with each implementation which supports this extension will describe the scheduling behavior guarantees. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_select_fprounding_mode.asciidoc b/ext/cl_khr_select_fprounding_mode.asciidoc index 5c1934a12..1432e07e7 100644 --- a/ext/cl_khr_select_fprounding_mode.asciidoc +++ b/ext/cl_khr_select_fprounding_mode.asciidoc @@ -10,9 +10,9 @@ It allows an application to specify the rounding mode for an instruction or grou **This extension was deprecated in OpenCL 1.1 and its use is not recommended.** -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_spir.asciidoc b/ext/cl_khr_spir.asciidoc index 4790158b9..f7c999307 100644 --- a/ext/cl_khr_spir.asciidoc +++ b/ext/cl_khr_spir.asciidoc @@ -15,9 +15,9 @@ This extension has been superseded by the SPIR-V intermediate representation, which is supported by the *cl_khr_il_program* extension, and is a core feature in OpenCL 2.1. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_srgb_image_writes.asciidoc b/ext/cl_khr_srgb_image_writes.asciidoc index 07c7da406..357d7d5e0 100644 --- a/ext/cl_khr_srgb_image_writes.asciidoc +++ b/ext/cl_khr_srgb_image_writes.asciidoc @@ -13,9 +13,9 @@ The sRGB image formats that may be written to will be returned by {clGetSupporte When the image is an sRGB image, the *write_imagef* built-in function will perform the linear to sRGB conversion. Only the R, G, and B components are converted from linear to sRGB; the A component is written as-is. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_subgroup_extensions.asciidoc b/ext/cl_khr_subgroup_extensions.asciidoc index a3f7f0bce..9f131cb0d 100644 --- a/ext/cl_khr_subgroup_extensions.asciidoc +++ b/ext/cl_khr_subgroup_extensions.asciidoc @@ -28,9 +28,9 @@ The functionality added by these extensions includes: This section describes changes to the OpenCL C Language for these extensions. There are no new API functions or enums added by these extensions. -=== General information +=== General Information -==== Version history +==== Version History For all of the extensions described in this section: diff --git a/ext/cl_khr_subgroup_named_barrier.asciidoc b/ext/cl_khr_subgroup_named_barrier.asciidoc index f04b7dec1..7140e7d8e 100644 --- a/ext/cl_khr_subgroup_named_barrier.asciidoc +++ b/ext/cl_khr_subgroup_named_barrier.asciidoc @@ -14,9 +14,9 @@ sub-groups named barriers in the SPIR-V intermediate representation, and to the OpenCL {cpp} specification for descriptions of the sub-group named barrier built-in functions in the OpenCL {cpp} kernel language. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_subgroups.asciidoc b/ext/cl_khr_subgroups.asciidoc index 91b93c868..8ea3cdd4a 100644 --- a/ext/cl_khr_subgroups.asciidoc +++ b/ext/cl_khr_subgroups.asciidoc @@ -16,9 +16,9 @@ Sub-groups were promoted to a core feature in OpenCL 2.1, however note that: * The sub-group OpenCL C built-in functions described by this extension must still be accessed as an OpenCL C extension in OpenCL 2.1. * Sub-group independent forward progress is an optional device property in OpenCL 2.1, see {CL_DEVICE_SUB_GROUP_INDEPENDENT_FORWARD_PROGRESS}. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_terminate_context.asciidoc b/ext/cl_khr_terminate_context.asciidoc index 250d4020a..4b3a7f816 100644 --- a/ext/cl_khr_terminate_context.asciidoc +++ b/ext/cl_khr_terminate_context.asciidoc @@ -27,9 +27,9 @@ terminate an OpenCL context and adds an API to terminate a context. The extension name is *cl_khr_terminate_context*. -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== diff --git a/ext/cl_khr_throttle_hints.asciidoc b/ext/cl_khr_throttle_hints.asciidoc index 728d8c441..ff734cae5 100644 --- a/ext/cl_khr_throttle_hints.asciidoc +++ b/ext/cl_khr_throttle_hints.asciidoc @@ -17,9 +17,9 @@ For example, a task may have high priority ({CL_QUEUE_PRIORITY_HIGH_KHR}) but should at the same time be executed at an optimized throttle setting ({CL_QUEUE_THROTTLE_LOW_KHR}). -=== General information +=== General Information -==== Version history +==== Version History [cols="1,1,3",options="header",] |==== From 21943846498be7792b7f6c370573c1333d671d72 Mon Sep 17 00:00:00 2001 From: Ben Ashbaugh Date: Wed, 29 Nov 2023 10:47:32 -0800 Subject: [PATCH 034/202] clarify async copies and wait group events must be convergent (#1015) --- OpenCL_C.txt | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/OpenCL_C.txt b/OpenCL_C.txt index 1492dc238..a25d94924 100644 --- a/OpenCL_C.txt +++ b/OpenCL_C.txt @@ -6382,6 +6382,23 @@ The OpenCL C programming language implements the <> that provide asynchronous copies between `global` and local memory and a prefetch from `global` memory. +The async copy and wait group events functions are performed by all work-items +in a work-group and therefore must be encountered by all work-items in a +work-group executing the kernel with the same argument values, otherwise the +results are undefined. +This rule applies to ND-ranges implemented with uniform and non-uniform +work-groups. + +If an async copy or wait group events function is inside a conditional statement +then all work-items in the work-group must enter the conditional if any +work-item in the work-group enters the conditional statement and executes the +async copy or wait group events function. + +If an async copy or wait group events function is inside a loop then all +work-items in the work-group must execute the async copy or wait group events +function on each iteration of the loop if any work-item executes the async copy +or wait group events function on that iteration. + We use the generic type name `gentype` to indicate the built-in data types `char`, `char__n__`, `uchar`, `uchar__n__`, `short`, `short__n__`, `ushort`, `ushort__n__`, `int`, `int__n__`, `uint`, @@ -6402,13 +6419,6 @@ _n_ is 2, 3 footnote:[{fn-vec3-async-copy}], 4, 8, or 16. const {local} gentype *_src_, size_t _num_gentypes_, event_t _event_) | Perform an async copy of _num_gentypes_ gentype elements from _src_ to _dst_. - The async copy is performed by all work-items in a work-group and this - built-in function must therefore be encountered by all work-items in a - work-group executing the kernel with the same argument values; - otherwise the results are undefined. - This rule applies to ND-ranges implemented with uniform and - non-uniform work-groups. - Returns an event object that can be used by *wait_group_events* to wait for the async copy to finish. The _event_ argument can also be used to associate the @@ -6436,12 +6446,6 @@ _n_ is 2, 3 footnote:[{fn-vec3-async-copy}], 4, 8, or 16. element read from _src_. The _dst_stride_ is the stride in elements for each `gentype` element written to _dst_. - The async gather is performed by all work-items in a work-group. - This built-in function must therefore be encountered by all work-items - in a work-group executing the kernel with the same argument values; - otherwise the results are undefined. - This rule applies to ND-ranges implemented with uniform and - non-uniform work-groups Returns an event object that can be used by *wait_group_events* to wait for the async copy to finish. @@ -6470,12 +6474,6 @@ _n_ is 2, 3 footnote:[{fn-vec3-async-copy}], 4, 8, or 16. to complete. The event objects specified in _event_list_ will be released after the wait is performed. - - This function must be encountered by all work-items in a work-group - executing the kernel with the same _num_events_ and event objects - specified in _event_list_; otherwise the results are undefined. - This rule applies to ND-ranges implemented with uniform and - non-uniform work-groups | | | void **prefetch**(const {global} gentype *_p_, size_t _num_gentypes_) | Prefetch `_num_gentypes_ * sizeof(gentype)` bytes into the global From 8cd620684bf8e7d59e4f410c577aada548c6ab55 Mon Sep 17 00:00:00 2001 From: Ben Ashbaugh Date: Wed, 29 Nov 2023 10:47:59 -0800 Subject: [PATCH 035/202] add old command queue APIs to the reference page table of contents (#985) --- man/toctail | 2 ++ 1 file changed, 2 insertions(+) diff --git a/man/toctail b/man/toctail index 3e96a3d03..82092a1ce 100644 --- a/man/toctail +++ b/man/toctail @@ -39,10 +39,12 @@