1#!/usr/bin/python 2# Copyright 2021 The ANGLE Project Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5# 6# gen_spirv_builder_and_parser.py: 7# Code generation for SPIR-V instruction builder and parser. 8# NOTE: don't run this script directly. Run scripts/run_code_generation.py. 9 10import itertools 11import json 12import os 13import sys 14 15# ANGLE uses SPIR-V 1.0 currently, so there's no reason to generate code for newer instructions. 16SPIRV_GRAMMAR_FILE = '../../../third_party/vulkan-deps/spirv-headers/src/include/spirv/1.0/spirv.core.grammar.json' 17 18# The script has two sets of outputs, a header and source file for SPIR-V code generation, and a 19# header and source file for SPIR-V parsing. 20SPIRV_BUILDER_FILE = 'spirv_instruction_builder' 21SPIRV_PARSER_FILE = 'spirv_instruction_parser' 22 23# The types are either defined in spirv_types.h (to use strong types), or are enums that are 24# defined by SPIR-V headers. 25ANGLE_DEFINED_TYPES = [ 26 'IdRef', 'IdResult', 'IdResultType', 'IdMemorySemantics', 'IdScope', 'LiteralInteger', 27 'LiteralString', 'LiteralContextDependentNumber', 'LiteralExtInstInteger', 28 'PairLiteralIntegerIdRef', 'PairIdRefLiteralInteger', 'PairIdRefIdRef' 29] 30 31HEADER_TEMPLATE = """// GENERATED FILE - DO NOT EDIT. 32// Generated by {script_name} using data from {data_source_name}. 33// 34// Copyright 2021 The ANGLE Project Authors. All rights reserved. 35// Use of this source code is governed by a BSD-style license that can be 36// found in the LICENSE file. 37// 38// {file_name}_autogen.h: 39// Functions to {verb} SPIR-V binary for each instruction. 40 41#ifndef COMMON_SPIRV_{file_name_capitalized}AUTOGEN_H_ 42#define COMMON_SPIRV_{file_name_capitalized}AUTOGEN_H_ 43 44#include <spirv/unified1/spirv.hpp> 45 46#include "spirv_types.h" 47 48namespace angle 49{{ 50namespace spirv 51{{ 52{prototype_list} 53}} // namespace spirv 54}} // namespace angle 55 56#endif // COMMON_SPIRV_{file_name_capitalized}AUTOGEN_H_ 57""" 58 59SOURCE_TEMPLATE = """// GENERATED FILE - DO NOT EDIT. 60// Generated by {script_name} using data from {data_source_name}. 61// 62// Copyright 2021 The ANGLE Project Authors. All rights reserved. 63// Use of this source code is governed by a BSD-style license that can be 64// found in the LICENSE file. 65// 66// {file_name}_autogen.cpp: 67// Functions to {verb} SPIR-V binary for each instruction. 68 69#include "{file_name}_autogen.h" 70 71#include <string.h> 72 73#include "common/debug.h" 74 75namespace angle 76{{ 77namespace spirv 78{{ 79{helper_functions} 80 81{function_list} 82}} // namespace spirv 83}} // namespace angle 84""" 85 86BUILDER_HELPER_FUNCTIONS = """namespace 87{ 88uint32_t MakeLengthOp(size_t length, spv::Op op) 89{ 90 ASSERT(length <= 0xFFFFu); 91 ASSERT(op <= 0xFFFFu); 92 93 return static_cast<uint32_t>(length) << 16 | op; 94} 95} // anonymous namespace 96 97void WriteSpirvHeader(std::vector<uint32_t> *blob, uint32_t idCount) 98{ 99 // Header: 100 // 101 // - Magic number 102 // - Version (1.0) 103 // - ANGLE's Generator number: 104 // * 24 for tool id (higher 16 bits) 105 // * 0 for tool version (lower 16 bits)) 106 // - Bound (idCount) 107 // - 0 (reserved) 108 constexpr uint32_t kANGLEGeneratorId = 24; 109 110 ASSERT(blob->empty()); 111 112 blob->push_back(spv::MagicNumber); 113 blob->push_back(0x00010000); 114 blob->push_back(kANGLEGeneratorId << 16 | 0); 115 blob->push_back(idCount); 116 blob->push_back(0x00000000); 117} 118""" 119 120BUILDER_HELPER_FUNCTION_PROTOTYPE = """ 121 void WriteSpirvHeader(std::vector<uint32_t> *blob, uint32_t idCount); 122""" 123 124PARSER_FIXED_FUNCTIONS_PROTOTYPES = """void GetInstructionOpAndLength(const uint32_t *_instruction, 125 spv::Op *opOut, uint32_t *lengthOut); 126""" 127 128PARSER_FIXED_FUNCTIONS = """void GetInstructionOpAndLength(const uint32_t *_instruction, 129 spv::Op *opOut, uint32_t *lengthOut) 130{ 131 constexpr uint32_t kOpMask = 0xFFFFu; 132 *opOut = static_cast<spv::Op>(_instruction[0] & kOpMask); 133 *lengthOut = _instruction[0] >> 16; 134} 135""" 136 137TEMPLATE_BUILDER_FUNCTION_PROTOTYPE = """void Write{op}(Blob *blob {param_list})""" 138TEMPLATE_BUILDER_FUNCTION_BODY = """{{ 139 const size_t startSize = blob->size(); 140 blob->push_back(0); 141 {push_back_lines} 142 (*blob)[startSize] = MakeLengthOp(blob->size() - startSize, spv::Op{op}); 143}} 144""" 145 146TEMPLATE_PARSER_FUNCTION_PROTOTYPE = """void Parse{op}(const uint32_t *_instruction {param_list})""" 147TEMPLATE_PARSER_FUNCTION_BODY = """{{ 148 spv::Op _op; 149 uint32_t _length; 150 GetInstructionOpAndLength(_instruction, &_op, &_length); 151 ASSERT(_op == spv::Op{op}); 152 uint32_t _o = 1; 153 {parse_lines} 154}} 155""" 156 157 158def load_grammar(grammar_file): 159 with open(grammar_file) as grammar_in: 160 grammar = json.loads(grammar_in.read()) 161 162 return grammar 163 164 165def remove_chars(string, chars): 166 return filter(lambda c: c not in chars, string) 167 168 169def make_camel_case(name): 170 return name[0].lower() + name[1:] 171 172 173class Writer: 174 175 def __init__(self): 176 self.path_prefix = os.path.dirname(os.path.realpath(__file__)) + os.path.sep 177 self.grammar = load_grammar(self.path_prefix + SPIRV_GRAMMAR_FILE) 178 179 # If an instruction has a parameter of these types, the instruction is ignored 180 self.unsupported_kinds = set(['LiteralSpecConstantOpInteger']) 181 # If an instruction requires a capability of these kinds, the instruction is ignored 182 self.unsupported_capabilities = set(['Kernel', 'Addresses']) 183 # If an instruction requires an extension other than these, the instruction is ignored 184 self.supported_extensions = set([]) 185 # List of bit masks. These have 'Mask' added to their typename in SPIR-V headers. 186 self.bit_mask_types = set([]) 187 188 # List of generated instructions builder/parser functions so far. 189 self.instruction_builder_prototypes = [BUILDER_HELPER_FUNCTION_PROTOTYPE] 190 self.instruction_builder_impl = [] 191 self.instruction_parser_prototypes = [PARSER_FIXED_FUNCTIONS_PROTOTYPES] 192 self.instruction_parser_impl = [PARSER_FIXED_FUNCTIONS] 193 194 def write_builder_and_parser(self): 195 """Generates four files, a set of header and source files for generating SPIR-V instructions 196 and a set for parsing SPIR-V instructions. Only Vulkan instructions are processed (and not 197 OpenCL for example), and some assumptions are made based on ANGLE's usage (for example that 198 constants always fit in one 32-bit unit, as GLES doesn't support double or 64-bit types). 199 200 Additionally, enums and other parameter 'kinds' are not parsed from the json file, but 201 rather use the definitions from the SPIR-V headers repository and the spirv_types.h file.""" 202 203 # Recurse through capabilities and accumulate ones that depend on unsupported ones. 204 self.accumulate_unsupported_capabilities() 205 206 self.find_bit_mask_types() 207 208 for instruction in self.grammar['instructions']: 209 self.generate_instruction_functions(instruction) 210 211 # Write out the files. 212 data_source_base_name = os.path.basename(SPIRV_GRAMMAR_FILE) 213 builder_template_args = { 214 'script_name': sys.argv[0], 215 'data_source_name': data_source_base_name, 216 'file_name': SPIRV_BUILDER_FILE, 217 'file_name_capitalized': remove_chars(SPIRV_BUILDER_FILE.upper(), '_'), 218 'verb': 'generate', 219 'helper_functions': BUILDER_HELPER_FUNCTIONS, 220 'prototype_list': ''.join(self.instruction_builder_prototypes), 221 'function_list': ''.join(self.instruction_builder_impl) 222 } 223 parser_template_args = { 224 'script_name': sys.argv[0], 225 'data_source_name': data_source_base_name, 226 'file_name': SPIRV_PARSER_FILE, 227 'file_name_capitalized': remove_chars(SPIRV_PARSER_FILE.upper(), '_'), 228 'verb': 'parse', 229 'helper_functions': '', 230 'prototype_list': ''.join(self.instruction_parser_prototypes), 231 'function_list': ''.join(self.instruction_parser_impl) 232 } 233 234 with (open(self.path_prefix + SPIRV_BUILDER_FILE + '_autogen.h', 'w')) as f: 235 f.write(HEADER_TEMPLATE.format(**builder_template_args)) 236 237 with (open(self.path_prefix + SPIRV_BUILDER_FILE + '_autogen.cpp', 'w')) as f: 238 f.write(SOURCE_TEMPLATE.format(**builder_template_args)) 239 240 with (open(self.path_prefix + SPIRV_PARSER_FILE + '_autogen.h', 'w')) as f: 241 f.write(HEADER_TEMPLATE.format(**parser_template_args)) 242 243 with (open(self.path_prefix + SPIRV_PARSER_FILE + '_autogen.cpp', 'w')) as f: 244 f.write(SOURCE_TEMPLATE.format(**parser_template_args)) 245 246 def requires_unsupported_capability(self, item): 247 depends = item.get('capabilities', []) 248 if len(depends) == 0: 249 return False 250 return all([dep in self.unsupported_capabilities for dep in depends]) 251 252 def requires_unsupported_extension(self, item): 253 extensions = item.get('extensions', []) 254 return any([ext not in self.supported_extensions for ext in extensions]) 255 256 def accumulate_unsupported_capabilities(self): 257 operand_kinds = self.grammar['operand_kinds'] 258 259 # Find the Capability enum 260 for kind in filter(lambda entry: entry['kind'] == 'Capability', operand_kinds): 261 capabilities = kind['enumerants'] 262 for capability in capabilities: 263 name = capability['enumerant'] 264 # For each capability, see if any of the capabilities they depend on is unsupported. 265 # If so, add the capability to the list of unsupported ones. 266 if self.requires_unsupported_capability(capability): 267 self.unsupported_capabilities.add(name) 268 continue 269 # Do the same for extensions 270 if self.requires_unsupported_extension(capability): 271 self.unsupported_capabilities.add(name) 272 273 def find_bit_mask_types(self): 274 operand_kinds = self.grammar['operand_kinds'] 275 276 # Find the BitEnum categories 277 for bitEnumEntry in filter(lambda entry: entry['category'] == 'BitEnum', operand_kinds): 278 self.bit_mask_types.add(bitEnumEntry['kind']) 279 280 def get_operand_name(self, operand): 281 kind = operand['kind'] 282 name = operand.get('name') 283 284 # If no name is given, derive the name from the kind 285 if name is None: 286 assert (kind.find(' ') == -1) 287 return make_camel_case(kind) 288 289 quantifier = operand.get('quantifier', '') 290 name = remove_chars(name, "'") 291 292 # First, a number of special-cases for optional lists 293 if quantifier == '*': 294 suffix = 'List' 295 296 # For IdRefs, change 'Xyz 1', +\n'Xyz 2', +\n...' to xyzList 297 if kind == 'IdRef': 298 if name.find(' ') != -1: 299 name = name[0:name.find(' ')] 300 301 # Otherwise, if it's a pair in the form of 'Xyz, Abc, ...', change it to xyzAbcPairList 302 elif kind.startswith('Pair'): 303 suffix = 'PairList' 304 305 # Otherwise, it's just a list, so change `xyz abc` to `xyzAbcList 306 307 name = remove_chars(name, " ,.") 308 return make_camel_case(name) + suffix 309 310 # Otherwise, remove invalid characters and make the first letter lower case. 311 name = remove_chars(name, " .,+\n~") 312 name = make_camel_case(name) 313 314 # Make sure the name is not a C++ keyword 315 return 'default_' if name == 'default' else name 316 317 def get_operand_namespace(self, kind): 318 return '' if kind in ANGLE_DEFINED_TYPES else 'spv::' 319 320 def get_operand_type_suffix(self, kind): 321 return 'Mask' if kind in self.bit_mask_types else '' 322 323 def get_kind_cpp_type(self, kind): 324 return self.get_operand_namespace(kind) + kind + self.get_operand_type_suffix(kind) 325 326 def get_operand_type_in_and_out(self, operand): 327 kind = operand['kind'] 328 quantifier = operand.get('quantifier', '') 329 330 is_array = quantifier == '*' 331 is_optional = quantifier == '?' 332 cpp_type = self.get_kind_cpp_type(kind) 333 334 if is_array: 335 type_in = 'const ' + cpp_type + 'List &' 336 type_out = cpp_type + 'List *' 337 elif is_optional: 338 type_in = 'const ' + cpp_type + ' *' 339 type_out = cpp_type + ' *' 340 else: 341 type_in = cpp_type 342 type_out = cpp_type + ' *' 343 344 return (type_in, type_out, is_array, is_optional) 345 346 def get_operand_push_back_line(self, operand, operand_name, is_array, is_optional): 347 kind = operand['kind'] 348 pre = '' 349 post = '' 350 accessor = '.' 351 item = operand_name 352 item_dereferenced = item 353 if is_optional: 354 # If optional, surround with an if. 355 pre = 'if (' + operand_name + ') {\n' 356 post = '\n}' 357 accessor = '->' 358 item_dereferenced = '*' + item 359 elif is_array: 360 # If array, surround with a loop. 361 pre = 'for (const auto &operand : ' + operand_name + ') {\n' 362 post = '\n}' 363 item = 'operand' 364 item_dereferenced = item 365 366 # Usually the operand is one uint32_t, but it may be pair. Handle the pairs especially. 367 if kind == 'PairLiteralIntegerIdRef': 368 line = 'blob->push_back(' + item + accessor + 'literal);' 369 line += 'blob->push_back(' + item + accessor + 'id);' 370 elif kind == 'PairIdRefLiteralInteger': 371 line = 'blob->push_back(' + item + accessor + 'id);' 372 line += 'blob->push_back(' + item + accessor + 'literal);' 373 elif kind == 'PairIdRefIdRef': 374 line = 'blob->push_back(' + item + accessor + 'id1);' 375 line += 'blob->push_back(' + item + accessor + 'id2);' 376 elif kind == 'LiteralString': 377 line = '{size_t d = blob->size();' 378 line += 'blob->resize(d + strlen(' + item_dereferenced + ') / 4 + 1, 0);' 379 # We currently don't have any big-endian devices in the list of supported platforms. 380 # Literal strings in SPIR-V are stored little-endian (SPIR-V 1.0 Section 2.2.1, Literal 381 # String), so if a big-endian device is to be supported, the string copy here should 382 # be adjusted. 383 line += 'ASSERT(IsLittleEndian());' 384 line += 'strcpy(reinterpret_cast<char *>(blob->data() + d), ' + item_dereferenced + ');}' 385 else: 386 line = 'blob->push_back(' + item_dereferenced + ');' 387 388 return pre + line + post 389 390 def get_operand_parse_line(self, operand, operand_name, is_array, is_optional): 391 kind = operand['kind'] 392 pre = '' 393 post = '' 394 accessor = '->' 395 396 if is_optional: 397 # If optional, surround with an if, both checking if argument is provided, and whether 398 # it exists. 399 pre = 'if (' + operand_name + ' && _o < _length) {\n' 400 post = '\n}' 401 elif is_array: 402 # If array, surround with an if and a loop. 403 pre = 'if (' + operand_name + ') {\n' 404 pre += 'while (_o < _length) {\n' 405 post = '\n}}' 406 accessor = '.' 407 408 # Usually the operand is one uint32_t, but it may be pair. Handle the pairs especially. 409 if kind == 'PairLiteralIntegerIdRef': 410 if is_array: 411 line = operand_name + '->emplace_back(' + kind + '{LiteralInteger(_instruction[_o]), IdRef(_instruction[_o + 1])});' 412 line += '_o += 2;' 413 else: 414 line = operand_name + '->literal = LiteralInteger(_instruction[_o++]);' 415 line += operand_name + '->id = IdRef(_instruction[_o++]);' 416 elif kind == 'PairIdRefLiteralInteger': 417 if is_array: 418 line = operand_name + '->emplace_back(' + kind + '{IdRef(_instruction[_o]), LiteralInteger(_instruction[_o + 1])});' 419 line += '_o += 2;' 420 else: 421 line = operand_name + '->id = IdRef(_instruction[_o++]);' 422 line += operand_name + '->literal = LiteralInteger(_instruction[_o++]);' 423 elif kind == 'PairIdRefIdRef': 424 if is_array: 425 line = operand_name + '->emplace_back(' + kind + '{IdRef(_instruction[_o]), IdRef(_instruction[_o + 1])});' 426 line += '_o += 2;' 427 else: 428 line = operand_name + '->id1 = IdRef(_instruction[_o++]);' 429 line += operand_name + '->id2 = IdRef(_instruction[_o++]);' 430 elif kind == 'LiteralString': 431 # The length of string in words is ceil((strlen(str) + 1) / 4). This is equal to 432 # (strlen(str)+1+3) / 4, which is equal to strlen(str)/4+1. 433 assert (not is_array) 434 # See handling of LiteralString in get_operand_push_back_line. 435 line = 'ASSERT(IsLittleEndian());' 436 line += '*' + operand_name + ' = reinterpret_cast<const char *>(&_instruction[_o]);' 437 line += '_o += strlen(*' + operand_name + ') / 4 + 1;' 438 else: 439 if is_array: 440 line = operand_name + '->emplace_back(_instruction[_o++]);' 441 else: 442 line = '*' + operand_name + ' = ' + self.get_kind_cpp_type( 443 kind) + '(_instruction[_o++]);' 444 445 return pre + line + post 446 447 def process_operand(self, operand, cpp_operands_in, cpp_operands_out, cpp_in_parse_lines, 448 cpp_out_push_back_lines): 449 operand_name = self.get_operand_name(operand) 450 type_in, type_out, is_array, is_optional = self.get_operand_type_in_and_out(operand) 451 452 # Make the parameter list 453 cpp_operands_in.append(', ' + type_in + ' ' + operand_name) 454 cpp_operands_out.append(', ' + type_out + ' ' + operand_name) 455 456 # Make the builder body lines 457 cpp_out_push_back_lines.append( 458 self.get_operand_push_back_line(operand, operand_name, is_array, is_optional)) 459 460 # Make the parser body lines 461 cpp_in_parse_lines.append( 462 self.get_operand_parse_line(operand, operand_name, is_array, is_optional)) 463 464 def generate_instruction_functions(self, instruction): 465 name = instruction['opname'] 466 assert (name.startswith('Op')) 467 name = name[2:] 468 469 # Skip if the instruction depends on a capability or extension we aren't interested in 470 if self.requires_unsupported_capability(instruction): 471 return 472 if self.requires_unsupported_extension(instruction): 473 return 474 475 operands = instruction.get('operands', []) 476 477 # Skip if any of the operands are not supported 478 if any([operand['kind'] in self.unsupported_kinds for operand in operands]): 479 return 480 481 cpp_operands_in = [] 482 cpp_operands_out = [] 483 cpp_in_parse_lines = [] 484 cpp_out_push_back_lines = [] 485 486 for operand in operands: 487 self.process_operand(operand, cpp_operands_in, cpp_operands_out, cpp_in_parse_lines, 488 cpp_out_push_back_lines) 489 490 # get_operand_parse_line relies on there only being one array parameter, and it being 491 # the last. 492 assert (operand.get('quantifier') != '*' or len(cpp_in_parse_lines) == len(operands)) 493 494 if operand['kind'] == 'Decoration': 495 # Special handling of Op*Decorate instructions with a Decoration operand. That 496 # operand always comes last, and implies a number of LiteralIntegers to follow. 497 assert (len(cpp_in_parse_lines) == len(operands)) 498 499 decoration_operands = { 500 'name': 'values', 501 'kind': 'LiteralInteger', 502 'quantifier': '*' 503 } 504 self.process_operand(decoration_operands, cpp_operands_in, cpp_operands_out, 505 cpp_in_parse_lines, cpp_out_push_back_lines) 506 507 elif operand['kind'] == 'ExecutionMode': 508 # Special handling of OpExecutionMode instruction with an ExecutionMode operand. 509 # That operand always comes last, and implies a number of LiteralIntegers to follow. 510 assert (len(cpp_in_parse_lines) == len(operands)) 511 512 execution_mode_operands = { 513 'name': 'operands', 514 'kind': 'LiteralInteger', 515 'quantifier': '*' 516 } 517 self.process_operand(execution_mode_operands, cpp_operands_in, cpp_operands_out, 518 cpp_in_parse_lines, cpp_out_push_back_lines) 519 520 elif operand['kind'] == 'ImageOperands': 521 # Special handling of OpImage* instructions with an ImageOperands operand. That 522 # operand always comes last, and implies a number of IdRefs to follow with different 523 # meanings based on the bits set in said operand. 524 assert (len(cpp_in_parse_lines) == len(operands)) 525 526 image_operands = {'name': 'imageOperandIds', 'kind': 'IdRef', 'quantifier': '*'} 527 self.process_operand(image_operands, cpp_operands_in, cpp_operands_out, 528 cpp_in_parse_lines, cpp_out_push_back_lines) 529 530 # Make the builder prototype body 531 builder_prototype = TEMPLATE_BUILDER_FUNCTION_PROTOTYPE.format( 532 op=name, param_list=''.join(cpp_operands_in)) 533 self.instruction_builder_prototypes.append(builder_prototype + ';\n') 534 self.instruction_builder_impl.append( 535 builder_prototype + '\n' + TEMPLATE_BUILDER_FUNCTION_BODY.format( 536 op=name, push_back_lines='\n'.join(cpp_out_push_back_lines))) 537 538 if len(operands) == 0: 539 return 540 541 parser_prototype = TEMPLATE_PARSER_FUNCTION_PROTOTYPE.format( 542 op=name, param_list=''.join(cpp_operands_out)) 543 self.instruction_parser_prototypes.append(parser_prototype + ';\n') 544 self.instruction_parser_impl.append( 545 parser_prototype + '\n' + TEMPLATE_PARSER_FUNCTION_BODY.format( 546 op=name, parse_lines='\n'.join(cpp_in_parse_lines))) 547 548 549def main(): 550 551 # auto_script parameters. 552 if len(sys.argv) > 1: 553 if sys.argv[1] == 'inputs': 554 print(SPIRV_GRAMMAR_FILE) 555 elif sys.argv[1] == 'outputs': 556 output_files_base = [SPIRV_BUILDER_FILE, SPIRV_PARSER_FILE] 557 output_files = [ 558 '_autogen.'.join(pair) 559 for pair in itertools.product(output_files_base, ['h', 'cpp']) 560 ] 561 print(','.join(output_files)) 562 else: 563 print('Invalid script parameters') 564 return 1 565 return 0 566 567 writer = Writer() 568 writer.write_builder_and_parser() 569 570 return 0 571 572 573if __name__ == '__main__': 574 sys.exit(main()) 575