1#!/usr/bin/env python3 2# Copyright (c) 2015-2019 The Khronos Group Inc. 3# Copyright (c) 2015-2019 Valve Corporation 4# Copyright (c) 2015-2019 LunarG, Inc. 5# Copyright (c) 2015-2019 Google Inc. 6# 7# Licensed under the Apache License, Version 2.0 (the "License"); 8# you may not use this file except in compliance with the License. 9# You may obtain a copy of the License at 10# 11# http://www.apache.org/licenses/LICENSE-2.0 12# 13# Unless required by applicable law or agreed to in writing, software 14# distributed under the License is distributed on an "AS IS" BASIS, 15# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16# See the License for the specific language governing permissions and 17# limitations under the License. 18# 19# Author: Tobin Ehlis <tobine@google.com> 20# Author: Dave Houlton <daveh@lunarg.com> 21# Author: Shannon McPherson <shannon@lunarg.com> 22 23import argparse 24import os 25import sys 26import operator 27import platform 28import json 29import re 30import csv 31import html 32import time 33from collections import defaultdict 34 35verbose_mode = False 36txt_db = False 37csv_db = False 38html_db = False 39txt_filename = "validation_error_database.txt" 40csv_filename = "validation_error_database.csv" 41html_filename = "validation_error_database.html" 42header_filename = "../layers/vk_validation_error_messages.h" 43test_file = '../tests/layer_validation_tests.cpp' 44vuid_prefixes = ['VUID-', 'UNASSIGNED-'] 45 46# Hard-coded flags that could be command line args, if we decide that's useful 47# replace KHR vuids with non-KHR during consistency checking 48dealias_khr = True 49ignore_unassigned = True # These are not found in layer code unless they appear explicitly (most don't), so produce false positives 50 51generated_layer_source_directories = [ 52'build', 53'dbuild', 54'release', 55'../build/Vulkan-ValidationLayers/' 56] 57generated_layer_source_files = [ 58'parameter_validation.cpp', 59'object_tracker.cpp', 60] 61layer_source_files = [ 62'../layers/buffer_validation.cpp', 63'../layers/core_validation.cpp', 64'../layers/descriptor_sets.cpp', 65'../layers/parameter_validation_utils.cpp', 66'../layers/object_tracker_utils.cpp', 67'../layers/shader_validation.cpp', 68'../layers/stateless_validation.h' 69] 70 71# This needs to be updated as new extensions roll in 72khr_aliases = { 73 'VUID-vkBindBufferMemory2KHR-device-parameter' : 'VUID-vkBindBufferMemory2-device-parameter', 74 'VUID-vkBindBufferMemory2KHR-pBindInfos-parameter' : 'VUID-vkBindBufferMemory2-pBindInfos-parameter', 75 'VUID-vkBindImageMemory2KHR-device-parameter' : 'VUID-vkBindImageMemory2-device-parameter', 76 'VUID-vkBindImageMemory2KHR-pBindInfos-parameter' : 'VUID-vkBindImageMemory2-pBindInfos-parameter', 77 'VUID-vkCmdDispatchBaseKHR-commandBuffer-parameter' : 'VUID-vkCmdDispatchBase-commandBuffer-parameter', 78 'VUID-vkCmdSetDeviceMaskKHR-commandBuffer-parameter' : 'VUID-vkCmdSetDeviceMask-commandBuffer-parameter', 79 'VUID-vkCreateDescriptorUpdateTemplateKHR-device-parameter' : 'VUID-vkCreateDescriptorUpdateTemplate-device-parameter', 80 'VUID-vkCreateDescriptorUpdateTemplateKHR-pDescriptorUpdateTemplate-parameter' : 'VUID-vkCreateDescriptorUpdateTemplate-pDescriptorUpdateTemplate-parameter', 81 'VUID-vkCreateSamplerYcbcrConversionKHR-device-parameter' : 'VUID-vkCreateSamplerYcbcrConversion-device-parameter', 82 'VUID-vkCreateSamplerYcbcrConversionKHR-pYcbcrConversion-parameter' : 'VUID-vkCreateSamplerYcbcrConversion-pYcbcrConversion-parameter', 83 'VUID-vkDestroyDescriptorUpdateTemplateKHR-descriptorUpdateTemplate-parameter' : 'VUID-vkDestroyDescriptorUpdateTemplate-descriptorUpdateTemplate-parameter', 84 'VUID-vkDestroyDescriptorUpdateTemplateKHR-descriptorUpdateTemplate-parent' : 'VUID-vkDestroyDescriptorUpdateTemplate-descriptorUpdateTemplate-parent', 85 'VUID-vkDestroyDescriptorUpdateTemplateKHR-device-parameter' : 'VUID-vkDestroyDescriptorUpdateTemplate-device-parameter', 86 'VUID-vkDestroySamplerYcbcrConversionKHR-device-parameter' : 'VUID-vkDestroySamplerYcbcrConversion-device-parameter', 87 'VUID-vkDestroySamplerYcbcrConversionKHR-ycbcrConversion-parameter' : 'VUID-vkDestroySamplerYcbcrConversion-ycbcrConversion-parameter', 88 'VUID-vkDestroySamplerYcbcrConversionKHR-ycbcrConversion-parent' : 'VUID-vkDestroySamplerYcbcrConversion-ycbcrConversion-parent', 89 'VUID-vkEnumeratePhysicalDeviceGroupsKHR-instance-parameter' : 'VUID-vkEnumeratePhysicalDeviceGroups-instance-parameter', 90 'VUID-vkEnumeratePhysicalDeviceGroupsKHR-pPhysicalDeviceGroupProperties-parameter' : 'VUID-vkEnumeratePhysicalDeviceGroups-pPhysicalDeviceGroupProperties-parameter', 91 'VUID-vkGetBufferMemoryRequirements2KHR-device-parameter' : 'VUID-vkGetBufferMemoryRequirements2-device-parameter', 92 'VUID-vkGetDescriptorSetLayoutSupportKHR-device-parameter' : 'VUID-vkGetDescriptorSetLayoutSupport-device-parameter', 93 'VUID-vkGetDeviceGroupPeerMemoryFeaturesKHR-device-parameter' : 'VUID-vkGetDeviceGroupPeerMemoryFeatures-device-parameter', 94 'VUID-vkGetDeviceGroupPeerMemoryFeaturesKHR-pPeerMemoryFeatures-parameter' : 'VUID-vkGetDeviceGroupPeerMemoryFeatures-pPeerMemoryFeatures-parameter', 95 'VUID-vkGetImageMemoryRequirements2KHR-device-parameter' : 'VUID-vkGetImageMemoryRequirements2-device-parameter', 96 'VUID-vkGetImageSparseMemoryRequirements2KHR-device-parameter' : 'VUID-vkGetImageSparseMemoryRequirements2-device-parameter', 97 'VUID-vkGetImageSparseMemoryRequirements2KHR-pSparseMemoryRequirements-parameter' : 'VUID-vkGetImageSparseMemoryRequirements2-pSparseMemoryRequirements-parameter', 98 'VUID-vkGetPhysicalDeviceExternalBufferPropertiesKHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceExternalBufferProperties-physicalDevice-parameter', 99 'VUID-vkGetPhysicalDeviceExternalFencePropertiesKHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceExternalFenceProperties-physicalDevice-parameter', 100 'VUID-vkGetPhysicalDeviceExternalSemaphorePropertiesKHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceExternalSemaphoreProperties-physicalDevice-parameter', 101 'VUID-vkGetPhysicalDeviceFeatures2KHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceFeatures2-physicalDevice-parameter', 102 'VUID-vkGetPhysicalDeviceFormatProperties2KHR-format-parameter' : 'VUID-vkGetPhysicalDeviceFormatProperties2-format-parameter', 103 'VUID-vkGetPhysicalDeviceFormatProperties2KHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceFormatProperties2-physicalDevice-parameter', 104 'VUID-vkGetPhysicalDeviceImageFormatProperties2KHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceImageFormatProperties2-physicalDevice-parameter', 105 'VUID-vkGetPhysicalDeviceMemoryProperties2KHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceMemoryProperties2-physicalDevice-parameter', 106 'VUID-vkGetPhysicalDeviceProperties2KHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceProperties2-physicalDevice-parameter', 107 'VUID-vkGetPhysicalDeviceQueueFamilyProperties2KHR-pQueueFamilyProperties-parameter' : 'VUID-vkGetPhysicalDeviceQueueFamilyProperties2-pQueueFamilyProperties-parameter', 108 'VUID-vkGetPhysicalDeviceSparseImageFormatProperties2KHR-pProperties-parameter' : 'VUID-vkGetPhysicalDeviceSparseImageFormatProperties2-pProperties-parameter', 109 'VUID-vkGetPhysicalDeviceSparseImageFormatProperties2KHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceSparseImageFormatProperties2-physicalDevice-parameter', 110 'VUID-vkTrimCommandPoolKHR-commandPool-parameter' : 'VUID-vkTrimCommandPool-commandPool-parameter', 111 'VUID-vkTrimCommandPoolKHR-commandPool-parent' : 'VUID-vkTrimCommandPool-commandPool-parent', 112 'VUID-vkTrimCommandPoolKHR-device-parameter' : 'VUID-vkTrimCommandPool-device-parameter', 113 'VUID-vkTrimCommandPoolKHR-flags-zerobitmask' : 'VUID-vkTrimCommandPool-flags-zerobitmask', 114 'VUID-vkUpdateDescriptorSetWithTemplateKHR-descriptorSet-parameter' : 'VUID-vkUpdateDescriptorSetWithTemplate-descriptorSet-parameter', 115 'VUID-vkUpdateDescriptorSetWithTemplateKHR-descriptorUpdateTemplate-parameter' : 'VUID-vkUpdateDescriptorSetWithTemplate-descriptorUpdateTemplate-parameter', 116 'VUID-vkUpdateDescriptorSetWithTemplateKHR-descriptorUpdateTemplate-parent' : 'VUID-vkUpdateDescriptorSetWithTemplate-descriptorUpdateTemplate-parent', 117 'VUID-vkUpdateDescriptorSetWithTemplateKHR-device-parameter' : 'VUID-vkUpdateDescriptorSetWithTemplate-device-parameter', 118 'VUID-vkCreateDescriptorUpdateTemplateKHR-pCreateInfo-parameter' : 'VUID-vkCreateDescriptorUpdateTemplate-pCreateInfo-parameter', 119 'VUID-vkCreateSamplerYcbcrConversionKHR-pCreateInfo-parameter' : 'VUID-vkCreateSamplerYcbcrConversion-pCreateInfo-parameter', 120 'VUID-vkGetBufferMemoryRequirements2KHR-pInfo-parameter' : 'VUID-vkGetBufferMemoryRequirements2-pInfo-parameter', 121 'VUID-vkGetBufferMemoryRequirements2KHR-pMemoryRequirements-parameter' : 'VUID-vkGetBufferMemoryRequirements2-pMemoryRequirements-parameter', 122 'VUID-vkGetDescriptorSetLayoutSupportKHR-pCreateInfo-parameter' : 'VUID-vkGetDescriptorSetLayoutSupport-pCreateInfo-parameter', 123 'VUID-vkGetDescriptorSetLayoutSupportKHR-pSupport-parameter' : 'VUID-vkGetDescriptorSetLayoutSupport-pSupport-parameter', 124 'VUID-vkGetImageMemoryRequirements2KHR-pInfo-parameter' : 'VUID-vkGetImageMemoryRequirements2-pInfo-parameter', 125 'VUID-vkGetImageMemoryRequirements2KHR-pMemoryRequirements-parameter' : 'VUID-vkGetImageMemoryRequirements2-pMemoryRequirements-parameter', 126 'VUID-vkGetImageSparseMemoryRequirements2KHR-pInfo-parameter' : 'VUID-vkGetImageSparseMemoryRequirements2-pInfo-parameter', 127 'VUID-vkGetPhysicalDeviceExternalBufferPropertiesKHR-pExternalBufferInfo-parameter' : 'VUID-vkGetPhysicalDeviceExternalBufferProperties-pExternalBufferInfo-parameter', 128 'VUID-vkGetPhysicalDeviceExternalBufferPropertiesKHR-pExternalBufferProperties-parameter' : 'VUID-vkGetPhysicalDeviceExternalBufferProperties-pExternalBufferProperties-parameter', 129 'VUID-vkGetPhysicalDeviceExternalFencePropertiesKHR-pExternalFenceInfo-parameter' : 'VUID-vkGetPhysicalDeviceExternalFenceProperties-pExternalFenceInfo-parameter', 130 'VUID-vkGetPhysicalDeviceExternalFencePropertiesKHR-pExternalFenceProperties-parameter' : 'VUID-vkGetPhysicalDeviceExternalFenceProperties-pExternalFenceProperties-parameter', 131 'VUID-vkGetPhysicalDeviceExternalSemaphorePropertiesKHR-pExternalSemaphoreInfo-parameter' : 'VUID-vkGetPhysicalDeviceExternalSemaphoreProperties-pExternalSemaphoreInfo-parameter', 132 'VUID-vkGetPhysicalDeviceExternalSemaphorePropertiesKHR-pExternalSemaphoreProperties-parameter' : 'VUID-vkGetPhysicalDeviceExternalSemaphoreProperties-pExternalSemaphoreProperties-parameter', 133 'VUID-vkGetPhysicalDeviceFeatures2KHR-pFeatures-parameter' : 'VUID-vkGetPhysicalDeviceFeatures2-pFeatures-parameter', 134 'VUID-vkGetPhysicalDeviceFormatProperties2KHR-pFormatProperties-parameter' : 'VUID-vkGetPhysicalDeviceFormatProperties2-pFormatProperties-parameter', 135 'VUID-vkGetPhysicalDeviceImageFormatProperties2KHR-pImageFormatInfo-parameter' : 'VUID-vkGetPhysicalDeviceImageFormatProperties2-pImageFormatInfo-parameter', 136 'VUID-vkGetPhysicalDeviceImageFormatProperties2KHR-pImageFormatProperties-parameter' : 'VUID-vkGetPhysicalDeviceImageFormatProperties2-pImageFormatProperties-parameter', 137 'VUID-vkGetPhysicalDeviceMemoryProperties2KHR-pMemoryProperties-parameter' : 'VUID-vkGetPhysicalDeviceMemoryProperties2-pMemoryProperties-parameter', 138 'VUID-vkGetPhysicalDeviceProperties2KHR-pProperties-parameter' : 'VUID-vkGetPhysicalDeviceProperties2-pProperties-parameter', 139 'VUID-vkGetPhysicalDeviceSparseImageFormatProperties2KHR-pFormatInfo-parameter' : 'VUID-vkGetPhysicalDeviceSparseImageFormatProperties2-pFormatInfo-parameter' } 140 141def printHelp(): 142 print ("Usage:") 143 print (" python vk_validation_stats.py <json_file>") 144 print (" [ -c ]") 145 print (" [ -todo ]") 146 print (" [ -vuid <vuid_name> ]") 147 print (" [ -text [ <text_out_filename>] ]") 148 print (" [ -csv [ <csv_out_filename>] ]") 149 print (" [ -html [ <html_out_filename>] ]") 150 print (" [ -export_header ]") 151 print (" [ -verbose ]") 152 print (" [ -help ]") 153 print ("\n The vk_validation_stats script parses validation layer source files to") 154 print (" determine the set of valid usage checks and tests currently implemented,") 155 print (" and generates coverage values by comparing against the full set of valid") 156 print (" usage identifiers in the Vulkan-Headers registry file 'validusage.json'") 157 print ("\nArguments: ") 158 print (" <json-file> (required) registry file 'validusage.json'") 159 print (" -c report consistency warnings") 160 print (" -todo report unimplemented VUIDs") 161 print (" -vuid <vuid_name> report status of individual VUID <vuid_name>") 162 print (" -text [filename] output the error database text to <text_database_filename>,") 163 print (" defaults to 'validation_error_database.txt'") 164 print (" -csv [filename] output the error database in csv to <csv_database_filename>,") 165 print (" defaults to 'validation_error_database.csv'") 166 print (" -html [filename] output the error database in html to <html_database_filename>,") 167 print (" defaults to 'validation_error_database.html'") 168 print (" -export_header export a new VUID error text header file to <%s>" % header_filename) 169 print (" -verbose show your work (to stdout)") 170 171class ValidationJSON: 172 def __init__(self, filename): 173 self.filename = filename 174 self.explicit_vuids = set() 175 self.implicit_vuids = set() 176 self.all_vuids = set() 177 self.vuid_db = defaultdict(list) # Maps VUID string to list of json-data dicts 178 self.apiversion = "" 179 self.duplicate_vuids = set() 180 181 # A set of specific regular expression substitutions needed to clean up VUID text 182 self.regex_dict = {} 183 self.regex_dict[re.compile('<.*?>|&(amp;)+lt;|&(amp;)+gt;')] = "" 184 self.regex_dict[re.compile(r'\\\(codeSize \\over 4\\\)')] = "(codeSize/4)" 185 self.regex_dict[re.compile(r'\\\(\\lceil{\\frac{height}{maxFragmentDensityTexelSize_{height}}}\\rceil\\\)')] = "the ceiling of height/maxFragmentDensityTexelSize.height" 186 self.regex_dict[re.compile(r'\\\(\\lceil{\\frac{width}{maxFragmentDensityTexelSize_{width}}}\\rceil\\\)')] = "the ceiling of width/maxFragmentDensityTexelSize.width" 187 self.regex_dict[re.compile(r'\\\(\\lceil{\\frac{maxFramebufferHeight}{minFragmentDensityTexelSize_{height}}}\\rceil\\\)')] = "the ceiling of maxFramebufferHeight/minFragmentDensityTexelSize.height" 188 self.regex_dict[re.compile(r'\\\(\\lceil{\\frac{maxFramebufferWidth}{minFragmentDensityTexelSize_{width}}}\\rceil\\\)')] = "the ceiling of maxFramebufferWidth/minFragmentDensityTexelSize.width" 189 self.regex_dict[re.compile(r'\\\(\\lceil\{\\mathit\{rasterizationSamples} \\over 32}\\rceil\\\)')] = "(rasterizationSamples/32)" 190 self.regex_dict[re.compile(r'\\\(\\textrm\{codeSize} \\over 4\\\)')] = "(codeSize/4)" 191 # Some fancy punctuation chars that break the Android build... 192 self.regex_dict[re.compile('→')] = "->" # Arrow char 193 self.regex_dict[re.compile('’')] = "'" # Left-slanting apostrophe to apostrophe 194 self.regex_dict[re.compile('̶(0|1);')] = "'" # L/R-slanting quotes to apostrophe 195 196 def read(self): 197 self.json_dict = {} 198 if os.path.isfile(self.filename): 199 json_file = open(self.filename, 'r', encoding='utf-8') 200 self.json_dict = json.load(json_file) 201 json_file.close() 202 if len(self.json_dict) == 0: 203 print("Error: Error loading validusage.json file <%s>" % self.filename) 204 sys.exit(-1) 205 try: 206 version = self.json_dict['version info'] 207 validation = self.json_dict['validation'] 208 self.apiversion = version['api version'] 209 except: 210 print("Error: Failure parsing validusage.json object") 211 sys.exit(-1) 212 213 # Parse vuid from json into local databases 214 for apiname in validation.keys(): 215 # print("entrypoint:%s"%apiname) 216 apidict = validation[apiname] 217 for ext in apidict.keys(): 218 vlist = apidict[ext] 219 for ventry in vlist: 220 vuid_string = ventry['vuid'] 221 if (vuid_string[-5:-1].isdecimal()): 222 self.explicit_vuids.add(vuid_string) # explicit end in 5 numeric chars 223 vtype = 'explicit' 224 else: 225 self.implicit_vuids.add(vuid_string) # otherwise, implicit 226 vtype = 'implicit' 227 vuid_text = ventry['text'] 228 for regex, replacement in self.regex_dict.items(): 229 vuid_text = re.sub(regex, replacement, vuid_text) # do regex substitution 230 vuid_text = html.unescape(vuid_text) # anything missed by the regex 231 self.vuid_db[vuid_string].append({'api':apiname, 'ext':ext, 'type':vtype, 'text':vuid_text}) 232 self.all_vuids = self.explicit_vuids | self.implicit_vuids 233 self.duplicate_vuids = set({v for v in self.vuid_db if len(self.vuid_db[v]) > 1}) 234 if len(self.duplicate_vuids) > 0: 235 print("Warning: duplicate VUIDs found in validusage.json") 236 237 238class ValidationSource: 239 def __init__(self, source_file_list, generated_source_file_list, generated_source_directories): 240 self.source_files = source_file_list 241 self.generated_source_files = generated_source_file_list 242 self.generated_source_dirs = generated_source_directories 243 self.vuid_count_dict = {} # dict of vuid values to the count of how much they're used, and location of where they're used 244 self.duplicated_checks = 0 245 self.explicit_vuids = set() 246 self.implicit_vuids = set() 247 self.unassigned_vuids = set() 248 self.all_vuids = set() 249 250 if len(self.generated_source_files) > 0: 251 qualified_paths = [] 252 for source in self.generated_source_files: 253 for build_dir in self.generated_source_dirs: 254 filepath = '../%s/layers/%s' % (build_dir, source) 255 if os.path.isfile(filepath): 256 qualified_paths.append(filepath) 257 break 258 if len(self.generated_source_files) != len(qualified_paths): 259 print("Error: Unable to locate one or more of the following source files in the %s directories" % (", ".join(generated_source_directories))) 260 print(self.generated_source_files) 261 print("Failed to locate one or more codegen files in layer source code - cannot proceed.") 262 exit(1) 263 else: 264 self.source_files.extend(qualified_paths) 265 266 def parse(self): 267 prepend = None 268 for sf in self.source_files: 269 line_num = 0 270 with open(sf) as f: 271 for line in f: 272 line_num = line_num + 1 273 if True in [line.strip().startswith(comment) for comment in ['//', '/*']]: 274 continue 275 # Find vuid strings 276 if prepend is not None: 277 line = prepend[:-2] + line.lstrip().lstrip('"') # join lines skipping CR, whitespace and trailing/leading quote char 278 prepend = None 279 if any(prefix in line for prefix in vuid_prefixes): 280 line_list = line.split() 281 282 # A VUID string that has been broken by clang will start with a vuid prefix and end with -, and will be last in the list 283 broken_vuid = line_list[-1].strip('"') 284 if any(broken_vuid.startswith(prefix) for prefix in vuid_prefixes) and broken_vuid.endswith('-'): 285 prepend = line 286 continue 287 288 vuid_list = [] 289 for str in line_list: 290 if any(prefix in str for prefix in vuid_prefixes): 291 vuid_list.append(str.strip(',);{}"')) 292 for vuid in vuid_list: 293 if vuid not in self.vuid_count_dict: 294 self.vuid_count_dict[vuid] = {} 295 self.vuid_count_dict[vuid]['count'] = 1 296 self.vuid_count_dict[vuid]['file_line'] = [] 297 else: 298 if self.vuid_count_dict[vuid]['count'] == 1: # only count first time duplicated 299 self.duplicated_checks = self.duplicated_checks + 1 300 self.vuid_count_dict[vuid]['count'] = self.vuid_count_dict[vuid]['count'] + 1 301 self.vuid_count_dict[vuid]['file_line'].append('%s,%d' % (sf, line_num)) 302 # Sort vuids by type 303 for vuid in self.vuid_count_dict.keys(): 304 if (vuid.startswith('VUID-')): 305 if (vuid[-5:-1].isdecimal()): 306 self.explicit_vuids.add(vuid) # explicit end in 5 numeric chars 307 else: 308 self.implicit_vuids.add(vuid) 309 elif (vuid.startswith('UNASSIGNED-')): 310 self.unassigned_vuids.add(vuid) 311 else: 312 print("Unable to categorize VUID: %s" % vuid) 313 print("Confused while parsing VUIDs in layer source code - cannot proceed. (FIXME)") 314 exit(-1) 315 self.all_vuids = self.explicit_vuids | self.implicit_vuids | self.unassigned_vuids 316 317# Class to parse the validation layer test source and store testnames 318class ValidationTests: 319 def __init__(self, test_file_list, test_group_name=['VkLayerTest', 'VkPositiveLayerTest', 'VkWsiEnabledLayerTest']): 320 self.test_files = test_file_list 321 self.test_trigger_txt_list = [] 322 for tg in test_group_name: 323 self.test_trigger_txt_list.append('TEST_F(%s' % tg) 324 self.explicit_vuids = set() 325 self.implicit_vuids = set() 326 self.unassigned_vuids = set() 327 self.all_vuids = set() 328 #self.test_to_vuids = {} # Map test name to VUIDs tested 329 self.vuid_to_tests = defaultdict(set) # Map VUIDs to set of test names where implemented 330 331 # Parse test files into internal data struct 332 def parse(self): 333 # For each test file, parse test names into set 334 grab_next_line = False # handle testname on separate line than wildcard 335 testname = '' 336 prepend = None 337 for test_file in self.test_files: 338 with open(test_file) as tf: 339 for line in tf: 340 if True in [line.strip().startswith(comment) for comment in ['//', '/*']]: 341 continue 342 343 # if line ends in a broken VUID string, fix that before proceeding 344 if prepend is not None: 345 line = prepend[:-2] + line.lstrip().lstrip('"') # join lines skipping CR, whitespace and trailing/leading quote char 346 prepend = None 347 if any(prefix in line for prefix in vuid_prefixes): 348 line_list = line.split() 349 350 # A VUID string that has been broken by clang will start with a vuid prefix and end with -, and will be last in the list 351 broken_vuid = line_list[-1].strip('"') 352 if any(broken_vuid.startswith(prefix) for prefix in vuid_prefixes) and broken_vuid.endswith('-'): 353 prepend = line 354 continue 355 356 if any(ttt in line for ttt in self.test_trigger_txt_list): 357 testname = line.split(',')[-1] 358 testname = testname.strip().strip(' {)') 359 if ('' == testname): 360 grab_next_line = True 361 continue 362 #self.test_to_vuids[testname] = [] 363 if grab_next_line: # test name on its own line 364 grab_next_line = False 365 testname = testname.strip().strip(' {)') 366 #self.test_to_vuids[testname] = [] 367 if any(prefix in line for prefix in vuid_prefixes): 368 line_list = re.split('[\s{}[\]()"]+',line) 369 for sub_str in line_list: 370 if any(prefix in sub_str for prefix in vuid_prefixes): 371 vuid_str = sub_str.strip(',);:"') 372 self.vuid_to_tests[vuid_str].add(testname) 373 #self.test_to_vuids[testname].append(vuid_str) 374 if (vuid_str.startswith('VUID-')): 375 if (vuid_str[-5:-1].isdecimal()): 376 self.explicit_vuids.add(vuid_str) # explicit end in 5 numeric chars 377 else: 378 self.implicit_vuids.add(vuid_str) 379 elif (vuid_str.startswith('UNASSIGNED-')): 380 self.unassigned_vuids.add(vuid_str) 381 else: 382 print("Unable to categorize VUID: %s" % vuid_str) 383 print("Confused while parsing VUIDs in test code - cannot proceed. (FIXME)") 384 exit(-1) 385 self.all_vuids = self.explicit_vuids | self.implicit_vuids | self.unassigned_vuids 386 387# Class to do consistency checking 388# 389class Consistency: 390 def __init__(self, all_json, all_checks, all_tests): 391 self.valid = all_json 392 self.checks = all_checks 393 self.tests = all_tests 394 395 if (dealias_khr): 396 dk = set() 397 for vuid in self.checks: 398 if vuid in khr_aliases: 399 dk.add(khr_aliases[vuid]) 400 else: 401 dk.add(vuid) 402 self.checks = dk 403 404 dk = set() 405 for vuid in self.tests: 406 if vuid in khr_aliases: 407 dk.add(khr_aliases[vuid]) 408 else: 409 dk.add(vuid) 410 self.tests = dk 411 412 # Report undefined VUIDs in source code 413 def undef_vuids_in_layer_code(self): 414 undef_set = self.checks - self.valid 415 undef_set.discard('VUID-Undefined') # don't report Undefined 416 if ignore_unassigned: 417 unassigned = set({uv for uv in undef_set if uv.startswith('UNASSIGNED-')}) 418 undef_set = undef_set - unassigned 419 if (len(undef_set) > 0): 420 print("\nFollowing VUIDs found in layer code are not defined in validusage.json (%d):" % len(undef_set)) 421 undef = list(undef_set) 422 undef.sort() 423 for vuid in undef: 424 print(" %s" % vuid) 425 return False 426 return True 427 428 # Report undefined VUIDs in tests 429 def undef_vuids_in_tests(self): 430 undef_set = self.tests - self.valid 431 undef_set.discard('VUID-Undefined') # don't report Undefined 432 if ignore_unassigned: 433 unassigned = set({uv for uv in undef_set if uv.startswith('UNASSIGNED-')}) 434 undef_set = undef_set - unassigned 435 if (len(undef_set) > 0): 436 ok = False 437 print("\nFollowing VUIDs found in layer tests are not defined in validusage.json (%d):" % len(undef_set)) 438 undef = list(undef_set) 439 undef.sort() 440 for vuid in undef: 441 print(" %s" % vuid) 442 return False 443 return True 444 445 # Report vuids in tests that are not in source 446 def vuids_tested_not_checked(self): 447 undef_set = self.tests - self.checks 448 undef_set.discard('VUID-Undefined') # don't report Undefined 449 if ignore_unassigned: 450 unassigned = set() 451 for vuid in undef_set: 452 if vuid.startswith('UNASSIGNED-'): 453 unassigned.add(vuid) 454 undef_set = undef_set - unassigned 455 if (len(undef_set) > 0): 456 ok = False 457 print("\nFollowing VUIDs found in tests but are not checked in layer code (%d):" % len(undef_set)) 458 undef = list(undef_set) 459 undef.sort() 460 for vuid in undef: 461 print(" %s" % vuid) 462 return False 463 return True 464 465 # TODO: Explicit checked VUIDs which have no test 466 # def explicit_vuids_checked_not_tested(self): 467 468 469# Class to output database in various flavors 470# 471class OutputDatabase: 472 def __init__(self, val_json, val_source, val_tests): 473 self.vj = val_json 474 self.vs = val_source 475 self.vt = val_tests 476 self.header_version = "/* THIS FILE IS GENERATED - DO NOT EDIT (scripts/vk_validation_stats.py) */" 477 self.header_version += "\n/* Vulkan specification version: %s */" % val_json.apiversion 478 self.header_version += "\n/* Header generated: %s */\n" % time.strftime('%Y-%m-%d %H:%M:%S') 479 self.header_preamble = """ 480/* 481 * Vulkan 482 * 483 * Copyright (c) 2016-2019 Google Inc. 484 * Copyright (c) 2016-2019 LunarG, Inc. 485 * 486 * Licensed under the Apache License, Version 2.0 (the "License"); 487 * you may not use this file except in compliance with the License. 488 * You may obtain a copy of the License at 489 * 490 * http://www.apache.org/licenses/LICENSE-2.0 491 * 492 * Unless required by applicable law or agreed to in writing, software 493 * distributed under the License is distributed on an "AS IS" BASIS, 494 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 495 * See the License for the specific language governing permissions and 496 * limitations under the License. 497 * 498 * Author: Tobin Ehlis <tobine@google.com> 499 * Author: Dave Houlton <daveh@lunarg.com> 500 */ 501 502#pragma once 503 504// Disable auto-formatting for generated file 505// clang-format off 506 507// Mapping from VUID string to the corresponding spec text 508typedef struct _vuid_spec_text_pair { 509 const char * vuid; 510 const char * spec_text; 511} vuid_spec_text_pair; 512 513static const vuid_spec_text_pair vuid_spec_text[] = { 514""" 515 self.header_postamble = """}; 516""" 517 self.spec_url = "https://www.khronos.org/registry/vulkan/specs/1.1-extensions/html/vkspec.html" 518 519 def dump_txt(self): 520 print("\n Dumping database to text file: %s" % txt_filename) 521 with open (txt_filename, 'w') as txt: 522 txt.write("## VUID Database\n") 523 txt.write("## Format: VUID_NAME | CHECKED | TEST | TYPE | API/STRUCT | EXTENSION | VUID_TEXT\n##\n") 524 vuid_list = list(self.vj.all_vuids) 525 vuid_list.sort() 526 for vuid in vuid_list: 527 db_list = self.vj.vuid_db[vuid] 528 db_list.sort(key=operator.itemgetter('ext')) # sort list to ease diffs of output file 529 for db_entry in db_list: 530 checked = 'N' 531 if vuid in self.vs.all_vuids: 532 checked = 'Y' 533 test = 'None' 534 if vuid in self.vt.vuid_to_tests: 535 test_list = list(self.vt.vuid_to_tests[vuid]) 536 test_list.sort() # sort tests, for diff-ability 537 sep = ', ' 538 test = sep.join(test_list) 539 540 txt.write("%s | %s | %s | %s | %s | %s | %s\n" % (vuid, checked, test, db_entry['type'], db_entry['api'], db_entry['ext'], db_entry['text'])) 541 542 def dump_csv(self): 543 print("\n Dumping database to csv file: %s" % csv_filename) 544 with open (csv_filename, 'w', newline='') as csvfile: 545 cw = csv.writer(csvfile) 546 cw.writerow(['VUID_NAME','CHECKED','TEST','TYPE','API/STRUCT','EXTENSION','VUID_TEXT']) 547 vuid_list = list(self.vj.all_vuids) 548 vuid_list.sort() 549 for vuid in vuid_list: 550 for db_entry in self.vj.vuid_db[vuid]: 551 row = [vuid] 552 if vuid in self.vs.all_vuids: 553 row.append('Y') 554 else: 555 row.append('N') 556 test = 'None' 557 if vuid in self.vt.vuid_to_tests: 558 sep = ', ' 559 test = sep.join(self.vt.vuid_to_tests[vuid]) 560 row.append(test) 561 row.append(db_entry['type']) 562 row.append(db_entry['api']) 563 row.append(db_entry['ext']) 564 row.append(db_entry['text']) 565 cw.writerow(row) 566 567 def dump_html(self): 568 print("\n Dumping database to html file: %s" % html_filename) 569 preamble = '<!DOCTYPE html>\n<html>\n<head>\n<style>\ntable, th, td {\n border: 1px solid black;\n border-collapse: collapse; \n}\n</style>\n<body>\n<h2>Valid Usage Database</h2>\n<font size="2" face="Arial">\n<table style="width:100%">\n' 570 headers = '<tr><th>VUID NAME</th><th>CHECKED</th><th>TEST</th><th>TYPE</th><th>API/STRUCT</th><th>EXTENSION</th><th>VUID TEXT</th></tr>\n' 571 with open (html_filename, 'w') as hfile: 572 hfile.write(preamble) 573 hfile.write(headers) 574 vuid_list = list(self.vj.all_vuids) 575 vuid_list.sort() 576 for vuid in vuid_list: 577 for db_entry in self.vj.vuid_db[vuid]: 578 hfile.write('<tr><th>%s</th>' % vuid) 579 checked = '<span style="color:red;">N</span>' 580 if vuid in self.vs.all_vuids: 581 checked = '<span style="color:limegreen;">Y</span>' 582 hfile.write('<th>%s</th>' % checked) 583 test = 'None' 584 if vuid in self.vt.vuid_to_tests: 585 sep = ', ' 586 test = sep.join(self.vt.vuid_to_tests[vuid]) 587 hfile.write('<th>%s</th>' % test) 588 hfile.write('<th>%s</th>' % db_entry['type']) 589 hfile.write('<th>%s</th>' % db_entry['api']) 590 hfile.write('<th>%s</th>' % db_entry['ext']) 591 hfile.write('<th>%s</th></tr>\n' % db_entry['text']) 592 hfile.write('</table>\n</body>\n</html>\n') 593 594 def export_header(self): 595 print("\n Exporting header file to: %s" % header_filename) 596 with open (header_filename, 'w') as hfile: 597 hfile.write(self.header_version) 598 hfile.write(self.header_preamble) 599 vuid_list = list(self.vj.all_vuids) 600 vuid_list.sort() 601 for vuid in vuid_list: 602 db_entry = self.vj.vuid_db[vuid][0] 603 hfile.write(' {"%s", "%s (%s#%s)"},\n' % (vuid, db_entry['text'].strip(' '), self.spec_url, vuid)) 604 # For multiply-defined VUIDs, include versions with extension appended 605 if len(self.vj.vuid_db[vuid]) > 1: 606 for db_entry in self.vj.vuid_db[vuid]: 607 hfile.write(' {"%s[%s]", "%s (%s#%s)"},\n' % (vuid, db_entry['ext'].strip(' '), db_entry['text'].strip(' '), self.spec_url, vuid)) 608 hfile.write(self.header_postamble) 609 610def main(argv): 611 global verbose_mode 612 global txt_filename 613 global csv_filename 614 global html_filename 615 616 run_consistency = False 617 report_unimplemented = False 618 get_vuid_status = '' 619 txt_out = False 620 csv_out = False 621 html_out = False 622 header_out = False 623 624 if (1 > len(argv)): 625 printHelp() 626 sys.exit() 627 628 # Parse script args 629 json_filename = argv[0] 630 i = 1 631 while (i < len(argv)): 632 arg = argv[i] 633 i = i + 1 634 if (arg == '-c'): 635 run_consistency = True 636 elif (arg == '-vuid'): 637 get_vuid_status = argv[i] 638 i = i + 1 639 elif (arg == '-todo'): 640 report_unimplemented = True 641 elif (arg == '-text'): 642 txt_out = True 643 # Set filename if supplied, else use default 644 if i < len(argv) and not argv[i].startswith('-'): 645 txt_filename = argv[i] 646 i = i + 1 647 elif (arg == '-csv'): 648 csv_out = True 649 # Set filename if supplied, else use default 650 if i < len(argv) and not argv[i].startswith('-'): 651 csv_filename = argv[i] 652 i = i + 1 653 elif (arg == '-html'): 654 html_out = True 655 # Set filename if supplied, else use default 656 if i < len(argv) and not argv[i].startswith('-'): 657 html_filename = argv[i] 658 i = i + 1 659 elif (arg == '-export_header'): 660 header_out = True 661 elif (arg in ['-verbose']): 662 verbose_mode = True 663 elif (arg in ['-help', '-h']): 664 printHelp() 665 sys.exit() 666 else: 667 print("Unrecognized argument: %s\n" % arg) 668 printHelp() 669 sys.exit() 670 671 result = 0 # Non-zero result indicates an error case 672 673 # Parse validusage json 674 val_json = ValidationJSON(json_filename) 675 val_json.read() 676 exp_json = len(val_json.explicit_vuids) 677 imp_json = len(val_json.implicit_vuids) 678 all_json = len(val_json.all_vuids) 679 if verbose_mode: 680 print("Found %d unique error vuids in validusage.json file." % all_json) 681 print(" %d explicit" % exp_json) 682 print(" %d implicit" % imp_json) 683 if len(val_json.duplicate_vuids) > 0: 684 print("%d VUIDs appear in validusage.json more than once." % len(val_json.duplicate_vuids)) 685 for vuid in val_json.duplicate_vuids: 686 print(" %s" % vuid) 687 for ext in val_json.vuid_db[vuid]: 688 print(" with extension: %s" % ext['ext']) 689 690 # Parse layer source files 691 val_source = ValidationSource(layer_source_files, generated_layer_source_files, generated_layer_source_directories) 692 val_source.parse() 693 exp_checks = len(val_source.explicit_vuids) 694 imp_checks = len(val_source.implicit_vuids) 695 all_checks = len(val_source.vuid_count_dict.keys()) 696 if verbose_mode: 697 print("Found %d unique vuid checks in layer source code." % all_checks) 698 print(" %d explicit" % exp_checks) 699 print(" %d implicit" % imp_checks) 700 print(" %d unassigned" % len(val_source.unassigned_vuids)) 701 print(" %d checks are implemented more that once" % val_source.duplicated_checks) 702 703 # Parse test files 704 val_tests = ValidationTests([test_file, ]) 705 val_tests.parse() 706 exp_tests = len(val_tests.explicit_vuids) 707 imp_tests = len(val_tests.implicit_vuids) 708 all_tests = len(val_tests.all_vuids) 709 if verbose_mode: 710 print("Found %d unique error vuids in test file %s." % (all_tests, test_file)) 711 print(" %d explicit" % exp_tests) 712 print(" %d implicit" % imp_tests) 713 print(" %d unassigned" % len(val_tests.unassigned_vuids)) 714 715 # Process stats 716 print("\nValidation Statistics (using validusage.json version %s)" % val_json.apiversion) 717 print(" VUIDs defined in JSON file: %04d explicit, %04d implicit, %04d total." % (exp_json, imp_json, all_json)) 718 print(" VUIDs checked in layer code: %04d explicit, %04d implicit, %04d total." % (exp_checks, imp_checks, all_checks)) 719 print(" VUIDs tested in layer tests: %04d explicit, %04d implicit, %04d total." % (exp_tests, imp_tests, all_tests)) 720 721 print("\nVUID check coverage") 722 print(" Explicit VUIDs checked: %.1f%% (%d checked vs %d defined)" % ((100.0 * exp_checks / exp_json), exp_checks, exp_json)) 723 print(" Implicit VUIDs checked: %.1f%% (%d checked vs %d defined)" % ((100.0 * imp_checks / imp_json), imp_checks, imp_json)) 724 print(" Overall VUIDs checked: %.1f%% (%d checked vs %d defined)" % ((100.0 * all_checks / all_json), all_checks, all_json)) 725 726 print("\nVUID test coverage") 727 print(" Explicit VUIDs tested: %.1f%% (%d tested vs %d checks)" % ((100.0 * exp_tests / exp_checks), exp_tests, exp_checks)) 728 print(" Implicit VUIDs tested: %.1f%% (%d tested vs %d checks)" % ((100.0 * imp_tests / imp_checks), imp_tests, imp_checks)) 729 print(" Overall VUIDs tested: %.1f%% (%d tested vs %d checks)" % ((100.0 * all_tests / all_checks), all_tests, all_checks)) 730 731 # Report status of a single VUID 732 if len(get_vuid_status) > 1: 733 print("\n\nChecking status of <%s>" % get_vuid_status); 734 if get_vuid_status not in val_json.all_vuids: 735 print(' Not a valid VUID string.') 736 else: 737 if get_vuid_status in val_source.explicit_vuids: 738 print(' Implemented!') 739 line_list = val_source.vuid_count_dict[get_vuid_status]['file_line'] 740 for line in line_list: 741 print(' => %s' % line) 742 elif get_vuid_status in val_source.implicit_vuids: 743 print(' Implemented! (Implicit)') 744 line_list = val_source.vuid_count_dict[get_vuid_status]['file_line'] 745 for line in line_list: 746 print(' => %s' % line) 747 else: 748 print(' Not implemented.') 749 if get_vuid_status in val_tests.all_vuids: 750 print(' Has a test!') 751 test_list = val_tests.vuid_to_tests[get_vuid_status] 752 for test in test_list: 753 print(' => %s' % test) 754 else: 755 print(' Not tested.') 756 757 # Report unimplemented explicit VUIDs 758 if report_unimplemented: 759 unim_explicit = val_json.explicit_vuids - val_source.explicit_vuids 760 print("\n\n%d explicit VUID checks remain unimplemented:" % len(unim_explicit)) 761 ulist = list(unim_explicit) 762 ulist.sort() 763 for vuid in ulist: 764 print(" => %s" % vuid) 765 766 # Consistency tests 767 if run_consistency: 768 print("\n\nRunning consistency tests...") 769 con = Consistency(val_json.all_vuids, val_source.all_vuids, val_tests.all_vuids) 770 ok = con.undef_vuids_in_layer_code() 771 ok &= con.undef_vuids_in_tests() 772 ok &= con.vuids_tested_not_checked() 773 774 if ok: 775 print(" OK! No inconsistencies found.") 776 777 # Output database in requested format(s) 778 db_out = OutputDatabase(val_json, val_source, val_tests) 779 if txt_out: 780 db_out.dump_txt() 781 if csv_out: 782 db_out.dump_csv() 783 if html_out: 784 db_out.dump_html() 785 if header_out: 786 db_out.export_header() 787 return result 788 789if __name__ == "__main__": 790 sys.exit(main(sys.argv[1:])) 791 792