1#!/usr/bin/env python 2# Copyright (c) 2012 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6'''The 'grit build' tool along with integration for this tool with the 7SCons build system. 8''' 9 10import filecmp 11import getopt 12import os 13import shutil 14import sys 15 16from grit import grd_reader 17from grit import util 18from grit.tool import interface 19from grit import shortcuts 20 21 22# It would be cleaner to have each module register itself, but that would 23# require importing all of them on every run of GRIT. 24'''Map from <output> node types to modules under grit.format.''' 25_format_modules = { 26 'android': 'android_xml', 27 'c_format': 'c_format', 28 'chrome_messages_json': 'chrome_messages_json', 29 'data_package': 'data_pack', 30 'js_map_format': 'js_map_format', 31 'rc_all': 'rc', 32 'rc_translateable': 'rc', 33 'rc_nontranslateable': 'rc', 34 'rc_header': 'rc_header', 35 'resource_map_header': 'resource_map', 36 'resource_map_source': 'resource_map', 37 'resource_file_map_source': 'resource_map', 38} 39_format_modules.update( 40 (type, 'policy_templates.template_formatter') for type in 41 [ 'adm', 'admx', 'adml', 'reg', 'doc', 'json', 42 'plist', 'plist_strings', 'ios_plist' ]) 43 44 45def GetFormatter(type): 46 modulename = 'grit.format.' + _format_modules[type] 47 __import__(modulename) 48 module = sys.modules[modulename] 49 try: 50 return module.Format 51 except AttributeError: 52 return module.GetFormatter(type) 53 54 55class RcBuilder(interface.Tool): 56 '''A tool that builds RC files and resource header files for compilation. 57 58Usage: grit build [-o OUTPUTDIR] [-D NAME[=VAL]]* 59 60All output options for this tool are specified in the input file (see 61'grit help' for details on how to specify the input file - it is a global 62option). 63 64Options: 65 66 -a FILE Assert that the given file is an output. There can be 67 multiple "-a" flags listed for multiple outputs. If a "-a" 68 or "--assert-file-list" argument is present, then the list 69 of asserted files must match the output files or the tool 70 will fail. The use-case is for the build system to maintain 71 separate lists of output files and to catch errors if the 72 build system's list and the grit list are out-of-sync. 73 74 --assert-file-list Provide a file listing multiple asserted output files. 75 There is one file name per line. This acts like specifying 76 each file with "-a" on the command line, but without the 77 possibility of running into OS line-length limits for very 78 long lists. 79 80 -o OUTPUTDIR Specify what directory output paths are relative to. 81 Defaults to the current directory. 82 83 -D NAME[=VAL] Specify a C-preprocessor-like define NAME with optional 84 value VAL (defaults to 1) which will be used to control 85 conditional inclusion of resources. 86 87 -E NAME=VALUE Set environment variable NAME to VALUE (within grit). 88 89 -f FIRSTIDSFILE Path to a python file that specifies the first id of 90 value to use for resources. A non-empty value here will 91 override the value specified in the <grit> node's 92 first_ids_file. 93 94 -w WHITELISTFILE Path to a file containing the string names of the 95 resources to include. Anything not listed is dropped. 96 97 -t PLATFORM Specifies the platform the build is targeting; defaults 98 to the value of sys.platform. The value provided via this 99 flag should match what sys.platform would report for your 100 target platform; see grit.node.base.EvaluateCondition. 101 102 -h HEADERFORMAT Custom format string to use for generating rc header files. 103 The string should have two placeholders: {textual_id} 104 and {numeric_id}. E.g. "#define {textual_id} {numeric_id}" 105 Otherwise it will use the default "#define SYMBOL 1234" 106 107Conditional inclusion of resources only affects the output of files which 108control which resources get linked into a binary, e.g. it affects .rc files 109meant for compilation but it does not affect resource header files (that define 110IDs). This helps ensure that values of IDs stay the same, that all messages 111are exported to translation interchange files (e.g. XMB files), etc. 112''' 113 114 def ShortDescription(self): 115 return 'A tool that builds RC files for compilation.' 116 117 def Run(self, opts, args): 118 self.output_directory = '.' 119 first_ids_file = None 120 whitelist_filenames = [] 121 assert_output_files = [] 122 target_platform = None 123 depfile = None 124 depdir = None 125 rc_header_format = None 126 (own_opts, args) = getopt.getopt(args, 'a:o:D:E:f:w:t:h:', 127 ('depdir=','depfile=','assert-file-list=')) 128 for (key, val) in own_opts: 129 if key == '-a': 130 assert_output_files.append(val) 131 elif key == '--assert-file-list': 132 with open(val) as f: 133 assert_output_files += f.read().splitlines() 134 elif key == '-o': 135 self.output_directory = val 136 elif key == '-D': 137 name, val = util.ParseDefine(val) 138 self.defines[name] = val 139 elif key == '-E': 140 (env_name, env_value) = val.split('=', 1) 141 os.environ[env_name] = env_value 142 elif key == '-f': 143 # TODO(joi@chromium.org): Remove this override once change 144 # lands in WebKit.grd to specify the first_ids_file in the 145 # .grd itself. 146 first_ids_file = val 147 elif key == '-w': 148 whitelist_filenames.append(val) 149 elif key == '-t': 150 target_platform = val 151 elif key == '-h': 152 rc_header_format = val 153 elif key == '--depdir': 154 depdir = val 155 elif key == '--depfile': 156 depfile = val 157 158 if len(args): 159 print 'This tool takes no tool-specific arguments.' 160 return 2 161 self.SetOptions(opts) 162 if self.scons_targets: 163 self.VerboseOut('Using SCons targets to identify files to output.\n') 164 else: 165 self.VerboseOut('Output directory: %s (absolute path: %s)\n' % 166 (self.output_directory, 167 os.path.abspath(self.output_directory))) 168 169 if whitelist_filenames: 170 self.whitelist_names = set() 171 for whitelist_filename in whitelist_filenames: 172 self.VerboseOut('Using whitelist: %s\n' % whitelist_filename); 173 whitelist_contents = util.ReadFile(whitelist_filename, util.RAW_TEXT) 174 self.whitelist_names.update(whitelist_contents.strip().split('\n')) 175 176 self.res = grd_reader.Parse(opts.input, 177 debug=opts.extra_verbose, 178 first_ids_file=first_ids_file, 179 defines=self.defines, 180 target_platform=target_platform) 181 # Set an output context so that conditionals can use defines during the 182 # gathering stage; we use a dummy language here since we are not outputting 183 # a specific language. 184 self.res.SetOutputLanguage('en') 185 if rc_header_format: 186 self.res.AssignRcHeaderFormat(rc_header_format) 187 self.res.RunGatherers() 188 self.Process() 189 190 if assert_output_files: 191 if not self.CheckAssertedOutputFiles(assert_output_files): 192 return 2 193 194 if depfile and depdir: 195 self.GenerateDepfile(depfile, depdir) 196 197 return 0 198 199 def __init__(self, defines=None): 200 # Default file-creation function is built-in open(). Only done to allow 201 # overriding by unit test. 202 self.fo_create = open 203 204 # key/value pairs of C-preprocessor like defines that are used for 205 # conditional output of resources 206 self.defines = defines or {} 207 208 # self.res is a fully-populated resource tree if Run() 209 # has been called, otherwise None. 210 self.res = None 211 212 # Set to a list of filenames for the output nodes that are relative 213 # to the current working directory. They are in the same order as the 214 # output nodes in the file. 215 self.scons_targets = None 216 217 # The set of names that are whitelisted to actually be included in the 218 # output. 219 self.whitelist_names = None 220 221 @staticmethod 222 def AddWhitelistTags(start_node, whitelist_names): 223 # Walk the tree of nodes added attributes for the nodes that shouldn't 224 # be written into the target files (skip markers). 225 from grit.node import include 226 from grit.node import message 227 for node in start_node: 228 # Same trick data_pack.py uses to see what nodes actually result in 229 # real items. 230 if (isinstance(node, include.IncludeNode) or 231 isinstance(node, message.MessageNode)): 232 text_ids = node.GetTextualIds() 233 # Mark the item to be skipped if it wasn't in the whitelist. 234 if text_ids and text_ids[0] not in whitelist_names: 235 node.SetWhitelistMarkedAsSkip(True) 236 237 @staticmethod 238 def ProcessNode(node, output_node, outfile): 239 '''Processes a node in-order, calling its formatter before and after 240 recursing to its children. 241 242 Args: 243 node: grit.node.base.Node subclass 244 output_node: grit.node.io.OutputNode 245 outfile: open filehandle 246 ''' 247 base_dir = util.dirname(output_node.GetOutputFilename()) 248 249 formatter = GetFormatter(output_node.GetType()) 250 formatted = formatter(node, output_node.GetLanguage(), output_dir=base_dir) 251 outfile.writelines(formatted) 252 253 254 def Process(self): 255 # Update filenames with those provided by SCons if we're being invoked 256 # from SCons. The list of SCons targets also includes all <structure> 257 # node outputs, but it starts with our output files, in the order they 258 # occur in the .grd 259 if self.scons_targets: 260 assert len(self.scons_targets) >= len(self.res.GetOutputFiles()) 261 outfiles = self.res.GetOutputFiles() 262 for ix in range(len(outfiles)): 263 outfiles[ix].output_filename = os.path.abspath( 264 self.scons_targets[ix]) 265 else: 266 for output in self.res.GetOutputFiles(): 267 output.output_filename = os.path.abspath(os.path.join( 268 self.output_directory, output.GetFilename())) 269 270 # If there are whitelisted names, tag the tree once up front, this way 271 # while looping through the actual output, it is just an attribute check. 272 if self.whitelist_names: 273 self.AddWhitelistTags(self.res, self.whitelist_names) 274 275 for output in self.res.GetOutputFiles(): 276 self.VerboseOut('Creating %s...' % output.GetFilename()) 277 278 # Microsoft's RC compiler can only deal with single-byte or double-byte 279 # files (no UTF-8), so we make all RC files UTF-16 to support all 280 # character sets. 281 if output.GetType() in ('rc_header', 'resource_map_header', 282 'resource_map_source', 'resource_file_map_source'): 283 encoding = 'cp1252' 284 elif output.GetType() in ('android', 'c_format', 'js_map_format', 'plist', 285 'plist_strings', 'doc', 'json'): 286 encoding = 'utf_8' 287 elif output.GetType() in ('chrome_messages_json'): 288 # Chrome Web Store currently expects BOM for UTF-8 files :-( 289 encoding = 'utf-8-sig' 290 else: 291 # TODO(gfeher) modify here to set utf-8 encoding for admx/adml 292 encoding = 'utf_16' 293 294 # Set the context, for conditional inclusion of resources 295 self.res.SetOutputLanguage(output.GetLanguage()) 296 self.res.SetOutputContext(output.GetContext()) 297 self.res.SetDefines(self.defines) 298 299 # Make the output directory if it doesn't exist. 300 self.MakeDirectoriesTo(output.GetOutputFilename()) 301 302 # Write the results to a temporary file and only overwrite the original 303 # if the file changed. This avoids unnecessary rebuilds. 304 outfile = self.fo_create(output.GetOutputFilename() + '.tmp', 'wb') 305 306 if output.GetType() != 'data_package': 307 outfile = util.WrapOutputStream(outfile, encoding) 308 309 # Iterate in-order through entire resource tree, calling formatters on 310 # the entry into a node and on exit out of it. 311 with outfile: 312 self.ProcessNode(self.res, output, outfile) 313 314 # Now copy from the temp file back to the real output, but on Windows, 315 # only if the real output doesn't exist or the contents of the file 316 # changed. This prevents identical headers from being written and .cc 317 # files from recompiling (which is painful on Windows). 318 if not os.path.exists(output.GetOutputFilename()): 319 os.rename(output.GetOutputFilename() + '.tmp', 320 output.GetOutputFilename()) 321 else: 322 # CHROMIUM SPECIFIC CHANGE. 323 # This clashes with gyp + vstudio, which expect the output timestamp 324 # to change on a rebuild, even if nothing has changed. 325 #files_match = filecmp.cmp(output.GetOutputFilename(), 326 # output.GetOutputFilename() + '.tmp') 327 #if (output.GetType() != 'rc_header' or not files_match 328 # or sys.platform != 'win32'): 329 shutil.copy2(output.GetOutputFilename() + '.tmp', 330 output.GetOutputFilename()) 331 os.remove(output.GetOutputFilename() + '.tmp') 332 333 self.VerboseOut(' done.\n') 334 335 # Print warnings if there are any duplicate shortcuts. 336 warnings = shortcuts.GenerateDuplicateShortcutsWarnings( 337 self.res.UberClique(), self.res.GetTcProject()) 338 if warnings: 339 print '\n'.join(warnings) 340 341 # Print out any fallback warnings, and missing translation errors, and 342 # exit with an error code if there are missing translations in a non-pseudo 343 # and non-official build. 344 warnings = (self.res.UberClique().MissingTranslationsReport(). 345 encode('ascii', 'replace')) 346 if warnings: 347 self.VerboseOut(warnings) 348 if self.res.UberClique().HasMissingTranslations(): 349 print self.res.UberClique().missing_translations_ 350 sys.exit(-1) 351 352 353 def CheckAssertedOutputFiles(self, assert_output_files): 354 '''Checks that the asserted output files are specified in the given list. 355 356 Returns true if the asserted files are present. If they are not, returns 357 False and prints the failure. 358 ''' 359 # Compare the absolute path names, sorted. 360 asserted = sorted([os.path.abspath(i) for i in assert_output_files]) 361 actual = sorted([ 362 os.path.abspath(os.path.join(self.output_directory, i.GetFilename())) 363 for i in self.res.GetOutputFiles()]) 364 365 if asserted != actual: 366 print '''Asserted file list does not match. 367 368Expected output files: %s 369 370Actual output files: %s 371''' % (asserted, actual) 372 return False 373 return True 374 375 376 def GenerateDepfile(self, depfile, depdir): 377 '''Generate a depfile that contains the imlicit dependencies of the input 378 grd. The depfile will be in the same format as a makefile, and will contain 379 references to files relative to |depdir|. It will be put in |depfile|. 380 381 For example, supposing we have three files in a directory src/ 382 383 src/ 384 blah.grd <- depends on input{1,2}.xtb 385 input1.xtb 386 input2.xtb 387 388 and we run 389 390 grit -i blah.grd -o ../out/gen --depdir ../out --depfile ../out/gen/blah.rd.d 391 392 from the directory src/ we will generate a depfile ../out/gen/blah.grd.d 393 that has the contents 394 395 gen/blah.h: ../src/input1.xtb ../src/input2.xtb 396 397 Where "gen/blah.h" is the first output (Ninja expects the .d file to list 398 the first output in cases where there is more than one). 399 400 Note that all paths in the depfile are relative to ../out, the depdir. 401 ''' 402 depfile = os.path.abspath(depfile) 403 depdir = os.path.abspath(depdir) 404 infiles = self.res.GetInputFiles() 405 406 # Get the first output file relative to the depdir. 407 outputs = self.res.GetOutputFiles() 408 output_file = os.path.relpath(os.path.join( 409 self.output_directory, outputs[0].GetFilename()), depdir) 410 411 # The path prefix to prepend to dependencies in the depfile. 412 prefix = os.path.relpath(os.getcwd(), depdir) 413 deps_text = ' '.join([os.path.join(prefix, i) for i in infiles]) 414 415 depfile_contents = output_file + ': ' + deps_text 416 self.MakeDirectoriesTo(depfile) 417 outfile = self.fo_create(depfile, 'wb') 418 outfile.writelines(depfile_contents) 419 420 @staticmethod 421 def MakeDirectoriesTo(file): 422 '''Creates directories necessary to contain |file|.''' 423 dir = os.path.split(file)[0] 424 if not os.path.exists(dir): 425 os.makedirs(dir) 426