1#!/usr/bin/env python3 2# Copyright 2020 The Pigweed Authors 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); you may not 5# use this file except in compliance with the License. You may obtain a copy of 6# the License at 7# 8# https://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13# License for the specific language governing permissions and limitations under 14# the License. 15"""Extracts build information from Arduino cores.""" 16 17import glob 18import logging 19import os 20import platform 21import pprint 22import re 23import sys 24import time 25from collections import OrderedDict 26from pathlib import Path 27from typing import List 28 29from pw_arduino_build import file_operations 30 31_LOG = logging.getLogger(__name__) 32 33_pretty_print = pprint.PrettyPrinter(indent=1, width=120).pprint 34_pretty_format = pprint.PrettyPrinter(indent=1, width=120).pformat 35 36 37def arduino_runtime_os_string(): 38 arduno_platform = { 39 "Linux": "linux", 40 "Windows": "windows", 41 "Darwin": "macosx" 42 } 43 return arduno_platform[platform.system()] 44 45 46class ArduinoBuilder: 47 """Used to interpret arduino boards.txt and platform.txt files.""" 48 # pylint: disable=too-many-instance-attributes,too-many-public-methods 49 50 BOARD_MENU_REGEX = re.compile( 51 r"^(?P<name>menu\.[^#=]+)=(?P<description>.*)$", re.MULTILINE) 52 53 BOARD_NAME_REGEX = re.compile( 54 r"^(?P<name>[^\s#\.]+)\.name=(?P<description>.*)$", re.MULTILINE) 55 56 VARIABLE_REGEX = re.compile(r"^(?P<name>[^\s#=]+)=(?P<value>.*)$", 57 re.MULTILINE) 58 59 MENU_OPTION_REGEX = re.compile( 60 r"^menu\." # starts with "menu" 61 r"(?P<menu_option_name>[^.]+)\." # first token after . 62 r"(?P<menu_option_value>[^.]+)$") # second (final) token after . 63 64 TOOL_NAME_REGEX = re.compile( 65 r"^tools\." # starts with "tools" 66 r"(?P<tool_name>[^.]+)\.") # first token after . 67 68 INTERPOLATED_VARIABLE_REGEX = re.compile(r"{[^}]+}", re.MULTILINE) 69 70 OBJCOPY_STEP_NAME_REGEX = re.compile(r"^recipe.objcopy.([^.]+).pattern$") 71 72 def __init__(self, 73 arduino_path, 74 package_name, 75 build_path=None, 76 project_path=None, 77 project_source_path=None, 78 library_path=None, 79 library_names=None, 80 build_project_name=None, 81 compiler_path_override=False): 82 self.arduino_path = arduino_path 83 self.arduino_package_name = package_name 84 self.selected_board = None 85 self.build_path = build_path 86 self.project_path = project_path 87 self.project_source_path = project_source_path 88 self.build_project_name = build_project_name 89 self.compiler_path_override = compiler_path_override 90 self.variant_includes = "" 91 self.build_variant_path = False 92 self.library_names = library_names 93 self.library_path = library_path 94 95 self.compiler_path_override_binaries = [] 96 if self.compiler_path_override: 97 self.compiler_path_override_binaries = file_operations.find_files( 98 self.compiler_path_override, "*") 99 100 # Container dicts for boards.txt and platform.txt file data. 101 self.board = OrderedDict() 102 self.platform = OrderedDict() 103 self.menu_options = OrderedDict({ 104 "global_options": {}, 105 "default_board_values": {}, 106 "selected": {} 107 }) 108 self.tools_variables = {} 109 110 # Set and check for valid hardware folder. 111 self.hardware_path = os.path.join(self.arduino_path, "hardware") 112 113 if not os.path.exists(self.hardware_path): 114 raise FileNotFoundError( 115 "Arduino package path '{}' does not exist.".format( 116 self.arduino_path)) 117 118 # Set and check for valid package name 119 self.package_path = os.path.join(self.arduino_path, "hardware", 120 package_name) 121 # {build.arch} is the first folder name of the package (upcased) 122 self.build_arch = os.path.split(package_name)[0].upper() 123 124 if not os.path.exists(self.package_path): 125 _LOG.error("Error: Arduino package name '%s' does not exist.", 126 package_name) 127 _LOG.error("Did you mean:\n") 128 # TODO(tonymd): On Windows concatenating "/" may not work 129 possible_alternatives = [ 130 d.replace(self.hardware_path + os.sep, "", 1) 131 for d in glob.glob(self.hardware_path + "/*/*") 132 ] 133 _LOG.error("\n".join(possible_alternatives)) 134 sys.exit(1) 135 136 # Populate library paths. 137 if not library_path: 138 self.library_path = [] 139 # Append core libraries directory. 140 core_lib_path = Path(self.package_path) / "libraries" 141 if core_lib_path.is_dir(): 142 self.library_path.append(Path(self.package_path) / "libraries") 143 if library_path: 144 self.library_path = [ 145 os.path.realpath(os.path.expanduser( 146 os.path.expandvars(l_path))) for l_path in library_path 147 ] 148 149 # Grab all folder names in the cores directory. These are typically 150 # sub-core source files. 151 self.sub_core_folders = os.listdir( 152 os.path.join(self.package_path, "cores")) 153 154 self._find_tools_variables() 155 156 self.boards_txt = os.path.join(self.package_path, "boards.txt") 157 self.platform_txt = os.path.join(self.package_path, "platform.txt") 158 159 def select_board(self, board_name, menu_option_overrides=False): 160 self.selected_board = board_name 161 162 # Load default menu options for a selected board. 163 if not self.selected_board in self.board.keys(): 164 _LOG.error("Error board: '%s' not supported.", self.selected_board) 165 # TODO(tonymd): Print supported boards here 166 sys.exit(1) 167 168 # Override default menu options if any are specified. 169 if menu_option_overrides: 170 for moption in menu_option_overrides: 171 if not self.set_menu_option(moption): 172 # TODO(tonymd): Print supported menu options here 173 sys.exit(1) 174 175 self._copy_default_menu_options_to_build_variables() 176 self._apply_recipe_overrides() 177 self._substitute_variables() 178 179 def set_variables(self, variable_list: List[str]): 180 # Convert the string list containing 'name=value' items into a dict 181 variable_source = {} 182 for var in variable_list: 183 var_name, value = var.split("=") 184 variable_source[var_name] = value 185 186 # Replace variables in platform 187 for var, value in self.platform.items(): 188 self.platform[var] = self._replace_variables( 189 value, variable_source) 190 191 def _apply_recipe_overrides(self): 192 # Override link recipes with per-core exceptions 193 # Teensyduino cores 194 if self.build_arch == "TEENSY": 195 # Change {build.path}/{archive_file} 196 # To {archive_file_path} (which should contain the core.a file) 197 new_link_line = self.platform["recipe.c.combine.pattern"].replace( 198 "{object_files} \"{build.path}/{archive_file}\"", 199 "{object_files} {archive_file_path}", 1) 200 # Add the teensy provided toolchain lib folder for link access to 201 # libarm_cortexM*_math.a 202 new_link_line = new_link_line.replace( 203 "\"-L{build.path}\"", 204 "\"-L{build.path}\" -L{compiler.path}/arm/arm-none-eabi/lib", 205 1) 206 self.platform["recipe.c.combine.pattern"] = new_link_line 207 # Remove the pre-compiled header include 208 self.platform["recipe.cpp.o.pattern"] = self.platform[ 209 "recipe.cpp.o.pattern"].replace("\"-I{build.path}/pch\"", "", 210 1) 211 212 # Adafruit-samd core 213 # TODO(tonymd): This build_arch may clash with Arduino-SAMD core 214 elif self.build_arch == "SAMD": 215 new_link_line = self.platform["recipe.c.combine.pattern"].replace( 216 "\"{build.path}/{archive_file}\" -Wl,--end-group", 217 "{archive_file_path} -Wl,--end-group", 1) 218 self.platform["recipe.c.combine.pattern"] = new_link_line 219 220 # STM32L4 Core: 221 # https://github.com/GrumpyOldPizza/arduino-STM32L4 222 elif self.build_arch == "STM32L4": 223 # TODO(tonymd): {build.path}/{archive_file} for the link step always 224 # seems to be core.a (except STM32 core) 225 line_to_delete = "-Wl,--start-group \"{build.path}/{archive_file}\"" 226 new_link_line = self.platform["recipe.c.combine.pattern"].replace( 227 line_to_delete, "-Wl,--start-group {archive_file_path}", 1) 228 self.platform["recipe.c.combine.pattern"] = new_link_line 229 230 # stm32duino core 231 elif self.build_arch == "STM32": 232 # Must link in SrcWrapper for all projects. 233 if not self.library_names: 234 self.library_names = [] 235 self.library_names.append("SrcWrapper") 236 237 def _copy_default_menu_options_to_build_variables(self): 238 # Clear existing options 239 self.menu_options["selected"] = {} 240 # Set default menu options for selected board 241 for menu_key, menu_dict in self.menu_options["default_board_values"][ 242 self.selected_board].items(): 243 for name, var in self.board[self.selected_board].items(): 244 starting_key = "{}.{}.".format(menu_key, menu_dict["name"]) 245 if name.startswith(starting_key): 246 new_var_name = name.replace(starting_key, "", 1) 247 self.menu_options["selected"][new_var_name] = var 248 249 def set_menu_option(self, moption): 250 if moption not in self.board[self.selected_board]: 251 _LOG.error("Error: '%s' is not a valid menu option.", moption) 252 return False 253 254 # Override default menu option with new value. 255 menu_match_result = self.MENU_OPTION_REGEX.match(moption) 256 if menu_match_result: 257 menu_match = menu_match_result.groupdict() 258 menu_value = menu_match["menu_option_value"] 259 menu_key = "menu.{}".format(menu_match["menu_option_name"]) 260 self.menu_options["default_board_values"][ 261 self.selected_board][menu_key]["name"] = menu_value 262 263 # Update build variables 264 self._copy_default_menu_options_to_build_variables() 265 return True 266 267 def _set_global_arduino_variables(self): 268 """Set some global variables defined by the Arduino-IDE. 269 270 See Docs: 271 https://arduino.github.io/arduino-cli/platform-specification/#global-predefined-properties 272 """ 273 274 # TODO(tonymd): Make sure these variables are replaced in recipe lines 275 # even if they are None: build_path, project_path, project_source_path, 276 # build_project_name 277 for current_board_name in self.board.keys(): 278 if self.build_path: 279 self.board[current_board_name]["build.path"] = self.build_path 280 if self.build_project_name: 281 self.board[current_board_name][ 282 "build.project_name"] = self.build_project_name 283 # {archive_file} is the final *.elf 284 archive_file = "{}.elf".format(self.build_project_name) 285 self.board[current_board_name]["archive_file"] = archive_file 286 # {archive_file_path} is the final core.a archive 287 if self.build_path: 288 self.board[current_board_name][ 289 "archive_file_path"] = os.path.join( 290 self.build_path, "core.a") 291 if self.project_source_path: 292 self.board[current_board_name][ 293 "build.source.path"] = self.project_source_path 294 295 self.board[current_board_name]["extra.time.local"] = str( 296 int(time.time())) 297 self.board[current_board_name]["runtime.ide.version"] = "10812" 298 self.board[current_board_name][ 299 "runtime.hardware.path"] = self.hardware_path 300 301 # Copy {runtime.tools.TOOL_NAME.path} vars 302 self._set_tools_variables(self.board[current_board_name]) 303 304 self.board[current_board_name][ 305 "runtime.platform.path"] = self.package_path 306 if self.platform["name"] == "Teensyduino": 307 # Teensyduino is installed into the arduino IDE folder 308 # rather than ~/.arduino15/packages/ 309 self.board[current_board_name][ 310 "runtime.hardware.path"] = os.path.join( 311 self.hardware_path, "teensy") 312 313 self.board[current_board_name]["build.system.path"] = os.path.join( 314 self.package_path, "system") 315 316 # Set the {build.core.path} variable that pointing to a sub-core 317 # folder. For Teensys this is: 318 # 'teensy/hardware/teensy/avr/cores/teensy{3,4}'. For other cores 319 # it's typically just the 'arduino' folder. For example: 320 # 'arduino-samd/hardware/samd/1.8.8/cores/arduino' 321 core_path = Path(self.package_path) / "cores" 322 core_path /= self.board[current_board_name].get( 323 "build.core", self.sub_core_folders[0]) 324 self.board[current_board_name][ 325 "build.core.path"] = core_path.as_posix() 326 327 self.board[current_board_name]["build.arch"] = self.build_arch 328 329 for name, var in self.board[current_board_name].items(): 330 self.board[current_board_name][name] = var.replace( 331 "{build.core.path}", core_path.as_posix()) 332 333 def load_board_definitions(self): 334 """Loads Arduino boards.txt and platform.txt files into dictionaries. 335 336 Populates the following dictionaries: 337 self.menu_options 338 self.boards 339 self.platform 340 """ 341 # Load platform.txt 342 with open(self.platform_txt, "r") as pfile: 343 platform_file = pfile.read() 344 platform_var_matches = self.VARIABLE_REGEX.finditer(platform_file) 345 for p_match in [m.groupdict() for m in platform_var_matches]: 346 self.platform[p_match["name"]] = p_match["value"] 347 348 # Load boards.txt 349 with open(self.boards_txt, "r") as bfile: 350 board_file = bfile.read() 351 # Get all top-level menu options, e.g. menu.usb=USB Type 352 board_menu_matches = self.BOARD_MENU_REGEX.finditer(board_file) 353 for menuitem in [m.groupdict() for m in board_menu_matches]: 354 self.menu_options["global_options"][menuitem["name"]] = { 355 "description": menuitem["description"] 356 } 357 358 # Get all board names, e.g. teensy40.name=Teensy 4.0 359 board_name_matches = self.BOARD_NAME_REGEX.finditer(board_file) 360 for b_match in [m.groupdict() for m in board_name_matches]: 361 self.board[b_match["name"]] = OrderedDict() 362 self.menu_options["default_board_values"][ 363 b_match["name"]] = OrderedDict() 364 365 # Get all board variables, e.g. teensy40.* 366 for current_board_name in self.board.keys(): 367 board_line_matches = re.finditer( 368 fr"^\s*{current_board_name}\." 369 fr"(?P<key>[^#=]+)=(?P<value>.*)$", board_file, 370 re.MULTILINE) 371 for b_match in [m.groupdict() for m in board_line_matches]: 372 # Check if this line is a menu option 373 # (e.g. 'menu.usb.serial') and save as default if it's the 374 # first one seen. 375 ArduinoBuilder.save_default_menu_option( 376 current_board_name, b_match["key"], b_match["value"], 377 self.menu_options) 378 self.board[current_board_name][ 379 b_match["key"]] = b_match["value"].strip() 380 381 self._set_global_arduino_variables() 382 383 @staticmethod 384 def save_default_menu_option(current_board_name, key, value, menu_options): 385 """Save a given menu option as the default. 386 387 Saves the key and value into menu_options["default_board_values"] 388 if it doesn't already exist. Assumes menu options are added in the order 389 specified in boards.txt. The first value for a menu key is the default. 390 """ 391 # Check if key is a menu option 392 # e.g. menu.usb.serial 393 # menu.usb.serial.build.usbtype 394 menu_match_result = re.match( 395 r'^menu\.' # starts with "menu" 396 r'(?P<menu_option_name>[^.]+)\.' # first token after . 397 r'(?P<menu_option_value>[^.]+)' # second token after . 398 r'(\.(?P<rest>.+))?', # optionally any trailing tokens after a . 399 key) 400 if menu_match_result: 401 menu_match = menu_match_result.groupdict() 402 current_menu_key = "menu.{}".format(menu_match["menu_option_name"]) 403 # If this is the first menu option seen for current_board_name, save 404 # as the default. 405 if current_menu_key not in menu_options["default_board_values"][ 406 current_board_name]: 407 menu_options["default_board_values"][current_board_name][ 408 current_menu_key] = { 409 "name": menu_match["menu_option_value"], 410 "description": value 411 } 412 413 def _replace_variables(self, line, variable_lookup_source): 414 """Replace {variables} from loaded boards.txt or platform.txt. 415 416 Replace interpolated variables surrounded by curly braces in line with 417 definitions from variable_lookup_source. 418 """ 419 new_line = line 420 for current_var_match in self.INTERPOLATED_VARIABLE_REGEX.findall( 421 line): 422 # {build.flags.c} --> build.flags.c 423 current_var = current_var_match.strip("{}") 424 425 # check for matches in board definition 426 if current_var in variable_lookup_source: 427 replacement = variable_lookup_source.get(current_var, "") 428 new_line = new_line.replace(current_var_match, replacement) 429 return new_line 430 431 def _find_tools_variables(self): 432 # Gather tool directories in order of increasing precedence 433 runtime_tool_paths = [] 434 435 # Check for tools installed in ~/.arduino15/packages/arduino/tools/ 436 # TODO(tonymd): Is this Mac & Linux specific? 437 runtime_tool_paths += glob.glob( 438 os.path.join( 439 os.path.realpath(os.path.expanduser(os.path.expandvars("~"))), 440 ".arduino15", "packages", "arduino", "tools", "*")) 441 442 # <ARDUINO_PATH>/tools/<OS_STRING>/<TOOL_NAMES> 443 runtime_tool_paths += glob.glob( 444 os.path.join(self.arduino_path, "tools", 445 arduino_runtime_os_string(), "*")) 446 # <ARDUINO_PATH>/tools/<TOOL_NAMES> 447 # This will grab linux/windows/macosx/share as <TOOL_NAMES>. 448 runtime_tool_paths += glob.glob( 449 os.path.join(self.arduino_path, "tools", "*")) 450 451 # Process package tools after arduino tools. 452 # They should overwrite vars & take precedence. 453 454 # <PACKAGE_PATH>/tools/<OS_STRING>/<TOOL_NAMES> 455 runtime_tool_paths += glob.glob( 456 os.path.join(self.package_path, "tools", 457 arduino_runtime_os_string(), "*")) 458 # <PACKAGE_PATH>/tools/<TOOL_NAMES> 459 # This will grab linux/windows/macosx/share as <TOOL_NAMES>. 460 runtime_tool_paths += glob.glob( 461 os.path.join(self.package_path, "tools", "*")) 462 463 for path in runtime_tool_paths: 464 # Make sure TOOL_NAME is not an OS string 465 if not (path.endswith("linux") or path.endswith("windows") 466 or path.endswith("macosx") or path.endswith("share")): 467 # TODO(tonymd): Check if a file & do nothing? 468 469 # Check if it's a directory with subdir(s) as a version string 470 # create all 'runtime.tools.{tool_folder}-{version.path}' 471 # (for each version) 472 # create 'runtime.tools.{tool_folder}.path' 473 # (with latest version) 474 if os.path.isdir(path): 475 # Grab the tool name (folder) by itself. 476 tool_folder = os.path.basename(path) 477 # Sort so that [-1] is the latest version. 478 version_paths = sorted(glob.glob(os.path.join(path, "*"))) 479 # Check if all sub folders start with a version string. 480 if len(version_paths) == sum( 481 bool(re.match(r"^[0-9.]+", os.path.basename(vp))) 482 for vp in version_paths): 483 for version_path in version_paths: 484 version_string = os.path.basename(version_path) 485 var_name = "runtime.tools.{}-{}.path".format( 486 tool_folder, version_string) 487 self.tools_variables[var_name] = os.path.join( 488 path, version_string) 489 var_name = "runtime.tools.{}.path".format(tool_folder) 490 self.tools_variables[var_name] = os.path.join( 491 path, os.path.basename(version_paths[-1])) 492 # Else set toolpath to path. 493 else: 494 var_name = "runtime.tools.{}.path".format(tool_folder) 495 self.tools_variables[var_name] = path 496 497 _LOG.debug("TOOL VARIABLES: %s", _pretty_format(self.tools_variables)) 498 499 # Copy self.tools_variables into destination 500 def _set_tools_variables(self, destination): 501 for key, value in self.tools_variables.items(): 502 destination[key] = value 503 504 def _substitute_variables(self): 505 """Perform variable substitution in board and platform metadata.""" 506 507 # menu -> board 508 # Copy selected menu variables into board definiton 509 for name, value in self.menu_options["selected"].items(): 510 self.board[self.selected_board][name] = value 511 512 # board -> board 513 # Replace any {vars} in the selected board with values defined within 514 # (and from copied in menu options). 515 for var, value in self.board[self.selected_board].items(): 516 self.board[self.selected_board][var] = self._replace_variables( 517 value, self.board[self.selected_board]) 518 519 # Check for build.variant variable 520 # This will be set in selected board after menu options substitution 521 build_variant = self.board[self.selected_board].get( 522 "build.variant", None) 523 if build_variant: 524 # Set build.variant.path 525 bvp = os.path.join(self.package_path, "variants", build_variant) 526 self.build_variant_path = bvp 527 self.board[self.selected_board]["build.variant.path"] = bvp 528 # Add the variant folder as an include directory 529 # (used in stm32l4 core) 530 self.variant_includes = "-I{}".format(bvp) 531 532 _LOG.debug("PLATFORM INITIAL: %s", _pretty_format(self.platform)) 533 534 # board -> platform 535 # Replace {vars} in platform from the selected board definition 536 for var, value in self.platform.items(): 537 self.platform[var] = self._replace_variables( 538 value, self.board[self.selected_board]) 539 540 # platform -> platform 541 # Replace any remaining {vars} in platform from platform 542 for var, value in self.platform.items(): 543 self.platform[var] = self._replace_variables(value, self.platform) 544 545 # Repeat platform -> platform for any lingering variables 546 # Example: {build.opt.name} in STM32 core 547 for var, value in self.platform.items(): 548 self.platform[var] = self._replace_variables(value, self.platform) 549 550 _LOG.debug("MENU_OPTIONS: %s", _pretty_format(self.menu_options)) 551 _LOG.debug("SELECTED_BOARD: %s", 552 _pretty_format(self.board[self.selected_board])) 553 _LOG.debug("PLATFORM: %s", _pretty_format(self.platform)) 554 555 def selected_board_spec(self): 556 return self.board[self.selected_board] 557 558 def get_menu_options(self): 559 all_options = [] 560 max_string_length = [0, 0] 561 562 for key_name, description in self.board[self.selected_board].items(): 563 menu_match_result = self.MENU_OPTION_REGEX.match(key_name) 564 if menu_match_result: 565 menu_match = menu_match_result.groupdict() 566 name = "menu.{}.{}".format(menu_match["menu_option_name"], 567 menu_match["menu_option_value"]) 568 if len(name) > max_string_length[0]: 569 max_string_length[0] = len(name) 570 if len(description) > max_string_length[1]: 571 max_string_length[1] = len(description) 572 all_options.append((name, description)) 573 574 return all_options, max_string_length 575 576 def get_default_menu_options(self): 577 default_options = [] 578 max_string_length = [0, 0] 579 580 for key_name, value in self.menu_options["default_board_values"][ 581 self.selected_board].items(): 582 full_key = key_name + "." + value["name"] 583 if len(full_key) > max_string_length[0]: 584 max_string_length[0] = len(full_key) 585 if len(value["description"]) > max_string_length[1]: 586 max_string_length[1] = len(value["description"]) 587 default_options.append((full_key, value["description"])) 588 589 return default_options, max_string_length 590 591 @staticmethod 592 def split_binary_from_arguments(compile_line): 593 compile_binary = None 594 rest_of_line = compile_line 595 596 compile_binary_match = re.search(r'^("[^"]+") ', compile_line) 597 if compile_binary_match: 598 compile_binary = compile_binary_match[1] 599 rest_of_line = compile_line.replace(compile_binary_match[0], "", 1) 600 601 return compile_binary, rest_of_line 602 603 def _strip_includes_source_file_object_file_vars(self, compile_line): 604 line = compile_line 605 if self.variant_includes: 606 line = compile_line.replace( 607 "{includes} \"{source_file}\" -o \"{object_file}\"", 608 self.variant_includes, 1) 609 else: 610 line = compile_line.replace( 611 "{includes} \"{source_file}\" -o \"{object_file}\"", "", 1) 612 return line 613 614 def _get_tool_name(self, line): 615 tool_match_result = self.TOOL_NAME_REGEX.match(line) 616 if tool_match_result: 617 return tool_match_result[1] 618 return False 619 620 def get_upload_tool_names(self): 621 return [ 622 self._get_tool_name(t) for t in self.platform.keys() 623 if self.TOOL_NAME_REGEX.match(t) and 'upload.pattern' in t 624 ] 625 626 # TODO(tonymd): Use these getters in _replace_variables() or 627 # _substitute_variables() 628 629 def _get_platform_variable(self, variable): 630 # TODO(tonymd): Check for '.macos' '.linux' '.windows' in variable name, 631 # compare with platform.system() and return that instead. 632 return self.platform.get(variable, False) 633 634 def _get_platform_variable_with_substitutions(self, variable, namespace): 635 line = self.platform.get(variable, False) 636 # Get all unique variables used in this line in line. 637 unique_vars = sorted( 638 set(self.INTERPOLATED_VARIABLE_REGEX.findall(line))) 639 # Search for each unique_vars in namespace and global. 640 for var in unique_vars: 641 v_raw_name = var.strip("{}") 642 643 # Check for namespace.variable 644 # eg: 'tools.stm32CubeProg.cmd' 645 possible_var_name = "{}.{}".format(namespace, v_raw_name) 646 result = self._get_platform_variable(possible_var_name) 647 # Check for os overriden variable 648 # eg: 649 # ('tools.stm32CubeProg.cmd', 'stm32CubeProg.sh'), 650 # ('tools.stm32CubeProg.cmd.windows', 'stm32CubeProg.bat'), 651 possible_var_name = "{}.{}.{}".format(namespace, v_raw_name, 652 arduino_runtime_os_string()) 653 os_override_result = self._get_platform_variable(possible_var_name) 654 655 if os_override_result: 656 line = line.replace(var, os_override_result) 657 elif result: 658 line = line.replace(var, result) 659 # Check for variable at top level? 660 # elif self._get_platform_variable(v_raw_name): 661 # line = line.replace(self._get_platform_variable(v_raw_name), 662 # result) 663 return line 664 665 def get_upload_line(self, tool_name, serial_port=False): 666 # TODO(tonymd): Error if tool_name does not exist 667 tool_namespace = "tools.{}".format(tool_name) 668 pattern = "tools.{}.upload.pattern".format(tool_name) 669 670 if not self._get_platform_variable(pattern): 671 _LOG.error("Error: upload tool '%s' does not exist.", tool_name) 672 tools = self.get_upload_tool_names() 673 _LOG.error("Valid tools: %s", ", ".join(tools)) 674 return sys.exit(1) 675 676 line = self._get_platform_variable_with_substitutions( 677 pattern, tool_namespace) 678 679 # TODO(tonymd): Teensy specific tool overrides. 680 if tool_name == "teensyloader": 681 # Remove un-necessary lines 682 # {serial.port.label} and {serial.port.protocol} are returned by 683 # the teensy_ports binary. 684 line = line.replace("\"-portlabel={serial.port.label}\"", "", 1) 685 line = line.replace("\"-portprotocol={serial.port.protocol}\"", "", 686 1) 687 688 if serial_port == "UNKNOWN" or not serial_port: 689 line = line.replace('"-port={serial.port}"', "", 1) 690 else: 691 line = line.replace("{serial.port}", serial_port, 1) 692 693 return line 694 695 def _get_binary_path(self, variable_pattern): 696 compile_line = self.replace_compile_binary_with_override_path( 697 self._get_platform_variable(variable_pattern)) 698 compile_binary, _ = ArduinoBuilder.split_binary_from_arguments( 699 compile_line) 700 return compile_binary 701 702 def get_cc_binary(self): 703 return self._get_binary_path("recipe.c.o.pattern") 704 705 def get_cxx_binary(self): 706 return self._get_binary_path("recipe.cpp.o.pattern") 707 708 def get_objcopy_binary(self): 709 objcopy_step_name = self.get_objcopy_step_names()[0] 710 objcopy_binary = self._get_binary_path(objcopy_step_name) 711 return objcopy_binary 712 713 def get_ar_binary(self): 714 return self._get_binary_path("recipe.ar.pattern") 715 716 def get_size_binary(self): 717 return self._get_binary_path("recipe.size.pattern") 718 719 def replace_command_args_with_compiler_override_path(self, compile_line): 720 if not self.compiler_path_override: 721 return compile_line 722 replacement_line = compile_line 723 replacement_line_args = compile_line.split() 724 for arg in replacement_line_args: 725 compile_binary_basename = os.path.basename(arg.strip("\"")) 726 if compile_binary_basename in self.compiler_path_override_binaries: 727 new_compiler = os.path.join(self.compiler_path_override, 728 compile_binary_basename) 729 replacement_line = replacement_line.replace( 730 arg, new_compiler, 1) 731 return replacement_line 732 733 def replace_compile_binary_with_override_path(self, compile_line): 734 replacement_compile_line = compile_line 735 736 # Change the compiler path if there's an override path set 737 if self.compiler_path_override: 738 compile_binary, line = ArduinoBuilder.split_binary_from_arguments( 739 compile_line) 740 compile_binary_basename = os.path.basename( 741 compile_binary.strip("\"")) 742 new_compiler = os.path.join(self.compiler_path_override, 743 compile_binary_basename) 744 if platform.system() == "Windows" and not re.match( 745 r".*\.exe$", new_compiler, flags=re.IGNORECASE): 746 new_compiler += ".exe" 747 748 if os.path.isfile(new_compiler): 749 replacement_compile_line = "\"{}\" {}".format( 750 new_compiler, line) 751 752 return replacement_compile_line 753 754 def get_c_compile_line(self): 755 _LOG.debug("ARDUINO_C_COMPILE: %s", 756 _pretty_format(self.platform["recipe.c.o.pattern"])) 757 758 compile_line = self.platform["recipe.c.o.pattern"] 759 compile_line = self._strip_includes_source_file_object_file_vars( 760 compile_line) 761 compile_line += " -I{}".format( 762 self.board[self.selected_board]["build.core.path"]) 763 764 compile_line = self.replace_compile_binary_with_override_path( 765 compile_line) 766 return compile_line 767 768 def get_s_compile_line(self): 769 _LOG.debug("ARDUINO_S_COMPILE %s", 770 _pretty_format(self.platform["recipe.S.o.pattern"])) 771 772 compile_line = self.platform["recipe.S.o.pattern"] 773 compile_line = self._strip_includes_source_file_object_file_vars( 774 compile_line) 775 compile_line += " -I{}".format( 776 self.board[self.selected_board]["build.core.path"]) 777 778 compile_line = self.replace_compile_binary_with_override_path( 779 compile_line) 780 return compile_line 781 782 def get_ar_compile_line(self): 783 _LOG.debug("ARDUINO_AR_COMPILE: %s", 784 _pretty_format(self.platform["recipe.ar.pattern"])) 785 786 compile_line = self.platform["recipe.ar.pattern"].replace( 787 "\"{object_file}\"", "", 1) 788 789 compile_line = self.replace_compile_binary_with_override_path( 790 compile_line) 791 return compile_line 792 793 def get_cpp_compile_line(self): 794 _LOG.debug("ARDUINO_CPP_COMPILE: %s", 795 _pretty_format(self.platform["recipe.cpp.o.pattern"])) 796 797 compile_line = self.platform["recipe.cpp.o.pattern"] 798 compile_line = self._strip_includes_source_file_object_file_vars( 799 compile_line) 800 compile_line += " -I{}".format( 801 self.board[self.selected_board]["build.core.path"]) 802 803 compile_line = self.replace_compile_binary_with_override_path( 804 compile_line) 805 return compile_line 806 807 def get_link_line(self): 808 _LOG.debug("ARDUINO_LINK: %s", 809 _pretty_format(self.platform["recipe.c.combine.pattern"])) 810 811 compile_line = self.platform["recipe.c.combine.pattern"] 812 813 compile_line = self.replace_compile_binary_with_override_path( 814 compile_line) 815 return compile_line 816 817 def get_objcopy_step_names(self): 818 names = [ 819 name for name, line in self.platform.items() 820 if self.OBJCOPY_STEP_NAME_REGEX.match(name) 821 ] 822 return names 823 824 def get_objcopy_steps(self) -> List[str]: 825 lines = [ 826 line for name, line in self.platform.items() 827 if self.OBJCOPY_STEP_NAME_REGEX.match(name) 828 ] 829 lines = [ 830 self.replace_compile_binary_with_override_path(line) 831 for line in lines 832 ] 833 return lines 834 835 # TODO(tonymd): These recipes are probably run in sorted order 836 def get_objcopy(self, suffix): 837 # Expected vars: 838 # teensy: 839 # recipe.objcopy.eep.pattern 840 # recipe.objcopy.hex.pattern 841 842 pattern = "recipe.objcopy.{}.pattern".format(suffix) 843 objcopy_step_names = self.get_objcopy_step_names() 844 845 objcopy_suffixes = [ 846 m[1] for m in [ 847 self.OBJCOPY_STEP_NAME_REGEX.match(line) 848 for line in objcopy_step_names 849 ] if m 850 ] 851 if pattern not in objcopy_step_names: 852 _LOG.error("Error: objcopy suffix '%s' does not exist.", suffix) 853 _LOG.error("Valid suffixes: %s", ", ".join(objcopy_suffixes)) 854 return sys.exit(1) 855 856 line = self._get_platform_variable(pattern) 857 858 _LOG.debug("ARDUINO_OBJCOPY_%s: %s", suffix, line) 859 860 line = self.replace_compile_binary_with_override_path(line) 861 862 return line 863 864 def get_objcopy_flags(self, suffix): 865 # TODO(tonymd): Possibly teensy specific variables. 866 flags = "" 867 if suffix == "hex": 868 flags = self.platform.get("compiler.elf2hex.flags", "") 869 elif suffix == "bin": 870 flags = self.platform.get("compiler.elf2bin.flags", "") 871 elif suffix == "eep": 872 flags = self.platform.get("compiler.objcopy.eep.flags", "") 873 return flags 874 875 # TODO(tonymd): There are more recipe hooks besides postbuild. 876 # They are run in sorted order. 877 # TODO(tonymd): Rename this to get_hooks(hook_name, step). 878 # TODO(tonymd): Add a list-hooks and or run-hooks command 879 def get_postbuild_line(self, step_number): 880 line = self.platform["recipe.hooks.postbuild.{}.pattern".format( 881 step_number)] 882 line = self.replace_command_args_with_compiler_override_path(line) 883 return line 884 885 def get_prebuild_steps(self) -> List[str]: 886 # Teensy core uses recipe.hooks.sketch.prebuild.1.pattern 887 # stm32 core uses recipe.hooks.prebuild.1.pattern 888 # TODO(tonymd): STM32 core uses recipe.hooks.prebuild.1.pattern.windows 889 # (should override non-windows key) 890 lines = [ 891 line for name, line in self.platform.items() if re.match( 892 r"^recipe.hooks.(?:sketch.)?prebuild.[^.]+.pattern$", name) 893 ] 894 # TODO(tonymd): Write a function to fetch/replace OS specific patterns 895 # (ending in an OS string) 896 lines = [ 897 self.replace_compile_binary_with_override_path(line) 898 for line in lines 899 ] 900 return lines 901 902 def get_postbuild_steps(self) -> List[str]: 903 lines = [ 904 line for name, line in self.platform.items() 905 if re.match(r"^recipe.hooks.postbuild.[^.]+.pattern$", name) 906 ] 907 908 lines = [ 909 self.replace_command_args_with_compiler_override_path(line) 910 for line in lines 911 ] 912 return lines 913 914 def get_s_flags(self): 915 compile_line = self.get_s_compile_line() 916 _, compile_line = ArduinoBuilder.split_binary_from_arguments( 917 compile_line) 918 compile_line = compile_line.replace("-c", "", 1) 919 return compile_line.strip() 920 921 def get_c_flags(self): 922 compile_line = self.get_c_compile_line() 923 _, compile_line = ArduinoBuilder.split_binary_from_arguments( 924 compile_line) 925 compile_line = compile_line.replace("-c", "", 1) 926 return compile_line.strip() 927 928 def get_cpp_flags(self): 929 compile_line = self.get_cpp_compile_line() 930 _, compile_line = ArduinoBuilder.split_binary_from_arguments( 931 compile_line) 932 compile_line = compile_line.replace("-c", "", 1) 933 return compile_line.strip() 934 935 def get_ar_flags(self): 936 compile_line = self.get_ar_compile_line() 937 _, compile_line = ArduinoBuilder.split_binary_from_arguments( 938 compile_line) 939 return compile_line.strip() 940 941 def get_ld_flags(self): 942 compile_line = self.get_link_line() 943 _, compile_line = ArduinoBuilder.split_binary_from_arguments( 944 compile_line) 945 946 # TODO(tonymd): This is teensy specific 947 line_to_delete = "-o \"{build.path}/{build.project_name}.elf\" " \ 948 "{object_files} \"-L{build.path}\"" 949 if self.build_path: 950 line_to_delete = line_to_delete.replace("{build.path}", 951 self.build_path) 952 if self.build_project_name: 953 line_to_delete = line_to_delete.replace("{build.project_name}", 954 self.build_project_name) 955 956 compile_line = compile_line.replace(line_to_delete, "", 1) 957 libs = re.findall(r'(-l[^ ]+ ?)', compile_line) 958 for lib in libs: 959 compile_line = compile_line.replace(lib, "", 1) 960 libs = [lib.strip() for lib in libs] 961 962 return compile_line.strip() 963 964 def get_ld_libs(self, name_only=False): 965 compile_line = self.get_link_line() 966 libs = re.findall(r'(?P<arg>-l(?P<name>[^ ]+) ?)', compile_line) 967 if name_only: 968 libs = [lib_name.strip() for lib_arg, lib_name in libs] 969 else: 970 libs = [lib_arg.strip() for lib_arg, lib_name in libs] 971 return " ".join(libs) 972 973 def library_folders(self): 974 # Arduino library format documentation: 975 # https://arduino.github.io/arduino-cli/library-specification/#layout-of-folders-and-files 976 # - If src folder exists, 977 # use that as the root include directory -Ilibraries/libname/src 978 # - Else lib folder as root include -Ilibraries/libname 979 # (exclude source files in the examples folder in this case) 980 981 if not self.library_names or not self.library_path: 982 return [] 983 984 folder_patterns = ["*"] 985 if self.library_names: 986 folder_patterns = self.library_names 987 988 library_folders = OrderedDict() 989 for library_dir in self.library_path: 990 found_library_names = file_operations.find_files( 991 library_dir, folder_patterns, directories_only=True) 992 _LOG.debug("Found Libraries %s: %s", library_dir, 993 found_library_names) 994 for lib_name in found_library_names: 995 lib_dir = os.path.join(library_dir, lib_name) 996 src_dir = os.path.join(lib_dir, "src") 997 if os.path.exists(src_dir) and os.path.isdir(src_dir): 998 library_folders[lib_name] = src_dir 999 else: 1000 library_folders[lib_name] = lib_dir 1001 1002 return list(library_folders.values()) 1003 1004 def library_include_dirs(self): 1005 return [Path(lib).as_posix() for lib in self.library_folders()] 1006 1007 def library_includes(self): 1008 include_args = [] 1009 library_folders = self.library_folders() 1010 for lib_dir in library_folders: 1011 include_args.append("-I{}".format(os.path.relpath(lib_dir))) 1012 return include_args 1013 1014 def library_files(self, pattern, only_library_name=None): 1015 sources = [] 1016 library_folders = self.library_folders() 1017 if only_library_name: 1018 library_folders = [ 1019 lf for lf in self.library_folders() if only_library_name in lf 1020 ] 1021 for lib_dir in library_folders: 1022 for file_path in file_operations.find_files(lib_dir, [pattern]): 1023 if not file_path.startswith("examples"): 1024 sources.append((Path(lib_dir) / file_path).as_posix()) 1025 return sources 1026 1027 def library_c_files(self): 1028 return self.library_files("**/*.c") 1029 1030 def library_s_files(self): 1031 return self.library_files("**/*.S") 1032 1033 def library_cpp_files(self): 1034 return self.library_files("**/*.cpp") 1035 1036 def get_core_path(self): 1037 return self.board[self.selected_board]["build.core.path"] 1038 1039 def core_files(self, pattern): 1040 sources = [] 1041 for file_path in file_operations.find_files(self.get_core_path(), 1042 [pattern]): 1043 sources.append(os.path.join(self.get_core_path(), file_path)) 1044 return sources 1045 1046 def core_c_files(self): 1047 return self.core_files("**/*.c") 1048 1049 def core_s_files(self): 1050 return self.core_files("**/*.S") 1051 1052 def core_cpp_files(self): 1053 return self.core_files("**/*.cpp") 1054 1055 def get_variant_path(self): 1056 return self.build_variant_path 1057 1058 def variant_files(self, pattern): 1059 sources = [] 1060 if self.build_variant_path: 1061 for file_path in file_operations.find_files( 1062 self.get_variant_path(), [pattern]): 1063 sources.append(os.path.join(self.get_variant_path(), 1064 file_path)) 1065 return sources 1066 1067 def variant_c_files(self): 1068 return self.variant_files("**/*.c") 1069 1070 def variant_s_files(self): 1071 return self.variant_files("**/*.S") 1072 1073 def variant_cpp_files(self): 1074 return self.variant_files("**/*.cpp") 1075 1076 def project_files(self, pattern): 1077 sources = [] 1078 for file_path in file_operations.find_files(self.project_path, 1079 [pattern]): 1080 if not file_path.startswith( 1081 "examples") and not file_path.startswith("libraries"): 1082 sources.append(file_path) 1083 return sources 1084 1085 def project_c_files(self): 1086 return self.project_files("**/*.c") 1087 1088 def project_cpp_files(self): 1089 return self.project_files("**/*.cpp") 1090 1091 def project_ino_files(self): 1092 return self.project_files("**/*.ino") 1093