From b93715ec385e42806b0e9eb57fa3ce3509a5a528 Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Thu, 7 Dec 2023 22:43:41 +0300 Subject: [PATCH 1/5] Tools - upload.py exception handling fixes - don't check exc_info() in `finally`, it only works without `except` with just `try` and `finally` see https://docs.python.org/3/reference/compound_stmts.html#try after `except Exception as e:` block, `e` is already deleted - handle a rare case when esptool code does not close 'erase_file'. printing paths may cause encoding issues, so just fall through silently --- tools/upload.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/tools/upload.py b/tools/upload.py index 760d4dbbef..f3802ec307 100755 --- a/tools/upload.py +++ b/tools/upload.py @@ -39,9 +39,11 @@ # https://github.com/esp8266/Arduino/issues/6755#issuecomment-553208688 if thisarg == "erase_flash": write_option = '--erase-all' + # 'erase_region' is using a temporary file filled with 0xff elif thisarg == 'erase_region': erase_addr = sys.argv.pop(0) erase_len = sys.argv.pop(0) + # 'write_flash' and everything else is used as-is elif thisarg == 'write_flash': write_addr = sys.argv.pop(0) binary = sys.argv.pop(0) @@ -56,19 +58,23 @@ erase_file = '' if erase_addr: - # Generate temporary empty (0xff) file - eraser = tempfile.mkstemp() - erase_file = eraser[1] - os.write(eraser[0], bytearray([0xff] * int(erase_len, 0))) - os.close(eraser[0]) + erase_fd, erase_file = tempfile.mkstemp() + os.write(erase_fd, b"\xff" * int(erase_len, 0)) + os.close(erase_fd) cmdline = cmdline + [erase_addr, erase_file] +exit_code = 0 + try: esptool.main(cmdline) except Exception as e: - sys.stderr.write('\nA fatal esptool.py error occurred: %s' % e) -finally: - if erase_file: + sys.stderr.write(f"\nA fatal upload.py error occurred: {repr(e)}\n") + exit_code = 2 + +if erase_file: + try: os.remove(erase_file) - if any(sys.exc_info()): - sys.exit(2) + except: + pass + +sys.exit(exit_code) From fe729285393716b1b219e64e6ce623efbf277f6d Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Thu, 29 May 2025 18:27:05 +0300 Subject: [PATCH 2/5] formatting, strict write_flash opts order, atexit & traceback replace temporaries tracking write_flash / erase_region with an append to a generic write_flash_args list. first are actual files, erased ones at the end using atexit for cleanup --- tools/upload.py | 147 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 98 insertions(+), 49 deletions(-) diff --git a/tools/upload.py b/tools/upload.py index f3802ec307..af2bfa6f59 100755 --- a/tools/upload.py +++ b/tools/upload.py @@ -7,74 +7,123 @@ # i.e. upload.py tools/pyserial tools/esptool write_flash file 0x0 import os +import atexit +import pathlib import sys import tempfile +import traceback -sys.argv.pop(0) # Remove executable name -toolspath = os.path.dirname(os.path.realpath(__file__)) +from typing import List + +# Add neighbouring pyserial & esptool to search path +MODULES = [ + "pyserial", + "esptool", +] + +PWD = pathlib.Path(__file__).resolve().parent +for m in MODULES: + sys.path.insert(0, (PWD / m).as_posix()) + +# If this fails, we can't continue and will bomb below try: - sys.path.insert(0, os.path.join(toolspath, "pyserial")) # Add pyserial dir to search path - sys.path.insert(0, os.path.join(toolspath, "esptool")) # Add esptool dir to search path - import esptool # If this fails, we can't continue and will bomb below -except ImportError: - sys.stderr.write("pyserial or esptool directories not found next to this upload.py tool.\n") + import esptool +except (ImportError, ModuleNotFoundError) as e: + sys.stderr.write( + "\n*** pyserial or esptool directories not found next to upload.py tool (this script) ***\n" + ) + traceback.print_exc(file=sys.stderr) + sys.stderr.flush() + sys.exit(1) -cmdline = [] -write_option = '' -write_addr = '0x0' -erase_addr = '' -erase_len = '' -while sys.argv: - thisarg = sys.argv.pop(0) +def make_erase_pair(addr: str, dest_size: int, block_size=2**16): + dest, path = tempfile.mkstemp() + + buffer = b"\xff" * block_size + while dest_size: + unaligned = dest_size % block_size + + src_size = block_size + if unaligned: + src = buffer[unaligned:] + src_size = unaligned + else: + src = buffer + + os.write(dest, src) + dest_size -= src_size + + os.close(dest) + + def maybe_remove(path): + try: + os.remove(path) + except: + pass + + atexit.register(maybe_remove, path) + return [addr, path] - # We silently replace the 921kbaud setting with 460k to enable backward + +argv = sys.argv[1:] # Remove executable name + +cmdline = [] # type: List[str] +write_options = ["--flash_size", "detect"] # type: List[str] +erase_options = [] + +thisarg = "" +lastarg = "" +while argv: + lastarg = thisarg + thisarg = argv.pop(0) + + # We silently replace the high-speed setting with 460k to enable backward # compatibility with the old esptool-ck.exe. Esptool.py doesn't seem - # work reliably at 921k, but is still significantly faster at 460kbaud. - if thisarg == "921600": + # work reliably, but 460kbaud is still plenty fast. + if lastarg == "--baud" and thisarg in ("921600", "3000000"): thisarg = "460800" # 'erase_flash' command is translated to the write_flash --erase-all option # https://github.com/esp8266/Arduino/issues/6755#issuecomment-553208688 if thisarg == "erase_flash": - write_option = '--erase-all' - # 'erase_region' is using a temporary file filled with 0xff - elif thisarg == 'erase_region': - erase_addr = sys.argv.pop(0) - erase_len = sys.argv.pop(0) - # 'write_flash' and everything else is used as-is - elif thisarg == 'write_flash': - write_addr = sys.argv.pop(0) - binary = sys.argv.pop(0) - elif thisarg: - cmdline = cmdline + [thisarg] + write_options.append("--erase-all") -cmdline = cmdline + ['write_flash'] -if write_option: - cmdline = cmdline + [write_option] -cmdline = cmdline + ['--flash_size', 'detect'] -cmdline = cmdline + [write_addr, binary] + # instead of providing esptool with separate targets, + # everything below becomes 'write_flash' [ ] pairs + + # 'erase_region' becomes a temporary file filled with 0xff + # this pair is appended *after* 'write_flash' pairs + elif thisarg == "erase_region": + addr = argv.pop(0) + size = int(argv.pop(0), 0) + erase_options.extend(make_erase_pair(addr, size)) + + # 'write_flash' pair taken in order it was specified + elif thisarg == "write_flash": + addr = argv.pop(0) + path = argv.pop(0) + write_options.extend([addr, path]) + + # everything else is used as-is + elif thisarg: + cmdline.append(thisarg) -erase_file = '' -if erase_addr: - erase_fd, erase_file = tempfile.mkstemp() - os.write(erase_fd, b"\xff" * int(erase_len, 0)) - os.close(erase_fd) - cmdline = cmdline + [erase_addr, erase_file] -exit_code = 0 +cmdline.append("write_flash") +for opts in (write_options, erase_options): + if opts: + cmdline.extend(opts) try: esptool.main(cmdline) -except Exception as e: - sys.stderr.write(f"\nA fatal upload.py error occurred: {repr(e)}\n") - exit_code = 2 +except: + etype, evalue, _ = sys.exc_info() + estring = "\n".join(traceback.format_exception_only(etype, value=evalue)) -if erase_file: - try: - os.remove(erase_file) - except: - pass + sys.stderr.write(f"\n*** A fatal upload.py error occurred ***\n") + sys.stderr.write(estring) + sys.stderr.flush() -sys.exit(exit_code) + sys.exit(2) From e5b89164964307247c28961be89ed75b51e57dac Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Thu, 29 May 2025 18:44:04 +0300 Subject: [PATCH 3/5] imports are automatic, comment refers to old version --- tools/upload.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tools/upload.py b/tools/upload.py index af2bfa6f59..0f39315a5e 100755 --- a/tools/upload.py +++ b/tools/upload.py @@ -1,10 +1,7 @@ #!/usr/bin/env python3 # Wrapper for Arduino core / others that can call esptool.py possibly multiple times -# Adds pyserial to sys.path automatically based on the path of the current file - -# First parameter is pyserial path, second is esptool path, then a series of command arguments -# i.e. upload.py tools/pyserial tools/esptool write_flash file 0x0 +# Adds pyserial & esptool that are in the same directory as the script to sys.path import os import atexit From c058de5394b21fa4ddbf971fe8b196c02b9aafc2 Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Thu, 29 May 2025 19:34:59 +0300 Subject: [PATCH 4/5] typos, typing, no bare except --- tools/upload.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/upload.py b/tools/upload.py index 0f39315a5e..ee16272ccd 100755 --- a/tools/upload.py +++ b/tools/upload.py @@ -25,7 +25,7 @@ # If this fails, we can't continue and will bomb below try: import esptool -except (ImportError, ModuleNotFoundError) as e: +except ImportError: sys.stderr.write( "\n*** pyserial or esptool directories not found next to upload.py tool (this script) ***\n" ) @@ -38,16 +38,16 @@ def make_erase_pair(addr: str, dest_size: int, block_size=2**16): dest, path = tempfile.mkstemp() - buffer = b"\xff" * block_size + buffer = bytearray(b"\xff" * block_size) while dest_size: unaligned = dest_size % block_size - src_size = block_size if unaligned: src = buffer[unaligned:] src_size = unaligned else: src = buffer + src_size = block_size os.write(dest, src) dest_size -= src_size @@ -57,7 +57,7 @@ def make_erase_pair(addr: str, dest_size: int, block_size=2**16): def maybe_remove(path): try: os.remove(path) - except: + except Exception: pass atexit.register(maybe_remove, path) @@ -66,9 +66,9 @@ def maybe_remove(path): argv = sys.argv[1:] # Remove executable name -cmdline = [] # type: List[str] -write_options = ["--flash_size", "detect"] # type: List[str] -erase_options = [] +cmdline: List[str] = [] +write_options: List[str] = ["--flash_size", "detect"] +erase_options: List[str] = [] thisarg = "" lastarg = "" @@ -115,11 +115,11 @@ def maybe_remove(path): try: esptool.main(cmdline) -except: +except Exception: etype, evalue, _ = sys.exc_info() estring = "\n".join(traceback.format_exception_only(etype, value=evalue)) - sys.stderr.write(f"\n*** A fatal upload.py error occurred ***\n") + sys.stderr.write("\n*** upload.py fatal error ***\n") sys.stderr.write(estring) sys.stderr.flush() From 2ee93d5de30598c317ff6025bd79d21cb8b5c61e Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Thu, 29 May 2025 23:29:22 +0300 Subject: [PATCH 5/5] typo in remainder offset --- tools/upload.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/upload.py b/tools/upload.py index ee16272ccd..a7b8c4f530 100755 --- a/tools/upload.py +++ b/tools/upload.py @@ -40,11 +40,11 @@ def make_erase_pair(addr: str, dest_size: int, block_size=2**16): buffer = bytearray(b"\xff" * block_size) while dest_size: - unaligned = dest_size % block_size + remainder = dest_size % block_size - if unaligned: - src = buffer[unaligned:] - src_size = unaligned + if remainder: + src = buffer[:remainder] + src_size = remainder else: src = buffer src_size = block_size