Merge pull request #473 from pycom/Dev

Push latest Dev changes into Release/v1.20

Author: Xykon

.gitignore

@@ -36,6 +36,7 @@ user.props
 .pydevproject
 .settings
 .launch
+.vscode
 
 # Key files (for Flash Encryption and Secure Boot)
 secure_boot_signing_key.pem

Jenkinsfile

@@ -9,9 +9,14 @@ node {
     stage('Checkout') {
         checkout scm
         sh 'rm -rf esp-idf'
-        sh 'git clone --depth=1 --recursive -b idf_v3.2 https://github.com/pycom/pycom-esp-idf.git esp-idf'
+        sh 'git clone --recursive -b idf_v3.3.1 https://github.com/pycom/pycom-esp-idf.git esp-idf'
+        IDF_HASH=get_idf_hash()
+        dir('esp-idf'){
+            sh 'git checkout ' + IDF_HASH
+            sh 'git submodule update --init --recursive'
+        }
     }
-    
+
     stage('git-tag') {
         PYCOM_VERSION=get_version()
         GIT_TAG = sh (script: 'git rev-parse --short HEAD', returnStdout: true).trim()
@@ -85,7 +90,8 @@ def flashBuild(short_name, version, variant) {
       unstash 'esp32Tools'
       unstash 'tests'
       unstash 'tools'
-      sh 'python esp32/tools/fw_updater/updater.py --port ' + device_name +' flash -t esp32/build-' + variant + '/' + board_name + '-' + version + '.tar.gz'
+      sh 'python esp32/tools/fw_updater/updater.py --noexit --port ' + device_name +' flash -t esp32/build-' + variant + '/' + board_name + '-' + version + '.tar.gz'
+      sh 'python esp32/tools/fw_updater/updater.py --port ' + device_name +' pybytes --auto_start False'
     }
   }
 }
@@ -114,6 +120,11 @@ def get_version() {
     matcher ? matcher[0][1].trim().replace('"','') : null
 }
 
+def get_idf_hash() {
+    def matcher = readFile('esp32/Makefile') =~ 'IDF_HASH=(.+)'
+    matcher ? matcher[0][1].trim().replace('"','') : null
+}
+
 def get_firmware_name(short_name) {
   node {
     def node_info = sh (script: 'cat ${JENKINS_HOME}/pycom-ic.conf || exit 0', returnStdout: true).trim()

README.md

@@ -26,7 +26,7 @@ board (PyBoard), the officially supported reference electronic circuit board.
 The following components are actively maintained by Pycom:
 - py/ -- the core Python implementation, including compiler, runtime, and
   core library.
-- exp32/ -- a version of MicroPython that runs on the ESP32 based boards from Pycom.
+- esp32/ -- a version of MicroPython that runs on the ESP32 based boards from Pycom.
 - tests/ -- test framework and test scripts.
 
 Additional components:
@@ -74,7 +74,7 @@ Then when you need the toolchain you can type ``get_esp32`` on the command line
 You also need the ESP IDF along side this repository in order to build the ESP32 port.
 To get it:
 
-    $ git clone --recursive -b idf_v3.2 https://github.com/pycom/pycom-esp-idf.git
+    $ git clone --recursive -b idf_v3.3.1 https://github.com/pycom/pycom-esp-idf.git
 
 After cloning, if you did not specify the --recursive option, make sure to checkout all the submodules:
 

docs/library/ussl.rst

@@ -114,6 +114,21 @@ network sockets, both client-side and server-side.
 
        SSL sockets inherit all methods and from the standard sockets, see the :mod:`usocket` module.
 
+    .. function:: ssl.save_session(sock)
+
+       Takes an instance sock of ssl.SSLSocket, and returns an instance of ssl.SSLSession representing saved session data from the socket, which can be used to resume a SSL session later. Example::
+
+          import socket
+          import ssl
+          addr = socket.getaddrinfo('www.google.com', 443)[0][-1]
+          sock_one = ssl.wrap_socket(socket.socket())
+          sock_one.connect(addr) # performs a full ssl handshake
+          session = ssl.save_session(sock_one)
+          sock_one.close()
+          sock_one = None
+          sock_two = ssl.wrap_socket(socket.socket(), saved_session=session)
+          sock_two.connect(addr) # resumes using saved session, resulting in a faster handshake
+
     Exceptions
     ----------
 

docs/license.rst

@@ -23,7 +23,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 
-Copyright (c) 2019, Pycom Limited.
+Copyright (c) 2020, Pycom Limited.
 
 This software is licensed under the GNU GPL version 3 or any
 later version, with permitted additional terms. For more information

drivers/sx127x/sx1272/sx1272.c

@@ -1,7 +1,7 @@
 /*
  * This file is derived from the MicroPython project, http://micropython.org/
  *
- * Copyright (c) 2019, Pycom Limited and its licensors.
+ * Copyright (c) 2020, Pycom Limited and its licensors.
  *
  * This software is licensed under the GNU GPL version 3 or any later version,
  * with permitted additional terms. For more information see the Pycom Licence

drivers/sx127x/sx1276/sx1276.c

@@ -1,7 +1,7 @@
 /*
  * This file is derived from the MicroPython project, http://micropython.org/
  *
- * Copyright (c) 2019, Pycom Limited and its licensors.
+ * Copyright (c) 2020, Pycom Limited and its licensors.
  *
  * This software is licensed under the GNU GPL version 3 or any later version,
  * with permitted additional terms. For more information see the Pycom Licence

esp32/Makefile

@@ -1,5 +1,5 @@
 #
-# Copyright © 2019, Pycom Limited.
+# Copyright © 2020, Pycom Limited.
 #
 # This software is licensed under the GNU GPL version 3 or any
 # later version, with permitted additional terms. For more information
@@ -14,7 +14,7 @@ ifeq ($(wildcard boards/$(BOARD)/.),)
     $(error Invalid BOARD specified)
 endif
 
-IDF_VERSION=3.2
+IDF_HASH=6ec081c
 
 TARGET ?= boot_app
 
@@ -103,17 +103,10 @@ LIBS = -L$(ESP_IDF_COMP_PATH)/esp32/lib -L$(ESP_IDF_COMP_PATH)/esp32/ld -L$(ESP_
        $(ESP_IDF_COMP_PATH)/newlib/lib/libc-psram-workaround.a \
        -lfreertos -ljson -ljsmn -llwip -lnewlib -lvfs -lopenssl -lmbedtls -lwpa_supplicant \
        -lxtensa-debug-module -lbt -lsdmmc -lsoc -lheap -lbootloader_support -lmicro-ecc \
-       -u ld_include_panic_highint_hdl -lsmartconfig_ack -lmesh -lesp_ringbuf -lcoap -lmdns
-ifeq ($(BOARD), $(filter $(BOARD), FIPY))
-    LIBS += sigfox/modsigfox_fipy.a
-endif
-
-ifeq ($(BOARD), $(filter $(BOARD), LOPY4))
-    LIBS += sigfox/modsigfox_lopy4.a
-endif
-
-ifeq ($(BOARD), $(filter $(BOARD), SIPY))
-    LIBS += sigfox/modsigfox_sipy.a
+       -u ld_include_panic_highint_hdl -lsmartconfig_ack -lmesh -lesp_ringbuf -lcoap -lmdns -lefuse -lespcoredump -lapp_update
+ifeq ($(BOARD), $(filter $(BOARD), SIPY LOPY4 FIPY))
+    LIBS += sigfox/modsigfox_$(BOARD).a
+$(BUILD)/application.elf: sigfox/modsigfox_$(BOARD).a
 endif
 
 ifeq ($(OPENTHREAD), on)
@@ -134,7 +127,7 @@ endif #ifeq ($(LTE_LOG_BUFF),1)
 
 B_LIBS = -Lbootloader/lib -Lbootloader -L$(BUILD)/bootloader -L$(ESP_IDF_COMP_PATH)/esp32/ld \
          -L$(ESP_IDF_COMP_PATH)/esp32/lib -llog -lcore -lbootloader_support \
-         -lspi_flash -lsoc -lmicro-ecc -lgcc -lstdc++ -lgcov
+         -lspi_flash -lsoc -lmicro-ecc -lgcc -lstdc++ -lgcov -lefuse
 
 # objcopy paramters, to transform a binary file into an object file
 OBJCOPY_EMBED_ARGS = --input-target binary --output-target elf32-xtensa-le --binary-architecture xtensa --rename-section .data=.rodata.embedded

esp32/PyJTAG/Readme.md

@@ -0,0 +1,96 @@
+# Short readme for how to use the PyJTAG
+
+## Setup
+Generally follow these rules to setup JTAG debugging on your OS: https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/jtag-debugging/index.html
+
+Download link for OpenOCD for ESP32 from Espressif: https://github.com/espressif/openocd-esp32/releases
+
+
+## Build the firmware
+Create the firmware with `BTYPE=debug` flag.
+
+Note: Do not use the default pins assigned to UART, SPI, CAN because they are used by the JTAG. Pins not to be used: P4, P9, P10, P23.
+
+## Setup the PyJTAG board
+
+PyJTAG's switches:
+ * ESP32 JTAG: all turned ON
+ * ESP32 B.LOADER: all turned ON except SAFE_BOOT_SW which is OFF
+ * TO LTE UART 1/2: does not matter
+ * CURRENT SHUNTS: connected
+
+Place the Pycom board with the reset button towards the Current Shunts. Now connect the PyJTAG via usb. You will see four new USB devices. On Linux this will look like this:
+```
+$ lsusb -d 0403:
+Bus 001 Device 010: ID 0403:6011 Future Technology Devices International, Ltd FT4232H Quad HS USB-UART/FIFO IC
+$ ls /dev/ttyUSB?
+/dev/ttyUSB0  /dev/ttyUSB1  /dev/ttyUSB2  /dev/ttyUSB3
+```
+
+## Start OCD
+
+Go to `esp32` folder in Firmware-Development repository and run:
+```
+PATH_TO_OPENOCD/bin/openocd -s PATH_TO_OPENOCD/share/openocd/scripts -s PyJTAG -f PyJTAG/interface/ftdi/esp32-pycom.cfg -f PyJTAG/board/esp32-pycom.cfg
+```
+
+Output should be like: 
+```
+Open On-Chip Debugger  v0.10.0-esp32-20191114 (2019-11-14-14:15)
+Licensed under GNU GPL v2
+For bug reports, read
+	http://openocd.org/doc/doxygen/bugs.html
+none separate
+adapter speed: 20000 kHz
+Info : Configured 2 cores
+Info : Listening on port 6666 for tcl connections
+Info : Listening on port 4444 for telnet connections
+Error: type 'esp32' is missing virt2phys
+Info : ftdi: if you experience problems at higher adapter clocks, try the command "ftdi_tdo_sample_edge falling"
+Info : clock speed 20000 kHz
+Info : JTAG tap: esp32.cpu0 tap/device found: 0x120034e5 (mfg: 0x272 (Tensilica), part: 0x2003, ver: 0x1)
+Info : JTAG tap: esp32.cpu1 tap/device found: 0x120034e5 (mfg: 0x272 (Tensilica), part: 0x2003, ver: 0x1)
+Info : Listening on port 3333 for gdb connections
+```
+
+## Start GDB
+
+When OpenOCD is running, start GDB from `esp32` folder. Assuming you have a FIPY:
+```
+xtensa-esp32-elf-gdb -x PyJTAG/gdbinit build/FIPY/debug/application.elf
+```
+
+In `PyJTAG/gdbinit` a breakpoint is configured at `TASK_Micropython`, so execution should stop there first:
+
+```
+Thread 1 hit Temporary breakpoint 1, TASK_Micropython (pvParameters=0x0) at mptask.c:136
+```
+
+
+## REPL
+
+Connect to `/dev/ttyUSB2` to reach the REPL terminal over usb serial. E.g. using pymakr in Atom. 
+
+## Troubleshooting
+If openocd says "Error: Connect failed", try to close gdb and openocd and start over.
+
+If `/dev/ttyUSB0` doesn't show up or disappears, disconnect the PyJTAG board, reconnect and start over.
+
+It can be advisable to use the `gdb` from the latest xtensa toolchain, even if an earlier version is used to build the firmware.
+
+If `gdb` does not reach the `Thread 1 hit Temporary breakpoint ...` line, close and reopen `gdb`.
+
+
+## Versions
+There are two generations of PyJTAG boards:
+
+1) First generation with green PCB has three blocks of switches. (Make sure SAFE_BOOT_SW is off.)
+2) Second generation with black PCB and two blocks of switches.
+
+Both generation boards can be equipped with pogo pins that connect to the bottom of the development board and allow LTE debugging. There can either be pins that connect to a GPy or pins that conenct to a FiPy.
+
+To reach the modem UART connect to `/dev/ttyUSB1`.
+
+## Extra
+A few more details are here: https://pycomiot.atlassian.net/wiki/spaces/FIR/pages/966295564/Usage+of+PyJTAG
+

esp32/PyJTAG/board/esp32-pycom.cfg

@@ -0,0 +1,2 @@
+set ESP32_FLASH_VOLTAGE 3.3
+source [find target/esp32-pycom.cfg]

esp32/PyJTAG/gdbinit

@@ -0,0 +1,5 @@
+target remote :3333
+mon reset halt
+flushregs
+thb TASK_Micropython
+c

esp32/PyJTAG/interface/ftdi/esp32-pycom.cfg

@@ -0,0 +1,35 @@
+#
+# Driver for the FT4232HL JTAG chip on the Pycom's PyJTAG board
+#
+ 
+
+interface ftdi
+ftdi_vid_pid 0x0403 0x6011
+
+# interface 1 is the uart
+ftdi_channel 0
+
+# TCK, TDI, TDO, TMS: ADBUS0-3
+# LEDs: ACBUS4-7
+
+ftdi_layout_init 0x0008 0xf00b
+#ftdi_layout_signal LED -data 0x1000
+#ftdi_layout_signal LED2 -data 0x2000
+#ftdi_layout_signal LED3 -data 0x4000
+#ftdi_layout_signal LED4 -data 0x8000
+
+# ESP32 series chips do not have a TRST input, and the SRST line is connected
+# to the EN pin. 
+# The target code doesn't handle SRST reset properly yet, so this is
+# commented out:
+# ftdi_layout_signal nSRST -oe 0x0020
+
+reset_config none
+
+# The speed of the JTAG interface, in KHz. If you get DSR/DIR errors (and they
+# do not relate to OpenOCD trying to read from a memory range without physical
+# memory being present there), you can try lowering this.
+#
+# On DevKit-J, this can go as high as 20MHz if CPU frequency is 80MHz, or 26MHz
+# if CPU frequency is 160MHz or 240MHz.
+adapter_khz 20000

esp32/PyJTAG/target/esp32-pycom.cfg

@@ -0,0 +1,69 @@
+# The ESP32 only supports JTAG.
+transport select jtag
+
+# Source the ESP common configuration file
+source [find target/esp_common.cfg]
+
+if { [info exists CHIPNAME] } {
+	set _CHIPNAME $CHIPNAME
+} else {
+	set _CHIPNAME esp32
+}
+
+if { [info exists CPUTAPID] } {
+	set _CPUTAPID $CPUTAPID
+} else {
+	set _CPUTAPID 0x120034e5
+}
+
+if { [info exists ESP32_ONLYCPU] } {
+	set _ONLYCPU $ESP32_ONLYCPU
+} else {
+	set _ONLYCPU 3
+}
+
+if { [info exists ESP32_FLASH_VOLTAGE] } {
+	set _FLASH_VOLTAGE $ESP32_FLASH_VOLTAGE
+} else {
+	set _FLASH_VOLTAGE 3.3
+}
+
+set _TARGETNAME $_CHIPNAME
+set _CPU0NAME cpu0
+set _CPU1NAME cpu1
+set _TAPNAME $_CHIPNAME.$_CPU0NAME
+
+jtag newtap $_CHIPNAME $_CPU0NAME -irlen 5 -expected-id $_CPUTAPID
+if { $_ONLYCPU != 1 } {
+	jtag newtap $_CHIPNAME $_CPU1NAME -irlen 5 -expected-id $_CPUTAPID
+} else {
+	jtag newtap $_CHIPNAME $_CPU1NAME -irlen 5 -disable -expected-id $_CPUTAPID
+}
+
+if { $_RTOS == "none" } {
+	target create $_TARGETNAME esp32 -endian little -chain-position $_TAPNAME
+} else {
+	target create $_TARGETNAME esp32 -endian little -chain-position $_TAPNAME -rtos $_RTOS
+}
+
+configure_esp_workarea $_TARGETNAME 0x40090000 0x3400 0x3FFC0000 0x6000
+configure_esp_flash_bank $_TARGETNAME $_TARGETNAME $_FLASH_SIZE
+
+esp32 flashbootstrap $_FLASH_VOLTAGE
+esp32 maskisr on
+if { $_SEMIHOST_BASEDIR != "" } {
+	esp32 semihost_basedir $_SEMIHOST_BASEDIR
+}
+if { $_FLASH_SIZE == 0 } {
+	gdb_breakpoint_override hard
+}
+
+# special function to program ESP32, it differs from the original 'program' that
+# it verifies written image by reading flash directly, instead of reading memory mapped flash regions
+proc program_esp32 {filename args} {
+	program_esp $filename $args
+}
+
+add_help_text program_esp32 "write an image to flash, address is only required for binary images. verify, reset, exit are optional"
+add_usage_text program_esp32 "<filename> \[address\] \[verify\] \[reset\] \[exit\]"
+