Merge branch 'feat/adopt_fast_reflashing' into 'master'

feat(idf_py): idf.py flash will reflash only changed data sectors by default

Closes IDF-11871

See merge request espressif/esp-idf!45542

components/esp_system/ld/ld.common

@@ -116,23 +116,23 @@ ASSERT((ADDR(NEXT_SECTION) == ADDR(PREV_SECTION) + SIZEOF(PREV_SECTION)), \
 #endif
 
 /*
- * FAST_REFLASHING_PADDING
+ * APP_BUILD_MINIMIZE_BINARY_CHANGES_PADDING
  *
- * Aligns the current location (.) to CONFIG_ESPTOOLPY_FAST_REFLASHING_PADDING.
+ * Aligns the current location (.) to CONFIG_APP_BUILD_MINIMIZE_BINARY_CHANGES_PADDING.
  * If more than half of the alignment block is already consumed,
  * extra padding is applied to skip to the next boundary.
  * For example, if the current location (.) is 0x1900 and the padding is 4 KB,
  * the location will be moved to 0x3000 instead of 0x2000 because less than
  * half of the current 4 KB block remains (0x2000 - 0x1900 < 2 KB).
  */
-#if CONFIG_ESPTOOLPY_FAST_REFLASHING
-#define FAST_REFLASHING_PADDING() \
+#if CONFIG_APP_BUILD_MINIMIZE_BINARY_CHANGES
+#define APP_BUILD_MINIMIZE_BINARY_CHANGES_PADDING() \
   . = ALIGN((. + ( \
-    (CONFIG_ESPTOOLPY_FAST_REFLASHING_PADDING - (ALIGN(CONFIG_ESPTOOLPY_FAST_REFLASHING_PADDING) - .)) + \
-    (CONFIG_ESPTOOLPY_FAST_REFLASHING_PADDING / 2) \
-  )), CONFIG_ESPTOOLPY_FAST_REFLASHING_PADDING);
+    (CONFIG_APP_BUILD_MINIMIZE_BINARY_CHANGES_PADDING - (ALIGN(CONFIG_APP_BUILD_MINIMIZE_BINARY_CHANGES_PADDING) - .)) + \
+    (CONFIG_APP_BUILD_MINIMIZE_BINARY_CHANGES_PADDING / 2) \
+  )), CONFIG_APP_BUILD_MINIMIZE_BINARY_CHANGES_PADDING);
 #else
-#define FAST_REFLASHING_PADDING()
+#define APP_BUILD_MINIMIZE_BINARY_CHANGES_PADDING()
 #endif
 
 #if CONFIG_LIBC_PICOLIBC
@@ -153,4 +153,4 @@ ASSERT((ADDR(NEXT_SECTION) == ADDR(PREV_SECTION) + SIZEOF(PREV_SECTION)), \
 
 #define SECTION_MAPPINGS_WITH_PADDING(SECTION_NAME) \
 \n  SECTION_MAPPINGS(SECTION_NAME) \
-\n  FAST_REFLASHING_PADDING()
+\n  APP_BUILD_MINIMIZE_BINARY_CHANGES_PADDING()

components/esp_system/ld/ld.dram.sections

@@ -95,7 +95,7 @@
 
     mapping[dram0_data]
     mutable[dram0_data]
-    FAST_REFLASHING_PADDING();
+    APP_BUILD_MINIMIZE_BINARY_CHANGES_PADDING();
 
     _data_end_high = ABSOLUTE(.);
   } > dram_high_seg

components/esptool_py/Kconfig.projbuild

@@ -214,64 +214,4 @@ menu "Serial flasher config"
         default ESP_CONSOLE_UART_BAUDRATE if ESP_CONSOLE_UART
         default 115200 if !ESP_CONSOLE_UART
 
-    menu "Fast Reflashing"
-
-        config ESPTOOLPY_FAST_REFLASHING
-            bool "Enable fast reflashing (Experimental)"
-            depends on IDF_EXPERIMENTAL_FEATURES
-            select COMPILER_NO_MERGE_CONSTANTS if IDF_TOOLCHAIN_GCC
-            help
-                Enabling this option classifies component libraries into two
-                categories: mutable and immutable. Mutable libraries are
-                expected to change frequently during development, while
-                immutable libraries are considered stable. All project
-                component libraries are treated as mutable; all other libraries
-                are treated as immutable.
-
-                In the generated linker script, input sections from immutable
-                libraries are placed before those from mutable libraries. This
-                layout helps localize changes in the output sections of the ELF
-                file, so that recompilation primarily affects a confined area
-                associated with the mutable libraries.
-
-                This enables the generation of binary images with large
-                unmodified regions across recompilations, allowing to flash
-                only the parts that have changed.
-
-                To support this, some compiler optimizations, such as constant
-                merging, are disabled, which helps minimize differences in the
-                binary image between builds.  However, this may increase the
-                overall image size and flash usage.
-
-                Additionally, this option inserts a padding, defined by the
-                ESPTOOLPY_FAST_REFLASHING_PADDING option, after the input
-                sections of mutable libraries. This allows further isolate
-                changes but also increases the size of the binary image and
-                flash consumption.
-
-        choice ESPTOOLPY_FAST_REFLASHING_PADDING_SIZE
-            prompt "Padding size"
-            default ESPTOOLPY_FAST_REFLASHING_PADDING_4KB
-            depends on ESPTOOLPY_FAST_REFLASHING
-            help
-                Selects the amount of padding (in kilobytes) to add after input
-                sections of mutable libraries when fast reflashing is enabled.
-
-            config ESPTOOLPY_FAST_REFLASHING_PADDING_2KB
-                bool "2 KB"
-            config ESPTOOLPY_FAST_REFLASHING_PADDING_4KB
-                bool "4 KB"
-            config ESPTOOLPY_FAST_REFLASHING_PADDING_8KB
-                bool "8 KB"
-        endchoice
-
-        config ESPTOOLPY_FAST_REFLASHING_PADDING
-            int
-            depends on ESPTOOLPY_FAST_REFLASHING
-            default 2048 if ESPTOOLPY_FAST_REFLASHING_PADDING_2KB
-            default 4096 if ESPTOOLPY_FAST_REFLASHING_PADDING_4KB
-            default 8192 if ESPTOOLPY_FAST_REFLASHING_PADDING_8KB
-
-    endmenu
-
 endmenu

components/esptool_py/project_include.cmake

@@ -197,6 +197,8 @@ function(esptool_py_flash_target target_name main_args sub_args)
         -D "IDF_PATH=${idf_path}"
         -D "SERIAL_TOOL=${esptool_py_cmd}"
         -D "SERIAL_TOOL_ARGS=${main_args};write-flash;@${filename_prefix}_args"
+        -D "SERIAL_TOOL_IS_WRITE_FLASH=1"
+        -D "SERIAL_TOOL_FLASH_ARGS_FILE=${filename_prefix}_args"
         -D "WORKING_DIRECTORY=${build_dir}"
         -P ${esptool_py_dir}/run_serial_tool.cmake
         WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
@@ -243,6 +245,8 @@ $<JOIN:$<TARGET_PROPERTY:${target_name},IMAGES>,\n>")
             -D "IDF_PATH=${idf_path}"
             -D "SERIAL_TOOL=${esptool_py_cmd}"
             -D "SERIAL_TOOL_ARGS=${main_args};write-flash;@encrypted_${filename_prefix}_args"
+            -D "SERIAL_TOOL_IS_WRITE_FLASH=1"
+            -D "SERIAL_TOOL_FLASH_ARGS_FILE=encrypted_${filename_prefix}_args"
             -D "WORKING_DIRECTORY=${build_dir}"
             -P ${esptool_py_dir}/run_serial_tool.cmake
             WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}

components/esptool_py/run_serial_tool.cmake

@@ -22,11 +22,60 @@ if(DEFINED ENV{IDF_ENV_FPGA})
     message("Note: IDF_ENV_FPGA is set, propagating to esptool with ESPTOOL_ENV_FPGA = 1")
 endif()
 
-set(serial_tool_cmd ${SERIAL_TOOL})
+# Extract binary file paths from esptool argfile content.
+# The argfile contains "0xOFFSET file" pairs; this returns just the file paths.
+function(_esptoolpy_extract_flash_files args_content out_var)
+    string(REPLACE "\r\n" "\n" args_content "${args_content}")
+    separate_arguments(_tokens NATIVE_COMMAND "${args_content}")
+    set(_result "")
+    set(_expect_file FALSE)
+    foreach(_tok IN LISTS _tokens)
+        if(_expect_file)
+            if(NOT _tok MATCHES "^-")
+                list(APPEND _result "${_tok}")
+            endif()
+            set(_expect_file FALSE)
+        elseif(_tok MATCHES "^0x[0-9a-fA-F]+$")
+            set(_expect_file TRUE)
+        endif()
+    endforeach()
+    set(${out_var} "${_result}" PARENT_SCOPE)
+endfunction()
+
+# Fast-reflashing support (esptool --diff-with and --skip-flashed).
+#
+# Build-time inputs (passed as -D flags from project_include.cmake):
+#   SERIAL_TOOL_IS_WRITE_FLASH  — set to 1 for write-flash targets
+#   SERIAL_TOOL_FLASH_ARGS_FILE — argfile name relative to WORKING_DIRECTORY
+#
+# Runtime inputs (environment variables, can change without reconfiguring CMake):
+#   IDF_FLASH_FULL=1            — disable fast reflash (idf.py flash -a/--all)
+#   IDF_TRUST_FLASH_CONTENT=1   — skip MD5 verification of unchanged files (idf.py flash -t)
+
+set(_flash_full FALSE)
+if(DEFINED ENV{IDF_FLASH_FULL} AND "$ENV{IDF_FLASH_FULL}" STREQUAL "1")
+    set(_flash_full TRUE)
+endif()
+
+set(_trust_flash_content FALSE)
+if(DEFINED ENV{IDF_TRUST_FLASH_CONTENT} AND "$ENV{IDF_TRUST_FLASH_CONTENT}" STREQUAL "1")
+    set(_trust_flash_content TRUE)
+endif()
+
+# Read the flash files list once (used for both pre-flash --diff-with and post-flash saving).
+set(_flash_files "")
+if(SERIAL_TOOL_IS_WRITE_FLASH AND SERIAL_TOOL_FLASH_ARGS_FILE
+        AND DEFINED WORKING_DIRECTORY AND WORKING_DIRECTORY
+        AND EXISTS "${WORKING_DIRECTORY}/${SERIAL_TOOL_FLASH_ARGS_FILE}")
+    file(READ "${WORKING_DIRECTORY}/${SERIAL_TOOL_FLASH_ARGS_FILE}" _args_content)
+    _esptoolpy_extract_flash_files("${_args_content}" _flash_files)
+endif()
 
 # Main purpose of this script: we can't expand these environment variables in the main IDF CMake build,
 # because we want to expand them at flashing time not at CMake runtime (so they can change
 # without needing a CMake re-run)
+set(serial_tool_cmd ${SERIAL_TOOL})
+
 set(ESPPORT $ENV{ESPPORT})
 if(NOT ESPPORT)
     message("Note: ${SERIAL_TOOL} will search for a serial port. "
@@ -43,26 +92,69 @@ else()
     list(APPEND serial_tool_cmd -b ${ESPBAUD})
 endif()
 
-# SERIAL_TOOL_ARGS is defined during the first cmake run
-# EXTRA_ARGS and EXTRA_PRE_CMD_ARGS are used for additional arguments from the command line during run-time
 list(APPEND serial_tool_cmd $ENV{SERIAL_TOOL_EXTRA_PRE_CMD_ARGS})
 list(APPEND serial_tool_cmd ${SERIAL_TOOL_ARGS})
+
+# Append fast-reflashing arguments when flash files are available.
+if(_flash_files AND NOT _flash_full)
+    # Build --diff-with list: use *_flashed.bin if present, otherwise "skip".
+    set(_diff_with_files "")
+    set(_have_any FALSE)
+    foreach(_file IN LISTS _flash_files)
+        string(REGEX REPLACE "\\.bin$" "_flashed.bin" _ref "${_file}")
+        if(EXISTS "${WORKING_DIRECTORY}/${_ref}")
+            list(APPEND _diff_with_files "${_ref}")
+            set(_have_any TRUE)
+        else()
+            list(APPEND _diff_with_files "skip")
+        endif()
+    endforeach()
+
+    if(_have_any)
+        list(APPEND serial_tool_cmd "--diff-with" ${_diff_with_files})
+        if(_trust_flash_content)
+            list(APPEND serial_tool_cmd "--trust-flash-content")
+        endif()
+    else()
+        if(_trust_flash_content)
+            message(WARNING "No previously flashed binaries found in build directory, "
+                    "--trust-flash-content will be ignored.")
+        endif()
+        list(APPEND serial_tool_cmd "--skip-flashed")
+    endif()
+endif()
+
 list(APPEND serial_tool_cmd $ENV{SERIAL_TOOL_EXTRA_ARGS})
 
+# Run the serial tool.
+set(_execute_extra_args "")
 if(${SERIAL_TOOL_SILENT})
-    execute_process(COMMAND ${serial_tool_cmd}
-        WORKING_DIRECTORY "${WORKING_DIRECTORY}"
-        RESULT_VARIABLE result
-        OUTPUT_VARIABLE SERIAL_TOOL_OUTPUT_LOG
-    )
-else()
-    execute_process(COMMAND ${serial_tool_cmd}
-        WORKING_DIRECTORY "${WORKING_DIRECTORY}"
-        RESULT_VARIABLE result
-    )
+    list(APPEND _execute_extra_args OUTPUT_VARIABLE SERIAL_TOOL_OUTPUT_LOG)
 endif()
 
+execute_process(COMMAND ${serial_tool_cmd}
+    WORKING_DIRECTORY "${WORKING_DIRECTORY}"
+    RESULT_VARIABLE result
+    ${_execute_extra_args}
+)
+
 if(${result})
-    # No way to have CMake silently fail, unfortunately
     message(FATAL_ERROR "${SERIAL_TOOL} failed. \n${SERIAL_TOOL_OUTPUT_LOG}")
 endif()
+
+# After successful flash, save copies of all flashed files as *_flashed.bin
+# for use with --diff-with on the next reflash.
+foreach(_file IN LISTS _flash_files)
+    if(EXISTS "${WORKING_DIRECTORY}/${_file}")
+        string(REGEX REPLACE "\\.bin$" "_flashed.bin" _dst "${_file}")
+        file(COPY_FILE
+            "${WORKING_DIRECTORY}/${_file}"
+            "${WORKING_DIRECTORY}/${_dst}"
+            ONLY_IF_DIFFERENT
+            RESULT _copy_result)
+        if(NOT _copy_result EQUAL "0")
+            message(WARNING "Could not save ${_dst}: ${_copy_result}, "
+                "next flash will run without comparison data.")
+        endif()
+    endif()
+endforeach()