Merge branch 'feature/add_utf_8_decoding_v5.2' into 'release/v5.2'

feat(tools): Enforced utf-8 encoding with Python open() functions (v5.2)

See merge request espressif/esp-idf!36127

components/efuse/efuse_table_gen.py

@@ -128,13 +128,13 @@ class FuseTable(list):
                 field_name = p.field_name + p.group
                 if field_name != '' and len(duplicates.intersection([field_name])) != 0:
                     fl_error = True
-                    print('Field at %s, %s, %s, %s have dublicate field_name' %
+                    print('Field at %s, %s, %s, %s have duplicate field_name' %
                           (p.field_name, p.efuse_block, p.bit_start, p.bit_count))
             if fl_error is True:
                 raise InputError('Field names must be unique')
 
     def check_struct_field_name(self):
-        # check that stuctured fields have a root field
+        # check that structured fields have a root field
         for p in self:
             if '.' in p.field_name:
                 name = ''
@@ -456,7 +456,7 @@ def process_input_file(file, type_table):
 
 def ckeck_md5_in_file(md5, filename):
     if os.path.exists(filename):
-        with open(filename, 'r') as f:
+        with open(filename, 'r', encoding='utf-8') as f:
             for line in f:
                 if md5 in line:
                     return True
@@ -480,12 +480,12 @@ def create_output_files(name, output_table, debug):
     if ckeck_md5_in_file(output_table.md5_digest_table, file_c_path) is False:
         status('Creating efuse *.h file ' + file_h_path + ' ...')
         output = output_table.to_header(file_name)
-        with open(file_h_path, 'w') as f:
+        with open(file_h_path, 'w', encoding='utf-8') as f:
             f.write(output)
 
         status('Creating efuse *.c file ' + file_c_path + ' ...')
         output = output_table.to_c_file(file_name, debug)
-        with open(file_c_path, 'w') as f:
+        with open(file_c_path, 'w', encoding='utf-8') as f:
             f.write(output)
     else:
         print('Source files do not require updating correspond to csv file.')

components/esp_hw_support/test_apps/esp_hw_support_unity_tests/main/gen_digital_signature_tests.py

@@ -67,7 +67,7 @@ def generate_tests_cases(target):  # type: (str) -> None
 
     messages = [random.randrange(0, 1 << max_key_size) for x in range(NUM_MESSAGES)]
 
-    with open('digital_signature_test_cases.h', 'w') as f:
+    with open('digital_signature_test_cases.h', 'w', encoding='utf-8') as f:
         f.write('/*\n')
         year = datetime.datetime.now().year
         f.write(' * SPDX-FileCopyrightText: {year} Espressif Systems (Shanghai) CO LTD\n'.format(year=year))

components/esp_system/check_system_init_priorities.py

@@ -50,7 +50,7 @@ def main() -> None:
         glob_iter = glob.glob(os.path.join(idf_path, 'components', '**', f'*.{extension}'), recursive=True)
         source_files_iters.append(glob_iter)
     for filename in itertools.chain(*source_files_iters):
-        with open(filename, 'r') as f_obj:
+        with open(filename, 'r', encoding='utf-8') as f_obj:
             file_contents = f_obj.read()
         if ESP_SYSTEM_INIT_FN_STR not in file_contents:
             continue
@@ -80,7 +80,7 @@ def main() -> None:
     # 3. Load startup entries list from STARTUP_ENTRIES_FILE, removing comments and empty lines
     #
     startup_entries_expected_lines = []
-    with open(os.path.join(idf_path, STARTUP_ENTRIES_FILE), 'r') as startup_entries_expected_file:
+    with open(os.path.join(idf_path, STARTUP_ENTRIES_FILE), 'r', encoding='utf-8') as startup_entries_expected_file:
         for line in startup_entries_expected_file:
             if line.startswith('#') or len(line.strip()) == 0:
                 continue

components/espcoredump/espcoredump.py

@@ -4,7 +4,6 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 #
-
 import json
 import logging
 import os.path
@@ -26,7 +25,7 @@ def get_prefix_map_gdbinit_path(prog_path):  # type: (str) -> Any
         logging.warning('%s does not exist. Please build the app with "idf.py build"', desc_path)
         return ''
 
-    with open(desc_path, 'r') as f:
+    with open(desc_path, 'r', encoding='utf-8') as f:
         project_desc = json.load(f)
 
     return project_desc.get('debug_prefix_map_gdbinit')

components/partition_table/gen_esp32part.py

@@ -591,7 +591,7 @@ def main():
 
     if input_is_binary:
         output = table.to_csv()
-        with sys.stdout if args.output == '-' else open(args.output, 'w') as f:
+        with sys.stdout if args.output == '-' else open(args.output, 'w', encoding='utf-8') as f:
             f.write(output)
     else:
         output = table.to_binary()

components/partition_table/gen_extra_subtypes_inc.py

@@ -1,14 +1,13 @@
 #!/usr/bin/env python
 # SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 import argparse
 
 
 def gen_header_file(path: str, subtypes: str) -> None:
     HDR_MESSAGE = '/* Automatically generated file. DO NOT EDIT. */\n\n'
     PARTTOOL_USAGE = 'If you want to use parttool.py manually, please use the following as an extra argument:'
-    with open(path, 'w') as f:
+    with open(path, 'w', encoding='utf-8') as f:
         f.write(HDR_MESSAGE)
         if subtypes:
             f.write('/*\n\t' + PARTTOOL_USAGE + '\n\t')

components/partition_table/parttool.py

@@ -92,7 +92,7 @@ class ParttoolTarget():
                     partition_table = gen.PartitionTable.from_binary(f.read())
 
             if partition_table is None:
-                with open(partition_table_file, 'r') as f:
+                with open(partition_table_file, 'r', encoding='utf-8') as f:
                     f.seek(0)
                     partition_table = gen.PartitionTable.from_csv(f.read())
         else:

components/ulp/esp32ulp_mapgen.py

@@ -1,12 +1,9 @@
 #!/usr/bin/env python
+# SPDX-FileCopyrightText: 2016-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+#
 # esp32ulp_mapgen utility converts a symbol list provided by nm into an export script
 # for the linker and a header file.
-#
-# SPDX-FileCopyrightText: 2016-2021 Espressif Systems (Shanghai) CO LTD
-# SPDX-License-Identifier: Apache-2.0
-
-from __future__ import print_function
-
 import argparse
 import os
 import textwrap
@@ -65,7 +62,7 @@ def main() -> None:
 
     args = parser.parse_args()
 
-    with open(args.outputfile + '.h', 'w') as f_h, open(args.outputfile + '.ld', 'w') as f_ld:
+    with open(args.outputfile + '.h', 'w', encoding='utf-8') as f_h, open(args.outputfile + '.ld', 'w', encoding='utf-8') as f_ld:
         gen_ld_h_from_sym(args.symfile, f_ld, f_h, int(args.base_addr, 0))