Merge branch 'feature/add_utf_8_decoding_v5.3' into 'release/v5.3'

feat(tools): Enforced utf-8 encoding with Python open() functions (v5.3)

See merge request espressif/esp-idf!36126

tools/idf_py_actions/create_ext.py

@@ -1,14 +1,12 @@
 # SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-from __future__ import print_function
-
 import os
 import re
 import sys
-from distutils.dir_util import copy_tree
 from typing import Dict
 
 import click
+from distutils.dir_util import copy_tree
 from idf_py_actions.tools import PropertyDict
 
 
@@ -17,7 +15,7 @@ def get_type(action: str) -> str:
 
 
 def replace_in_file(filename: str, pattern: str, replacement: str) -> None:
-    with open(filename, 'r+') as f:
+    with open(filename, 'r+', encoding='utf-8') as f:
         content = f.read()
         overwritten_content = re.sub(pattern, replacement, content, flags=re.M)
         f.seek(0)

tools/idf_py_actions/debug_ext.py

@@ -70,7 +70,7 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict:
                     if p.poll() is not None:
                         print('OpenOCD exited with {}'.format(p.poll()))
                         break
-                    with open(name, 'r') as f:
+                    with open(name, 'r', encoding='utf-8') as f:
                         content = f.read()
                         if re.search(r'Listening on port \d+ for gdb connections', content):
                             # expect OpenOCD has started successfully - stop watching
@@ -78,7 +78,7 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict:
                     time.sleep(0.5)
 
                 # OpenOCD exited or is not listening -> print full log and terminate
-                with open(name, 'r') as f:
+                with open(name, 'r', encoding='utf-8') as f:
                     print(f.read())
 
                 raise FatalError('Action "{}" failed due to errors in OpenOCD'.format(target), ctx)
@@ -194,7 +194,7 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict:
                 name = processes[target + '_outfile_name']
                 pos = 0
                 while True:
-                    with open(name, 'r') as f:
+                    with open(name, 'r', encoding='utf-8') as f:
                         f.seek(pos)
                         for line in f:
                             print(line.rstrip())
@@ -212,7 +212,7 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict:
         desc_path = os.path.join(args.build_dir, 'project_description.json')
         if not os.path.exists(desc_path):
             ensure_build_directory(args, ctx.info_name)
-        with open(desc_path, 'r') as f:
+        with open(desc_path, 'r', encoding='utf-8') as f:
             project_desc = json.load(f)
             return project_desc
 
@@ -237,7 +237,7 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict:
         local_dir = project_desc['build_dir']
         args = ['openocd'] + shlex.split(openocd_arguments)
         openocd_out_name = os.path.join(local_dir, OPENOCD_OUT_FILE)
-        openocd_out = open(openocd_out_name, 'w')
+        openocd_out = open(openocd_out_name, 'w', encoding='utf-8')
         try:
             process = subprocess.Popen(args, stdout=openocd_out, stderr=subprocess.STDOUT, bufsize=1)
         except Exception as e:
@@ -352,7 +352,7 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict:
         if gdbgui_port is not None:
             gdbgui_args += ['--port', gdbgui_port]
         gdbgui_out_name = os.path.join(local_dir, GDBGUI_OUT_FILE)
-        gdbgui_out = open(gdbgui_out_name, 'w')
+        gdbgui_out = open(gdbgui_out_name, 'w', encoding='utf-8')
         env = os.environ.copy()
         # The only known solution for https://github.com/cs01/gdbgui/issues/359 is to set the following environment
         # variable. The greenlet package cannot be downgraded for compatibility with other requirements (gdbgui,

tools/idf_py_actions/qemu_ext.py

@@ -188,7 +188,7 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict:
         desc_path = os.path.join(args.build_dir, 'project_description.json')
         if not os.path.exists(desc_path):
             ensure_build_directory(args, ctx.info_name)
-        with open(desc_path, 'r') as f:
+        with open(desc_path, 'r', encoding='utf-8') as f:
             project_desc = json.load(f)
         return project_desc
 

tools/idf_py_actions/serial_ext.py

@@ -51,7 +51,7 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict:
         desc_path = os.path.join(args.build_dir, 'project_description.json')
         if not os.path.exists(desc_path):
             ensure_build_directory(args, ctx.info_name)
-        with open(desc_path, 'r') as f:
+        with open(desc_path, 'r', encoding='utf-8') as f:
             project_desc = json.load(f)
         return project_desc
 
@@ -67,7 +67,7 @@ def action_extensions(base_actions: Dict, project_path: str) -> Dict:
         result += ['-p', args.port]
         result += ['-b', str(args.baud)]
 
-        with open(os.path.join(args.build_dir, 'flasher_args.json')) as f:
+        with open(os.path.join(args.build_dir, 'flasher_args.json'), encoding='utf-8') as f:
             flasher_args = json.load(f)
 
         extra_esptool_args = flasher_args['extra_esptool_args']

tools/idf_py_actions/tools.py

@@ -64,7 +64,7 @@ def _set_build_context(args: 'PropertyDict') -> None:
 
     proj_desc_fn = f'{args.build_dir}/project_description.json'
     try:
-        with open(proj_desc_fn, 'r') as f:
+        with open(proj_desc_fn, 'r', encoding='utf-8') as f:
             ctx['proj_desc'] = json.load(f)
     except (OSError, ValueError) as e:
         raise FatalError(f'Cannot load {proj_desc_fn}: {e}')
@@ -85,7 +85,7 @@ def _idf_version_from_cmake() -> Optional[str]:
     regex = re.compile(r'^\s*set\s*\(\s*IDF_VERSION_([A-Z]{5})\s+(\d+)')
     ver = {}
     try:
-        with open(version_path) as f:
+        with open(version_path, encoding='utf-8') as f:
             for line in f:
                 m = regex.match(line)
 
@@ -183,7 +183,7 @@ def load_hints() -> Dict:
     }
 
     current_module_dir = os.path.dirname(__file__)
-    with open(os.path.join(current_module_dir, 'hints.yml'), 'r') as file:
+    with open(os.path.join(current_module_dir, 'hints.yml'), 'r', encoding='utf-8') as file:
         hints['yml'] = yaml.safe_load(file)
 
     hint_modules_dir = os.path.join(current_module_dir, 'hint_modules')
@@ -257,7 +257,7 @@ def generate_hints(*filenames: str) -> Generator:
     """Getting output files and printing hints on how to resolve errors based on the output."""
     hints = load_hints()
     for file_name in filenames:
-        with open(file_name, 'r') as file:
+        with open(file_name, 'r', encoding='utf-8') as file:
             yield from generate_hints_buffer(file.read(), hints)
 
 
@@ -685,7 +685,7 @@ def get_sdkconfig_filename(args: 'PropertyDict', cache_cmdl: Optional[Dict]=None
 
     proj_desc_path = os.path.join(args.build_dir, 'project_description.json')
     try:
-        with open(proj_desc_path, 'r') as f:
+        with open(proj_desc_path, 'r', encoding='utf-8') as f:
             proj_desc = json.load(f)
         return str(proj_desc['config_file'])
     except (OSError, KeyError):
@@ -706,7 +706,7 @@ def get_sdkconfig_value(sdkconfig_file: str, key: str) -> Optional[str]:
     value = None
     # if the value is quoted, this excludes the quotes from the value
     pattern = re.compile(r"^{}=\"?([^\"]*)\"?$".format(key))
-    with open(sdkconfig_file, 'r') as f:
+    with open(sdkconfig_file, 'r', encoding='utf-8') as f:
         for line in f:
             match = re.match(pattern, line)
             if match: