Mirrors/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2026-08-18 06:35:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix: Improved encoding requirement check

Browse Source 
Closes https://github.com/espressif/esp-idf/issues/17542
This commit is contained in:
Jakub Kocka
2025-11-06 14:04:22 +01:00
parent 49aa2d5bd7
commit 73cfe97054
2 changed files with 140 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
tools/idf.py
View File

@@ -978,47 +978,94 @@ def expand_file_arguments(argv: list[Any]) -> list[Any]:
return argv
def _valid_unicode_config() -> codecs.CodecInfo | bool:
# With python 3 unicode environment is required
def _valid_unicode_config() -> bool:
try:
return codecs.lookup(locale.getpreferredencoding()).name != 'ascii'
locale_name, encoding = locale.getlocale()
if encoding is None or locale_name is None:
return False
encoding_name = codecs.lookup(encoding).name
# LC_CTYPE has issues with case conversion for ASCII characters in these locales
if locale_name.lower().startswith(('tr', 'az', 'lt', 'kk')):
# Turkish (tr) & Azerbaijani (az): dotless I (ı/I vs i/İ) breaks strcasecmp for "INTERRUPT"
# https://unicode.org/mail-arch/unicode-ml/Archives-Old/UML009/0619.html
# https://en.wikipedia.org/wiki/Dotted_and_dotless_I_in_computing
# Lithuanian (lt): special rules for i with accents affect case conversion
# Kazakh (kk): may use dotless I in Latin script
# These cause issues when parsing assembly code with strcasecmp on registers/opcodes
# so we are introducing this workaround to change the locale to a Unicode-capable one
return False
# Check if the encoding is Unicode-capable
return encoding_name.lower().startswith('utf')
except Exception:
return False
def _find_usable_locale() -> str:
def _find_usable_locale() -> str | None:
"""
Find the best locale for Unicode support.
All the locales available on the system (via locale -a) are checked.
The unicode locales are filtered.
The best locale is selected based on the proiority (the lowest number, the higher priority):
- user preferred Unicode (priority -1)
- c.utf8 (priority 0)
- universal Unicode (priority 1)
- any english Unicode (priority 2)
this concludes for the priority <= 1 to be sufficient in the search
"""
locale_name, encoding = locale.getlocale()
err_msg = (
'Support for Unicode is required, '
'locale '
f'with {str(encoding) if locale_name is not None else "default"} ' # C locale can be interpreted as None
'encoding found on your system is not sufficient.'
' Please refer to the manual for your operating system for details on locale installation.'
)
# Get available locales from the system
try:
locales = (
subprocess.Popen(['locale', '-a'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
.communicate()[0]
.decode('ascii', 'replace')
)
except OSError:
locales = ''
result = subprocess.run(['locale', '-a'], capture_output=True, text=True, check=True)
system_locales = result.stdout.strip().split('\n')
except (subprocess.CalledProcessError, FileNotFoundError):
raise FatalError('Failed to get available locales from the system')
usable_locales: list[str] = []
for line in locales.splitlines():
locale = line.strip()
locale_name = locale.lower().replace('-', '')
# (priority, locale_name, encoding) - lower priority = higher preference
best_locale: tuple[int, str, str] | None = None
found_locale: tuple[int, str, str] | None = None
for lcl in system_locales:
lcl = lcl.strip()
if 'utf' in str(lcl).lower().replace('-', ''):
# filter unicode locales
lcl_alias_name = lcl.lower()
if str(locale_name).lower().replace(' ', '-') in lcl_alias_name:
# user preferred language has unicode encoding (highest priority -1)
if str(locale_name).lower().startswith(('tr', 'az', 'lt', 'kk')):
print_warning(
f'Your locale "{locale_name}" has potential issues with case conversion for ASCII characters'
)
continue
found_locale = (-1, lcl_alias_name, lcl)
elif 'c.utf8' == lcl_alias_name:
# c.utf should be on most systems (second priority 0)
found_locale = (0, lcl_alias_name, lcl)
elif 'univ' in lcl_alias_name:
# universal unicode locale (third priority 1)
found_locale = (1, lcl_alias_name, lcl)
elif 'en_' in lcl_alias_name:
# any english unicode locale (fourth priority 2)
found_locale = (2, lcl_alias_name, lcl)
if found_locale is not None and (best_locale is None or found_locale[0] < best_locale[0]):
best_locale = found_locale
if best_locale[0] <= 1:
# if there is a hit for the best locale to satisfy the unicode support (priority <= 1)
# we can break the loop and use the best locale
break
# C.UTF-8 is the best option, if supported
if locale_name == 'c.utf8':
return locale
if best_locale is None:
raise FatalError(err_msg)
if locale_name.endswith('.utf8'):
# Make a preference of english locales
if locale.startswith('en_'):
usable_locales.insert(0, locale)
else:
usable_locales.append(locale)
if not usable_locales:
raise FatalError(
'Support for Unicode filenames is required, but no suitable UTF-8 locale was found on your system.'
' Please refer to the manual for your operating system for details on locale reconfiguration.'
)
return usable_locales[0]
# return encoding of the best locale
return str(best_locale[2])
if __name__ == '__main__':
@@ -1029,15 +1076,20 @@ if __name__ == '__main__':
'documentation in order to set up a suitiable environment, or continue at your own risk.'
)
elif os.name == 'posix' and not _valid_unicode_config():
# Trying to find best utf-8 locale available on the system and restart python with it
# Trying to find best unicode locale available on the system and restart python with
best_locale = _find_usable_locale()
print_warning(
'Your environment is not configured to handle unicode filenames outside of ASCII range.'
f' Environment variable LC_ALL is temporary set to {best_locale} for unicode support.'
)
os.environ['LC_ALL'] = best_locale
# Unset LC_ALL if it exists, as it takes precedence over LC_CTYPE
# This prevents infinite loops when LC_ALL is set to a non-Unicode locale
if best_locale:
print_warning(
'Your environment is not configured to handle unicode characters.'
' Environment variable LC_CTYPE is temporary set to '
f'{best_locale} (found on the system) for unicode support.'
)
if 'LC_ALL' in os.environ:
del os.environ['LC_ALL']
os.environ['LC_CTYPE'] = best_locale
ret = subprocess.call([sys.executable] + sys.argv, env=os.environ)
if ret:
raise SystemExit(ret)

49
tools/test_idf_py/test_idf_py.py
View File

@@ -11,6 +11,7 @@ from typing import Any
from unittest import TestCase
from unittest import main
from unittest import mock
from unittest import skipIf
import jsonschema
@@ -623,5 +624,53 @@ class TestUF2Commands(TestWrapperCommands):
os.environ.pop('ESPPORT')
@skipIf(os.name == 'nt', 'Locale handling differs on Windows')
class TestUserLocale(TestWrapperCommands):
"""
Test if user locale check works as expected.
"""
def test_user_locale_no_unicode(self):
import locale
original_locale = locale.getlocale()
try:
for lcl in locale.locale_alias.items():
lcl_encoding = str(lcl[1]).lower().replace('-', '')
if 'utf' not in lcl_encoding:
# Try to set any non-unicode locale - break if successful
try:
locale.setlocale(locale.LC_ALL, lcl[1])
os.environ['LC_ALL'] = lcl[1] # Testing with LC_ALL if it is ignored
os.environ['LC_CTYPE'] = lcl[1]
break
except locale.Error:
pass
command = [sys.executable, idf_py_path, '--help']
try:
output = subprocess.check_output(command, env=os.environ, stderr=subprocess.STDOUT).decode(
'utf-8', 'ignore'
)
except subprocess.CalledProcessError as e:
# Process may exit with code 2 if no UTF locale is found, but warning should still be printed
output = e.output.decode('utf-8', 'ignore')
self.assertTrue(
'Your environment is not configured to handle unicode characters' in output
or 'Support for Unicode is required' in output,
'Expected unicode configuration message not found in output',
)
finally:
try:
locale.setlocale(locale.LC_ALL, original_locale)
except locale.Error:
pass
if 'LC_ALL' in os.environ:
del os.environ['LC_ALL']
if 'LC_CTYPE' in os.environ:
del os.environ['LC_CTYPE']
if __name__ == '__main__':
main()