diff --git a/tools/ci/check_callgraph.py b/tools/ci/check_callgraph.py index ab74b903234..9fcafcab4f6 100755 --- a/tools/ci/check_callgraph.py +++ b/tools/ci/check_callgraph.py @@ -7,103 +7,90 @@ import argparse import fnmatch import os import re +from collections.abc import Callable +from collections.abc import Generator from functools import partial from typing import BinaryIO -from typing import Callable -from typing import Dict -from typing import Generator -from typing import List -from typing import Optional -from typing import Tuple import elftools from elftools.elf import elffile -FUNCTION_REGEX = re.compile( - r'^;; Function (?P.*)\s+\((?P\S+)(,.*)?\).*$' -) +FUNCTION_REGEX = re.compile(r'^;; Function (?P.*)\s+\((?P\S+)(,.*)?\).*$') CALL_REGEX = re.compile(r'^.*\(call.*"(?P.*)".*$') SYMBOL_REF_REGEX = re.compile(r'^.*\(symbol_ref[^()]*\("(?P.*)"\).*$') -class RtlFunction(object): +class RtlFunction: def __init__(self, name: str, rtl_filename: str, tu_filename: str) -> None: self.name = name self.rtl_filename = rtl_filename self.tu_filename = tu_filename - self.refs: List[str] = list() + self.refs: list[str] = list() self.sym = None -class SectionAddressRange(object): +class SectionAddressRange: def __init__(self, name: str, addr: int, size: int) -> None: self.name = name self.low = addr self.high = addr + size def __str__(self) -> str: - return '{}: 0x{:08x} - 0x{:08x}'.format(self.name, self.low, self.high) + return f'{self.name}: 0x{self.low:08x} - 0x{self.high:08x}' def contains_address(self, addr: int) -> bool: return self.low <= addr < self.high -TARGET_SECTIONS: Dict[str, List[SectionAddressRange]] = { +TARGET_SECTIONS: dict[str, list[SectionAddressRange]] = { 'esp32': [ SectionAddressRange('.rom.text', 0x40000000, 0x70000), - SectionAddressRange('.rom.rodata', 0x3ff96000, 0x9018) + SectionAddressRange('.rom.rodata', 0x3FF96000, 0x9018), ], 'esp32s2': [ - SectionAddressRange('.rom.text', 0x40000000, 0x1bed0), - SectionAddressRange('.rom.rodata', 0x3ffac600, 0x392c) + SectionAddressRange('.rom.text', 0x40000000, 0x1BED0), + SectionAddressRange('.rom.rodata', 0x3FFAC600, 0x392C), ], 'esp32s3': [ - SectionAddressRange('.rom.text', 0x40000000, 0x568d0), - SectionAddressRange('.rom.rodata', 0x3ff071c0, 0x8e30) - ] + SectionAddressRange('.rom.text', 0x40000000, 0x568D0), + SectionAddressRange('.rom.rodata', 0x3FF071C0, 0x8E30), + ], } -class Symbol(object): - def __init__(self, name: str, addr: int, local: bool, filename: Optional[str], section: Optional[str]) -> None: +class Symbol: + def __init__(self, name: str, addr: int, local: bool, filename: str | None, section: str | None) -> None: self.name = name self.addr = addr self.local = local self.filename = filename self.section = section - self.refers_to: List[Symbol] = list() - self.referred_from: List[Symbol] = list() + self.refers_to: list[Symbol] = list() + self.referred_from: list[Symbol] = list() def __str__(self) -> str: return '{} @0x{:08x} [{}]{} {}'.format( - self.name, - self.addr, - self.section or 'unknown', - ' (local)' if self.local else '', - self.filename + self.name, self.addr, self.section or 'unknown', ' (local)' if self.local else '', self.filename ) -class Reference(object): +class Reference: def __init__(self, from_sym: Symbol, to_sym: Symbol) -> None: self.from_sym = from_sym self.to_sym = to_sym def __str__(self) -> str: - return '{} @0x{:08x} ({}) -> {} @0x{:08x} ({})'.format( - self.from_sym.name, - self.from_sym.addr, - self.from_sym.section, - self.to_sym.name, - self.to_sym.addr, - self.to_sym.section + return ( + f'{self.from_sym.name} @0x{self.from_sym.addr:08x} ({self.from_sym.section}) -> ' + f'{self.to_sym.name} @0x{self.to_sym.addr:08x} ({self.to_sym.section})' ) -class IgnorePair(): +class IgnorePair: """ A pair of symbol names which should be ignored when checking references. """ + def __init__(self, pair: str) -> None: try: self.source, self.dest = pair.split('/') @@ -111,14 +98,14 @@ class IgnorePair(): raise ValueError(f'Invalid ignore pair: {pair}. Must be in the form "source/dest".') -class ElfInfo(object): +class ElfInfo: def __init__(self, elf_file: BinaryIO) -> None: self.elf_file = elf_file self.elf_obj = elffile.ELFFile(self.elf_file) self.section_ranges = self._load_sections() self.symbols = self._load_symbols() - def _load_symbols(self) -> List[Symbol]: + def _load_symbols(self) -> list[Symbol]: symbols = [] for s in self.elf_obj.iter_sections(): if not isinstance(s, elftools.elf.sections.SymbolTableSection): @@ -142,18 +129,14 @@ class ElfInfo(object): ) return symbols - def _load_sections(self) -> List[SectionAddressRange]: + def _load_sections(self) -> list[SectionAddressRange]: result = [] for segment in self.elf_obj.iter_segments(): if segment['p_type'] == 'PT_LOAD': for section in self.elf_obj.iter_sections(): if not segment.section_in_segment(section): continue - result.append( - SectionAddressRange( - section.name, section['sh_addr'], section['sh_size'] - ) - ) + result.append(SectionAddressRange(section.name, section['sh_addr'], section['sh_size'])) target = os.environ.get('IDF_TARGET') if target in TARGET_SECTIONS: @@ -161,22 +144,22 @@ class ElfInfo(object): return result - def symbols_by_name(self, name: str) -> List['Symbol']: + def symbols_by_name(self, name: str) -> list['Symbol']: res = [] for sym in self.symbols: if sym.name == name: res.append(sym) return res - def section_for_addr(self, sym_addr: int) -> Optional[str]: + def section_for_addr(self, sym_addr: int) -> str | None: for sar in self.section_ranges: if sar.contains_address(sym_addr): return sar.name return None -def load_rtl_file(rtl_filename: str, tu_filename: str, functions: List[RtlFunction]) -> None: - last_function: Optional[RtlFunction] = None +def load_rtl_file(rtl_filename: str, tu_filename: str, functions: list[RtlFunction]) -> None: + last_function: RtlFunction | None = None for line in open(rtl_filename): # Find function definition match = re.match(FUNCTION_REGEX, line) @@ -212,7 +195,7 @@ def rtl_filename_matches_sym_filename(rtl_filename: str, symbol_filename: str) - return os.path.basename(rtl_filename).startswith(symbol_filename) -def filter_ignore_pairs(function: RtlFunction, ignore_pairs: List[IgnorePair]) -> None: +def filter_ignore_pairs(function: RtlFunction, ignore_pairs: list[IgnorePair]) -> None: """ Given a function S0 and a list of ignore pairs (S, T), remove all references to T for which S==S0. @@ -228,7 +211,7 @@ class SymbolNotFound(RuntimeError): pass -def find_symbol_by_name(name: str, elfinfo: ElfInfo, local_func_matcher: Callable[[Symbol], bool]) -> Optional[Symbol]: +def find_symbol_by_name(name: str, elfinfo: ElfInfo, local_func_matcher: Callable[[Symbol], bool]) -> Symbol | None: """ Find an ELF symbol for the given name. local_func_matcher is a callback function which checks is the candidate local symbol is suitable. @@ -280,9 +263,11 @@ def match_local_target_func(rtl_filename: str, sym_from: Symbol, sym: Symbol) -> return rtl_filename_matches_sym_filename(rtl_filename, sym.filename) -def match_rtl_funcs_to_symbols(rtl_functions: List[RtlFunction], elfinfo: ElfInfo) -> Tuple[List[Symbol], List[Reference]]: - symbols: List[Symbol] = [] - refs: List[Reference] = [] +def match_rtl_funcs_to_symbols( + rtl_functions: list[RtlFunction], elfinfo: ElfInfo +) -> tuple[list[Symbol], list[Reference]]: + symbols: list[Symbol] = [] + refs: list[Reference] = [] # General idea: # - iterate over RTL functions. @@ -292,7 +277,9 @@ def match_rtl_funcs_to_symbols(rtl_functions: List[RtlFunction], elfinfo: ElfInf # - record every pair (sym_from, sym_to) as a Reference object for source_rtl_func in rtl_functions: - maybe_sym_from = find_symbol_by_name(source_rtl_func.name, elfinfo, partial(match_local_source_func, source_rtl_func.rtl_filename)) + maybe_sym_from = find_symbol_by_name( + source_rtl_func.name, elfinfo, partial(match_local_source_func, source_rtl_func.rtl_filename) + ) if maybe_sym_from is None: # RTL references a symbol, but the symbol is not defined in the generated object file. # This means that the symbol was likely removed (or not included) at link time. @@ -307,7 +294,9 @@ def match_rtl_funcs_to_symbols(rtl_functions: List[RtlFunction], elfinfo: ElfInf if '*.LC' in target_rtl_func_name: # skip local labels continue - maybe_sym_to = find_symbol_by_name(target_rtl_func_name, elfinfo, partial(match_local_target_func, source_rtl_func.rtl_filename, sym_from)) + maybe_sym_to = find_symbol_by_name( + target_rtl_func_name, elfinfo, partial(match_local_target_func, source_rtl_func.rtl_filename, sym_from) + ) if not maybe_sym_to: # This may happen for a extern reference in the RTL file, if the reference was later removed # by one of the optimization passes, and the external definition got garbage-collected. @@ -325,10 +314,12 @@ def match_rtl_funcs_to_symbols(rtl_functions: List[RtlFunction], elfinfo: ElfInf return symbols, refs -def get_symbols_and_refs(rtl_list: List[str], elf_file: BinaryIO, ignore_pairs: List[IgnorePair]) -> Tuple[List[Symbol], List[Reference]]: +def get_symbols_and_refs( + rtl_list: list[str], elf_file: BinaryIO, ignore_pairs: list[IgnorePair] +) -> tuple[list[Symbol], list[Reference]]: elfinfo = ElfInfo(elf_file) - rtl_functions: List[RtlFunction] = [] + rtl_functions: list[RtlFunction] = [] for file_name in rtl_list: load_rtl_file(file_name, file_name, rtl_functions) @@ -338,11 +329,12 @@ def get_symbols_and_refs(rtl_list: List[str], elf_file: BinaryIO, ignore_pairs: return match_rtl_funcs_to_symbols(rtl_functions, elfinfo) -def list_refs_from_to_sections(refs: List[Reference], from_sections: List[str], to_sections: List[str]) -> int: +def list_refs_from_to_sections(refs: list[Reference], from_sections: list[str], to_sections: list[str]) -> int: found = 0 for ref in refs: - if (not from_sections or ref.from_sym.section in from_sections) and \ - (not to_sections or ref.to_sym.section in to_sections): + if (not from_sections or ref.from_sym.section in from_sections) and ( + not to_sections or ref.to_sym.section in to_sections + ): print(str(ref)) found += 1 return found @@ -376,19 +368,15 @@ def main() -> None: action_sub = parser.add_subparsers(dest='action') find_refs_parser = action_sub.add_parser( 'find-refs', - help='List the references coming from a given list of source sections' - 'to a given list of target sections.', + help='List the references coming from a given list of source sectionsto a given list of target sections.', ) + find_refs_parser.add_argument('--from-sections', help='comma-separated list of source sections') + find_refs_parser.add_argument('--to-sections', help='comma-separated list of target sections') find_refs_parser.add_argument( - '--from-sections', help='comma-separated list of source sections' - ) - find_refs_parser.add_argument( - '--to-sections', help='comma-separated list of target sections' - ) - find_refs_parser.add_argument( - '--ignore-refs', help='Comma-separated list of symbol pairs to exclude from the references list.' - 'The caller and the callee are separated by a slash. ' - 'Wildcards are supported. Example: my_lib_*/some_lib_in_flash_*.' + '--ignore-refs', + help='Comma-separated list of symbol pairs to exclude from the references list.' + 'The caller and the callee are separated by a slash. ' + 'Wildcards are supported. Example: my_lib_*/some_lib_in_flash_*.', ) find_refs_parser.add_argument( '--exit-code', @@ -403,15 +391,15 @@ def main() -> None: parser.parse_args() args = parser.parse_args() if args.rtl_list: - with open(args.rtl_list, 'r') as rtl_list_file: + with open(args.rtl_list) as rtl_list_file: rtl_list = [line.strip() for line in rtl_list_file] else: if not args.rtl_dirs: raise RuntimeError('Either --rtl-list or --rtl-dirs must be specified') rtl_dirs = args.rtl_dirs.split(',') rtl_list = [] - for dir in rtl_dirs: - rtl_list.extend(list(find_files_recursive(dir, '.expand'))) + for rtl_dir in rtl_dirs: + rtl_list.extend(list(find_files_recursive(rtl_dir, '.expand'))) if not rtl_list: raise RuntimeError('No RTL files specified') @@ -426,9 +414,7 @@ def main() -> None: if args.action == 'find-refs': from_sections = args.from_sections.split(',') if args.from_sections else [] to_sections = args.to_sections.split(',') if args.to_sections else [] - found = list_refs_from_to_sections( - refs, from_sections, to_sections - ) + found = list_refs_from_to_sections(refs, from_sections, to_sections) if args.exit_code and found: raise SystemExit(1) elif args.action == 'all-refs': diff --git a/tools/ci/check_codeowners.py b/tools/ci/check_codeowners.py index 89f3cf6df8a..75382e842db 100755 --- a/tools/ci/check_codeowners.py +++ b/tools/ci/check_codeowners.py @@ -10,7 +10,6 @@ import os import re import subprocess import sys -from typing import List, Optional from idf_ci_utils import IDF_PATH @@ -18,7 +17,7 @@ CODEOWNERS_PATH = os.path.join(IDF_PATH, '.gitlab', 'CODEOWNERS') CODEOWNER_GROUP_PREFIX = '@esp-idf-codeowners/' -def get_all_files() -> List[str]: +def get_all_files() -> list[str]: """ Get list of all file paths in the repository. """ @@ -40,34 +39,36 @@ def pattern_to_regex(pattern: str) -> str: # Convert the glob pattern into a regular expression: # first into intermediate tokens - pattern = (pattern.replace('**/', ':REGLOB:') - .replace('**', ':INVALID:') - .replace('*', ':GLOB:') - .replace('.', ':DOT:') - .replace('?', ':ANY:')) + pattern = ( + pattern.replace('**/', ':REGLOB:') + .replace('**', ':INVALID:') + .replace('*', ':GLOB:') + .replace('.', ':DOT:') + .replace('?', ':ANY:') + ) if pattern.find(':INVALID:') >= 0: - raise ValueError("Likely invalid pattern '{}': '**' should be followed by '/'".format(orig_pattern)) + raise ValueError(f"Likely invalid pattern '{orig_pattern}': '**' should be followed by '/'") # then into the final regex pattern: - re_pattern = (pattern.replace(':REGLOB:', '(?:.*/)?') - .replace(':GLOB:', '[^/]*') - .replace(':DOT:', '[.]') - .replace(':ANY:', '.') + '$') + re_pattern = ( + pattern.replace(':REGLOB:', '(?:.*/)?').replace(':GLOB:', '[^/]*').replace(':DOT:', '[.]').replace(':ANY:', '.') + + '$' + ) if re_pattern.startswith('/'): re_pattern = '^' + re_pattern return re_pattern -def files_by_regex(all_files: List, regex: re.Pattern) -> List: +def files_by_regex(all_files: list, regex: re.Pattern) -> list: """ - Return all files in the repository matching the given regular expresion. + Return all files in the repository matching the given regular expression. """ return [file for file in all_files if regex.search('/' + file)] -def files_by_pattern(all_files: list, pattern: Optional[str]=None) -> List: +def files_by_pattern(all_files: list, pattern: str | None = None) -> list: """ Return all the files in the repository matching the given CODEOWNERS pattern. """ @@ -111,7 +112,7 @@ def action_ci_check(args: argparse.Namespace) -> None: errors = [] def add_error(msg: str) -> None: - errors.append('{}:{}: {}'.format(CODEOWNERS_PATH, line_no, msg)) + errors.append(f'{CODEOWNERS_PATH}:{line_no}: {msg}') all_files = get_all_files() prev_path_pattern = '' @@ -122,10 +123,12 @@ def action_ci_check(args: argparse.Namespace) -> None: if line.startswith('# sort-order-reset'): prev_path_pattern = '' - if (not line - or line.startswith('#') # comment - or line.startswith('[') # file group - or line.startswith('^[')): # optional file group + if ( + not line + or line.startswith('#') # comment + or line.startswith('[') # file group + or line.startswith('^[') + ): # optional file group continue # Each line has a form of " +" @@ -133,22 +136,22 @@ def action_ci_check(args: argparse.Namespace) -> None: path_pattern = tokens[0] owners = tokens[1:] if not owners: - add_error('no owners specified for {}'.format(path_pattern)) + add_error(f'no owners specified for {path_pattern}') # Check that the file is sorted by path patterns if not in_order(prev_path_pattern, path_pattern): - add_error('file is not sorted: {} < {}'.format(path_pattern, prev_path_pattern)) + add_error(f'file is not sorted: {path_pattern} < {prev_path_pattern}') prev_path_pattern = path_pattern # Check that the pattern matches at least one file files = files_by_pattern(all_files, path_pattern) if not files: - add_error('no files matched by pattern {}'.format(path_pattern)) + add_error(f'no files matched by pattern {path_pattern}') for o in owners: # Sanity-check the owner group name if not o.startswith(CODEOWNER_GROUP_PREFIX): - add_error("owner {} doesn't start with {}".format(o, CODEOWNER_GROUP_PREFIX)) + add_error(f"owner {o} doesn't start with {CODEOWNER_GROUP_PREFIX}") if not errors: print('No errors found.') @@ -180,7 +183,7 @@ def in_order(prev: str, current: str) -> bool: return c in '?*' # looping until we see a different character - for a,b in zip(prev, current): + for a, b in zip(prev, current): if is_separator(a) and is_separator(b): continue if is_wildcard(a) or is_wildcard(b): @@ -214,9 +217,11 @@ def main() -> None: test_pattern = subparsers.add_parser( 'test-pattern', - help='Print files in the repository for a given CODEOWNERS pattern. Useful when adding new rules.' + help='Print files in the repository for a given CODEOWNERS pattern. Useful when adding new rules.', + ) + test_pattern.add_argument( + '--regex', action='store_true', help='Print the equivalent regular expression instead of the file list.' ) - test_pattern.add_argument('--regex', action='store_true', help='Print the equivalent regular expression instead of the file list.') test_pattern.add_argument('pattern', help='Path pattern to get the list of files for') args = parser.parse_args() diff --git a/tools/ci/check_copyright_ignore.txt b/tools/ci/check_copyright_ignore.txt index 3f838f4cfd0..a9bd4dde19e 100644 --- a/tools/ci/check_copyright_ignore.txt +++ b/tools/ci/check_copyright_ignore.txt @@ -903,6 +903,5 @@ tools/test_apps/system/bootloader_sections/main/test_main.c tools/test_apps/system/cxx_no_except/main/main.cpp tools/test_apps/system/gdb_loadable_elf/main/hello_world_main.c tools/test_apps/system/longjmp_test/main/hello_world_main.c -tools/test_apps/system/no_embedded_paths/check_for_file_paths.py tools/test_apps/system/no_embedded_paths/main/test_no_embedded_paths_main.c tools/test_apps/system/startup/main/test_startup_main.c diff --git a/tools/ci/check_executables.py b/tools/ci/check_executables.py index ea9ee35cf2d..13d4dfe900e 100755 --- a/tools/ci/check_executables.py +++ b/tools/ci/check_executables.py @@ -6,7 +6,7 @@ import argparse import os import sys -from typing import Iterable, List +from collections.abc import Iterable try: from idf_ci_utils import is_executable @@ -16,7 +16,7 @@ except ImportError: from idf_ci_utils import is_executable -def _strip_each_item(iterable: Iterable) -> List: +def _strip_each_item(iterable: Iterable) -> list: res = [] for item in iterable: if item: @@ -33,30 +33,34 @@ def check_executable_list() -> int: ret = 0 for index, fn in enumerate(known_executables): if not os.path.exists(os.path.join(IDF_PATH, fn)): - print('{}:{} {} not exists. Please remove it manually'.format(EXECUTABLE_LIST_FN, index + 1, fn)) + print(f'{EXECUTABLE_LIST_FN}:{index + 1} {fn} not exists. Please remove it manually') ret = 1 return ret -def check_executables(files: List) -> int: +def check_executables(files: list) -> int: ret = 0 for fn in files: fn_executable = is_executable(fn) fn_in_list = fn in known_executables if fn_executable and not fn_in_list: - print('"{}" is not in {}'.format(fn, EXECUTABLE_LIST_FN)) + print(f'"{fn}" is not in {EXECUTABLE_LIST_FN}') ret = 1 if not fn_executable and fn_in_list: - print('"{}" is not executable but is in {}'.format(fn, EXECUTABLE_LIST_FN)) + print(f'"{fn}" is not executable but is in {EXECUTABLE_LIST_FN}') ret = 1 return ret def check() -> int: parser = argparse.ArgumentParser() - parser.add_argument('--action', choices=['executables', 'list'], required=True, - help='if "executables", pass all your executables to see if it\'s in the list.' - 'if "list", check if all items on your list exist') + parser.add_argument( + '--action', + choices=['executables', 'list'], + required=True, + help='if "executables", pass all your executables to see if it\'s in the list.' + 'if "list", check if all items on your list exist', + ) parser.add_argument('filenames', nargs='*', help='Filenames to check.') args = parser.parse_args() diff --git a/tools/ci/check_requirement_files.py b/tools/ci/check_requirement_files.py index 85e0571ce79..f7dd38404a4 100755 --- a/tools/ci/check_requirement_files.py +++ b/tools/ci/check_requirement_files.py @@ -16,18 +16,18 @@ RE_FEATURE = re.compile(r'requirements\.([\w-]+)\.txt') def action_validate(req_obj: Any) -> None: # "Any" because we are checking this in this script - ''' + """ Check that the parsed JSON object is valid according to the JSON schema provided - ''' - with open(SCHEMA_PATH, 'r') as schema_file: + """ + with open(SCHEMA_PATH) as schema_file: schema_json = json.load(schema_file) jsonschema.validate(req_obj, schema_json) def action_check_directory(req_obj: Any) -> None: # "Any" because we are checking this in this script - ''' + """ Check that all directory items are listed in the JSON file - ''' + """ features = set(d['name'] for d in req_obj['features']) features_found = set() @@ -38,7 +38,7 @@ def action_check_directory(req_obj: Any) -> None: # "Any" because we are checki raise RuntimeError(f'Cannot find a feature for {file_name} in {JSON_PATH}') features_found.add(m.group(1)) else: - raise RuntimeError(f'{file_name} in {REQ_DIR} doesn\'t match the expected name') + raise RuntimeError(f"{file_name} in {REQ_DIR} doesn't match the expected name") features_not_found = features - features_found @@ -47,7 +47,7 @@ def action_check_directory(req_obj: Any) -> None: # "Any" because we are checki def main() -> None: - with open(JSON_PATH, 'r') as f: + with open(JSON_PATH) as f: req_obj = json.load(f) action_validate(req_obj) action_check_directory(req_obj) diff --git a/tools/ci/check_rules_components_patterns.py b/tools/ci/check_rules_components_patterns.py index 8c8b709dace..e1dce8586fc 100755 --- a/tools/ci/check_rules_components_patterns.py +++ b/tools/ci/check_rules_components_patterns.py @@ -4,12 +4,11 @@ """ Check patterns-build_components including all components files except 'test*/' """ + import glob import os import subprocess from functools import lru_cache -from typing import List -from typing import Set import yaml from idf_ci_utils import IDF_PATH @@ -18,15 +17,15 @@ RULES_FILE = os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml') @lru_cache(1) -def get_component_git_files() -> Set[str]: +def get_component_git_files() -> set[str]: output = subprocess.check_output(['git', 'ls-files', 'components/'], cwd=IDF_PATH).decode() - component_git_files = set((_l.strip() for _l in output.strip().splitlines())) + component_git_files = set(_l.strip() for _l in output.strip().splitlines()) assert component_git_files, 'Can not list git files' return component_git_files -def get_component_files_from_patterns(patterns: List[str]) -> Set[str]: - files: Set[str] = set() +def get_component_files_from_patterns(patterns: list[str]) -> set[str]: + files: set[str] = set() for pattern in patterns: # NOTE: version 3.11: Added the include_hidden parameter. # NOTE: version 3.10: Added the root_dir and dir_fd parameters. @@ -41,7 +40,7 @@ def get_component_files_from_patterns(patterns: List[str]) -> Set[str]: return files -def get_components_test_files() -> Set[str]: +def get_components_test_files() -> set[str]: patterns = [ 'components/*/test/**/*', 'components/*/test_apps/**/*', @@ -62,12 +61,12 @@ def get_components_test_files() -> Set[str]: return get_component_files_from_patterns(patterns) -def get_all_components_files() -> Set[str]: +def get_all_components_files() -> set[str]: return get_component_files_from_patterns(['components/**/*']) def main() -> None: - with open(RULES_FILE, 'r') as f: + with open(RULES_FILE) as f: rules = yaml.load(f.read(), Loader=yaml.FullLoader) patterns_components = [p for p in rules['.patterns-build_components'] if p.startswith('components/')] submodules = [p for p in rules['.patterns-submodule'] if p.startswith('components/')] diff --git a/tools/ci/check_soc_struct_headers.py b/tools/ci/check_soc_struct_headers.py index efdcfe86c2b..56608e5446d 100755 --- a/tools/ci/check_soc_struct_headers.py +++ b/tools/ci/check_soc_struct_headers.py @@ -24,14 +24,13 @@ import os import re import sys from typing import Any -from typing import Optional class MemberField: member_type = '' bitfield = None - def __init__(self, m_type: str, m_bits: Optional[int]=None) -> None: + def __init__(self, m_type: str, m_bits: int | None = None) -> None: self.member_type = m_type self.bitfield = m_bits @@ -43,8 +42,8 @@ class MemberField: def __str__(self) -> str: if self.bitfield is None: - return '"Field type={}"'.format(self.member_type) - return '"Field type={} bit={}"'.format(self.member_type, self.bitfield) + return f'"Field type={self.member_type}"' + return f'"Field type={self.member_type} bit={self.bitfield}"' class SoCStructureHeaderChecker: @@ -55,8 +54,10 @@ class SoCStructureHeaderChecker: # capture: type_var_name __REGEXP_MATCH_STRUCT_UNION_END_NAME = r'^[\s]*}[\s]*([\w\[\]\*]*)[\s]*;[\s\S]*$' # capture: type, name, bitfield - __REGEXP_MATCH_BITFIELD_MEMBER = (r'^[\s]*(?:(?:volatile[\s]+)|(?:))([\w\*]+)[\s]+([\w\*]+(?:(?:\[[\s\S]*\])|(?:)))' - r'[\s]*(?:(?:[\s]*;)|(?::[\s]*([\d]+)[\s]*;))[\s\S]*$') + __REGEXP_MATCH_BITFIELD_MEMBER = ( + r'^[\s]*(?:(?:volatile[\s]+)|(?:))([\w\*]+)[\s]+([\w\*]+(?:(?:\[[\s\S]*\])|(?:)))' + r'[\s]*(?:(?:[\s]*;)|(?::[\s]*([\d]+)[\s]*;))[\s\S]*$' + ) # should be useless and can be safely deleted __REGEXP_MATCH_MULTILINE_COMMENT = r'^[\s]*[\/]{0,2}\*[\/]?[\s\S]*$' __REGEX_MATCH_SIMPLE_VAL_FIELD = r'^[\s]*(?:(?:volatile[\s]+)|(?:))([\w]+)[\s]+([\w\[\]\*]+)[\s]*;[\s]*$' @@ -64,16 +65,16 @@ class SoCStructureHeaderChecker: __REGEX_MATCH_ROOT_EXTERNAL = r'^[\s]*extern[\s]+([\w]+)[\s]+([\w]+)[\s]*;[\s]*$' __linecount = 0 - __fd = None # type: Any + __fd = None # type: Any __is_eof = False # generated reference tree - __ref_tree = dict() # type: dict + __ref_tree = dict() # type: dict # middle result of generated tree, shared # named typedef, or named struct/union. referd but will not delete - __temp_ref_types = dict() # type: dict + __temp_ref_types = dict() # type: dict - def __expand_type(self, member_type: str, bitfield: Optional[int]=None) -> Any: + def __expand_type(self, member_type: str, bitfield: int | None = None) -> Any: if member_type == 'uint32_t': return MemberField(member_type, bitfield) if bitfield is not None: @@ -83,7 +84,7 @@ class SoCStructureHeaderChecker: return self.__temp_ref_types[member_type] return None - def __getline(self, incomment:bool=False) -> Any: + def __getline(self, incomment: bool = False) -> Any: rawline = self.__fd.readline() if not rawline: self.__is_eof = True @@ -94,7 +95,7 @@ class SoCStructureHeaderChecker: pos = rawline.find('*/') if pos != -1: # set string that is behind comment - rawline = rawline[pos + 2:] + rawline = rawline[pos + 2 :] else: # continue multiple line return self.__getline(True) @@ -111,7 +112,7 @@ class SoCStructureHeaderChecker: pos = match_obj.groups()[1].find('*/') if pos != -1: # apply string that is behind comment - rawline = rawline + match_obj.groups()[1][pos + 2:] + rawline = rawline + match_obj.groups()[1][pos + 2 :] else: # multiple line return self.__getline(True) @@ -120,7 +121,7 @@ class SoCStructureHeaderChecker: # skip empty line return self.__getline() if rawline.count(';') > 1: - print('\033[0;34mINFO\033[0m: line: {}: possibly multiple expression within same line'.format(self.__linecount)) + print(f'\033[0;34mINFO\033[0m: line: {self.__linecount}: possibly multiple expression within same line') print(rawline) return rawline @@ -128,7 +129,7 @@ class SoCStructureHeaderChecker: ret_val = 0 # first check for anonymous register structs if is_typedef and is_volatile and name is None: - print('\033[0;31mERROR\033[0m: line {}: anonymous struct'.format(self.__linecount)) + print(f'\033[0;31mERROR\033[0m: line {self.__linecount}: anonymous struct') ret_val = -1 node_tree = dict() bitcount = 0 @@ -144,7 +145,8 @@ class SoCStructureHeaderChecker: if match_obj is not None: has_nested_struct_union = True ret, inherited_node_tree = self.__process_structure( - match_obj.groups()[2], match_obj.groups()[0] == 'typedef', match_obj.groups()[1] == 'volatile') + match_obj.groups()[2], match_obj.groups()[0] == 'typedef', match_obj.groups()[1] == 'volatile' + ) if ret != 0: ret_val = -2 if inherited_node_tree is not None: @@ -154,7 +156,9 @@ class SoCStructureHeaderChecker: match_obj = re.match(self.__REGEXP_MATCH_UNION_BEGIN, rawline) if match_obj is not None: has_nested_struct_union = True - ret, inherited_node_tree = self.__process_union(match_obj.groups()[2], match_obj.groups()[0] == 'typedef', match_obj.groups()[1] == 'volatile') + ret, inherited_node_tree = self.__process_union( + match_obj.groups()[2], match_obj.groups()[0] == 'typedef', match_obj.groups()[1] == 'volatile' + ) if ret != 0: ret_val = -2 if inherited_node_tree is not None: @@ -168,16 +172,21 @@ class SoCStructureHeaderChecker: if bitcount not in (0, 32): ret_val = -2 if is_typedef: - print('\033[0;31mERROR\033[0m: line {}: bitfield count is {}, type {}'.format(self.__linecount, bitcount, match_obj.groups()[0])) + print( + f'\033[0;31mERROR\033[0m: line {self.__linecount}: ' + f'bitfield count is {bitcount}, type {match_obj.groups()[0]}' + ) else: - print('\033[0;31mERROR\033[0m: line {}: bitfield count is {}, type {}, varname "{}"' - .format(self.__linecount, bitcount, name, match_obj.groups()[0])) + print( + f'\033[0;31mERROR\033[0m: line {self.__linecount}: ' + f'bitfield count is {bitcount}, type {name}, varname "{match_obj.groups()[0]}"' + ) parsed_varname = match_obj.groups()[0] if is_typedef: # is a typedef if match_obj.groups()[0] == '' or match_obj.groups()[0].find('[') != -1: # should be c error - print('\033[0;31mERROR\033[0m: line {}: C error'.format(self.__linecount)) + print(f'\033[0;31mERROR\033[0m: line {self.__linecount}: C error') ret_val = -3 if match_obj.groups()[0] in self.__temp_ref_types: # duplication, script bug: we are putting all types into same namespace @@ -211,7 +220,10 @@ class SoCStructureHeaderChecker: bitcount += field_bit # bitfield should be u32 if match_obj.groups()[0] != 'uint32_t': - print('\033[0;33mWARN\033[0m: line: {}: {} has type {}'.format(self.__linecount, match_obj.groups()[1], match_obj.groups()[0])) + print( + f'\033[0;33mWARN\033[0m: line: {self.__linecount}: ' + f'{match_obj.groups()[1]} has type {match_obj.groups()[0]}' + ) else: has_non_bitfield_member = True # append to node tree @@ -224,9 +236,9 @@ class SoCStructureHeaderChecker: node_tree[match_obj.groups()[1]] = member_node else: if '*' not in match_obj.groups()[0]: - print('\033[0;33mWARN\033[0m: line {}: unknown type {}'.format(self.__linecount, match_obj.groups()[0])) + print(f'\033[0;33mWARN\033[0m: line {self.__linecount}: unknown type {match_obj.groups()[0]}') else: - print('\033[0;33mWARN\033[0m: line {}: pointer type {}'.format(self.__linecount, match_obj.groups()[0])) + print(f'\033[0;33mWARN\033[0m: line {self.__linecount}: pointer type {match_obj.groups()[0]}') continue # check comments match_obj = re.match(self.__REGEXP_MATCH_MULTILINE_COMMENT, rawline) @@ -234,24 +246,29 @@ class SoCStructureHeaderChecker: # code comments continue # dump out unmatched condition - print(('\033[0;33mWARN\033[0m: line: {}: unexpected expression: {}'.format(self.__linecount, rawline)).replace('\n', '')) + print( + (f'\033[0;33mWARN\033[0m: line: {self.__linecount}: unexpected expression: {rawline}').replace('\n', '') + ) if bitcount != 0 and has_nested_struct_union: - print('\033[0;33mWARN\033[0m: line: {}: mixed bitfield member and nested structure/union'.format(self.__linecount)) + print(f'\033[0;33mWARN\033[0m: line: {self.__linecount}: mixed bitfield member and nested structure/union') if bitcount != 0 and has_non_bitfield_member: - print('\033[0;33mWARN\033[0m: line: {}: mixed bitfield member and non-bitfield member'.format(self.__linecount)) + print(f'\033[0;33mWARN\033[0m: line: {self.__linecount}: mixed bitfield member and non-bitfield member') if is_typedef and is_volatile and name is None: if parsed_varname != '': print('SUGGEST: {}'.format(parsed_varname.rstrip('t') + 's')) if name is not None and is_typedef and is_volatile and parsed_varname.rstrip('t') != name.rstrip('s'): - print('\033[0;33mWARN\033[0m: line: {}: different type and typedef name: {} {}'.format(self.__linecount, name, parsed_varname)) + print( + f'\033[0;33mWARN\033[0m: line: {self.__linecount}: ' + f'different type and typedef name: {name} {parsed_varname}' + ) return ret_val, node_tree def __process_union(self, name: str, is_typedef: bool, is_volatile: bool) -> Any: ret_val = 0 # first check for anonymous register structs if is_typedef and is_volatile and name is None: - print('\033[0;31mERROR\033[0m: line {}: anonymous union'.format(self.__linecount)) + print(f'\033[0;31mERROR\033[0m: line {self.__linecount}: anonymous union') ret_val = -1 node_tree = dict() # type: Any has_struct_count = 0 @@ -265,7 +282,8 @@ class SoCStructureHeaderChecker: if match_obj is not None: has_struct_count += 1 ret, inherited_node_tree = self.__process_structure( - match_obj.groups()[2], match_obj.groups()[0] == 'typedef', match_obj.groups()[1] == 'volatile') + match_obj.groups()[2], match_obj.groups()[0] == 'typedef', match_obj.groups()[1] == 'volatile' + ) if ret != 0: ret_val = -2 if inherited_node_tree is not None: @@ -275,7 +293,9 @@ class SoCStructureHeaderChecker: match_obj = re.match(self.__REGEXP_MATCH_UNION_BEGIN, rawline) if match_obj is not None: has_struct_count += 1 - ret, inherited_node_tree = self.__process_union(match_obj.groups()[2], match_obj.groups()[0] == 'typedef', match_obj.groups()[1] == 'volatile') + ret, inherited_node_tree = self.__process_union( + match_obj.groups()[2], match_obj.groups()[0] == 'typedef', match_obj.groups()[1] == 'volatile' + ) if ret != 0: ret_val = -2 if inherited_node_tree is not None: @@ -290,7 +310,7 @@ class SoCStructureHeaderChecker: # is a typedef if match_obj.groups()[0] == '': # should be c error - print('\033[0;31mERROR\033[0m: line {}: C error'.format(self.__linecount)) + print(f'\033[0;31mERROR\033[0m: line {self.__linecount}: C error') ret_val = -3 if match_obj.groups()[0] in self.__temp_ref_types: # duplication, script bug: we are putting all types into same namespace @@ -324,7 +344,11 @@ class SoCStructureHeaderChecker: if match_obj is not None: # expecting to see 'uint32_t val;' if match_obj.groups()[0] != 'uint32_t' or match_obj.groups()[1] != 'val': - print(('\033[0;33mWARN\033[0m: unexpected union member at {}: {}'.format(self.__linecount, rawline)).replace('\n', '')) + print( + (f'\033[0;33mWARN\033[0m: unexpected union member at {self.__linecount}: {rawline}').replace( + '\n', '' + ) + ) else: has_val_field_count += 1 # append to node tree @@ -333,21 +357,28 @@ class SoCStructureHeaderChecker: node_tree[match_obj.groups()[1]] = member_node else: if '*' not in match_obj.groups()[0]: - print('\033[0;31mWARN\033[0m: line {}: unknown type {}'.format(self.__linecount, match_obj.groups()[0])) + print(f'\033[0;31mWARN\033[0m: line {self.__linecount}: unknown type {match_obj.groups()[0]}') else: - print('\033[0;33mWARN\033[0m: line {}: pointer type {}'.format(self.__linecount, match_obj.groups()[0])) + print(f'\033[0;33mWARN\033[0m: line {self.__linecount}: pointer type {match_obj.groups()[0]}') continue # dump out unmatched condition - print(('\033[0;33mWARN\033[0m: line: {}: unexpected expression: {}'.format(self.__linecount, rawline)).replace('\n', '')) + print( + (f'\033[0;33mWARN\033[0m: line: {self.__linecount}: unexpected expression: {rawline}').replace('\n', '') + ) if not (has_struct_count == 1 and has_val_field_count == 1): - print('\033[0;34mINFO\033[0m: line: {}: not a typical union: {} nested structures, {} u32 val member' - .format(self.__linecount, has_struct_count, has_val_field_count)) + print( + f'\033[0;34mINFO\033[0m: line: {self.__linecount}: not a typical union: ' + f'{has_struct_count} nested structures, {has_val_field_count} u32 val member' + ) if is_typedef and is_volatile and name is None: if parsed_varname != '': print('SUGGEST: {}'.format(parsed_varname.rstrip('t') + 's')) if name is not None and is_typedef and is_volatile and parsed_varname.rstrip('t') != name.rstrip('s'): - print('\033[0;33mWARN\033[0m: line: {}: different type and typedef name: {} {}'.format(self.__linecount, name, parsed_varname)) + print( + f'\033[0;33mWARN\033[0m: line: {self.__linecount}: ' + f'different type and typedef name: {name} {parsed_varname}' + ) return ret_val, node_tree def __process_root(self) -> int: @@ -362,7 +393,8 @@ class SoCStructureHeaderChecker: match_obj = re.match(self.__REGEXP_MATCH_STRUCTURE_BEGIN, rawline) if match_obj is not None: ret, inherited_node_tree = self.__process_structure( - match_obj.groups()[2], match_obj.groups()[0] == 'typedef', match_obj.groups()[1] == 'volatile') + match_obj.groups()[2], match_obj.groups()[0] == 'typedef', match_obj.groups()[1] == 'volatile' + ) if ret != 0: ret_val = -2 if inherited_node_tree is not None: @@ -371,7 +403,9 @@ class SoCStructureHeaderChecker: continue match_obj = re.match(self.__REGEXP_MATCH_UNION_BEGIN, rawline) if match_obj is not None: - ret, inherited_node_tree = self.__process_union(match_obj.groups()[2], match_obj.groups()[0] == 'typedef', match_obj.groups()[1] == 'volatile') + ret, inherited_node_tree = self.__process_union( + match_obj.groups()[2], match_obj.groups()[0] == 'typedef', match_obj.groups()[1] == 'volatile' + ) if ret != 0: ret_val = -2 if inherited_node_tree is not None: @@ -386,7 +420,7 @@ class SoCStructureHeaderChecker: return ret_val def check(self, file: str) -> int: - self.__fd = open(file, 'r', encoding='utf8') + self.__fd = open(file, encoding='utf8') self.__linecount = 0 self.__is_eof = False @@ -394,9 +428,9 @@ class SoCStructureHeaderChecker: self.__fd.close() if ret_val != 0: - print('\033[0;31mCHECK FAILED\033[0m:\t{}'.format(file)) + print(f'\033[0;31mCHECK FAILED\033[0m:\t{file}') else: - print('\033[0;32mCHECK PASSED\033[0m:\t{}'.format(file)) + print(f'\033[0;32mCHECK PASSED\033[0m:\t{file}') return ret_val def get_ref_tree(self) -> Any: @@ -409,7 +443,7 @@ def main() -> None: print('file not exist') exit(-1) checker = SoCStructureHeaderChecker() - print('CHECKING:\t{}'.format(sys.argv[1])) + print(f'CHECKING:\t{sys.argv[1]}') ret = checker.check(sys.argv[1]) if len(sys.argv) == 3 and sys.argv[2] == 'print': print(checker.get_ref_tree()) diff --git a/tools/ci/check_tools_files_patterns.py b/tools/ci/check_tools_files_patterns.py index 89e593fa736..7a99635d170 100755 --- a/tools/ci/check_tools_files_patterns.py +++ b/tools/ci/check_tools_files_patterns.py @@ -7,20 +7,20 @@ import argparse import os import sys from pathlib import Path -from typing import Set, Tuple import yaml -from idf_ci_utils import IDF_PATH, get_git_files +from idf_ci_utils import IDF_PATH +from idf_ci_utils import get_git_files -def check(pattern_yml: str, exclude_list: str) -> Tuple[Set, Set]: +def check(pattern_yml: str, exclude_list: str) -> tuple[set, set]: rules_dict = yaml.load(open(pattern_yml), Loader=yaml.FullLoader) rules_patterns_set = set() for k, v in rules_dict.items(): if k.startswith('.pattern') and k != '.patterns-python-files' and isinstance(v, list): rules_patterns_set.update(v) - rules_files_set: Set = set() + rules_files_set: set = set() idf_path = Path(IDF_PATH) for pat in rules_patterns_set: rules_files_set.update(idf_path.glob(pat)) @@ -31,7 +31,7 @@ def check(pattern_yml: str, exclude_list: str) -> Tuple[Set, Set]: if pat: exclude_patterns_set.add(pat) - exclude_files_set: Set = set() + exclude_files_set: set = set() for pat in exclude_patterns_set: exclude_files_set.update(idf_path.glob(pat)) @@ -39,7 +39,11 @@ def check(pattern_yml: str, exclude_list: str) -> Tuple[Set, Set]: git_files = get_git_files(os.path.join(IDF_PATH, 'tools'), full_path=True) for f in git_files: f = Path(f) - if f in rules_files_set or f in exclude_files_set or str(f).startswith(os.path.join(IDF_PATH, 'tools', 'test_apps')): + if ( + f in rules_files_set + or f in exclude_files_set + or str(f).startswith(os.path.join(IDF_PATH, 'tools', 'test_apps')) + ): continue missing_files.add(os.path.relpath(f, IDF_PATH)) @@ -48,12 +52,18 @@ def check(pattern_yml: str, exclude_list: str) -> Tuple[Set, Set]: if __name__ == '__main__': parser = argparse.ArgumentParser(description='check if all tools files are in rules patterns or exclude list') - parser.add_argument('-c', '--pattern-yml', - default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml'), - help='yml file path included file patterns') - parser.add_argument('-e', '--exclude-list', - default=os.path.join(IDF_PATH, 'tools', 'ci', 'exclude_check_tools_files.txt'), - help='exclude list path') + parser.add_argument( + '-c', + '--pattern-yml', + default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml'), + help='yml file path included file patterns', + ) + parser.add_argument( + '-e', + '--exclude-list', + default=os.path.join(IDF_PATH, 'tools', 'ci', 'exclude_check_tools_files.txt'), + help='exclude list path', + ) args = parser.parse_args() @@ -61,15 +71,19 @@ if __name__ == '__main__': not_included_files, dup_patterns = check(args.pattern_yml, args.exclude_list) if not_included_files or dup_patterns: res = 1 - print('This test is used for making sure of all the tools dir files are recorded in .gitlab/ci/rules.yml to ' - 'trigger the related tests, except those files should be excluded.') + print( + 'This test is used for making sure of all the tools dir files are recorded in .gitlab/ci/rules.yml to ' + 'trigger the related tests, except those files should be excluded.' + ) if not_included_files: print('Missing Files:') for file in not_included_files: print('\t' + str(file)) - print('Please add these files or glob patterns to ".gitlab/ci/rules.yml" and put related files under ' - '".patterns-" block to trigger related tests.\n' - 'Or add them to "tools/ci/exclude_check_tools_files.txt" to exclude them.') + print( + 'Please add these files or glob patterns to ".gitlab/ci/rules.yml" and put related files under ' + '".patterns-" block to trigger related tests.\n' + 'Or add them to "tools/ci/exclude_check_tools_files.txt" to exclude them.' + ) if dup_patterns: print('Duplicated Patterns:') diff --git a/tools/ci/checkout_project_ref.py b/tools/ci/checkout_project_ref.py index 4a987c2eee1..753d60501cb 100755 --- a/tools/ci/checkout_project_ref.py +++ b/tools/ci/checkout_project_ref.py @@ -10,12 +10,11 @@ import json import os import re import subprocess -from typing import List, Tuple IDF_GIT_DESCRIBE_PATTERN = re.compile(r'^v(\d)\.(\d)') -def _idf_version_from_cmake() -> Tuple[int, int]: +def _idf_version_from_cmake() -> tuple[int, int]: version_path = os.path.join(os.environ['IDF_PATH'], 'tools/cmake/version.cmake') regex = re.compile(r'^\s*set\s*\(\s*IDF_VERSION_([A-Z]{5})\s+(\d+)') try: @@ -46,7 +45,7 @@ def get_customized_project_revision(proj_name: str) -> str: return revision -def target_branch_candidates(proj_name: str) -> List: +def target_branch_candidates(proj_name: str) -> list: """ :return: a list of target branch candidates, from highest priority to lowest priority. """ @@ -64,22 +63,19 @@ def target_branch_candidates(proj_name: str) -> List: # branch name read from IDF major_revision, minor_revision = _idf_version_from_cmake() # release branch - candidates.append('release/v{}.{}'.format(major_revision, minor_revision)) + candidates.append(f'release/v{major_revision}.{minor_revision}') # branch to match all major branches, like v3.x or v3 - candidates.append('release/v{}.x'.format(major_revision)) - candidates.append('release/v{}'.format(major_revision)) + candidates.append(f'release/v{major_revision}.x') + candidates.append(f'release/v{major_revision}') return [c for c in candidates if c] # filter out null value if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('project', - help='the name of project') - parser.add_argument('project_relative_path', - help='relative path of project to IDF repository directory') - parser.add_argument('--customized_only', action='store_true', - help='Only to find customized revision') + parser.add_argument('project', help='the name of project') + parser.add_argument('project_relative_path', help='relative path of project to IDF repository directory') + parser.add_argument('--customized_only', action='store_true', help='Only to find customized revision') args = parser.parse_args() @@ -96,13 +92,17 @@ if __name__ == '__main__': for candidate in candidate_branches: # check if the branch, tag or commit exists try: - subprocess.check_call(['git', 'cat-file', '-t', 'origin/{}'.format(candidate)], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + subprocess.check_call( + ['git', 'cat-file', '-t', f'origin/{candidate}'], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) ref_to_use = candidate break except subprocess.CalledProcessError: try: # For customized commits - subprocess.check_call(['git', 'cat-file', '-t', candidate], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + subprocess.check_call( + ['git', 'cat-file', '-t', candidate], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) ref_to_use = candidate break except subprocess.CalledProcessError: @@ -112,7 +112,7 @@ if __name__ == '__main__': if ref_to_use: try: subprocess.check_call(['git', 'checkout', '-f', ref_to_use], stdout=subprocess.PIPE) # not print the stdout - print('CI using ref {} for project {}'.format(ref_to_use, args.project)) + print(f'CI using ref {ref_to_use} for project {args.project}') except subprocess.CalledProcessError: pass else: diff --git a/tools/ci/ci_fetch_submodule.py b/tools/ci/ci_fetch_submodule.py index 2137e1ef78a..fa6dcb691a5 100644 --- a/tools/ci/ci_fetch_submodule.py +++ b/tools/ci/ci_fetch_submodule.py @@ -12,7 +12,6 @@ import shutil import subprocess import time from typing import Any -from typing import List import gitlab_api @@ -25,7 +24,7 @@ SUBMODULE_ARCHIVE_TEMP_FOLDER = 'submodule_archive' SUBMODULE_ARCHIVE_CACHE_DIR = '.cache/submodule_archives' -class SubModule(object): +class SubModule: # We don't need to support recursive submodule clone now GIT_LS_TREE_OUTPUT_PATTERN = re.compile(r'\d+\s+commit\s+([0-9a-f]+)\s+') @@ -52,7 +51,7 @@ class SubModule(object): return project_id def download_archive(self) -> None: - print('Update submodule: {}: {}'.format(self.path, self.commit_id)) + print(f'Update submodule: {self.path}: {self.commit_id}') path_name = self.gitlab_inst.download_archive( self.commit_id, SUBMODULE_ARCHIVE_TEMP_FOLDER, self.project_id, SUBMODULE_ARCHIVE_CACHE_DIR ) @@ -62,10 +61,10 @@ class SubModule(object): shutil.move(renamed_path, os.path.dirname(self.path)) -def update_submodule(git_module_file: str, submodules_to_update: List) -> None: +def update_submodule(git_module_file: str, submodules_to_update: list) -> None: gitlab_inst = gitlab_api.Gitlab() submodules = [] - with open(git_module_file, 'r') as f: + with open(git_module_file) as f: data = f.read() match = SUBMODULE_PATTERN.search(data) if match is not None: @@ -118,4 +117,4 @@ if __name__ == '__main__': else: _submodules = args.submodule.split(';') update_submodule(os.path.join(args.repo_path, '.gitmodules'), _submodules) - print('total time spent on update submodule: {:.02f}s'.format(time.time() - start_time)) + print(f'total time spent on update submodule: {time.time() - start_time:.02f}s') diff --git a/tools/ci/cleanup_ignore_lists.py b/tools/ci/cleanup_ignore_lists.py index 68f718f40dd..a3dbdeb2559 100755 --- a/tools/ci/cleanup_ignore_lists.py +++ b/tools/ci/cleanup_ignore_lists.py @@ -11,7 +11,7 @@ from pathlib import Path from idf_ci_utils import IDF_PATH -def print_list(_list: t.Iterable[t.Any], title: t.Optional[str] = None) -> None: +def print_list(_list: t.Iterable[t.Any], title: str | None = None) -> None: if not _list: return @@ -24,7 +24,7 @@ def print_list(_list: t.Iterable[t.Any], title: t.Optional[str] = None) -> None: if __name__ == '__main__': os.chdir(IDF_PATH) - ignore_lists: t.Set[Path] = set() + ignore_lists: set[Path] = set() ignore_lists.update(Path('tools', 'ci').glob('**/*.txt')) ignore_lists.remove(Path('tools', 'ci', 'ignore_build_warnings.txt')) ignore_lists.remove(Path('tools', 'ci', 'check_ldgen_mapping_exceptions.txt')) diff --git a/tools/ci/idf_pytest/utils.py b/tools/ci/idf_pytest/utils.py index 22ade14792f..c200687c4d5 100644 --- a/tools/ci/idf_pytest/utils.py +++ b/tools/ci/idf_pytest/utils.py @@ -2,12 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 import logging import os -import typing as t from xml.etree import ElementTree as ET def format_case_id( - target: t.Optional[str], config: t.Optional[str], case: str, is_qemu: bool = False, params: t.Optional[dict] = None + target: str | None, config: str | None, case: str, is_qemu: bool = False, params: dict | None = None ) -> str: parts = [] if target: @@ -22,16 +21,16 @@ def format_case_id( return '.'.join(parts) -def merge_junit_files(junit_files: t.List[str], target_path: str) -> None: +def merge_junit_files(junit_files: list[str], target_path: str) -> None: if len(junit_files) <= 1: return merged_testsuite: ET.Element = ET.Element('testsuite') - testcases: t.Dict[str, ET.Element] = {} + testcases: dict[str, ET.Element] = {} for junit in junit_files: logging.info(f'Merging {junit} to {target_path}') - tree: ET.ElementTree = ET.parse(junit) - testsuite: ET.Element = tree.getroot() + tree = ET.parse(junit) + testsuite = tree.getroot() for testcase in testsuite.findall('testcase'): name: str = testcase.get('name') if testcase.get('name') else '' # type: ignore @@ -62,7 +61,7 @@ def merge_junit_files(junit_files: t.List[str], target_path: str) -> None: fw.write(ET.tostring(merged_testsuite)) -def normalize_testcase_file_path(file: str, app_path: t.Union[str, tuple]) -> str: +def normalize_testcase_file_path(file: str, app_path: str | tuple) -> str: """ Normalize file paths to a consistent format, resolving relative paths based on the `app_path`. diff --git a/tools/ci/python_packages/common_test_methods.py b/tools/ci/python_packages/common_test_methods.py index d18f3f0e683..100bc36985a 100644 --- a/tools/ci/python_packages/common_test_methods.py +++ b/tools/ci/python_packages/common_test_methods.py @@ -4,7 +4,6 @@ import logging import os import socket from typing import Any -from typing import List from idf_ci_utils import IDF_PATH @@ -22,7 +21,7 @@ ENV_CONFIG_FILE_SEARCH = [ os.path.join(IDF_PATH, 'EnvConfig.yml'), os.path.join(IDF_PATH, 'ci-test-runner-configs', os.environ.get('CI_RUNNER_DESCRIPTION', ''), 'EnvConfig.yml'), ] -ENV_CONFIG_TEMPLATE = ''' +ENV_CONFIG_TEMPLATE = """ $IDF_PATH/EnvConfig.yml: ```yaml : @@ -31,19 +30,19 @@ $IDF_PATH/EnvConfig.yml: : key: var ``` -''' +""" def get_host_ip_by_interface(interface_name: str, ip_type: int = netifaces.AF_INET) -> str: if ip_type == netifaces.AF_INET: for _addr in netifaces.ifaddresses(interface_name)[ip_type]: - host_ip = _addr['addr'].replace('%{}'.format(interface_name), '') + host_ip = _addr['addr'].replace(f'%{interface_name}', '') assert isinstance(host_ip, str) return host_ip elif ip_type == netifaces.AF_INET6: - ip6_addrs: List[str] = [] + ip6_addrs: list[str] = [] for _addr in netifaces.ifaddresses(interface_name)[ip_type]: - host_ip = _addr['addr'].replace('%{}'.format(interface_name), '') + host_ip = _addr['addr'].replace(f'%{interface_name}', '') assert isinstance(host_ip, str) # prefer to use link local address due to example settings if host_ip.startswith('FE80::'): @@ -89,7 +88,7 @@ def get_my_interface_by_dest_ip(dest_ip: str = '') -> str: if my_ip in [a['addr'] for a in addresses]: assert isinstance(interface, str) return interface - logging.error('Can not get interface, dest ip is {}'.format(dest_ip)) + logging.error(f'Can not get interface, dest ip is {dest_ip}') return '' @@ -109,7 +108,7 @@ def get_env_config_variable(env_name: str, key: str, default: Any = None) -> Any config = dict() for _file in ENV_CONFIG_FILE_SEARCH: try: - with open(_file, 'r') as f: + with open(_file) as f: data = yaml.load(f.read(), Loader=yaml.SafeLoader) config = data[env_name] break @@ -124,7 +123,7 @@ def get_env_config_variable(env_name: str, key: str, default: Any = None) -> Any logging.info(f'Env config file template: {ENV_CONFIG_TEMPLATE}') if not os.environ.get('CI_JOB_ID'): # Try to get variable from stdin - var = input(f'You can input the variable now:') + var = input('You can input the variable now:') else: raise ValueError(f'Env variable not found: {env_name}/{key}') logging.debug(f'Got env variable {env_name}/{key}: {var}') diff --git a/tools/ci/python_packages/idf_http_server_test/adder.py b/tools/ci/python_packages/idf_http_server_test/adder.py index 85ff255fa5e..5e591f8f218 100644 --- a/tools/ci/python_packages/idf_http_server_test/adder.py +++ b/tools/ci/python_packages/idf_http_server_test/adder.py @@ -21,11 +21,11 @@ def getreq(conn, path, verbose=False): resp = conn.getresponse() data = resp.read() if verbose: - logging.info('GET : {}'.format(path)) - logging.info('Status : {}'.format(resp.status)) - logging.info('Reason : {}'.format(resp.reason)) - logging.info('Data length : {}'.format(len(data))) - logging.info('Data content : {}'.format(data)) + logging.info(f'GET : {path}') + logging.info(f'Status : {resp.status}') + logging.info(f'Reason : {resp.reason}') + logging.info(f'Data length : {len(data)}') + logging.info(f'Data content : {data}') return data @@ -34,11 +34,11 @@ def postreq(conn, path, data, verbose=False): resp = conn.getresponse() data = resp.read() if verbose: - logging.info('POST : {}'.format(data)) - logging.info('Status : {}'.format(resp.status)) - logging.info('Reason : {}'.format(resp.reason)) - logging.info('Data length : {}'.format(len(data))) - logging.info('Data content : {}'.format(data)) + logging.info(f'POST : {data}') + logging.info(f'Status : {resp.status}') + logging.info(f'Reason : {resp.reason}') + logging.info(f'Data length : {len(data)}') + logging.info(f'Data content : {data}') return data @@ -47,26 +47,26 @@ def putreq(conn, path, body, verbose=False): resp = conn.getresponse() data = resp.read() if verbose: - logging.info('PUT : {} {}'.format(path, body)) - logging.info('Status : {}'.format(resp.status)) - logging.info('Reason : {}'.format(resp.reason)) - logging.info('Data length : {}'.format(len(data))) - logging.info('Data content : {}'.format(data)) + logging.info(f'PUT : {path} {body}') + logging.info(f'Status : {resp.status}') + logging.info(f'Reason : {resp.reason}') + logging.info(f'Data length : {len(data)}') + logging.info(f'Data content : {data}') return data if __name__ == '__main__': # Configure argument parser parser = argparse.ArgumentParser(description='Run HTTPd Test') - parser.add_argument('IP', metavar='IP', type=str, help='Server IP') - parser.add_argument('port', metavar='port', type=str, help='Server port') - parser.add_argument('N', metavar='integer', type=int, help='Integer to sum upto') + parser.add_argument('IP', metavar='IP', type=str, help='Server IP') + parser.add_argument('port', metavar='port', type=str, help='Server port') + parser.add_argument('N', metavar='integer', type=int, help='Integer to sum upto') args = vars(parser.parse_args()) # Get arguments - ip = args['IP'] + ip = args['IP'] port = args['port'] - N = args['N'] + N = args['N'] # Establish HTTP connection logging.info('Connecting to => ' + ip + ':' + port) @@ -79,7 +79,7 @@ if __name__ == '__main__': putreq(conn, '/adder', str(0)) # Sum numbers from 1 to specified value(N) - logging.info('Summing numbers from 1 to {}'.format(N)) + logging.info(f'Summing numbers from 1 to {N}') for i in range(1, N + 1): postreq(conn, '/adder', str(i)) diff --git a/tools/ci/python_packages/idf_http_server_test/client.py b/tools/ci/python_packages/idf_http_server_test/client.py index a8dd67e1bff..a5d90e8a291 100644 --- a/tools/ci/python_packages/idf_http_server_test/client.py +++ b/tools/ci/python_packages/idf_http_server_test/client.py @@ -10,16 +10,16 @@ import logging def verbose_print(verbosity, *args): - if (verbosity): + if verbosity: logging.info(''.join(str(elems) for elems in args)) def test_val(text, expected, received): if expected != received: logging.info(' Fail!') - logging.info(' [reason] {} :'.format(text)) - logging.info(' expected: {}'.format(expected)) - logging.info(' received: {}'.format(received)) + logging.info(f' [reason] {text} :') + logging.info(f' expected: {expected}') + logging.info(f' received: {received}') return False return True @@ -32,7 +32,7 @@ def test_get_handler(ip, port, verbosity=False): uri = '/hello?query1=value1&query2=value2&query3=value3' # GET hello response - test_headers = {'Test-Header-1':'Test-Value-1', 'Test-Header-2':'Test-Value-2'} + test_headers = {'Test-Header-1': 'Test-Value-1', 'Test-Header-2': 'Test-Value-2'} verbose_print(verbosity, 'Sending GET to URI : ', uri) verbose_print(verbosity, 'Sending additional headers : ') for k, v in test_headers.items(): @@ -45,10 +45,10 @@ def test_get_handler(ip, port, verbosity=False): sess.close() if not ( - test_val('Status code mismatch', 200, resp.status) and - test_val('Response mismatch', 'Custom-Value-1', resp.getheader('Custom-Header-1')) and - test_val('Response mismatch', 'Custom-Value-2', resp.getheader('Custom-Header-2')) and - test_val('Response mismatch', 'Hello World!', resp_data) + test_val('Status code mismatch', 200, resp.status) + and test_val('Response mismatch', 'Custom-Value-1', resp.getheader('Custom-Header-1')) + and test_val('Response mismatch', 'Custom-Value-2', resp.getheader('Custom-Header-2')) + and test_val('Response mismatch', 'Hello World!', resp_data) ): return False @@ -144,7 +144,7 @@ def test_put_handler(ip, port, verbosity=False): except http.client.HTTPException: # Catch socket error as we tried to communicate with an already closed socket pass - except IOError as err: + except OSError as err: if err.errno == errno.EPIPE: # Sometimes Broken Pipe error is returned # when sending data to a closed socket @@ -240,19 +240,17 @@ def test_custom_uri_query(ip, port, query, verbosity=False): if __name__ == '__main__': # Configure argument parser parser = argparse.ArgumentParser(description='Run HTTPd Test') - parser.add_argument('IP', metavar='IP', type=str, help='Server IP') - parser.add_argument('port', metavar='port', type=str, help='Server port') - parser.add_argument('msg', metavar='message', type=str, help='Message to be sent to server') + parser.add_argument('IP', metavar='IP', type=str, help='Server IP') + parser.add_argument('port', metavar='port', type=str, help='Server port') + parser.add_argument('msg', metavar='message', type=str, help='Message to be sent to server') args = vars(parser.parse_args()) # Get arguments - ip = args['IP'] + ip = args['IP'] port = args['port'] - msg = args['msg'] + msg = args['msg'] if not ( - test_get_handler(ip, port, True) and - test_put_handler(ip, port, True) and - test_post_handler(ip, port, msg, True) + test_get_handler(ip, port, True) and test_put_handler(ip, port, True) and test_post_handler(ip, port, msg, True) ): logging.info('Failed!') diff --git a/tools/ci/python_packages/idf_iperf_test_util/Attenuator.py b/tools/ci/python_packages/idf_iperf_test_util/Attenuator.py index 565cea6c7c4..ba79164e229 100644 --- a/tools/ci/python_packages/idf_iperf_test_util/Attenuator.py +++ b/tools/ci/python_packages/idf_iperf_test_util/Attenuator.py @@ -37,10 +37,10 @@ def set_att(port, att, att_fix=False): serial_port = serial.Serial(port, baudrate=9600, rtscts=False, timeout=0.1) if serial_port.isOpen() is False: - raise IOError('attenuator control, failed to open att port') + raise OSError('attenuator control, failed to open att port') - cmd_hex = '7e7e10{:02x}{:x}'.format(att_t, 0x10 + att_t) - exp_res_hex = '7e7e20{:02x}00{:x}'.format(att_t, 0x20 + att_t) + cmd_hex = f'7e7e10{att_t:02x}{0x10 + att_t:x}' + exp_res_hex = f'7e7e20{att_t:02x}00{0x20 + att_t:x}' cmd = codecs.decode(cmd_hex, 'hex') exp_res = codecs.decode(exp_res_hex, 'hex') diff --git a/tools/ci/python_packages/idf_iperf_test_util/LineChart.py b/tools/ci/python_packages/idf_iperf_test_util/LineChart.py index 88271f40204..f1b2d2ac894 100644 --- a/tools/ci/python_packages/idf_iperf_test_util/LineChart.py +++ b/tools/ci/python_packages/idf_iperf_test_util/LineChart.py @@ -29,19 +29,21 @@ def draw_line_chart(file_name, title, x_label, y_label, data_series, range_list) _data[str(key)] = data_series[item][key] _data = list(_data.values()) try: - legend = item + ' (max: {:.02f})'.format(max([x for x in _data if x])) + legend = item + f' (max: {max([x for x in _data if x]):.02f})' except TypeError: legend = item - line.add_yaxis(legend, _data, is_smooth=True, is_connect_nones=True, - label_opts=opts.LabelOpts(is_show=False)) + line.add_yaxis(legend, _data, is_smooth=True, is_connect_nones=True, label_opts=opts.LabelOpts(is_show=False)) line.set_global_opts( datazoom_opts=opts.DataZoomOpts(range_start=0, range_end=100), title_opts=opts.TitleOpts(title=title, pos_left='center'), legend_opts=opts.LegendOpts(pos_top='10%', pos_left='right', orient='vertical'), tooltip_opts=opts.TooltipOpts(trigger='axis'), xaxis_opts=opts.AxisOpts(type_='category', name=x_label, splitline_opts=opts.SplitLineOpts(is_show=True)), - yaxis_opts=opts.AxisOpts(type_='value', name=y_label, - axistick_opts=opts.AxisTickOpts(is_show=True), - splitline_opts=opts.SplitLineOpts(is_show=True)), + yaxis_opts=opts.AxisOpts( + type_='value', + name=y_label, + axistick_opts=opts.AxisTickOpts(is_show=True), + splitline_opts=opts.SplitLineOpts(is_show=True), + ), ) line.render(file_name) diff --git a/tools/ci/python_packages/idf_iperf_test_util/PowerControl.py b/tools/ci/python_packages/idf_iperf_test_util/PowerControl.py index 38d4c081217..1647f88850c 100644 --- a/tools/ci/python_packages/idf_iperf_test_util/PowerControl.py +++ b/tools/ci/python_packages/idf_iperf_test_util/PowerControl.py @@ -10,19 +10,19 @@ This file implements controlling APC PDU via telnet. import telnetlib -class Control(object): - """ control APC via telnet """ +class Control: + """control APC via telnet""" @classmethod def apc_telnet_make_choice(cls, telnet, choice): - """ select a choice """ + """select a choice""" telnet.read_until(b'Event Log') telnet.read_until(b'>') telnet.write(choice.encode() + b'\r\n') @classmethod def apc_telnet_common_action(cls, telnet, check_str, action): - """ wait until a pattern and then write a line """ + """wait until a pattern and then write a line""" telnet.read_until(check_str.encode()) telnet.write(action.encode() + b'\r\n') diff --git a/tools/ci/python_packages/idf_iperf_test_util/TestReport.py b/tools/ci/python_packages/idf_iperf_test_util/TestReport.py index 43e667b6503..35d9b91b4b0 100644 --- a/tools/ci/python_packages/idf_iperf_test_util/TestReport.py +++ b/tools/ci/python_packages/idf_iperf_test_util/TestReport.py @@ -8,10 +8,11 @@ The test report contains 2 parts: 1. throughput with different configs 2. throughput with RSSI """ + import os -class ThroughputForConfigsReport(object): +class ThroughputForConfigsReport: THROUGHPUT_TYPES = ['tcp_tx', 'tcp_rx', 'udp_tx', 'udp_rx'] REPORT_FILE_NAME = 'ThroughputForConfigs.md' @@ -21,13 +22,13 @@ class ThroughputForConfigsReport(object): :param ap_ssid: the ap we expected to use :param throughput_results: config with the following type:: { - "config_name": { - "tcp_tx": result, - "tcp_rx": result, - "udp_tx": result, - "udp_rx": result, + 'config_name': { + 'tcp_tx': result, + 'tcp_rx': result, + 'udp_tx': result, + 'udp_rx': result, }, - "config_name2": {}, + 'config_name2': {}, } """ self.output_path = output_path @@ -44,7 +45,7 @@ class ThroughputForConfigsReport(object): @staticmethod def _parse_config_file(config_file_path): sdkconfig = {} - with open(config_file_path, 'r') as f: + with open(config_file_path) as f: for line in f: if not line.isspace(): if line[0] == '#': @@ -83,7 +84,7 @@ class ThroughputForConfigsReport(object): except KeyError: _new_value = 'null' if _base_value != _new_value: - _difference[_config] = '{} -> {}'.format(_base_value, _new_value) + _difference[_config] = f'{_base_value} -> {_new_value}' return _difference for i, _config_name in enumerate(self.sort_order): @@ -98,9 +99,9 @@ class ThroughputForConfigsReport(object): if previous_config: # log the difference difference = find_difference(previous_config, current_config) - data += '* {} (compared to {}):\r\n'.format(_config_name, previous_config_name) + data += f'* {_config_name} (compared to {previous_config_name}):\r\n' for diff_name in difference: - data += ' * `{}`: {}\r\n'.format(diff_name, difference[diff_name]) + data += f' * `{diff_name}`: {difference[diff_name]}\r\n' return data def _generate_report_for_one_type(self, throughput_type): @@ -123,11 +124,11 @@ class ThroughputForConfigsReport(object): for config in self.sort_order: try: result = self.results[config][throughput_type] - throughput = '{:.02f}'.format(max(result.throughput_by_att[self.ap_ssid].values())) + throughput = f'{max(result.throughput_by_att[self.ap_ssid].values()):.02f}' heap_size = str(result.heap_size) # although markdown table will do alignment # do align here for better text editor presentation - ret += '| {:<12}| {:<18}| {:<23}|\r\n'.format(config, throughput, heap_size) + ret += f'| {config:<12}| {throughput:<18}| {heap_size:<23}|\r\n' empty = False except KeyError: pass @@ -135,7 +136,7 @@ class ThroughputForConfigsReport(object): def generate_report(self): data = '# Throughput for different configs\r\n' - data += '\r\nAP: {}\r\n'.format(self.ap_ssid) + data += f'\r\nAP: {self.ap_ssid}\r\n' for throughput_type in self.THROUGHPUT_TYPES: data += self._generate_report_for_one_type(throughput_type) @@ -147,18 +148,17 @@ class ThroughputForConfigsReport(object): f.write(data) -class ThroughputVsRssiReport(object): - +class ThroughputVsRssiReport: REPORT_FILE_NAME = 'ThroughputVsRssi.md' def __init__(self, output_path, throughput_results): """ :param throughput_results: config with the following type:: { - "tcp_tx": result, - "tcp_rx": result, - "udp_tx": result, - "udp_rx": result, + 'tcp_tx': result, + 'tcp_rx': result, + 'udp_tx': result, + 'udp_rx': result, } """ self.output_path = output_path @@ -193,9 +193,10 @@ class ThroughputVsRssiReport(object): _max_for_ap = max(result.throughput_by_rssi[ap_ssid].values()) if _max_for_ap > max_throughput: max_throughput = _max_for_ap - max_throughput = '{:.02f}'.format(max_throughput) - ret += '| {:<8}| {:<15}| {:<22}|\r\n'.format('{}_{}'.format(result.proto, result.direction), - curve_analysis, max_throughput) + max_throughput = f'{max_throughput:.02f}' + ret += '| {:<8}| {:<15}| {:<22}|\r\n'.format( + f'{result.proto}_{result.direction}', curve_analysis, max_throughput + ) return ret def _generate_report_for_one_type(self, result): @@ -219,13 +220,13 @@ class ThroughputVsRssiReport(object): """ result.post_analysis() - ret = '\r\n### {} {}\r\n'.format(result.proto, result.direction) + ret = f'\r\n### {result.proto} {result.direction}\r\n' if result.error_list: ret += '\r\nErrors:\r\n\r\n' for error in result.error_list: ret += '* ' + error + '\r\n' for ap_ssid in result.throughput_by_rssi: - ret += '\r\nAP: {}\r\n'.format(ap_ssid) + ret += f'\r\nAP: {ap_ssid}\r\n' # draw figure file_name = result.draw_throughput_figure(self.raw_data_path, ap_ssid, 'rssi') result.draw_throughput_figure(self.raw_data_path, ap_ssid, 'att') diff --git a/tools/ci/python_packages/wifi_tools.py b/tools/ci/python_packages/wifi_tools.py index 55a2569abc0..3255dddc716 100644 --- a/tools/ci/python_packages/wifi_tools.py +++ b/tools/ci/python_packages/wifi_tools.py @@ -38,8 +38,9 @@ class wpa_cli: dbus.mainloop.glib.DBusGMainLoop(set_as_default=True) bus = dbus.SystemBus() - service = dbus.Interface(bus.get_object('fi.w1.wpa_supplicant1', '/fi/w1/wpa_supplicant1'), - 'fi.w1.wpa_supplicant1') + service = dbus.Interface( + bus.get_object('fi.w1.wpa_supplicant1', '/fi/w1/wpa_supplicant1'), 'fi.w1.wpa_supplicant1' + ) iface_path = service.GetInterface(self.iface_name) self.iface_obj = bus.get_object('fi.w1.wpa_supplicant1', iface_path) self.iface_ifc = dbus.Interface(self.iface_obj, 'fi.w1.wpa_supplicant1.Interface') @@ -48,7 +49,7 @@ class wpa_cli: raise RuntimeError('supplicant : Failed to fetch interface') self.old_network = self._get_iface_property('CurrentNetwork') - print('Old network is %s' % self.old_network) + print(f'Old network is {self.old_network}') if self.old_network == '/': self.old_network = None @@ -56,10 +57,10 @@ class wpa_cli: self.connected = True except Exception as err: - raise Exception('Failure in wpa_cli init: {}'.format(err)) + raise Exception(f'Failure in wpa_cli init: {err}') def _get_iface_property(self, name): - """ Read the property with 'name' from the wi-fi interface object + """Read the property with 'name' from the wi-fi interface object Note: The result is a dbus wrapped type, so should usually convert it to the corresponding native Python type @@ -75,7 +76,11 @@ class wpa_cli: if self.new_network is not None: self.iface_ifc.RemoveNetwork(self.new_network) - print('Pre-connect state is %s, IP is %s' % (self._get_iface_property('State'), get_wiface_IPv4(self.iface_name))) + print( + 'Pre-connect state is {}, IP is {}'.format( + self._get_iface_property('State'), get_wiface_IPv4(self.iface_name) + ) + ) self.new_network = self.iface_ifc.AddNetwork({'ssid': ssid, 'psk': password}) self.iface_ifc.SelectNetwork(self.new_network) @@ -86,11 +91,11 @@ class wpa_cli: while retry > 0: time.sleep(5) state = str(self._get_iface_property('State')) - print('wpa iface state %s (scanning %s)' % (state, bool(self._get_iface_property('Scanning')))) + print('wpa iface state {} (scanning {})'.format(state, bool(self._get_iface_property('Scanning')))) if state in ['disconnected', 'inactive']: self.iface_ifc.Reconnect() ip = get_wiface_IPv4(self.iface_name) - print('wpa iface %s IP %s' % (self.iface_name, ip)) + print(f'wpa iface {self.iface_name} IP {ip}') if ip is not None: self.connected = True return ip @@ -101,7 +106,7 @@ class wpa_cli: print('wpa_cli : Connection failed') except Exception as err: - raise Exception('Failure in wpa_cli init: {}'.format(err)) + raise Exception(f'Failure in wpa_cli init: {err}') def reset(self): if self.iface_ifc is not None: diff --git a/tools/ci/test_autocomplete/test_autocomplete.py b/tools/ci/test_autocomplete/test_autocomplete.py index 96e12e6b9ac..42e299cef52 100755 --- a/tools/ci/test_autocomplete/test_autocomplete.py +++ b/tools/ci/test_autocomplete/test_autocomplete.py @@ -12,47 +12,26 @@ Test = collections.namedtuple('Test', ['name', 'shell', 'term', 'prefix', 'patte IDF_PATH = os.environ['IDF_PATH'] TESTS = ( - Test('base', - 'fish', - 'vt100', - '', - 'all.*app.*app-flash.*bootloader.*', - 'fish'), - Test('base', - 'bash', - 'xterm-256color', - '', - 'all.*app.*app-flash.*bootloader.*bootloader-flash.*build-system-targets.*clean.*', - 'sh'), - Test('base', - 'zsh', - '', - '', - 'all.*app.*app-flash.*bootloader.*bootloader-flash.*build-system-targets.*clean.*', - 'sh'), + Test('base', 'fish', 'vt100', '', 'all.*app.*app-flash.*bootloader.*', 'fish'), + Test( + 'base', + 'bash', + 'xterm-256color', + '', + 'all.*app.*app-flash.*bootloader.*bootloader-flash.*build-system-targets.*clean.*', + 'sh', + ), + Test( + 'base', 'zsh', '', '', 'all.*app.*app-flash.*bootloader.*bootloader-flash.*build-system-targets.*clean.*', 'sh' + ), # Tests for file autocompletion - Test('file', - 'fish', - 'vt100', - f'@{IDF_PATH}/README', - '.*README.*md.*', - 'fish'), - Test('file', - 'bash', - 'xterm-256color', - f'@{IDF_PATH}/README', - '.*README.*md.*', - 'sh'), - Test('file', - 'zsh', - '', - f'@{IDF_PATH}/README', - '.*README.*md.*', - 'sh') + Test('file', 'fish', 'vt100', f'@{IDF_PATH}/README', '.*README.*md.*', 'fish'), + Test('file', 'bash', 'xterm-256color', f'@{IDF_PATH}/README', '.*README.*md.*', 'sh'), + Test('file', 'zsh', '', f'@{IDF_PATH}/README', '.*README.*md.*', 'sh'), ) -# Additional positional arguments for all child.expect() calls are constant so we can rely on the order and print message -# about which pattern was matched +# Additional positional arguments for all child.expect() calls are constant so we can rely on the order and print +# message about which pattern was matched pargs = (pexpect.EOF, pexpect.TIMEOUT) @@ -68,7 +47,6 @@ def get_fail_msg(pproc: pexpect.spawn, msg: str, index: int) -> str: class UTTest(unittest.TestCase): - def test_shell(self) -> None: env = os.environ.copy() for test in TESTS: @@ -85,7 +63,9 @@ class UTTest(unittest.TestCase): self.assertEqual(i, 0, get_fail_msg(pproc, 'Export was not successful!', i)) pproc.send(f'idf.py {test.prefix}\t\t') i = pproc.expect([test.pattern, *pargs], timeout=100) - self.assertEqual(i, 0, get_fail_msg(pproc, f'Autocompletion for idf.py failed in {test.shell}!', i)) + self.assertEqual( + i, 0, get_fail_msg(pproc, f'Autocompletion for idf.py failed in {test.shell}!', i) + ) if __name__ == '__main__': diff --git a/tools/idf_py_actions/hint_modules/component_requirements.py b/tools/idf_py_actions/hint_modules/component_requirements.py index 8fe538a5e05..91f7b99a8d8 100644 --- a/tools/idf_py_actions/hint_modules/component_requirements.py +++ b/tools/idf_py_actions/hint_modules/component_requirements.py @@ -2,11 +2,10 @@ # SPDX-License-Identifier: Apache-2.0 import os import re -from typing import Optional from idf_py_actions.tools import get_build_context -''' +""" glossary: original_component: component which compilation failed source_component: component containing file which is including the missing header file @@ -14,15 +13,13 @@ candidate_component: component which contain the missing header file original_filename: abs path of file(compilation unit) in original_component source_filename: abs path of file in source_component which is including the missing header file missing_header: filename of the missing header included in source_filename -''' +""" # Regex to find source_filename in preprocessor's error message -ENOENT_RE = re.compile(r'^(.+):\d+:\d+: fatal error: (.+): No such file or directory$', - flags=re.MULTILINE) +ENOENT_RE = re.compile(r'^(.+):\d+:\d+: fatal error: (.+): No such file or directory$', flags=re.MULTILINE) # Regex to find full preprocessor's error message to identify the original_filename # in case the missing_header is reported in indirect include. -ENOENT_FULL_RE = re.compile(r'^(In file included.*?No such file or directory)$', - flags=re.MULTILINE | re.DOTALL) +ENOENT_FULL_RE = re.compile(r'^(In file included.*?No such file or directory)$', flags=re.MULTILINE | re.DOTALL) # Regex to find original_filename in preprocessor's error message ORIGINAL_FILE_RE = re.compile(r'.*from (.*):[\d]+:') @@ -40,7 +37,7 @@ def _get_absolute_path(filename: str, base: str) -> str: return filename -def generate_hint(output: str) -> Optional[str]: +def generate_hint(output: str) -> str | None: # get the project description proj_desc = get_build_context().get('proj_desc') if not proj_desc: @@ -101,15 +98,13 @@ def generate_hint(output: str) -> Optional[str]: # Original component not found. This should never happen, because the # original_filename has to part of some component, which was added to # the build. - return _bug((f'cannot find original component for source ' - f'component {found_source_component_name}')) + return _bug(f'cannot find original component for source component {found_source_component_name}') else: # We should never reach this path. It would probably mean # the preprocessor output was changed or we received malformed # output. - return _bug((f'cannot match original component filename for ' - f'source component {found_source_component_name}')) + return _bug(f'cannot match original component filename for source component {found_source_component_name}') # look for the header file in the public include directories of all components found_dep_component_names = [] @@ -142,11 +137,13 @@ def generate_hint(output: str) -> Optional[str]: if candidate_component_include_dirs: candidates = ', '.join(candidate_component_include_dirs) - return (f'Missing "{missing_header}" file name found in the following component(s): {candidates}. ' - f'Maybe one of the components needs to add the missing header directory to INCLUDE_DIRS ' - f'of idf_component_register call in CMakeLists.txt. Another possibility may be that the ' - f'component or its feature is not enabled in the configuration. Use "idf.py menuconfig" ' - f'to check if the required options are enabled.') + return ( + f'Missing "{missing_header}" file name found in the following component(s): {candidates}. ' + f'Maybe one of the components needs to add the missing header directory to INCLUDE_DIRS ' + f'of idf_component_register call in CMakeLists.txt. Another possibility may be that the ' + f'component or its feature is not enabled in the configuration. Use "idf.py menuconfig" ' + f'to check if the required options are enabled.' + ) # The missing header not found anywhere, nothing much we can do here. return None @@ -155,22 +152,22 @@ def generate_hint(output: str) -> Optional[str]: assert found_original_component_info is not None # to help mypy # Sanity check: verify we didn't somehow find a component which is already in the requirements list - all_reqs = (found_source_component_info['reqs'] - + found_source_component_info['managed_reqs']) + all_reqs = found_source_component_info['reqs'] + found_source_component_info['managed_reqs'] if found_original_component_name == found_source_component_name: # Add also private reqs, but only if source_component is same original_component. # The missing_header may be part of component which is already added as private # req for source_component. Meaning it's not part of source_component public # interface. - all_reqs += (found_source_component_info['priv_reqs'] - + found_source_component_info['managed_priv_reqs']) + all_reqs += found_source_component_info['priv_reqs'] + found_source_component_info['managed_priv_reqs'] for dep_component_name in found_dep_component_names: if dep_component_name in all_reqs: # Oops. This component is already in the requirements list. # How did this happen? - return _bug((f'{missing_header} found in component {dep_component_name} ' - f'which is already in the requirements list of {found_source_component_name}')) + return _bug( + f'{missing_header} found in component {dep_component_name} ' + f'which is already in the requirements list of {found_source_component_name}' + ) # try to figure out the correct require type: REQUIRES or PRIV_REQUIRES requires_type = None @@ -203,8 +200,7 @@ def generate_hint(output: str) -> Optional[str]: # already there. If so, we suggest to move it from PRIV_REQUIRES to REQUIRES. # This is done only if there is one component in found_dep_component_names, because # otherwise we cannot be sure which component should be moved. - priv_reqs = (found_source_component_info['priv_reqs'] - + found_source_component_info['managed_priv_reqs']) + priv_reqs = found_source_component_info['priv_reqs'] + found_source_component_info['managed_priv_reqs'] if found_dep_component_names[0] in priv_reqs: source_component_has_priv_dep = True @@ -213,19 +209,22 @@ def generate_hint(output: str) -> Optional[str]: cmakelists_file_to_fix = os.path.normpath(os.path.join(found_source_component_info['dir'], 'CMakeLists.txt')) problem_description = ( f'Compilation failed because {source_filename_short} (in "{found_source_component_name}" component) ' - f'includes {missing_header}, provided by {found_dep_component_names_list} component(s).\n') + f'includes {missing_header}, provided by {found_dep_component_names_list} component(s).\n' + ) if source_component_has_priv_dep: problem_solution = ( f'However, {found_dep_component_names_list} component(s) is in the private requirements list ' f'of "{found_source_component_name}".\n' f'To fix this, move {found_dep_component_names_list} from PRIV_REQUIRES into ' - f'REQUIRES list of idf_component_register call in {cmakelists_file_to_fix}.') + f'REQUIRES list of idf_component_register call in {cmakelists_file_to_fix}.' + ) else: problem_solution = ( f'However, {found_dep_component_names_list} component(s) is not in the requirements list ' f'of "{found_source_component_name}".\n' f'To fix this, add {found_dep_component_names_list} to {requires_type} list ' - f'of idf_component_register call in {cmakelists_file_to_fix}.') + f'of idf_component_register call in {cmakelists_file_to_fix}.' + ) return problem_description + problem_solution diff --git a/tools/install_util.py b/tools/install_util.py index a1762738697..063eb9be8b8 100644 --- a/tools/install_util.py +++ b/tools/install_util.py @@ -57,7 +57,7 @@ def action_print_help(script_extension: str) -> None: # extract the list of features from ./requirements.json thisdir = os.path.dirname(os.path.realpath(__file__)) - with open(f'{thisdir}/requirements.json', 'r', encoding='utf-8') as f: + with open(f'{thisdir}/requirements.json', encoding='utf-8') as f: json_data = json.load(f) features = [feat['name'] for feat in json_data['features']] @@ -101,7 +101,7 @@ def main() -> None: action_print_help(args.extension) sys.exit(0) - action_func = globals()['action_{}_{}'.format(args.action, args.type)] + action_func = globals()[f'action_{args.action}_{args.type}'] str_to_parse = vars(args)['str-to-parse'] or '' action_func(' '.join(chain([str_to_parse], unknown_args))) diff --git a/tools/ldgen/test/test_entity.py b/tools/ldgen/test/test_entity.py index 25ac8b6c611..acb661e6205 100755 --- a/tools/ldgen/test/test_entity.py +++ b/tools/ldgen/test/test_entity.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding=utf-8 # # SPDX-FileCopyrightText: 2018-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 @@ -10,14 +9,15 @@ import sys import unittest try: - from ldgen.entity import Entity, EntityDB + from ldgen.entity import Entity + from ldgen.entity import EntityDB except ImportError: sys.path.append(os.path.dirname(os.path.dirname(__file__))) - from ldgen.entity import Entity, EntityDB + from ldgen.entity import Entity + from ldgen.entity import EntityDB class EntityTest(unittest.TestCase): - def test_create_none(self): entity = Entity(Entity.ALL) self.assertEqual(Entity.Specificity.NONE, entity.specificity) @@ -128,7 +128,7 @@ class EntityTest(unittest.TestCase): self.assertEqual(entity_a, entity_b) def test_compare_none_vs_all(self): - # Two entities might have the same specifity whether + # Two entities might have the same specificity whether # Entity.ALL is used or not specified; the latter is # considered less than the former. entity_a = Entity() @@ -186,7 +186,6 @@ class EntityTest(unittest.TestCase): class EntityDBTest(unittest.TestCase): - def setUp(self): self.entities = EntityDB() diff --git a/tools/ldgen/test/test_fragments.py b/tools/ldgen/test/test_fragments.py index c7e72892eef..d36a1360e15 100755 --- a/tools/ldgen/test/test_fragments.py +++ b/tools/ldgen/test/test_fragments.py @@ -10,14 +10,25 @@ import unittest from io import StringIO try: - from ldgen.fragments import Align, Flag, Keep, Sort, Surround, parse_fragment_file + from ldgen.fragments import Align + from ldgen.fragments import Flag + from ldgen.fragments import Keep + from ldgen.fragments import Sort + from ldgen.fragments import Surround + from ldgen.fragments import parse_fragment_file from ldgen.sdkconfig import SDKConfig except ImportError: sys.path.append(os.path.dirname(os.path.dirname(__file__))) - from ldgen.fragments import Align, Flag, Keep, Sort, Surround, parse_fragment_file + from ldgen.fragments import Align + from ldgen.fragments import Flag + from ldgen.fragments import Keep + from ldgen.fragments import Sort + from ldgen.fragments import Surround + from ldgen.fragments import parse_fragment_file from ldgen.sdkconfig import SDKConfig -from pyparsing import ParseException, ParseFatalException +from pyparsing import ParseException +from pyparsing import ParseFatalException class FragmentTest(unittest.TestCase): @@ -51,7 +62,7 @@ class FragmentTest(unittest.TestCase): return f def test_basic(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] entries: value_1 @@ -67,7 +78,7 @@ entries: self.assertEqual(fragment_file.fragments[0].entries, {'value_1', 'value_2', 'value_3', 'value_a'}) def test_conditional(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] entries: value_1 @@ -84,7 +95,7 @@ entries: self.assertEqual(fragment_file.fragments[0].name, 'test') self.assertEqual(fragment_file.fragments[0].entries, {'value_1', 'value_2', 'value_3', 'value_5'}) - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] entries: value_1 @@ -103,7 +114,7 @@ entries: self.assertEqual(fragment_file.fragments[0].name, 'test') self.assertEqual(fragment_file.fragments[0].entries, {'value_1', 'value_3', 'value_6'}) - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] entries: value_1 @@ -121,10 +132,11 @@ entries: fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(fragment_file.fragments[0].name, 'test') - self.assertEqual(fragment_file.fragments[0].entries, - {'value_1', 'value_2', 'value_4', 'value_5', 'value_6', 'value_7'}) + self.assertEqual( + fragment_file.fragments[0].entries, {'value_1', 'value_2', 'value_4', 'value_5', 'value_6', 'value_7'} + ) - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] entries: if PERFORMANCE_A = n: @@ -134,14 +146,14 @@ entries: parse_fragment_file(test_fragment, self.sdkconfig) def test_empty_file(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" """) fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(len(fragment_file.fragments), 0) def test_setting_indent(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] entries: value_1 @@ -154,7 +166,7 @@ entries: self.assertEqual(fragment_file.fragments[0].entries, {'value_1', 'value_2', 'value_3'}) def test_settings_unmatch_indent(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] entries: value_1 @@ -165,7 +177,7 @@ entries: parse_fragment_file(test_fragment, self.sdkconfig) def test_unsupported_key(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] key_1: value_a @@ -174,14 +186,14 @@ key_1: parse_fragment_file(test_fragment, self.sdkconfig) def test_empty_fragment(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] """) with self.assertRaises(ParseException): parse_fragment_file(test_fragment, self.sdkconfig) def test_empty_conditional(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] entries: if PERFORMANCE_B = y: @@ -191,7 +203,7 @@ entries: with self.assertRaises(ParseException): parse_fragment_file(test_fragment, self.sdkconfig) - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] entries: if PERFORMANCE_B = y: @@ -201,7 +213,7 @@ entries: with self.assertRaises(ParseFatalException): parse_fragment_file(test_fragment, self.sdkconfig) - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] entries: if PERFORMANCE_B = y: @@ -214,7 +226,7 @@ entries: parse_fragment_file(test_fragment, self.sdkconfig) def test_out_of_order_conditional(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] entries: elif PERFORMANCE_B = y: @@ -225,7 +237,7 @@ entries: with self.assertRaises(ParseException): parse_fragment_file(test_fragment, self.sdkconfig) - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] entries: else: @@ -235,7 +247,7 @@ entries: parse_fragment_file(test_fragment, self.sdkconfig) def test_multiple_fragments(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test1] entries: value_1 @@ -252,7 +264,7 @@ entries: self.assertEqual(fragment_file.fragments[1].entries, {('section', 'target')}) def test_whole_conditional_fragment(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" if PERFORMANCE_B = y: [sections:test1] entries: @@ -290,7 +302,7 @@ entries: self.assertEqual(fragment_file.fragments[3].entries, {'value_5'}) def test_equivalent_conditional_fragment(self): - test_fragment1 = self.create_fragment_file(u""" + test_fragment1 = self.create_fragment_file(""" if PERFORMANCE_A = y: [sections:test1] entries: @@ -305,7 +317,7 @@ else: self.assertEqual(fragment_file1.fragments[0].name, 'test1') self.assertEqual(fragment_file1.fragments[0].entries, {'value_1'}) - test_fragment2 = self.create_fragment_file(u""" + test_fragment2 = self.create_fragment_file(""" [sections:test1] entries: if PERFORMANCE_A = y: @@ -320,9 +332,8 @@ entries: class SectionsTest(FragmentTest): - def test_basic(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] entries: .section1 @@ -332,7 +343,7 @@ entries: self.assertEqual(fragment_file.fragments[0].entries, {'.section1', '.section2'}) def test_duplicate_entries(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] entries: .section1 @@ -344,14 +355,14 @@ entries: self.assertEqual(fragment_file.fragments[0].entries, {'.section1', '.section2', '.section3'}) def test_empty_entries(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] entries: """) with self.assertRaises(ParseException): parse_fragment_file(test_fragment, self.sdkconfig) - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] entries: if PERFORMANCE_B = y: @@ -361,7 +372,7 @@ entries: parse_fragment_file(test_fragment, self.sdkconfig) def test_entries_grammar(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] entries: _valid1 @@ -369,11 +380,10 @@ entries: .valid3_- """) fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) - self.assertEqual(fragment_file.fragments[0].entries, - {'_valid1', 'valid2.', '.valid3_-'}) + self.assertEqual(fragment_file.fragments[0].entries, {'_valid1', 'valid2.', '.valid3_-'}) # invalid starting char - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] entries: 1invalid @@ -381,7 +391,7 @@ entries: with self.assertRaises(ParseException): parse_fragment_file(test_fragment, self.sdkconfig) - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] entries: -invalid @@ -390,16 +400,15 @@ entries: parse_fragment_file(test_fragment, self.sdkconfig) # + notation - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] entries: valid+ """) fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) - self.assertEqual(fragment_file.fragments[0].entries, - {'valid+'}) + self.assertEqual(fragment_file.fragments[0].entries, {'valid+'}) - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [sections:test] entries: inva+lid+ @@ -409,9 +418,8 @@ entries: class SchemeTest(FragmentTest): - def test_basic(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [scheme:test] entries: sections1 -> target1 @@ -419,12 +427,10 @@ entries: """) fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) - self.assertEqual(fragment_file.fragments[0].entries, - {('sections1', 'target1'), - ('sections2', 'target2')}) + self.assertEqual(fragment_file.fragments[0].entries, {('sections1', 'target1'), ('sections2', 'target2')}) def test_duplicate_entries(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [scheme:test] entries: sections1 -> target1 @@ -433,19 +439,17 @@ entries: """) fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) - self.assertEqual(fragment_file.fragments[0].entries, - {('sections1', 'target1'), - ('sections2', 'target2')}) + self.assertEqual(fragment_file.fragments[0].entries, {('sections1', 'target1'), ('sections2', 'target2')}) def test_empty_entries(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [scheme:test] entries: """) with self.assertRaises(ParseException): parse_fragment_file(test_fragment, self.sdkconfig) - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [scheme:test] entries: if PERFORMANCE_B = y: @@ -455,7 +459,7 @@ entries: parse_fragment_file(test_fragment, self.sdkconfig) def test_improper_grammar(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [scheme:test] entries: sections1, target1 # improper separator @@ -466,9 +470,8 @@ entries: class MappingTest(FragmentTest): - def test_basic(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:test] archive: lib.a entries: @@ -479,17 +482,19 @@ entries: * (noflash) """) - expected = {('obj', 'symbol', 'noflash'), - ('obj', None, 'noflash'), - ('obj', 'symbol_2', 'noflash'), - ('obj_2', None, 'noflash'), - ('*', None, 'noflash')} + expected = { + ('obj', 'symbol', 'noflash'), + ('obj', None, 'noflash'), + ('obj', 'symbol_2', 'noflash'), + ('obj_2', None, 'noflash'), + ('*', None, 'noflash'), + } fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(expected, fragment_file.fragments[0].entries) def test_archive(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:test] archive: entries: @@ -498,7 +503,7 @@ entries: with self.assertRaises(ParseException): parse_fragment_file(test_fragment, self.sdkconfig) - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:test] archive: lib1.a @@ -510,7 +515,7 @@ entries: parse_fragment_file(test_fragment, self.sdkconfig) def test_archive_allowed_names(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:test] archive: libstdc++.a @@ -521,7 +526,7 @@ entries: self.assertEqual('libstdc++.a', fragment_file.fragments[0].archive) def test_empty_entries(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:test] archive: lib.a @@ -535,7 +540,7 @@ entries: fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) self.assertEqual(expected, fragment_file.fragments[0].entries) - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:test] archive: lib.a @@ -546,7 +551,7 @@ entries: parse_fragment_file(test_fragment, self.sdkconfig) def test_duplicate_entries(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:test] archive: lib.a @@ -561,7 +566,7 @@ entries: self.assertEqual(expected, fragment_file.fragments[0].entries) def test_invalid_grammar(self): - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:test] archive: lib.a @@ -569,7 +574,7 @@ archive: with self.assertRaises(ParseException): parse_fragment_file(test_fragment, self.sdkconfig) - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:test] entries: * (default) @@ -577,7 +582,7 @@ entries: with self.assertRaises(ParseException): parse_fragment_file(test_fragment, self.sdkconfig) - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:test] archive: lib.a entries: @@ -586,7 +591,7 @@ entries: with self.assertRaises(ParseException): parse_fragment_file(test_fragment, self.sdkconfig) - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:test] archive: lib.a entries: @@ -595,7 +600,7 @@ entries: with self.assertRaises(ParseException): parse_fragment_file(test_fragment, self.sdkconfig) - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:test] archive: lib.a entries: @@ -604,7 +609,7 @@ entries: with self.assertRaises(ParseException): parse_fragment_file(test_fragment, self.sdkconfig) - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:test] archive: lib.a entries: @@ -613,7 +618,7 @@ entries: with self.assertRaises(ParseException): parse_fragment_file(test_fragment, self.sdkconfig) - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:test] archive: lib.a entries: @@ -622,7 +627,7 @@ entries: with self.assertRaises(ParseException): parse_fragment_file(test_fragment, self.sdkconfig) - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:test] archive: lib.a entries: @@ -631,7 +636,7 @@ entries: with self.assertRaises(ParseException): parse_fragment_file(test_fragment, self.sdkconfig) - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:test] archive: lib.a entries: @@ -642,7 +647,7 @@ entries: def test_keep_flag(self): # Test parsing combinations and orders of flags - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:map] archive: libmain.a entries: @@ -654,15 +659,14 @@ entries: fragment = fragment_file.fragments[0] - expected = [Flag('text', 'flash_text', [Keep()]), - Flag('rodata', 'flash_rodata', [Keep(), Keep()])] + expected = [Flag('text', 'flash_text', [Keep()]), Flag('rodata', 'flash_rodata', [Keep(), Keep()])] actual = fragment.flags[('obj1', None, 'default')] self.assertEqual(expected, actual) def test_align_flag(self): # Test parsing combinations and orders of flags - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:map] archive: libmain.a entries: @@ -677,17 +681,19 @@ entries: fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) fragment = fragment_file.fragments[0] - expected = [Flag('text', 'flash_text', [Align(8, True, False)]), - Flag('rodata', 'flash_rodata', [Align(8, True, False)]), - Flag('data', 'dram0_data', [Align(8, True, True)]), - Flag('bss', 'dram0_bss', [Align(8, False, True)]), - Flag('common', 'dram0_bss', [Align(8, True, True), Align(8, True, False)])] + expected = [ + Flag('text', 'flash_text', [Align(8, True, False)]), + Flag('rodata', 'flash_rodata', [Align(8, True, False)]), + Flag('data', 'dram0_data', [Align(8, True, True)]), + Flag('bss', 'dram0_bss', [Align(8, False, True)]), + Flag('common', 'dram0_bss', [Align(8, True, True), Align(8, True, False)]), + ] actual = fragment.flags[('obj1', None, 'default')] self.assertEqual(expected, actual) # Wrong post, pre order - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:map] archive: libmain.a entries: @@ -700,7 +706,7 @@ entries: def test_sort_flag(self): # Test parsing combinations and orders of flags - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:map] archive: libmain.a entries: @@ -718,20 +724,22 @@ entries: fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) fragment = fragment_file.fragments[0] - expected = [Flag('text', 'flash_text', [Sort()]), - Flag('text', 'flash_text', [Sort('name')]), - Flag('rodata', 'flash_rodata', [Sort('alignment')]), - Flag('data', 'dram0_data', [Sort('init_priority')]), - Flag('bss', 'dram0_bss', [Sort('name', 'alignment')]), - Flag('common', 'dram0_bss', [Sort('alignment', 'name')]), - Flag('iram', 'iram0_text', [Sort('name', 'name')]), - Flag('dram', 'dram0_data', [Sort('alignment', 'alignment')])] + expected = [ + Flag('text', 'flash_text', [Sort()]), + Flag('text', 'flash_text', [Sort('name')]), + Flag('rodata', 'flash_rodata', [Sort('alignment')]), + Flag('data', 'dram0_data', [Sort('init_priority')]), + Flag('bss', 'dram0_bss', [Sort('name', 'alignment')]), + Flag('common', 'dram0_bss', [Sort('alignment', 'name')]), + Flag('iram', 'iram0_text', [Sort('name', 'name')]), + Flag('dram', 'dram0_data', [Sort('alignment', 'alignment')]), + ] actual = fragment.flags[('obj1', None, 'default')] self.assertEqual(expected, actual) def test_surround_flag(self): # Test parsing combinations and orders of flags - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:map] archive: libmain.a entries: @@ -747,7 +755,7 @@ entries: def test_flag_order(self): # Test that the order in which the flags are specified is retained - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:map] archive: libmain.a entries: @@ -758,18 +766,26 @@ entries: fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) fragment = fragment_file.fragments[0] - expected = [Flag('text', 'flash_text', [Align(4, True, False), - Keep(), - Surround('sym1'), - Align(8, True, False), - Sort('name')]), - Flag('rodata', 'flash_rodata', [Keep(), - Align(4, True, False), - Keep(), - Surround('sym1'), - Align(8, True, False), - Align(4, True, False), - Sort('name')])] + expected = [ + Flag( + 'text', + 'flash_text', + [Align(4, True, False), Keep(), Surround('sym1'), Align(8, True, False), Sort('name')], + ), + Flag( + 'rodata', + 'flash_rodata', + [ + Keep(), + Align(4, True, False), + Keep(), + Surround('sym1'), + Align(8, True, False), + Align(4, True, False), + Sort('name'), + ], + ), + ] actual = fragment.flags[('obj1', None, 'default')] self.assertEqual(expected, actual) @@ -777,7 +793,7 @@ entries: # Not an error, generation step handles this, since # it that step has a more complete information # about all mappings. - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:map] archive: libmain.a entries: @@ -788,14 +804,10 @@ entries: fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) fragment = fragment_file.fragments[0] - expected = [Flag('text', 'flash_text', [Align(4, True, False), - Keep(), - Surround('sym1'), - Sort('name')]), - Flag('text', 'flash_text', [Align(4, True, False), - Keep(), - Surround('sym1'), - Sort('name')])] + expected = [ + Flag('text', 'flash_text', [Align(4, True, False), Keep(), Surround('sym1'), Sort('name')]), + Flag('text', 'flash_text', [Align(4, True, False), Keep(), Surround('sym1'), Sort('name')]), + ] actual = fragment.flags[('obj1', None, 'default')] self.assertEqual(expected, actual) @@ -804,7 +816,7 @@ entries: # it that step has a more complete information # about all mappings. This can happen across multiple # mapping fragments. - test_fragment = self.create_fragment_file(u""" + test_fragment = self.create_fragment_file(""" [mapping:map] archive: libmain.a entries: @@ -816,14 +828,10 @@ entries: fragment_file = parse_fragment_file(test_fragment, self.sdkconfig) fragment = fragment_file.fragments[0] - expected = [Flag('text', 'flash_text', [Align(4, True, False), - Keep(), - Surround('sym1'), - Sort('name')]), - Flag('text', 'flash_text', [Align(4, True, False), - Keep(), - Surround('sym1'), - Sort('name')])] + expected = [ + Flag('text', 'flash_text', [Align(4, True, False), Keep(), Surround('sym1'), Sort('name')]), + Flag('text', 'flash_text', [Align(4, True, False), Keep(), Surround('sym1'), Sort('name')]), + ] actual = fragment.flags[('obj1', None, 'default')] self.assertEqual(expected, actual) diff --git a/tools/ldgen/test/test_output_commands.py b/tools/ldgen/test/test_output_commands.py index 27906d1ed1b..8b1acbe1c95 100755 --- a/tools/ldgen/test/test_output_commands.py +++ b/tools/ldgen/test/test_output_commands.py @@ -9,11 +9,15 @@ import unittest try: from ldgen.entity import Entity - from ldgen.output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress + from ldgen.output_commands import AlignAtAddress + from ldgen.output_commands import InputSectionDesc + from ldgen.output_commands import SymbolAtAddress except ImportError: sys.path.append(os.path.dirname(os.path.dirname(__file__))) from ldgen.entity import Entity - from ldgen.output_commands import AlignAtAddress, InputSectionDesc, SymbolAtAddress + from ldgen.output_commands import AlignAtAddress + from ldgen.output_commands import InputSectionDesc + from ldgen.output_commands import SymbolAtAddress SECTIONS = ['.text', '.text.*', '.literal', '.literal.*'] @@ -23,7 +27,6 @@ CROUTINE = Entity('libfreertos.a', 'croutine') class InputSectionDescTest(unittest.TestCase): - def test_catch_all_placement(self): # Test default (catch-all) command expected = '*(.literal .literal.* .text .text.*)' @@ -78,44 +81,56 @@ class InputSectionDescTest(unittest.TestCase): InputSectionDesc(Entity('libfreertos.a', 'croutine'), SECTIONS, [Entity()]) with self.assertRaises(AssertionError): - InputSectionDesc(Entity('libfreertos.a', 'croutine'), SECTIONS, [Entity('libfreertos.a', 'croutine', 'prvCheckPendingReadyList')]) + InputSectionDesc( + Entity('libfreertos.a', 'croutine'), + SECTIONS, + [Entity('libfreertos.a', 'croutine', 'prvCheckPendingReadyList')], + ) def test_exclusions(self): # Test exclusions # Library - expected = ('*libfreertos.a:croutine.*' - '(EXCLUDE_FILE(*libfreertos.a) ' - '.literal EXCLUDE_FILE(*libfreertos.a) ' - '.literal.* EXCLUDE_FILE(*libfreertos.a) ' - '.text EXCLUDE_FILE(*libfreertos.a) .text.*)') + expected = ( + '*libfreertos.a:croutine.*' + '(EXCLUDE_FILE(*libfreertos.a) ' + '.literal EXCLUDE_FILE(*libfreertos.a) ' + '.literal.* EXCLUDE_FILE(*libfreertos.a) ' + '.text EXCLUDE_FILE(*libfreertos.a) .text.*)' + ) desc = InputSectionDesc(CROUTINE, SECTIONS, [FREERTOS]) self.assertEqual(expected, str(desc)) # Object - expected = ('*libfreertos.a:croutine.*' - '(EXCLUDE_FILE(*libfreertos.a:croutine.*) ' - '.literal EXCLUDE_FILE(*libfreertos.a:croutine.*) ' - '.literal.* EXCLUDE_FILE(*libfreertos.a:croutine.*) ' - '.text EXCLUDE_FILE(*libfreertos.a:croutine.*) .text.*)') + expected = ( + '*libfreertos.a:croutine.*' + '(EXCLUDE_FILE(*libfreertos.a:croutine.*) ' + '.literal EXCLUDE_FILE(*libfreertos.a:croutine.*) ' + '.literal.* EXCLUDE_FILE(*libfreertos.a:croutine.*) ' + '.text EXCLUDE_FILE(*libfreertos.a:croutine.*) .text.*)' + ) desc = InputSectionDesc(CROUTINE, SECTIONS, [CROUTINE]) self.assertEqual(expected, str(desc)) # Multiple exclusions - expected = ('*libfreertos.a:croutine.*' - '(EXCLUDE_FILE(*libfreertos.a *libfreertos.a:croutine.*) ' - '.literal EXCLUDE_FILE(*libfreertos.a *libfreertos.a:croutine.*) ' - '.literal.* EXCLUDE_FILE(*libfreertos.a *libfreertos.a:croutine.*) ' - '.text EXCLUDE_FILE(*libfreertos.a *libfreertos.a:croutine.*) .text.*)') + expected = ( + '*libfreertos.a:croutine.*' + '(EXCLUDE_FILE(*libfreertos.a *libfreertos.a:croutine.*) ' + '.literal EXCLUDE_FILE(*libfreertos.a *libfreertos.a:croutine.*) ' + '.literal.* EXCLUDE_FILE(*libfreertos.a *libfreertos.a:croutine.*) ' + '.text EXCLUDE_FILE(*libfreertos.a *libfreertos.a:croutine.*) .text.*)' + ) desc = InputSectionDesc(CROUTINE, SECTIONS, [FREERTOS, CROUTINE]) self.assertEqual(expected, str(desc)) # Disambugated exclusion - expected = ('*libfreertos.a:croutine.*' - '(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) ' - '.literal EXCLUDE_FILE(*libfreertos.a:croutine.c.*) ' - '.literal.* EXCLUDE_FILE(*libfreertos.a:croutine.c.*) ' - '.text EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text.*)') + expected = ( + '*libfreertos.a:croutine.*' + '(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) ' + '.literal EXCLUDE_FILE(*libfreertos.a:croutine.c.*) ' + '.literal.* EXCLUDE_FILE(*libfreertos.a:croutine.c.*) ' + '.text EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text.*)' + ) desc = InputSectionDesc(CROUTINE, SECTIONS, [Entity('libfreertos.a', 'croutine.c')]) self.assertEqual(expected, str(desc)) @@ -135,94 +150,124 @@ class InputSectionDescTest(unittest.TestCase): def test_sort(self): # Test sort - expected = ('*libfreertos.a:croutine.*(' - 'SORT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal) ' - 'SORT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal.*) ' - 'SORT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text) ' - 'SORT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text.*)' - ')') - desc = InputSectionDesc(CROUTINE, SECTIONS, [Entity('libfreertos.a', 'croutine.c')], keep=False, sort=(None, None)) + expected = ( + '*libfreertos.a:croutine.*(' + 'SORT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal) ' + 'SORT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal.*) ' + 'SORT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text) ' + 'SORT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text.*)' + ')' + ) + desc = InputSectionDesc( + CROUTINE, SECTIONS, [Entity('libfreertos.a', 'croutine.c')], keep=False, sort=(None, None) + ) self.assertEqual(expected, str(desc)) - expected = ('*libfreertos.a:croutine.*(' - 'SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal) ' - 'SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal.*) ' - 'SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text) ' - 'SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text.*)' - ')') - desc = InputSectionDesc(CROUTINE, SECTIONS, [Entity('libfreertos.a', 'croutine.c')], keep=False, sort=('name', None)) + expected = ( + '*libfreertos.a:croutine.*(' + 'SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal) ' + 'SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal.*) ' + 'SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text) ' + 'SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text.*)' + ')' + ) + desc = InputSectionDesc( + CROUTINE, SECTIONS, [Entity('libfreertos.a', 'croutine.c')], keep=False, sort=('name', None) + ) self.assertEqual(expected, str(desc)) - expected = ('*libfreertos.a:croutine.*(' - 'SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal) ' - 'SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal.*) ' - 'SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text) ' - 'SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text.*)' - ')') - desc = InputSectionDesc(CROUTINE, SECTIONS, [Entity('libfreertos.a', 'croutine.c')], keep=False, sort=('alignment', None)) + expected = ( + '*libfreertos.a:croutine.*(' + 'SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal) ' + 'SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal.*) ' + 'SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text) ' + 'SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text.*)' + ')' + ) + desc = InputSectionDesc( + CROUTINE, SECTIONS, [Entity('libfreertos.a', 'croutine.c')], keep=False, sort=('alignment', None) + ) self.assertEqual(expected, str(desc)) - expected = ('*libfreertos.a:croutine.*(' - 'SORT_BY_INIT_PRIORITY(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal) ' - 'SORT_BY_INIT_PRIORITY(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal.*) ' - 'SORT_BY_INIT_PRIORITY(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text) ' - 'SORT_BY_INIT_PRIORITY(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text.*)' - ')') - desc = InputSectionDesc(CROUTINE, SECTIONS, [Entity('libfreertos.a', 'croutine.c')], keep=False, sort=('init_priority', None)) + expected = ( + '*libfreertos.a:croutine.*(' + 'SORT_BY_INIT_PRIORITY(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal) ' + 'SORT_BY_INIT_PRIORITY(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal.*) ' + 'SORT_BY_INIT_PRIORITY(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text) ' + 'SORT_BY_INIT_PRIORITY(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text.*)' + ')' + ) + desc = InputSectionDesc( + CROUTINE, SECTIONS, [Entity('libfreertos.a', 'croutine.c')], keep=False, sort=('init_priority', None) + ) self.assertEqual(expected, str(desc)) - expected = ('*libfreertos.a:croutine.*(' - 'SORT_BY_NAME(SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal)) ' - 'SORT_BY_NAME(SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal.*)) ' - 'SORT_BY_NAME(SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text)) ' - 'SORT_BY_NAME(SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text.*))' - ')') - desc = InputSectionDesc(CROUTINE, SECTIONS, [Entity('libfreertos.a', 'croutine.c')], keep=False, sort=('name', 'alignment')) + expected = ( + '*libfreertos.a:croutine.*(' + 'SORT_BY_NAME(SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal)) ' + 'SORT_BY_NAME(SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal.*)) ' + 'SORT_BY_NAME(SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text)) ' + 'SORT_BY_NAME(SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text.*))' + ')' + ) + desc = InputSectionDesc( + CROUTINE, SECTIONS, [Entity('libfreertos.a', 'croutine.c')], keep=False, sort=('name', 'alignment') + ) self.assertEqual(expected, str(desc)) - expected = ('*libfreertos.a:croutine.*(' - 'SORT_BY_NAME(SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal)) ' - 'SORT_BY_NAME(SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal.*)) ' - 'SORT_BY_NAME(SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text)) ' - 'SORT_BY_NAME(SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text.*))' - ')') - desc = InputSectionDesc(CROUTINE, SECTIONS, [Entity('libfreertos.a', 'croutine.c')], keep=False, sort=('name', 'name')) + expected = ( + '*libfreertos.a:croutine.*(' + 'SORT_BY_NAME(SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal)) ' + 'SORT_BY_NAME(SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal.*)) ' + 'SORT_BY_NAME(SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text)) ' + 'SORT_BY_NAME(SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text.*))' + ')' + ) + desc = InputSectionDesc( + CROUTINE, SECTIONS, [Entity('libfreertos.a', 'croutine.c')], keep=False, sort=('name', 'name') + ) self.assertEqual(expected, str(desc)) - expected = ('*libfreertos.a:croutine.*(' - 'SORT_BY_ALIGNMENT(SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal)) ' - 'SORT_BY_ALIGNMENT(SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal.*)) ' - 'SORT_BY_ALIGNMENT(SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text)) ' - 'SORT_BY_ALIGNMENT(SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text.*))' - ')') - desc = InputSectionDesc(CROUTINE, SECTIONS, [Entity('libfreertos.a', 'croutine.c')], keep=False, sort=('alignment', 'name')) + expected = ( + '*libfreertos.a:croutine.*(' + 'SORT_BY_ALIGNMENT(SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal)) ' + 'SORT_BY_ALIGNMENT(SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal.*)) ' + 'SORT_BY_ALIGNMENT(SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text)) ' + 'SORT_BY_ALIGNMENT(SORT_BY_NAME(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text.*))' + ')' + ) + desc = InputSectionDesc( + CROUTINE, SECTIONS, [Entity('libfreertos.a', 'croutine.c')], keep=False, sort=('alignment', 'name') + ) self.assertEqual(expected, str(desc)) - expected = ('*libfreertos.a:croutine.*(' - 'SORT_BY_ALIGNMENT(SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal)) ' - 'SORT_BY_ALIGNMENT(SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal.*)) ' - 'SORT_BY_ALIGNMENT(SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text)) ' - 'SORT_BY_ALIGNMENT(SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text.*))' - ')') - desc = InputSectionDesc(CROUTINE, SECTIONS, [Entity('libfreertos.a', 'croutine.c')], keep=False, sort=('alignment', 'alignment')) + expected = ( + '*libfreertos.a:croutine.*(' + 'SORT_BY_ALIGNMENT(SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal)) ' + 'SORT_BY_ALIGNMENT(SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .literal.*)) ' + 'SORT_BY_ALIGNMENT(SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text)) ' + 'SORT_BY_ALIGNMENT(SORT_BY_ALIGNMENT(EXCLUDE_FILE(*libfreertos.a:croutine.c.*) .text.*))' + ')' + ) + desc = InputSectionDesc( + CROUTINE, SECTIONS, [Entity('libfreertos.a', 'croutine.c')], keep=False, sort=('alignment', 'alignment') + ) self.assertEqual(expected, str(desc)) class SymbolAtAddressTest(unittest.TestCase): - def test_symbol(self): symbol = 'test_symbol' - expected = '%s = ABSOLUTE(.);' % symbol + expected = f'{symbol} = ABSOLUTE(.);' desc = SymbolAtAddress(symbol) self.assertEqual(expected, str(desc)) class AlignAtAddressTest(unittest.TestCase): - def test_align(self): align = 8 - expected = '. = ALIGN(%d);' % 8 + expected = '. = ALIGN(8);' desc = AlignAtAddress(align) self.assertEqual(expected, str(desc)) diff --git a/tools/test_apps/system/log/pytest_log.py b/tools/test_apps/system/log/pytest_log.py index 0f0b29a8ead..41083126c1a 100644 --- a/tools/test_apps/system/log/pytest_log.py +++ b/tools/test_apps/system/log/pytest_log.py @@ -176,9 +176,10 @@ def test_log(dut: Dut) -> None: if dut.app.sdkconfig.get('BOOTLOADER_LOG_MODE_BINARY_EN'): monitor_cmd += ' ' + os.path.join(dut.app.binary_path, 'bootloader', 'bootloader.elf') monitor_log_path = os.path.join(dut.logdir, 'monitor.txt') - with open(monitor_log_path, 'w') as log, pexpect.spawn( - monitor_cmd, logfile=log, timeout=3, encoding='utf-8', codec_errors='ignore' - ) as p: + with ( + open(monitor_log_path, 'w') as log, + pexpect.spawn(monitor_cmd, logfile=log, timeout=3, encoding='utf-8', codec_errors='ignore') as p, + ): check_output(p) else: check_output(dut) diff --git a/tools/test_apps/system/mmu_page_size/conftest.py b/tools/test_apps/system/mmu_page_size/conftest.py index 21d1a1145e1..361409d5f30 100644 --- a/tools/test_apps/system/mmu_page_size/conftest.py +++ b/tools/test_apps/system/mmu_page_size/conftest.py @@ -23,7 +23,7 @@ class FlashBootloader(IdfSerial): bootloader_path = os.path.join(binary_path, 'bootloader', 'bootloader.bin') logging.info(bootloader_path) offs = int(self.app.sdkconfig.get('BOOTLOADER_OFFSET_IN_FLASH', 0)) - logging.info('bootloader offset is {0}'.format(hex(offs))) + logging.info(f'bootloader offset is {hex(offs)}') prev_flash_files = self.app.flash_files flash_files = [] flash_files.append( diff --git a/tools/test_apps/system/no_embedded_paths/check_for_file_paths.py b/tools/test_apps/system/no_embedded_paths/check_for_file_paths.py index ca0bd21b9c3..fca406e0098 100755 --- a/tools/test_apps/system/no_embedded_paths/check_for_file_paths.py +++ b/tools/test_apps/system/no_embedded_paths/check_for_file_paths.py @@ -13,19 +13,8 @@ # # # -# Copyright 2019 Espressif Systems (Shanghai) PTE LTD -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: 2019-2026 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 # import os import re @@ -54,27 +43,29 @@ def main(): # type: () -> None assert os.path.exists(idf_path) assert os.path.exists(build_dir) - print('Checking object files in {} for mentions of {}...'.format(build_dir, idf_path)) + print(f'Checking object files in {build_dir} for mentions of {idf_path}...') # note: once ESP-IDF moves to Python >=3.6 then this can be simplified to use 'glob' and f'{build_dir}**/*.obj' files = [] - for (dirpath, _, filepaths) in os.walk(build_dir): + for dirpath, _, filepaths in os.walk(build_dir): files += [os.path.join(dirpath, filepath) for filepath in filepaths if filepath.endswith('.obj')] - print('Found {} object files...'.format(len(files))) + print(f'Found {len(files)} object files...') - idf_path_binary = idf_path.encode() # we're going to be checking binary streams (note: probably non-ascii IDF_PATH will not match OK) + idf_path_binary = ( + idf_path.encode() + ) # we're going to be checking binary streams (note: probably non-ascii IDF_PATH will not match OK) failures = 0 for obj_file in files: if not any(re.search(exception, obj_file) for exception in EXCEPTIONS): failures += check_file(obj_file, idf_path_binary) if failures > 0: - raise SystemExit('{} source files are embedding file paths, see list above.'.format(failures)) + raise SystemExit(f'{failures} source files are embedding file paths, see list above.') print('No embedded file paths found') -def check_file(obj_file, idf_path): # type: (str, bytes) -> int +def check_file(obj_file, idf_path): # type: (str, bytes) -> int failures = 0 with open(obj_file, 'rb') as f: elf = ELFFile(f) @@ -84,7 +75,7 @@ def check_file(obj_file, idf_path): # type: (str, bytes) -> int if '.rodata' in sec.name or '.dram' in sec.name: contents = sec.data() if idf_path in contents: - print('error: {} contains an unwanted __FILE__ macro'.format(obj_file)) + print(f'error: {obj_file} contains an unwanted __FILE__ macro') failures += 1 break return failures diff --git a/tools/test_apps/system/unicore_bootloader/conftest.py b/tools/test_apps/system/unicore_bootloader/conftest.py index 72c33c8687a..3c00244ae8b 100644 --- a/tools/test_apps/system/unicore_bootloader/conftest.py +++ b/tools/test_apps/system/unicore_bootloader/conftest.py @@ -24,7 +24,7 @@ class FlashBootloader(IdfSerial): bootloader_path = os.path.join(binary_path, 'bootloader', 'bootloader.bin') logging.info(bootloader_path) offs = int(self.app.sdkconfig.get('BOOTLOADER_OFFSET_IN_FLASH', 0)) - logging.info('bootloader offset is {0}'.format(hex(offs))) + logging.info(f'bootloader offset is {hex(offs)}') prev_flash_files = self.app.flash_files flash_files = [] flash_files.append( diff --git a/tools/test_bsasm/test_bsasm.py b/tools/test_bsasm/test_bsasm.py index 275aeb0a830..8e41454f1aa 100755 --- a/tools/test_bsasm/test_bsasm.py +++ b/tools/test_bsasm/test_bsasm.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 @@ -12,8 +11,6 @@ import subprocess import tempfile import unittest from typing import Any -from typing import List -from typing import Tuple from typing import TypedDict @@ -28,7 +25,7 @@ class DecompInstOp(TypedDict, total=False): class DecompInst(TypedDict, total=False): - mux_val: List[int] + mux_val: list[int] opcode: DecompInstOp read_in: int wr_out: int @@ -48,8 +45,8 @@ class UnpackedBinary(TypedDict, total=False): trailing_bits: int eof_on: int padding: int - inst: List[DecompInst] - lut: List[int] + inst: list[DecompInst] + lut: list[int] current_dir = os.path.dirname(os.path.realpath(__file__)) @@ -57,11 +54,11 @@ bsasm_path = os.path.join(current_dir, '..', 'bsasm.py') class TestAssembler(unittest.TestCase): - def bit_from_inst(self, data: Tuple[int, int], off: int) -> int: + def bit_from_inst(self, data: tuple[int, int], off: int) -> int: return data[off // 8] & (1 << (off & 0x7)) # returns a field from a list of bytes. Starts at bit off, field is of size length. - def bits_from_inst(self, data: Tuple[int, int], off: int, length: int) -> int: + def bits_from_inst(self, data: tuple[int, int], off: int, length: int) -> int: ret = 0 for i in range(0, length): if self.bit_from_inst(data, off + i) != 0: @@ -69,7 +66,7 @@ class TestAssembler(unittest.TestCase): return ret # returns a decomposed instruction - def decode_inst(self, data: Tuple[int, int]) -> DecompInst: + def decode_inst(self, data: tuple[int, int]) -> DecompInst: ret: DecompInst = {} mux_val = [] for i in range(0, 32): @@ -140,18 +137,16 @@ class TestAssembler(unittest.TestCase): inst.append(self.decode_inst(struct.unpack_from('B' * 36, data, off))) off += 36 unpacked['inst'] = inst - unpacked['lut'] = list( - struct.unpack_from('<' + 'L' * unpacked['lut_size_words'], data, off) - ) + unpacked['lut'] = list(struct.unpack_from('<' + 'L' * unpacked['lut_size_words'], data, off)) return unpacked def compare(self, out: Any, js: Any, base: str) -> None: self.assertEqual(type(out), type(js), ' Diverging types between json and decoded obj: ' + base) - if type(js) == dict: + if type(js) is dict: for k in js: self.assertTrue(k in out, ' Key not found in decoded output: ' + base + '.' + k) self.compare(out[k], js[k], base + '.' + k) - elif type(js) == list: + elif type(js) is list: for k in range(0, len(js)): self.compare(out[k], js[k], f'{base}[{k}]') else: @@ -188,7 +183,7 @@ class TestAssembler(unittest.TestCase): if not should_fail: self.assertEqual(p.returncode, 0) else: - print('Note: THE TEST EXPECTED BSASM TO ERROR OUT. If there\'s error text above, that is expected.') + print("Note: THE TEST EXPECTED BSASM TO ERROR OUT. If there's error text above, that is expected.") self.assertNotEqual(p.returncode, 0) if not should_fail: @@ -197,7 +192,8 @@ class TestAssembler(unittest.TestCase): try: with open(jsfn) as out_desc_f: out_desc = json.load(out_desc_f) - # We were able to open the JSON file. See if the keys in it match up with the ones in the decoded fields. + # We were able to open the JSON file. See if the keys in it match up with the ones in the decoded + # fields. self.compare(b, out_desc, '') except FileNotFoundError: print(f'File not found: {jsfn}. Printing out decoded contents instead.') diff --git a/tools/test_build_system/conftest.py b/tools/test_build_system/conftest.py index 71ee0022d0f..868a8135211 100644 --- a/tools/test_build_system/conftest.py +++ b/tools/test_build_system/conftest.py @@ -191,9 +191,7 @@ def _session_work_dir(request: FixtureRequest) -> typing.Generator[tuple[Path, b if work_dir: # resolve allows using relative paths with --work-dir option - work_dir_path = Path(work_dir).resolve() / datetime.datetime.now(datetime.timezone.utc).strftime( - '%Y-%m-%d_%H-%M-%S' - ) + work_dir_path = Path(work_dir).resolve() / datetime.datetime.now(datetime.UTC).strftime('%Y-%m-%d_%H-%M-%S') logging.debug(f'using work directory: {work_dir_path}') os.makedirs(work_dir_path, exist_ok=True) clean_dir = None diff --git a/tools/test_build_system/test_bootloader.py b/tools/test_build_system/test_bootloader.py index 4e7abc07dbe..be3c5751cb8 100644 --- a/tools/test_build_system/test_bootloader.py +++ b/tools/test_build_system/test_bootloader.py @@ -3,16 +3,19 @@ import logging import shutil from pathlib import Path -from typing import Union -from test_build_system_helpers import EnvDict, IdfPyFunc, bin_file_contains, file_contains, replace_in_file +from test_build_system_helpers import EnvDict +from test_build_system_helpers import IdfPyFunc +from test_build_system_helpers import bin_file_contains +from test_build_system_helpers import file_contains +from test_build_system_helpers import replace_in_file -def get_two_header_bytes(file_path: Union[str, Path]) -> str: - ''' +def get_two_header_bytes(file_path: str | Path) -> str: + """ get the bytes 3-4 of the given file https://docs.espressif.com/projects/esptool/en/latest/esp32/advanced-topics/firmware-image-format.html - ''' + """ data = b'' with open(file_path, 'rb') as f: data = f.read(4) @@ -25,18 +28,26 @@ def test_bootloader_custom_overrides_original(test_app_copy: Path, idf_py: IdfPy idf_path = Path(default_idf_env.get('IDF_PATH')) shutil.copytree(idf_path / 'components' / 'bootloader', test_app_copy / 'components' / 'bootloader') # Because of relative include of Kconfig, also esp_bootloader_format needs to be copied. - shutil.copytree(idf_path / 'components' / 'esp_bootloader_format', test_app_copy / 'components' / 'esp_bootloader_format') + shutil.copytree( + idf_path / 'components' / 'esp_bootloader_format', test_app_copy / 'components' / 'esp_bootloader_format' + ) idf_py('bootloader') - assert file_contains(test_app_copy / 'build' / 'bootloader' / 'compile_commands.json', - (test_app_copy / 'components' / 'bootloader' / 'subproject' / 'main' / 'bootloader_start.c')) + assert file_contains( + test_app_copy / 'build' / 'bootloader' / 'compile_commands.json', + (test_app_copy / 'components' / 'bootloader' / 'subproject' / 'main' / 'bootloader_start.c'), + ) -def test_bootloader_custom_ignores_extra_component(test_app_copy: Path, idf_py: IdfPyFunc, default_idf_env: EnvDict) -> None: +def test_bootloader_custom_ignores_extra_component( + test_app_copy: Path, idf_py: IdfPyFunc, default_idf_env: EnvDict +) -> None: logging.info('Custom bootloader can ignore extra components') idf_path = Path(default_idf_env.get('IDF_PATH')) # Import the main bootloader component that overrides the default one - shutil.copytree(idf_path / 'examples' / 'custom_bootloader' / 'bootloader_override' / 'bootloader_components', - test_app_copy / 'bootloader_components') + shutil.copytree( + idf_path / 'examples' / 'custom_bootloader' / 'bootloader_override' / 'bootloader_components', + test_app_copy / 'bootloader_components', + ) # Compile it normally and make sure the bootloader is overridden for now idf_py('bootloader') # Search for 'Custom bootloader message defined in the KConfig file.' in the resulting binary @@ -46,9 +57,11 @@ def test_bootloader_custom_ignores_extra_component(test_app_copy: Path, idf_py: # Clean the project idf_py('fullclean') # Ignore bootloader component and rebuild - replace_in_file(test_app_copy / 'CMakeLists.txt', - '# placeholder_after_include_project_cmake', - 'set(BOOTLOADER_IGNORE_EXTRA_COMPONENT "main")') + replace_in_file( + test_app_copy / 'CMakeLists.txt', + '# placeholder_after_include_project_cmake', + 'set(BOOTLOADER_IGNORE_EXTRA_COMPONENT "main")', + ) idf_py('bootloader') # The overridden message must not be present anymore assert not bin_file_contains(test_app_copy / 'build' / 'bootloader' / 'bootloader.bin', default_message) diff --git a/tools/test_build_system/test_build_system_helpers/file_utils.py b/tools/test_build_system/test_build_system_helpers/file_utils.py index e79a8b62f48..e5278fa2050 100644 --- a/tools/test_build_system/test_build_system_helpers/file_utils.py +++ b/tools/test_build_system/test_build_system_helpers/file_utils.py @@ -6,26 +6,26 @@ from pathlib import Path from pathlib import WindowsPath -def append_to_file(filename: t.Union[str, Path], what: str) -> None: +def append_to_file(filename: str | Path, what: str) -> None: with open(filename, 'a', encoding='utf-8') as f: f.write(what) -def replace_in_file(filename: t.Union[str, Path], search: str, replace: str) -> None: - with open(filename, 'r', encoding='utf-8') as f: +def replace_in_file(filename: str | Path, search: str, replace: str) -> None: + with open(filename, encoding='utf-8') as f: data = f.read() result = data.replace(search, replace) with open(filename, 'w', encoding='utf-8') as f: f.write(result) -def file_contains(filename: t.Union[str, Path], what: t.Union[t.Union[str, Path], t.Pattern]) -> bool: +def file_contains(filename: str | Path, what: str | Path | t.Pattern) -> bool: """ Returns true if file contains required object :param filename: path to file where lookup is executed :param what: searched substring or regex object """ - with open(filename, 'r', encoding='utf-8') as f: + with open(filename, encoding='utf-8') as f: data = f.read() if isinstance(what, t.Pattern): return re.search(what, data) is not None @@ -39,7 +39,7 @@ def file_contains(filename: t.Union[str, Path], what: t.Union[t.Union[str, Path] return what_str in data -def bin_file_contains(filename: t.Union[str, Path], what: bytearray) -> bool: +def bin_file_contains(filename: str | Path, what: bytearray) -> bool: """ Returns true if the binary file contains the given string :param filename: path to file where lookup is executed @@ -50,7 +50,7 @@ def bin_file_contains(filename: t.Union[str, Path], what: bytearray) -> bool: return data.find(what) != -1 -def bin_files_differ(filename1: t.Union[str, Path], filename2: t.Union[str, Path]) -> bool: +def bin_files_differ(filename1: str | Path, filename2: str | Path) -> bool: """ Checks if two binary files are different :param filename1: path to first file diff --git a/tools/test_build_system/test_build_system_helpers/snapshot.py b/tools/test_build_system/test_build_system_helpers/snapshot.py index 79ee0ffa700..c2c8f51688f 100644 --- a/tools/test_build_system/test_build_system_helpers/snapshot.py +++ b/tools/test_build_system/test_build_system_helpers/snapshot.py @@ -3,7 +3,6 @@ import fnmatch import glob import os -from typing import List, Optional, Tuple, Union class Snapshot: @@ -11,9 +10,10 @@ class Snapshot: Helper class for working with build snapshots. A snapshot is a set of files along with their modification timestamps. """ - def __init__(self, files: List[str]) -> None: + + def __init__(self, files: list[str]) -> None: """Create a snapshot of the given list of files, recording their mtimes""" - self.info: List[Tuple[str, int]] = [] + self.info: list[tuple[str, int]] = [] for file in sorted(files): st = os.stat(file) self.info.append((file, st.st_mtime_ns)) @@ -22,7 +22,9 @@ class Snapshot: """Assert that this snapshot has the same files and mtimes as the other snapshot""" for my_info, other_info in zip(self.info, other.info): assert my_info[0] == other_info[0], f'Snapshots mismatch, expected file {my_info[0]}, found {other_info[0]}' - assert my_info[1] == other_info[1], f'File {my_info[0]} timestamp has changed! diff={(other_info[1]-my_info[1])/10**9}' + assert my_info[1] == other_info[1], ( + f'File {my_info[0]} timestamp has changed! diff={(other_info[1] - my_info[1]) / 10**9}' + ) def assert_different(self, other: 'Snapshot') -> None: """Assert that this snapshot has the same files as the other snapshot, but all mtimes are different""" @@ -31,9 +33,10 @@ class Snapshot: assert my_info[1] != other_info[1], f'File {my_info[0]} timestamp has not changed! ({my_info[1]})' -def get_snapshot(glob_patterns: Union[str, List[str]], - exclude_patterns: Optional[Union[str, List[str]]] = None) -> Snapshot: - """Return a snapshot including the files matched by glob_patterns, and excluding those matched by exclude_patterns""" +def get_snapshot(glob_patterns: str | list[str], exclude_patterns: str | list[str] | None = None) -> Snapshot: + """ + Return a snapshot including the files matched by glob_patterns, and excluding those matched by exclude_patterns + """ if isinstance(glob_patterns, str): glob_patterns = [glob_patterns] @@ -48,13 +51,11 @@ def get_snapshot(glob_patterns: Union[str, List[str]], # whether the path found by glob.glob should be excluded? def should_exclude(filename: str) -> bool: - return os.path.isdir(filename) or \ - any((fnmatch.fnmatch(filename, pattern) for pattern in exclude_patterns_list)) + return os.path.isdir(filename) or any(fnmatch.fnmatch(filename, pattern) for pattern in exclude_patterns_list) - files_to_snapshot: List[str] = [] + files_to_snapshot: list[str] = [] for pattern in glob_patterns: - found = filter(lambda f: not should_exclude(f), - glob.glob(pattern, recursive=True)) + found = filter(lambda f: not should_exclude(f), glob.glob(pattern, recursive=True)) if not found: raise RuntimeError(f'failed to match any files with pattern {pattern}') files_to_snapshot.extend(found) diff --git a/tools/test_build_system/test_kconfig.py b/tools/test_build_system/test_kconfig.py index 5ed0f33f0ce..821bbfecef7 100644 --- a/tools/test_build_system/test_kconfig.py +++ b/tools/test_build_system/test_kconfig.py @@ -4,14 +4,14 @@ import logging import os import shutil import textwrap +from collections.abc import Iterator from contextlib import contextmanager from pathlib import Path -from typing import Iterator import pytest +from test_build_system_helpers import IdfPyFunc from test_build_system_helpers import append_to_file from test_build_system_helpers import file_contains -from test_build_system_helpers import IdfPyFunc @contextmanager @@ -43,7 +43,9 @@ def test_kconfig_deprecated_options(idf_py: IdfPyFunc, test_app_copy: Path) -> N idf_py('reconfigure') append_to_file((test_app_copy / 'sdkconfig'), 'CONFIG_TEST_OLD_OPTION=y') (idf_path / 'sdkconfig.rename').write_text('CONFIG_TEST_OLD_OPTION CONFIG_TEST_NEW_OPTION') - append_to_file((idf_path / 'Kconfig'), textwrap.dedent(""" + append_to_file( + (idf_path / 'Kconfig'), + textwrap.dedent(""" menu "test" config TEST_NEW_OPTION bool "test" @@ -51,19 +53,34 @@ def test_kconfig_deprecated_options(idf_py: IdfPyFunc, test_app_copy: Path) -> N help TEST_NEW_OPTION description endmenu - """)) + """), + ) idf_py('reconfigure') - assert all([file_contains((test_app_copy / 'sdkconfig'), x) for x in ['CONFIG_TEST_OLD_OPTION=y', - 'CONFIG_TEST_NEW_OPTION=y']]) - assert all([file_contains((test_app_copy / 'build' / 'config' / 'sdkconfig.h'), x) for x in ['#define CONFIG_TEST_NEW_OPTION 1', - '#define CONFIG_TEST_OLD_OPTION CONFIG_TEST_NEW_OPTION']]) - assert all([file_contains((test_app_copy / 'build' / 'config' / 'sdkconfig.cmake'), x) for x in ['set(CONFIG_TEST_OLD_OPTION "y")', - 'set(CONFIG_TEST_NEW_OPTION "y")']]) + assert all( + [ + file_contains((test_app_copy / 'sdkconfig'), x) + for x in ['CONFIG_TEST_OLD_OPTION=y', 'CONFIG_TEST_NEW_OPTION=y'] + ] + ) + assert all( + [ + file_contains((test_app_copy / 'build' / 'config' / 'sdkconfig.h'), x) + for x in ['#define CONFIG_TEST_NEW_OPTION 1', '#define CONFIG_TEST_OLD_OPTION CONFIG_TEST_NEW_OPTION'] + ] + ) + assert all( + [ + file_contains((test_app_copy / 'build' / 'config' / 'sdkconfig.cmake'), x) + for x in ['set(CONFIG_TEST_OLD_OPTION "y")', 'set(CONFIG_TEST_NEW_OPTION "y")'] + ] + ) logging.info('Handling deprecated Kconfig options in sdkconfig.defaults') (test_app_copy / 'sdkconfig.defaults').write_text('CONFIG_TEST_OLD_OPTION=7') (idf_path / 'sdkconfig.rename').write_text('CONFIG_TEST_OLD_OPTION CONFIG_TEST_NEW_OPTION') - append_to_file((idf_path / 'Kconfig'), textwrap.dedent(""" + append_to_file( + (idf_path / 'Kconfig'), + textwrap.dedent(""" menu "test" config TEST_NEW_OPTION int "TEST_NEW_OPTION" @@ -72,10 +89,15 @@ def test_kconfig_deprecated_options(idf_py: IdfPyFunc, test_app_copy: Path) -> N help TEST_NEW_OPTION description endmenu - """)) + """), + ) idf_py('reconfigure') - assert all([file_contains((test_app_copy / 'sdkconfig'), x) for x in ['CONFIG_TEST_OLD_OPTION=7', - 'CONFIG_TEST_NEW_OPTION=7']]) + assert all( + [ + file_contains((test_app_copy / 'sdkconfig'), x) + for x in ['CONFIG_TEST_OLD_OPTION=7', 'CONFIG_TEST_NEW_OPTION=7'] + ] + ) @pytest.mark.usefixtures('idf_copy') @@ -88,9 +110,15 @@ def test_kconfig_multiple_and_target_specific_options(idf_py: IdfPyFunc, test_ap (idf_path / 'sdkconfig.rename').write_text('') idf_py('reconfigure') append_to_file((test_app_copy / 'sdkconfig'), 'CONFIG_TEST_NEW_OPTION=y') - append_to_file((idf_path / 'sdkconfig.rename'), '\n'.join(['CONFIG_TEST_OLD_OPTION_1 CONFIG_TEST_NEW_OPTION', - 'CONFIG_TEST_OLD_OPTION_2 CONFIG_TEST_NEW_OPTION'])) - append_to_file((idf_path / 'Kconfig'), textwrap.dedent(""" + append_to_file( + (idf_path / 'sdkconfig.rename'), + '\n'.join( + ['CONFIG_TEST_OLD_OPTION_1 CONFIG_TEST_NEW_OPTION', 'CONFIG_TEST_OLD_OPTION_2 CONFIG_TEST_NEW_OPTION'] + ), + ) + append_to_file( + (idf_path / 'Kconfig'), + textwrap.dedent(""" menu "test" config TEST_NEW_OPTION bool "test" @@ -98,20 +126,40 @@ def test_kconfig_multiple_and_target_specific_options(idf_py: IdfPyFunc, test_ap help TEST_NEW_OPTION description endmenu - """)) + """), + ) idf_py('reconfigure') - assert all([file_contains((test_app_copy / 'sdkconfig'), x) for x in ['CONFIG_TEST_OLD_OPTION_1=y', - 'CONFIG_TEST_OLD_OPTION_2=y']]) - assert all([file_contains((test_app_copy / 'build' / 'config' / 'sdkconfig.h'), x) for x in ['#define CONFIG_TEST_OLD_OPTION_1 CONFIG_TEST_NEW_OPTION', - '#define CONFIG_TEST_OLD_OPTION_2 CONFIG_TEST_NEW_OPTION' - ]]) - assert all([file_contains((test_app_copy / 'build' / 'config' / 'sdkconfig.cmake'), x) for x in ['set(CONFIG_TEST_OLD_OPTION_1 "y")', - 'set(CONFIG_TEST_OLD_OPTION_2 "y")']]) + assert all( + [ + file_contains((test_app_copy / 'sdkconfig'), x) + for x in ['CONFIG_TEST_OLD_OPTION_1=y', 'CONFIG_TEST_OLD_OPTION_2=y'] + ] + ) + assert all( + [ + file_contains((test_app_copy / 'build' / 'config' / 'sdkconfig.h'), x) + for x in [ + '#define CONFIG_TEST_OLD_OPTION_1 CONFIG_TEST_NEW_OPTION', + '#define CONFIG_TEST_OLD_OPTION_2 CONFIG_TEST_NEW_OPTION', + ] + ] + ) + assert all( + [ + file_contains((test_app_copy / 'build' / 'config' / 'sdkconfig.cmake'), x) + for x in ['set(CONFIG_TEST_OLD_OPTION_1 "y")', 'set(CONFIG_TEST_OLD_OPTION_2 "y")'] + ] + ) logging.info('Can have target specific deprecated Kconfig options') (test_app_copy / 'sdkconfig').write_text('CONFIG_TEST_OLD_OPTION=y') - append_to_file((idf_path / 'components' / 'esp_system' / 'sdkconfig.rename.esp32s2'), 'CONFIG_TEST_OLD_OPTION CONFIG_TEST_NEW_OPTION') - append_to_file((idf_path / 'Kconfig'), textwrap.dedent(""" + append_to_file( + (idf_path / 'components' / 'esp_system' / 'sdkconfig.rename.esp32s2'), + 'CONFIG_TEST_OLD_OPTION CONFIG_TEST_NEW_OPTION', + ) + append_to_file( + (idf_path / 'Kconfig'), + textwrap.dedent(""" menu "test" config TEST_NEW_OPTION bool "TEST_NEW_OPTION" @@ -119,11 +167,16 @@ def test_kconfig_multiple_and_target_specific_options(idf_py: IdfPyFunc, test_ap help TEST_NEW_OPTION description endmenu - """)) + """), + ) idf_py('set-target', 'esp32') assert not file_contains((test_app_copy / 'sdkconfig'), 'CONFIG_TEST_OLD_OPTION=y') assert file_contains((test_app_copy / 'sdkconfig'), 'CONFIG_TEST_NEW_OPTION=y') (test_app_copy / 'sdkconfig').unlink() idf_py('set-target', 'esp32s2') - assert all([file_contains((test_app_copy / 'sdkconfig'), x) for x in ['CONFIG_TEST_NEW_OPTION=y', - 'CONFIG_TEST_OLD_OPTION=y']]) + assert all( + [ + file_contains((test_app_copy / 'sdkconfig'), x) + for x in ['CONFIG_TEST_NEW_OPTION=y', 'CONFIG_TEST_OLD_OPTION=y'] + ] + ) diff --git a/tools/test_build_system/test_sdkconfig.py b/tools/test_build_system/test_sdkconfig.py index f84502575c4..f4197c2fbe9 100644 --- a/tools/test_build_system/test_sdkconfig.py +++ b/tools/test_build_system/test_sdkconfig.py @@ -4,13 +4,13 @@ import logging import os import re from pathlib import Path -from typing import Optional import pytest -from test_build_system_helpers import IdfPyFunc, file_contains +from test_build_system_helpers import IdfPyFunc +from test_build_system_helpers import file_contains -def idf_version_from_cmake() -> Optional[str]: +def idf_version_from_cmake() -> str | None: version_path = os.path.join(os.environ['IDF_PATH'], 'tools/cmake/version.cmake') regex = re.compile(r'^\s*set\s*\(\s*IDF_VERSION_([A-Z]{5})\s+(\d+)') ver = {} @@ -22,21 +22,30 @@ def idf_version_from_cmake() -> Optional[str]: if m: ver[m.group(1)] = m.group(2) - return '%s.%s.%s' % (ver['MAJOR'], ver['MINOR'], ver['PATCH']) + return '{}.{}.{}'.format(ver['MAJOR'], ver['MINOR'], ver['PATCH']) except (KeyError, OSError): pytest.fail('Cannot find ESP-IDF version in tools/cmake/version.cmake') return None def test_sdkconfig_contains_all_files(idf_py: IdfPyFunc, test_app_copy: Path) -> None: - logging.info('sdkconfig should have contents of all files: sdkconfig, sdkconfig.defaults, sdkconfig.defaults.IDF_TARGET') + logging.info( + 'sdkconfig should have contents of all files: sdkconfig, sdkconfig.defaults, sdkconfig.defaults.IDF_TARGET' + ) (test_app_copy / 'sdkconfig').write_text('CONFIG_PARTITION_TABLE_TWO_OTA=y') (test_app_copy / 'sdkconfig.defaults').write_text('CONFIG_PARTITION_TABLE_OFFSET=0x10000') (test_app_copy / 'sdkconfig.defaults.esp32').write_text('CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y') idf_py('reconfigure') - assert all([file_contains((test_app_copy / 'sdkconfig'), x) for x in ['CONFIG_PARTITION_TABLE_TWO_OTA=y', - 'CONFIG_PARTITION_TABLE_OFFSET=0x10000', - 'CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y']]) + assert all( + [ + file_contains((test_app_copy / 'sdkconfig'), x) + for x in [ + 'CONFIG_PARTITION_TABLE_TWO_OTA=y', + 'CONFIG_PARTITION_TABLE_OFFSET=0x10000', + 'CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y', + ] + ] + ) def test_sdkconfig_multiple_default_files(idf_py: IdfPyFunc, test_app_copy: Path) -> None: @@ -44,8 +53,12 @@ def test_sdkconfig_multiple_default_files(idf_py: IdfPyFunc, test_app_copy: Path (test_app_copy / 'sdkconfig.defaults1').write_text('CONFIG_PARTITION_TABLE_OFFSET=0x10000') (test_app_copy / 'sdkconfig.defaults2').write_text('CONFIG_PARTITION_TABLE_TWO_OTA=y') idf_py('-DSDKCONFIG_DEFAULTS=sdkconfig.defaults1;sdkconfig.defaults2', 'reconfigure') - assert all([file_contains((test_app_copy / 'sdkconfig'), x) for x in ['CONFIG_PARTITION_TABLE_TWO_OTA=y', - 'CONFIG_PARTITION_TABLE_OFFSET=0x10000']]) + assert all( + [ + file_contains((test_app_copy / 'sdkconfig'), x) + for x in ['CONFIG_PARTITION_TABLE_TWO_OTA=y', 'CONFIG_PARTITION_TABLE_OFFSET=0x10000'] + ] + ) def test_keep_idf_init_version(idf_py: IdfPyFunc, test_app_copy: Path) -> None: diff --git a/tools/test_idf_diag/test_idf_diag.py b/tools/test_idf_diag/test_idf_diag.py index a67a527d9a8..0e4c4803d21 100644 --- a/tools/test_idf_diag/test_idf_diag.py +++ b/tools/test_idf_diag/test_idf_diag.py @@ -7,30 +7,26 @@ import sys from pathlib import Path from shutil import copytree from subprocess import PIPE -from subprocess import run from subprocess import STDOUT +from subprocess import run from tempfile import TemporaryDirectory from typing import Any -from typing import Optional -from typing import Tuple -from typing import Union - IDF_PATH = Path(os.environ['IDF_PATH']) IDF_PY_PATH = IDF_PATH / 'tools' / 'idf.py' IDF_DIAG_PY_PATH = IDF_PATH / 'tools' / 'idf_diag.py' HELLO_WORLD_PATH = IDF_PATH / 'examples' / 'get-started' / 'hello_world' -PathLike = Union[str, Path] +PathLike = str | Path -def run_cmd(*cmd: PathLike, cwd: Optional[PathLike]=None, check: bool=True, text: bool=True) -> Tuple[int, str]: +def run_cmd(*cmd: PathLike, cwd: PathLike | None = None, check: bool = True, text: bool = True) -> tuple[int, str]: logging.info('running: {}'.format(' '.join([str(arg) for arg in cmd]))) p = run(cmd, stdout=PIPE, stderr=STDOUT, cwd=cwd, check=check, text=text) return p.returncode, p.stdout -def run_idf_py(*args: PathLike, **kwargs: Any) -> Tuple[int, str]: +def run_idf_py(*args: PathLike, **kwargs: Any) -> tuple[int, str]: return run_cmd(sys.executable, IDF_PY_PATH, *args, **kwargs) @@ -70,6 +66,6 @@ def test_idf_diag() -> None: idf_component_path.write_text('https://username:password@github.com/username/repository.git') run_idf_py('diag', '--force', '--output', report_path, cwd=app_path) idf_component_path.unlink() - with open(report_path / 'manager' / 'idf_component' / 'idf_component.yml', 'r') as f: + with open(report_path / 'manager' / 'idf_component' / 'idf_component.yml') as f: data = f.read() assert 'https://[XXX]@github.com/username/repository.git' in data diff --git a/tools/test_idf_py/test_idf_extensions/test_ext/test_extension.py b/tools/test_idf_py/test_idf_extensions/test_ext/test_extension.py index e5c99bf0945..a51d4d0934d 100644 --- a/tools/test_idf_py/test_idf_extensions/test_ext/test_extension.py +++ b/tools/test_idf_py/test_idf_extensions/test_ext/test_extension.py @@ -8,23 +8,22 @@ def action_extensions(base_actions, project_path=os.getcwd()): print('!!! From test_subcommand') def test_global_callback(ctx, global_args, tasks): - print('!!! From test global callback: %s' % global_args.test_extension_option) + print(f'!!! From test global callback: {global_args.test_extension_option}') return { - 'global_options': [{ - 'names': ['--test-extension-option'], - 'help': 'Help for option --test-extension-option', - 'default': 'test', - }], + 'global_options': [ + { + 'names': ['--test-extension-option'], + 'help': 'Help for option --test-extension-option', + 'default': 'test', + } + ], 'global_action_callbacks': [test_global_callback], 'actions': { 'test_subcommand': { 'callback': test_callback, 'help': 'Help for test subcommand.', }, - 'hidden_one': { - 'callback': test_callback, - 'hidden': True - } - } + 'hidden_one': {'callback': test_callback, 'hidden': True}, + }, } diff --git a/tools/test_idf_size/test_idf_size.py b/tools/test_idf_size/test_idf_size.py index 3d88641c2d2..9dc0c219d88 100644 --- a/tools/test_idf_size/test_idf_size.py +++ b/tools/test_idf_size/test_idf_size.py @@ -7,34 +7,30 @@ import sys from pathlib import Path from shutil import copytree from subprocess import PIPE -from subprocess import run from subprocess import STDOUT +from subprocess import run from tempfile import TemporaryDirectory from typing import Any -from typing import Optional -from typing import Tuple -from typing import Union - IDF_PATH = Path(os.environ['IDF_PATH']) IDF_PY_PATH = IDF_PATH / 'tools' / 'idf.py' IDF_SIZE_PY_PATH = IDF_PATH / 'tools' / 'idf_size.py' HELLO_WORLD_PATH = IDF_PATH / 'examples' / 'get-started' / 'hello_world' -PathLike = Union[str, Path] +PathLike = str | Path -def run_cmd(*cmd: PathLike, cwd: Optional[PathLike]=None, check: bool=True, text: bool=True) -> Tuple[int, str]: +def run_cmd(*cmd: PathLike, cwd: PathLike | None = None, check: bool = True, text: bool = True) -> tuple[int, str]: logging.info('running: {}'.format(' '.join([str(arg) for arg in cmd]))) p = run(cmd, stdout=PIPE, stderr=STDOUT, cwd=cwd, check=check, text=text) return p.returncode, p.stdout -def run_idf_py(*args: PathLike, **kwargs: Any) -> Tuple[int, str]: +def run_idf_py(*args: PathLike, **kwargs: Any) -> tuple[int, str]: return run_cmd(sys.executable, IDF_PY_PATH, *args, **kwargs) -def run_idf_size_py(*args: PathLike, **kwargs: Any) -> Tuple[int, str]: +def run_idf_size_py(*args: PathLike, **kwargs: Any) -> tuple[int, str]: return run_cmd(sys.executable, IDF_SIZE_PY_PATH, *args, **kwargs)