From 11a0dfcc7c0a5e91a33aa56ea0f33a03c902e38d Mon Sep 17 00:00:00 2001 From: Kapil Gupta Date: Fri, 24 Apr 2026 15:16:32 +0530 Subject: [PATCH] ci(esp_remote): Add wifi-remote pre-commit hook --- .pre-commit-config.yaml | 15 ++ .../remote/scripts/generate_and_check.py | 248 ++++++++++++------ 2 files changed, 177 insertions(+), 86 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7cbc826751c..b74191dfbb7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -174,6 +174,21 @@ repos: pass_filenames: false additional_dependencies: - PyYAML == 5.3.1 + - id: check-wifi-remote-api + name: Check wifi-remote API generation + description: Runs generate_and_check.py and fails if any generated files differ from the index. + entry: bash -c "cd components/esp_wifi/remote/scripts/ && python3 generate_and_check.py" + language: system + files: > + (?x)^( + components/esp_wifi/remote/scripts/(ignore_extensions\.h|copyright_header\.h)| + components/esp_wifi/include/(esp_wifi.*|esp_mesh.*|esp_now.*)\.h| + components/esp_wifi/Kconfig| + components/wpa_supplicant/esp_supplicant/include/esp_eap_client\.h| + components/soc/.+/include/soc/Kconfig\.soc_caps\.in + )$ + pass_filenames: false + verbose: true - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.0.1 hooks: diff --git a/components/esp_wifi/remote/scripts/generate_and_check.py b/components/esp_wifi/remote/scripts/generate_and_check.py index a4be4f97019..4e02e714f16 100644 --- a/components/esp_wifi/remote/scripts/generate_and_check.py +++ b/components/esp_wifi/remote/scripts/generate_and_check.py @@ -13,12 +13,6 @@ from typing import IO from typing import Any from typing import cast -from idf_build_apps.constants import PREVIEW_TARGETS -from idf_build_apps.constants import SUPPORTED_TARGETS -from pycparser import c_ast -from pycparser import c_parser -from pycparser import preprocess_file - script_dir = os.path.dirname(os.path.abspath(__file__)) current_dir = os.path.abspath(os.getcwd()) if script_dir != current_dir: @@ -26,6 +20,10 @@ if script_dir != current_dir: print(f'Current working directory is: {current_dir}') sys.exit(1) +idf_path = os.getenv('IDF_PATH') +if idf_path is None: + idf_path = os.path.realpath(os.path.join(script_dir, '..', '..', '..', '..')) + Param = namedtuple('Param', ['ptr', 'array', 'qual', 'type', 'name']) component_path = os.path.normpath(os.path.join(os.path.realpath(__file__), '..', '..')) @@ -38,61 +36,63 @@ wifi_configs = [] FunctionPrototypes = dict[str, tuple[str, list[Any]]] -class FunctionVisitor(c_ast.NodeVisitor): - def __init__(self, header: str, prefixes: str | list[str] | tuple[str, ...]) -> None: - self.function_prototypes: FunctionPrototypes = {} - self.ptr = 0 - self.array = 0 - self.content = open(header).read() - self.prefixes = prefixes if isinstance(prefixes, list | tuple) else [prefixes] - - def get_type(self, node: Any, suffix: str = 'param') -> tuple[str, str, Any]: - if suffix == 'param': - self.ptr = 0 - self.array = 0 - - if isinstance(node.type, c_ast.TypeDecl): - typename = node.type.declname - quals = '' - if node.type.quals: - quals = ' '.join(node.type.quals) - if node.type.type.names: - c_type = ' '.join(node.type.type.names) - return quals, c_type, typename - if isinstance(node.type, c_ast.PtrDecl): - quals, c_type, name = self.get_type(node.type, 'ptr') - self.ptr += 1 - return quals, c_type, name - - if isinstance(node.type, c_ast.ArrayDecl): - quals, c_type, name = self.get_type(node.type, 'array') - self.array = int(node.type.dim.value) - return quals, c_type, name - return '', '', None - - def visit_FuncDecl(self, node: c_ast.FuncDecl) -> None: - if isinstance(node.type, c_ast.TypeDecl): - func_name = node.type.declname - if ( - any(func_name.startswith(prefix) for prefix in self.prefixes) - and not func_name.endswith('_t') - and func_name in self.content - ): - if func_name in DEPRECATED_API: - return - ret = node.type.type.names[0] - args = [] - for param in node.args.params: - quals, c_type, name = self.get_type(param) - param = Param(ptr=self.ptr, array=self.array, qual=quals, type=c_type, name=name) - args.append(param) - self.function_prototypes[func_name] = (ret, args) - - # Parse the header file and extract function prototypes def extract_function_prototypes( header_code: str, header: str, prefixes: str | list[str] | tuple[str, ...] ) -> FunctionPrototypes: + from pycparser import c_ast + from pycparser import c_parser + + class FunctionVisitor(c_ast.NodeVisitor): + def __init__(self, header: str, prefixes: str | list[str] | tuple[str, ...]) -> None: + self.function_prototypes: FunctionPrototypes = {} + self.ptr = 0 + self.array = 0 + self.content = open(header).read() + self.prefixes = prefixes if isinstance(prefixes, list | tuple) else [prefixes] + + def get_type(self, node: Any, suffix: str = 'param') -> tuple[str, str, Any]: + if suffix == 'param': + self.ptr = 0 + self.array = 0 + + if isinstance(node.type, c_ast.TypeDecl): + typename = node.type.declname + quals = '' + if node.type.quals: + quals = ' '.join(node.type.quals) + if node.type.type.names: + c_type = ' '.join(node.type.type.names) + return quals, c_type, typename + if isinstance(node.type, c_ast.PtrDecl): + quals, c_type, name = self.get_type(node.type, 'ptr') + self.ptr += 1 + return quals, c_type, name + + if isinstance(node.type, c_ast.ArrayDecl): + quals, c_type, name = self.get_type(node.type, 'array') + self.array = int(node.type.dim.value) + return quals, c_type, name + return '', '', None + + def visit_FuncDecl(self, node: c_ast.FuncDecl) -> None: + if isinstance(node.type, c_ast.TypeDecl): + func_name = node.type.declname + if ( + any(func_name.startswith(prefix) for prefix in self.prefixes) + and not func_name.endswith('_t') + and func_name in self.content + ): + if func_name in DEPRECATED_API: + return + ret = node.type.type.names[0] + args = [] + for param in node.args.params: + quals, c_type, name = self.get_type(param) + param = Param(ptr=self.ptr, array=self.array, qual=quals, type=c_type, name=name) + args.append(param) + self.function_prototypes[func_name] = (ret, args) + parser = c_parser.CParser() # Set debug parameter to False ast = parser.parse(header_code) visitor = FunctionVisitor(header, prefixes) @@ -114,34 +114,91 @@ def exec_cmd(what: list[str], out_file: IO[str] | None = None) -> tuple[int, str return rc, output, err, ' '.join(what) +_cached_include_dir_flags: list[str] | None = None + + +def _handle_missing_tool(msg: str) -> None: + YELLOW = '\033[33m' + RESET = '\033[0m' + full_msg = ( + f'{msg}. WiFi-remote API generation check could not be performed.\n' + 'If you have modified WiFi or Supplicant headers, the corresponding ' + 'wifi-remote files may need to be updated.\n' + 'Please set up your ESP-IDF environment (run export.sh) and run this script manually:\n' + f'cd {os.path.relpath(script_dir, idf_path)} && python3 {os.path.basename(__file__)}\n' + 'Then commit the resulting changes.' + ) + print(f'{YELLOW}SKIPPED: {full_msg}{RESET}') + sys.exit(0) + + +try: + import idf_build_apps # noqa: F401 + import pycparser # noqa: F401 +except ImportError: + _handle_missing_tool('ESP-IDF environment not found (missing python dependencies)') + + def preprocess(idf_path: str, header: str) -> str: + global _cached_include_dir_flags project_dir = os.path.join(idf_path, 'examples', 'wifi', 'getting_started', 'station') build_dir = os.path.join(project_dir, 'build') - # Clean up build artifacts - if os.path.exists(build_dir): - shutil.rmtree(build_dir) - sdkconfig = os.path.join(project_dir, 'sdkconfig') - if os.path.exists(sdkconfig): - os.remove(sdkconfig) + if _cached_include_dir_flags is None: + # Clean up build artifacts ONLY on the first run to ensure a fresh state if needed, + # but idf.py reconfigure is usually good at incremental updates. + # To be safe and fast, we only clean if we don't have a build dir yet. + if not os.path.exists(build_dir): + sdkconfig = os.path.join(project_dir, 'sdkconfig') + if os.path.exists(sdkconfig): + os.remove(sdkconfig) + + idf_py = shutil.which('idf.py') + if idf_py is None: + idf_py_path = os.path.join(idf_path, 'tools', 'idf.py') + if os.path.exists(idf_py_path): + idf_py = f'{sys.executable} {idf_py_path}' + else: + _handle_missing_tool('ESP-IDF environment not found') + + assert idf_py is not None + try: + if isinstance(idf_py, str) and idf_py.startswith(sys.executable): + subprocess.run( + idf_py.split() + ['-B', build_dir, 'reconfigure'], + cwd=project_dir, + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + else: + subprocess.run( + [idf_py, '-B', build_dir, 'reconfigure'], + cwd=project_dir, + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + except (subprocess.CalledProcessError, FileNotFoundError): + _handle_missing_tool('ESP-IDF environment not found') + + build_commands_json = os.path.join(build_dir, 'compile_commands.json') + if not os.path.exists(build_commands_json): + _handle_missing_tool('ESP-IDF environment not found') + + with open(build_commands_json, encoding='utf-8') as f: + build_command = json.load(f)[0]['command'].split() + _cached_include_dir_flags = [] + # process compilation flags (includes and defines) + for item in build_command: + if item.startswith('-I'): + _cached_include_dir_flags.append(item) + if item.startswith('-D'): + _cached_include_dir_flags.append( + item.replace('\\', '') + ) # removes escaped quotes, eg: -DMBEDTLS_CONFIG_FILE=\\\"mbedtls/esp_config.h\\\" + _cached_include_dir_flags.append('-I' + os.path.join(build_dir, 'config')) - subprocess.check_call(['idf.py', '-B', build_dir, 'reconfigure'], cwd=project_dir) - build_commands_json = os.path.join(build_dir, 'compile_commands.json') - with open(build_commands_json, encoding='utf-8') as f: - build_command = json.load(f)[0]['command'].split() - include_dir_flags = [] - include_dirs = [] - # process compilation flags (includes and defines) - for item in build_command: - if item.startswith('-I'): - include_dir_flags.append(item) - if 'components' in item: - include_dirs.append(item[2:]) # Removing the leading "-I" - if item.startswith('-D'): - include_dir_flags.append( - item.replace('\\', '') - ) # removes escaped quotes, eg: -DMBEDTLS_CONFIG_FILE=\\\"mbedtls/esp_config.h\\\" - include_dir_flags.append('-I' + os.path.join(build_dir, 'config')) temp_file = 'esp_wifi_preprocessed.h' with open(temp_file, 'w') as f: f.write('#define asm\n') @@ -149,12 +206,18 @@ def preprocess(idf_path: str, header: str) -> str: f.write('#define __asm__\n') f.write('#define __volatile__\n') with open(temp_file, 'a') as f: + gcc_cmd = 'xtensa-esp32-elf-gcc' + if shutil.which(gcc_cmd) is None: + _handle_missing_tool(f'{gcc_cmd} not found') + rc, out, err, cmd = exec_cmd( - ['xtensa-esp32-elf-gcc', '-w', '-P', '-include', 'ignore_extensions.h', '-E', header] + include_dir_flags, f + [gcc_cmd, '-w', '-P', '-include', 'ignore_extensions.h', '-E', header] + _cached_include_dir_flags, f ) if rc != 0: print(f'command {cmd} failed!') print(err) + from pycparser import preprocess_file + preprocessed_code = preprocess_file(temp_file) return cast(str, preprocessed_code) @@ -205,6 +268,9 @@ def get_vars(parameters: list[Any]) -> tuple[str, str]: def generate_kconfig_wifi_caps(idf_path: str, component_path: str) -> list[str]: + from idf_build_apps.constants import PREVIEW_TARGETS + from idf_build_apps.constants import SUPPORTED_TARGETS + kconfig = os.path.join(component_path, 'Kconfig.soc_wifi_caps.in') slave_select = os.path.join(component_path, 'Kconfig.slave_select.in') @@ -642,9 +708,6 @@ making changes you might need to modify 'copyright_header.h' in the script direc parser.add_argument('--base-dir', help='Base directory to compare generated files against') args = parser.parse_args() - idf_path = os.getenv('IDF_PATH') - if idf_path is None: - raise RuntimeError("Environment variable 'IDF_PATH' wasn't set.") header = os.path.join(idf_path, 'components', 'esp_wifi', 'include', 'esp_wifi.h') eap_header = os.path.join(idf_path, 'components', 'wpa_supplicant', 'esp_supplicant', 'include', 'esp_eap_client.h') function_prototypes = extract_function_prototypes(preprocess(idf_path, header), header, ['esp_wifi_']) @@ -661,11 +724,24 @@ making changes you might need to modify 'copyright_header.h' in the script direc files_to_check += generate_wifi_native(idf_path, component_path) files_to_check += generate_kconfig(idf_path, component_path) + modified_files = [] for f in files_to_check: - print(f) + if os.path.exists(f): + # Check if file is modified relative to index + rc, _, _, _ = exec_cmd(['git', 'diff', '--exit-code', f]) + if rc != 0: + modified_files.append(f) + + if modified_files: + print('WiFi-remote API files were updated:') + for f in modified_files: + print(f' modified: {os.path.relpath(f, idf_path)}') + print('\nPlease stage these changes and try committing again.') if args.skip_check or args.base_dir is None: - exit(0) + if modified_files: + sys.exit(1) + sys.exit(0) failures = compare_files(args.base_dir, component_path, files_to_check)