diff --git a/components/fatfs/fatfs_utils/exceptions.py b/components/fatfs/fatfs_utils/exceptions.py index a3a27df5d6e..6915c35997a 100644 --- a/components/fatfs/fatfs_utils/exceptions.py +++ b/components/fatfs/fatfs_utils/exceptions.py @@ -1,10 +1,25 @@ -# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 +from esp_pylib.errors import FatalError + +__all__ = [ + 'FatalError', + 'InconsistentFATAttributes', + 'LowerCaseException', + 'NoFreeClusterException', + 'NotInitialized', + 'TooLongNameException', + 'WLNotInitialized', + 'WriteDirectoryException', +] + + class WriteDirectoryException(Exception): """ Exception is raised when the user tries to write the content into the directory instead of file """ + pass @@ -12,6 +27,7 @@ class NoFreeClusterException(Exception): """ Exception is raised when the user tries allocate cluster but no free one is available """ + pass @@ -19,6 +35,7 @@ class LowerCaseException(Exception): """ Exception is raised when the user tries to write file or directory with lower case """ + pass @@ -26,6 +43,7 @@ class TooLongNameException(Exception): """ Exception is raised when long name support is not enabled and user tries to write file longer then allowed """ + pass @@ -33,6 +51,7 @@ class NotInitialized(Exception): """ Exception is raised when the user tries to access not initialized property """ + pass @@ -40,10 +59,7 @@ class WLNotInitialized(Exception): """ Exception is raised when the user tries to write fatfs not initialized with wear levelling """ - pass - -class FatalError(Exception): pass @@ -51,4 +67,5 @@ class InconsistentFATAttributes(Exception): """ Caused by e.g. wrong number of clusters for given FAT type """ + pass diff --git a/components/fatfs/fatfs_utils/fatfs_state.py b/components/fatfs/fatfs_utils/fatfs_state.py index 4ba571324a9..4981a24190e 100644 --- a/components/fatfs/fatfs_utils/fatfs_state.py +++ b/components/fatfs/fatfs_utils/fatfs_state.py @@ -1,13 +1,22 @@ -# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 from textwrap import dedent -from typing import Optional + +from esp_pylib.logger import log from .exceptions import InconsistentFATAttributes -from .utils import (ALLOWED_SECTOR_SIZES, FAT12, FAT12_MAX_CLUSTERS, FAT16, FAT16_MAX_CLUSTERS, - RESERVED_CLUSTERS_COUNT, FATDefaults, get_fat_sectors_count, get_fatfs_type, - get_non_data_sectors_cnt, number_of_clusters) +from .utils import ALLOWED_SECTOR_SIZES +from .utils import FAT12 +from .utils import FAT12_MAX_CLUSTERS +from .utils import FAT16 +from .utils import FAT16_MAX_CLUSTERS +from .utils import RESERVED_CLUSTERS_COUNT +from .utils import FATDefaults +from .utils import get_fat_sectors_count +from .utils import get_fatfs_type +from .utils import get_non_data_sectors_cnt +from .utils import number_of_clusters class FATFSState: @@ -15,45 +24,51 @@ class FATFSState: The class represents the state and the configuration of the FATFS. """ - def __init__(self, - sector_size: int, - reserved_sectors_cnt: int, - root_dir_sectors_cnt: int, - size: int, - media_type: int, - sectors_per_cluster: int, - volume_label: str, - oem_name: str, - fat_tables_cnt: int, - sec_per_track: int, - num_heads: int, - hidden_sectors: int, - file_sys_type: str, - use_default_datetime: bool, - explicit_fat_type: Optional[int] = None, - long_names_enabled: bool = False): - self.boot_sector_state = BootSectorState(oem_name=oem_name, - sector_size=sector_size, - sectors_per_cluster=sectors_per_cluster, - reserved_sectors_cnt=reserved_sectors_cnt, - fat_tables_cnt=fat_tables_cnt, - root_dir_sectors_cnt=root_dir_sectors_cnt, - sectors_count=size // sector_size, - media_type=media_type, - sec_per_track=sec_per_track, - num_heads=num_heads, - hidden_sectors=hidden_sectors, - volume_label=volume_label, - file_sys_type=file_sys_type, - volume_uuid=-1) + def __init__( + self, + sector_size: int, + reserved_sectors_cnt: int, + root_dir_sectors_cnt: int, + size: int, + media_type: int, + sectors_per_cluster: int, + volume_label: str, + oem_name: str, + fat_tables_cnt: int, + sec_per_track: int, + num_heads: int, + hidden_sectors: int, + file_sys_type: str, + use_default_datetime: bool, + explicit_fat_type: int | None = None, + long_names_enabled: bool = False, + ): + self.boot_sector_state = BootSectorState( + oem_name=oem_name, + sector_size=sector_size, + sectors_per_cluster=sectors_per_cluster, + reserved_sectors_cnt=reserved_sectors_cnt, + fat_tables_cnt=fat_tables_cnt, + root_dir_sectors_cnt=root_dir_sectors_cnt, + sectors_count=size // sector_size, + media_type=media_type, + sec_per_track=sec_per_track, + num_heads=num_heads, + hidden_sectors=hidden_sectors, + volume_label=volume_label, + file_sys_type=file_sys_type, + volume_uuid=-1, + ) - self._explicit_fat_type: Optional[int] = explicit_fat_type + self._explicit_fat_type: int | None = explicit_fat_type self.long_names_enabled: bool = long_names_enabled self.use_default_datetime: bool = use_default_datetime if (size // sector_size) * sectors_per_cluster in (FAT12_MAX_CLUSTERS, FAT16_MAX_CLUSTERS): - print('WARNING: It is not recommended to create FATFS with bounding ' - f'count of clusters: {FAT12_MAX_CLUSTERS} or {FAT16_MAX_CLUSTERS}') + log.warn( + 'It is not recommended to create FATFS with bounding ' + f'count of clusters: {FAT12_MAX_CLUSTERS} or {FAT16_MAX_CLUSTERS}' + ) self.check_fat_type() @property @@ -67,31 +82,36 @@ class FATFSState: def check_fat_type(self) -> None: _type = self.boot_sector_state.fatfs_type if self._explicit_fat_type is not None and self._explicit_fat_type != _type: - raise InconsistentFATAttributes(dedent( - f"""FAT type you specified is inconsistent with other attributes of the system. + raise InconsistentFATAttributes( + dedent( + f"""FAT type you specified is inconsistent with other attributes of the system. The specified FATFS type: FAT{self._explicit_fat_type} - The actual FATFS type: FAT{_type}""")) + The actual FATFS type: FAT{_type}""" + ) + ) if _type not in (FAT12, FAT16): raise NotImplementedError('FAT32 is currently not supported.') class BootSectorState: # pylint: disable=too-many-instance-attributes - def __init__(self, - oem_name: str, - sector_size: int, - sectors_per_cluster: int, - reserved_sectors_cnt: int, - fat_tables_cnt: int, - root_dir_sectors_cnt: int, - sectors_count: int, - media_type: int, - sec_per_track: int, - num_heads: int, - hidden_sectors: int, - volume_label: str, - file_sys_type: str, - volume_uuid: int = -1) -> None: + def __init__( + self, + oem_name: str, + sector_size: int, + sectors_per_cluster: int, + reserved_sectors_cnt: int, + fat_tables_cnt: int, + root_dir_sectors_cnt: int, + sectors_count: int, + media_type: int, + sec_per_track: int, + num_heads: int, + hidden_sectors: int, + volume_label: str, + file_sys_type: str, + volume_uuid: int = -1, + ) -> None: self.oem_name: str = oem_name self.sector_size: int = sector_size assert self.sector_size in ALLOWED_SECTOR_SIZES @@ -150,10 +170,9 @@ class BootSectorState: @property def non_data_sectors(self) -> int: - non_data_sectors_: int = get_non_data_sectors_cnt(self.reserved_sectors_cnt, - self.sectors_per_fat_cnt, - self.fat_tables_cnt, - self.root_dir_sectors_cnt) + non_data_sectors_: int = get_non_data_sectors_cnt( + self.reserved_sectors_cnt, self.sectors_per_fat_cnt, self.fat_tables_cnt, self.root_dir_sectors_cnt + ) return non_data_sectors_ @property @@ -167,5 +186,7 @@ class BootSectorState: @property def root_directory_start(self) -> int: - root_dir_start: int = (self.reserved_sectors_cnt + self.sectors_per_fat_cnt * self.fat_tables_cnt) * self.sector_size + root_dir_start: int = ( + self.reserved_sectors_cnt + self.sectors_per_fat_cnt * self.fat_tables_cnt + ) * self.sector_size return root_dir_start diff --git a/components/fatfs/fatfs_utils/utils.py b/components/fatfs/fatfs_utils/utils.py index c44d790b1f6..2b9791515f4 100644 --- a/components/fatfs/fatfs_utils/utils.py +++ b/components/fatfs/fatfs_utils/utils.py @@ -1,15 +1,18 @@ # SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 -import argparse import binascii import os import re import uuid +from dataclasses import dataclass from datetime import datetime +from typing import Any +from typing import cast from construct import BitsInteger from construct import BitStruct from construct import Int16ul +from esp_pylib.logger import log # the regex pattern defines symbols that are allowed by long file names but not by short file names INVALID_SFN_CHARS_PATTERN = re.compile(r'[.+,;=\[\]]') @@ -203,77 +206,174 @@ def split_content_into_sectors(content: bytes, sector_size: int) -> list[bytes]: return result -def get_args_for_partition_generator(desc: str, wl: bool) -> argparse.Namespace: - parser: argparse.ArgumentParser = argparse.ArgumentParser(description=desc) - parser.add_argument('input_directory', help='Path to the directory that will be encoded into fatfs image') - parser.add_argument('--output_file', default='fatfs_image.img', help='Filename of the generated fatfs image') - parser.add_argument( +@dataclass +class PartitionGeneratorArgs: + input_directory: str + output_file: str + partition_size: int + sector_size: int + sectors_per_cluster: int + root_entry_count: int + long_name_support: bool + use_default_datetime: bool + fat_type: int | None + fat_count: int + wl_mode: str | None + + +def _normalize_partition_size(partition_size: str | int, wl: bool) -> int: + if partition_size == 'detect' and not wl: + return -1 + return int(str(partition_size), 0) + + +def _partition_generator_args( # type: ignore[no-untyped-def] + input_directory, + output_file, + partition_size, + sector_size, + sectors_per_cluster, + root_entry_count, + long_name_support, + use_default_datetime, + fat_type, + fat_count, + wl_mode, + wl: bool, +) -> PartitionGeneratorArgs: + if not os.path.isdir(input_directory): + log.die(f'The target directory `{input_directory}` does not exist!') + if wl_mode is not None and sector_size != 512: + log.die('Wear levelling mode can be set only for sector size 512') + return PartitionGeneratorArgs( + input_directory=input_directory, + output_file=output_file, + partition_size=_normalize_partition_size(partition_size, wl), + sector_size=sector_size, + sectors_per_cluster=sectors_per_cluster, + root_entry_count=int(root_entry_count), + long_name_support=long_name_support, + use_default_datetime=use_default_datetime, + fat_type=None if fat_type == 0 else fat_type, + fat_count=fat_count, + wl_mode=wl_mode, + ) + + +def _build_partition_generator_cli(desc: str, wl: bool) -> Any: + import rich_click as click + from esp_pylib.cli_types import AnyIntType + + sector_choices = ALLOWED_WL_SECTOR_SIZES if wl else ALLOWED_SECTOR_SIZES + partition_size_help = 'Size of the partition in bytes.' + if not wl: + partition_size_help += ' Use `--partition_size detect` for detecting the minimal partition size.' + + @click.command( + context_settings={'help_option_names': ['-h', '--help']}, + help=desc, + ) + @click.argument('input_directory', type=click.Path(exists=False, file_okay=False)) + @click.option( + '--output_file', + default='fatfs_image.img', + show_default=True, + help='Filename of the generated fatfs image', + ) + @click.option( '--partition_size', - default=FATDefaults.SIZE, - help='Size of the partition in bytes.' - + ('' if wl else ' Use `--partition_size detect` for detecting the minimal partition size.'), + default=str(FATDefaults.SIZE), + show_default=True, + help=partition_size_help, ) - parser.add_argument( + @click.option( '--sector_size', - default=FATDefaults.SECTOR_SIZE, - type=int, - choices=ALLOWED_WL_SECTOR_SIZES if wl else ALLOWED_SECTOR_SIZES, - help='Size of the partition in bytes', + type=click.Choice([str(s) for s in sector_choices], case_sensitive=False), + default=str(FATDefaults.SECTOR_SIZE), + show_default=True, + help='Size of the partition sector in bytes', ) - parser.add_argument( + @click.option( '--sectors_per_cluster', - default=1, - type=int, - choices=ALLOWED_SECTORS_PER_CLUSTER, + type=click.Choice([str(s) for s in ALLOWED_SECTORS_PER_CLUSTER], case_sensitive=False), + default='1', + show_default=True, help='Number of sectors per cluster', ) - parser.add_argument( - '--root_entry_count', default=FATDefaults.ROOT_ENTRIES_COUNT, help='Number of entries in the root directory' + @click.option( + '--root_entry_count', + default=str(FATDefaults.ROOT_ENTRIES_COUNT), + show_default=True, + type=AnyIntType(), + help='Number of entries in the root directory', ) - parser.add_argument('--long_name_support', action='store_true', help='Set flag to enable long names support.') - parser.add_argument( + @click.option( + '--long_name_support', + is_flag=True, + default=False, + help='Set flag to enable long names support.', + ) + @click.option( '--use_default_datetime', - action='store_true', + is_flag=True, + default=False, help='For test purposes. If the flag is set the files are created with ' 'the default timestamp that is the 1st of January 1980', ) - parser.add_argument( + @click.option( '--fat_type', - default=0, - type=int, - choices=[FAT12, FAT16, 0], - help=""" - Type of the FAT file-system. Select '12' for FAT12, '16' for FAT16. - Leave unset or select 0 for automatic file-system type detection. - """, + type=click.Choice(['12', '16', '0'], case_sensitive=False), + default='0', + show_default=True, + help='Type of the FAT file-system. Select 12 for FAT12, 16 for FAT16. ' + 'Leave unset or select 0 for automatic file-system type detection.', ) - parser.add_argument( + @click.option( '--fat_count', - default=FATDefaults.FAT_TABLES_COUNT, - type=int, - choices=[1, 2], + type=click.Choice(['1', '2'], case_sensitive=False), + default=str(FATDefaults.FAT_TABLES_COUNT), + show_default=True, help='Number of file allocation tables (FATs) in the filesystem.', ) - parser.add_argument( + @click.option( '--wl_mode', + type=click.Choice(['safe', 'perf'], case_sensitive=False), default=None, - type=str, - choices=['safe', 'perf'], help='Wear levelling mode to use. Safe or performance. Only for sector size of 512', ) + def cli( # type: ignore[no-untyped-def] + input_directory, + output_file, + partition_size, + sector_size, + sectors_per_cluster, + root_entry_count, + long_name_support, + use_default_datetime, + fat_type, + fat_count, + wl_mode, + ): + return _partition_generator_args( + input_directory=input_directory, + output_file=output_file, + partition_size=partition_size, + sector_size=int(sector_size), + sectors_per_cluster=int(sectors_per_cluster), + root_entry_count=root_entry_count, + long_name_support=long_name_support, + use_default_datetime=use_default_datetime, + fat_type=int(fat_type), + fat_count=int(fat_count), + wl_mode=wl_mode, + wl=wl, + ) - args = parser.parse_args() - if args.fat_type == 0: - args.fat_type = None - if args.partition_size == 'detect' and not wl: - args.partition_size = -1 - args.partition_size = int(str(args.partition_size), 0) - if not os.path.isdir(args.input_directory): - raise NotADirectoryError(f'The target directory `{args.input_directory}` does not exist!') - if args.wl_mode is not None: - if args.sector_size != 512: - raise ValueError('Wear levelling mode can be set only for sector size 512') - return args + return cli + + +def get_args_for_partition_generator(desc: str, wl: bool) -> PartitionGeneratorArgs: + return cast(PartitionGeneratorArgs, _build_partition_generator_cli(desc, wl)(standalone_mode=False)) def read_filesystem(path: str) -> bytearray: diff --git a/components/fatfs/fatfsgen.py b/components/fatfs/fatfsgen.py index 30d274f5260..70fec7f406b 100755 --- a/components/fatfs/fatfsgen.py +++ b/components/fatfs/fatfsgen.py @@ -1,11 +1,9 @@ #!/usr/bin/env python -# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import os from datetime import datetime from typing import Any -from typing import List -from typing import Optional from fatfs_utils.boot_sector import BootSector from fatfs_utils.exceptions import NoFreeClusterException @@ -14,15 +12,15 @@ from fatfs_utils.fatfs_state import FATFSState from fatfs_utils.fs_object import Directory from fatfs_utils.long_filename_utils import get_required_lfn_entries_count from fatfs_utils.utils import BYTES_PER_DIRECTORY_ENTRY -from fatfs_utils.utils import FATDefaults from fatfs_utils.utils import FATFS_INCEPTION from fatfs_utils.utils import FATFS_MIN_ALLOC_UNIT +from fatfs_utils.utils import RESERVED_CLUSTERS_COUNT +from fatfs_utils.utils import FATDefaults from fatfs_utils.utils import get_args_for_partition_generator from fatfs_utils.utils import get_fat_sectors_count from fatfs_utils.utils import get_non_data_sectors_cnt from fatfs_utils.utils import read_filesystem from fatfs_utils.utils import required_clusters_count -from fatfs_utils.utils import RESERVED_CLUSTERS_COUNT def duplicate_fat_decorator(func): # type: ignore @@ -30,6 +28,7 @@ def duplicate_fat_decorator(func): # type: ignore func(self, *args, **kwargs) if isinstance(self, FATFS): self.duplicate_fat() + return wrapper @@ -39,68 +38,78 @@ class FATFS: It contains reference to the FAT table and to the root directory. """ - def __init__(self, - binary_image_path: Optional[str] = None, - size: int = FATDefaults.SIZE, - reserved_sectors_cnt: int = FATDefaults.RESERVED_SECTORS_COUNT, - fat_tables_cnt: int = FATDefaults.FAT_TABLES_COUNT, - sectors_per_cluster: int = FATDefaults.SECTORS_PER_CLUSTER, - sector_size: int = FATDefaults.SECTOR_SIZE, - hidden_sectors: int = FATDefaults.HIDDEN_SECTORS, - long_names_enabled: bool = False, - use_default_datetime: bool = True, - num_heads: int = FATDefaults.NUM_HEADS, - oem_name: str = FATDefaults.OEM_NAME, - sec_per_track: int = FATDefaults.SEC_PER_TRACK, - volume_label: str = FATDefaults.VOLUME_LABEL, - file_sys_type: str = FATDefaults.FILE_SYS_TYPE, - root_entry_count: int = FATDefaults.ROOT_ENTRIES_COUNT, - explicit_fat_type: Optional[int] = None, - media_type: int = FATDefaults.MEDIA_TYPE) -> None: + def __init__( + self, + binary_image_path: str | None = None, + size: int = FATDefaults.SIZE, + reserved_sectors_cnt: int = FATDefaults.RESERVED_SECTORS_COUNT, + fat_tables_cnt: int = FATDefaults.FAT_TABLES_COUNT, + sectors_per_cluster: int = FATDefaults.SECTORS_PER_CLUSTER, + sector_size: int = FATDefaults.SECTOR_SIZE, + hidden_sectors: int = FATDefaults.HIDDEN_SECTORS, + long_names_enabled: bool = False, + use_default_datetime: bool = True, + num_heads: int = FATDefaults.NUM_HEADS, + oem_name: str = FATDefaults.OEM_NAME, + sec_per_track: int = FATDefaults.SEC_PER_TRACK, + volume_label: str = FATDefaults.VOLUME_LABEL, + file_sys_type: str = FATDefaults.FILE_SYS_TYPE, + root_entry_count: int = FATDefaults.ROOT_ENTRIES_COUNT, + explicit_fat_type: int | None = None, + media_type: int = FATDefaults.MEDIA_TYPE, + ) -> None: # root directory bytes should be aligned by sector size assert (int(root_entry_count) * BYTES_PER_DIRECTORY_ENTRY) % sector_size == 0 # number of bytes in the root dir must be even multiple of BPB_BytsPerSec - if (int(root_entry_count) > 128): + if int(root_entry_count) > 128: assert ((int(root_entry_count) * BYTES_PER_DIRECTORY_ENTRY) // sector_size) % 2 == 0 root_dir_sectors_cnt: int = (int(root_entry_count) * BYTES_PER_DIRECTORY_ENTRY) // sector_size - self.state: FATFSState = FATFSState(sector_size=sector_size, - explicit_fat_type=explicit_fat_type, - reserved_sectors_cnt=reserved_sectors_cnt, - root_dir_sectors_cnt=root_dir_sectors_cnt, - size=size, - file_sys_type=file_sys_type, - num_heads=num_heads, - fat_tables_cnt=fat_tables_cnt, - sectors_per_cluster=sectors_per_cluster, - media_type=media_type, - hidden_sectors=hidden_sectors, - sec_per_track=sec_per_track, - long_names_enabled=long_names_enabled, - volume_label=volume_label, - oem_name=oem_name, - use_default_datetime=use_default_datetime) + self.state: FATFSState = FATFSState( + sector_size=sector_size, + explicit_fat_type=explicit_fat_type, + reserved_sectors_cnt=reserved_sectors_cnt, + root_dir_sectors_cnt=root_dir_sectors_cnt, + size=size, + file_sys_type=file_sys_type, + num_heads=num_heads, + fat_tables_cnt=fat_tables_cnt, + sectors_per_cluster=sectors_per_cluster, + media_type=media_type, + hidden_sectors=hidden_sectors, + sec_per_track=sec_per_track, + long_names_enabled=long_names_enabled, + volume_label=volume_label, + oem_name=oem_name, + use_default_datetime=use_default_datetime, + ) binary_image: bytes = bytearray( - read_filesystem(binary_image_path) if binary_image_path else self.create_empty_fatfs()) + read_filesystem(binary_image_path) if binary_image_path else self.create_empty_fatfs() + ) self.state.binary_image = binary_image self.fat: FAT = FAT(boot_sector_state=self.state.boot_sector_state, init_=True) root_dir_size = self.state.boot_sector_state.root_dir_sectors_cnt * self.state.boot_sector_state.sector_size - self.root_directory: Directory = Directory(name='A', # the name is not important, must be string - size=root_dir_size, - fat=self.fat, - cluster=self.fat.clusters[1], - fatfs_state=self.state) + self.root_directory: Directory = Directory( + name='A', # the name is not important, must be string + size=root_dir_size, + fat=self.fat, + cluster=self.fat.clusters[1], + fatfs_state=self.state, + ) self.root_directory.init_directory() @duplicate_fat_decorator - def create_file(self, name: str, - extension: str = '', - path_from_root: Optional[List[str]] = None, - object_timestamp_: datetime = FATFS_INCEPTION, - is_empty: bool = False) -> None: + def create_file( + self, + name: str, + extension: str = '', + path_from_root: list[str] | None = None, + object_timestamp_: datetime = FATFS_INCEPTION, + is_empty: bool = False, + ) -> None: """ This method allocates necessary clusters and creates a new file record in the directory required. The directory must exists. @@ -113,16 +122,18 @@ class FATFS: :param object_timestamp_: is not None, this will be propagated to the file's entry :param is_empty: True if there is no need to allocate any cluster, otherwise False """ - self.root_directory.new_file(name=name, - extension=extension, - path_from_root=path_from_root, - object_timestamp_=object_timestamp_, - is_empty=is_empty) + self.root_directory.new_file( + name=name, + extension=extension, + path_from_root=path_from_root, + object_timestamp_=object_timestamp_, + is_empty=is_empty, + ) @duplicate_fat_decorator - def create_directory(self, name: str, - path_from_root: Optional[List[str]] = None, - object_timestamp_: datetime = FATFS_INCEPTION) -> None: + def create_directory( + self, name: str, path_from_root: list[str] | None = None, object_timestamp_: datetime = FATFS_INCEPTION + ) -> None: """ Initially recursively finds a parent of the new directory and then create a new directory inside the parent. @@ -139,13 +150,12 @@ class FATFS: if path_from_root: parent_dir = self.root_directory.recursive_search(path_from_root, self.root_directory) - self.root_directory.new_directory(name=name, - parent=parent_dir, - path_from_root=path_from_root, - object_timestamp_=object_timestamp_) + self.root_directory.new_directory( + name=name, parent=parent_dir, path_from_root=path_from_root, object_timestamp_=object_timestamp_ + ) @duplicate_fat_decorator - def write_content(self, path_from_root: List[str], content: bytes) -> None: + def write_content(self, path_from_root: list[str], content: bytes) -> None: """ fat fs invokes root directory to recursively find the required file and writes the content """ @@ -165,8 +175,8 @@ class FATFS: fat_start = boot_sec_st.reserved_sectors_cnt * boot_sec_st.sector_size fat_end = fat_start + boot_sec_st.sectors_per_fat_cnt * boot_sec_st.sector_size second_fat_shift = boot_sec_st.sectors_per_fat_cnt * boot_sec_st.sector_size - self.state.binary_image[fat_start + second_fat_shift: fat_end + second_fat_shift] = ( - self.state.binary_image[fat_start: fat_end] + self.state.binary_image[fat_start + second_fat_shift : fat_end + second_fat_shift] = ( + self.state.binary_image[fat_start:fat_end] ) def write_filesystem(self, output_path: str) -> None: @@ -174,10 +184,9 @@ class FATFS: output.write(bytearray(self.state.binary_image)) @duplicate_fat_decorator - def _generate_partition_from_folder(self, - folder_relative_path: str, - folder_path: str = '', - is_dir: bool = False) -> None: + def _generate_partition_from_folder( + self, folder_relative_path: str, folder_path: str = '', is_dir: bool = False + ) -> None: """ Given path to folder and folder name recursively encodes folder into binary image. Used by method generate. @@ -196,17 +205,19 @@ class FATFS: content = file.read() file_name, extension = os.path.splitext(split_path[-1]) extension = extension[1:] # remove the dot from the extension - self.create_file(name=file_name, - extension=extension, - path_from_root=split_path[1:-1] or None, - object_timestamp_=object_timestamp, - is_empty=len(content) == 0) + self.create_file( + name=file_name, + extension=extension, + path_from_root=split_path[1:-1] or None, + object_timestamp_=object_timestamp, + is_empty=len(content) == 0, + ) self.write_content(split_path[1:], content) elif os.path.isdir(real_path): if not is_dir: - self.create_directory(name=split_path[-1], - path_from_root=split_path[1:-1], - object_timestamp_=object_timestamp) + self.create_directory( + name=split_path[-1], path_from_root=split_path[1:-1], object_timestamp_=object_timestamp + ) # sorting files for better testability dir_content = list(sorted(os.listdir(real_path))) @@ -221,11 +232,9 @@ class FATFS: self._generate_partition_from_folder(folder_name, folder_path=path_to_folder, is_dir=True) -def calculate_min_space(path: List[str], - fs_entity: str, - sector_size: int = 0x1000, - long_file_names: bool = False, - is_root: bool = False) -> int: +def calculate_min_space( + path: list[str], fs_entity: str, sector_size: int = 0x1000, long_file_names: bool = False, is_root: bool = False +) -> int: if os.path.isfile(os.path.join(*path, fs_entity)): with open(os.path.join(*path, fs_entity), 'rb') as file_: content = file_.read() @@ -255,26 +264,33 @@ def main() -> None: clusters = calculate_min_space([], args.input_directory, args.sector_size, long_file_names=True, is_root=True) fats = get_fat_sectors_count(clusters, args.sector_size) root_dir_sectors = (FATDefaults.ROOT_ENTRIES_COUNT * FATDefaults.ENTRY_SIZE) // args.sector_size - args.partition_size = max(FATFS_MIN_ALLOC_UNIT * args.sector_size, - (clusters + fats + get_non_data_sectors_cnt(RESERVED_CLUSTERS_COUNT, - fats, - args.fat_count, - root_dir_sectors) - ) * args.sector_size - ) + args.partition_size = max( + FATFS_MIN_ALLOC_UNIT * args.sector_size, + ( + clusters + + fats + + get_non_data_sectors_cnt(RESERVED_CLUSTERS_COUNT, fats, args.fat_count, root_dir_sectors) + ) + * args.sector_size, + ) - fatfs = FATFS(size=args.partition_size, - fat_tables_cnt=args.fat_count, - sectors_per_cluster=args.sectors_per_cluster, - sector_size=args.sector_size, - long_names_enabled=args.long_name_support, - use_default_datetime=args.use_default_datetime, - root_entry_count=args.root_entry_count, - explicit_fat_type=args.fat_type) + fatfs = FATFS( + size=args.partition_size, + fat_tables_cnt=args.fat_count, + sectors_per_cluster=args.sectors_per_cluster, + sector_size=args.sector_size, + long_names_enabled=args.long_name_support, + use_default_datetime=args.use_default_datetime, + root_entry_count=args.root_entry_count, + explicit_fat_type=args.fat_type, + ) fatfs.generate(args.input_directory) fatfs.write_filesystem(args.output_file) if __name__ == '__main__': + from esp_pylib.excepthook import install_exception_reporting + + install_exception_reporting() main() diff --git a/components/fatfs/fatfsparse.py b/components/fatfs/fatfsparse.py index 5e48bef11e8..6ac2fe5956b 100755 --- a/components/fatfs/fatfsparse.py +++ b/components/fatfs/fatfsparse.py @@ -1,10 +1,12 @@ #!/usr/bin/env python # SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 -import argparse import os +from typing import Any import construct +from esp_pylib.cli_options import MutuallyExclusiveOption +from esp_pylib.logger import log from fatfs_utils.boot_sector import BootSector from fatfs_utils.entry import Entry from fatfs_utils.fat import FAT @@ -24,14 +26,20 @@ def build_file_name(name1: bytes, name2: bytes, name3: bytes) -> str: return full_name_.rstrip(FULL_BYTE).decode(LONG_NAMES_ENCODING).rstrip('\x00') -def get_obj_name(obj_: dict, directory_bytes_: bytes, entry_position_: int, lfn_checksum_: int) -> str: +def get_obj_name( + obj_: dict, + directory_bytes_: bytes, + entry_position_: int, + lfn_checksum_: int, + long_name_support: bool, +) -> str: obj_ext_ = obj_['DIR_Name_ext'].rstrip(chr(PAD_CHAR)) ext_ = f'.{obj_ext_}' if len(obj_ext_) > 0 else '' obj_name_: str = obj_['DIR_Name'].rstrip(chr(PAD_CHAR)) + ext_ # short entry name # if LFN was detected, the record is considered as single SFN record only if DIR_NTRes == 0x18 (LDIR_DIR_NTRES) # if LFN was not detected, the record cannot be part of the LFN, no matter the value of DIR_NTRes - if not args.long_name_support or obj_['DIR_NTRes'] == Entry.LDIR_DIR_NTRES: + if not long_name_support or obj_['DIR_NTRes'] == Entry.LDIR_DIR_NTRES: return obj_name_ full_name = {} @@ -48,12 +56,18 @@ def get_obj_name(obj_: dict, directory_bytes_: bytes, entry_position_: int, lfn_ def traverse_folder_tree( - directory_bytes_: bytes, name: str, state_: BootSectorState, fat_: FAT, binary_array_: bytes -) -> None: + directory_bytes_: bytes, + name: str, + state_: BootSectorState, + fat_: FAT, + binary_array_: bytes, + long_name_support: bool, +) -> bool: os.makedirs(name) assert len(directory_bytes_) % FATDefaults.ENTRY_SIZE == 0 entries_count_: int = len(directory_bytes_) // FATDefaults.ENTRY_SIZE + enable_long_names = long_name_support for i in range(entries_count_): obj_address_: int = FATDefaults.ENTRY_SIZE * i @@ -62,7 +76,7 @@ def traverse_folder_tree( directory_bytes_[obj_address_ : obj_address_ + FATDefaults.ENTRY_SIZE] ) except (construct.core.ConstError, UnicodeDecodeError, construct.core.StringError): - args.long_name_support = True + enable_long_names = True continue if obj_['DIR_Attr'] == 0: # empty entry @@ -73,6 +87,7 @@ def traverse_folder_tree( directory_bytes_, entry_position_=i, lfn_checksum_=lfn_checksum(obj_['DIR_Name'] + obj_['DIR_Name_ext']), + long_name_support=enable_long_names, ) if obj_['DIR_Attr'] == Entry.ATTR_ARCHIVE: content_ = b'' @@ -85,14 +100,17 @@ def traverse_folder_tree( if obj_name_ in ('.', '..'): continue child_directory_bytes_ = fat_.get_chained_content(cluster_id_=obj_['DIR_FstClusLO']) - traverse_folder_tree( + enable_long_names = traverse_folder_tree( directory_bytes_=child_directory_bytes_, name=os.path.join(name, obj_name_), state_=state_, fat_=fat_, binary_array_=binary_array_, + long_name_support=enable_long_names, ) + return enable_long_names + def remove_wear_levelling_if_exists(fs_: bytes) -> bytes: """ @@ -113,57 +131,24 @@ def remove_wear_levelling_if_exists(fs_: bytes) -> bytes: return plain_fs -if __name__ == '__main__': - desc = 'Tool for parsing fatfs image and extracting directory structure on host.' - argument_parser: argparse.ArgumentParser = argparse.ArgumentParser(description=desc) - argument_parser.add_argument('input_image', help='Path to the image that will be parsed and extracted.') - argument_parser.add_argument('--long-name-support', action='store_true', help=argparse.SUPPRESS) +def _parse_fatfs_image( + input_image: str, + long_name_support: bool, + wl_layer: str, + verbose: bool, +) -> None: + fs = read_filesystem(input_image) - # ensures backward compatibility - argument_parser.add_argument('--wear-leveling', action='store_true', help=argparse.SUPPRESS) - argument_parser.add_argument( - '--wl-layer', - choices=['detect', 'enabled', 'disabled'], - default=None, - help="If detection doesn't work correctly, you can force analyzer to or not to assume WL.", - ) - argument_parser.add_argument('--verbose', action='store_true', help='Prints details about FAT image.') - - args = argument_parser.parse_args() - - # if wear levelling is detected or user explicitly sets the parameter `--wl_layer enabled` - # the partition with wear levelling is transformed to partition without WL for convenient parsing - # in some cases the partitions with and without wear levelling can be 100% equivalent - # and only user can break this tie by explicitly setting - # the parameter --wl-layer to enabled, respectively disabled - if args.wear_leveling and args.wl_layer: - raise NotImplementedError('Argument --wear-leveling cannot be combined with --wl-layer!') - if args.wear_leveling: - args.wl_layer = 'enabled' - args.wl_layer = args.wl_layer or 'detect' - - fs = read_filesystem(args.input_image) - - # An algorithm for removing wear levelling: - # 1. find an remove dummy sector: - # a) dummy sector is at the position defined by the number of records in the state sector - # b) dummy may not be placed in state nor cfg sectors - # c) first (boot) sector position (boot_s_pos) is calculated using value of move count - # boot_s_pos = - mc - # 2. remove state sectors (trivial) - # 3. remove cfg sector (trivial) - # 4. valid fs is then old_fs[-mc:] + old_fs[:-mc] - if args.wl_layer == 'enabled': + if wl_layer == 'enabled': fs = remove_wl(fs) - elif args.wl_layer != 'disabled': - # wear levelling is removed to enable parsing using common algorithm + elif wl_layer != 'disabled': fs = remove_wear_levelling_if_exists(fs) boot_sector_ = BootSector() boot_sector_.parse_boot_sector(fs) - if args.verbose: - print(str(boot_sector_)) + if verbose: + log.print(str(boot_sector_)) fat = FAT(boot_sector_.boot_sector_state, init_=False) @@ -176,4 +161,55 @@ if __name__ == '__main__': boot_sector_.boot_sector_state, fat, fs, + long_name_support=long_name_support, ) + + +def _build_cli() -> Any: + import rich_click as click + + desc = 'Tool for parsing fatfs image and extracting directory structure on host.' + + @click.command( + context_settings={'help_option_names': ['-h', '--help']}, + help=desc, + ) + @click.argument('input_image', type=click.Path(exists=False, dir_okay=False)) + @click.option('--long-name-support', is_flag=True, default=False, hidden=True) + @click.option( + '--wear-leveling', + is_flag=True, + default=False, + hidden=True, + cls=MutuallyExclusiveOption, + exclusive_with=['wl_layer'], + ) + @click.option( + '--wl-layer', + type=click.Choice(['detect', 'enabled', 'disabled'], case_sensitive=False), + default=None, + help="If detection doesn't work correctly, you can force analyzer to or not to assume WL.", + cls=MutuallyExclusiveOption, + exclusive_with=['wear_leveling'], + ) + @click.option('--verbose', is_flag=True, default=False, help='Prints details about FAT image.') + def cli(input_image, long_name_support, wear_leveling, wl_layer, verbose): # type: ignore[no-untyped-def] + if wear_leveling and wl_layer is not None: + log.die('Argument --wear-leveling cannot be combined with --wl-layer!') + if wear_leveling: + wl_layer = 'enabled' + wl_layer = wl_layer or 'detect' + _parse_fatfs_image(input_image, long_name_support, wl_layer, verbose) + + return cli + + +def main() -> None: + _build_cli()() + + +if __name__ == '__main__': + from esp_pylib.excepthook import install_exception_reporting + + install_exception_reporting() + main() diff --git a/components/fatfs/test_fatfsgen/test_fatfsgen.py b/components/fatfs/test_fatfsgen/test_fatfsgen.py index ca162f45b45..3665a3c79ae 100755 --- a/components/fatfs/test_fatfsgen/test_fatfsgen.py +++ b/components/fatfs/test_fatfsgen/test_fatfsgen.py @@ -737,19 +737,17 @@ class FatFSGen(unittest.TestCase): def test_boundary_clusters12(self) -> None: output: bytes = check_output( - ['python', '../fatfsgen.py', '--partition_size', '16732160', 'test_dir'], stderr=STDOUT - ) - self.assertEqual( - output, b'WARNING: It is not recommended to create FATFS with bounding count of clusters: 4085 or 65525\n' + [sys.executable, '../fatfsgen.py', '--partition_size', '16732160', 'test_dir'], stderr=STDOUT ) + self.assertIn(b'WARNING:', output) + self.assertIn(b'4085 or 65525', output) def test_boundary_clusters16(self) -> None: output: bytes = check_output( - ['python', '../fatfsgen.py', '--partition_size', '268390400', 'test_dir'], stderr=STDOUT - ) - self.assertEqual( - output, b'WARNING: It is not recommended to create FATFS with bounding count of clusters: 4085 or 65525\n' + [sys.executable, '../fatfsgen.py', '--partition_size', '268390400', 'test_dir'], stderr=STDOUT ) + self.assertIn(b'WARNING:', output) + self.assertIn(b'4085 or 65525', output) def test_boundary_clusters_fat32(self) -> None: self.assertRaises(NotImplementedError, fatfsgen.FATFS, size=268419193) diff --git a/components/fatfs/wl_fatfsgen.py b/components/fatfs/wl_fatfsgen.py index f4ded5cba76..9c2ceb135e4 100755 --- a/components/fatfs/wl_fatfsgen.py +++ b/components/fatfs/wl_fatfsgen.py @@ -227,7 +227,7 @@ class WLFATFS: output.write(bytearray(self.fatfs_binary_image)) -if __name__ == '__main__': +def main() -> None: desc = 'Create a FAT filesystem with support for wear levelling and populate it with directory content' args = get_args_for_partition_generator(desc, wl=True) wl_fatfs = WLFATFS( @@ -245,3 +245,10 @@ if __name__ == '__main__': wl_fatfs.plain_fatfs.generate(args.input_directory) wl_fatfs.init_wl() wl_fatfs.wl_write_filesystem(args.output_file) + + +if __name__ == '__main__': + from esp_pylib.excepthook import install_exception_reporting + + install_exception_reporting() + main()