Mirrors/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2026-08-18 06:35:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix(fatfs): normalize fatfs Python tool CLI output

Browse Source
This commit is contained in:
sonika.rathi
2026-06-04 12:10:02 +02:00
parent 7fac3ca134
commit 2820aa64c3
7 changed files with 469 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
components/fatfs/fatfs_utils/exceptions.py
View File

@@ -1,10 +1,25 @@
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
from esp_pylib.errors import FatalError
__all__ = [
'FatalError',
'InconsistentFATAttributes',
'LowerCaseException',
'NoFreeClusterException',
'NotInitialized',
'TooLongNameException',
'WLNotInitialized',
'WriteDirectoryException',
]
class WriteDirectoryException(Exception):
"""
Exception is raised when the user tries to write the content into the directory instead of file
"""
pass
@@ -12,6 +27,7 @@ class NoFreeClusterException(Exception):
"""
Exception is raised when the user tries allocate cluster but no free one is available
"""
pass
@@ -19,6 +35,7 @@ class LowerCaseException(Exception):
"""
Exception is raised when the user tries to write file or directory with lower case
"""
pass
@@ -26,6 +43,7 @@ class TooLongNameException(Exception):
"""
Exception is raised when long name support is not enabled and user tries to write file longer then allowed
"""
pass
@@ -33,6 +51,7 @@ class NotInitialized(Exception):
"""
Exception is raised when the user tries to access not initialized property
"""
pass
@@ -40,10 +59,7 @@ class WLNotInitialized(Exception):
"""
Exception is raised when the user tries to write fatfs not initialized with wear levelling
"""
pass
class FatalError(Exception):
pass
@@ -51,4 +67,5 @@ class InconsistentFATAttributes(Exception):
"""
Caused by e.g. wrong number of clusters for given FAT type
"""
pass

145
components/fatfs/fatfs_utils/fatfs_state.py
View File

@@ -1,13 +1,22 @@
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
from textwrap import dedent
from typing import Optional
from esp_pylib.logger import log
from .exceptions import InconsistentFATAttributes
from .utils import (ALLOWED_SECTOR_SIZES, FAT12, FAT12_MAX_CLUSTERS, FAT16, FAT16_MAX_CLUSTERS,
RESERVED_CLUSTERS_COUNT, FATDefaults, get_fat_sectors_count, get_fatfs_type,
get_non_data_sectors_cnt, number_of_clusters)
from .utils import ALLOWED_SECTOR_SIZES
from .utils import FAT12
from .utils import FAT12_MAX_CLUSTERS
from .utils import FAT16
from .utils import FAT16_MAX_CLUSTERS
from .utils import RESERVED_CLUSTERS_COUNT
from .utils import FATDefaults
from .utils import get_fat_sectors_count
from .utils import get_fatfs_type
from .utils import get_non_data_sectors_cnt
from .utils import number_of_clusters
class FATFSState:
@@ -15,45 +24,51 @@ class FATFSState:
The class represents the state and the configuration of the FATFS.
"""
def __init__(self,
sector_size: int,
reserved_sectors_cnt: int,
root_dir_sectors_cnt: int,
size: int,
media_type: int,
sectors_per_cluster: int,
volume_label: str,
oem_name: str,
fat_tables_cnt: int,
sec_per_track: int,
num_heads: int,
hidden_sectors: int,
file_sys_type: str,
use_default_datetime: bool,
explicit_fat_type: Optional[int] = None,
long_names_enabled: bool = False):
self.boot_sector_state = BootSectorState(oem_name=oem_name,
sector_size=sector_size,
sectors_per_cluster=sectors_per_cluster,
reserved_sectors_cnt=reserved_sectors_cnt,
fat_tables_cnt=fat_tables_cnt,
root_dir_sectors_cnt=root_dir_sectors_cnt,
sectors_count=size // sector_size,
media_type=media_type,
sec_per_track=sec_per_track,
num_heads=num_heads,
hidden_sectors=hidden_sectors,
volume_label=volume_label,
file_sys_type=file_sys_type,
volume_uuid=-1)
def __init__(
self,
sector_size: int,
reserved_sectors_cnt: int,
root_dir_sectors_cnt: int,
size: int,
media_type: int,
sectors_per_cluster: int,
volume_label: str,
oem_name: str,
fat_tables_cnt: int,
sec_per_track: int,
num_heads: int,
hidden_sectors: int,
file_sys_type: str,
use_default_datetime: bool,
explicit_fat_type: int | None = None,
long_names_enabled: bool = False,
):
self.boot_sector_state = BootSectorState(
oem_name=oem_name,
sector_size=sector_size,
sectors_per_cluster=sectors_per_cluster,
reserved_sectors_cnt=reserved_sectors_cnt,
fat_tables_cnt=fat_tables_cnt,
root_dir_sectors_cnt=root_dir_sectors_cnt,
sectors_count=size // sector_size,
media_type=media_type,
sec_per_track=sec_per_track,
num_heads=num_heads,
hidden_sectors=hidden_sectors,
volume_label=volume_label,
file_sys_type=file_sys_type,
volume_uuid=-1,
)
self._explicit_fat_type: Optional[int] = explicit_fat_type
self._explicit_fat_type: int | None = explicit_fat_type
self.long_names_enabled: bool = long_names_enabled
self.use_default_datetime: bool = use_default_datetime
if (size // sector_size) * sectors_per_cluster in (FAT12_MAX_CLUSTERS, FAT16_MAX_CLUSTERS):
print('WARNING: It is not recommended to create FATFS with bounding '
f'count of clusters: {FAT12_MAX_CLUSTERS} or {FAT16_MAX_CLUSTERS}')
log.warn(
'It is not recommended to create FATFS with bounding '
f'count of clusters: {FAT12_MAX_CLUSTERS} or {FAT16_MAX_CLUSTERS}'
)
self.check_fat_type()
@property
@@ -67,31 +82,36 @@ class FATFSState:
def check_fat_type(self) -> None:
_type = self.boot_sector_state.fatfs_type
if self._explicit_fat_type is not None and self._explicit_fat_type != _type:
raise InconsistentFATAttributes(dedent(
f"""FAT type you specified is inconsistent with other attributes of the system.
raise InconsistentFATAttributes(
dedent(
f"""FAT type you specified is inconsistent with other attributes of the system.
The specified FATFS type: FAT{self._explicit_fat_type}
The actual FATFS type: FAT{_type}"""))
The actual FATFS type: FAT{_type}"""
)
)
if _type not in (FAT12, FAT16):
raise NotImplementedError('FAT32 is currently not supported.')
class BootSectorState:
# pylint: disable=too-many-instance-attributes
def __init__(self,
oem_name: str,
sector_size: int,
sectors_per_cluster: int,
reserved_sectors_cnt: int,
fat_tables_cnt: int,
root_dir_sectors_cnt: int,
sectors_count: int,
media_type: int,
sec_per_track: int,
num_heads: int,
hidden_sectors: int,
volume_label: str,
file_sys_type: str,
volume_uuid: int = -1) -> None:
def __init__(
self,
oem_name: str,
sector_size: int,
sectors_per_cluster: int,
reserved_sectors_cnt: int,
fat_tables_cnt: int,
root_dir_sectors_cnt: int,
sectors_count: int,
media_type: int,
sec_per_track: int,
num_heads: int,
hidden_sectors: int,
volume_label: str,
file_sys_type: str,
volume_uuid: int = -1,
) -> None:
self.oem_name: str = oem_name
self.sector_size: int = sector_size
assert self.sector_size in ALLOWED_SECTOR_SIZES
@@ -150,10 +170,9 @@ class BootSectorState:
@property
def non_data_sectors(self) -> int:
non_data_sectors_: int = get_non_data_sectors_cnt(self.reserved_sectors_cnt,
self.sectors_per_fat_cnt,
self.fat_tables_cnt,
self.root_dir_sectors_cnt)
non_data_sectors_: int = get_non_data_sectors_cnt(
self.reserved_sectors_cnt, self.sectors_per_fat_cnt, self.fat_tables_cnt, self.root_dir_sectors_cnt
)
return non_data_sectors_
@property
@@ -167,5 +186,7 @@ class BootSectorState:
@property
def root_directory_start(self) -> int:
root_dir_start: int = (self.reserved_sectors_cnt + self.sectors_per_fat_cnt * self.fat_tables_cnt) * self.sector_size
root_dir_start: int = (
self.reserved_sectors_cnt + self.sectors_per_fat_cnt * self.fat_tables_cnt
) * self.sector_size
return root_dir_start

200
components/fatfs/fatfs_utils/utils.py
View File

@@ -1,15 +1,18 @@
# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import argparse
import binascii
import os
import re
import uuid
from dataclasses import dataclass
from datetime import datetime
from typing import Any
from typing import cast
from construct import BitsInteger
from construct import BitStruct
from construct import Int16ul
from esp_pylib.logger import log
# the regex pattern defines symbols that are allowed by long file names but not by short file names
INVALID_SFN_CHARS_PATTERN = re.compile(r'[.+,;=\[\]]')
@@ -203,77 +206,174 @@ def split_content_into_sectors(content: bytes, sector_size: int) -> list[bytes]:
return result
def get_args_for_partition_generator(desc: str, wl: bool) -> argparse.Namespace:
parser: argparse.ArgumentParser = argparse.ArgumentParser(description=desc)
parser.add_argument('input_directory', help='Path to the directory that will be encoded into fatfs image')
parser.add_argument('--output_file', default='fatfs_image.img', help='Filename of the generated fatfs image')
parser.add_argument(
@dataclass
class PartitionGeneratorArgs:
input_directory: str
output_file: str
partition_size: int
sector_size: int
sectors_per_cluster: int
root_entry_count: int
long_name_support: bool
use_default_datetime: bool
fat_type: int | None
fat_count: int
wl_mode: str | None
def _normalize_partition_size(partition_size: str | int, wl: bool) -> int:
if partition_size == 'detect' and not wl:
return -1
return int(str(partition_size), 0)
def _partition_generator_args( # type: ignore[no-untyped-def]
input_directory,
output_file,
partition_size,
sector_size,
sectors_per_cluster,
root_entry_count,
long_name_support,
use_default_datetime,
fat_type,
fat_count,
wl_mode,
wl: bool,
) -> PartitionGeneratorArgs:
if not os.path.isdir(input_directory):
log.die(f'The target directory `{input_directory}` does not exist!')
if wl_mode is not None and sector_size != 512:
log.die('Wear levelling mode can be set only for sector size 512')
return PartitionGeneratorArgs(
input_directory=input_directory,
output_file=output_file,
partition_size=_normalize_partition_size(partition_size, wl),
sector_size=sector_size,
sectors_per_cluster=sectors_per_cluster,
root_entry_count=int(root_entry_count),
long_name_support=long_name_support,
use_default_datetime=use_default_datetime,
fat_type=None if fat_type == 0 else fat_type,
fat_count=fat_count,
wl_mode=wl_mode,
)
def _build_partition_generator_cli(desc: str, wl: bool) -> Any:
import rich_click as click
from esp_pylib.cli_types import AnyIntType
sector_choices = ALLOWED_WL_SECTOR_SIZES if wl else ALLOWED_SECTOR_SIZES
partition_size_help = 'Size of the partition in bytes.'
if not wl:
partition_size_help += ' Use `--partition_size detect` for detecting the minimal partition size.'
@click.command(
context_settings={'help_option_names': ['-h', '--help']},
help=desc,
)
@click.argument('input_directory', type=click.Path(exists=False, file_okay=False))
@click.option(
'--output_file',
default='fatfs_image.img',
show_default=True,
help='Filename of the generated fatfs image',
)
@click.option(
'--partition_size',
default=FATDefaults.SIZE,
help='Size of the partition in bytes.'
+ ('' if wl else ' Use `--partition_size detect` for detecting the minimal partition size.'),
default=str(FATDefaults.SIZE),
show_default=True,
help=partition_size_help,
)
parser.add_argument(
@click.option(
'--sector_size',
default=FATDefaults.SECTOR_SIZE,
type=int,
choices=ALLOWED_WL_SECTOR_SIZES if wl else ALLOWED_SECTOR_SIZES,
help='Size of the partition in bytes',
type=click.Choice([str(s) for s in sector_choices], case_sensitive=False),
default=str(FATDefaults.SECTOR_SIZE),
show_default=True,
help='Size of the partition sector in bytes',
)
parser.add_argument(
@click.option(
'--sectors_per_cluster',
default=1,
type=int,
choices=ALLOWED_SECTORS_PER_CLUSTER,
type=click.Choice([str(s) for s in ALLOWED_SECTORS_PER_CLUSTER], case_sensitive=False),
default='1',
show_default=True,
help='Number of sectors per cluster',
)
parser.add_argument(
'--root_entry_count', default=FATDefaults.ROOT_ENTRIES_COUNT, help='Number of entries in the root directory'
@click.option(
'--root_entry_count',
default=str(FATDefaults.ROOT_ENTRIES_COUNT),
show_default=True,
type=AnyIntType(),
help='Number of entries in the root directory',
)
parser.add_argument('--long_name_support', action='store_true', help='Set flag to enable long names support.')
parser.add_argument(
@click.option(
'--long_name_support',
is_flag=True,
default=False,
help='Set flag to enable long names support.',
)
@click.option(
'--use_default_datetime',
action='store_true',
is_flag=True,
default=False,
help='For test purposes. If the flag is set the files are created with '
'the default timestamp that is the 1st of January 1980',
)
parser.add_argument(
@click.option(
'--fat_type',
default=0,
type=int,
choices=[FAT12, FAT16, 0],
help="""
Type of the FAT file-system. Select '12' for FAT12, '16' for FAT16.
Leave unset or select 0 for automatic file-system type detection.
""",
type=click.Choice(['12', '16', '0'], case_sensitive=False),
default='0',
show_default=True,
help='Type of the FAT file-system. Select 12 for FAT12, 16 for FAT16. '
'Leave unset or select 0 for automatic file-system type detection.',
)
parser.add_argument(
@click.option(
'--fat_count',
default=FATDefaults.FAT_TABLES_COUNT,
type=int,
choices=[1, 2],
type=click.Choice(['1', '2'], case_sensitive=False),
default=str(FATDefaults.FAT_TABLES_COUNT),
show_default=True,
help='Number of file allocation tables (FATs) in the filesystem.',
)
parser.add_argument(
@click.option(
'--wl_mode',
type=click.Choice(['safe', 'perf'], case_sensitive=False),
default=None,
type=str,
choices=['safe', 'perf'],
help='Wear levelling mode to use. Safe or performance. Only for sector size of 512',
)
def cli( # type: ignore[no-untyped-def]
input_directory,
output_file,
partition_size,
sector_size,
sectors_per_cluster,
root_entry_count,
long_name_support,
use_default_datetime,
fat_type,
fat_count,
wl_mode,
):
return _partition_generator_args(
input_directory=input_directory,
output_file=output_file,
partition_size=partition_size,
sector_size=int(sector_size),
sectors_per_cluster=int(sectors_per_cluster),
root_entry_count=root_entry_count,
long_name_support=long_name_support,
use_default_datetime=use_default_datetime,
fat_type=int(fat_type),
fat_count=int(fat_count),
wl_mode=wl_mode,
wl=wl,
)
args = parser.parse_args()
if args.fat_type == 0:
args.fat_type = None
if args.partition_size == 'detect' and not wl:
args.partition_size = -1
args.partition_size = int(str(args.partition_size), 0)
if not os.path.isdir(args.input_directory):
raise NotADirectoryError(f'The target directory `{args.input_directory}` does not exist!')
if args.wl_mode is not None:
if args.sector_size != 512:
raise ValueError('Wear levelling mode can be set only for sector size 512')
return args
return cli
def get_args_for_partition_generator(desc: str, wl: bool) -> PartitionGeneratorArgs:
return cast(PartitionGeneratorArgs, _build_partition_generator_cli(desc, wl)(standalone_mode=False))
def read_filesystem(path: str) -> bytearray:

212
components/fatfs/fatfsgen.py
View File

@@ -1,11 +1,9 @@
#!/usr/bin/env python
# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import os
from datetime import datetime
from typing import Any
from typing import List
from typing import Optional
from fatfs_utils.boot_sector import BootSector
from fatfs_utils.exceptions import NoFreeClusterException
@@ -14,15 +12,15 @@ from fatfs_utils.fatfs_state import FATFSState
from fatfs_utils.fs_object import Directory
from fatfs_utils.long_filename_utils import get_required_lfn_entries_count
from fatfs_utils.utils import BYTES_PER_DIRECTORY_ENTRY
from fatfs_utils.utils import FATDefaults
from fatfs_utils.utils import FATFS_INCEPTION
from fatfs_utils.utils import FATFS_MIN_ALLOC_UNIT
from fatfs_utils.utils import RESERVED_CLUSTERS_COUNT
from fatfs_utils.utils import FATDefaults
from fatfs_utils.utils import get_args_for_partition_generator
from fatfs_utils.utils import get_fat_sectors_count
from fatfs_utils.utils import get_non_data_sectors_cnt
from fatfs_utils.utils import read_filesystem
from fatfs_utils.utils import required_clusters_count
from fatfs_utils.utils import RESERVED_CLUSTERS_COUNT
def duplicate_fat_decorator(func): # type: ignore
@@ -30,6 +28,7 @@ def duplicate_fat_decorator(func): # type: ignore
func(self, *args, **kwargs)
if isinstance(self, FATFS):
self.duplicate_fat()
return wrapper
@@ -39,68 +38,78 @@ class FATFS:
It contains reference to the FAT table and to the root directory.
"""
def __init__(self,
binary_image_path: Optional[str] = None,
size: int = FATDefaults.SIZE,
reserved_sectors_cnt: int = FATDefaults.RESERVED_SECTORS_COUNT,
fat_tables_cnt: int = FATDefaults.FAT_TABLES_COUNT,
sectors_per_cluster: int = FATDefaults.SECTORS_PER_CLUSTER,
sector_size: int = FATDefaults.SECTOR_SIZE,
hidden_sectors: int = FATDefaults.HIDDEN_SECTORS,
long_names_enabled: bool = False,
use_default_datetime: bool = True,
num_heads: int = FATDefaults.NUM_HEADS,
oem_name: str = FATDefaults.OEM_NAME,
sec_per_track: int = FATDefaults.SEC_PER_TRACK,
volume_label: str = FATDefaults.VOLUME_LABEL,
file_sys_type: str = FATDefaults.FILE_SYS_TYPE,
root_entry_count: int = FATDefaults.ROOT_ENTRIES_COUNT,
explicit_fat_type: Optional[int] = None,
media_type: int = FATDefaults.MEDIA_TYPE) -> None:
def __init__(
self,
binary_image_path: str | None = None,
size: int = FATDefaults.SIZE,
reserved_sectors_cnt: int = FATDefaults.RESERVED_SECTORS_COUNT,
fat_tables_cnt: int = FATDefaults.FAT_TABLES_COUNT,
sectors_per_cluster: int = FATDefaults.SECTORS_PER_CLUSTER,
sector_size: int = FATDefaults.SECTOR_SIZE,
hidden_sectors: int = FATDefaults.HIDDEN_SECTORS,
long_names_enabled: bool = False,
use_default_datetime: bool = True,
num_heads: int = FATDefaults.NUM_HEADS,
oem_name: str = FATDefaults.OEM_NAME,
sec_per_track: int = FATDefaults.SEC_PER_TRACK,
volume_label: str = FATDefaults.VOLUME_LABEL,
file_sys_type: str = FATDefaults.FILE_SYS_TYPE,
root_entry_count: int = FATDefaults.ROOT_ENTRIES_COUNT,
explicit_fat_type: int | None = None,
media_type: int = FATDefaults.MEDIA_TYPE,
) -> None:
# root directory bytes should be aligned by sector size
assert (int(root_entry_count) * BYTES_PER_DIRECTORY_ENTRY) % sector_size == 0
# number of bytes in the root dir must be even multiple of BPB_BytsPerSec
if (int(root_entry_count) > 128):
if int(root_entry_count) > 128:
assert ((int(root_entry_count) * BYTES_PER_DIRECTORY_ENTRY) // sector_size) % 2 == 0
root_dir_sectors_cnt: int = (int(root_entry_count) * BYTES_PER_DIRECTORY_ENTRY) // sector_size
self.state: FATFSState = FATFSState(sector_size=sector_size,
explicit_fat_type=explicit_fat_type,
reserved_sectors_cnt=reserved_sectors_cnt,
root_dir_sectors_cnt=root_dir_sectors_cnt,
size=size,
file_sys_type=file_sys_type,
num_heads=num_heads,
fat_tables_cnt=fat_tables_cnt,
sectors_per_cluster=sectors_per_cluster,
media_type=media_type,
hidden_sectors=hidden_sectors,
sec_per_track=sec_per_track,
long_names_enabled=long_names_enabled,
volume_label=volume_label,
oem_name=oem_name,
use_default_datetime=use_default_datetime)
self.state: FATFSState = FATFSState(
sector_size=sector_size,
explicit_fat_type=explicit_fat_type,
reserved_sectors_cnt=reserved_sectors_cnt,
root_dir_sectors_cnt=root_dir_sectors_cnt,
size=size,
file_sys_type=file_sys_type,
num_heads=num_heads,
fat_tables_cnt=fat_tables_cnt,
sectors_per_cluster=sectors_per_cluster,
media_type=media_type,
hidden_sectors=hidden_sectors,
sec_per_track=sec_per_track,
long_names_enabled=long_names_enabled,
volume_label=volume_label,
oem_name=oem_name,
use_default_datetime=use_default_datetime,
)
binary_image: bytes = bytearray(
read_filesystem(binary_image_path) if binary_image_path else self.create_empty_fatfs())
read_filesystem(binary_image_path) if binary_image_path else self.create_empty_fatfs()
)
self.state.binary_image = binary_image
self.fat: FAT = FAT(boot_sector_state=self.state.boot_sector_state, init_=True)
root_dir_size = self.state.boot_sector_state.root_dir_sectors_cnt * self.state.boot_sector_state.sector_size
self.root_directory: Directory = Directory(name='A', # the name is not important, must be string
size=root_dir_size,
fat=self.fat,
cluster=self.fat.clusters[1],
fatfs_state=self.state)
self.root_directory: Directory = Directory(
name='A', # the name is not important, must be string
size=root_dir_size,
fat=self.fat,
cluster=self.fat.clusters[1],
fatfs_state=self.state,
)
self.root_directory.init_directory()
@duplicate_fat_decorator
def create_file(self, name: str,
extension: str = '',
path_from_root: Optional[List[str]] = None,
object_timestamp_: datetime = FATFS_INCEPTION,
is_empty: bool = False) -> None:
def create_file(
self,
name: str,
extension: str = '',
path_from_root: list[str] | None = None,
object_timestamp_: datetime = FATFS_INCEPTION,
is_empty: bool = False,
) -> None:
"""
This method allocates necessary clusters and creates a new file record in the directory required.
The directory must exists.
@@ -113,16 +122,18 @@ class FATFS:
:param object_timestamp_: is not None, this will be propagated to the file's entry
:param is_empty: True if there is no need to allocate any cluster, otherwise False
"""
self.root_directory.new_file(name=name,
extension=extension,
path_from_root=path_from_root,
object_timestamp_=object_timestamp_,
is_empty=is_empty)
self.root_directory.new_file(
name=name,
extension=extension,
path_from_root=path_from_root,
object_timestamp_=object_timestamp_,
is_empty=is_empty,
)
@duplicate_fat_decorator
def create_directory(self, name: str,
path_from_root: Optional[List[str]] = None,
object_timestamp_: datetime = FATFS_INCEPTION) -> None:
def create_directory(
self, name: str, path_from_root: list[str] | None = None, object_timestamp_: datetime = FATFS_INCEPTION
) -> None:
"""
Initially recursively finds a parent of the new directory
and then create a new directory inside the parent.
@@ -139,13 +150,12 @@ class FATFS:
if path_from_root:
parent_dir = self.root_directory.recursive_search(path_from_root, self.root_directory)
self.root_directory.new_directory(name=name,
parent=parent_dir,
path_from_root=path_from_root,
object_timestamp_=object_timestamp_)
self.root_directory.new_directory(
name=name, parent=parent_dir, path_from_root=path_from_root, object_timestamp_=object_timestamp_
)
@duplicate_fat_decorator
def write_content(self, path_from_root: List[str], content: bytes) -> None:
def write_content(self, path_from_root: list[str], content: bytes) -> None:
"""
fat fs invokes root directory to recursively find the required file and writes the content
"""
@@ -165,8 +175,8 @@ class FATFS:
fat_start = boot_sec_st.reserved_sectors_cnt * boot_sec_st.sector_size
fat_end = fat_start + boot_sec_st.sectors_per_fat_cnt * boot_sec_st.sector_size
second_fat_shift = boot_sec_st.sectors_per_fat_cnt * boot_sec_st.sector_size
self.state.binary_image[fat_start + second_fat_shift: fat_end + second_fat_shift] = (
self.state.binary_image[fat_start: fat_end]
self.state.binary_image[fat_start + second_fat_shift : fat_end + second_fat_shift] = (
self.state.binary_image[fat_start:fat_end]
)
def write_filesystem(self, output_path: str) -> None:
@@ -174,10 +184,9 @@ class FATFS:
output.write(bytearray(self.state.binary_image))
@duplicate_fat_decorator
def _generate_partition_from_folder(self,
folder_relative_path: str,
folder_path: str = '',
is_dir: bool = False) -> None:
def _generate_partition_from_folder(
self, folder_relative_path: str, folder_path: str = '', is_dir: bool = False
) -> None:
"""
Given path to folder and folder name recursively encodes folder into binary image.
Used by method generate.
@@ -196,17 +205,19 @@ class FATFS:
content = file.read()
file_name, extension = os.path.splitext(split_path[-1])
extension = extension[1:] # remove the dot from the extension
self.create_file(name=file_name,
extension=extension,
path_from_root=split_path[1:-1] or None,
object_timestamp_=object_timestamp,
is_empty=len(content) == 0)
self.create_file(
name=file_name,
extension=extension,
path_from_root=split_path[1:-1] or None,
object_timestamp_=object_timestamp,
is_empty=len(content) == 0,
)
self.write_content(split_path[1:], content)
elif os.path.isdir(real_path):
if not is_dir:
self.create_directory(name=split_path[-1],
path_from_root=split_path[1:-1],
object_timestamp_=object_timestamp)
self.create_directory(
name=split_path[-1], path_from_root=split_path[1:-1], object_timestamp_=object_timestamp
)
# sorting files for better testability
dir_content = list(sorted(os.listdir(real_path)))
@@ -221,11 +232,9 @@ class FATFS:
self._generate_partition_from_folder(folder_name, folder_path=path_to_folder, is_dir=True)
def calculate_min_space(path: List[str],
fs_entity: str,
sector_size: int = 0x1000,
long_file_names: bool = False,
is_root: bool = False) -> int:
def calculate_min_space(
path: list[str], fs_entity: str, sector_size: int = 0x1000, long_file_names: bool = False, is_root: bool = False
) -> int:
if os.path.isfile(os.path.join(*path, fs_entity)):
with open(os.path.join(*path, fs_entity), 'rb') as file_:
content = file_.read()
@@ -255,26 +264,33 @@ def main() -> None:
clusters = calculate_min_space([], args.input_directory, args.sector_size, long_file_names=True, is_root=True)
fats = get_fat_sectors_count(clusters, args.sector_size)
root_dir_sectors = (FATDefaults.ROOT_ENTRIES_COUNT * FATDefaults.ENTRY_SIZE) // args.sector_size
args.partition_size = max(FATFS_MIN_ALLOC_UNIT * args.sector_size,
(clusters + fats + get_non_data_sectors_cnt(RESERVED_CLUSTERS_COUNT,
fats,
args.fat_count,
root_dir_sectors)
) * args.sector_size
)
args.partition_size = max(
FATFS_MIN_ALLOC_UNIT * args.sector_size,
(
clusters
+ fats
+ get_non_data_sectors_cnt(RESERVED_CLUSTERS_COUNT, fats, args.fat_count, root_dir_sectors)
)
* args.sector_size,
)
fatfs = FATFS(size=args.partition_size,
fat_tables_cnt=args.fat_count,
sectors_per_cluster=args.sectors_per_cluster,
sector_size=args.sector_size,
long_names_enabled=args.long_name_support,
use_default_datetime=args.use_default_datetime,
root_entry_count=args.root_entry_count,
explicit_fat_type=args.fat_type)
fatfs = FATFS(
size=args.partition_size,
fat_tables_cnt=args.fat_count,
sectors_per_cluster=args.sectors_per_cluster,
sector_size=args.sector_size,
long_names_enabled=args.long_name_support,
use_default_datetime=args.use_default_datetime,
root_entry_count=args.root_entry_count,
explicit_fat_type=args.fat_type,
)
fatfs.generate(args.input_directory)
fatfs.write_filesystem(args.output_file)
if __name__ == '__main__':
from esp_pylib.excepthook import install_exception_reporting
install_exception_reporting()
main()

138
components/fatfs/fatfsparse.py
View File

@@ -1,10 +1,12 @@
#!/usr/bin/env python
# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import argparse
import os
from typing import Any
import construct
from esp_pylib.cli_options import MutuallyExclusiveOption
from esp_pylib.logger import log
from fatfs_utils.boot_sector import BootSector
from fatfs_utils.entry import Entry
from fatfs_utils.fat import FAT
@@ -24,14 +26,20 @@ def build_file_name(name1: bytes, name2: bytes, name3: bytes) -> str:
return full_name_.rstrip(FULL_BYTE).decode(LONG_NAMES_ENCODING).rstrip('\x00')
def get_obj_name(obj_: dict, directory_bytes_: bytes, entry_position_: int, lfn_checksum_: int) -> str:
def get_obj_name(
obj_: dict,
directory_bytes_: bytes,
entry_position_: int,
lfn_checksum_: int,
long_name_support: bool,
) -> str:
obj_ext_ = obj_['DIR_Name_ext'].rstrip(chr(PAD_CHAR))
ext_ = f'.{obj_ext_}' if len(obj_ext_) > 0 else ''
obj_name_: str = obj_['DIR_Name'].rstrip(chr(PAD_CHAR)) + ext_ # short entry name
# if LFN was detected, the record is considered as single SFN record only if DIR_NTRes == 0x18 (LDIR_DIR_NTRES)
# if LFN was not detected, the record cannot be part of the LFN, no matter the value of DIR_NTRes
if not args.long_name_support or obj_['DIR_NTRes'] == Entry.LDIR_DIR_NTRES:
if not long_name_support or obj_['DIR_NTRes'] == Entry.LDIR_DIR_NTRES:
return obj_name_
full_name = {}
@@ -48,12 +56,18 @@ def get_obj_name(obj_: dict, directory_bytes_: bytes, entry_position_: int, lfn_
def traverse_folder_tree(
directory_bytes_: bytes, name: str, state_: BootSectorState, fat_: FAT, binary_array_: bytes
) -> None:
directory_bytes_: bytes,
name: str,
state_: BootSectorState,
fat_: FAT,
binary_array_: bytes,
long_name_support: bool,
) -> bool:
os.makedirs(name)
assert len(directory_bytes_) % FATDefaults.ENTRY_SIZE == 0
entries_count_: int = len(directory_bytes_) // FATDefaults.ENTRY_SIZE
enable_long_names = long_name_support
for i in range(entries_count_):
obj_address_: int = FATDefaults.ENTRY_SIZE * i
@@ -62,7 +76,7 @@ def traverse_folder_tree(
directory_bytes_[obj_address_ : obj_address_ + FATDefaults.ENTRY_SIZE]
)
except (construct.core.ConstError, UnicodeDecodeError, construct.core.StringError):
args.long_name_support = True
enable_long_names = True
continue
if obj_['DIR_Attr'] == 0: # empty entry
@@ -73,6 +87,7 @@ def traverse_folder_tree(
directory_bytes_,
entry_position_=i,
lfn_checksum_=lfn_checksum(obj_['DIR_Name'] + obj_['DIR_Name_ext']),
long_name_support=enable_long_names,
)
if obj_['DIR_Attr'] == Entry.ATTR_ARCHIVE:
content_ = b''
@@ -85,14 +100,17 @@ def traverse_folder_tree(
if obj_name_ in ('.', '..'):
continue
child_directory_bytes_ = fat_.get_chained_content(cluster_id_=obj_['DIR_FstClusLO'])
traverse_folder_tree(
enable_long_names = traverse_folder_tree(
directory_bytes_=child_directory_bytes_,
name=os.path.join(name, obj_name_),
state_=state_,
fat_=fat_,
binary_array_=binary_array_,
long_name_support=enable_long_names,
)
return enable_long_names
def remove_wear_levelling_if_exists(fs_: bytes) -> bytes:
"""
@@ -113,57 +131,24 @@ def remove_wear_levelling_if_exists(fs_: bytes) -> bytes:
return plain_fs
if __name__ == '__main__':
desc = 'Tool for parsing fatfs image and extracting directory structure on host.'
argument_parser: argparse.ArgumentParser = argparse.ArgumentParser(description=desc)
argument_parser.add_argument('input_image', help='Path to the image that will be parsed and extracted.')
argument_parser.add_argument('--long-name-support', action='store_true', help=argparse.SUPPRESS)
def _parse_fatfs_image(
input_image: str,
long_name_support: bool,
wl_layer: str,
verbose: bool,
) -> None:
fs = read_filesystem(input_image)
# ensures backward compatibility
argument_parser.add_argument('--wear-leveling', action='store_true', help=argparse.SUPPRESS)
argument_parser.add_argument(
'--wl-layer',
choices=['detect', 'enabled', 'disabled'],
default=None,
help="If detection doesn't work correctly, you can force analyzer to or not to assume WL.",
)
argument_parser.add_argument('--verbose', action='store_true', help='Prints details about FAT image.')
args = argument_parser.parse_args()
# if wear levelling is detected or user explicitly sets the parameter `--wl_layer enabled`
# the partition with wear levelling is transformed to partition without WL for convenient parsing
# in some cases the partitions with and without wear levelling can be 100% equivalent
# and only user can break this tie by explicitly setting
# the parameter --wl-layer to enabled, respectively disabled
if args.wear_leveling and args.wl_layer:
raise NotImplementedError('Argument --wear-leveling cannot be combined with --wl-layer!')
if args.wear_leveling:
args.wl_layer = 'enabled'
args.wl_layer = args.wl_layer or 'detect'
fs = read_filesystem(args.input_image)
# An algorithm for removing wear levelling:
# 1. find an remove dummy sector:
# a) dummy sector is at the position defined by the number of records in the state sector
# b) dummy may not be placed in state nor cfg sectors
# c) first (boot) sector position (boot_s_pos) is calculated using value of move count
# boot_s_pos = - mc
# 2. remove state sectors (trivial)
# 3. remove cfg sector (trivial)
# 4. valid fs is then old_fs[-mc:] + old_fs[:-mc]
if args.wl_layer == 'enabled':
if wl_layer == 'enabled':
fs = remove_wl(fs)
elif args.wl_layer != 'disabled':
# wear levelling is removed to enable parsing using common algorithm
elif wl_layer != 'disabled':
fs = remove_wear_levelling_if_exists(fs)
boot_sector_ = BootSector()
boot_sector_.parse_boot_sector(fs)
if args.verbose:
print(str(boot_sector_))
if verbose:
log.print(str(boot_sector_))
fat = FAT(boot_sector_.boot_sector_state, init_=False)
@@ -176,4 +161,55 @@ if __name__ == '__main__':
boot_sector_.boot_sector_state,
fat,
fs,
long_name_support=long_name_support,
)
def _build_cli() -> Any:
import rich_click as click
desc = 'Tool for parsing fatfs image and extracting directory structure on host.'
@click.command(
context_settings={'help_option_names': ['-h', '--help']},
help=desc,
)
@click.argument('input_image', type=click.Path(exists=False, dir_okay=False))
@click.option('--long-name-support', is_flag=True, default=False, hidden=True)
@click.option(
'--wear-leveling',
is_flag=True,
default=False,
hidden=True,
cls=MutuallyExclusiveOption,
exclusive_with=['wl_layer'],
)
@click.option(
'--wl-layer',
type=click.Choice(['detect', 'enabled', 'disabled'], case_sensitive=False),
default=None,
help="If detection doesn't work correctly, you can force analyzer to or not to assume WL.",
cls=MutuallyExclusiveOption,
exclusive_with=['wear_leveling'],
)
@click.option('--verbose', is_flag=True, default=False, help='Prints details about FAT image.')
def cli(input_image, long_name_support, wear_leveling, wl_layer, verbose): # type: ignore[no-untyped-def]
if wear_leveling and wl_layer is not None:
log.die('Argument --wear-leveling cannot be combined with --wl-layer!')
if wear_leveling:
wl_layer = 'enabled'
wl_layer = wl_layer or 'detect'
_parse_fatfs_image(input_image, long_name_support, wl_layer, verbose)
return cli
def main() -> None:
_build_cli()()
if __name__ == '__main__':
from esp_pylib.excepthook import install_exception_reporting
install_exception_reporting()
main()

14
components/fatfs/test_fatfsgen/test_fatfsgen.py
View File

@@ -737,19 +737,17 @@ class FatFSGen(unittest.TestCase):
def test_boundary_clusters12(self) -> None:
output: bytes = check_output(
['python', '../fatfsgen.py', '--partition_size', '16732160', 'test_dir'], stderr=STDOUT
)
self.assertEqual(
output, b'WARNING: It is not recommended to create FATFS with bounding count of clusters: 4085 or 65525\n'
[sys.executable, '../fatfsgen.py', '--partition_size', '16732160', 'test_dir'], stderr=STDOUT
)
self.assertIn(b'WARNING:', output)
self.assertIn(b'4085 or 65525', output)
def test_boundary_clusters16(self) -> None:
output: bytes = check_output(
['python', '../fatfsgen.py', '--partition_size', '268390400', 'test_dir'], stderr=STDOUT
)
self.assertEqual(
output, b'WARNING: It is not recommended to create FATFS with bounding count of clusters: 4085 or 65525\n'
[sys.executable, '../fatfsgen.py', '--partition_size', '268390400', 'test_dir'], stderr=STDOUT
)
self.assertIn(b'WARNING:', output)
self.assertIn(b'4085 or 65525', output)
def test_boundary_clusters_fat32(self) -> None:
self.assertRaises(NotImplementedError, fatfsgen.FATFS, size=268419193)

9
components/fatfs/wl_fatfsgen.py
View File

@@ -227,7 +227,7 @@ class WLFATFS:
output.write(bytearray(self.fatfs_binary_image))
if __name__ == '__main__':
def main() -> None:
desc = 'Create a FAT filesystem with support for wear levelling and populate it with directory content'
args = get_args_for_partition_generator(desc, wl=True)
wl_fatfs = WLFATFS(
@@ -245,3 +245,10 @@ if __name__ == '__main__':
wl_fatfs.plain_fatfs.generate(args.input_directory)
wl_fatfs.init_wl()
wl_fatfs.wl_write_filesystem(args.output_file)
if __name__ == '__main__':
from esp_pylib.excepthook import install_exception_reporting
install_exception_reporting()
main()