fix(fatfs): move readdir-stat cache to per-DIR stream

Move cached_fileinfo and dir_path from vfs_fat_ctx_t to vfs_fat_dir_t so
each open DIR* has its own readdir→stat cache.
This commit is contained in:
sonika.rathi
2026-07-08 12:28:36 +02:00
parent 0581c1e046
commit 2d6ac630a5
5 changed files with 410 additions and 53 deletions

View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
* SPDX-FileCopyrightText: 2015-2026 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
@@ -281,6 +281,34 @@ TEST_CASE("(WL) readdir, stat work as expected", "[fatfs][wear_levelling]")
test_teardown();
}
TEST_CASE("(WL) readdir stat cache with two open directories", "[fatfs][wear_levelling]")
{
test_setup();
test_fatfs_readdir_stat_dual_opendir("/spiflash/readdir_cache");
test_teardown();
}
TEST_CASE("(WL) readdir stat cache stale after truncate", "[fatfs][wear_levelling]")
{
test_setup();
test_fatfs_readdir_stat_stale_after_truncate("/spiflash/readdir_cache");
test_teardown();
}
TEST_CASE("(WL) readdir stat cache stale after unlink", "[fatfs][wear_levelling]")
{
test_setup();
test_fatfs_readdir_stat_stale_after_unlink("/spiflash/readdir_cache");
test_teardown();
}
TEST_CASE("(WL) readdir stat cache concurrent dual opendir", "[fatfs][wear_levelling]")
{
test_setup();
test_fatfs_readdir_stat_concurrent_dual_opendir("/spiflash/readdir_cache");
test_teardown();
}
TEST_CASE("(WL) opendir, readdir, rewinddir, seekdir work as expected", "[fatfs][wear_levelling]")
{
test_setup();

View File

@@ -1,2 +1,3 @@
CONFIG_COMPILER_OPTIMIZATION_SIZE=y
CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_SILENT=y
CONFIG_ESP_MAIN_TASK_STACK_SIZE=4096

View File

@@ -818,6 +818,251 @@ void test_fatfs_readdir_stat(const char* dir_prefix)
}
}
void test_fatfs_readdir_stat_dual_opendir(const char* base_prefix)
{
char dir_a[128];
char dir_b[128];
char file_a[320];
char file_b[320];
snprintf(dir_a, sizeof(dir_a), "%s/dirA", base_prefix);
snprintf(dir_b, sizeof(dir_b), "%s/dirB", base_prefix);
snprintf(file_a, sizeof(file_a), "%s/same_name.txt", dir_a);
snprintf(file_b, sizeof(file_b), "%s/same_name.txt", dir_b);
unlink(file_a);
unlink(file_b);
rmdir(dir_a);
rmdir(dir_b);
rmdir(base_prefix);
TEST_ASSERT_EQUAL(0, mkdir(base_prefix, 0755));
TEST_ASSERT_EQUAL(0, mkdir(dir_a, 0755));
TEST_ASSERT_EQUAL(0, mkdir(dir_b, 0755));
test_fatfs_create_file_with_text(file_a, "AAAA");
test_fatfs_create_file_with_text(file_b, "BBBBBBBB");
DIR* dir_a_handle = opendir(dir_a);
DIR* dir_b_handle = opendir(dir_b);
TEST_ASSERT_NOT_NULL(dir_a_handle);
TEST_ASSERT_NOT_NULL(dir_b_handle);
struct dirent* de = NULL;
bool found = false;
while ((de = readdir(dir_a_handle)) != NULL) {
if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) {
continue;
}
TEST_ASSERT_EQUAL_STRING("same_name.txt", de->d_name);
found = true;
break;
}
TEST_ASSERT_TRUE(found);
struct stat st = {0};
TEST_ASSERT_EQUAL(0, stat(file_b, &st));
TEST_ASSERT_EQUAL(8, st.st_size);
TEST_ASSERT_EQUAL(0, closedir(dir_a_handle));
TEST_ASSERT_EQUAL(0, closedir(dir_b_handle));
unlink(file_a);
unlink(file_b);
rmdir(dir_a);
rmdir(dir_b);
rmdir(base_prefix);
}
void test_fatfs_readdir_stat_stale_after_truncate(const char* dir_prefix)
{
char filepath[320];
snprintf(filepath, sizeof(filepath), "%s/stale_truncate.txt", dir_prefix);
unlink(filepath);
rmdir(dir_prefix);
TEST_ASSERT_EQUAL(0, mkdir(dir_prefix, 0755));
test_fatfs_create_file_with_text(filepath, "0123456789");
DIR* dir = opendir(dir_prefix);
TEST_ASSERT_NOT_NULL(dir);
struct dirent* de = NULL;
bool found = false;
while ((de = readdir(dir)) != NULL) {
if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) {
continue;
}
TEST_ASSERT_EQUAL_STRING("stale_truncate.txt", de->d_name);
found = true;
break;
}
TEST_ASSERT_TRUE(found);
TEST_ASSERT_EQUAL(0, truncate(filepath, 4));
struct stat st = {0};
TEST_ASSERT_EQUAL(0, stat(filepath, &st));
TEST_ASSERT_EQUAL(4, st.st_size);
TEST_ASSERT_EQUAL(0, closedir(dir));
unlink(filepath);
rmdir(dir_prefix);
}
void test_fatfs_readdir_stat_stale_after_unlink(const char* dir_prefix)
{
char filepath[320];
snprintf(filepath, sizeof(filepath), "%s/stale_unlink.txt", dir_prefix);
unlink(filepath);
rmdir(dir_prefix);
TEST_ASSERT_EQUAL(0, mkdir(dir_prefix, 0755));
test_fatfs_create_file_with_text(filepath, "gone");
DIR* dir = opendir(dir_prefix);
TEST_ASSERT_NOT_NULL(dir);
struct dirent* de = NULL;
bool found = false;
while ((de = readdir(dir)) != NULL) {
if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) {
continue;
}
TEST_ASSERT_EQUAL_STRING("stale_unlink.txt", de->d_name);
found = true;
break;
}
TEST_ASSERT_TRUE(found);
TEST_ASSERT_EQUAL(0, unlink(filepath));
struct stat st = {0};
TEST_ASSERT_EQUAL(-1, stat(filepath, &st));
TEST_ASSERT_EQUAL(ENOENT, errno);
TEST_ASSERT_EQUAL(0, closedir(dir));
rmdir(dir_prefix);
}
typedef struct {
const char* dir_path;
size_t expected_size;
SemaphoreHandle_t done;
esp_err_t result;
} readdir_stat_task_arg_t;
static void readdir_stat_task(void* param)
{
readdir_stat_task_arg_t* arg = (readdir_stat_task_arg_t*) param;
char filepath[320];
struct stat st = {0};
struct dirent* de;
DIR* dir = opendir(arg->dir_path);
if (dir == NULL) {
arg->result = ESP_FAIL;
xSemaphoreGive(arg->done);
vTaskDelete(NULL);
return;
}
while ((de = readdir(dir)) != NULL) {
if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) {
continue;
}
snprintf(filepath, sizeof(filepath), "%s/%s", arg->dir_path, de->d_name);
if (stat(filepath, &st) != 0) {
arg->result = ESP_FAIL;
closedir(dir);
xSemaphoreGive(arg->done);
vTaskDelete(NULL);
return;
}
if ((size_t) st.st_size != arg->expected_size) {
arg->result = ESP_FAIL;
closedir(dir);
xSemaphoreGive(arg->done);
vTaskDelete(NULL);
return;
}
}
closedir(dir);
arg->result = ESP_OK;
xSemaphoreGive(arg->done);
vTaskDelete(NULL);
}
void test_fatfs_readdir_stat_concurrent_dual_opendir(const char* base_prefix)
{
char dir_a[128];
char dir_b[128];
char file_a[320];
char file_b[320];
snprintf(dir_a, sizeof(dir_a), "%s/conc_dirA", base_prefix);
snprintf(dir_b, sizeof(dir_b), "%s/conc_dirB", base_prefix);
snprintf(file_a, sizeof(file_a), "%s/same_name.txt", dir_a);
snprintf(file_b, sizeof(file_b), "%s/same_name.txt", dir_b);
unlink(file_a);
unlink(file_b);
rmdir(dir_a);
rmdir(dir_b);
rmdir(base_prefix);
TEST_ASSERT_EQUAL(0, mkdir(base_prefix, 0755));
TEST_ASSERT_EQUAL(0, mkdir(dir_a, 0755));
TEST_ASSERT_EQUAL(0, mkdir(dir_b, 0755));
test_fatfs_create_file_with_text(file_a, "AAAA");
test_fatfs_create_file_with_text(file_b, "BBBBBBBB");
readdir_stat_task_arg_t args_a = {
.dir_path = dir_a,
.expected_size = 4,
.done = xSemaphoreCreateBinary(),
.result = ESP_FAIL,
};
readdir_stat_task_arg_t args_b = {
.dir_path = dir_b,
.expected_size = 8,
.done = xSemaphoreCreateBinary(),
.result = ESP_FAIL,
};
TEST_ASSERT_NOT_NULL(args_a.done);
TEST_ASSERT_NOT_NULL(args_b.done);
const int stack_size = 4096;
const int cpuid_0 = 0;
const int cpuid_1 = CONFIG_FREERTOS_NUMBER_OF_CORES - 1;
xTaskCreatePinnedToCore(readdir_stat_task, "readdir_stat_a", stack_size, &args_a, 3, NULL, cpuid_0);
xTaskCreatePinnedToCore(readdir_stat_task, "readdir_stat_b", stack_size, &args_b, 3, NULL, cpuid_1);
xSemaphoreTake(args_a.done, portMAX_DELAY);
xSemaphoreTake(args_b.done, portMAX_DELAY);
TEST_ASSERT_EQUAL(ESP_OK, args_a.result);
TEST_ASSERT_EQUAL(ESP_OK, args_b.result);
vSemaphoreDelete(args_a.done);
vSemaphoreDelete(args_b.done);
unlink(file_a);
unlink(file_b);
rmdir(dir_a);
rmdir(dir_b);
rmdir(base_prefix);
}
void test_fatfs_opendir_readdir_rewinddir(const char* dir_prefix)
{
char name_dir_inner_file[64];

View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2015-2022 Espressif Systems (Shanghai) CO LTD
* SPDX-FileCopyrightText: 2015-2026 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
@@ -82,3 +82,11 @@ void test_fatfs_create_contiguous_file(const char* base_path, const char* full_p
#endif
void test_fatfs_readdir_stat(const char* path);
void test_fatfs_readdir_stat_dual_opendir(const char* base_prefix);
void test_fatfs_readdir_stat_stale_after_truncate(const char* dir_prefix);
void test_fatfs_readdir_stat_stale_after_unlink(const char* dir_prefix);
void test_fatfs_readdir_stat_concurrent_dual_opendir(const char* base_prefix);

View File

@@ -14,6 +14,9 @@
#include <sys/fcntl.h>
#include <sys/lock.h>
#include "esp_vfs_fat.h"
#ifdef CONFIG_VFS_SUPPORT_DIR
#include <sys/queue.h>
#endif
#include "vfs_fat_internal.h"
#include "esp_vfs.h"
#include "esp_log.h"
@@ -23,21 +26,32 @@
#define F_WRITE_MALLOC_ZEROING_BUF_SIZE_LIMIT 512
#ifdef CONFIG_VFS_SUPPORT_DIR
struct cached_data{
#if FF_USE_LFN
char file_path[FILENAME_MAX+1+FF_LFN_BUF+1]; //FILENAME_MAX+1: for dir_path, FF_LFN_BUF+1: for file name
#else
char file_path[FILENAME_MAX+1+FF_SFN_BUF+1]; //FILENAME_MAX+1: for dir_path, FF_LFN_BUF+1: for file name
#endif
FILINFO fileinfo;
};
#endif // CONFIG_VFS_SUPPORT_DIR
#if !defined(FILENAME_MAX)
#define FILENAME_MAX 255
#endif
#ifdef CONFIG_VFS_SUPPORT_DIR
struct cached_data {
FILINFO cached_fileinfo;
char *cached_file_path;
size_t cached_file_path_size;
};
typedef struct vfs_fat_dir_t {
DIR dir;
long offset;
FF_DIR ffdir;
FILINFO filinfo;
struct dirent cur_dirent;
const char *dir_path;
struct cached_data cached_data;
SLIST_ENTRY(vfs_fat_dir_t) open_dirs_entry;
char path_buf[]; /* dir_path, then room for one cached entry path */
} vfs_fat_dir_t;
SLIST_HEAD(vfs_fat_open_dirs, vfs_fat_dir_t);
#endif // CONFIG_VFS_SUPPORT_DIR
typedef struct {
char fat_drive[8]; /* FAT drive name */
char base_path[ESP_VFS_PATH_MAX]; /* base path in VFS where partition is registered */
@@ -48,20 +62,11 @@ typedef struct {
char tmp_path_buf2[FILENAME_MAX+3]; /* as above; used in functions which take two path arguments */
uint32_t *flags; /* file descriptor flags, array of max_files size */
#ifdef CONFIG_VFS_SUPPORT_DIR
char dir_path[FILENAME_MAX]; /* variable to store path of opened directory*/
struct cached_data cached_fileinfo;
struct vfs_fat_open_dirs open_dirs; /* list of open directory streams */
#endif
FIL files[]; /* array with max_files entries; must be the final member of the structure */
} vfs_fat_ctx_t;
typedef struct {
DIR dir;
long offset;
FF_DIR ffdir;
FILINFO filinfo;
struct dirent cur_dirent;
} vfs_fat_dir_t;
/* Date and time storage formats in FAT */
typedef union {
struct {
@@ -201,6 +206,9 @@ esp_err_t esp_vfs_fat_register(const esp_vfs_fat_conf_t* conf, FATFS** out_fs)
return ESP_ERR_NO_MEM;
}
memset(fat_ctx, 0, ctx_size);
#ifdef CONFIG_VFS_SUPPORT_DIR
SLIST_INIT(&fat_ctx->open_dirs);
#endif
fat_ctx->flags = ff_memalloc(max_files * sizeof(*fat_ctx->flags));
if (fat_ctx->flags == NULL) {
free(fat_ctx);
@@ -765,6 +773,74 @@ static void update_stat_struct(struct stat *st, FILINFO *info)
st->st_ctime = 0;
}
static vfs_fat_dir_t *vfs_fat_dir_alloc(const char *dir_path)
{
const size_t dir_len = strlen(dir_path);
#if FF_USE_LFN
const size_t buf_size = dir_len + 1 + FF_LFN_BUF + 2;
#else
const size_t buf_size = dir_len + 1 + FF_SFN_BUF + 2;
#endif
vfs_fat_dir_t *fat_dir = ff_memalloc(sizeof(vfs_fat_dir_t) + buf_size);
if (fat_dir == NULL) {
return NULL;
}
memset(fat_dir, 0, sizeof(*fat_dir));
strcpy(fat_dir->path_buf, dir_path);
fat_dir->dir_path = fat_dir->path_buf;
fat_dir->cached_data.cached_file_path = fat_dir->path_buf + dir_len + 1;
fat_dir->cached_data.cached_file_path_size = buf_size - (dir_len + 1);
return fat_dir;
}
static void vfs_fat_clear_dir_stat_cache(vfs_fat_dir_t *fat_dir)
{
memset(&fat_dir->cached_data.cached_fileinfo, 0, sizeof(fat_dir->cached_data.cached_fileinfo));
if (fat_dir->cached_data.cached_file_path != NULL) {
fat_dir->cached_data.cached_file_path[0] = '\0';
}
}
static void vfs_fat_invalidate_stat_cache(vfs_fat_ctx_t *fat_ctx, const char *path)
{
vfs_fat_dir_t *fat_dir;
SLIST_FOREACH(fat_dir, &fat_ctx->open_dirs, open_dirs_entry) {
if (fat_dir->cached_data.cached_file_path != NULL &&
fat_dir->cached_data.cached_file_path[0] != '\0' &&
strcmp(path, fat_dir->cached_data.cached_file_path) == 0) {
vfs_fat_clear_dir_stat_cache(fat_dir);
}
}
}
static bool vfs_fat_consume_stat_cache(vfs_fat_ctx_t *fat_ctx, const char *path, struct stat *st)
{
vfs_fat_dir_t *fat_dir;
SLIST_FOREACH(fat_dir, &fat_ctx->open_dirs, open_dirs_entry) {
if (fat_dir->cached_data.cached_file_path != NULL &&
fat_dir->cached_data.cached_file_path[0] != '\0' &&
strcmp(path, fat_dir->cached_data.cached_file_path) == 0) {
update_stat_struct(st, &fat_dir->cached_data.cached_fileinfo);
vfs_fat_clear_dir_stat_cache(fat_dir);
return true;
}
}
return false;
}
static void vfs_fat_cache_readdir_entry(vfs_fat_dir_t *fat_dir)
{
vfs_fat_clear_dir_stat_cache(fat_dir);
if (strcmp(fat_dir->dir_path, "/") == 0) {
snprintf(fat_dir->cached_data.cached_file_path, fat_dir->cached_data.cached_file_path_size,
"/%s", fat_dir->filinfo.fname);
} else {
snprintf(fat_dir->cached_data.cached_file_path, fat_dir->cached_data.cached_file_path_size,
"%s/%s", fat_dir->dir_path, fat_dir->filinfo.fname);
}
fat_dir->cached_data.cached_fileinfo = fat_dir->filinfo;
}
static int vfs_fat_stat(void* ctx, const char * path, struct stat * st)
{
if (strcmp(path, "/") == 0) {
@@ -778,16 +854,12 @@ static int vfs_fat_stat(void* ctx, const char * path, struct stat * st)
vfs_fat_ctx_t* fat_ctx = (vfs_fat_ctx_t*) ctx;
//If fileinfo is already cached by readdir for requested filename,
//then return the same info else obtain fileinfo with f_stat function
if (strcmp(path, fat_ctx->cached_fileinfo.file_path) == 0) {
update_stat_struct(st, &fat_ctx->cached_fileinfo.fileinfo);
memset(&fat_ctx->cached_fileinfo, 0 ,sizeof(FILINFO));
_lock_acquire(&fat_ctx->lock);
if (vfs_fat_consume_stat_cache(fat_ctx, path, st)) {
_lock_release(&fat_ctx->lock);
return 0;
}
memset(&fat_ctx->cached_fileinfo, 0 ,sizeof(fat_ctx->cached_fileinfo));
_lock_acquire(&fat_ctx->lock);
prepend_drive_to_path(fat_ctx, &path, NULL);
FILINFO info;
FRESULT res = f_stat(path, &info);
@@ -806,6 +878,7 @@ static int vfs_fat_unlink(void* ctx, const char *path)
{
vfs_fat_ctx_t* fat_ctx = (vfs_fat_ctx_t*) ctx;
_lock_acquire(&fat_ctx->lock);
vfs_fat_invalidate_stat_cache(fat_ctx, path);
prepend_drive_to_path(fat_ctx, &path, NULL);
FRESULT res = f_unlink(path);
_lock_release(&fat_ctx->lock);
@@ -907,6 +980,8 @@ static int vfs_fat_rename(void* ctx, const char *src, const char *dst)
{
vfs_fat_ctx_t* fat_ctx = (vfs_fat_ctx_t*) ctx;
_lock_acquire(&fat_ctx->lock);
vfs_fat_invalidate_stat_cache(fat_ctx, src);
vfs_fat_invalidate_stat_cache(fat_ctx, dst);
prepend_drive_to_path(fat_ctx, &src, &dst);
FRESULT res = f_rename(src, dst);
_lock_release(&fat_ctx->lock);
@@ -921,32 +996,41 @@ static int vfs_fat_rename(void* ctx, const char *src, const char *dst)
static DIR* vfs_fat_opendir(void* ctx, const char* name)
{
vfs_fat_ctx_t* fat_ctx = (vfs_fat_ctx_t*) ctx;
strlcpy(fat_ctx->dir_path, name, sizeof(fat_ctx->dir_path));
_lock_acquire(&fat_ctx->lock);
prepend_drive_to_path(fat_ctx, &name, NULL);
vfs_fat_dir_t* fat_dir = ff_memalloc(sizeof(vfs_fat_dir_t));
vfs_fat_dir_t* fat_dir = vfs_fat_dir_alloc(name);
if (!fat_dir) {
_lock_release(&fat_ctx->lock);
errno = ENOMEM;
return NULL;
}
memset(fat_dir, 0, sizeof(*fat_dir));
prepend_drive_to_path(fat_ctx, &name, NULL);
FRESULT res = f_opendir(&fat_dir->ffdir, name);
_lock_release(&fat_ctx->lock);
if (res != FR_OK) {
_lock_release(&fat_ctx->lock);
free(fat_dir);
ESP_LOGD(TAG, "%s: fresult=%d", __func__, res);
errno = fresult_to_errno(res);
return NULL;
}
SLIST_INSERT_HEAD(&fat_ctx->open_dirs, fat_dir, open_dirs_entry);
_lock_release(&fat_ctx->lock);
return (DIR*) fat_dir;
}
static int vfs_fat_closedir(void* ctx, DIR* pdir)
{
assert(pdir);
vfs_fat_ctx_t* fat_ctx = (vfs_fat_ctx_t*) ctx;
vfs_fat_dir_t* fat_dir = (vfs_fat_dir_t*) pdir;
_lock_acquire(&fat_ctx->lock);
SLIST_REMOVE(&fat_ctx->open_dirs, fat_dir, vfs_fat_dir_t, open_dirs_entry);
vfs_fat_clear_dir_stat_cache(fat_dir);
_lock_release(&fat_ctx->lock);
FRESULT res = f_closedir(&fat_dir->ffdir);
free(pdir);
if (res != FR_OK) {
@@ -969,25 +1053,15 @@ static struct dirent* vfs_fat_readdir(void* ctx, DIR* pdir)
errno = err;
return NULL;
}
//Store the FILEINFO in the cached_fileinfo. If the stat function is invoked immediately afterward,
//the cached_fileinfo will provide the FILEINFO directly, as it was already obtained during the readdir operation.
//During directory size calculation, this optimization can reduce the computation time.
memset(&fat_ctx->cached_fileinfo, 0 ,sizeof(fat_ctx->cached_fileinfo));
if (strcmp(fat_ctx->dir_path, "/") == 0) {
snprintf(fat_ctx->cached_fileinfo.file_path, sizeof(fat_ctx->cached_fileinfo.file_path),
"/%s", fat_dir->filinfo.fname);
} else {
char *temp_file_path = (char*) ff_memalloc(sizeof(fat_ctx->cached_fileinfo.file_path));
if (temp_file_path == NULL) {
return out_dirent;
}
snprintf(temp_file_path, sizeof(fat_ctx->cached_fileinfo.file_path),
"%s/%s", fat_ctx->dir_path, fat_dir->filinfo.fname);
memcpy(fat_ctx->cached_fileinfo.file_path, temp_file_path, sizeof(fat_ctx->cached_fileinfo.file_path));
ff_memfree(temp_file_path);
if (out_dirent == NULL) {
return NULL;
}
fat_ctx->cached_fileinfo.fileinfo = fat_dir->filinfo;
/* Cache FILINFO per directory stream so a following stat() on the same path
* can reuse data already obtained by readdir (see GH #10220). */
_lock_acquire(&fat_ctx->lock);
vfs_fat_cache_readdir_entry(fat_dir);
_lock_release(&fat_ctx->lock);
return out_dirent;
}
@@ -1174,6 +1248,7 @@ static int vfs_fat_truncate(void* ctx, const char *path, off_t length)
}
_lock_acquire(&fat_ctx->lock);
vfs_fat_invalidate_stat_cache(fat_ctx, path);
prepend_drive_to_path(fat_ctx, &path, NULL);
file = (FIL*) ff_memalloc(sizeof(FIL));