From 56a01ca822f3cf6656e9ab4fca818810302a3340 Mon Sep 17 00:00:00 2001 From: "radek.tandler" Date: Thu, 4 Jun 2026 15:00:45 +0200 Subject: [PATCH] docs(nvs_flash): improved description of NVS space consumption - Improved description of space required to store data types into NVS partition - Example showing nvs statistics extended with a code demonstrating the fragmentation effect --- docs/en/api-reference/storage/nvs_flash.rst | 70 ++++-- examples/storage/nvs/nvs_statistics/README.md | 150 ++++++------ .../nvs/nvs_statistics/main/Kconfig.projbuild | 21 ++ .../main/nvs_statistics_example.c | 223 +++++++++++++++++- .../storage/nvs/nvs_statistics/partitions.csv | 8 + .../nvs_statistics/pytest_nvs_statistics.py | 7 +- .../nvs/nvs_statistics/sdkconfig.defaults | 3 + 7 files changed, 397 insertions(+), 85 deletions(-) create mode 100644 examples/storage/nvs/nvs_statistics/main/Kconfig.projbuild create mode 100644 examples/storage/nvs/nvs_statistics/partitions.csv create mode 100644 examples/storage/nvs/nvs_statistics/sdkconfig.defaults diff --git a/docs/en/api-reference/storage/nvs_flash.rst b/docs/en/api-reference/storage/nvs_flash.rst index 8f1d28a07b9..736b5695262 100644 --- a/docs/en/api-reference/storage/nvs_flash.rst +++ b/docs/en/api-reference/storage/nvs_flash.rst @@ -32,31 +32,44 @@ Keys and Values NVS operates on key-value pairs. Keys are ASCII strings; the maximum key length is currently 15 characters. Values can have one of the following types: -- integer types: ``uint8_t``, ``int8_t``, ``uint16_t``, ``int16_t``, ``uint32_t``, ``int32_t``, ``uint64_t``, ``int64_t`` -- zero-terminated string -- variable length binary data (blob) -- floating point types: ``float`` and ``double`` +- integer types: ``uint8_t``, ``int8_t``, ``uint16_t``, ``int16_t``, ``uint32_t``, ``int32_t``, ``uint64_t``, ``int64_t`` +- floating point types: ``float`` and ``double`` +- zero-terminated C-like string +- variable length binary data - blob .. note:: - NVS works best for storing many small values, rather than a few large values of the type ``string`` and ``blob``. If you need to store large blobs or strings, consider using the facilities provided by the FAT filesystem on top of the wear levelling library. - -.. note:: - - String values are currently limited to 4000 bytes. This includes the null terminator. Blob values are limited to 508,000 bytes or 97.6% of the partition size - 4000 bytes, whichever is lower. - -.. note:: - - Before setting new or updating existing key-value pair, free entries in nvs pages have to be available. For integer types, at least one free entry has to be available. For the string value, at least one page capable of keeping the whole string in a contiguous row of free entries has to be available. For the blob value, the size of new data has to be available in free entries. + NVS works best for storing a moderate, relatively stable set of small values — such as device configuration, calibration data or state flags — rather than a few large ``string`` or ``blob`` values. "Small values" here does not imply that NVS is a good fit for continuously growing or frequently rewritten datasets, such as event logs or periodic running measurements: accumulating such records quickly fills and fragments the partition, makes space reclaim run more often, and increases flash wear. If you need to store large blobs or strings, or to keep appending data over time, consider using one of the filesystems available in ESP-IDF instead. .. note:: The floating point types ``float`` and ``double`` are supported regardless of the FPU presence on a particular SoC. -Keys are required to be unique. Assigning a new value to an existing key replaces the old value and data type with the value and data type specified by a write operation. +Keys must be unique within their namespace. Writing a new value to an existing key replaces the previous key-value pair. The actual data type is determined by the most recent write operation. -A data type check is performed when reading a value. An error is returned if the data type expected by read operation does not match the data type of entry found for the key provided. +A data type check is performed when reading a value. An error ``ESP_ERR_NVS_TYPE_MISMATCH`` is returned if the data type expected by the read operation does not match the data type of the entry found for the key provided. +Record Size Limitations +^^^^^^^^^^^^^^^^^^^^^^^ + +The maximum size of a single stored value depends on its data type: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Data type + - Maximum value size + * - Integer and floating point + - Fixed by the type (1 to 8 bytes); always stored in a single entry. + * - String + - 4000 bytes, including the null terminator. + * - Blob + - 508,000 bytes, or 97.6% of the partition size minus 4000 bytes, whichever is lower. + +.. note:: + + The string and blob limits above are absolute upper bounds valid on an empty (non-fragmented) data partition. The size that can actually be stored at run time is typically lower and depends on how the partition is fragmented. See :ref:`nvs_space_consumption` for how NVS allocates entries and why fragmentation matters. Namespaces ^^^^^^^^^^ @@ -261,6 +274,8 @@ You can find code examples in the :example:`storage/nvs` directory of ESP-IDF ex Usage statistics are obtained prior to and post writing, with the differences being compared to expected values of newly used entries. + The second part of example shows the effect of NVS partition fragmentation to the blob storage overhead. + :example:`storage/nvs/nvs_iteration` This example demonstrates how to iterate over entries of specific (or any) NVS data type and how to obtain info about such entries. @@ -281,7 +296,7 @@ NVS stores key-value pairs sequentially, with new key-value pairs being added at .. note:: - NVS component includes flash wear levelling by design. Set operations are appending new data to the free space after existing entries. Invalidation of old values doesn't require immediate flash erase operations. The organization of NVS space to pages and entries effectively reduces the frequency of flash erase to flash write operations for data types fitting one entry by a factor of 126. + The NVS component includes flash wear levelling by design. Set operations append new data to the free space after existing entries, and invalidation of old values does not trigger immediate flash erase operations. The organization of NVS space into pages and entries reduces the frequency of flash erase to flash write operations for data types fitting one entry by up to a factor of 126 in the ideal case (one page of single-entry writes per erase). The actual factor is lower in practice and is primarily driven by how full the partition is: as live data grows, NVS space reclaim runs more often, and the erase/write ratio worsens. Large, never-overwritten data chunks may occupy the same NVS page indefinitely — reclaim only selects pages that contain erased entries, so such pages never participate in the erase cycle and therefore reduce the share of flash memory subject to wear levelling. Pages and Entries ^^^^^^^^^^^^^^^^^ @@ -442,6 +457,29 @@ Data Variable length values (strings and blobs) are written into subsequent entries, 32 bytes per entry. The ``Span`` field of the first entry indicates how many entries are used. +.. _nvs_space_consumption: + +Space Consumption +^^^^^^^^^^^^^^^^^ + +NVS stores every record as one or more 32-byte entries within a 4096-byte data page (126 usable entries per page). The number of entries a value occupies, and the number of free entries required to store it, depends on the data type: + +- Integer and floating point values use one self-contained entry; a single free entry available anywhere is enough to store them. +- A string uses one metadata entry followed by ``ceil(payload_size / entry_size)`` data entries (the null terminator counts toward the payload). All of them have to be available as a contiguous run within a single page. +- A blob uses one ``BLOB_INDEX`` metadata entry plus one or more data chunks; each chunk is a metadata entry followed by its payload entries and lives on a different page. Storing a blob therefore needs ``1 + k + ceil(blob_size / entry_size)`` entries, where ``k`` is the number of pages the data is split across. + +Before setting a new key-value pair or updating an existing one, NVS looks for a page with enough free (or reclaimable) entries. The space reclaim algorithm is designed for sudden-power-off resiliency and consolidates free space on a single candidate page per call, so the largest contiguous run of available entries is determined on a per-page basis. This has two consequences: + +- The effective maximum string length is bound by the highest sum of free and deleted entries offered by any single page. +- A blob is split into chunks sized to the entries available on each page after reclaim, and the split continues until the whole value is stored. This lets NVS reuse pages with as few as two free entries, at the cost of one metadata entry per chunk. In the extreme case, the metadata overhead can exceed 100% of the payload size. + +Because of this per-page behavior, the actual limits are lower than the absolute maximums listed in `Record Size Limitations`_ and get tighter as the partition fills up and fragments. The related effect on flash endurance is described in the wear levelling note under `Log of Key-Value Pairs`_. + +.. note:: + + As it is difficult to resize the NVS partition on devices deployed in the field, make its initial size large enough to accommodate the current needs as well as the potential growth of keys or their data. It is also recommended to run a sufficient number of tests that realistically reflect the frequency of writing and updating NVS keys. Before testing a software update (e.g., via OTA), start these tests on a data partition already fragmented by the previous version of the software. + + Namespaces ^^^^^^^^^^ diff --git a/examples/storage/nvs/nvs_statistics/README.md b/examples/storage/nvs/nvs_statistics/README.md index 05a599d4ea5..0f2969f1b0e 100644 --- a/examples/storage/nvs/nvs_statistics/README.md +++ b/examples/storage/nvs/nvs_statistics/README.md @@ -18,6 +18,42 @@ Statistics obtained via [nvs_get_stats()](https://docs.espressif.com/projects/es Detailed functional description of NVS and API is provided in [documentation](https://docs.espressif.com/projects/esp-idf/en/latest/api-reference/storage/nvs_flash.html). +## Blob Storage-Overhead Measurement + +In addition to the basic statistics demonstration, the example can measure how much usable storage a blob actually consumes, taking the NVS metadata and free-space fragmentation into account. This part is enabled by default and can be turned off via `idf.py menuconfig` → *Example Configuration* → *Run NVS blob storage-overhead measurement*. + +The measurement sweeps a matrix of **partition sizes** × **blob sizes** and, for each cell, fills the partition to capacity and reports the heap demand, NVS entry usage and the resulting storage overhead. + +### Variable partition size + +The example uses a custom partition table (`partitions.csv`) that defines several NVS partitions of different sizes (`nvs_16k`, `nvs_32k`, `nvs_64k`). The measurement iterates over them using `nvs_flash_init_partition()` / `nvs_get_stats()`, so the influence of the partition size on the relative overhead becomes directly visible. + +### Variable blob size + +For each partition, blobs of increasing size (128, 256, 512, 1024, 2048 and 4096 bytes) are stored with unique keys until `nvs_set_blob()` returns `ESP_ERR_NVS_NOT_ENOUGH_SPACE`. The number of stored blobs is then compared against the theoretical (ideal) count derived from the documented per-blob entry cost: + +``` +entries_per_blob = 1 (BLOB_INDEX) + k (per-page BLOB_DATA chunk headers) + ceil(blob_size / 32) +``` + +where `k` is the number of pages the blob data is split across. + +### Worst-case fragmentation + +By design, each NVS page is a 4096-byte flash sector holding 126 usable 32-byte entries. A string occupies `1 + ceil((len + 1) / 32)` entries and must fit contiguously within a single page, while a blob may split its data into per-page chunks. + +To demonstrate the worst case, the example optionally pre-populates a partition so that **every page is filled up to its last 2 entries** (enabled via *Add a worst-case (pre-fragmented) measurement pass*). This is achieved by writing large strings: + +* the first string is sized to leave 2 free entries on page 0 while accounting for the namespace entry, +* every following string fills a fresh page to 124 entries, leaving exactly 2 free. + +After this step the partition reports a large amount of `free_entries`, but the largest contiguous run of free entries on any page is only 2. The consequences are then measured by filling the partition with blobs: + +* each blob chunk can only use 1 chunk-header + 1 data entry per page, so roughly half of the consumed space becomes metadata overhead, +* because the number of chunks per blob is bounded, large blobs may become unstoreable even though many `free_entries` remain. + +This makes the relationship between fragmentation, remaining free space and resulting overhead measurable, instead of presenting a single (best-case) number that could create false expectations. + ## How to use example ### Hardware required @@ -38,6 +74,8 @@ See the Getting Started Guide for full steps to configure and use ESP-IDF to bui ## Example Output +The first part of the output shows the basic statistics demonstration: + ``` ... I (265) nvs_statistics_example: Erasing the contents of the default NVS partition... @@ -49,71 +87,49 @@ I (485) nvs_statistics_example: Free NVS entries: 755 I (495) nvs_statistics_example: Available NVS entries: 629 I (495) nvs_statistics_example: Total NVS entries: 756 I (505) nvs_statistics_example: Namespace count: 1 -I (505) nvs_statistics_example: Writing mock data key-value pairs to NVS namespace '_mock_data'... -I (525) nvs_statistics_example: Committing data in NVS namespace '_mock_data'... -I (525) nvs_statistics_example: Getting post-commit NVS statistics... -I (525) nvs_statistics_example: NVS statistics: -I (535) nvs_statistics_example: Used NVS entries: 30 -I (535) nvs_statistics_example: Free NVS entries: 726 -I (545) nvs_statistics_example: Available NVS entries: 600 -I (545) nvs_statistics_example: Total NVS entries: 756 -I (555) nvs_statistics_example: Namespace count: 1 -I (555) nvs_statistics_example: Newly used entries match expectation. -I (565) nvs_statistics_example: Newly used entries: 29, expected: 29. -I (575) nvs_statistics_example: NVS handle for namespace '_mock_data' closed. -I (575) nvs_statistics_example: Opening Non-Volatile Storage (NVS) handle for namespace '_mock_data'... -I (585) nvs_statistics_example: Reading stored data from NVS namespace '_mock_data'... -I (595) nvs_statistics_example: Read key-value pair from NVS: 'wifi_ssid':'HomeNetwork' -I (605) nvs_statistics_example: Read key-value pair from NVS: 'wifi_pass':'MySecretPass' -I (605) nvs_statistics_example: Read key-value pair from NVS: 'dev_name':'LivingRoomThermostat' -I (615) nvs_statistics_example: Read key-value pair from NVS: 'temp_unit':'Celsius' -I (625) nvs_statistics_example: Read key-value pair from NVS: 'target_temp':'22' -I (635) nvs_statistics_example: Read key-value pair from NVS: 'eco_mode':'false' -I (635) nvs_statistics_example: Read key-value pair from NVS: 'fw_version':'1.2.3' -I (645) nvs_statistics_example: Read key-value pair from NVS: 'led_bright':'80' -I (655) nvs_statistics_example: Read key-value pair from NVS: 'auto_update':'true' -I (665) nvs_statistics_example: Read key-value pair from NVS: 'last_sync':'2025-01-01T08:00:00Z' -I (665) nvs_statistics_example: Read key-value pair from NVS: 'user_lang':'en' -I (675) nvs_statistics_example: Read key-value pair from NVS: 'long_token':'2f8c1e7b5a4d9c6e3b0f1a8e5d7c2b6f4e1a9c7b' -I (685) nvs_statistics_example: Read key-value pair from NVS: 'very_long_token':'7e2b1c9f5a4d8e3b0f1a6c7e2d9b5a4c8e1f7b2d6c3a9e5b0f1a8c7e2d9b5a4c8e1f7b2d6c3a9e5b' -I (705) nvs_statistics_example: NVS handle for namespace '_mock_data' closed. -I (705) nvs_statistics_example: Opening Non-Volatile Storage (NVS) handle for namespace '_mock_backup'... -I (715) nvs_statistics_example: Getting NVS statistics... -I (725) nvs_statistics_example: NVS statistics: -I (725) nvs_statistics_example: Used NVS entries: 31 -I (735) nvs_statistics_example: Free NVS entries: 725 -I (735) nvs_statistics_example: Available NVS entries: 599 -I (745) nvs_statistics_example: Total NVS entries: 756 -I (745) nvs_statistics_example: Namespace count: 2 -I (755) nvs_statistics_example: Writing mock data key-value pairs to NVS namespace '_mock_backup'... -I (765) nvs_statistics_example: Committing data in NVS namespace '_mock_backup'... -I (765) nvs_statistics_example: Getting post-commit NVS statistics... -I (775) nvs_statistics_example: NVS statistics: -I (775) nvs_statistics_example: Used NVS entries: 60 -I (785) nvs_statistics_example: Free NVS entries: 696 -I (785) nvs_statistics_example: Available NVS entries: 570 -I (795) nvs_statistics_example: Total NVS entries: 756 -I (795) nvs_statistics_example: Namespace count: 2 -I (805) nvs_statistics_example: Newly used entries match expectation. -I (805) nvs_statistics_example: Newly used entries: 29, expected: 29. -I (815) nvs_statistics_example: NVS handle for namespace '_mock_backup' closed. -I (825) nvs_statistics_example: Opening Non-Volatile Storage (NVS) handle for namespace '_mock_backup'... -I (835) nvs_statistics_example: Reading stored data from NVS namespace '_mock_backup'... -I (835) nvs_statistics_example: Read key-value pair from NVS: 'wifi_ssid':'HomeNetwork' -I (845) nvs_statistics_example: Read key-value pair from NVS: 'wifi_pass':'MySecretPass' -I (855) nvs_statistics_example: Read key-value pair from NVS: 'dev_name':'LivingRoomThermostat' -I (865) nvs_statistics_example: Read key-value pair from NVS: 'temp_unit':'Celsius' -I (865) nvs_statistics_example: Read key-value pair from NVS: 'target_temp':'22' -I (875) nvs_statistics_example: Read key-value pair from NVS: 'eco_mode':'false' -I (885) nvs_statistics_example: Read key-value pair from NVS: 'fw_version':'1.2.3' -I (895) nvs_statistics_example: Read key-value pair from NVS: 'led_bright':'80' -I (895) nvs_statistics_example: Read key-value pair from NVS: 'auto_update':'true' -I (905) nvs_statistics_example: Read key-value pair from NVS: 'last_sync':'2025-01-01T08:00:00Z' -I (915) nvs_statistics_example: Read key-value pair from NVS: 'user_lang':'en' -I (925) nvs_statistics_example: Read key-value pair from NVS: 'long_token':'2f8c1e7b5a4d9c6e3b0f1a8e5d7c2b6f4e1a9c7b' -I (935) nvs_statistics_example: Read key-value pair from NVS: 'very_long_token':'7e2b1c9f5a4d8e3b0f1a6c7e2d9b5a4c8e1f7b2d6c3a9e5b0f1a8c7e2d9b5a4c8e1f7b2d6c3a9e5b' -I (945) nvs_statistics_example: NVS handle for namespace '_mock_backup' closed. -I (955) nvs_statistics_example: Returning from app_main(). -I (955) main_task: Returned from app_main() ... -``` \ No newline at end of file +I (565) nvs_statistics_example: Newly used entries match expectation. +... +``` + +The second part reports the blob storage overhead for each partition / blob size, in both a pristine and a pre-fragmented partition: + +``` +... +I (1265) nvs_statistics_example: Starting NVS blob storage-overhead measurement... + +NVS BLOB TEST - 128 B (partition 'nvs_16k', pristine): +====================== +heap before NVS init: 299220 B +heap after NVS init: 255472 B (diff 43748 B) + +available heap after fill: 254100 B (diff 1372 B) +expected blobs count: 21 +stored blobs count: 20 +used_entries: 120 (3840 B) +free_entries: 6 (192 B) +total_entries: 126 (4032 B) +STORAGE OVERHEAD: 36% + +NVS BLOB TEST - 128 B (partition 'nvs_16k', fragmented): +====================== +heap before NVS init: 299220 B +heap after NVS init: 255472 B (diff 43748 B) +fragmentation strings written: 3 + +available heap after fill: 254100 B (diff 1372 B) +expected blobs count: 21 +stored blobs count: 2 +used_entries: 124 (3968 B) +free_entries: 2 (64 B) +total_entries: 126 (4032 B) +STORAGE OVERHEAD: 98% +... +I (9000) nvs_statistics_example: NVS blob storage-overhead measurement done. +I (9010) nvs_statistics_example: Returning from app_main(). +... +``` + +> The exact numbers depend on the target, partition size and blob size; the values above are illustrative. The key takeaway is the difference in `STORAGE OVERHEAD` and `stored blobs count` between the pristine and fragmented passes. + +To reset NVS data, erase the contents of flash memory using `idf.py erase-flash`, then upload the program again as described above. diff --git a/examples/storage/nvs/nvs_statistics/main/Kconfig.projbuild b/examples/storage/nvs/nvs_statistics/main/Kconfig.projbuild new file mode 100644 index 00000000000..aedccbb9f12 --- /dev/null +++ b/examples/storage/nvs/nvs_statistics/main/Kconfig.projbuild @@ -0,0 +1,21 @@ +menu "Example Configuration" + + config EXAMPLE_RUN_OVERHEAD_MEASUREMENT + bool "Run NVS blob storage-overhead measurement" + default y + help + When enabled, the example sweeps several NVS partition sizes and blob + sizes, fills each partition to capacity with blobs and reports the + resulting heap demand, entry usage and storage overhead. + + config EXAMPLE_INDUCE_FRAGMENTATION + bool "Add a worst-case (pre-fragmented) measurement pass" + depends on EXAMPLE_RUN_OVERHEAD_MEASUREMENT + default y + help + When enabled, every measured partition is first pre-populated with + large strings that leave only two free entries per NVS page. This + scatters the free space into non-coalescable gaps and demonstrates + the worst-case blob storage overhead caused by fragmentation. + +endmenu diff --git a/examples/storage/nvs/nvs_statistics/main/nvs_statistics_example.c b/examples/storage/nvs/nvs_statistics/main/nvs_statistics_example.c index 9b74d99d784..106bc52f8c7 100644 --- a/examples/storage/nvs/nvs_statistics/main/nvs_statistics_example.c +++ b/examples/storage/nvs/nvs_statistics/main/nvs_statistics_example.c @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2025-2026 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Unlicense OR CC0-1.0 */ @@ -15,7 +15,10 @@ CONDITIONS OF ANY KIND, either express or implied. */ #include +#include +#include #include +#include "sdkconfig.h" #include "freertos/FreeRTOS.h" #include "freertos/task.h" #include "esp_check.h" @@ -27,6 +30,26 @@ #define MOCK_DATA_NAMESPACE "_mock_data" #define MOCK_DATA_BACKUP_NAMESPACE "_mock_backup" +/* NVS on-flash geometry. These are by-design constants of the NVS format (one + * 4096-byte flash sector per page, 32-byte entries, 126 usable entries per + * page). They are not exposed through the public API, so they are mirrored here + * to allow precise control over page-level fragmentation below. */ +#define NVS_ENTRY_SIZE 32 +#define NVS_ENTRIES_PER_PAGE 126 +#define NVS_PAGE_CHUNK_MAX_SIZE (NVS_ENTRY_SIZE * (NVS_ENTRIES_PER_PAGE - 1)) // 4000 B + +/* String lengths (strlen, excluding the NUL terminator) used to fragment a + * partition. A string of length L occupies 1 header entry + ceil((L+1)/32) data + * entries and must fit contiguously within a single page. + * - FRAG_STR_LEN fills a fresh page to 124 entries, leaving exactly 2 free. + * - FRAG_STR_FIRST_LEN is one entry shorter to account for the namespace entry + * that is written on the first page, so that page also keeps 2 free entries. */ +#define FRAG_STR_LEN (123 * NVS_ENTRY_SIZE - 1) // 3935 -> 124 entries +#define FRAG_STR_FIRST_LEN (122 * NVS_ENTRY_SIZE - 1) // 3903 -> 123 entries + +#define FRAG_NAMESPACE "_frag" +#define BLOB_NAMESPACE "_blobs" + static const char *TAG = "nvs_statistics_example"; // Maximum key character length is 15 (NVS_KEY_NAME_MAX_SIZE-1) @@ -195,6 +218,200 @@ static esp_err_t read_mock_data_from_namespace(const char* namespace_name) return ESP_OK; } +#if CONFIG_EXAMPLE_RUN_OVERHEAD_MEASUREMENT + +// Partitions of various sizes (declared in partitions.csv) swept by the measurement. +static const char* measured_partitions[] = { + "nvs_16k", + "nvs_32k", + "nvs_64k", +}; + +// Blob sizes (in bytes) measured for each partition. +static const size_t blob_sizes[] = {128, 256, 512, 1024, 2048, 4096}; + +// Theoretical entries consumed by a single blob in a non-fragmented partition: +// 1 BLOB_INDEX entry + 'chunks' chunk-header entries + ceil(size/32) data entries. +static size_t entries_per_blob_ideal(size_t blob_size) +{ + size_t chunks = (blob_size + NVS_PAGE_CHUNK_MAX_SIZE - 1) / NVS_PAGE_CHUNK_MAX_SIZE; + if (chunks == 0) { + chunks = 1; + } + size_t data_entries = (blob_size + NVS_ENTRY_SIZE - 1) / NVS_ENTRY_SIZE; + return 1 + chunks + data_entries; +} + +// Pre-populate a partition with large strings so that every page is filled up to +// its last 2 entries. Returns the number of strings written. +static size_t fragment_partition(const char* partition_name) +{ + nvs_handle_t handle; + esp_err_t err = nvs_open_from_partition(partition_name, FRAG_NAMESPACE, NVS_READWRITE, &handle); + if (err != ESP_OK) { + ESP_LOGE(TAG, "Error (%s) opening fragmentation handle on '%s'!", esp_err_to_name(err), partition_name); + return 0; + } + + char* buffer = malloc(FRAG_STR_LEN + 1); + if (buffer == NULL) { + ESP_LOGE(TAG, "Failed to allocate fragmentation buffer!"); + nvs_close(handle); + return 0; + } + memset(buffer, 'A', FRAG_STR_LEN); + buffer[FRAG_STR_LEN] = '\0'; + + size_t count = 0; + char key[16]; + + // First string is one entry shorter to compensate for the namespace entry + // written on page 0, so that page also retains exactly 2 free entries. + buffer[FRAG_STR_FIRST_LEN] = '\0'; + snprintf(key, sizeof(key), "f%05u", (unsigned)count); + err = nvs_set_str(handle, key, buffer); + buffer[FRAG_STR_FIRST_LEN] = 'A'; + if (err == ESP_OK && nvs_commit(handle) == ESP_OK) { + count++; + } + + // Remaining full-page strings until the partition cannot hold another one. + while (true) { + snprintf(key, sizeof(key), "f%05u", (unsigned)count); + err = nvs_set_str(handle, key, buffer); + if (err == ESP_ERR_NVS_NOT_ENOUGH_SPACE) { + break; + } + if (err != ESP_OK) { + ESP_LOGE(TAG, "Error (%s) writing fragmentation string!", esp_err_to_name(err)); + break; + } + if (nvs_commit(handle) != ESP_OK) { + break; + } + count++; + } + + free(buffer); + nvs_close(handle); + return count; +} + +// Fill a partition with same-sized blobs until it runs out of space. +// Returns the number of blobs successfully stored. +static size_t fill_with_blobs(const char* partition_name, size_t blob_size) +{ + nvs_handle_t handle; + esp_err_t err = nvs_open_from_partition(partition_name, BLOB_NAMESPACE, NVS_READWRITE, &handle); + if (err != ESP_OK) { + ESP_LOGE(TAG, "Error (%s) opening blob handle on '%s'!", esp_err_to_name(err), partition_name); + return 0; + } + + uint8_t* blob = malloc(blob_size); + if (blob == NULL) { + ESP_LOGE(TAG, "Failed to allocate %u B blob buffer!", (unsigned)blob_size); + nvs_close(handle); + return 0; + } + memset(blob, 0x5A, blob_size); + + size_t count = 0; + char key[16]; + while (true) { + snprintf(key, sizeof(key), "b%05u", (unsigned)count); + err = nvs_set_blob(handle, key, blob, blob_size); + if (err == ESP_ERR_NVS_NOT_ENOUGH_SPACE) { + break; + } + if (err != ESP_OK) { + ESP_LOGE(TAG, "Error (%s) writing blob!", esp_err_to_name(err)); + break; + } + err = nvs_commit(handle); + if (err == ESP_ERR_NVS_NOT_ENOUGH_SPACE) { + break; + } + count++; + } + + free(blob); + nvs_close(handle); + return count; +} + +// Run one measurement cell: erase + init a partition, optionally fragment it, +// fill it with blobs of the given size and report heap/entry/overhead statistics. +static void measure_blob_overhead(const char* partition_name, size_t blob_size, bool fragment) +{ + ESP_ERROR_CHECK(nvs_flash_erase_partition(partition_name)); + + uint32_t heap_before_init = esp_get_free_heap_size(); + ESP_ERROR_CHECK(nvs_flash_init_partition(partition_name)); + uint32_t heap_after_init = esp_get_free_heap_size(); + + nvs_stats_t stats; + ESP_ERROR_CHECK(nvs_get_stats(partition_name, &stats)); + size_t total_entries = stats.total_entries; + + size_t frag_strings = 0; + if (fragment) { + frag_strings = fragment_partition(partition_name); + } + + size_t stored = fill_with_blobs(partition_name, blob_size); + uint32_t heap_after_fill = esp_get_free_heap_size(); + + ESP_ERROR_CHECK(nvs_get_stats(partition_name, &stats)); + + size_t per_blob = entries_per_blob_ideal(blob_size); + size_t expected = (per_blob != 0) ? (total_entries / per_blob) : 0; + size_t payload = stored * blob_size; + size_t capacity = total_entries * NVS_ENTRY_SIZE; + int overhead_pct = (capacity != 0) ? (int)(100 - (100ULL * payload) / capacity) : 0; + + printf("\n"); + printf("NVS BLOB TEST - %u B (partition '%s', %s):\n", + (unsigned)blob_size, partition_name, fragment ? "fragmented" : "pristine"); + printf("======================\n"); + printf("heap before NVS init: %" PRIu32 " B\n", heap_before_init); + printf("heap after NVS init: %" PRIu32 " B (diff %" PRId32 " B)\n", + heap_after_init, (int32_t)(heap_before_init - heap_after_init)); + if (fragment) { + printf("fragmentation strings written: %u\n", (unsigned)frag_strings); + } + printf("\n"); + printf("available heap after fill: %" PRIu32 " B (diff %" PRId32 " B)\n", + heap_after_fill, (int32_t)(heap_after_init - heap_after_fill)); + printf("expected blobs count: %u\n", (unsigned)expected); + printf("stored blobs count: %u\n", (unsigned)stored); + printf("used_entries: %u (%u B)\n", stats.used_entries, (unsigned)(stats.used_entries * NVS_ENTRY_SIZE)); + printf("free_entries: %u (%u B)\n", stats.free_entries, (unsigned)(stats.free_entries * NVS_ENTRY_SIZE)); + printf("total_entries: %u (%u B)\n", stats.total_entries, (unsigned)(stats.total_entries * NVS_ENTRY_SIZE)); + printf("STORAGE OVERHEAD: %d%%\n", overhead_pct); + + ESP_ERROR_CHECK(nvs_flash_deinit_partition(partition_name)); +} + +static void run_overhead_measurement(void) +{ + const size_t partition_count = sizeof(measured_partitions) / sizeof(measured_partitions[0]); + const size_t blob_size_count = sizeof(blob_sizes) / sizeof(blob_sizes[0]); + + ESP_LOGI(TAG, "Starting NVS blob storage-overhead measurement..."); + for (size_t p = 0; p < partition_count; p++) { + for (size_t b = 0; b < blob_size_count; b++) { + measure_blob_overhead(measured_partitions[p], blob_sizes[b], false); +#if CONFIG_EXAMPLE_INDUCE_FRAGMENTATION + measure_blob_overhead(measured_partitions[p], blob_sizes[b], true); +#endif + } + } + ESP_LOGI(TAG, "NVS blob storage-overhead measurement done."); +} + +#endif // CONFIG_EXAMPLE_RUN_OVERHEAD_MEASUREMENT + void app_main(void) { // Erase the contents of the default NVS partition for clean run of this example @@ -230,5 +447,9 @@ void app_main(void) ESP_LOGE(TAG, "Error (%s) reading back stored data from namespace '%s'!", esp_err_to_name(ret), MOCK_DATA_BACKUP_NAMESPACE); } +#if CONFIG_EXAMPLE_RUN_OVERHEAD_MEASUREMENT + run_overhead_measurement(); +#endif + ESP_LOGI(TAG, "Returning from app_main()."); } diff --git a/examples/storage/nvs/nvs_statistics/partitions.csv b/examples/storage/nvs/nvs_statistics/partitions.csv new file mode 100644 index 00000000000..77fbef5c3e9 --- /dev/null +++ b/examples/storage/nvs/nvs_statistics/partitions.csv @@ -0,0 +1,8 @@ +# Name, Type, SubType, Offset, Size +factory, app, factory, 0x10000, 1M +nvs, data, nvs, , 0x6000 +phy_init, data, phy, , 0x1000 +# Partitions of different sizes used by the storage-overhead measurement. +nvs_16k, data, nvs, , 0x4000 +nvs_32k, data, nvs, , 0x8000 +nvs_64k, data, nvs, , 0x10000 diff --git a/examples/storage/nvs/nvs_statistics/pytest_nvs_statistics.py b/examples/storage/nvs/nvs_statistics/pytest_nvs_statistics.py index 17fdfe738b6..e51c8befce4 100644 --- a/examples/storage/nvs/nvs_statistics/pytest_nvs_statistics.py +++ b/examples/storage/nvs/nvs_statistics/pytest_nvs_statistics.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2025-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 import pytest from pytest_embedded import Dut @@ -14,4 +14,9 @@ def test_examples_nvs_statistics(dut: Dut) -> None: dut.expect('Getting post-commit NVS statistics...', timeout=5) dut.expect('Newly used entries match expectation.', timeout=5) + # Blob storage-overhead measurement (sweeps partition and blob sizes). + dut.expect('Starting NVS blob storage-overhead measurement...', timeout=10) + dut.expect('STORAGE OVERHEAD:', timeout=60) + dut.expect('NVS blob storage-overhead measurement done.', timeout=120) + dut.expect('Returning from app_main().', timeout=5) diff --git a/examples/storage/nvs/nvs_statistics/sdkconfig.defaults b/examples/storage/nvs/nvs_statistics/sdkconfig.defaults new file mode 100644 index 00000000000..fc3cd2ecfdb --- /dev/null +++ b/examples/storage/nvs/nvs_statistics/sdkconfig.defaults @@ -0,0 +1,3 @@ +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" +CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y