Merge branch 'feature/storage_nvs_perf_blob_v5.3' into 'release/v5.3'

Improvement of NVS Blob performance (v5.3)

See merge request espressif/esp-idf!41576

components/nvs_flash/host_test/nvs_host_test/main/test_nvs.cpp

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -3805,6 +3805,82 @@ TEST_CASE("nvs multiple write with same key but different types", "[nvs][xxx]")
     TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME));
 }
 
+TEST_CASE("nvs multiple write with same key blob and string involved", "[nvs]")
+{
+    PartitionEmulationFixture f(0, 10);
+
+    nvs_handle_t handle_1;
+    const uint32_t NVS_FLASH_SECTOR = 6;
+    const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3;
+    TEMPORARILY_DISABLED(f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN);)
+
+    for (uint16_t j = NVS_FLASH_SECTOR; j < NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN; ++j) {
+        f.erase(j);
+    }
+    TEST_ESP_OK(nvs::NVSPartitionManager::get_instance()->init_custom(f.part(),
+                                                                      NVS_FLASH_SECTOR,
+                                                                      NVS_FLASH_SECTOR_COUNT_MIN));
+
+    TEST_ESP_OK(nvs_open("namespace1", NVS_READWRITE, &handle_1));
+
+    nvs_erase_all(handle_1);
+
+    const char key_name[] = "foo";
+
+    // integer variables
+    int32_t v32;
+    int8_t v8;
+
+    // string
+    #define str_data_len 64
+    const char str_data[] = "string data";
+    char str_buf[str_data_len] = {0};
+    size_t str_len = str_data_len;
+
+    // blob
+    #define blob_data_len 64
+    uint8_t blob_data[blob_data_len] = {0};
+    uint8_t blob_buf[blob_data_len] = {0};
+    size_t blob_read_size;
+
+    // first write is i32
+    TEST_ESP_OK(nvs_set_i32(handle_1, key_name, (int32_t)12345678));
+
+    TEST_ESP_ERR(nvs_get_i8(handle_1, key_name, &v8), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_OK(nvs_get_i32(handle_1, key_name, &v32));
+    TEST_ESP_ERR(nvs_get_str(handle_1, key_name, str_buf, &str_len), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_ERR(nvs_get_blob(handle_1, key_name, blob_buf, &blob_read_size), ESP_ERR_NVS_NOT_FOUND);
+
+
+    // second write is string
+    TEST_ESP_OK(nvs_set_str(handle_1, key_name, str_data));
+
+    TEST_ESP_ERR(nvs_get_i8(handle_1, key_name, &v8), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_ERR(nvs_get_i32(handle_1, key_name, &v32), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_OK(nvs_get_str(handle_1, key_name, str_buf, &str_len));
+    TEST_ESP_ERR(nvs_get_blob(handle_1, key_name, blob_buf, &blob_read_size), ESP_ERR_NVS_NOT_FOUND);
+
+    // third write is blob
+    TEST_ESP_OK(nvs_set_blob(handle_1, key_name, blob_data, blob_data_len));
+
+    TEST_ESP_ERR(nvs_get_i8(handle_1, key_name, &v8), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_ERR(nvs_get_i32(handle_1, key_name, &v32), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_ERR(nvs_get_str(handle_1, key_name, str_buf, &str_len), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_OK(nvs_get_blob(handle_1, key_name, blob_buf, &blob_read_size));
+
+    // fourth write is i8
+    TEST_ESP_OK(nvs_set_i8(handle_1, key_name, (int8_t)12));
+
+    TEST_ESP_OK(nvs_get_i8(handle_1, key_name, &v8));
+    TEST_ESP_ERR(nvs_get_i32(handle_1, key_name, &v32), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_ERR(nvs_get_str(handle_1, key_name, str_buf, &str_len), ESP_ERR_NVS_NOT_FOUND);
+    TEST_ESP_ERR(nvs_get_blob(handle_1, key_name, blob_buf, &blob_read_size), ESP_ERR_NVS_NOT_FOUND);
+
+    nvs_close(handle_1);
+
+    TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME));
+}
+
 TEST_CASE("nvs find key tests", "[nvs]")
 {
     const size_t buff_len = 4096;

components/nvs_flash/src/nvs_page.cpp

@@ -247,6 +247,43 @@ esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, c
     return ESP_OK;
 }
 
+// Reads the data entries of the variable length item.
+// The metadata entry is already read in the item object.
+// index is the index of the metadata entry on the page.
+// data is pointer to the buffer where the data will be copied to. It has to be at least
+// item.varLength.dataSize bytes long.
+// The function returns ESP_OK if the data was read successfully, or an error code if there was an error.
+esp_err_t Page::readVariableLengthItemData(const Item& item, const size_t index, void* data)
+{
+    if (mState == PageState::INVALID) {
+        return ESP_ERR_NVS_INVALID_STATE;
+    }
+
+    esp_err_t rc;
+    uint8_t* dst = reinterpret_cast<uint8_t*>(data);
+    size_t left = item.varLength.dataSize;
+    for (size_t i = index + 1; i < index + item.span; ++i) {
+        Item ditem;
+        rc = readEntry(i, ditem);
+        if (rc != ESP_OK) {
+            return rc;
+        }
+        size_t willCopy = ENTRY_SIZE;
+        willCopy = (left < willCopy) ? left : willCopy;
+        memcpy(dst, ditem.rawData, willCopy);
+        left -= willCopy;
+        dst += willCopy;
+    }
+    if (Item::calculateCrc32(reinterpret_cast<uint8_t * >(data), item.varLength.dataSize) != item.varLength.dataCrc32) {
+        rc = eraseEntryAndSpan(index);
+        if (rc != ESP_OK) {
+            return rc;
+        }
+        return ESP_ERR_NVS_NOT_FOUND;
+    }
+    return ESP_OK;
+}
+
 esp_err_t Page::readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize, uint8_t chunkIdx, VerOffset chunkStart)
 {
     size_t index = 0;
@@ -274,28 +311,7 @@ esp_err_t Page::readItem(uint8_t nsIndex, ItemType datatype, const char* key, vo
         return ESP_ERR_NVS_INVALID_LENGTH;
     }
 
-    uint8_t* dst = reinterpret_cast<uint8_t*>(data);
-    size_t left = item.varLength.dataSize;
-    for (size_t i = index + 1; i < index + item.span; ++i) {
-        Item ditem;
-        rc = readEntry(i, ditem);
-        if (rc != ESP_OK) {
-            return rc;
-        }
-        size_t willCopy = ENTRY_SIZE;
-        willCopy = (left < willCopy) ? left : willCopy;
-        memcpy(dst, ditem.rawData, willCopy);
-        left -= willCopy;
-        dst += willCopy;
-    }
-    if (Item::calculateCrc32(reinterpret_cast<uint8_t * >(data), item.varLength.dataSize) != item.varLength.dataCrc32) {
-        rc = eraseEntryAndSpan(index);
-        if (rc != ESP_OK) {
-            return rc;
-        }
-        return ESP_ERR_NVS_NOT_FOUND;
-    }
-    return ESP_OK;
+    return readVariableLengthItemData(item, index, data);
 }
 
 esp_err_t Page::cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx, VerOffset chunkStart)
@@ -327,9 +343,23 @@ esp_err_t Page::cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, con
         return ESP_ERR_NVS_INVALID_LENGTH;
     }
 
+    // We have metadata of the variable length data chunk. It contains the length of the data and the crc32.
+    // As a first step we can calculate the crc32 of the data buffer to be compared with the crc32 of the item in the flash.
+    // If they are not equal, immediately return ESP_ERR_NVS_CONTENT_DIFFERS.
+    // If they are equal, to avoid crc32 collision false positive, we will read the data from the flash entry by entry and compare
+    // it with the respective chunk of input data buffer. The crc32 of the data read from the flash will be calculated on the fly.
+    // At the end, we will compare the crc32 of the data read from the flash with the crc32 of the metadata item in the flash to make sure
+    // that the data in the flash is not corrupted.
+    if (Item::calculateCrc32(reinterpret_cast<const uint8_t * >(data), item.varLength.dataSize) != item.varLength.dataCrc32) {
+        return ESP_ERR_NVS_CONTENT_DIFFERS;
+    }
+
     const uint8_t* dst = reinterpret_cast<const uint8_t*>(data);
     size_t left = item.varLength.dataSize;
-    for (size_t i = index + 1; i < index + item.span; ++i) {
+    uint32_t accumulatedCRC32;
+    size_t initial_index = index + 1;
+
+    for (size_t i = initial_index; i < index + item.span; ++i) {
         Item ditem;
         rc = readEntry(i, ditem);
         if (rc != ESP_OK) {
@@ -340,11 +370,18 @@ esp_err_t Page::cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, con
         if (memcmp(dst, ditem.rawData, willCopy)) {
             return ESP_ERR_NVS_CONTENT_DIFFERS;
         }
+
+        // Calculate the crc32 of the actual ditem.rawData buffer. Do not pass accumulatedCRC32 in the first call.
+        // In the first call, calculateCrc32 will use its default. In the subsequent calls, accumulatedCRC32 is the crc32 of the previous buffer.
+        accumulatedCRC32 = Item::calculateCrc32(ditem.rawData, willCopy, (i == initial_index) ? nullptr : &accumulatedCRC32);
+
         left -= willCopy;
         dst += willCopy;
     }
-    if (Item::calculateCrc32(reinterpret_cast<const uint8_t * >(data), item.varLength.dataSize) != item.varLength.dataCrc32) {
-        return ESP_ERR_NVS_NOT_FOUND;
+    // Check if the CRC32 calculated on the fly matches the variable length data CRC32 indicated in the metadata entry.
+    // If they are not equal, it means the data in the flash is corrupt, we will return ESP_ERR_NVS_CONTENT_DIFFERS.
+    if (accumulatedCRC32 != item.varLength.dataCrc32) {
+        return ESP_ERR_NVS_CONTENT_DIFFERS;
     }
 
     return ESP_OK;

components/nvs_flash/src/nvs_page.hpp

@@ -88,6 +88,8 @@ public:
 
     esp_err_t writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY);
 
+    esp_err_t readVariableLengthItemData(const Item& item, const size_t index, void* data);
+
     esp_err_t readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY);
 
     esp_err_t cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY);

components/nvs_flash/src/nvs_storage.hpp

@@ -153,7 +153,7 @@ protected:
 
     void fillEntryInfo(Item &item, nvs_entry_info_t &info);
 
-    esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx = Page::CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY);
+    esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx = Page::CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY, size_t* itemIndex = NULL);
 
 protected:
     Partition *mPartition;

components/nvs_flash/src/nvs_types.cpp

@@ -33,9 +33,12 @@ uint32_t Item::calculateCrc32WithoutValue() const
     return result;
 }
 
-uint32_t Item::calculateCrc32(const uint8_t* data, size_t size)
+uint32_t Item::calculateCrc32(const uint8_t* data, size_t size, uint32_t* initial_crc32)
 {
     uint32_t result = 0xffffffff;
+    if(initial_crc32) {
+        result = *initial_crc32;
+    }
     result = esp_rom_crc32_le(result, data, size);
     return result;
 }

components/nvs_flash/src/nvs_types.hpp

@@ -93,7 +93,7 @@ public:
 
     uint32_t calculateCrc32() const;
     uint32_t calculateCrc32WithoutValue() const;
-    static uint32_t calculateCrc32(const uint8_t* data, size_t size);
+    static uint32_t calculateCrc32(const uint8_t* data, size_t size, uint32_t* initial_crc32 = nullptr);
 
     void getKey(char* dst, size_t dstSize)
     {