Initial public version

components/nvs_flash/src/compressed_enum_table.hpp

@@ -0,0 +1,79 @@
+// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef compressed_enum_table_h
+#define compressed_enum_table_h
+
+#include <cstdint>
+#include <cassert>
+#include <type_traits>
+
+template<typename Tenum, size_t Nbits, size_t Nitems>
+class CompressedEnumTable
+{
+public:
+    uint32_t* data()
+    {
+        return mData;
+    }
+
+    const uint32_t* data() const
+    {
+        return mData;
+    }
+
+    Tenum get(size_t index)
+    {
+        assert(index >= 0 && index < Nitems);
+        size_t wordIndex = index / ITEMS_PER_WORD;
+        size_t offset = (index % ITEMS_PER_WORD) * Nbits;
+
+        return static_cast<Tenum>((mData[wordIndex] >> offset) & VALUE_MASK);
+    }
+
+    void set(size_t index, Tenum val)
+    {
+        assert(index >= 0 && index < Nitems);
+        size_t wordIndex = index / ITEMS_PER_WORD;
+        size_t offset = (index % ITEMS_PER_WORD) * Nbits;
+
+        uint32_t v = static_cast<uint32_t>(val) << offset;
+        mData[wordIndex] = (mData[wordIndex] & ~(VALUE_MASK << offset)) | v;
+    }
+
+    static constexpr size_t getWordIndex(size_t index)
+    {
+        return index / ITEMS_PER_WORD;
+    }
+
+    static constexpr size_t byteSize()
+    {
+        return WORD_COUNT * 4;
+    }
+
+    static constexpr size_t count()
+    {
+        return Nitems;
+    }
+
+
+protected:
+    static_assert(32 % Nbits == 0, "Nbits must divide 32");
+    static const size_t ITEMS_PER_WORD = 32 / Nbits;
+    static const size_t WORD_COUNT = ( Nbits * Nitems + 31 ) / 32;
+    static const uint32_t VALUE_MASK = (1 << Nbits) - 1;
+    uint32_t mData[WORD_COUNT];
+};
+
+#endif /* compressed_enum_table_h */

components/nvs_flash/src/intrusive_list.h

@@ -0,0 +1,248 @@
+// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef intrusive_list_h
+#define intrusive_list_h
+
+#include <cassert>
+
+template <typename T>
+class intrusive_list;
+
+template <typename T>
+class intrusive_list_node
+{
+protected:
+    friend class intrusive_list<T>;
+    T* mPrev = nullptr;
+    T* mNext = nullptr;
+};
+
+template <typename T>
+class intrusive_list
+{
+    typedef intrusive_list_node<T> TNode;
+    static_assert(std::is_base_of<TNode, T>::value, "");
+
+public:
+
+    class iterator : public std::iterator<std::forward_iterator_tag, T>
+    {
+    public:
+
+        iterator() : mPos(nullptr) {}
+
+        iterator(T* pos) : mPos(pos) {}
+
+        iterator operator++(int)
+        {
+            auto result = *this;
+            mPos = mPos->mNext;
+            return result;
+        }
+
+        iterator operator--(int)
+        {
+            auto result = *this;
+            mPos = mPos->mPrev;
+            return result;
+        }
+
+        iterator& operator++()
+        {
+            mPos = mPos->mNext;
+            return *this;
+        }
+
+        iterator& operator--()
+        {
+            mPos = mPos->mPrev;
+            return *this;
+        }
+
+
+        bool operator==(const iterator& other) const
+        {
+            return mPos == other.mPos;
+        }
+
+        bool operator!=(const iterator& other) const
+        {
+            return !(*this == other);
+        }
+
+        T& operator*()
+        {
+            return *mPos;
+        }
+
+        const T& operator*() const
+        {
+            return *mPos;
+        }
+
+        T* operator->()
+        {
+            return mPos;
+        }
+
+        const T* operator->() const
+        {
+            return mPos;
+        }
+
+        operator T*()
+        {
+            return mPos;
+        }
+
+        operator const T*() const
+        {
+            return mPos;
+        }
+
+
+    protected:
+        T* mPos;
+    };
+
+    void push_back(T* node)
+    {
+        if (mLast) {
+            mLast->mNext = node;
+        }
+        node->mPrev = mLast;
+        node->mNext = nullptr;
+        mLast = node;
+        if (mFirst == nullptr) {
+            mFirst = node;
+        }
+        ++mSize;
+    }
+
+    void push_front(T* node)
+    {
+        node->mPrev = nullptr;
+        node->mNext = mFirst;
+        if (mFirst) {
+            mFirst->mPrev = node;
+        }
+        mFirst = node;
+        if (mLast == nullptr) {
+            mLast = node;
+        }
+        ++mSize;
+    }
+
+    T& back()
+    {
+        return *mLast;
+    }
+
+    const T& back() const
+    {
+        return *mLast;
+    }
+
+    T& front()
+    {
+        return *mFirst;
+    }
+
+    const T& front() const
+    {
+        return *mFirst;
+    }
+
+    void pop_front()
+    {
+        erase(mFirst);
+    }
+
+    void pop_back()
+    {
+        erase(mLast);
+    }
+
+    void insert(iterator next, T* node)
+    {
+        if (static_cast<T*>(next) == nullptr) {
+            push_back(node);
+        } else {
+            auto prev = next->mPrev;
+            if (!prev) {
+                push_front(node);
+            } else {
+                prev->mNext = node;
+                next->mPrev = node;
+                node->mNext = next;
+                node->mPrev = &(*prev);
+                ++mSize;
+            }
+        }
+    }
+
+    void erase(iterator it)
+    {
+        auto prev = it->mPrev;
+        auto next = it->mNext;
+
+        if (prev) {
+            prev->mNext = next;
+        } else {
+            mFirst = next;
+        }
+        if (next) {
+            next->mPrev = prev;
+        } else {
+            mLast = prev;
+        }
+        --mSize;
+    }
+
+    iterator begin()
+    {
+        return iterator(mFirst);
+    }
+
+    iterator end()
+    {
+        return iterator(nullptr);
+    }
+
+    size_t size() const
+    {
+        return mSize;
+    }
+
+    bool empty() const
+    {
+        return mSize == 0;
+    }
+    
+    
+    void clear()
+    {
+        while (mFirst) {
+            erase(mFirst);
+        }
+    }
+
+protected:
+    T* mFirst = nullptr;
+    T* mLast = nullptr;
+    size_t mSize = 0;
+};
+
+
+#endif /* intrusive_list_h */

components/nvs_flash/src/nvs.hpp

@@ -0,0 +1,25 @@
+// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef nvs_hpp
+#define nvs_hpp
+
+
+#include <memory>
+#include "nvs.h"
+#include "nvs_types.hpp"
+#include "nvs_page.hpp"
+#include "nvs_pagemanager.hpp"
+#include "nvs_storage.hpp"
+
+#endif /* nvs_hpp */

components/nvs_flash/src/nvs_api.cpp

@@ -0,0 +1,278 @@
+// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "nvs.hpp"
+#include "nvs_flash.h"
+#include "nvs_storage.hpp"
+#include "intrusive_list.h"
+#include "nvs_platform.hpp"
+
+class HandleEntry : public intrusive_list_node<HandleEntry>
+{
+public:
+    HandleEntry(){}
+    
+    HandleEntry(nvs_handle handle, bool readOnly, uint8_t nsIndex) :
+    mHandle(handle),
+    mReadOnly(readOnly),
+    mNsIndex(nsIndex)
+    {
+    }
+    
+    nvs_handle mHandle;
+    uint8_t mReadOnly;
+    uint8_t mNsIndex;
+};
+
+#ifdef ESP_PLATFORM
+SemaphoreHandle_t nvs::Lock::mSemaphore = NULL;
+#endif
+
+using namespace std;
+using namespace nvs;
+
+static intrusive_list<HandleEntry> s_nvs_handles;
+static uint32_t s_nvs_next_handle = 1;
+static nvs::Storage s_nvs_storage;
+
+extern "C" esp_err_t nvs_flash_init(uint32_t baseSector, uint32_t sectorCount)
+{
+    Lock::init();
+    Lock lock;
+    NVS_DEBUGV("%s %d %d\r\n", __func__, baseSector, sectorCount);
+    return s_nvs_storage.init(baseSector, sectorCount);
+}
+
+static esp_err_t nvs_find_ns_handle(nvs_handle handle, HandleEntry& entry)
+{
+    auto it = find_if(begin(s_nvs_handles), end(s_nvs_handles), [=](HandleEntry& e) -> bool {
+        return e.mHandle == handle;
+    });
+    if (it == end(s_nvs_handles)) {
+        return ESP_ERR_NVS_INVALID_HANDLE;
+    }
+    entry = *it;
+    return ESP_OK;
+}
+
+extern "C" esp_err_t nvs_open(const char* name, nvs_open_mode open_mode, nvs_handle *out_handle)
+{
+    Lock lock;
+    NVS_DEBUGV("%s %s %d\r\n", __func__, name, open_mode);
+    uint8_t nsIndex;
+    esp_err_t err = s_nvs_storage.createOrOpenNamespace(name, open_mode == NVS_READWRITE, nsIndex);
+    if (err != ESP_OK) {
+        return err;
+    }
+
+    uint32_t handle = s_nvs_next_handle;
+    ++s_nvs_next_handle;
+    *out_handle = handle;
+
+    s_nvs_handles.push_back(new HandleEntry(handle, open_mode==NVS_READONLY, nsIndex));
+    return ESP_OK;
+}
+
+extern "C" void nvs_close(nvs_handle handle)
+{
+    Lock lock;
+    NVS_DEBUGV("%s %d\r\n", __func__, handle);
+    auto it = find_if(begin(s_nvs_handles), end(s_nvs_handles), [=](HandleEntry& e) -> bool {
+        return e.mHandle == handle;
+    });
+    if (it == end(s_nvs_handles)) {
+        return;
+    }
+    s_nvs_handles.erase(it);
+}
+
+template<typename T>
+static esp_err_t nvs_set(nvs_handle handle, const char* key, T value)
+{
+    Lock lock;
+    NVS_DEBUGV("%s %s %d %d\r\n", __func__, key, sizeof(T), (uint32_t) value);
+    HandleEntry entry;
+    auto err = nvs_find_ns_handle(handle, entry);
+    if (err != ESP_OK) {
+        return err;
+    }
+    if (entry.mReadOnly) {
+        return ESP_ERR_NVS_READ_ONLY;
+    }
+    return s_nvs_storage.writeItem(entry.mNsIndex, key, value);
+}
+
+extern "C" esp_err_t nvs_set_i8  (nvs_handle handle, const char* key, int8_t value)
+{
+    return nvs_set(handle, key, value);
+}
+
+extern "C" esp_err_t nvs_set_u8  (nvs_handle handle, const char* key, uint8_t value)
+{
+    return nvs_set(handle, key, value);
+}
+
+extern "C" esp_err_t nvs_set_i16 (nvs_handle handle, const char* key, int16_t value)
+{
+    return nvs_set(handle, key, value);
+}
+
+extern "C" esp_err_t nvs_set_u16 (nvs_handle handle, const char* key, uint16_t value)
+{
+    return nvs_set(handle, key, value);
+}
+
+extern "C" esp_err_t nvs_set_i32 (nvs_handle handle, const char* key, int32_t value)
+{
+    return nvs_set(handle, key, value);
+}
+
+extern "C" esp_err_t nvs_set_u32 (nvs_handle handle, const char* key, uint32_t value)
+{
+    return nvs_set(handle, key, value);
+}
+
+extern "C" esp_err_t nvs_set_i64 (nvs_handle handle, const char* key, int64_t value)
+{
+    return nvs_set(handle, key, value);
+}
+
+extern "C" esp_err_t nvs_set_u64 (nvs_handle handle, const char* key, uint64_t value)
+{
+    return nvs_set(handle, key, value);
+}
+
+extern "C" esp_err_t nvs_commit(nvs_handle handle)
+{
+    Lock lock;
+    // no-op for now, to be used when intermediate cache is added
+    HandleEntry entry;
+    return nvs_find_ns_handle(handle, entry);
+}
+
+extern "C" esp_err_t nvs_set_str(nvs_handle handle, const char* key, const char* value)
+{
+    Lock lock;
+    NVS_DEBUGV("%s %s %s\r\n", __func__, key, value);
+    HandleEntry entry;
+    auto err = nvs_find_ns_handle(handle, entry);
+    if (err != ESP_OK) {
+        return err;
+    }
+    return s_nvs_storage.writeItem(entry.mNsIndex, nvs::ItemType::SZ, key, value, strlen(value) + 1);
+}
+
+extern "C" esp_err_t nvs_set_blob(nvs_handle handle, const char* key, const void* value, size_t length)
+{
+    Lock lock;
+    NVS_DEBUGV("%s %s %d\r\n", __func__, key, length);
+    HandleEntry entry;
+    auto err = nvs_find_ns_handle(handle, entry);
+    if (err != ESP_OK) {
+        return err;
+    }
+    return s_nvs_storage.writeItem(entry.mNsIndex, nvs::ItemType::BLOB, key, value, length);
+}
+
+
+template<typename T>
+static esp_err_t nvs_get(nvs_handle handle, const char* key, T* out_value)
+{
+    Lock lock;
+    NVS_DEBUGV("%s %s %d\r\n", __func__, key, sizeof(T));
+    HandleEntry entry;
+    auto err = nvs_find_ns_handle(handle, entry);
+    if (err != ESP_OK) {
+        return err;
+    }
+    return s_nvs_storage.readItem(entry.mNsIndex, key, *out_value);
+}
+
+extern "C" esp_err_t nvs_get_i8  (nvs_handle handle, const char* key, int8_t* out_value)
+{
+    return nvs_get(handle, key, out_value);
+}
+
+extern "C" esp_err_t nvs_get_u8  (nvs_handle handle, const char* key, uint8_t* out_value)
+{
+    return nvs_get(handle, key, out_value);
+}
+
+extern "C" esp_err_t nvs_get_i16 (nvs_handle handle, const char* key, int16_t* out_value)
+{
+    return nvs_get(handle, key, out_value);
+}
+
+extern "C" esp_err_t nvs_get_u16 (nvs_handle handle, const char* key, uint16_t* out_value)
+{
+    return nvs_get(handle, key, out_value);
+}
+
+extern "C" esp_err_t nvs_get_i32 (nvs_handle handle, const char* key, int32_t* out_value)
+{
+    return nvs_get(handle, key, out_value);
+}
+
+extern "C" esp_err_t nvs_get_u32 (nvs_handle handle, const char* key, uint32_t* out_value)
+{
+    return nvs_get(handle, key, out_value);
+}
+
+extern "C" esp_err_t nvs_get_i64 (nvs_handle handle, const char* key, int64_t* out_value)
+{
+    return nvs_get(handle, key, out_value);
+}
+
+extern "C" esp_err_t nvs_get_u64 (nvs_handle handle, const char* key, uint64_t* out_value)
+{
+    return nvs_get(handle, key, out_value);
+}
+
+static esp_err_t nvs_get_str_or_blob(nvs_handle handle, nvs::ItemType type, const char* key, void* out_value, size_t* length)
+{
+    Lock lock;
+    NVS_DEBUGV("%s %s\r\n", __func__, key);
+    HandleEntry entry;
+    auto err = nvs_find_ns_handle(handle, entry);
+    if (err != ESP_OK) {
+        return err;
+    }
+
+    size_t dataSize;
+    err = s_nvs_storage.getItemDataSize(entry.mNsIndex, type, key, dataSize);
+    if (err != ESP_OK) {
+        return err;
+    }
+
+    if (length != nullptr && out_value == nullptr) {
+        *length = dataSize;
+        return ESP_OK;
+    }
+
+    if (length == nullptr || *length < dataSize) {
+        return ESP_ERR_NVS_INVALID_LENGTH;
+    }
+
+    return s_nvs_storage.readItem(entry.mNsIndex, type, key, out_value, dataSize);
+}
+
+extern "C" esp_err_t nvs_get_str(nvs_handle handle, const char* key, char* out_value, size_t* length)
+{
+    return nvs_get_str_or_blob(handle, nvs::ItemType::SZ, key, out_value, length);
+}
+
+extern "C" esp_err_t nvs_get_blob(nvs_handle handle, const char* key, void* out_value, size_t* length)
+{
+    return nvs_get_str_or_blob(handle, nvs::ItemType::BLOB, key, out_value, length);
+}
+

components/nvs_flash/src/nvs_page.cpp

@@ -0,0 +1,689 @@
+// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "nvs_page.hpp"
+#if defined(ESP_PLATFORM)
+#include <rom/crc.h>
+#else
+#include "crc.h"
+#endif
+
+
+namespace nvs
+{
+
+uint32_t Page::Header::calculateCrc32()
+{
+    return crc32_le(0xffffffff,
+                    reinterpret_cast<uint8_t*>(this) + offsetof(Header, mSeqNumber),
+                    offsetof(Header, mCrc32) - offsetof(Header, mSeqNumber));
+}
+    
+esp_err_t Page::load(uint32_t sectorNumber)
+{
+    mBaseAddress = sectorNumber * SEC_SIZE;
+    mUsedEntryCount = 0;
+    mErasedEntryCount = 0;
+
+    Header header;
+    auto rc = spi_flash_read(mBaseAddress, reinterpret_cast<uint32_t*>(&header), sizeof(header));
+    if (rc != ESP_OK) {
+        mState = PageState::INVALID;
+        return rc;
+    }
+    if (header.mState == PageState::UNINITIALIZED) {
+        mState = header.mState;
+        // check if the whole page is really empty
+        // reading the whole page takes ~40 times less than erasing it
+        uint32_t line[8];
+        for (uint32_t i = 0; i < SPI_FLASH_SEC_SIZE; i += sizeof(line)) {
+            rc = spi_flash_read(mBaseAddress + i, line, sizeof(line));
+            if (rc != ESP_OK) {
+                mState = PageState::INVALID;
+                return rc;
+            }
+            if (std::any_of(line, line + 4, [](uint32_t val) -> bool { return val != 0xffffffff; })) {
+                // page isn't as empty after all, mark it as corrupted
+                mState = PageState::CORRUPT;
+                break;
+            }
+        }
+    }
+    else if (header.mCrc32 != header.calculateCrc32()) {
+        header.mState = PageState::CORRUPT;
+    }
+    else {
+        mState = header.mState;
+        mSeqNumber = header.mSeqNumber;
+    }
+    
+    switch (mState) {
+    case PageState::UNINITIALIZED:
+        break;
+
+    case PageState::FULL:
+    case PageState::ACTIVE:
+    case PageState::FREEING:
+        mLoadEntryTable();
+        break;
+
+    default:
+        mState = PageState::CORRUPT;
+        break;
+    }
+
+    return ESP_OK;
+}
+
+esp_err_t Page::writeEntry(const Item& item)
+{
+    auto rc = spi_flash_write(getEntryAddress(mNextFreeEntry), reinterpret_cast<const uint32_t*>(&item), sizeof(item));
+    if (rc != ESP_OK) {
+        mState = PageState::INVALID;
+        return rc;
+    }
+
+    auto err = alterEntryState(mNextFreeEntry, EntryState::WRITTEN);
+    if (err != ESP_OK) {
+        return err;
+    }
+
+    if (mNextFreeEntry == 0) {
+        mFirstUsedEntry = 0;
+    }
+
+    ++mUsedEntryCount;
+
+    if (++mNextFreeEntry == ENTRY_COUNT) {
+        alterPageState(PageState::FULL);
+    }
+
+    return ESP_OK;
+}
+
+esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize)
+{
+    Item item;
+
+    esp_err_t err;
+    if (mState == PageState::UNINITIALIZED) {
+        err = initialize();
+        if (err != ESP_OK) {
+            return err;
+        }
+    }
+
+    if (mState == PageState::FULL) {
+        return ESP_ERR_NVS_PAGE_FULL;
+    }
+
+    const size_t keySize = strlen(key);
+    if (keySize > Item::MAX_KEY_LENGTH) {
+        return ESP_ERR_NVS_KEY_TOO_LONG;
+    }
+
+    size_t totalSize = ENTRY_SIZE;
+    size_t entriesCount = 1;
+    if (datatype == ItemType::SZ || datatype == ItemType::BLOB) {
+        size_t roundedSize = (dataSize + ENTRY_SIZE - 1) & ~(ENTRY_SIZE - 1);
+        totalSize += roundedSize;
+        entriesCount += roundedSize / ENTRY_SIZE;
+    }
+
+    // primitive types should fit into one entry
+    assert(totalSize == ENTRY_SIZE || datatype == ItemType::BLOB || datatype == ItemType::SZ);
+
+    if (mNextFreeEntry + entriesCount > ENTRY_COUNT) {
+        // page will not fit this amount of data
+        return ESP_ERR_NVS_PAGE_FULL;
+    }
+
+    // write first item
+
+    item.nsIndex = nsIndex;
+    item.datatype = datatype;
+    item.span = (totalSize + ENTRY_SIZE - 1) / ENTRY_SIZE;
+    item.reserved = 0xff;
+
+    std::fill_n(reinterpret_cast<uint32_t*>(item.key),  sizeof(item.key)  / 4, 0xffffffff);
+    std::fill_n(reinterpret_cast<uint32_t*>(item.data), sizeof(item.data) / 4, 0xffffffff);
+
+    strlcpy(item.key, key, Item::MAX_KEY_LENGTH + 1);
+
+
+    if (datatype != ItemType::SZ && datatype != ItemType::BLOB) {
+        memcpy(item.data, data, dataSize);
+        item.crc32 = item.calculateCrc32();
+        err = writeEntry(item);
+        if (err != ESP_OK) {
+            return err;
+        }
+    } else {
+        const uint8_t* src = reinterpret_cast<const uint8_t*>(data);
+        item.varLength.dataCrc32 = Item::calculateCrc32(src, dataSize);
+        item.varLength.dataSize = dataSize;
+        item.varLength.reserved2 = 0xffff;
+        item.crc32 = item.calculateCrc32();
+        err = writeEntry(item);
+        if (err != ESP_OK) {
+            return err;
+        }
+
+        size_t left = dataSize;
+        while (left != 0) {
+            size_t willWrite = Page::ENTRY_SIZE;
+            willWrite = (left < willWrite)?left:willWrite;
+            memcpy(item.rawData, src, willWrite);
+            src += willWrite;
+            left -= willWrite;
+            err = writeEntry(item);
+            if (err != ESP_OK) {
+                return err;
+            }
+        }
+    }
+    return ESP_OK;
+}
+
+esp_err_t Page::readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize)
+{
+    size_t index = 0;
+    Item item;
+    esp_err_t rc = findItem(nsIndex, datatype, key, index, item);
+    if (rc != ESP_OK) {
+        return rc;
+    }
+
+    if (datatype != ItemType::SZ && datatype != ItemType::BLOB) {
+        if (dataSize != getAlignmentForType(datatype)) {
+            return ESP_ERR_NVS_TYPE_MISMATCH;
+        }
+
+        memcpy(data, item.data, dataSize);
+        return ESP_OK;
+    }
+
+    if (dataSize < static_cast<size_t>(item.varLength.dataSize)) {
+        return ESP_ERR_NVS_INVALID_LENGTH;
+    }
+
+    uint8_t* dst = reinterpret_cast<uint8_t*>(data);
+    size_t left = item.varLength.dataSize;
+    for (size_t i = index + 1; i < index + item.span; ++i) {
+        Item ditem;
+        rc = readEntry(i, ditem);
+        if (rc != ESP_OK) {
+            return rc;
+        }
+        size_t willCopy = ENTRY_SIZE;
+        willCopy = (left < willCopy)?left:willCopy;
+        memcpy(dst, ditem.rawData, willCopy);
+        left -= willCopy;
+        dst += willCopy;
+    }
+    if (Item::calculateCrc32(reinterpret_cast<uint8_t*>(data), item.varLength.dataSize) != item.varLength.dataCrc32) {
+        rc = eraseEntryAndSpan(index);
+        if (rc != ESP_OK) {
+            return rc;
+        }
+        return ESP_ERR_NVS_NOT_FOUND;
+    }
+    return ESP_OK;
+}
+
+esp_err_t Page::eraseItem(uint8_t nsIndex, ItemType datatype, const char* key)
+{
+    size_t index = 0;
+    Item item;
+    esp_err_t rc = findItem(nsIndex, datatype, key, index, item);
+    if (rc != ESP_OK) {
+        return rc;
+    }
+    if (CachedFindInfo(nsIndex, datatype, key) == mFindInfo) {
+        invalidateCache();
+    }
+    return eraseEntryAndSpan(index);
+}
+
+esp_err_t Page::findItem(uint8_t nsIndex, ItemType datatype, const char* key)
+{
+    size_t index = 0;
+    Item item;
+    return findItem(nsIndex, datatype, key, index, item);
+}
+
+esp_err_t Page::eraseEntry(size_t index)
+{
+    auto state = mEntryTable.get(index);
+    assert(state == EntryState::WRITTEN || state == EntryState::EMPTY);
+    
+    auto rc = alterEntryState(index, EntryState::ERASED);
+    if (rc != ESP_OK) {
+        return rc;
+    }
+    
+    return ESP_OK;
+}
+    
+esp_err_t Page::eraseEntryAndSpan(size_t index)
+{
+    auto state = mEntryTable.get(index);
+    assert(state == EntryState::WRITTEN || state == EntryState::EMPTY);
+
+    size_t span = 1;
+    if (state == EntryState::WRITTEN) {
+        Item item;
+        auto rc = readEntry(index, item);
+        if (rc != ESP_OK) {
+            return rc;
+        }
+        if (item.calculateCrc32() != item.crc32) {
+            rc = alterEntryState(index, EntryState::ERASED);
+            if (rc != ESP_OK) {
+                return rc;
+            }
+        } else {
+            span = item.span;
+            for (ptrdiff_t i = index + span - 1; i >= static_cast<ptrdiff_t>(index); --i) {
+                rc = alterEntryState(i, EntryState::ERASED);
+                if (rc != ESP_OK) {
+                    return rc;
+                }
+            }
+        }
+    }
+    else {
+        auto rc = alterEntryState(index, EntryState::ERASED);
+        if (rc != ESP_OK) {
+            return rc;
+        }
+    }
+
+    if (index == mFirstUsedEntry) {
+        updateFirstUsedEntry(index, span);
+    }
+
+    mErasedEntryCount += span;
+    mUsedEntryCount -= span;
+
+    return ESP_OK;
+}
+
+void Page::updateFirstUsedEntry(size_t index, size_t span)
+{
+    assert(index == mFirstUsedEntry);
+    mFirstUsedEntry = INVALID_ENTRY;
+    for (size_t i = index + span; i < mNextFreeEntry; ++i) {
+        if (mEntryTable.get(i) == EntryState::WRITTEN) {
+            mFirstUsedEntry = i;
+            break;
+        }
+    }
+}
+    
+esp_err_t Page::moveItem(Page& other)
+{
+    if (mFirstUsedEntry == INVALID_ENTRY) {
+        return ESP_ERR_NVS_NOT_FOUND;
+    }
+
+    if (mFindInfo.itemIndex() == mFirstUsedEntry) {
+        invalidateCache();
+    }
+    
+    if (other.mState == PageState::UNINITIALIZED) {
+        auto err = other.initialize();
+        if (err != ESP_OK) {
+            return err;
+        }
+    }
+
+    Item entry;
+    auto err = readEntry(mFirstUsedEntry, entry);
+    if (err != ESP_OK) {
+        return err;
+    }
+    err = other.writeEntry(entry);
+    if (err != ESP_OK) {
+        return err;
+    }
+    err = eraseEntry(mFirstUsedEntry);
+    if (err != ESP_OK) {
+        return err;
+    }
+
+    size_t span = entry.span;
+    size_t end = mFirstUsedEntry + span;
+    
+    assert(mFirstUsedEntry != INVALID_ENTRY || span == 1);
+    
+    for (size_t i = mFirstUsedEntry + 1; i < end; ++i) {
+        readEntry(i, entry);
+        err = other.writeEntry(entry);
+        if (err != ESP_OK) {
+            return err;
+        }
+        err = eraseEntry(i);
+        if (err != ESP_OK) {
+            return err;
+        }
+    }
+    updateFirstUsedEntry(mFirstUsedEntry, span);
+    mErasedEntryCount += span;
+    mUsedEntryCount -= span;
+    
+    return ESP_OK;
+}
+
+esp_err_t Page::mLoadEntryTable()
+{
+    // for states where we actually care about data in the page, read entry state table
+    if (mState == PageState::ACTIVE ||
+            mState == PageState::FULL ||
+            mState == PageState::FREEING) {
+        auto rc = spi_flash_read(mBaseAddress + ENTRY_TABLE_OFFSET, mEntryTable.data(),
+                                 static_cast<uint32_t>(mEntryTable.byteSize()));
+        if (rc != ESP_OK) {
+            mState = PageState::INVALID;
+            return rc;
+        }
+    }
+
+    mErasedEntryCount = 0;
+    mUsedEntryCount = 0;
+    for (size_t i = 0; i < ENTRY_COUNT; ++i) {
+        auto s = mEntryTable.get(i);
+        if (s == EntryState::WRITTEN) {
+            if (mFirstUsedEntry == INVALID_ENTRY) {
+                mFirstUsedEntry = i;
+            }
+            ++mUsedEntryCount;
+        } else if (s == EntryState::ERASED) {
+            ++mErasedEntryCount;
+        }
+    }
+
+    // for PageState::ACTIVE, we may have more data written to this page
+    // as such, we need to figure out where the first unused entry is
+    if (mState == PageState::ACTIVE) {
+        for (size_t i = 0; i < ENTRY_COUNT; ++i) {
+            if (mEntryTable.get(i) == EntryState::EMPTY) {
+                mNextFreeEntry = i;
+                break;
+            }
+        }
+
+        // however, if power failed after some data was written into the entry.
+        // but before the entry state table was altered, the entry locacted via
+        // entry state table may actually be half-written.
+        // this is easy to check by reading EntryHeader (i.e. first word)
+        uint32_t entryAddress = mBaseAddress + ENTRY_DATA_OFFSET +
+                                static_cast<uint32_t>(mNextFreeEntry) * ENTRY_SIZE;
+        uint32_t header;
+        auto rc = spi_flash_read(entryAddress, &header, sizeof(header));
+        if (rc != ESP_OK) {
+            mState = PageState::INVALID;
+            return rc;
+        }
+        if (header != 0xffffffff) {
+            auto err = alterEntryState(mNextFreeEntry, EntryState::ERASED);
+            if (err != ESP_OK) {
+                mState = PageState::INVALID;
+                return err;
+            }
+            ++mNextFreeEntry;
+        }
+
+        // check that all variable-length items are written or erased fully
+        for (size_t i = 0; i < mNextFreeEntry; ++i) {
+            if (mEntryTable.get(i) == EntryState::ERASED) {
+                continue;
+            }
+
+            Item item;
+            auto err = readEntry(i, item);
+            if (err != ESP_OK) {
+                mState = PageState::INVALID;
+                return err;
+            }
+
+            if (item.crc32 != item.calculateCrc32()) {
+                err = eraseEntryAndSpan(i);
+                if (err != ESP_OK) {
+                    mState = PageState::INVALID;
+                    return err;
+                }
+                continue;
+            }
+
+            if (item.datatype != ItemType::BLOB && item.datatype != ItemType::SZ) {
+                continue;
+            }
+
+            size_t span = item.span;
+            bool needErase = false;
+            for (size_t j = i; j < i + span; ++j) {
+                if (mEntryTable.get(j) != EntryState::WRITTEN) {
+                    needErase = true;
+                    break;
+                }
+            }
+            if (needErase) {
+                eraseEntryAndSpan(i);
+            }
+            i += span - 1;
+        }
+
+    }
+
+    return ESP_OK;
+}
+
+
+esp_err_t Page::initialize()
+{
+    assert(mState == PageState::UNINITIALIZED);
+    mState = PageState::ACTIVE;
+    Header header;
+    header.mState = mState;
+    header.mSeqNumber = mSeqNumber;
+    header.mCrc32 = header.calculateCrc32();
+    
+    auto rc = spi_flash_write(mBaseAddress, reinterpret_cast<uint32_t*>(&header), sizeof(header));
+    if (rc != ESP_OK) {
+        mState = PageState::INVALID;
+        return rc;
+    }
+
+    mNextFreeEntry = 0;
+    std::fill_n(mEntryTable.data(), mEntryTable.byteSize() / sizeof(uint32_t), 0xffffffff);
+    invalidateCache();
+    return ESP_OK;
+}
+
+esp_err_t Page::alterEntryState(size_t index, EntryState state)
+{
+    assert(index < ENTRY_COUNT);
+    mEntryTable.set(index, state);
+    size_t wordToWrite = mEntryTable.getWordIndex(index);
+    uint32_t word = mEntryTable.data()[wordToWrite];
+    auto rc = spi_flash_write(mBaseAddress + ENTRY_TABLE_OFFSET + static_cast<uint32_t>(wordToWrite) * 4, &word, 4);
+    if (rc != ESP_OK) {
+        mState = PageState::INVALID;
+        return rc;
+    }
+    return ESP_OK;
+}
+
+esp_err_t Page::alterPageState(PageState state)
+{
+    auto rc = spi_flash_write(mBaseAddress, reinterpret_cast<uint32_t*>(&state), sizeof(state));
+    if (rc != ESP_OK) {
+        mState = PageState::INVALID;
+        return rc;
+    }
+    mState = (PageState) state;
+    return ESP_OK;
+}
+
+esp_err_t Page::readEntry(size_t index, Item& dst)
+{
+    auto rc = spi_flash_read(getEntryAddress(index), reinterpret_cast<uint32_t*>(&dst), sizeof(dst));
+    if (rc != ESP_OK) {
+        return rc;
+    }
+    return ESP_OK;
+}
+
+esp_err_t Page::findItem(uint8_t nsIndex, ItemType datatype, const char* key, size_t &itemIndex, Item& item)
+{
+    if (mState == PageState::CORRUPT || mState == PageState::INVALID || mState == PageState::UNINITIALIZED) {
+        return ESP_ERR_NVS_NOT_FOUND;
+    }
+
+    CachedFindInfo findInfo(nsIndex, datatype, key);
+    if (mFindInfo == findInfo) {
+        itemIndex = mFindInfo.itemIndex();
+    }
+
+    size_t start = mFirstUsedEntry;
+    if (itemIndex > mFirstUsedEntry && itemIndex < ENTRY_COUNT) {
+        start = itemIndex;
+    }
+
+    size_t next;
+    for (size_t i = start; i < mNextFreeEntry; i = next) {
+        next = i + 1;
+        if (mEntryTable.get(i) != EntryState::WRITTEN) {
+            continue;
+        }
+
+        auto rc = readEntry(i, item);
+        if (rc != ESP_OK) {
+            mState = PageState::INVALID;
+            return rc;
+        }
+
+        auto crc32 = item.calculateCrc32();
+        if (item.crc32 != crc32) {
+            eraseEntryAndSpan(i);
+            continue;
+        }
+
+        if (item.datatype == ItemType::BLOB || item.datatype == ItemType::SZ) {
+            next = i + item.span;
+        }
+
+        if (nsIndex != NS_ANY && item.nsIndex != nsIndex) {
+            continue;
+        }
+
+        if (key != nullptr && strncmp(key, item.key, Item::MAX_KEY_LENGTH) != 0) {
+            continue;
+        }
+
+        if (datatype != ItemType::ANY && item.datatype != datatype) {
+            return ESP_ERR_NVS_TYPE_MISMATCH;
+        }
+
+        itemIndex = i;
+        findInfo.setItemIndex(static_cast<uint32_t>(itemIndex));
+        mFindInfo = findInfo;
+
+        return ESP_OK;
+    }
+
+    return ESP_ERR_NVS_NOT_FOUND;
+}
+
+esp_err_t Page::getSeqNumber(uint32_t& seqNumber) const
+{
+    if (mState != PageState::UNINITIALIZED && mState != PageState::INVALID && mState != PageState::CORRUPT) {
+        seqNumber = mSeqNumber;
+        return ESP_OK;
+    }
+    return ESP_ERR_NVS_NOT_INITIALIZED;
+}
+
+
+esp_err_t Page::setSeqNumber(uint32_t seqNumber)
+{
+    if (mState != PageState::UNINITIALIZED) {
+        return ESP_ERR_NVS_INVALID_STATE;
+    }
+    mSeqNumber = seqNumber;
+    return ESP_OK;
+}
+
+esp_err_t Page::erase()
+{
+    auto sector = mBaseAddress / SPI_FLASH_SEC_SIZE;
+    auto rc = spi_flash_erase_sector(sector);
+    if (rc != ESP_OK) {
+        mState = PageState::INVALID;
+        return rc;
+    }
+    return load(sector);
+}
+
+esp_err_t Page::markFreeing()
+{
+    if (mState != PageState::FULL && mState != PageState::ACTIVE) {
+        return ESP_ERR_NVS_INVALID_STATE;
+    }
+    return alterPageState(PageState::FREEING);
+}
+
+esp_err_t Page::markFull()
+{
+    if (mState != PageState::ACTIVE) {
+        return ESP_ERR_NVS_INVALID_STATE;
+    }
+    return alterPageState(PageState::FULL);
+}
+
+
+void Page::invalidateCache()
+{
+    mFindInfo = CachedFindInfo();
+}
+    
+void Page::debugDump()
+{
+    printf("state=%x addr=%x seq=%d\nfirstUsed=%d nextFree=%d used=%d erased=%d\n", mState, mBaseAddress, mSeqNumber, static_cast<int>(mFirstUsedEntry), static_cast<int>(mNextFreeEntry), mUsedEntryCount, mErasedEntryCount);
+    size_t skip = 0;
+    for (size_t i = 0; i < ENTRY_COUNT; ++i) {
+        printf("%3d: ", static_cast<int>(i));
+        EntryState state = mEntryTable.get(i);
+        if (state == EntryState::EMPTY) {
+            printf("E\n");
+        }
+        else if (state == EntryState::ERASED) {
+            printf("X\n");
+        }
+        else if (state == EntryState::WRITTEN) {
+            Item item;
+            readEntry(i, item);
+            if (skip == 0) {
+                printf("W ns=%2u type=%2u span=%3u key=\"%s\"\n", item.nsIndex, static_cast<unsigned>(item.datatype), item.span, item.key);
+                skip = item.span - 1;
+            }
+            else {
+                printf("D\n");
+                skip--;
+            }
+        }
+    }
+}
+
+} // namespace nvs

components/nvs_flash/src/nvs_page.hpp

@@ -0,0 +1,246 @@
+// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef nvs_page_hpp
+#define nvs_page_hpp
+
+#include "nvs.h"
+#include "nvs_types.hpp"
+#include <cstdint>
+#include <type_traits>
+#include <cstring>
+#include <algorithm>
+#include "esp_spi_flash.h"
+#include "compressed_enum_table.hpp"
+#include "intrusive_list.h"
+
+namespace nvs
+{
+
+class CachedFindInfo
+{
+public:
+    CachedFindInfo() { }
+    CachedFindInfo(uint8_t nsIndex, ItemType type, const char* key) :
+        mKeyPtr(key),
+        mNsIndex(nsIndex),
+        mType(type)
+    {
+    }
+
+    bool operator==(const CachedFindInfo& other) const
+    {
+        return mKeyPtr == other.mKeyPtr && mType == other.mType && mNsIndex == other.mNsIndex;
+    }
+
+    void setItemIndex(uint32_t index)
+    {
+        mItemIndex = index;
+    }
+    uint32_t itemIndex() const
+    {
+        return mItemIndex;
+    }
+
+protected:
+    uint32_t mItemIndex = 0;
+    const char* mKeyPtr = nullptr;
+    uint8_t mNsIndex = 0;
+    ItemType mType;
+
+};
+
+class Page : public intrusive_list_node<Page>
+{
+public:
+    static const uint32_t PSB_INIT = 0x1;
+    static const uint32_t PSB_FULL = 0x2;
+    static const uint32_t PSB_FREEING = 0x4;
+    static const uint32_t PSB_CORRUPT = 0x8;
+
+    static const uint32_t ESB_WRITTEN = 0x1;
+    static const uint32_t ESB_ERASED = 0x2;
+
+    static const uint32_t SEC_SIZE = SPI_FLASH_SEC_SIZE;
+
+    static const size_t ENTRY_SIZE  = 32;
+    static const size_t ENTRY_COUNT = 126;
+    static const uint32_t INVALID_ENTRY = 0xffffffff;
+
+    static const uint8_t NS_INDEX = 0;
+    static const uint8_t NS_ANY = 255;
+
+    enum class PageState : uint32_t {
+        // All bits set, default state after flash erase. Page has not been initialized yet.
+        UNINITIALIZED = 0xffffffff,
+
+        // Page is initialized, and will accept writes.
+        ACTIVE        = UNINITIALIZED & ~PSB_INIT,
+
+        // Page is marked as full and will not accept new writes.
+        FULL          = ACTIVE & ~PSB_FULL,
+
+        // Data is being moved from this page to a new one.
+        FREEING       = FULL & ~PSB_FREEING,
+
+        // Page was found to be in a corrupt and unrecoverable state.
+        // Instead of being erased immediately, it will be kept for diagnostics and data recovery.
+        // It will be erased once we run out out free pages.
+        CORRUPT       = FREEING & ~PSB_CORRUPT,
+
+        // Page object wasn't loaded from flash memory
+        INVALID       = 0
+    };
+
+    PageState state() const
+    {
+        return mState;
+    }
+
+    esp_err_t load(uint32_t sectorNumber);
+
+    esp_err_t getSeqNumber(uint32_t& seqNumber) const;
+
+    esp_err_t setSeqNumber(uint32_t seqNumber);
+
+    esp_err_t writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize);
+
+    esp_err_t readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize);
+
+    esp_err_t eraseItem(uint8_t nsIndex, ItemType datatype, const char* key);
+
+    esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key);
+
+    esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, size_t &itemIndex, Item& item);
+
+    template<typename T>
+    esp_err_t writeItem(uint8_t nsIndex, const char* key, const T& value)
+    {
+        return writeItem(nsIndex, itemTypeOf(value), key, &value, sizeof(value));
+    }
+
+    template<typename T>
+    esp_err_t readItem(uint8_t nsIndex, const char* key, T& value)
+    {
+        return readItem(nsIndex, itemTypeOf(value), key, &value, sizeof(value));
+    }
+
+    template<typename T>
+    esp_err_t eraseItem(uint8_t nsIndex, const char* key)
+    {
+        return eraseItem(nsIndex, itemTypeOf<T>(), key);
+    }
+
+    size_t getUsedEntryCount() const
+    {
+        return mUsedEntryCount;
+    }
+
+    size_t getErasedEntryCount() const
+    {
+        return mErasedEntryCount;
+    }
+
+
+    esp_err_t markFull();
+
+    esp_err_t markFreeing();
+
+    esp_err_t moveItem(Page& other);
+
+    esp_err_t erase();
+
+    void invalidateCache();
+    
+    void debugDump();
+
+protected:
+
+    class Header {
+    public:
+        Header() {
+            std::fill_n(mReserved, sizeof(mReserved)/sizeof(mReserved[0]), UINT32_MAX);
+        }
+        
+        PageState mState;       // page state
+        uint32_t mSeqNumber;    // sequence number of this page
+        uint32_t mReserved[5];  // unused, must be 0xffffffff
+        uint32_t mCrc32;        // crc of everything except mState
+        
+        uint32_t calculateCrc32();
+    };
+    
+    enum class EntryState {
+        EMPTY   = 0x3, // 0b11, default state after flash erase
+        WRITTEN = EMPTY & ~ESB_WRITTEN, // entry was written
+        ERASED  = WRITTEN & ~ESB_ERASED, // entry was written and then erased
+        INVALID = 0x4 // entry is in inconsistent state (write started but ESB_WRITTEN has not been set yet)
+    };
+
+    esp_err_t mLoadEntryTable();
+
+    esp_err_t initialize();
+
+    esp_err_t alterEntryState(size_t index, EntryState state);
+
+    esp_err_t alterPageState(PageState state);
+
+    esp_err_t readEntry(size_t index, Item& dst);
+
+    esp_err_t writeEntry(const Item& item);
+
+    esp_err_t eraseEntry(size_t index);
+    
+    esp_err_t eraseEntryAndSpan(size_t index);
+    
+    void updateFirstUsedEntry(size_t index, size_t span);
+
+    static constexpr size_t getAlignmentForType(ItemType type)
+    {
+        return static_cast<uint8_t>(type) & 0x0f;
+    }
+
+    uint32_t getEntryAddress(size_t entry)
+    {
+        assert(entry < ENTRY_COUNT);
+        return mBaseAddress + ENTRY_DATA_OFFSET + static_cast<uint32_t>(entry) * ENTRY_SIZE;
+    }
+
+
+protected:
+    uint32_t mBaseAddress = 0;
+    PageState mState = PageState::INVALID;
+    uint32_t mSeqNumber = UINT32_MAX;
+    typedef CompressedEnumTable<EntryState, 2, ENTRY_COUNT> TEntryTable;
+    TEntryTable mEntryTable;
+    size_t mNextFreeEntry = INVALID_ENTRY;
+    size_t mFirstUsedEntry = INVALID_ENTRY;
+    uint16_t mUsedEntryCount = 0;
+    uint16_t mErasedEntryCount = 0;
+
+    CachedFindInfo mFindInfo;
+
+    static const uint32_t HEADER_OFFSET = 0;
+    static const uint32_t ENTRY_TABLE_OFFSET = HEADER_OFFSET + 32;
+    static const uint32_t ENTRY_DATA_OFFSET = ENTRY_TABLE_OFFSET + 32;
+
+    static_assert(sizeof(Header) == 32, "header size must be 32 bytes");
+    static_assert(ENTRY_TABLE_OFFSET % 32 == 0, "entry table offset should be aligned");
+    static_assert(ENTRY_DATA_OFFSET % 32 == 0, "entry data offset should be aligned");
+
+}; // class Page
+
+} // namespace nvs
+
+
+#endif /* nvs_page_hpp */

components/nvs_flash/src/nvs_pagemanager.cpp

@@ -0,0 +1,137 @@
+// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "nvs_pagemanager.hpp"
+
+namespace nvs
+{
+esp_err_t PageManager::load(uint32_t baseSector, uint32_t sectorCount)
+{
+    mBaseSector = baseSector;
+    mPageCount = sectorCount;
+    mPageList.clear();
+    mFreePageList.clear();
+    mPages.reset(new Page[sectorCount]);
+
+    for (uint32_t i = 0; i < sectorCount; ++i) {
+        auto err = mPages[i].load(baseSector + i);
+        if (err != ESP_OK) {
+            return err;
+        }
+        uint32_t seqNumber;
+        if (mPages[i].getSeqNumber(seqNumber) != ESP_OK) {
+            mFreePageList.push_back(&mPages[i]);
+        } else {
+            auto pos = std::find_if(std::begin(mPageList), std::end(mPageList), [=](const Page& page) -> bool {
+                uint32_t otherSeqNumber;
+                return page.getSeqNumber(otherSeqNumber) == ESP_OK && otherSeqNumber > seqNumber;
+            });
+            if (pos == mPageList.end()) {
+                mPageList.push_back(&mPages[i]);
+            } else {
+                mPageList.insert(pos, &mPages[i]);
+            }
+        }
+    }
+
+    if (mPageList.empty()) {
+        mSeqNumber = 0;
+        return activatePage();
+    }
+    else {
+        uint32_t lastSeqNo;
+        assert(mPageList.back().getSeqNumber(lastSeqNo) == ESP_OK);
+        mSeqNumber = lastSeqNo + 1;
+    }
+
+    return ESP_OK;
+}
+
+esp_err_t PageManager::requestNewPage()
+{
+    if (mFreePageList.empty()) {
+        return ESP_ERR_NVS_INVALID_STATE;
+    }
+
+    // do we have at least two free pages? in that case no erasing is required
+    if (mFreePageList.size() >= 2) {
+        return activatePage();
+    }
+
+    // find the page with the higest number of erased items
+    TPageListIterator maxErasedItemsPageIt;
+    size_t maxErasedItems = 0;
+    for (auto it = begin(); it != end(); ++it) {
+        auto erased = it->getErasedEntryCount();
+        if (erased > maxErasedItems) {
+            maxErasedItemsPageIt = it;
+            maxErasedItems = erased;
+        }
+    }
+
+    if (maxErasedItems == 0) {
+        return ESP_ERR_NVS_NOT_ENOUGH_SPACE;
+    }
+
+    esp_err_t err = activatePage();
+    if (err != ESP_OK) {
+        return err;
+    }
+    
+    Page* newPage = &mPageList.back();
+
+    Page* erasedPage = maxErasedItemsPageIt;
+    err = erasedPage->markFreeing();
+    if (err != ESP_OK) {
+        return err;
+    }
+    while (true) {
+        err = erasedPage->moveItem(*newPage);
+        if (err == ESP_ERR_NVS_NOT_FOUND) {
+            break;
+        } else if (err != ESP_OK) {
+            return err;
+        }
+    }
+
+    err = erasedPage->erase();
+    if (err != ESP_OK) {
+        return err;
+    }
+
+    mPageList.erase(maxErasedItemsPageIt);
+    mFreePageList.push_back(erasedPage);
+
+    return ESP_OK;
+}
+
+esp_err_t PageManager::activatePage()
+{
+    if (mFreePageList.empty()) {
+        return ESP_ERR_NVS_NOT_ENOUGH_SPACE;
+    }
+    Page* p = &mFreePageList.front();
+    if (p->state() == Page::PageState::CORRUPT) {
+        auto err = p->erase();
+        if (err != ESP_OK) {
+            return err;
+        }
+    }
+    mFreePageList.pop_front();
+    mPageList.push_back(p);
+    p->setSeqNumber(mSeqNumber);
+    ++mSeqNumber;
+    return ESP_OK;
+}
+    
+} // namespace nvs

components/nvs_flash/src/nvs_pagemanager.hpp

@@ -0,0 +1,70 @@
+// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef nvs_pagemanager_hpp
+#define nvs_pagemanager_hpp
+
+#include <memory>
+#include <list>
+#include "nvs_types.hpp"
+#include "nvs_page.hpp"
+#include "nvs_pagemanager.hpp"
+#include "intrusive_list.h"
+
+namespace nvs
+{
+class PageManager
+{
+    using TPageList = intrusive_list<Page>;
+    using TPageListIterator = TPageList::iterator;
+public:
+
+    PageManager() {}
+
+    esp_err_t load(uint32_t baseSector, uint32_t sectorCount);
+
+    TPageListIterator begin()
+    {
+        return mPageList.begin();
+    }
+
+    TPageListIterator end()
+    {
+        return mPageList.end();
+    }
+
+    Page& back()
+    {
+        return mPageList.back();
+    }
+
+    esp_err_t requestNewPage();
+
+protected:
+    friend class Iterator;
+    
+    esp_err_t activatePage();
+
+    TPageList mPageList;
+    TPageList mFreePageList;
+    std::unique_ptr<Page[]> mPages;
+    uint32_t mBaseSector;
+    uint32_t mPageCount;
+    uint32_t mSeqNumber;
+}; // class PageManager
+
+
+} // namespace nvs
+
+
+#endif /* nvs_pagemanager_hpp */

components/nvs_flash/src/nvs_platform.hpp

@@ -0,0 +1,84 @@
+// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef nvs_platform_h
+#define nvs_platform_h
+
+
+#ifdef ESP_PLATFORM
+#define NVS_DEBUGV(...) ets_printf(__VA_ARGS__)
+
+#include "rom/ets_sys.h"
+#include "freertos/FreeRTOS.h"
+#include "freertos/semphr.h"
+
+namespace nvs
+{
+
+class Lock
+{
+public:
+    Lock()
+    {
+        assert(mSemaphore);
+        xSemaphoreTake(mSemaphore, portMAX_DELAY);
+    }
+
+    ~Lock()
+    {
+        assert(mSemaphore);
+        xSemaphoreGive(mSemaphore);
+    }
+
+    static esp_err_t init()
+    {
+        assert(mSemaphore == nullptr);
+        mSemaphore = xSemaphoreCreateMutex();
+        if (!mSemaphore) {
+            return ESP_ERR_NO_MEM;
+        }
+        return ESP_OK;
+    }
+
+    static void uninit()
+    {
+        vSemaphoreDelete(mSemaphore);
+        mSemaphore = nullptr;
+    }
+
+    static SemaphoreHandle_t mSemaphore;
+};
+} // namespace nvs
+
+#else // ESP_PLATFORM
+#define NVS_DEBUGV(...) printf(__VA_ARGS__)
+namespace nvs
+{
+class Lock
+{
+public:
+    Lock() { }
+    ~Lock() { }
+    static void init() {}
+    static void uninit() {}
+};
+} // namespace nvs
+#endif // ESP_PLATFORM
+
+#ifndef CONFIG_NVS_DEBUG
+#undef NVS_DEBUGV
+#define NVS_DEBUGV(...)
+#endif
+
+
+#endif /* nvs_platform_h */

components/nvs_flash/src/nvs_storage.cpp

@@ -0,0 +1,195 @@
+// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "nvs_storage.hpp"
+
+namespace nvs
+{
+
+Storage::~Storage()
+{
+    clearNamespaces();
+}
+
+void Storage::clearNamespaces()
+{
+    for (auto it = std::begin(mNamespaces); it != std::end(mNamespaces); ) {
+        auto tmp = it;
+        ++it;
+        mNamespaces.erase(tmp);
+        delete static_cast<NamespaceEntry*>(tmp);
+    }
+}
+
+esp_err_t Storage::init(uint32_t baseSector, uint32_t sectorCount)
+{
+    auto err = mPageManager.load(baseSector, sectorCount);
+    if (err != ESP_OK) {
+        mState = StorageState::INVALID;
+        return err;
+    }
+
+    clearNamespaces();
+    std::fill_n(mNamespaceUsage.data(), mNamespaceUsage.byteSize() / 4, 0);
+    for (auto it = mPageManager.begin(); it != mPageManager.end(); ++it) {
+        Page& p = *it;
+        size_t itemIndex = 0;
+        Item item;
+        while(p.findItem(Page::NS_INDEX, ItemType::U8, nullptr, itemIndex, item) == ESP_OK) {
+            NamespaceEntry* entry = new NamespaceEntry;
+            item.getKey(entry->mName, sizeof(entry->mName) - 1);
+            item.getValue(entry->mIndex);
+            mNamespaces.push_back(entry);
+            mNamespaceUsage.set(entry->mIndex, true);
+        }
+    }
+    mNamespaceUsage.set(0, true);
+    mNamespaceUsage.set(255, true);
+    mState = StorageState::ACTIVE;
+    return ESP_OK;
+}
+
+esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize)
+{
+    if (mState != StorageState::ACTIVE) {
+        return ESP_ERR_NVS_NOT_INITIALIZED;
+    }
+
+    Page* findPage = nullptr;
+    Item item;
+    auto err = findItem(nsIndex, datatype, key, findPage, item);
+    if (err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) {
+        return err;
+    }
+
+    Page& page = getCurrentPage();
+    err = page.writeItem(nsIndex, datatype, key, data, dataSize);
+    if (err == ESP_ERR_NVS_PAGE_FULL) {
+        page.markFull();
+        err = mPageManager.requestNewPage();
+        if (err != ESP_OK) {
+            return err;
+        }
+
+        err = getCurrentPage().writeItem(nsIndex, datatype, key, data, dataSize);
+        if (err != ESP_OK) {
+            return err;
+        }
+    }
+
+    if (findPage) {
+        if (findPage->state() == Page::PageState::UNINITIALIZED) {
+            auto err = findItem(nsIndex, datatype, key, findPage, item);
+            assert(err == ESP_OK);
+        }
+        err = findPage->eraseItem(nsIndex, datatype, key);
+        if (err != ESP_OK) {
+            return err;
+        }
+    }
+
+    return ESP_OK;
+}
+
+esp_err_t Storage::createOrOpenNamespace(const char* nsName, bool canCreate, uint8_t& nsIndex)
+{
+    if (mState != StorageState::ACTIVE) {
+        return ESP_ERR_NVS_NOT_INITIALIZED;
+    }
+    auto it = std::find_if(mNamespaces.begin(), mNamespaces.end(), [=] (const NamespaceEntry& e) -> bool {
+        return strncmp(nsName, e.mName, sizeof(e.mName) - 1) == 0;
+    });
+    if (it == std::end(mNamespaces)) {
+        if (!canCreate) {
+            return ESP_ERR_NVS_NOT_FOUND;
+        }
+
+        uint8_t ns;
+        for (ns = 1; ns < 255; ++ns) {
+            if (mNamespaceUsage.get(ns) == false) {
+                break;
+            }
+        }
+
+        if (ns == 255) {
+            return ESP_ERR_NVS_NOT_ENOUGH_SPACE;
+        }
+
+        auto err = writeItem(Page::NS_INDEX, ItemType::U8, nsName, &ns, sizeof(ns));
+        if (err != ESP_OK) {
+            return err;
+        }
+        mNamespaceUsage.set(ns, true);
+        nsIndex = ns;
+        
+        NamespaceEntry* entry = new NamespaceEntry;
+        entry->mIndex = ns;
+        strlcpy(entry->mName, nsName, sizeof(entry->mName));
+        mNamespaces.push_back(entry);
+
+    } else {
+        nsIndex = it->mIndex;
+    }
+    return ESP_OK;
+}
+
+esp_err_t Storage::readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize)
+{
+    if (mState != StorageState::ACTIVE) {
+        return ESP_ERR_NVS_NOT_INITIALIZED;
+    }
+
+    Item item;
+    Page* findPage = nullptr;
+    auto err = findItem(nsIndex, datatype, key, findPage, item);
+    if (err != ESP_OK) {
+        return err;
+    }
+
+    return findPage->readItem(nsIndex, datatype, key, data, dataSize);
+}
+
+esp_err_t Storage::eraseItem(uint8_t nsIndex, ItemType datatype, const char* key)
+{
+    if (mState != StorageState::ACTIVE) {
+        return ESP_ERR_NVS_NOT_INITIALIZED;
+    }
+
+    Item item;
+    Page* findPage = nullptr;
+    auto err = findItem(nsIndex, datatype, key, findPage, item);
+    if (err != ESP_OK) {
+        return err;
+    }
+
+    return findPage->eraseItem(nsIndex, datatype, key);
+}
+
+esp_err_t Storage::getItemDataSize(uint8_t nsIndex, ItemType datatype, const char* key, size_t& dataSize)
+{
+    if (mState != StorageState::ACTIVE) {
+        return ESP_ERR_NVS_NOT_INITIALIZED;
+    }
+
+    Item item;
+    Page* findPage = nullptr;
+    auto err = findItem(nsIndex, datatype, key, findPage, item);
+    if (err != ESP_OK) {
+        return err;
+    }
+
+    dataSize = item.varLength.dataSize;
+    return ESP_OK;
+}
+
+}

components/nvs_flash/src/nvs_storage.hpp

@@ -0,0 +1,114 @@
+// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef nvs_storage_hpp
+#define nvs_storage_hpp
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include "nvs.hpp"
+#include "nvs_types.hpp"
+#include "nvs_page.hpp"
+#include "nvs_pagemanager.hpp"
+
+//extern void dumpBytes(const uint8_t* data, size_t count);
+
+namespace nvs
+{
+
+class Storage
+{
+    enum class StorageState : uint32_t {
+        INVALID,
+        ACTIVE,
+    };
+
+    struct NamespaceEntry : public intrusive_list_node<NamespaceEntry> {
+    public:
+        char mName[Item::MAX_KEY_LENGTH + 1];
+        uint8_t mIndex;
+    };
+
+    typedef intrusive_list<NamespaceEntry> TNamespaces;
+
+public:
+    ~Storage();
+
+    esp_err_t init(uint32_t baseSector, uint32_t sectorCount);
+
+    esp_err_t createOrOpenNamespace(const char* nsName, bool canCreate, uint8_t& nsIndex);
+
+    esp_err_t writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize);
+
+    esp_err_t readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize);
+
+    esp_err_t getItemDataSize(uint8_t nsIndex, ItemType datatype, const char* key, size_t& dataSize);
+
+    esp_err_t eraseItem(uint8_t nsIndex, ItemType datatype, const char* key);
+
+    template<typename T>
+    esp_err_t writeItem(uint8_t nsIndex, const char* key, const T& value)
+    {
+        return writeItem(nsIndex, itemTypeOf(value), key, &value, sizeof(value));
+    }
+
+    template<typename T>
+    esp_err_t readItem(uint8_t nsIndex, const char* key, T& value)
+    {
+        return readItem(nsIndex, itemTypeOf(value), key, &value, sizeof(value));
+    }
+
+    template<typename T>
+    esp_err_t eraseItem(uint8_t nsIndex, const char* key)
+    {
+        return eraseItem(nsIndex, itemTypeOf<T>(), key);
+    }
+
+
+protected:
+
+    Page& getCurrentPage()
+    {
+        return mPageManager.back();
+    }
+
+    void clearNamespaces();
+
+    esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item)
+    {
+        size_t itemIndex = 0;
+        for (auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) {
+            auto err = it->findItem(nsIndex, datatype, key, itemIndex, item);
+            if (err == ESP_OK) {
+                page = it;
+                return ESP_OK;
+            }
+        }
+        return ESP_ERR_NVS_NOT_FOUND;
+    }
+
+
+protected:
+    size_t mPageCount;
+    PageManager mPageManager;
+    TNamespaces mNamespaces;
+    CompressedEnumTable<bool, 1, 256> mNamespaceUsage;
+    StorageState mState = StorageState::INVALID;
+};
+
+} // namespace nvs
+
+
+
+#endif /* nvs_storage_hpp */

components/nvs_flash/src/nvs_types.cpp

@@ -0,0 +1,42 @@
+// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "nvs_types.hpp"
+
+#if defined(ESP_PLATFORM)
+#include <rom/crc.h>
+#else
+#include "crc.h"
+#endif
+
+namespace nvs
+{
+uint32_t Item::calculateCrc32()
+{
+    uint32_t result = 0xffffffff;
+    const uint8_t* p = reinterpret_cast<const uint8_t*>(this);
+    result = crc32_le(result, p + offsetof(Item, nsIndex),
+                      offsetof(Item, crc32) - offsetof(Item, nsIndex));
+    result = crc32_le(result, p + offsetof(Item, key), sizeof(key));
+    result = crc32_le(result, p + offsetof(Item, data), sizeof(data));
+    return result;
+}
+
+uint32_t Item::calculateCrc32(const uint8_t* data, size_t size)
+{
+    uint32_t result = 0xffffffff;
+    result = crc32_le(result, data, size);
+    return result;
+}
+
+} // namespace nvs

components/nvs_flash/src/nvs_types.hpp

@@ -0,0 +1,100 @@
+// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef nvs_types_h
+#define nvs_types_h
+
+#include <cstdint>
+#include <type_traits>
+#include <cstring>
+#include <cassert>
+#include <algorithm>
+#include "nvs.h"
+#include "compressed_enum_table.hpp"
+
+
+namespace nvs
+{
+
+enum class ItemType : uint8_t {
+    U8   = 0x01,
+    I8   = 0x11,
+    U16  = 0x02,
+    I16  = 0x12,
+    U32  = 0x04,
+    I32  = 0x14,
+    U64  = 0x08,
+    I64  = 0x18,
+    SZ   = 0x21,
+    BLOB = 0x41,
+    ANY  = 0xff
+};
+
+template<typename T, typename std::enable_if<std::is_integral<T>::value, void*>::type = nullptr>
+constexpr ItemType itemTypeOf()
+{
+    return static_cast<ItemType>(((std::is_signed<T>::value)?0x10:0x00) | sizeof(T));
+}
+
+template<typename T>
+constexpr ItemType itemTypeOf(const T&)
+{
+    return itemTypeOf<T>();
+}
+
+class Item
+{
+public:
+    union {
+        struct {
+            uint8_t  nsIndex;
+            ItemType datatype;
+            uint8_t  span;
+            uint8_t  reserved;
+            uint32_t crc32;
+            char     key[16];
+            union {
+                struct {
+                    uint16_t dataSize;
+                    uint16_t reserved2;
+                    uint32_t dataCrc32;
+                } varLength;
+                uint8_t data[8];
+            };
+        };
+        uint8_t rawData[32];
+    };
+
+    static const size_t MAX_KEY_LENGTH = sizeof(key) - 1;
+
+    uint32_t calculateCrc32();
+    static uint32_t calculateCrc32(const uint8_t* data, size_t size);
+
+    void getKey(char* dst, size_t dstSize)
+    {
+        strncpy(dst, key, (dstSize<MAX_KEY_LENGTH)?dstSize:MAX_KEY_LENGTH);
+    }
+
+    template<typename T>
+    void getValue(T& dst)
+    {
+        assert(itemTypeOf(dst) == datatype);
+        dst = *reinterpret_cast<T*>(data);
+    }
+};
+
+} // namespace nvs
+
+
+
+#endif /* nvs_types_h */