fs: introduce ZMS a new Memory storage system

ZMS is the abreviation of Zephyr Memory Storage.
It is a storage developed to target especially the non erasable devices.

The new memory storage system inherit from the NVS storage multiple
features and introduce new ones :
* Inherited features :
 - light key-value based storage
 - cache for entries
 - Wear Leveling of flash memory
 - Resilience to power failures
* New features :
 - cycle counter for non erasable devices (instead of erase emulation)
 - Keys up to 32-bit
 - Built-in support of CRC32 for data
 - Small size data (<= 8 bytes) integrated within entries

Signed-off-by: Riadh Ghaddab <rghaddab@baylibre.com>

include/zephyr/fs/zms.h

@@ -0,0 +1,215 @@
+/*  ZMS: Zephyr Memory Storage
+ *
+ * Copyright (c) 2024 BayLibre SAS
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#ifndef ZEPHYR_INCLUDE_FS_ZMS_H_
+#define ZEPHYR_INCLUDE_FS_ZMS_H_
+
+#include <zephyr/drivers/flash.h>
+#include <sys/types.h>
+#include <zephyr/kernel.h>
+#include <zephyr/device.h>
+#include <zephyr/toolchain.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @brief Zephyr Memory Storage (ZMS)
+ * @defgroup zms Zephyr Memory Storage (ZMS)
+ * @ingroup file_system_storage
+ * @{
+ * @}
+ */
+
+/**
+ * @brief Zephyr Memory Storage Data Structures
+ * @defgroup zms_data_structures Zephyr Memory Storage Data Structures
+ * @ingroup zms
+ * @{
+ */
+
+/**
+ * @brief Zephyr Memory Storage File system structure
+ */
+struct zms_fs {
+	/** File system offset in flash **/
+	off_t offset;
+	/** Allocation table entry write address.
+	 * Addresses are stored as uint64_t:
+	 * - high 4 bytes correspond to the sector
+	 * - low 4 bytes are the offset in the sector
+	 */
+	uint64_t ate_wra;
+	/** Data write address */
+	uint64_t data_wra;
+	/** Storage system is split into sectors, each sector size must be multiple of erase-blocks
+	 *  if the device has erase capabilities
+	 */
+	uint32_t sector_size;
+	/** Number of sectors in the file system */
+	uint32_t sector_count;
+	/** Current cycle counter of the active sector (pointed by ate_wra)*/
+	uint8_t sector_cycle;
+	/** Flag indicating if the file system is initialized */
+	bool ready;
+	/** Mutex */
+	struct k_mutex zms_lock;
+	/** Flash device runtime structure */
+	const struct device *flash_device;
+	/** Flash memory parameters structure */
+	const struct flash_parameters *flash_parameters;
+	/** Size of an Allocation Table Entry */
+	size_t ate_size;
+#if CONFIG_ZMS_LOOKUP_CACHE
+	/** Lookup table used to cache ATE address of a written ID */
+	uint64_t lookup_cache[CONFIG_ZMS_LOOKUP_CACHE_SIZE];
+#endif
+};
+
+/**
+ * @}
+ */
+
+/**
+ * @brief Zephyr Memory Storage APIs
+ * @defgroup zms_high_level_api Zephyr Memory Storage APIs
+ * @ingroup zms
+ * @{
+ */
+
+/**
+ * @brief Mount a ZMS file system onto the device specified in @p fs.
+ *
+ * @param fs Pointer to file system
+ * @retval 0 Success
+ * @retval -ERRNO errno code if error
+ */
+int zms_mount(struct zms_fs *fs);
+
+/**
+ * @brief Clear the ZMS file system from device.
+ *
+ * @param fs Pointer to file system
+ * @retval 0 Success
+ * @retval -ERRNO errno code if error
+ */
+int zms_clear(struct zms_fs *fs);
+
+/**
+ * @brief Write an entry to the file system.
+ *
+ * @note  When @p len parameter is equal to @p 0 then entry is effectively removed (it is
+ * equivalent to calling of zms_delete). It is not possible to distinguish between a deleted
+ * entry and an entry with data of length 0.
+ *
+ * @param fs Pointer to file system
+ * @param id Id of the entry to be written
+ * @param data Pointer to the data to be written
+ * @param len Number of bytes to be written (maximum 64 KB)
+ *
+ * @return Number of bytes written. On success, it will be equal to the number of bytes requested
+ * to be written. When a rewrite of the same data already stored is attempted, nothing is written
+ * to flash, thus 0 is returned. On error, returns negative value of errno.h defined error codes.
+ */
+ssize_t zms_write(struct zms_fs *fs, uint32_t id, const void *data, size_t len);
+
+/**
+ * @brief Delete an entry from the file system
+ *
+ * @param fs Pointer to file system
+ * @param id Id of the entry to be deleted
+ * @retval 0 Success
+ * @retval -ERRNO errno code if error
+ */
+int zms_delete(struct zms_fs *fs, uint32_t id);
+
+/**
+ * @brief Read an entry from the file system.
+ *
+ * @param fs Pointer to file system
+ * @param id Id of the entry to be read
+ * @param data Pointer to data buffer
+ * @param len Number of bytes to be read (or size of the allocated read buffer)
+ *
+ * @return Number of bytes read. On success, it will be equal to the number of bytes requested
+ * to be read. When the return value is less than the number of bytes requested to read this
+ * indicates that ATE contain less data than requested. On error, returns negative value of
+ * errno.h defined error codes.
+ */
+ssize_t zms_read(struct zms_fs *fs, uint32_t id, void *data, size_t len);
+
+/**
+ * @brief Read a history entry from the file system.
+ *
+ * @param fs Pointer to file system
+ * @param id Id of the entry to be read
+ * @param data Pointer to data buffer
+ * @param len Number of bytes to be read
+ * @param cnt History counter: 0: latest entry, 1: one before latest ...
+ *
+ * @return Number of bytes read. On success, it will be equal to the number of bytes requested
+ * to be read. When the return value is larger than the number of bytes requested to read this
+ * indicates not all bytes were read, and more data is available. On error, returns negative
+ * value of errno.h defined error codes.
+ */
+ssize_t zms_read_hist(struct zms_fs *fs, uint32_t id, void *data, size_t len, uint32_t cnt);
+
+/**
+ * @brief Gets the data size that is stored in an entry with a given id
+ *
+ * @param fs Pointer to file system
+ * @param id Id of the entry that we want to get its data length
+ *
+ * @return Data length contained in the ATE. On success, it will be equal to the number of bytes
+ * in the ATE. On error, returns negative value of errno.h defined error codes.
+ */
+ssize_t zms_get_data_length(struct zms_fs *fs, uint32_t id);
+/**
+ * @brief Calculate the available free space in the file system.
+ *
+ * @param fs Pointer to file system
+ *
+ * @return Number of bytes free. On success, it will be equal to the number of bytes that can
+ * still be written to the file system.
+ * Calculating the free space is a time consuming operation, especially on spi flash.
+ * On error, returns negative value of errno.h defined error codes.
+ */
+ssize_t zms_calc_free_space(struct zms_fs *fs);
+
+/**
+ * @brief Tell how many contiguous free space remains in the currently active ZMS sector.
+ *
+ * @param fs Pointer to the file system.
+ *
+ * @return Number of free bytes.
+ */
+size_t zms_sector_max_data_size(struct zms_fs *fs);
+
+/**
+ * @brief Close the currently active sector and switch to the next one.
+ *
+ * @note The garbage collector is called on the new sector.
+ *
+ * @warning This routine is made available for specific use cases.
+ * It collides with the ZMS goal of avoiding any unnecessary flash erase operations.
+ * Using this routine extensively can result in premature failure of the flash device.
+ *
+ * @param fs Pointer to the file system.
+ *
+ * @return 0 on success. On error, returns negative value of errno.h defined error codes.
+ */
+int zms_sector_use_next(struct zms_fs *fs);
+
+/**
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ZEPHYR_INCLUDE_FS_ZMS_H_ */

subsys/fs/CMakeLists.txt

@@ -27,6 +27,7 @@ endif()
 
 add_subdirectory_ifdef(CONFIG_FCB  ./fcb)
 add_subdirectory_ifdef(CONFIG_NVS  ./nvs)
+add_subdirectory_ifdef(CONFIG_ZMS  ./zms)
 
 if(CONFIG_FUSE_FS_ACCESS)
   zephyr_library_named(FS_FUSE)

subsys/fs/Kconfig

@@ -110,5 +110,6 @@ endif # FILE_SYSTEM
 
 rsource "fcb/Kconfig"
 rsource "nvs/Kconfig"
+rsource "zms/Kconfig"
 
 endmenu

subsys/fs/zms/CMakeLists.txt

@@ -0,0 +1,3 @@
+#SPDX-License-Identifier: Apache-2.0
+
+zephyr_sources(zms.c)

subsys/fs/zms/Kconfig

@@ -0,0 +1,57 @@
+#Zephyr Memory Storage ZMS
+
+#Copyright (c) 2024 BayLibre SAS
+
+#SPDX-License-Identifier: Apache-2.0
+
+config ZMS
+	bool "Zephyr Memory Storage"
+	select CRC
+	help
+	  Enable support of Zephyr Memory Storage.
+
+if ZMS
+
+config ZMS_LOOKUP_CACHE
+	bool "ZMS lookup cache"
+	help
+	  Enable ZMS cache to reduce the ZMS data lookup time.
+	  Each cache entry holds an address of the most recent allocation
+	  table entry (ATE) for all ZMS IDs that fall into that cache position.
+
+config ZMS_LOOKUP_CACHE_SIZE
+	int "ZMS Storage lookup cache size"
+	default 128
+	range 1 65536
+	depends on ZMS_LOOKUP_CACHE
+	help
+	  Number of entries in ZMS lookup cache.
+	  It is recommended that it should be a power of 2.
+	  Every additional entry in cache will add 8 bytes in RAM
+
+config ZMS_DATA_CRC
+	bool "ZMS DATA CRC"
+	help
+	  Enables DATA CRC
+
+config ZMS_CUSTOM_BLOCK_SIZE
+	bool "Custom buffer size used by ZMS for reads and writes"
+	help
+	  ZMS uses internal buffers to read/write and compare stored data.
+	  Increasing the size of these buffers should be done carefully in order to not
+	  overflow the stack.
+	  Increasing this buffer means as well that ZMS could work with storage devices
+	  that have larger write-block-size which decreases ZMS performance
+
+config ZMS_MAX_BLOCK_SIZE
+	int "ZMS internal buffer size"
+	default 32
+	depends on ZMS_CUSTOM_BLOCK_SIZE
+	help
+	  Changes the internal buffer size of ZMS
+
+module = ZMS
+module-str = zms
+source "subsys/logging/Kconfig.template.log_config"
+
+endif # ZMS

subsys/fs/zms/zms_priv.h

@@ -0,0 +1,76 @@
+/*  ZMS: Zephyr Memory Storage
+ *
+ * Copyright (c) 2024 BayLibre SAS
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#ifndef __ZMS_PRIV_H_
+#define __ZMS_PRIV_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * MASKS AND SHIFT FOR ADDRESSES
+ * an address in zms is an uint64_t where:
+ *   high 4 bytes represent the sector number
+ *   low 4 bytes represent the offset in a sector
+ */
+#define ADDR_SECT_MASK   GENMASK64(63, 32)
+#define ADDR_SECT_SHIFT  32
+#define ADDR_OFFS_MASK   GENMASK64(31, 0)
+#define SECTOR_NUM(x)    FIELD_GET(ADDR_SECT_MASK, x)
+#define SECTOR_OFFSET(x) FIELD_GET(ADDR_OFFS_MASK, x)
+
+#if defined(CONFIG_ZMS_CUSTOM_BLOCK_SIZE)
+#define ZMS_BLOCK_SIZE CONFIG_ZMS_MAX_BLOCK_SIZE
+#else
+#define ZMS_BLOCK_SIZE 32
+#endif
+
+#define ZMS_LOOKUP_CACHE_NO_ADDR GENMASK64(63, 0)
+#define ZMS_HEAD_ID              GENMASK(31, 0)
+
+#define ZMS_VERSION_MASK        GENMASK(7, 0)
+#define ZMS_GET_VERSION(x)      FIELD_GET(ZMS_VERSION_MASK, x)
+#define ZMS_DEFAULT_VERSION     1
+#define ZMS_MAGIC_NUMBER        0x42 /* murmur3a hash of "ZMS" (MSB) */
+#define ZMS_MAGIC_NUMBER_MASK   GENMASK(15, 8)
+#define ZMS_GET_MAGIC_NUMBER(x) FIELD_GET(ZMS_MAGIC_NUMBER_MASK, x)
+#define ZMS_MIN_ATE_NUM         5
+
+#define ZMS_INVALID_SECTOR_NUM -1
+#define ZMS_DATA_IN_ATE_SIZE   8
+
+struct zms_ate {
+	uint8_t crc8;      /* crc8 check of the entry */
+	uint8_t cycle_cnt; /* cycle counter for non erasable devices */
+	uint32_t id;       /* data id */
+	uint16_t len;      /* data len within sector */
+	union {
+		uint8_t data[8]; /* used to store small size data */
+		struct {
+			uint32_t offset; /* data offset within sector */
+			union {
+				uint32_t data_crc; /*
+						    * crc for data: The data CRC is checked only
+						    * when the whole data of the element is read.
+						    * The data CRC is not checked for a partial
+						    * read, as it is computed for the complete
+						    * set of data.
+						    */
+				uint32_t metadata; /*
+						    * Used to store metadata information
+						    * such as storage version.
+						    */
+			};
+		};
+	};
+} __packed;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __ZMS_PRIV_H_ */