fs: introduce ZMS a new Memory storage system
ZMS is the abreviation of Zephyr Memory Storage. It is a storage developed to target especially the non erasable devices. The new memory storage system inherit from the NVS storage multiple features and introduce new ones : * Inherited features : - light key-value based storage - cache for entries - Wear Leveling of flash memory - Resilience to power failures * New features : - cycle counter for non erasable devices (instead of erase emulation) - Keys up to 32-bit - Built-in support of CRC32 for data - Small size data (<= 8 bytes) integrated within entries Signed-off-by: Riadh Ghaddab <rghaddab@baylibre.com>
This commit is contained in:
Riadh Ghaddab
Fabio Baltieri
parent
96f08d53a4
commit
d4e246dfa1
7 changed files with 2105 additions and 0 deletions
215
include/zephyr/fs/zms.h
Normal file
215
include/zephyr/fs/zms.h
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
/* ZMS: Zephyr Memory Storage
|
||||
*
|
||||
* Copyright (c) 2024 BayLibre SAS
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
#ifndef ZEPHYR_INCLUDE_FS_ZMS_H_
|
||||
#define ZEPHYR_INCLUDE_FS_ZMS_H_
|
||||
|
||||
#include <zephyr/drivers/flash.h>
|
||||
#include <sys/types.h>
|
||||
#include <zephyr/kernel.h>
|
||||
#include <zephyr/device.h>
|
||||
#include <zephyr/toolchain.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/**
|
||||
* @brief Zephyr Memory Storage (ZMS)
|
||||
* @defgroup zms Zephyr Memory Storage (ZMS)
|
||||
* @ingroup file_system_storage
|
||||
* @{
|
||||
* @}
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief Zephyr Memory Storage Data Structures
|
||||
* @defgroup zms_data_structures Zephyr Memory Storage Data Structures
|
||||
* @ingroup zms
|
||||
* @{
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief Zephyr Memory Storage File system structure
|
||||
*/
|
||||
struct zms_fs {
|
||||
/** File system offset in flash **/
|
||||
off_t offset;
|
||||
/** Allocation table entry write address.
|
||||
* Addresses are stored as uint64_t:
|
||||
* - high 4 bytes correspond to the sector
|
||||
* - low 4 bytes are the offset in the sector
|
||||
*/
|
||||
uint64_t ate_wra;
|
||||
/** Data write address */
|
||||
uint64_t data_wra;
|
||||
/** Storage system is split into sectors, each sector size must be multiple of erase-blocks
|
||||
* if the device has erase capabilities
|
||||
*/
|
||||
uint32_t sector_size;
|
||||
/** Number of sectors in the file system */
|
||||
uint32_t sector_count;
|
||||
/** Current cycle counter of the active sector (pointed by ate_wra)*/
|
||||
uint8_t sector_cycle;
|
||||
/** Flag indicating if the file system is initialized */
|
||||
bool ready;
|
||||
/** Mutex */
|
||||
struct k_mutex zms_lock;
|
||||
/** Flash device runtime structure */
|
||||
const struct device *flash_device;
|
||||
/** Flash memory parameters structure */
|
||||
const struct flash_parameters *flash_parameters;
|
||||
/** Size of an Allocation Table Entry */
|
||||
size_t ate_size;
|
||||
#if CONFIG_ZMS_LOOKUP_CACHE
|
||||
/** Lookup table used to cache ATE address of a written ID */
|
||||
uint64_t lookup_cache[CONFIG_ZMS_LOOKUP_CACHE_SIZE];
|
||||
#endif
|
||||
};
|
||||
|
||||
/**
|
||||
* @}
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief Zephyr Memory Storage APIs
|
||||
* @defgroup zms_high_level_api Zephyr Memory Storage APIs
|
||||
* @ingroup zms
|
||||
* @{
|
||||
*/
|
||||
|
||||
/**
|
||||
* @brief Mount a ZMS file system onto the device specified in @p fs.
|
||||
*
|
||||
* @param fs Pointer to file system
|
||||
* @retval 0 Success
|
||||
* @retval -ERRNO errno code if error
|
||||
*/
|
||||
int zms_mount(struct zms_fs *fs);
|
||||
|
||||
/**
|
||||
* @brief Clear the ZMS file system from device.
|
||||
*
|
||||
* @param fs Pointer to file system
|
||||
* @retval 0 Success
|
||||
* @retval -ERRNO errno code if error
|
||||
*/
|
||||
int zms_clear(struct zms_fs *fs);
|
||||
|
||||
/**
|
||||
* @brief Write an entry to the file system.
|
||||
*
|
||||
* @note When @p len parameter is equal to @p 0 then entry is effectively removed (it is
|
||||
* equivalent to calling of zms_delete). It is not possible to distinguish between a deleted
|
||||
* entry and an entry with data of length 0.
|
||||
*
|
||||
* @param fs Pointer to file system
|
||||
* @param id Id of the entry to be written
|
||||
* @param data Pointer to the data to be written
|
||||
* @param len Number of bytes to be written (maximum 64 KB)
|
||||
*
|
||||
* @return Number of bytes written. On success, it will be equal to the number of bytes requested
|
||||
* to be written. When a rewrite of the same data already stored is attempted, nothing is written
|
||||
* to flash, thus 0 is returned. On error, returns negative value of errno.h defined error codes.
|
||||
*/
|
||||
ssize_t zms_write(struct zms_fs *fs, uint32_t id, const void *data, size_t len);
|
||||
|
||||
/**
|
||||
* @brief Delete an entry from the file system
|
||||
*
|
||||
* @param fs Pointer to file system
|
||||
* @param id Id of the entry to be deleted
|
||||
* @retval 0 Success
|
||||
* @retval -ERRNO errno code if error
|
||||
*/
|
||||
int zms_delete(struct zms_fs *fs, uint32_t id);
|
||||
|
||||
/**
|
||||
* @brief Read an entry from the file system.
|
||||
*
|
||||
* @param fs Pointer to file system
|
||||
* @param id Id of the entry to be read
|
||||
* @param data Pointer to data buffer
|
||||
* @param len Number of bytes to be read (or size of the allocated read buffer)
|
||||
*
|
||||
* @return Number of bytes read. On success, it will be equal to the number of bytes requested
|
||||
* to be read. When the return value is less than the number of bytes requested to read this
|
||||
* indicates that ATE contain less data than requested. On error, returns negative value of
|
||||
* errno.h defined error codes.
|
||||
*/
|
||||
ssize_t zms_read(struct zms_fs *fs, uint32_t id, void *data, size_t len);
|
||||
|
||||
/**
|
||||
* @brief Read a history entry from the file system.
|
||||
*
|
||||
* @param fs Pointer to file system
|
||||
* @param id Id of the entry to be read
|
||||
* @param data Pointer to data buffer
|
||||
* @param len Number of bytes to be read
|
||||
* @param cnt History counter: 0: latest entry, 1: one before latest ...
|
||||
*
|
||||
* @return Number of bytes read. On success, it will be equal to the number of bytes requested
|
||||
* to be read. When the return value is larger than the number of bytes requested to read this
|
||||
* indicates not all bytes were read, and more data is available. On error, returns negative
|
||||
* value of errno.h defined error codes.
|
||||
*/
|
||||
ssize_t zms_read_hist(struct zms_fs *fs, uint32_t id, void *data, size_t len, uint32_t cnt);
|
||||
|
||||
/**
|
||||
* @brief Gets the data size that is stored in an entry with a given id
|
||||
*
|
||||
* @param fs Pointer to file system
|
||||
* @param id Id of the entry that we want to get its data length
|
||||
*
|
||||
* @return Data length contained in the ATE. On success, it will be equal to the number of bytes
|
||||
* in the ATE. On error, returns negative value of errno.h defined error codes.
|
||||
*/
|
||||
ssize_t zms_get_data_length(struct zms_fs *fs, uint32_t id);
|
||||
/**
|
||||
* @brief Calculate the available free space in the file system.
|
||||
*
|
||||
* @param fs Pointer to file system
|
||||
*
|
||||
* @return Number of bytes free. On success, it will be equal to the number of bytes that can
|
||||
* still be written to the file system.
|
||||
* Calculating the free space is a time consuming operation, especially on spi flash.
|
||||
* On error, returns negative value of errno.h defined error codes.
|
||||
*/
|
||||
ssize_t zms_calc_free_space(struct zms_fs *fs);
|
||||
|
||||
/**
|
||||
* @brief Tell how many contiguous free space remains in the currently active ZMS sector.
|
||||
*
|
||||
* @param fs Pointer to the file system.
|
||||
*
|
||||
* @return Number of free bytes.
|
||||
*/
|
||||
size_t zms_sector_max_data_size(struct zms_fs *fs);
|
||||
|
||||
/**
|
||||
* @brief Close the currently active sector and switch to the next one.
|
||||
*
|
||||
* @note The garbage collector is called on the new sector.
|
||||
*
|
||||
* @warning This routine is made available for specific use cases.
|
||||
* It collides with the ZMS goal of avoiding any unnecessary flash erase operations.
|
||||
* Using this routine extensively can result in premature failure of the flash device.
|
||||
*
|
||||
* @param fs Pointer to the file system.
|
||||
*
|
||||
* @return 0 on success. On error, returns negative value of errno.h defined error codes.
|
||||
*/
|
||||
int zms_sector_use_next(struct zms_fs *fs);
|
||||
|
||||
/**
|
||||
* @}
|
||||
*/
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ZEPHYR_INCLUDE_FS_ZMS_H_ */
|
||||
|
|
@ -27,6 +27,7 @@ endif()
|
|||
|
||||
add_subdirectory_ifdef(CONFIG_FCB ./fcb)
|
||||
add_subdirectory_ifdef(CONFIG_NVS ./nvs)
|
||||
add_subdirectory_ifdef(CONFIG_ZMS ./zms)
|
||||
|
||||
if(CONFIG_FUSE_FS_ACCESS)
|
||||
zephyr_library_named(FS_FUSE)
|
||||
|
|
|
|||
|
|
@ -110,5 +110,6 @@ endif # FILE_SYSTEM
|
|||
|
||||
rsource "fcb/Kconfig"
|
||||
rsource "nvs/Kconfig"
|
||||
rsource "zms/Kconfig"
|
||||
|
||||
endmenu
|
||||
|
|
|
|||
3
subsys/fs/zms/CMakeLists.txt
Normal file
3
subsys/fs/zms/CMakeLists.txt
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
#SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
zephyr_sources(zms.c)
|
||||
57
subsys/fs/zms/Kconfig
Normal file
57
subsys/fs/zms/Kconfig
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
#Zephyr Memory Storage ZMS
|
||||
|
||||
#Copyright (c) 2024 BayLibre SAS
|
||||
|
||||
#SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
config ZMS
|
||||
bool "Zephyr Memory Storage"
|
||||
select CRC
|
||||
help
|
||||
Enable support of Zephyr Memory Storage.
|
||||
|
||||
if ZMS
|
||||
|
||||
config ZMS_LOOKUP_CACHE
|
||||
bool "ZMS lookup cache"
|
||||
help
|
||||
Enable ZMS cache to reduce the ZMS data lookup time.
|
||||
Each cache entry holds an address of the most recent allocation
|
||||
table entry (ATE) for all ZMS IDs that fall into that cache position.
|
||||
|
||||
config ZMS_LOOKUP_CACHE_SIZE
|
||||
int "ZMS Storage lookup cache size"
|
||||
default 128
|
||||
range 1 65536
|
||||
depends on ZMS_LOOKUP_CACHE
|
||||
help
|
||||
Number of entries in ZMS lookup cache.
|
||||
It is recommended that it should be a power of 2.
|
||||
Every additional entry in cache will add 8 bytes in RAM
|
||||
|
||||
config ZMS_DATA_CRC
|
||||
bool "ZMS DATA CRC"
|
||||
help
|
||||
Enables DATA CRC
|
||||
|
||||
config ZMS_CUSTOM_BLOCK_SIZE
|
||||
bool "Custom buffer size used by ZMS for reads and writes"
|
||||
help
|
||||
ZMS uses internal buffers to read/write and compare stored data.
|
||||
Increasing the size of these buffers should be done carefully in order to not
|
||||
overflow the stack.
|
||||
Increasing this buffer means as well that ZMS could work with storage devices
|
||||
that have larger write-block-size which decreases ZMS performance
|
||||
|
||||
config ZMS_MAX_BLOCK_SIZE
|
||||
int "ZMS internal buffer size"
|
||||
default 32
|
||||
depends on ZMS_CUSTOM_BLOCK_SIZE
|
||||
help
|
||||
Changes the internal buffer size of ZMS
|
||||
|
||||
module = ZMS
|
||||
module-str = zms
|
||||
source "subsys/logging/Kconfig.template.log_config"
|
||||
|
||||
endif # ZMS
|
||||
1752
subsys/fs/zms/zms.c
Normal file
1752
subsys/fs/zms/zms.c
Normal file
File diff suppressed because it is too large
Load diff
76
subsys/fs/zms/zms_priv.h
Normal file
76
subsys/fs/zms/zms_priv.h
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
/* ZMS: Zephyr Memory Storage
|
||||
*
|
||||
* Copyright (c) 2024 BayLibre SAS
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
#ifndef __ZMS_PRIV_H_
|
||||
#define __ZMS_PRIV_H_
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*
|
||||
* MASKS AND SHIFT FOR ADDRESSES
|
||||
* an address in zms is an uint64_t where:
|
||||
* high 4 bytes represent the sector number
|
||||
* low 4 bytes represent the offset in a sector
|
||||
*/
|
||||
#define ADDR_SECT_MASK GENMASK64(63, 32)
|
||||
#define ADDR_SECT_SHIFT 32
|
||||
#define ADDR_OFFS_MASK GENMASK64(31, 0)
|
||||
#define SECTOR_NUM(x) FIELD_GET(ADDR_SECT_MASK, x)
|
||||
#define SECTOR_OFFSET(x) FIELD_GET(ADDR_OFFS_MASK, x)
|
||||
|
||||
#if defined(CONFIG_ZMS_CUSTOM_BLOCK_SIZE)
|
||||
#define ZMS_BLOCK_SIZE CONFIG_ZMS_MAX_BLOCK_SIZE
|
||||
#else
|
||||
#define ZMS_BLOCK_SIZE 32
|
||||
#endif
|
||||
|
||||
#define ZMS_LOOKUP_CACHE_NO_ADDR GENMASK64(63, 0)
|
||||
#define ZMS_HEAD_ID GENMASK(31, 0)
|
||||
|
||||
#define ZMS_VERSION_MASK GENMASK(7, 0)
|
||||
#define ZMS_GET_VERSION(x) FIELD_GET(ZMS_VERSION_MASK, x)
|
||||
#define ZMS_DEFAULT_VERSION 1
|
||||
#define ZMS_MAGIC_NUMBER 0x42 /* murmur3a hash of "ZMS" (MSB) */
|
||||
#define ZMS_MAGIC_NUMBER_MASK GENMASK(15, 8)
|
||||
#define ZMS_GET_MAGIC_NUMBER(x) FIELD_GET(ZMS_MAGIC_NUMBER_MASK, x)
|
||||
#define ZMS_MIN_ATE_NUM 5
|
||||
|
||||
#define ZMS_INVALID_SECTOR_NUM -1
|
||||
#define ZMS_DATA_IN_ATE_SIZE 8
|
||||
|
||||
struct zms_ate {
|
||||
uint8_t crc8; /* crc8 check of the entry */
|
||||
uint8_t cycle_cnt; /* cycle counter for non erasable devices */
|
||||
uint32_t id; /* data id */
|
||||
uint16_t len; /* data len within sector */
|
||||
union {
|
||||
uint8_t data[8]; /* used to store small size data */
|
||||
struct {
|
||||
uint32_t offset; /* data offset within sector */
|
||||
union {
|
||||
uint32_t data_crc; /*
|
||||
* crc for data: The data CRC is checked only
|
||||
* when the whole data of the element is read.
|
||||
* The data CRC is not checked for a partial
|
||||
* read, as it is computed for the complete
|
||||
* set of data.
|
||||
*/
|
||||
uint32_t metadata; /*
|
||||
* Used to store metadata information
|
||||
* such as storage version.
|
||||
*/
|
||||
};
|
||||
};
|
||||
};
|
||||
} __packed;
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* __ZMS_PRIV_H_ */
|
||||
Loading…
Reference in a new issue