From 86fc43c939c623472c512914cd2631579d89d768 Mon Sep 17 00:00:00 2001 From: Christopher Friedt Date: Sun, 28 May 2023 12:34:14 -0400 Subject: [PATCH] drivers: dma: add emulated dma driver Add an emulated DMA driver. Emulation drivers are great to have for each driver API for multiple reasons: - providing an ideal / model driver for reference - potential for configurable backend support - seamless integration with device tree - multi-instance, etc, for all supported boards - fast regression testing of app and library code Since many other drivers and lbraries depend on DMA, this might help us to increase test coverage. Signed-off-by: Christopher Friedt --- drivers/dma/CMakeLists.txt | 1 + drivers/dma/Kconfig | 2 + drivers/dma/Kconfig.emul | 9 + drivers/dma/dma_emul.c | 617 +++++++++++++++++++++++++++++++++++++ 4 files changed, 629 insertions(+) create mode 100644 drivers/dma/Kconfig.emul create mode 100644 drivers/dma/dma_emul.c diff --git a/drivers/dma/CMakeLists.txt b/drivers/dma/CMakeLists.txt index e65c3726622..75eef49a32c 100644 --- a/drivers/dma/CMakeLists.txt +++ b/drivers/dma/CMakeLists.txt @@ -38,3 +38,4 @@ zephyr_library_sources_ifdef(CONFIG_DMA_ANDES_ATCDMAC300 dma_andes_atcdmac300.c) zephyr_library_sources_ifdef(CONFIG_DMA_SEDI dma_sedi.c) zephyr_library_sources_ifdef(CONFIG_DMA_SMARTBOND dma_smartbond.c) zephyr_library_sources_ifdef(CONFIG_DMA_NXP_SOF_HOST_DMA dma_nxp_sof_host_dma.c) +zephyr_library_sources_ifdef(CONFIG_DMA_EMUL dma_emul.c) diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index e24386558cd..7d2b7f94d90 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -72,4 +72,6 @@ source "drivers/dma/Kconfig.smartbond" source "drivers/dma/Kconfig.nxp_sof_host_dma" +source "drivers/dma/Kconfig.emul" + endif # DMA diff --git a/drivers/dma/Kconfig.emul b/drivers/dma/Kconfig.emul new file mode 100644 index 00000000000..595f577f3d1 --- /dev/null +++ b/drivers/dma/Kconfig.emul @@ -0,0 +1,9 @@ +# Copyright (c) 2023 Meta +# SPDX-License-Identifier: Apache-2.0 + +config DMA_EMUL + bool "Emulated DMA driver [EXPERIMENTAL]" + depends on DT_HAS_ZEPHYR_DMA_EMUL_ENABLED + select EXPERIMENTAL + help + Emulated DMA Driver diff --git a/drivers/dma/dma_emul.c b/drivers/dma/dma_emul.c new file mode 100644 index 00000000000..235593df624 --- /dev/null +++ b/drivers/dma/dma_emul.c @@ -0,0 +1,617 @@ +/* + * Copyright (c) 2023, Meta + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#define DT_DRV_COMPAT zephyr_dma_emul + +#ifdef CONFIG_DMA_64BIT +#define dma_addr_t uint64_t +#else +#define dma_addr_t uint32_t +#endif + +enum dma_emul_channel_state { + DMA_EMUL_CHANNEL_UNUSED, + DMA_EMUL_CHANNEL_LOADED, + DMA_EMUL_CHANNEL_STARTED, + DMA_EMUL_CHANNEL_STOPPED, +}; + +struct dma_emul_xfer_desc { + struct dma_config config; +}; + +struct dma_emul_work { + const struct device *dev; + uint32_t channel; + struct k_work work; +}; + +struct dma_emul_config { + uint32_t channel_mask; + size_t num_channels; + size_t num_requests; + size_t addr_align; + size_t size_align; + size_t copy_align; + + k_thread_stack_t *work_q_stack; + size_t work_q_stack_size; + int work_q_priority; + + /* points to an array of size num_channels */ + struct dma_emul_xfer_desc *xfer; + /* points to an array of size num_channels * num_requests */ + struct dma_block_config *block; +}; + +struct dma_emul_data { + struct dma_context dma_ctx; + atomic_t *channels_atomic; + struct k_spinlock lock; + struct k_work_q work_q; + struct dma_emul_work work; +}; + +static void dma_emul_work_handler(struct k_work *work); + +LOG_MODULE_REGISTER(dma_emul, CONFIG_DMA_LOG_LEVEL); + +static inline bool dma_emul_xfer_is_error_status(int status) +{ + return status < 0; +} + +static inline const char *const dma_emul_channel_state_to_string(enum dma_emul_channel_state state) +{ + switch (state) { + case DMA_EMUL_CHANNEL_UNUSED: + return "UNUSED"; + case DMA_EMUL_CHANNEL_LOADED: + return "LOADED"; + case DMA_EMUL_CHANNEL_STARTED: + return "STARTED"; + case DMA_EMUL_CHANNEL_STOPPED: + return "STOPPED"; + default: + return "(invalid)"; + }; +} + +/* + * Repurpose the "_reserved" field for keeping track of internal + * channel state. + * + * Note: these must be called with data->lock locked! + */ +static enum dma_emul_channel_state dma_emul_get_channel_state(const struct device *dev, + uint32_t channel) +{ + const struct dma_emul_config *config = dev->config; + + __ASSERT_NO_MSG(channel < config->num_channels); + + return (enum dma_emul_channel_state)config->xfer[channel].config._reserved; +} + +static void dma_emul_set_channel_state(const struct device *dev, uint32_t channel, + enum dma_emul_channel_state state) +{ + const struct dma_emul_config *config = dev->config; + + LOG_DBG("setting channel %u state to %s", channel, dma_emul_channel_state_to_string(state)); + + __ASSERT_NO_MSG(channel < config->num_channels); + __ASSERT_NO_MSG(state >= DMA_EMUL_CHANNEL_UNUSED && state <= DMA_EMUL_CHANNEL_STOPPED); + + config->xfer[channel].config._reserved = state; +} + +static const char *dma_emul_xfer_config_to_string(const struct dma_config *cfg) +{ + static char buffer[1024]; + + snprintf(buffer, sizeof(buffer), + "{" + "\n\tslot: %u" + "\n\tchannel_direction: %u" + "\n\tcomplete_callback_en: %u" + "\n\terror_callback_en: %u" + "\n\tsource_handshake: %u" + "\n\tdest_handshake: %u" + "\n\tchannel_priority: %u" + "\n\tsource_chaining_en: %u" + "\n\tdest_chaining_en: %u" + "\n\tlinked_channel: %u" + "\n\tcyclic: %u" + "\n\t_reserved: %u" + "\n\tsource_data_size: %u" + "\n\tdest_data_size: %u" + "\n\tsource_burst_length: %u" + "\n\tdest_burst_length: %u" + "\n\tblock_count: %u" + "\n\thead_block: %p" + "\n\tuser_data: %p" + "\n\tdma_callback: %p" + "\n}", + cfg->dma_slot, cfg->channel_direction, cfg->complete_callback_en, + cfg->error_callback_en, cfg->source_handshake, cfg->dest_handshake, + cfg->channel_priority, cfg->source_chaining_en, cfg->dest_chaining_en, + cfg->linked_channel, cfg->cyclic, cfg->_reserved, cfg->source_data_size, + cfg->dest_data_size, cfg->source_burst_length, cfg->dest_burst_length, + cfg->block_count, cfg->head_block, cfg->user_data, cfg->dma_callback); + + return buffer; +} + +static const char *dma_emul_block_config_to_string(const struct dma_block_config *cfg) +{ + static char buffer[1024]; + + snprintf(buffer, sizeof(buffer), + "{" + "\n\tsource_address: %p" + "\n\tdest_address: %p" + "\n\tsource_gather_interval: %u" + "\n\tdest_scatter_interval: %u" + "\n\tdest_scatter_count: %u" + "\n\tsource_gather_count: %u" + "\n\tblock_size: %u" + "\n\tnext_block: %p" + "\n\tsource_gather_en: %u" + "\n\tdest_scatter_en: %u" + "\n\tsource_addr_adj: %u" + "\n\tdest_addr_adj: %u" + "\n\tsource_reload_en: %u" + "\n\tdest_reload_en: %u" + "\n\tfifo_mode_control: %u" + "\n\tflow_control_mode: %u" + "\n\t_reserved: %u" + "\n}", + (void *)cfg->source_address, (void *)cfg->dest_address, + cfg->source_gather_interval, cfg->dest_scatter_interval, cfg->dest_scatter_count, + cfg->source_gather_count, cfg->block_size, cfg->next_block, cfg->source_gather_en, + cfg->dest_scatter_en, cfg->source_addr_adj, cfg->dest_addr_adj, + cfg->source_reload_en, cfg->dest_reload_en, cfg->fifo_mode_control, + cfg->flow_control_mode, cfg->_reserved + + ); + + return buffer; +} + +static void dma_emul_work_handler(struct k_work *work) +{ + size_t i; + size_t bytes; + uint32_t channel; + k_spinlock_key_t key; + struct dma_block_config block; + struct dma_config xfer_config; + enum dma_emul_channel_state state; + struct dma_emul_xfer_desc *xfer; + struct dma_emul_work *dma_work = CONTAINER_OF(work, struct dma_emul_work, work); + const struct device *dev = dma_work->dev; + struct dma_emul_data *data = dev->data; + const struct dma_emul_config *config = dev->config; + + channel = dma_work->channel; + + do { + key = k_spin_lock(&data->lock); + xfer = &config->xfer[channel]; + /* + * copy the dma_config so we don't have to worry about + * it being asynchronously updated. + */ + memcpy(&xfer_config, &xfer->config, sizeof(xfer_config)); + k_spin_unlock(&data->lock, key); + + LOG_DBG("processing xfer %p for channel %u", xfer, channel); + for (i = 0; i < xfer_config.block_count; ++i) { + + LOG_DBG("processing block %zu", i); + + key = k_spin_lock(&data->lock); + /* + * copy the dma_block_config so we don't have to worry about + * it being asynchronously updated. + */ + memcpy(&block, + &config->block[channel * config->num_requests + + xfer_config.dma_slot + i], + sizeof(block)); + k_spin_unlock(&data->lock, key); + + /* transfer data in bursts */ + for (bytes = MIN(block.block_size, xfer_config.dest_burst_length); + bytes > 0; block.block_size -= bytes, block.source_address += bytes, + block.dest_address += bytes, + bytes = MIN(block.block_size, xfer_config.dest_burst_length)) { + + key = k_spin_lock(&data->lock); + state = dma_emul_get_channel_state(dev, channel); + k_spin_unlock(&data->lock, key); + + if (state == DMA_EMUL_CHANNEL_STOPPED) { + LOG_DBG("asynchronously canceled"); + if (xfer_config.error_callback_en) { + xfer_config.dma_callback(dev, xfer_config.user_data, + channel, -ECANCELED); + } else { + LOG_DBG("error_callback_en is not set (async " + "cancel)"); + } + goto out; + } + + __ASSERT_NO_MSG(state == DMA_EMUL_CHANNEL_STARTED); + + /* + * FIXME: create a backend API (memcpy, TCP/UDP socket, etc) + * Simple copy for now + */ + memcpy((void *)(uintptr_t)block.dest_address, + (void *)(uintptr_t)block.source_address, bytes); + } + } + + key = k_spin_lock(&data->lock); + dma_emul_set_channel_state(dev, channel, DMA_EMUL_CHANNEL_STOPPED); + k_spin_unlock(&data->lock, key); + + /* FIXME: tests/drivers/dma/chan_blen_transfer/ does not set complete_callback_en */ + if (true) { + xfer_config.dma_callback(dev, xfer_config.user_data, channel, + DMA_STATUS_COMPLETE); + } else { + LOG_DBG("complete_callback_en is not set"); + } + + if (xfer_config.source_chaining_en || xfer_config.dest_chaining_en) { + LOG_DBG("%s(): Linked channel %u -> %u", __func__, channel, + xfer_config.linked_channel); + __ASSERT_NO_MSG(channel != xfer_config.linked_channel); + channel = xfer_config.linked_channel; + } else { + LOG_DBG("%s(): done!", __func__); + break; + } + } while (true); + +out: + return; +} + +static bool dma_emul_config_valid(const struct device *dev, uint32_t channel, + const struct dma_config *xfer_config) +{ + size_t i; + struct dma_block_config *block; + const struct dma_emul_config *config = dev->config; + + if (xfer_config->dma_slot >= config->num_requests) { + LOG_ERR("invalid dma_slot %u", xfer_config->dma_slot); + return false; + } + + if (channel >= config->num_channels) { + LOG_ERR("invalid DMA channel %u", channel); + return false; + } + + if (xfer_config->dest_burst_length != xfer_config->source_burst_length) { + LOG_ERR("burst length does not agree. source: %u dest: %u ", + xfer_config->source_burst_length, xfer_config->dest_burst_length); + return false; + } + + for (i = 0, block = xfer_config->head_block; i < xfer_config->block_count; + ++i, block = block->next_block) { + if (block == NULL) { + LOG_ERR("block %zu / %u is NULL", i + 1, xfer_config->block_count); + return false; + } + + if (i >= config->num_requests) { + LOG_ERR("not enough slots to store block %zu / %u", i + 1, + xfer_config->block_count); + return false; + } + } + + /* + * FIXME: + * + * Need to verify all of the fields in struct dma_config with different DT + * configurations so that the driver model is at least consistent and + * verified by CI. + */ + + return true; +} + +static int dma_emul_configure(const struct device *dev, uint32_t channel, + struct dma_config *xfer_config) +{ + size_t i; + int ret = 0; + size_t block_idx; + k_spinlock_key_t key; + struct dma_block_config *block; + struct dma_block_config *block_it; + enum dma_emul_channel_state state; + struct dma_emul_xfer_desc *xfer; + struct dma_emul_data *data = dev->data; + const struct dma_emul_config *config = dev->config; + + if (!dma_emul_config_valid(dev, channel, xfer_config)) { + return -EINVAL; + } + + key = k_spin_lock(&data->lock); + xfer = &config->xfer[channel]; + + LOG_DBG("%s():\nchannel: %u\nconfig: %s", __func__, channel, + dma_emul_xfer_config_to_string(xfer_config)); + + block_idx = channel * config->num_requests + xfer_config->dma_slot; + + block = &config->block[channel * config->num_requests + xfer_config->dma_slot]; + state = dma_emul_get_channel_state(dev, channel); + switch (state) { + case DMA_EMUL_CHANNEL_UNUSED: + case DMA_EMUL_CHANNEL_STOPPED: + /* copy the configuration into the driver */ + memcpy(&xfer->config, xfer_config, sizeof(xfer->config)); + + /* copy all blocks into slots */ + for (i = 0, block_it = xfer_config->head_block; i < xfer_config->block_count; + ++i, block_it = block_it->next_block, ++block) { + __ASSERT_NO_MSG(block_it != NULL); + + LOG_DBG("block_config %s", dma_emul_block_config_to_string(block_it)); + + memcpy(block, block_it, sizeof(*block)); + } + dma_emul_set_channel_state(dev, channel, DMA_EMUL_CHANNEL_LOADED); + + break; + default: + LOG_ERR("attempt to configure DMA in state %d", state); + ret = -EBUSY; + } + k_spin_unlock(&data->lock, key); + + return ret; +} + +static int dma_emul_reload(const struct device *dev, uint32_t channel, dma_addr_t src, + dma_addr_t dst, size_t size) +{ + LOG_DBG("%s()", __func__); + + return -ENOSYS; +} + +static int dma_emul_start(const struct device *dev, uint32_t channel) +{ + int ret = 0; + k_spinlock_key_t key; + enum dma_emul_channel_state state; + struct dma_emul_xfer_desc *xfer; + struct dma_config *xfer_config; + struct dma_emul_data *data = dev->data; + const struct dma_emul_config *config = dev->config; + + LOG_DBG("%s(channel: %u)", __func__, channel); + + if (channel >= config->num_channels) { + return -EINVAL; + } + + key = k_spin_lock(&data->lock); + xfer = &config->xfer[channel]; + state = dma_emul_get_channel_state(dev, channel); + switch (state) { + case DMA_EMUL_CHANNEL_STARTED: + /* start after being started already is a no-op */ + break; + case DMA_EMUL_CHANNEL_LOADED: + case DMA_EMUL_CHANNEL_STOPPED: + data->work.channel = channel; + while (true) { + dma_emul_set_channel_state(dev, channel, DMA_EMUL_CHANNEL_STARTED); + + xfer_config = &config->xfer[channel].config; + if (xfer_config->source_chaining_en || xfer_config->dest_chaining_en) { + LOG_DBG("%s(): Linked channel %u -> %u", __func__, channel, + xfer_config->linked_channel); + channel = xfer_config->linked_channel; + } else { + break; + } + } + ret = k_work_submit_to_queue(&data->work_q, &data->work.work); + ret = (ret < 0) ? ret : 0; + break; + default: + LOG_ERR("attempt to start dma in invalid state %d", state); + ret = -EIO; + break; + } + k_spin_unlock(&data->lock, key); + + return ret; +} + +static int dma_emul_stop(const struct device *dev, uint32_t channel) +{ + k_spinlock_key_t key; + struct dma_emul_data *data = dev->data; + + key = k_spin_lock(&data->lock); + dma_emul_set_channel_state(dev, channel, DMA_EMUL_CHANNEL_STOPPED); + k_spin_unlock(&data->lock, key); + + return 0; +} + +static int dma_emul_suspend(const struct device *dev, uint32_t channel) +{ + LOG_DBG("%s()", __func__); + + return -ENOSYS; +} + +static int dma_emul_resume(const struct device *dev, uint32_t channel) +{ + LOG_DBG("%s()", __func__); + + return -ENOSYS; +} + +static int dma_emul_get_status(const struct device *dev, uint32_t channel, + struct dma_status *status) +{ + LOG_DBG("%s()", __func__); + + return -ENOSYS; +} + +static int dma_emul_get_attribute(const struct device *dev, uint32_t type, uint32_t *value) +{ + LOG_DBG("%s()", __func__); + + return -ENOSYS; +} + +static bool dma_emul_chan_filter(const struct device *dev, int channel, void *filter_param) +{ + bool success; + k_spinlock_key_t key; + struct dma_emul_data *data = dev->data; + + key = k_spin_lock(&data->lock); + /* lets assume the struct dma_context handles races properly */ + success = dma_emul_get_channel_state(dev, channel) == DMA_EMUL_CHANNEL_UNUSED; + k_spin_unlock(&data->lock, key); + + return success; +} + +static const struct dma_driver_api dma_emul_driver_api = { + .config = dma_emul_configure, + .reload = dma_emul_reload, + .start = dma_emul_start, + .stop = dma_emul_stop, + .suspend = dma_emul_suspend, + .resume = dma_emul_resume, + .get_status = dma_emul_get_status, + .get_attribute = dma_emul_get_attribute, + .chan_filter = dma_emul_chan_filter, +}; + +#ifdef CONFIG_PM_DEVICE +static int gpio_emul_pm_device_pm_action(const struct device *dev, enum pm_device_action action) +{ + ARG_UNUSED(dev); + ARG_UNUSED(action); + + return 0; +} +#endif + +static int dma_emul_init(const struct device *dev) +{ + struct dma_emul_data *data = dev->data; + const struct dma_emul_config *config = dev->config; + + data->work.dev = dev; + data->dma_ctx.magic = DMA_MAGIC; + data->dma_ctx.dma_channels = config->num_channels; + data->dma_ctx.atomic = data->channels_atomic; + + k_work_queue_init(&data->work_q); + k_work_init(&data->work.work, dma_emul_work_handler); + k_work_queue_start(&data->work_q, config->work_q_stack, config->work_q_stack_size, + config->work_q_priority, NULL); + + return 0; +} + +#define DMA_EMUL_INST_HAS_PROP(_inst, _prop) DT_NODE_HAS_PROP(DT_DRV_INST(_inst), _prop) + +#define DMA_EMUL_INST_CHANNEL_MASK(_inst) \ + DT_INST_PROP_OR(_inst, dma_channel_mask, \ + DMA_EMUL_INST_HAS_PROP(_inst, dma_channels) \ + ? ((DT_INST_PROP(_inst, dma_channels) > 0) \ + ? BIT_MASK(DT_INST_PROP_OR(_inst, dma_channels, 0)) \ + : 0) \ + : 0) + +#define DMA_EMUL_INST_NUM_CHANNELS(_inst) \ + DT_INST_PROP_OR(_inst, dma_channels, \ + DMA_EMUL_INST_HAS_PROP(_inst, dma_channel_mask) \ + ? POPCOUNT(DT_INST_PROP_OR(_inst, dma_channel_mask, 0)) \ + : 0) + +#define DMA_EMUL_INST_NUM_REQUESTS(_inst) DT_INST_PROP_OR(_inst, dma_requests, 1) + +#define DEFINE_DMA_EMUL(_inst) \ + BUILD_ASSERT(DMA_EMUL_INST_HAS_PROP(_inst, dma_channel_mask) || \ + DMA_EMUL_INST_HAS_PROP(_inst, dma_channels), \ + "at least one of dma_channel_mask or dma_channels must be provided"); \ + \ + BUILD_ASSERT(DMA_EMUL_INST_NUM_CHANNELS(_inst) <= 32, "invalid dma-channels property"); \ + \ + static K_THREAD_STACK_DEFINE(work_q_stack_##_inst, DT_INST_PROP(_inst, stack_size)); \ + \ + static struct dma_emul_xfer_desc \ + dma_emul_xfer_desc_##_inst[DMA_EMUL_INST_NUM_CHANNELS(_inst)]; \ + \ + static struct dma_block_config \ + dma_emul_block_config_##_inst[DMA_EMUL_INST_NUM_CHANNELS(_inst) * \ + DMA_EMUL_INST_NUM_REQUESTS(_inst)]; \ + \ + static const struct dma_emul_config dma_emul_config_##_inst = { \ + .channel_mask = DMA_EMUL_INST_CHANNEL_MASK(_inst), \ + .num_channels = DMA_EMUL_INST_NUM_CHANNELS(_inst), \ + .num_requests = DMA_EMUL_INST_NUM_REQUESTS(_inst), \ + .addr_align = DT_INST_PROP_OR(_inst, dma_buf_addr_alignment, 1), \ + .size_align = DT_INST_PROP_OR(_inst, dma_buf_size_alignment, 1), \ + .copy_align = DT_INST_PROP_OR(_inst, dma_copy_alignment, 1), \ + .work_q_stack = (k_thread_stack_t *)&work_q_stack_##_inst, \ + .work_q_stack_size = K_THREAD_STACK_SIZEOF(work_q_stack_##_inst), \ + .work_q_priority = DT_INST_PROP_OR(_inst, priority, 0), \ + .xfer = dma_emul_xfer_desc_##_inst, \ + .block = dma_emul_block_config_##_inst, \ + }; \ + \ + static ATOMIC_DEFINE(dma_emul_channels_atomic_##_inst, \ + DT_INST_PROP_OR(_inst, dma_channels, 0)); \ + \ + static struct dma_emul_data dma_emul_data_##_inst = { \ + .channels_atomic = dma_emul_channels_atomic_##_inst, \ + }; \ + \ + PM_DEVICE_DT_INST_DEFINE(_inst, dma_emul_pm_device_pm_action); \ + \ + DEVICE_DT_INST_DEFINE(_inst, dma_emul_init, PM_DEVICE_DT_INST_GET(_inst), \ + &dma_emul_data_##_inst, &dma_emul_config_##_inst, POST_KERNEL, \ + CONFIG_DMA_INIT_PRIORITY, &dma_emul_driver_api); + +DT_INST_FOREACH_STATUS_OKAY(DEFINE_DMA_EMUL)