From e348415d1cd76d92503439e2a12803c15b790096 Mon Sep 17 00:00:00 2001 From: Tomasz Bursztyka Date: Wed, 30 Nov 2022 13:39:26 +0100 Subject: [PATCH] drivers/nvme: Support data bigger than a memory page Pre-allocating PRP list for such purpose. Which PRP list is relevantly filled in depending on the data size and data pointer page alignment. Signed-off-by: Tomasz Bursztyka --- drivers/disk/nvme/Kconfig | 9 +++ drivers/disk/nvme/nvme_cmd.c | 89 ++++++++++++++++++++++++++++-- drivers/disk/nvme/nvme_cmd.h | 15 +++++ drivers/disk/nvme/nvme_namespace.h | 2 +- 4 files changed, 110 insertions(+), 5 deletions(-) diff --git a/drivers/disk/nvme/Kconfig b/drivers/disk/nvme/Kconfig index 8291d4ab5c5..a2f6bedddd7 100644 --- a/drivers/disk/nvme/Kconfig +++ b/drivers/disk/nvme/Kconfig @@ -52,6 +52,15 @@ config NVME_REQUEST_TIMEOUT This sets the waiting time for a request to succeed. Do not touch this unless you know what you are doing. +config NVME_PRP_LIST_AMOUNT + int "Number of allocated PRP list" + default 2 + help + This sets the amount of pre-allocated PRP list. Each list + can be used in a NVMe command to address memory where to + read or write data. Each PRP list is of page size be careful + on this number as it may take a sensible amount of memory. + config NVME_MAX_NAMESPACES int "Maximum namespace to allocate" range 1 16 diff --git a/drivers/disk/nvme/nvme_cmd.c b/drivers/disk/nvme/nvme_cmd.c index d8c801ac232..f6633b7dec1 100644 --- a/drivers/disk/nvme/nvme_cmd.c +++ b/drivers/disk/nvme/nvme_cmd.c @@ -15,6 +15,9 @@ LOG_MODULE_DECLARE(nvme, CONFIG_NVME_LOG_LEVEL); #include "nvme.h" #include "nvme_helpers.h" +static struct nvme_prp_list prp_list_pool[CONFIG_NVME_PRP_LIST_AMOUNT]; +static sys_dlist_t free_prp_list; + static struct nvme_request request_pool[NVME_REQUEST_AMOUNT]; static sys_dlist_t free_request; static sys_dlist_t pending_request; @@ -29,10 +32,36 @@ void nvme_cmd_init(void) sys_dlist_init(&free_request); sys_dlist_init(&pending_request); + sys_dlist_init(&free_prp_list); for (idx = 0; idx < NVME_REQUEST_AMOUNT; idx++) { sys_dlist_append(&free_request, &request_pool[idx].node); } + + for (idx = 0; idx < CONFIG_NVME_PRP_LIST_AMOUNT; idx++) { + sys_dlist_append(&free_prp_list, &prp_list_pool[idx].node); + } +} + +static struct nvme_prp_list *nvme_prp_list_alloc(void) +{ + sys_dnode_t *node; + + node = sys_dlist_peek_head(&free_prp_list); + if (!node) { + LOG_ERR("Could not allocate PRP list"); + return NULL; + } + + sys_dlist_remove(node); + + return CONTAINER_OF(node, struct nvme_prp_list, node); +} + +static void nvme_prp_list_free(struct nvme_prp_list *prp_list) +{ + memset(prp_list, 0, sizeof(struct nvme_prp_list)); + sys_dlist_append(&free_prp_list, &prp_list->node); } void nvme_cmd_request_free(struct nvme_request *request) @@ -41,6 +70,10 @@ void nvme_cmd_request_free(struct nvme_request *request) sys_dlist_remove(&request->node); } + if (request->prp_list != NULL) { + nvme_prp_list_free(request->prp_list); + } + memset(request, 0, sizeof(struct nvme_request)); sys_dlist_append(&free_request, &request->node); } @@ -336,6 +369,36 @@ void nvme_cmd_qpair_reset(struct nvme_cmd_qpair *qpair) qpair->num_entries * sizeof(struct nvme_completion)); } +static int nvme_cmd_qpair_fill_prp_list(struct nvme_cmd_qpair *qpair, + struct nvme_request *request, + int n_prp) +{ + struct nvme_prp_list *prp_list; + uintptr_t p_addr; + int idx; + + prp_list = nvme_prp_list_alloc(); + if (prp_list == NULL) { + return -ENOMEM; + } + + p_addr = (uintptr_t)request->payload; + request->cmd.dptr.prp1 = + (uint64_t)sys_cpu_to_le64(p_addr); + request->cmd.dptr.prp2 = + (uint64_t)sys_cpu_to_le64(&prp_list->prp); + p_addr = NVME_PRP_NEXT_PAGE(p_addr); + + for (idx = 0; idx < n_prp; idx++) { + prp_list->prp[idx] = (uint64_t)sys_cpu_to_le64(p_addr); + p_addr = NVME_PRP_NEXT_PAGE(p_addr); + } + + request->prp_list = prp_list; + + return 0; +} + static int nvme_cmd_qpair_fill_dptr(struct nvme_cmd_qpair *qpair, struct nvme_request *request) { @@ -343,16 +406,34 @@ static int nvme_cmd_qpair_fill_dptr(struct nvme_cmd_qpair *qpair, case NVME_REQUEST_NULL: break; case NVME_REQUEST_VADDR: + int n_prp; + if (request->payload_size > qpair->ctrlr->max_xfer_size) { LOG_ERR("VADDR request's payload too big"); return -EINVAL; } - request->cmd.dptr.prp1 = - (uint64_t)sys_cpu_to_le64(request->payload); - request->cmd.dptr.prp2 = 0; + n_prp = request->payload_size / qpair->ctrlr->page_size; + if ((request->payload_size % qpair->ctrlr->page_size) || + ((uintptr_t)request->payload & NVME_PBAO_MASK)) { + n_prp++; + } - /* ToDo: handle > page_size payload through prp list */ + if (n_prp <= 2) { + request->cmd.dptr.prp1 = + (uint64_t)sys_cpu_to_le64(request->payload); + if ((uintptr_t)request->payload & NVME_PBAO_MASK) { + request->cmd.dptr.prp2 = + NVME_PRP_NEXT_PAGE( + (uintptr_t)request->payload); + } else { + request->cmd.dptr.prp2 = 0; + } + + break; + } + + return nvme_cmd_qpair_fill_prp_list(qpair, request, n_prp); default: break; } diff --git a/drivers/disk/nvme/nvme_cmd.h b/drivers/disk/nvme/nvme_cmd.h index c662b323595..c5cd7f8190d 100644 --- a/drivers/disk/nvme/nvme_cmd.h +++ b/drivers/disk/nvme/nvme_cmd.h @@ -305,6 +305,19 @@ enum nvme_feature { #define CACHE_LINE_SIZE CONFIG_DCACHE_LINE_SIZE #endif +/* Assuming page size it always 4Kib + * ToDo: define it accorditng to CONFIG_MMU_PAGE_SIZE + */ +#define NVME_PBAO_MASK 0xFFF + +#define NVME_PRP_NEXT_PAGE(_addr) ((_addr & (~NVME_PBAO_MASK)) + 0x1000) + +struct nvme_prp_list { + uintptr_t prp[CONFIG_MMU_PAGE_SIZE / sizeof(uintptr_t)] + __aligned(0x1000); + sys_dnode_t node; +}; + struct nvme_cmd_qpair { struct nvme_controller *ctrlr; uint32_t id; @@ -354,6 +367,8 @@ struct nvme_request { nvme_cb_fn_t cb_fn; void *cb_arg; + struct nvme_prp_list *prp_list; + sys_dnode_t node; }; diff --git a/drivers/disk/nvme/nvme_namespace.h b/drivers/disk/nvme/nvme_namespace.h index 82d6c88d838..38cd603eaaf 100644 --- a/drivers/disk/nvme/nvme_namespace.h +++ b/drivers/disk/nvme/nvme_namespace.h @@ -155,7 +155,7 @@ void nvme_namespace_data_swapbytes(struct nvme_namespace_data *s) struct nvme_namespace { struct nvme_controller *ctrlr; - struct nvme_namespace_data data __aligned(0x1000); + struct nvme_namespace_data data; uint32_t id; uint32_t flags; uint32_t boundary;