drivers/nvme: Support data bigger than a memory page

Pre-allocating PRP list for such purpose. Which PRP list is relevantly
filled in depending on the data size and data pointer page alignment.

Signed-off-by: Tomasz Bursztyka <tomasz.bursztyka@linux.intel.com>

drivers/disk/nvme/Kconfig

@@ -52,6 +52,15 @@ config NVME_REQUEST_TIMEOUT
 	  This sets the waiting time for a request to succeed.
 	  Do not touch this unless you know what you are doing.
 
+config NVME_PRP_LIST_AMOUNT
+	int "Number of allocated PRP list"
+	default 2
+	help
+	  This sets the amount of pre-allocated PRP list. Each list
+	  can be used in a NVMe command to address memory where to
+	  read or write data. Each PRP list is of page size be careful
+	  on this number as it may take a sensible amount of memory.
+
 config NVME_MAX_NAMESPACES
 	int "Maximum namespace to allocate"
 	range 1 16

drivers/disk/nvme/nvme_cmd.c

@@ -15,6 +15,9 @@ LOG_MODULE_DECLARE(nvme, CONFIG_NVME_LOG_LEVEL);
 #include "nvme.h"
 #include "nvme_helpers.h"
 
+static struct nvme_prp_list prp_list_pool[CONFIG_NVME_PRP_LIST_AMOUNT];
+static sys_dlist_t free_prp_list;
+
 static struct nvme_request request_pool[NVME_REQUEST_AMOUNT];
 static sys_dlist_t free_request;
 static sys_dlist_t pending_request;
@@ -29,10 +32,36 @@ void nvme_cmd_init(void)
 
 	sys_dlist_init(&free_request);
 	sys_dlist_init(&pending_request);
+	sys_dlist_init(&free_prp_list);
 
 	for (idx = 0; idx < NVME_REQUEST_AMOUNT; idx++) {
 		sys_dlist_append(&free_request, &request_pool[idx].node);
 	}
+
+	for (idx = 0; idx < CONFIG_NVME_PRP_LIST_AMOUNT; idx++) {
+		sys_dlist_append(&free_prp_list, &prp_list_pool[idx].node);
+	}
+}
+
+static struct nvme_prp_list *nvme_prp_list_alloc(void)
+{
+	sys_dnode_t *node;
+
+	node = sys_dlist_peek_head(&free_prp_list);
+	if (!node) {
+		LOG_ERR("Could not allocate PRP list");
+		return NULL;
+	}
+
+	sys_dlist_remove(node);
+
+	return CONTAINER_OF(node, struct nvme_prp_list, node);
+}
+
+static void nvme_prp_list_free(struct nvme_prp_list *prp_list)
+{
+	memset(prp_list, 0, sizeof(struct nvme_prp_list));
+	sys_dlist_append(&free_prp_list, &prp_list->node);
 }
 
 void nvme_cmd_request_free(struct nvme_request *request)
@@ -41,6 +70,10 @@ void nvme_cmd_request_free(struct nvme_request *request)
 		sys_dlist_remove(&request->node);
 	}
 
+	if (request->prp_list != NULL) {
+		nvme_prp_list_free(request->prp_list);
+	}
+
 	memset(request, 0, sizeof(struct nvme_request));
 	sys_dlist_append(&free_request, &request->node);
 }
@@ -336,6 +369,36 @@ void nvme_cmd_qpair_reset(struct nvme_cmd_qpair *qpair)
 	       qpair->num_entries * sizeof(struct nvme_completion));
 }
 
+static int nvme_cmd_qpair_fill_prp_list(struct nvme_cmd_qpair *qpair,
+					struct nvme_request *request,
+					int n_prp)
+{
+	struct nvme_prp_list *prp_list;
+	uintptr_t p_addr;
+	int idx;
+
+	prp_list = nvme_prp_list_alloc();
+	if (prp_list == NULL) {
+		return -ENOMEM;
+	}
+
+	p_addr = (uintptr_t)request->payload;
+	request->cmd.dptr.prp1 =
+		(uint64_t)sys_cpu_to_le64(p_addr);
+	request->cmd.dptr.prp2 =
+		(uint64_t)sys_cpu_to_le64(&prp_list->prp);
+	p_addr = NVME_PRP_NEXT_PAGE(p_addr);
+
+	for (idx = 0; idx < n_prp; idx++) {
+		prp_list->prp[idx] = (uint64_t)sys_cpu_to_le64(p_addr);
+		p_addr = NVME_PRP_NEXT_PAGE(p_addr);
+	}
+
+	request->prp_list = prp_list;
+
+	return 0;
+}
+
 static int nvme_cmd_qpair_fill_dptr(struct nvme_cmd_qpair *qpair,
 				    struct nvme_request *request)
 {
@@ -343,16 +406,34 @@ static int nvme_cmd_qpair_fill_dptr(struct nvme_cmd_qpair *qpair,
 	case NVME_REQUEST_NULL:
 		break;
 	case NVME_REQUEST_VADDR:
+		int n_prp;
+
 		if (request->payload_size > qpair->ctrlr->max_xfer_size) {
 			LOG_ERR("VADDR request's payload too big");
 			return -EINVAL;
 		}
 
-		request->cmd.dptr.prp1 =
-			(uint64_t)sys_cpu_to_le64(request->payload);
-		request->cmd.dptr.prp2 = 0;
+		n_prp = request->payload_size / qpair->ctrlr->page_size;
+		if ((request->payload_size % qpair->ctrlr->page_size) ||
+		    ((uintptr_t)request->payload & NVME_PBAO_MASK)) {
+			n_prp++;
+		}
 
-		/* ToDo: handle > page_size payload through prp list */
+		if (n_prp <= 2) {
+			request->cmd.dptr.prp1 =
+				(uint64_t)sys_cpu_to_le64(request->payload);
+			if ((uintptr_t)request->payload & NVME_PBAO_MASK) {
+				request->cmd.dptr.prp2 =
+					NVME_PRP_NEXT_PAGE(
+						(uintptr_t)request->payload);
+			} else {
+				request->cmd.dptr.prp2 = 0;
+			}
+
+			break;
+		}
+
+		return nvme_cmd_qpair_fill_prp_list(qpair, request, n_prp);
 	default:
 		break;
 	}

drivers/disk/nvme/nvme_cmd.h

@@ -305,6 +305,19 @@ enum nvme_feature {
 #define CACHE_LINE_SIZE				CONFIG_DCACHE_LINE_SIZE
 #endif
 
+/* Assuming page size it always 4Kib
+ * ToDo: define it accorditng to CONFIG_MMU_PAGE_SIZE
+ */
+#define NVME_PBAO_MASK 0xFFF
+
+#define NVME_PRP_NEXT_PAGE(_addr) ((_addr & (~NVME_PBAO_MASK)) + 0x1000)
+
+struct nvme_prp_list {
+	uintptr_t prp[CONFIG_MMU_PAGE_SIZE / sizeof(uintptr_t)]
+						__aligned(0x1000);
+	sys_dnode_t node;
+};
+
 struct nvme_cmd_qpair {
 	struct nvme_controller	*ctrlr;
 	uint32_t		id;
@@ -354,6 +367,8 @@ struct nvme_request {
 	nvme_cb_fn_t			cb_fn;
 	void				*cb_arg;
 
+	struct nvme_prp_list		*prp_list;
+
 	sys_dnode_t			node;
 };
 

drivers/disk/nvme/nvme_namespace.h

@@ -155,7 +155,7 @@ void nvme_namespace_data_swapbytes(struct nvme_namespace_data *s)
 
 struct nvme_namespace {
 	struct nvme_controller *ctrlr;
-	struct nvme_namespace_data data __aligned(0x1000);
+	struct nvme_namespace_data data;
 	uint32_t id;
 	uint32_t flags;
 	uint32_t boundary;