lib: Factor-out TLB management from IPI management

This patch factor-out TLB management from IPI management to separate
sources sbi_tlb.c and sbi_tlb.h.

Signed-off-by: Anup Patel <anup.patel@wdc.com>
Reviewed-by: Atish Patra <atish.patra@wdc.com>

include/sbi/riscv_asm.h

@@ -32,9 +32,6 @@
 #define PAGE_SIZE       (_AC(1, UL) << PAGE_SHIFT)
 #define PAGE_MASK       (~(PAGE_SIZE - 1))
 
-#define SBI_TLB_FLUSH_ALL	((unsigned long)-1)
-#define SBI_TLB_FLUSH_MAX_SIZE	(1UL << 30)
-
 #define REG_L		__REG_SEL(ld, lw)
 #define REG_S		__REG_SEL(sd, sw)
 #define SZREG		__REG_SEL(8, 4)

include/sbi/sbi_ipi.h

@@ -22,28 +22,12 @@
 
 /* clang-format on */
 
-#define SBI_TLB_FIFO_NUM_ENTRIES 4
-enum sbi_tlb_info_types {
-	SBI_TLB_FLUSH_VMA,
-	SBI_TLB_FLUSH_VMA_ASID,
-	SBI_TLB_FLUSH_VMA_VMID
-};
-
 struct sbi_scratch;
 
 struct sbi_ipi_data {
 	unsigned long ipi_type;
 };
 
-struct sbi_tlb_info {
-	unsigned long start;
-	unsigned long size;
-	unsigned long asid;
-	unsigned long type;
-};
-
-#define SBI_TLB_INFO_SIZE sizeof(struct sbi_tlb_info)
-
 int sbi_ipi_send_many(struct sbi_scratch *scratch, ulong *pmask, u32 event,
 		      void *data);
 

include/sbi/sbi_tlb.h

@@ -0,0 +1,48 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2019 Western Digital Corporation or its affiliates.
+ *
+ * Authors:
+ *   Atish Patra <atish.patra@wdc.com>
+ *   Anup Patel <anup.patel@wdc.com>
+ */
+
+#ifndef __SBI_TLB_H__
+#define __SBI_TLB_H__
+
+#include <sbi/sbi_types.h>
+
+/* clang-format off */
+
+#define SBI_TLB_FLUSH_ALL			((unsigned long)-1)
+#define SBI_TLB_FLUSH_MAX_SIZE			(1UL << 30)
+
+/* clang-format on */
+
+#define SBI_TLB_FIFO_NUM_ENTRIES		4
+
+enum sbi_tlb_info_types {
+	SBI_TLB_FLUSH_VMA,
+	SBI_TLB_FLUSH_VMA_ASID,
+	SBI_TLB_FLUSH_VMA_VMID
+};
+
+struct sbi_scratch;
+
+struct sbi_tlb_info {
+	unsigned long start;
+	unsigned long size;
+	unsigned long asid;
+	unsigned long type;
+};
+
+#define SBI_TLB_INFO_SIZE			sizeof(struct sbi_tlb_info)
+
+int sbi_tlb_fifo_update(struct sbi_scratch *scratch, u32 event, void *data);
+
+void sbi_tlb_fifo_process(struct sbi_scratch *scratch, u32 event);
+
+int sbi_tlb_fifo_init(struct sbi_scratch *scratch, bool cold_boot);
+
+#endif

lib/objects.mk

@@ -24,6 +24,7 @@ lib-objs-y += sbi_misaligned_ldst.o
 lib-objs-y += sbi_scratch.o
 lib-objs-y += sbi_system.o
 lib-objs-y += sbi_timer.o
+lib-objs-y += sbi_tlb.o
 lib-objs-y += sbi_trap.o
 
 # External Libraries to include

lib/sbi_ecall.c

@@ -14,6 +14,7 @@
 #include <sbi/sbi_ipi.h>
 #include <sbi/sbi_system.h>
 #include <sbi/sbi_timer.h>
+#include <sbi/sbi_tlb.h>
 #include <sbi/sbi_trap.h>
 
 #define SBI_ECALL_VERSION_MAJOR 0

lib/sbi_ipi.c

@@ -13,92 +13,23 @@
 #include <sbi/riscv_atomic.h>
 #include <sbi/riscv_unpriv.h>
 #include <sbi/sbi_error.h>
-#include <sbi/sbi_fifo.h>
-#include <sbi/sbi_hart.h>
 #include <sbi/sbi_bitops.h>
+#include <sbi/sbi_hart.h>
 #include <sbi/sbi_ipi.h>
 #include <sbi/sbi_platform.h>
 #include <sbi/sbi_timer.h>
+#include <sbi/sbi_tlb.h>
 #include <plat/string.h>
 
 static unsigned long ipi_data_off;
-static unsigned long ipi_tlb_fifo_off;
-static unsigned long ipi_tlb_fifo_mem_off;
-
-static inline int __sbi_tlb_fifo_range_check(struct sbi_tlb_info *curr,
-					     struct sbi_tlb_info *next)
-{
-	unsigned long curr_end;
-	unsigned long next_end;
-	int ret = SBI_FIFO_UNCHANGED;
-
-	if (!curr || !next)
-		return ret;
-
-	next_end = next->start + next->size;
-	curr_end = curr->start + curr->size;
-	if (next->start <= curr->start && next_end > curr_end) {
-		curr->start = next->start;
-		curr->size  = next->size;
-		ret	    = SBI_FIFO_UPDATED;
-	} else if (next->start >= curr->start && next_end <= curr_end) {
-		ret = SBI_FIFO_SKIP;
-	}
-
-	return ret;
-}
-
-/**
- * Call back to decide if an inplace fifo update is required or next entry can
- * can be skipped. Here are the different cases that are being handled.
- *
- * Case1:
- *	if next flush request range lies within one of the existing entry, skip
- *	the next entry.
- * Case2:
- *	if flush request range in current fifo entry lies within next flush
- *	request, update the current entry.
- * Case3:
-	if a complete vma flush is requested, then all entries can be deleted
-	and new request can be enqueued. This will not be done for ASID case
-	as that means we have to iterate again in the fifo to figure out which
-	entries belong to that ASID.
- */
-int sbi_tlb_fifo_update_cb(void *in, void *data)
-{
-	struct sbi_tlb_info *curr;
-	struct sbi_tlb_info *next;
-	int ret = SBI_FIFO_UNCHANGED;
-
-	if (!in && !!data)
-		return ret;
-
-	curr = (struct sbi_tlb_info *)data;
-	next = (struct sbi_tlb_info *)in;
-	if (next->type == SBI_TLB_FLUSH_VMA_ASID &&
-	    curr->type == SBI_TLB_FLUSH_VMA_ASID) {
-		if (next->asid == curr->asid)
-			ret = __sbi_tlb_fifo_range_check(curr, next);
-	} else if (next->type == SBI_TLB_FLUSH_VMA &&
-		   curr->type == SBI_TLB_FLUSH_VMA) {
-		if (next->size == SBI_TLB_FLUSH_ALL)
-			ret = SBI_FIFO_RESET;
-		else
-			ret = __sbi_tlb_fifo_range_check(curr, next);
-	}
-
-	return ret;
-}
 
 static int sbi_ipi_send(struct sbi_scratch *scratch, u32 hartid, u32 event,
 			void *data)
 {
+	int ret;
 	struct sbi_scratch *remote_scratch = NULL;
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 	struct sbi_ipi_data *ipi_data;
-	struct sbi_fifo *ipi_tlb_fifo;
-	struct sbi_tlb_info *tinfo = data;
-	int ret = SBI_FIFO_UNCHANGED;
 
 	if (sbi_platform_hart_disabled(plat, hartid))
 		return -1;
@@ -109,45 +40,21 @@ static int sbi_ipi_send(struct sbi_scratch *scratch, u32 hartid, u32 event,
 	 */
 	remote_scratch = sbi_hart_id_to_scratch(scratch, hartid);
 	ipi_data = sbi_scratch_offset_ptr(remote_scratch, ipi_data_off);
-	ipi_tlb_fifo = sbi_scratch_offset_ptr(remote_scratch,
-					      ipi_tlb_fifo_off);
 	if (event == SBI_IPI_EVENT_SFENCE_VMA ||
 	    event == SBI_IPI_EVENT_SFENCE_VMA_ASID) {
-		/*
+		ret = sbi_tlb_fifo_update(remote_scratch, event, data);
-		 * If address range to flush is too big then simply
+		if (ret > 0)
-		 * upgrade it to flush all because we can only flush
-		 * 4KB at a time.
-		 */
-		if (tinfo->size >= SBI_TLB_FLUSH_MAX_SIZE) {
-			tinfo->start = 0;
-			tinfo->size = SBI_TLB_FLUSH_ALL;
-		}
-
-		ret = sbi_fifo_inplace_update(ipi_tlb_fifo, data,
-					      sbi_tlb_fifo_update_cb);
-		if (ret == SBI_FIFO_SKIP || ret == SBI_FIFO_UPDATED) {
 			goto done;
-		}
+		else if (ret < 0)
-		while (sbi_fifo_enqueue(ipi_tlb_fifo, data) < 0) {
+			return ret;
-			/**
-			 * For now, Busy loop until there is space in the fifo.
-			 * There may be case where target hart is also
-			 * enqueue in source hart's fifo. Both hart may busy
-			 * loop leading to a deadlock.
-			 * TODO: Introduce a wait/wakeup event mechansim to handle
-			 * this properly.
-			 */
-			__asm__ __volatile("nop");
-			__asm__ __volatile("nop");
-		}
 	}
 	atomic_raw_set_bit(event, &ipi_data->ipi_type);
 	mb();
 	sbi_platform_ipi_send(plat, hartid);
 	if (event != SBI_IPI_EVENT_SOFT)
 		sbi_platform_ipi_sync(plat, hartid);
-done:
 
+done:
 	return 0;
 }
 
@@ -180,69 +87,13 @@ void sbi_ipi_clear_smode(struct sbi_scratch *scratch)
 	csr_clear(CSR_MIP, MIP_SSIP);
 }
 
-static void sbi_ipi_tlb_flush_all()
-{
-	__asm__ __volatile("sfence.vma");
-}
-
-static void sbi_ipi_sfence_vma(struct sbi_tlb_info *tinfo)
-{
-	unsigned long start = tinfo->start;
-	unsigned long size  = tinfo->size;
-	unsigned long i;
-
-	if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) {
-		sbi_ipi_tlb_flush_all();
-		return;
-	}
-
-	for (i = 0; i < size; i += PAGE_SIZE) {
-		__asm__ __volatile__("sfence.vma %0"
-				     :
-				     : "r"(start + i)
-				     : "memory");
-	}
-}
-
-static void sbi_ipi_sfence_vma_asid(struct sbi_tlb_info *tinfo)
-{
-	unsigned long start = tinfo->start;
-	unsigned long size  = tinfo->size;
-	unsigned long asid  = tinfo->asid;
-	unsigned long i;
-
-	if (start == 0 && size == 0) {
-		sbi_ipi_tlb_flush_all();
-		return;
-	}
-
-	/* Flush entire MM context for a given ASID */
-	if (size == SBI_TLB_FLUSH_ALL) {
-		__asm__ __volatile__("sfence.vma x0, %0"
-				     :
-				     : "r"(asid)
-				     : "memory");
-		return;
-	}
-
-	for (i = 0; i < size; i += PAGE_SIZE) {
-		__asm__ __volatile__("sfence.vma %0, %1"
-				     :
-				     : "r"(start + i), "r"(asid)
-				     : "memory");
-	}
-}
-
 void sbi_ipi_process(struct sbi_scratch *scratch)
 {
 	volatile unsigned long ipi_type;
-	struct sbi_tlb_info tinfo;
 	unsigned int ipi_event;
 	const struct sbi_platform *plat = sbi_platform_ptr(scratch);
 	struct sbi_ipi_data *ipi_data =
 			sbi_scratch_offset_ptr(scratch, ipi_data_off);
-	struct sbi_fifo *ipi_tlb_fifo =
-			sbi_scratch_offset_ptr(scratch, ipi_tlb_fifo_off);
 
 	u32 hartid = sbi_current_hartid();
 	sbi_platform_ipi_clear(plat, hartid);
@@ -260,13 +111,7 @@ void sbi_ipi_process(struct sbi_scratch *scratch)
 			break;
 		case SBI_IPI_EVENT_SFENCE_VMA:
 		case SBI_IPI_EVENT_SFENCE_VMA_ASID:
-			while (!sbi_fifo_dequeue(ipi_tlb_fifo, &tinfo)) {
+			sbi_tlb_fifo_process(scratch, ipi_event);
-				if (tinfo.type == SBI_TLB_FLUSH_VMA)
-					sbi_ipi_sfence_vma(&tinfo);
-				else if (tinfo.type == SBI_TLB_FLUSH_VMA_ASID)
-					sbi_ipi_sfence_vma_asid(&tinfo);
-				memset(&tinfo, 0, SBI_TLB_INFO_SIZE);
-			}
 			break;
 		case SBI_IPI_EVENT_HALT:
 			sbi_hart_hang();
@@ -278,8 +123,7 @@ void sbi_ipi_process(struct sbi_scratch *scratch)
 
 int sbi_ipi_init(struct sbi_scratch *scratch, bool cold_boot)
 {
-	void *ipi_tlb_mem;
+	int ret;
-	struct sbi_fifo *ipi_tlb_q;
 	struct sbi_ipi_data *ipi_data;
 
 	if (cold_boot) {
@@ -287,34 +131,17 @@ int sbi_ipi_init(struct sbi_scratch *scratch, bool cold_boot)
 							"IPI_DATA");
 		if (!ipi_data_off)
 			return SBI_ENOMEM;
-		ipi_tlb_fifo_off = sbi_scratch_alloc_offset(sizeof(*ipi_tlb_q),
-							    "IPI_TLB_FIFO");
-		if (!ipi_tlb_fifo_off) {
-			sbi_scratch_free_offset(ipi_data_off);
-			return SBI_ENOMEM;
-		}
-		ipi_tlb_fifo_mem_off = sbi_scratch_alloc_offset(
-				SBI_TLB_FIFO_NUM_ENTRIES * SBI_TLB_INFO_SIZE,
-				"IPI_TLB_FIFO_MEM");
-		if (!ipi_tlb_fifo_mem_off) {
-			sbi_scratch_free_offset(ipi_tlb_fifo_off);
-			sbi_scratch_free_offset(ipi_data_off);
-			return SBI_ENOMEM;
-		}
 	} else {
-		if (!ipi_data_off ||
+		if (!ipi_data_off)
-		    !ipi_tlb_fifo_off ||
-		    !ipi_tlb_fifo_mem_off)
 			return SBI_ENOMEM;
 	}
 
 	ipi_data = sbi_scratch_offset_ptr(scratch, ipi_data_off);
-	ipi_tlb_q = sbi_scratch_offset_ptr(scratch, ipi_tlb_fifo_off);
-	ipi_tlb_mem = sbi_scratch_offset_ptr(scratch, ipi_tlb_fifo_mem_off);
-
 	ipi_data->ipi_type = 0x00;
-	sbi_fifo_init(ipi_tlb_q, ipi_tlb_mem,
+
-		      SBI_TLB_FIFO_NUM_ENTRIES, SBI_TLB_INFO_SIZE);
+	ret = sbi_tlb_fifo_init(scratch, cold_boot);
+	if (ret)
+		return ret;
 
 	/* Enable software interrupts */
 	csr_set(CSR_MIE, MIP_MSIP);

lib/sbi_tlb.c

@@ -0,0 +1,227 @@
+/*
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2019 Western Digital Corporation or its affiliates.
+ *
+ * Authors:
+ *   Atish Patra <atish.patra@wdc.com>
+ *   Anup Patel <anup.patel@wdc.com>
+ */
+
+#include <sbi/riscv_asm.h>
+#include <sbi/riscv_barrier.h>
+#include <sbi/sbi_error.h>
+#include <sbi/sbi_fifo.h>
+#include <sbi/sbi_hart.h>
+#include <sbi/sbi_bitops.h>
+#include <sbi/sbi_scratch.h>
+#include <sbi/sbi_tlb.h>
+#include <plat/string.h>
+
+static unsigned long ipi_tlb_fifo_off;
+static unsigned long ipi_tlb_fifo_mem_off;
+
+static inline int __sbi_tlb_fifo_range_check(struct sbi_tlb_info *curr,
+					     struct sbi_tlb_info *next)
+{
+	unsigned long curr_end;
+	unsigned long next_end;
+	int ret = SBI_FIFO_UNCHANGED;
+
+	if (!curr || !next)
+		return ret;
+
+	next_end = next->start + next->size;
+	curr_end = curr->start + curr->size;
+	if (next->start <= curr->start && next_end > curr_end) {
+		curr->start = next->start;
+		curr->size  = next->size;
+		ret	    = SBI_FIFO_UPDATED;
+	} else if (next->start >= curr->start && next_end <= curr_end) {
+		ret = SBI_FIFO_SKIP;
+	}
+
+	return ret;
+}
+
+/**
+ * Call back to decide if an inplace fifo update is required or next entry can
+ * can be skipped. Here are the different cases that are being handled.
+ *
+ * Case1:
+ *	if next flush request range lies within one of the existing entry, skip
+ *	the next entry.
+ * Case2:
+ *	if flush request range in current fifo entry lies within next flush
+ *	request, update the current entry.
+ * Case3:
+	if a complete vma flush is requested, then all entries can be deleted
+	and new request can be enqueued. This will not be done for ASID case
+	as that means we have to iterate again in the fifo to figure out which
+	entries belong to that ASID.
+ */
+static int sbi_tlb_fifo_update_cb(void *in, void *data)
+{
+	struct sbi_tlb_info *curr;
+	struct sbi_tlb_info *next;
+	int ret = SBI_FIFO_UNCHANGED;
+
+	if (!in && !!data)
+		return ret;
+
+	curr = (struct sbi_tlb_info *)data;
+	next = (struct sbi_tlb_info *)in;
+	if (next->type == SBI_TLB_FLUSH_VMA_ASID &&
+	    curr->type == SBI_TLB_FLUSH_VMA_ASID) {
+		if (next->asid == curr->asid)
+			ret = __sbi_tlb_fifo_range_check(curr, next);
+	} else if (next->type == SBI_TLB_FLUSH_VMA &&
+		   curr->type == SBI_TLB_FLUSH_VMA) {
+		if (next->size == SBI_TLB_FLUSH_ALL)
+			ret = SBI_FIFO_RESET;
+		else
+			ret = __sbi_tlb_fifo_range_check(curr, next);
+	}
+
+	return ret;
+}
+
+int sbi_tlb_fifo_update(struct sbi_scratch *scratch, u32 event, void *data)
+{
+	int ret;
+	struct sbi_fifo *ipi_tlb_fifo;
+	struct sbi_tlb_info *tinfo = data;
+
+	ipi_tlb_fifo = sbi_scratch_offset_ptr(scratch,
+					      ipi_tlb_fifo_off);
+	/*
+	 * If address range to flush is too big then simply
+	 * upgrade it to flush all because we can only flush
+	 * 4KB at a time.
+	 */
+	if (tinfo->size >= SBI_TLB_FLUSH_MAX_SIZE) {
+		tinfo->start = 0;
+		tinfo->size = SBI_TLB_FLUSH_ALL;
+	}
+
+	ret = sbi_fifo_inplace_update(ipi_tlb_fifo, data,
+				      sbi_tlb_fifo_update_cb);
+	if (ret == SBI_FIFO_SKIP || ret == SBI_FIFO_UPDATED) {
+		return 1;
+	}
+
+	while (sbi_fifo_enqueue(ipi_tlb_fifo, data) < 0) {
+		/**
+		 * For now, Busy loop until there is space in the fifo.
+		 * There may be case where target hart is also
+		 * enqueue in source hart's fifo. Both hart may busy
+		 * loop leading to a deadlock.
+		 * TODO: Introduce a wait/wakeup event mechansim to handle
+		 * this properly.
+		 */
+		__asm__ __volatile("nop");
+		__asm__ __volatile("nop");
+	}
+
+	return 0;
+}
+
+static void sbi_tlb_flush_all(void)
+{
+	__asm__ __volatile("sfence.vma");
+}
+
+static void sbi_tlb_fifo_sfence_vma(struct sbi_tlb_info *tinfo)
+{
+	unsigned long start = tinfo->start;
+	unsigned long size  = tinfo->size;
+	unsigned long i;
+
+	if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) {
+		sbi_tlb_flush_all();
+		return;
+	}
+
+	for (i = 0; i < size; i += PAGE_SIZE) {
+		__asm__ __volatile__("sfence.vma %0"
+				     :
+				     : "r"(start + i)
+				     : "memory");
+	}
+}
+
+static void sbi_tlb_fifo_sfence_vma_asid(struct sbi_tlb_info *tinfo)
+{
+	unsigned long start = tinfo->start;
+	unsigned long size  = tinfo->size;
+	unsigned long asid  = tinfo->asid;
+	unsigned long i;
+
+	if (start == 0 && size == 0) {
+		sbi_tlb_flush_all();
+		return;
+	}
+
+	/* Flush entire MM context for a given ASID */
+	if (size == SBI_TLB_FLUSH_ALL) {
+		__asm__ __volatile__("sfence.vma x0, %0"
+				     :
+				     : "r"(asid)
+				     : "memory");
+		return;
+	}
+
+	for (i = 0; i < size; i += PAGE_SIZE) {
+		__asm__ __volatile__("sfence.vma %0, %1"
+				     :
+				     : "r"(start + i), "r"(asid)
+				     : "memory");
+	}
+}
+
+void sbi_tlb_fifo_process(struct sbi_scratch *scratch, u32 event)
+{
+	struct sbi_tlb_info tinfo;
+	struct sbi_fifo *ipi_tlb_fifo =
+			sbi_scratch_offset_ptr(scratch, ipi_tlb_fifo_off);
+
+	while (!sbi_fifo_dequeue(ipi_tlb_fifo, &tinfo)) {
+		if (tinfo.type == SBI_TLB_FLUSH_VMA)
+			sbi_tlb_fifo_sfence_vma(&tinfo);
+		else if (tinfo.type == SBI_TLB_FLUSH_VMA_ASID)
+			sbi_tlb_fifo_sfence_vma_asid(&tinfo);
+		memset(&tinfo, 0, SBI_TLB_INFO_SIZE);
+	}
+}
+
+int sbi_tlb_fifo_init(struct sbi_scratch *scratch, bool cold_boot)
+{
+	void *ipi_tlb_mem;
+	struct sbi_fifo *ipi_tlb_q;
+
+	if (cold_boot) {
+		ipi_tlb_fifo_off = sbi_scratch_alloc_offset(sizeof(*ipi_tlb_q),
+							    "IPI_TLB_FIFO");
+		if (!ipi_tlb_fifo_off)
+			return SBI_ENOMEM;
+		ipi_tlb_fifo_mem_off = sbi_scratch_alloc_offset(
+				SBI_TLB_FIFO_NUM_ENTRIES * SBI_TLB_INFO_SIZE,
+				"IPI_TLB_FIFO_MEM");
+		if (!ipi_tlb_fifo_mem_off) {
+			sbi_scratch_free_offset(ipi_tlb_fifo_off);
+			return SBI_ENOMEM;
+		}
+	} else {
+		if (!ipi_tlb_fifo_off ||
+		    !ipi_tlb_fifo_mem_off)
+			return SBI_ENOMEM;
+	}
+
+	ipi_tlb_q = sbi_scratch_offset_ptr(scratch, ipi_tlb_fifo_off);
+	ipi_tlb_mem = sbi_scratch_offset_ptr(scratch, ipi_tlb_fifo_mem_off);
+
+	sbi_fifo_init(ipi_tlb_q, ipi_tlb_mem,
+		      SBI_TLB_FIFO_NUM_ENTRIES, SBI_TLB_INFO_SIZE);
+
+	return 0;
+}