soc: sifive: ccache: Add non-coherent DMA handling

Add functions to flush the caches and handle non-coherent DMA.

Signed-off-by: Emil Renner Berthing <kernel@esmil.dk>

arch/riscv/Kconfig

@@ -220,6 +220,16 @@ config PGTABLE_LEVELS
 config LOCKDEP_SUPPORT
 	def_bool y
 
+config RISCV_DMA_NONCOHERENT
+	bool "Support non-coherent DMA"
+	default SOC_STARFIVE
+	select ARCH_HAS_DMA_PREP_COHERENT
+	select ARCH_HAS_DMA_SET_UNCACHED
+	select ARCH_HAS_DMA_CLEAR_UNCACHED
+	select ARCH_HAS_SYNC_DMA_FOR_DEVICE
+	select ARCH_HAS_SYNC_DMA_FOR_CPU
+	select ARCH_HAS_SETUP_DMA_OPS
+
 source "arch/riscv/Kconfig.socs"
 source "arch/riscv/Kconfig.erratas"
 

arch/riscv/mm/Makefile

@@ -30,3 +30,4 @@ endif
 endif
 
 obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o
+obj-$(CONFIG_RISCV_DMA_NONCOHERENT) += dma-noncoherent.o

arch/riscv/mm/dma-noncoherent.c

@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * RISC-V specific functions to support DMA for non-coherent devices
+ *
+ * Copyright (c) 2021 Western Digital Corporation or its affiliates.
+ */
+
+#include <linux/dma-map-ops.h>
+
+#include <soc/sifive/sifive_l2_cache.h>
+
+void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, enum dma_data_direction dir)
+{
+	if (sifive_l2_handle_noncoherent())
+		sifive_l2_flush_range(paddr, size);
+}
+
+void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, enum dma_data_direction dir)
+{
+	if (sifive_l2_handle_noncoherent())
+		sifive_l2_flush_range(paddr, size);
+}
+
+void *arch_dma_set_uncached(void *addr, size_t size)
+{
+	if (sifive_l2_handle_noncoherent())
+		return sifive_l2_set_uncached(addr, size);
+
+	return addr;
+}
+
+void arch_dma_clear_uncached(void *addr, size_t size)
+{
+	if (sifive_l2_handle_noncoherent())
+		sifive_l2_clear_uncached(addr, size);
+}
+
+void arch_dma_prep_coherent(struct page *page, size_t size)
+{
+	void *flush_addr = page_address(page);
+
+	memset(flush_addr, 0, size);
+	if (sifive_l2_handle_noncoherent())
+		sifive_l2_flush_range(__pa(flush_addr), size);
+}
+
+void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
+		const struct iommu_ops *iommu, bool coherent)
+{
+	/* If a specific device is dma-coherent, set it here */
+	dev->dma_coherent = coherent;
+}

drivers/soc/sifive/sifive_l2_cache.c

@@ -5,12 +5,14 @@
  * Copyright (C) 2018-2019 SiFive, Inc.
  *
  */
+#include <linux/align.h>
 #include <linux/debugfs.h>
 #include <linux/interrupt.h>
 #include <linux/of_irq.h>
 #include <linux/of_address.h>
 #include <linux/device.h>
 #include <asm/cacheinfo.h>
+#include <asm/page.h>
 #include <soc/sifive/sifive_l2_cache.h>
 
 #define SIFIVE_L2_DIRECCFIX_LOW 0x100
@@ -30,6 +32,7 @@
 #define SIFIVE_L2_DATECCFAIL_COUNT 0x168
 
 #define SIFIVE_L2_FLUSH64 0x200
+#define SIFIVE_L2_FLUSH32 0x240
 
 #define SIFIVE_L2_CONFIG 0x00
 #define SIFIVE_L2_WAYENABLE 0x08
@@ -41,6 +44,8 @@
 static void __iomem *l2_base;
 static int g_irq[SIFIVE_L2_MAX_ECCINTR];
 static struct riscv_cacheinfo_ops l2_cache_ops;
+static phys_addr_t uncached_offset;
+DEFINE_STATIC_KEY_FALSE(sifive_l2_handle_noncoherent_key);
 
 enum {
 	DIR_CORR = 0,
@@ -119,6 +124,42 @@ int unregister_sifive_l2_error_notifier(struct notifier_block *nb)
 }
 EXPORT_SYMBOL_GPL(unregister_sifive_l2_error_notifier);
 
+void sifive_l2_flush_range(phys_addr_t start, size_t len)
+{
+	phys_addr_t end = start + len;
+	phys_addr_t line;
+
+	if (!len)
+		return;
+
+	mb();
+	for (line = ALIGN_DOWN(start, SIFIVE_L2_FLUSH64_LINE_LEN); line < end;
+	     line += SIFIVE_L2_FLUSH64_LINE_LEN) {
+#ifdef CONFIG_32BIT
+		writel(line >> 4, l2_base + SIFIVE_L2_FLUSH32);
+#else
+		writeq(line, l2_base + SIFIVE_L2_FLUSH64);
+#endif
+		mb();
+	}
+}
+EXPORT_SYMBOL_GPL(sifive_l2_flush_range);
+
+void *sifive_l2_set_uncached(void *addr, size_t size)
+{
+	phys_addr_t phys_addr = __pa(addr) + uncached_offset;
+	void *mem_base;
+
+	mem_base = memremap(phys_addr, size, MEMREMAP_WT);
+	if (!mem_base) {
+		pr_err("%s memremap failed for addr %p\n", __func__, addr);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return mem_base;
+}
+EXPORT_SYMBOL_GPL(sifive_l2_set_uncached);
+
 #ifdef CONFIG_SIFIVE_L2_FLUSH
 void sifive_l2_flush64_range(unsigned long start, unsigned long len)
 {
@@ -235,6 +276,7 @@ static int __init sifive_l2_init(void)
 	struct device_node *np;
 	struct resource res;
 	int i, rc, intr_num;
+	u64 offset;
 
 	np = of_find_matching_node(NULL, sifive_l2_ids);
 	if (!np)
@@ -268,6 +310,11 @@ static int __init sifive_l2_init(void)
 	}
 	of_node_put(np);
 
+	if (!of_property_read_u64(np, "uncached-offset", &offset)) {
+		uncached_offset = offset;
+		static_branch_enable(&sifive_l2_handle_noncoherent_key);
+	}
+
 	l2_config_read();
 
 	l2_cache_ops.get_priv_group = l2_get_priv_group;

include/soc/sifive/sifive_l2_cache.h

@@ -7,12 +7,33 @@
 #ifndef __SOC_SIFIVE_L2_CACHE_H
 #define __SOC_SIFIVE_L2_CACHE_H
 
+#include <linux/io.h>
+#include <linux/jump_label.h>
+
 extern int register_sifive_l2_error_notifier(struct notifier_block *nb);
 extern int unregister_sifive_l2_error_notifier(struct notifier_block *nb);
 
 #define SIFIVE_L2_ERR_TYPE_CE 0
 #define SIFIVE_L2_ERR_TYPE_UE 1
 
+DECLARE_STATIC_KEY_FALSE(sifive_l2_handle_noncoherent_key);
+
+static inline bool sifive_l2_handle_noncoherent(void)
+{
+#ifdef CONFIG_SIFIVE_L2
+	return static_branch_unlikely(&sifive_l2_handle_noncoherent_key);
+#else
+	return false;
+#endif
+}
+
+void sifive_l2_flush_range(phys_addr_t start, size_t len);
+void *sifive_l2_set_uncached(void *addr, size_t size);
+static inline void sifive_l2_clear_uncached(void *addr, size_t size)
+{
+	memunmap(addr);
+}
+
 #ifdef CONFIG_SIFIVE_L2_FLUSH
 void sifive_l2_flush64_range(unsigned long start, unsigned long len);
 #endif