Merge git://git.denx.de/u-boot-riscv

- RISC-V arch support SMP.
- Support Andestech's PLIC and PLMT.
- qemu, fu54e, ae350 boards enable SMP by default.
- Fix CONFIG_DEFAULT_DEVICE_TREE failure.

arch/riscv/Kconfig

@@ -109,6 +109,24 @@ config SIFIVE_CLINT
 	  The SiFive CLINT block holds memory-mapped control and status registers
 	  associated with software and timer interrupts.
 
+config ANDES_PLIC
+	bool
+	depends on RISCV_MMODE
+	select REGMAP
+	select SYSCON
+	help
+	  The Andes PLIC block holds memory-mapped claim and pending registers
+	  associated with software interrupt.
+
+config ANDES_PLMT
+	bool
+	depends on RISCV_MMODE
+	select REGMAP
+	select SYSCON
+	help
+	  The Andes PLMT block holds memory-mapped mtime register
+	  associated with timer tick.
+
 config RISCV_RDTIME
 	bool
 	default y if RISCV_SMODE
@@ -120,4 +138,32 @@ config RISCV_RDTIME
 config SYS_MALLOC_F_LEN
 	default 0x1000
 
+config SMP
+	bool "Symmetric Multi-Processing"
+	help
+	  This enables support for systems with more than one CPU. If
+	  you say N here, U-Boot will run on single and multiprocessor
+	  machines, but will use only one CPU of a multiprocessor
+	  machine. If you say Y here, U-Boot will run on many, but not
+	  all, single processor machines.
+
+config NR_CPUS
+	int "Maximum number of CPUs (2-32)"
+	range 2 32
+	depends on SMP
+	default 8
+	help
+	  On multiprocessor machines, U-Boot sets up a stack for each CPU.
+	  Stack memory is pre-allocated. U-Boot must therefore know the
+	  maximum number of CPUs that may be present.
+
+config SBI_IPI
+	bool
+	default y if RISCV_SMODE
+	depends on SMP
+
+config STACK_SIZE_SHIFT
+	int
+	default 13
+
 endmenu

arch/riscv/cpu/ax25/Kconfig

@@ -1,5 +1,11 @@
 config RISCV_NDS
 	bool
+	select ARCH_EARLY_INIT_R
+	imply CPU
+	imply CPU_RISCV
+	imply RISCV_TIMER
+	imply ANDES_PLIC if RISCV_MMODE
+	imply ANDES_PLMT if RISCV_MMODE
 	help
 	  Run U-Boot on AndeStar V5 platforms and use some specific features
 	  which are provided by Andes Technology AndeStar V5 families.
@@ -8,6 +14,7 @@ if RISCV_NDS
 
 config RISCV_NDS_CACHE
 	bool "AndeStar V5 families specific cache support"
+	depends on RISCV_MMODE
 	help
 	  Provide Andes Technology AndeStar V5 families specific cache support.
 

arch/riscv/cpu/cpu.c

@@ -12,10 +12,17 @@
 #include <dm/uclass-internal.h>
 
 /*
- * prior_stage_fdt_address must be stored in the data section since it is used
+ * The variables here must be stored in the data section since they are used
  * before the bss section is available.
  */
 phys_addr_t prior_stage_fdt_address __attribute__((section(".data")));
+u32 hart_lottery __attribute__((section(".data"))) = 0;
+
+/*
+ * The main hart running U-Boot has acquired available_harts_lock until it has
+ * finished initialization of global data.
+ */
+u32 available_harts_lock = 1;
 
 static inline bool supports_extension(char ext)
 {

arch/riscv/cpu/start.S

@@ -13,6 +13,7 @@
 #include <config.h>
 #include <common.h>
 #include <elf.h>
+#include <asm/csr.h>
 #include <asm/encoding.h>
 #include <generated/asm-offsets.h>
 
@@ -32,11 +33,19 @@
 #define SYM_SIZE		0x18
 #endif
 
+.section .data
+secondary_harts_relocation_error:
+	.ascii "Relocation of secondary harts has failed, error %d\n"
+
 .section .text
 .globl _start
 _start:
+#ifdef CONFIG_RISCV_MMODE
+	csrr	a0, mhartid
+#endif
+
 	/* save hart id and dtb pointer */
-	mv	s0, a0
+	mv	tp, a0
 	mv	s1, a1
 
 	la	t0, trap_entry
@@ -45,9 +54,22 @@ _start:
 	/* mask all interrupts */
 	csrw	MODE_PREFIX(ie), zero
 
-	/* Enable cache */
+#ifdef CONFIG_SMP
-	jal	icache_enable
+	/* check if hart is within range */
-	jal	dcache_enable
+	/* tp: hart id */
+	li	t0, CONFIG_NR_CPUS
+	bge	tp, t0, hart_out_of_bounds_loop
+#endif
+
+#ifdef CONFIG_SMP
+	/* set xSIE bit to receive IPIs */
+#ifdef CONFIG_RISCV_MMODE
+	li	t0, MIE_MSIE
+#else
+	li	t0, SIE_SSIE
+#endif
+	csrs	MODE_PREFIX(ie), t0
+#endif
 
 /*
  * Set stackpointer in internal/ex RAM to call board_init_f
@@ -57,14 +79,33 @@ call_board_init_f:
 	li	t1, CONFIG_SYS_INIT_SP_ADDR
 	and	sp, t1, t0		/* force 16 byte alignment */
 
-#ifdef CONFIG_DEBUG_UART
-	jal	debug_uart_init
-#endif
-
 call_board_init_f_0:
 	mv	a0, sp
 	jal	board_init_f_alloc_reserve
+
+	/*
+	 * Set global data pointer here for all harts, uninitialized at this
+	 * point.
+	 */
+	mv	gp, a0
+
+	/* setup stack */
+#ifdef CONFIG_SMP
+	/* tp: hart id */
+	slli	t0, tp, CONFIG_STACK_SIZE_SHIFT
+	sub	sp, a0, t0
+#else
 	mv	sp, a0
+#endif
+
+	/*
+	 * Pick hart to initialize global data and run U-Boot. The other harts
+	 * wait for initialization to complete.
+	 */
+	la	t0, hart_lottery
+	li	s2, 1
+	amoswap.w s2, t1, 0(t0)
+	bnez	s2, wait_for_gd_init
 
 	la	t0, prior_stage_fdt_address
 	SREG	s1, 0(t0)
@@ -72,7 +113,42 @@ call_board_init_f_0:
 	jal	board_init_f_init_reserve
 
 	/* save the boot hart id to global_data */
-	SREG	s0, GD_BOOT_HART(gp)
+	SREG	tp, GD_BOOT_HART(gp)
+
+	la	t0, available_harts_lock
+	fence	rw, w
+	amoswap.w zero, zero, 0(t0)
+
+wait_for_gd_init:
+	la	t0, available_harts_lock
+	li	t1, 1
+1:	amoswap.w t1, t1, 0(t0)
+	fence	r, rw
+	bnez	t1, 1b
+
+	/* register available harts in the available_harts mask */
+	li	t1, 1
+	sll	t1, t1, tp
+	LREG	t2, GD_AVAILABLE_HARTS(gp)
+	or	t2, t2, t1
+	SREG	t2, GD_AVAILABLE_HARTS(gp)
+
+	fence	rw, w
+	amoswap.w zero, zero, 0(t0)
+
+	/*
+	 * Continue on hart lottery winner, others branch to
+	 * secondary_hart_loop.
+	 */
+	bnez	s2, secondary_hart_loop
+
+	/* Enable cache */
+	jal	icache_enable
+	jal	dcache_enable
+
+#ifdef CONFIG_DEBUG_UART
+	jal	debug_uart_init
+#endif
 
 	mv	a0, zero		/* a0 <-- boot_flags = 0 */
 	la	t5, board_init_f
@@ -95,7 +171,14 @@ relocate_code:
  *Set up the stack
  */
 stack_setup:
+#ifdef CONFIG_SMP
+	/* tp: hart id */
+	slli	t0, tp, CONFIG_STACK_SIZE_SHIFT
+	sub	sp, s2, t0
+#else
 	mv	sp, s2
+#endif
+
 	la	t0, _start
 	sub	t6, s4, t0		/* t6 <- relocation offset */
 	beq	t0, s4, clear_bss	/* skip relocation */
@@ -175,13 +258,37 @@ clear_bss:
 	add	t0, t0, t6		/* t0 <- rel __bss_start in RAM */
 	la	t1, __bss_end		/* t1 <- rel __bss_end in FLASH */
 	add	t1, t1, t6		/* t1 <- rel __bss_end in RAM */
-	beq	t0, t1, call_board_init_r
+	beq	t0, t1, relocate_secondary_harts
 
 clbss_l:
 	SREG	zero, 0(t0)		/* clear loop... */
 	addi	t0, t0, REGBYTES
 	bne	t0, t1, clbss_l
 
+relocate_secondary_harts:
+#ifdef CONFIG_SMP
+	/* send relocation IPI */
+	la	t0, secondary_hart_relocate
+	add	a0, t0, t6
+
+	/* store relocation offset */
+	mv	s5, t6
+
+	mv	a1, s2
+	mv	a2, s3
+	jal	smp_call_function
+
+	/* hang if relocation of secondary harts has failed */
+	beqz	a0, 1f
+	mv	a1, a0
+	la	a0, secondary_harts_relocation_error
+	jal	printf
+	jal	hang
+
+	/* restore relocation offset */
+1:	mv	t6, s5
+#endif
+
 /*
  * We are done. Do not return, instead branch to second part of board
  * initialization, now running from RAM.
@@ -202,3 +309,43 @@ call_board_init_r:
  * jump to it ...
  */
 	jr	t4			/* jump to board_init_r() */
+
+#ifdef CONFIG_SMP
+hart_out_of_bounds_loop:
+	/* Harts in this loop are out of bounds, increase CONFIG_NR_CPUS. */
+	wfi
+	j	hart_out_of_bounds_loop
+#endif
+
+#ifdef CONFIG_SMP
+/* SMP relocation entry */
+secondary_hart_relocate:
+	/* a1: new sp */
+	/* a2: new gd */
+	/* tp: hart id */
+
+	/* setup stack */
+	slli	t0, tp, CONFIG_STACK_SIZE_SHIFT
+	sub	sp, a1, t0
+
+	/* update global data pointer */
+	mv	gp, a2
+#endif
+
+secondary_hart_loop:
+	wfi
+
+#ifdef CONFIG_SMP
+	csrr	t0, MODE_PREFIX(ip)
+#ifdef CONFIG_RISCV_MMODE
+	andi	t0, t0, MIE_MSIE
+#else
+	andi	t0, t0, SIE_SSIE
+#endif
+	beqz	t0, secondary_hart_loop
+
+	mv	a0, tp
+	jal	handle_ipi
+#endif
+
+	j	secondary_hart_loop

arch/riscv/dts/Makefile

@@ -1,5 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0+
 
+dtb-$(CONFIG_TARGET_AX25_AE350) += ae350_32.dtb ae350_64.dtb
+
 targets += $(dtb-y)
 
 DTC_FLAGS += -R 4 -p 0x1000

arch/riscv/dts/ae350_32.dts

@@ -26,16 +26,49 @@
 			status = "okay";
 			compatible = "riscv";
 			riscv,isa = "rv32imafdc";
+			riscv,priv-major = <1>;
+			riscv,priv-minor = <10>;
 			mmu-type = "riscv,sv32";
 			clock-frequency = <60000000>;
+			i-cache-size = <0x8000>;
+			i-cache-line-size = <32>;
 			d-cache-size = <0x8000>;
 			d-cache-line-size = <32>;
+			next-level-cache = <&L2>;
 			CPU0_intc: interrupt-controller {
 				#interrupt-cells = <1>;
 				interrupt-controller;
 				compatible = "riscv,cpu-intc";
 			};
 		};
+		CPU1: cpu@1 {
+			device_type = "cpu";
+			reg = <1>;
+			status = "okay";
+			compatible = "riscv";
+			riscv,isa = "rv32imafdc";
+			riscv,priv-major = <1>;
+			riscv,priv-minor = <10>;
+			mmu-type = "riscv,sv32";
+			clock-frequency = <60000000>;
+			i-cache-size = <0x8000>;
+			i-cache-line-size = <32>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <32>;
+			next-level-cache = <&L2>;
+			CPU1_intc: interrupt-controller {
+				#interrupt-cells = <1>;
+				interrupt-controller;
+				compatible = "riscv,cpu-intc";
+			};
+		};
+
+		L2: l2-cache@e0500000 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-size = <0x40000>;
+			reg = <0x0 0xe0500000 0x0 0x40000>;
+		};
 	};
 
 	memory@0 {
@@ -46,32 +79,32 @@
 	soc {
 		#address-cells = <1>;
 		#size-cells = <1>;
-		compatible = "andestech,riscv-ae350-soc";
+		compatible = "simple-bus";
 		ranges;
 
-	plic0: interrupt-controller@e4000000 {
+		plic0: interrupt-controller@e4000000 {
-		compatible = "riscv,plic0";
+			compatible = "riscv,plic0";
-		#address-cells = <1>;
+			#address-cells = <1>;
-		#interrupt-cells = <1>;
+			#interrupt-cells = <1>;
-		interrupt-controller;
+			interrupt-controller;
-		reg = <0xe4000000 0x2000000>;
+			reg = <0xe4000000 0x2000000>;
-		riscv,ndev=<71>;
+			riscv,ndev=<71>;
-		interrupts-extended = <&CPU0_intc 11 &CPU0_intc 9>;
+			interrupts-extended = <&CPU0_intc 11 &CPU0_intc 9 &CPU1_intc 11 &CPU1_intc 9>;
-	};
+		};
 
-	plic1: interrupt-controller@e6400000 {
+		plic1: interrupt-controller@e6400000 {
-		compatible = "riscv,plic1";
+			compatible = "riscv,plic1";
-		#address-cells = <1>;
+			#address-cells = <1>;
-		#interrupt-cells = <1>;
+			#interrupt-cells = <1>;
-		interrupt-controller;
+			interrupt-controller;
-		reg = <0xe6400000 0x400000>;
+			reg = <0xe6400000 0x400000>;
-		riscv,ndev=<1>;
+			riscv,ndev=<2>;
-		interrupts-extended = <&CPU0_intc 3>;
+			interrupts-extended = <&CPU0_intc 3 &CPU1_intc 3>;
-	};
+		};
 
-	plmt0@e6000000 {
+		plmt0@e6000000 {
-		compatible = "riscv,plmt0";
+			compatible = "riscv,plmt0";
-			interrupts-extended = <&CPU0_intc 7>;
+			interrupts-extended = <&CPU0_intc 7 &CPU1_intc 7>;
 			reg = <0xe6000000 0x100000>;
 		};
 	};
@@ -146,6 +179,10 @@
 		interrupt-parent = <&plic0>;
 	};
 
+	pmu {
+		compatible = "riscv,base-pmu";
+	};
+
 	virtio_mmio@fe007000 {
 		interrupts = <0x17 0x4>;
 		interrupt-parent = <0x2>;

arch/riscv/dts/ae350_64.dts

@@ -26,16 +26,49 @@
 			status = "okay";
 			compatible = "riscv";
 			riscv,isa = "rv64imafdc";
+			riscv,priv-major = <1>;
+			riscv,priv-minor = <10>;
 			mmu-type = "riscv,sv39";
 			clock-frequency = <60000000>;
+			i-cache-size = <0x8000>;
+			i-cache-line-size = <32>;
 			d-cache-size = <0x8000>;
 			d-cache-line-size = <32>;
+			next-level-cache = <&L2>;
 			CPU0_intc: interrupt-controller {
 				#interrupt-cells = <1>;
 				interrupt-controller;
 				compatible = "riscv,cpu-intc";
 			};
 		};
+		CPU1: cpu@1 {
+			device_type = "cpu";
+			reg = <1>;
+			status = "okay";
+			compatible = "riscv";
+			riscv,isa = "rv64imafdc";
+			riscv,priv-major = <1>;
+			riscv,priv-minor = <10>;
+			mmu-type = "riscv,sv39";
+			clock-frequency = <60000000>;
+			i-cache-size = <0x8000>;
+			i-cache-line-size = <32>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <32>;
+			next-level-cache = <&L2>;
+			CPU1_intc: interrupt-controller {
+				#interrupt-cells = <1>;
+				interrupt-controller;
+				compatible = "riscv,cpu-intc";
+			};
+		};
+
+		L2: l2-cache@e0500000 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-size = <0x40000>;
+			reg = <0x0 0xe0500000 0x0 0x40000>;
+		};
 	};
 
 	memory@0 {
@@ -46,32 +79,32 @@
 	soc {
 		#address-cells = <2>;
 		#size-cells = <2>;
-		compatible = "andestech,riscv-ae350-soc";
+		compatible = "simple-bus";
 		ranges;
 
-	plic0: interrupt-controller@e4000000 {
+		plic0: interrupt-controller@e4000000 {
-		compatible = "riscv,plic0";
+			compatible = "riscv,plic0";
-		#address-cells = <2>;
+			#address-cells = <2>;
-		#interrupt-cells = <2>;
+			#interrupt-cells = <2>;
-		interrupt-controller;
+			interrupt-controller;
-		reg = <0x0 0xe4000000 0x0 0x2000000>;
+			reg = <0x0 0xe4000000 0x0 0x2000000>;
-		riscv,ndev=<71>;
+			riscv,ndev=<71>;
-		interrupts-extended = <&CPU0_intc 11 &CPU0_intc 9>;
+			interrupts-extended = <&CPU0_intc 11 &CPU0_intc 9 &CPU1_intc 11 &CPU1_intc 9>;
-	};
+		};
 
-	plic1: interrupt-controller@e6400000 {
+		plic1: interrupt-controller@e6400000 {
-		compatible = "riscv,plic1";
+			compatible = "riscv,plic1";
-		#address-cells = <2>;
+			#address-cells = <2>;
-		#interrupt-cells = <2>;
+			#interrupt-cells = <2>;
-		interrupt-controller;
+			interrupt-controller;
-		reg = <0x0 0xe6400000 0x0 0x400000>;
+			reg = <0x0 0xe6400000 0x0 0x400000>;
-		riscv,ndev=<1>;
+			riscv,ndev=<2>;
-		interrupts-extended = <&CPU0_intc 3>;
+			interrupts-extended = <&CPU0_intc 3 &CPU1_intc 3>;
-	};
+		};
 
-	plmt0@e6000000 {
+		plmt0@e6000000 {
-		compatible = "riscv,plmt0";
+			compatible = "riscv,plmt0";
-			interrupts-extended = <&CPU0_intc 7>;
+			interrupts-extended = <&CPU0_intc 7 &CPU1_intc 7>;
 			reg = <0x0 0xe6000000 0x0 0x100000>;
 		};
 	};
@@ -146,6 +179,10 @@
 		interrupt-parent = <&plic0>;
 	};
 
+	pmu {
+		compatible = "riscv,base-pmu";
+	};
+
 	virtio_mmio@fe007000 {
 		interrupts = <0x17 0x4>;
 		interrupt-parent = <0x2>;

arch/riscv/include/asm/csr.h

@@ -46,6 +46,7 @@
 #endif
 
 /* Interrupt Enable and Interrupt Pending flags */
+#define MIE_MSIE	_AC(0x00000008, UL) /* Software Interrupt Enable */
 #define SIE_SSIE	_AC(0x00000002, UL) /* Software Interrupt Enable */
 #define SIE_STIE	_AC(0x00000020, UL) /* Timer Interrupt Enable */
 

arch/riscv/include/asm/global_data.h

@@ -10,12 +10,24 @@
 #ifndef	__ASM_GBL_DATA_H
 #define __ASM_GBL_DATA_H
 
+#include <asm/smp.h>
+
 /* Architecture-specific global data */
 struct arch_global_data {
 	long boot_hart;		/* boot hart id */
 #ifdef CONFIG_SIFIVE_CLINT
 	void __iomem *clint;	/* clint base address */
 #endif
+#ifdef CONFIG_ANDES_PLIC
+	void __iomem *plic;	/* plic base address */
+#endif
+#ifdef CONFIG_ANDES_PLMT
+	void __iomem *plmt;	/* plmt base address */
+#endif
+#ifdef CONFIG_SMP
+	struct ipi_data ipi[CONFIG_NR_CPUS];
+#endif
+	ulong available_harts;
 };
 
 #include <asm-generic/global_data.h>

arch/riscv/include/asm/sbi.h

@@ -0,0 +1,94 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2015 Regents of the University of California
+ *
+ * Taken from Linux arch/riscv/include/asm/sbi.h
+ */
+
+#ifndef _ASM_RISCV_SBI_H
+#define _ASM_RISCV_SBI_H
+
+#include <linux/types.h>
+
+#define SBI_SET_TIMER 0
+#define SBI_CONSOLE_PUTCHAR 1
+#define SBI_CONSOLE_GETCHAR 2
+#define SBI_CLEAR_IPI 3
+#define SBI_SEND_IPI 4
+#define SBI_REMOTE_FENCE_I 5
+#define SBI_REMOTE_SFENCE_VMA 6
+#define SBI_REMOTE_SFENCE_VMA_ASID 7
+#define SBI_SHUTDOWN 8
+
+#define SBI_CALL(which, arg0, arg1, arg2) ({			\
+	register uintptr_t a0 asm ("a0") = (uintptr_t)(arg0);	\
+	register uintptr_t a1 asm ("a1") = (uintptr_t)(arg1);	\
+	register uintptr_t a2 asm ("a2") = (uintptr_t)(arg2);	\
+	register uintptr_t a7 asm ("a7") = (uintptr_t)(which);	\
+	asm volatile ("ecall"					\
+		      : "+r" (a0)				\
+		      : "r" (a1), "r" (a2), "r" (a7)		\
+		      : "memory");				\
+	a0;							\
+})
+
+/* Lazy implementations until SBI is finalized */
+#define SBI_CALL_0(which) SBI_CALL(which, 0, 0, 0)
+#define SBI_CALL_1(which, arg0) SBI_CALL(which, arg0, 0, 0)
+#define SBI_CALL_2(which, arg0, arg1) SBI_CALL(which, arg0, arg1, 0)
+
+static inline void sbi_console_putchar(int ch)
+{
+	SBI_CALL_1(SBI_CONSOLE_PUTCHAR, ch);
+}
+
+static inline int sbi_console_getchar(void)
+{
+	return SBI_CALL_0(SBI_CONSOLE_GETCHAR);
+}
+
+static inline void sbi_set_timer(uint64_t stime_value)
+{
+#if __riscv_xlen == 32
+	SBI_CALL_2(SBI_SET_TIMER, stime_value, stime_value >> 32);
+#else
+	SBI_CALL_1(SBI_SET_TIMER, stime_value);
+#endif
+}
+
+static inline void sbi_shutdown(void)
+{
+	SBI_CALL_0(SBI_SHUTDOWN);
+}
+
+static inline void sbi_clear_ipi(void)
+{
+	SBI_CALL_0(SBI_CLEAR_IPI);
+}
+
+static inline void sbi_send_ipi(const unsigned long *hart_mask)
+{
+	SBI_CALL_1(SBI_SEND_IPI, hart_mask);
+}
+
+static inline void sbi_remote_fence_i(const unsigned long *hart_mask)
+{
+	SBI_CALL_1(SBI_REMOTE_FENCE_I, hart_mask);
+}
+
+static inline void sbi_remote_sfence_vma(const unsigned long *hart_mask,
+					 unsigned long start,
+					 unsigned long size)
+{
+	SBI_CALL_1(SBI_REMOTE_SFENCE_VMA, hart_mask);
+}
+
+static inline void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask,
+					      unsigned long start,
+					      unsigned long size,
+					      unsigned long asid)
+{
+	SBI_CALL_1(SBI_REMOTE_SFENCE_VMA_ASID, hart_mask);
+}
+
+#endif

arch/riscv/include/asm/smp.h

@@ -0,0 +1,53 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2019 Fraunhofer AISEC,
+ * Lukas Auer <lukas.auer@aisec.fraunhofer.de>
+ */
+
+#ifndef _ASM_RISCV_SMP_H
+#define _ASM_RISCV_SMP_H
+
+/**
+ * struct ipi_data - Inter-processor interrupt (IPI) data structure
+ *
+ * IPIs are used for SMP support to communicate to other harts what function to
+ * call. Functions are in the form
+ * void (*addr)(ulong hart, ulong arg0, ulong arg1).
+ *
+ * The function address and the two arguments, arg0 and arg1, are stored in the
+ * IPI data structure. The hart ID is inserted by the hart handling the IPI and
+ * calling the function.
+ *
+ * @addr: Address of function
+ * @arg0: First argument of function
+ * @arg1: Second argument of function
+ */
+struct ipi_data {
+	ulong addr;
+	ulong arg0;
+	ulong arg1;
+};
+
+/**
+ * handle_ipi() - interrupt handler for software interrupts
+ *
+ * The IPI interrupt handler must be called to handle software interrupts. It
+ * calls the function specified in the hart's IPI data structure.
+ *
+ * @hart: Hart ID of the current hart
+ */
+void handle_ipi(ulong hart);
+
+/**
+ * smp_call_function() - Call a function on all other harts
+ *
+ * Send IPIs with the specified function call to all harts.
+ *
+ * @addr: Address of function
+ * @arg0: First argument of function
+ * @arg1: Second argument of function
+ * @return 0 if OK, -ve on error
+ */
+int smp_call_function(ulong addr, ulong arg0, ulong arg1);
+
+#endif

arch/riscv/include/asm/syscon.h

@@ -8,12 +8,12 @@
 
 /*
  * System controllers in a RISC-V system
- *
- * So far only SiFive's Core Local Interruptor (CLINT) is defined.
  */
 enum {
 	RISCV_NONE,
 	RISCV_SYSCON_CLINT,	/* Core Local Interruptor (CLINT) */
+	RISCV_SYSCON_PLIC,	/* Platform Level Interrupt Controller (PLIC) */
+	RISCV_SYSCON_PLMT,	/* Platform Level Machine Timer (PLMT) */
 };
 
 #endif /* _ASM_SYSCON_H */

arch/riscv/lib/Makefile

@@ -11,9 +11,13 @@ obj-$(CONFIG_CMD_GO) += boot.o
 obj-y	+= cache.o
 obj-$(CONFIG_RISCV_RDTIME) += rdtime.o
 obj-$(CONFIG_SIFIVE_CLINT) += sifive_clint.o
+obj-$(CONFIG_ANDES_PLIC) += andes_plic.o
+obj-$(CONFIG_ANDES_PLMT) += andes_plmt.o
 obj-y	+= interrupts.o
 obj-y	+= reset.o
+obj-$(CONFIG_SBI_IPI) += sbi_ipi.o
 obj-y   += setjmp.o
+obj-$(CONFIG_SMP) += smp.o
 
 # For building EFI apps
 CFLAGS_$(EFI_CRT0) := $(CFLAGS_EFI)

arch/riscv/lib/andes_plic.c

@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2019, Rick Chen <rick@andestech.com>
+ *
+ * U-Boot syscon driver for Andes's Platform Level Interrupt Controller (PLIC).
+ * The PLIC block holds memory-mapped claim and pending registers
+ * associated with software interrupt.
+ */
+
+#include <common.h>
+#include <dm.h>
+#include <dm/device-internal.h>
+#include <dm/lists.h>
+#include <dm/uclass-internal.h>
+#include <regmap.h>
+#include <syscon.h>
+#include <asm/io.h>
+#include <asm/syscon.h>
+#include <cpu.h>
+
+/* pending register */
+#define PENDING_REG(base, hart)	((ulong)(base) + 0x1000 + (hart) * 8)
+/* enable register */
+#define ENABLE_REG(base, hart)	((ulong)(base) + 0x2000 + (hart) * 0x80)
+/* claim register */
+#define CLAIM_REG(base, hart)	((ulong)(base) + 0x200004 + (hart) * 0x1000)
+
+#define ENABLE_HART_IPI         (0x80808080)
+#define SEND_IPI_TO_HART(hart)  (0x80 >> (hart))
+
+DECLARE_GLOBAL_DATA_PTR;
+static int init_plic(void);
+
+#define PLIC_BASE_GET(void)						\
+	do {								\
+		long *ret;						\
+									\
+		if (!gd->arch.plic) {					\
+			ret = syscon_get_first_range(RISCV_SYSCON_PLIC); \
+			if (IS_ERR(ret))				\
+				return PTR_ERR(ret);			\
+			gd->arch.plic = ret;				\
+			init_plic();					\
+		}							\
+	} while (0)
+
+static int enable_ipi(int harts)
+{
+	int i;
+	int en = ENABLE_HART_IPI;
+
+	for (i = 0; i < harts; i++) {
+		en = en >> i;
+		writel(en, (void __iomem *)ENABLE_REG(gd->arch.plic, i));
+	}
+
+	return 0;
+}
+
+static int init_plic(void)
+{
+	struct udevice *dev;
+	int ret;
+
+	ret = uclass_find_first_device(UCLASS_CPU, &dev);
+	if (ret)
+		return ret;
+
+	if (ret == 0 && dev) {
+		ret = cpu_get_count(dev);
+		if (ret < 0)
+			return ret;
+
+		enable_ipi(ret);
+		return 0;
+	}
+
+	return -ENODEV;
+}
+
+int riscv_send_ipi(int hart)
+{
+	PLIC_BASE_GET();
+
+	writel(SEND_IPI_TO_HART(hart),
+	       (void __iomem *)PENDING_REG(gd->arch.plic, gd->arch.boot_hart));
+
+	return 0;
+}
+
+int riscv_clear_ipi(int hart)
+{
+	u32 source_id;
+
+	PLIC_BASE_GET();
+
+	source_id = readl((void __iomem *)CLAIM_REG(gd->arch.plic, hart));
+	writel(source_id, (void __iomem *)CLAIM_REG(gd->arch.plic, hart));
+
+	return 0;
+}
+
+static const struct udevice_id andes_plic_ids[] = {
+	{ .compatible = "riscv,plic1", .data = RISCV_SYSCON_PLIC },
+	{ }
+};
+
+U_BOOT_DRIVER(andes_plic) = {
+	.name		= "andes_plic",
+	.id		= UCLASS_SYSCON,
+	.of_match	= andes_plic_ids,
+	.flags		= DM_FLAG_PRE_RELOC,
+};

arch/riscv/lib/andes_plmt.c

@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2019, Rick Chen <rick@andestech.com>
+ *
+ * U-Boot syscon driver for Andes's Platform Level Machine Timer (PLMT).
+ * The PLMT block holds memory-mapped mtime register
+ * associated with timer tick.
+ */
+
+#include <common.h>
+#include <dm.h>
+#include <regmap.h>
+#include <syscon.h>
+#include <asm/io.h>
+#include <asm/syscon.h>
+
+/* mtime register */
+#define MTIME_REG(base)			((ulong)(base))
+
+DECLARE_GLOBAL_DATA_PTR;
+
+#define PLMT_BASE_GET(void)						\
+	do {								\
+		long *ret;						\
+									\
+		if (!gd->arch.plmt) {					\
+			ret = syscon_get_first_range(RISCV_SYSCON_PLMT); \
+			if (IS_ERR(ret))				\
+				return PTR_ERR(ret);			\
+			gd->arch.plmt = ret;				\
+		}							\
+	} while (0)
+
+int riscv_get_time(u64 *time)
+{
+	PLMT_BASE_GET();
+
+	*time = readq((void __iomem *)MTIME_REG(gd->arch.plmt));
+
+	return 0;
+}
+
+static const struct udevice_id andes_plmt_ids[] = {
+	{ .compatible = "riscv,plmt0", .data = RISCV_SYSCON_PLMT },
+	{ }
+};
+
+U_BOOT_DRIVER(andes_plmt) = {
+	.name		= "andes_plmt",
+	.id		= UCLASS_SYSCON,
+	.of_match	= andes_plmt_ids,
+	.flags		= DM_FLAG_PRE_RELOC,
+};

arch/riscv/lib/asm-offsets.c

@@ -14,6 +14,7 @@
 int main(void)
 {
 	DEFINE(GD_BOOT_HART, offsetof(gd_t, arch.boot_hart));
+	DEFINE(GD_AVAILABLE_HARTS, offsetof(gd_t, arch.available_harts));
 
 	return 0;
 }

arch/riscv/lib/bootm.c

@@ -13,6 +13,7 @@
 #include <image.h>
 #include <asm/byteorder.h>
 #include <asm/csr.h>
+#include <asm/smp.h>
 #include <dm/device.h>
 #include <dm/root.h>
 #include <u-boot/zlib.h>
@@ -81,6 +82,9 @@ static void boot_jump_linux(bootm_headers_t *images, int flag)
 {
 	void (*kernel)(ulong hart, void *dtb);
 	int fake = (flag & BOOTM_STATE_OS_FAKE_GO);
+#ifdef CONFIG_SMP
+	int ret;
+#endif
 
 	kernel = (void (*)(ulong, void *))images->ep;
 
@@ -92,8 +96,15 @@ static void boot_jump_linux(bootm_headers_t *images, int flag)
 	announce_and_cleanup(fake);
 
 	if (!fake) {
-		if (IMAGE_ENABLE_OF_LIBFDT && images->ft_len)
+		if (IMAGE_ENABLE_OF_LIBFDT && images->ft_len) {
+#ifdef CONFIG_SMP
+			ret = smp_call_function(images->ep,
+						(ulong)images->ft_addr, 0);
+			if (ret)
+				hang();
+#endif
 			kernel(gd->arch.boot_hart, images->ft_addr);
+		}
 	}
 }
 

arch/riscv/lib/sbi_ipi.c

@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2019 Fraunhofer AISEC,
+ * Lukas Auer <lukas.auer@aisec.fraunhofer.de>
+ */
+
+#include <common.h>
+#include <asm/sbi.h>
+
+int riscv_send_ipi(int hart)
+{
+	ulong mask;
+
+	mask = 1UL << hart;
+	sbi_send_ipi(&mask);
+
+	return 0;
+}
+
+int riscv_clear_ipi(int hart)
+{
+	sbi_clear_ipi();
+
+	return 0;
+}

arch/riscv/lib/smp.c

@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2019 Fraunhofer AISEC,
+ * Lukas Auer <lukas.auer@aisec.fraunhofer.de>
+ */
+
+#include <common.h>
+#include <dm.h>
+#include <asm/barrier.h>
+#include <asm/smp.h>
+
+DECLARE_GLOBAL_DATA_PTR;
+
+/**
+ * riscv_send_ipi() - Send inter-processor interrupt (IPI)
+ *
+ * Platform code must provide this function.
+ *
+ * @hart: Hart ID of receiving hart
+ * @return 0 if OK, -ve on error
+ */
+extern int riscv_send_ipi(int hart);
+
+/**
+ * riscv_clear_ipi() - Clear inter-processor interrupt (IPI)
+ *
+ * Platform code must provide this function.
+ *
+ * @hart: Hart ID of hart to be cleared
+ * @return 0 if OK, -ve on error
+ */
+extern int riscv_clear_ipi(int hart);
+
+static int send_ipi_many(struct ipi_data *ipi)
+{
+	ofnode node, cpus;
+	u32 reg;
+	int ret;
+
+	cpus = ofnode_path("/cpus");
+	if (!ofnode_valid(cpus)) {
+		pr_err("Can't find cpus node!\n");
+		return -EINVAL;
+	}
+
+	ofnode_for_each_subnode(node, cpus) {
+		/* skip if hart is marked as not available in the device tree */
+		if (!ofnode_is_available(node))
+			continue;
+
+		/* read hart ID of CPU */
+		ret = ofnode_read_u32(node, "reg", &reg);
+		if (ret)
+			continue;
+
+		/* skip if it is the hart we are running on */
+		if (reg == gd->arch.boot_hart)
+			continue;
+
+		if (reg >= CONFIG_NR_CPUS) {
+			pr_err("Hart ID %d is out of range, increase CONFIG_NR_CPUS\n",
+			       reg);
+			continue;
+		}
+
+		/* skip if hart is not available */
+		if (!(gd->arch.available_harts & (1 << reg)))
+			continue;
+
+		gd->arch.ipi[reg].addr = ipi->addr;
+		gd->arch.ipi[reg].arg0 = ipi->arg0;
+		gd->arch.ipi[reg].arg1 = ipi->arg1;
+
+		ret = riscv_send_ipi(reg);
+		if (ret) {
+			pr_err("Cannot send IPI to hart %d\n", reg);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+void handle_ipi(ulong hart)
+{
+	int ret;
+	void (*smp_function)(ulong hart, ulong arg0, ulong arg1);
+
+	if (hart >= CONFIG_NR_CPUS)
+		return;
+
+	ret = riscv_clear_ipi(hart);
+	if (ret) {
+		pr_err("Cannot clear IPI of hart %ld\n", hart);
+		return;
+	}
+
+	__smp_mb();
+
+	smp_function = (void (*)(ulong, ulong, ulong))gd->arch.ipi[hart].addr;
+	invalidate_icache_all();
+
+	smp_function(hart, gd->arch.ipi[hart].arg0, gd->arch.ipi[hart].arg1);
+}
+
+int smp_call_function(ulong addr, ulong arg0, ulong arg1)
+{
+	int ret = 0;
+	struct ipi_data ipi;
+
+	ipi.addr = addr;
+	ipi.arg0 = arg0;
+	ipi.arg1 = arg1;
+
+	ret = send_ipi_many(&ipi);
+
+	return ret;
+}

board/AndesTech/ax25-ae350/Kconfig

@@ -24,5 +24,6 @@ config ENV_OFFSET
 config BOARD_SPECIFIC_OPTIONS # dummy
 	def_bool y
 	select RISCV_NDS
+	imply SMP
 
 endif

board/emulation/qemu-riscv/Kconfig

@@ -34,5 +34,6 @@ config BOARD_SPECIFIC_OPTIONS # dummy
 	imply BOARD_LATE_INIT
 	imply OF_BOARD_SETUP
 	imply SIFIVE_SERIAL
+	imply SMP
 
 endif

board/sifive/fu540/Kconfig

@@ -38,5 +38,6 @@ config BOARD_SPECIFIC_OPTIONS # dummy
 	imply PHY_LIB
 	imply PHY_MSCC
 	imply SIFIVE_SERIAL
+	imply SMP
 
 endif

configs/ae350_rv32_defconfig

@@ -34,4 +34,3 @@ CONFIG_BAUDRATE=38400
 CONFIG_SYS_NS16550=y
 CONFIG_SPI=y
 CONFIG_ATCSPI200_SPI=y
-CONFIG_ATCPIT100_TIMER=y

configs/ae350_rv64_defconfig

@@ -35,4 +35,3 @@ CONFIG_BAUDRATE=38400
 CONFIG_SYS_NS16550=y
 CONFIG_SPI=y
 CONFIG_ATCSPI200_SPI=y
-CONFIG_ATCPIT100_TIMER=y