1/*
  2 * SMP initialisation and IPI support
  3 * Based on arch/arm64/kernel/smp.c
  4 *
  5 * Copyright (C) 2012 ARM Ltd.
  6 * Copyright (C) 2015 Regents of the University of California
  7 * Copyright (C) 2017 SiFive
  8 *
  9 * This program is free software; you can redistribute it and/or modify
 10 * it under the terms of the GNU General Public License version 2 as
 11 * published by the Free Software Foundation.
 12 *
 13 * This program is distributed in the hope that it will be useful,
 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 16 * GNU General Public License for more details.
 17 *
 18 * You should have received a copy of the GNU General Public License
 19 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 20 */
 21
 22#include <linux/interrupt.h>
 23#include <linux/smp.h>
 24#include <linux/sched.h>
 25
 26#include <asm/sbi.h>
 27#include <asm/tlbflush.h>
 28#include <asm/cacheflush.h>
 29
 30/* A collection of single bit ipi messages.  */
 31static struct {
 32	unsigned long bits ____cacheline_aligned;
 33} ipi_data[NR_CPUS] __cacheline_aligned;
 34
 35enum ipi_message_type {
 36	IPI_RESCHEDULE,
 37	IPI_CALL_FUNC,
 38	IPI_MAX
 39};
 40
 41
 42/* Unsupported */
 43int setup_profiling_timer(unsigned int multiplier)
 44{
 45	return -EINVAL;
 46}
 47
 48irqreturn_t handle_ipi(void)
 49{
 50	unsigned long *pending_ipis = &ipi_data[smp_processor_id()].bits;
 51
 52	/* Clear pending IPI */
 53	csr_clear(sip, SIE_SSIE);
 54
 55	while (true) {
 56		unsigned long ops;
 57
 58		/* Order bit clearing and data access. */
 59		mb();
 60
 61		ops = xchg(pending_ipis, 0);
 62		if (ops == 0)
 63			return IRQ_HANDLED;
 64
 65		if (ops & (1 << IPI_RESCHEDULE))
 66			scheduler_ipi();
 67
 68		if (ops & (1 << IPI_CALL_FUNC))
 69			generic_smp_call_function_interrupt();
 70
 71		BUG_ON((ops >> IPI_MAX) != 0);
 72
 73		/* Order data access and bit testing. */
 74		mb();
 75	}
 76
 77	return IRQ_HANDLED;
 78}
 79
 80static void
 81send_ipi_message(const struct cpumask *to_whom, enum ipi_message_type operation)
 82{
 83	int i;
 84
 85	mb();
 86	for_each_cpu(i, to_whom)
 87		set_bit(operation, &ipi_data[i].bits);
 88
 89	mb();
 90	sbi_send_ipi(cpumask_bits(to_whom));
 91}
 92
 93void arch_send_call_function_ipi_mask(struct cpumask *mask)
 94{
 95	send_ipi_message(mask, IPI_CALL_FUNC);
 96}
 97
 98void arch_send_call_function_single_ipi(int cpu)
 99{
100	send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC);
101}
102
103static void ipi_stop(void *unused)
104{
105	while (1)
106		wait_for_interrupt();
107}
108
109void smp_send_stop(void)
110{
111	on_each_cpu(ipi_stop, NULL, 1);
112}
113
114void smp_send_reschedule(int cpu)
115{
116	send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
117}
118
119/*
120 * Performs an icache flush for the given MM context.  RISC-V has no direct
121 * mechanism for instruction cache shoot downs, so instead we send an IPI that
122 * informs the remote harts they need to flush their local instruction caches.
123 * To avoid pathologically slow behavior in a common case (a bunch of
124 * single-hart processes on a many-hart machine, ie 'make -j') we avoid the
125 * IPIs for harts that are not currently executing a MM context and instead
126 * schedule a deferred local instruction cache flush to be performed before
127 * execution resumes on each hart.
128 */
129void flush_icache_mm(struct mm_struct *mm, bool local)
130{
131	unsigned int cpu;
132	cpumask_t others, *mask;
133
134	preempt_disable();
135
136	/* Mark every hart's icache as needing a flush for this MM. */
137	mask = &mm->context.icache_stale_mask;
138	cpumask_setall(mask);
139	/* Flush this hart's I$ now, and mark it as flushed. */
140	cpu = smp_processor_id();
141	cpumask_clear_cpu(cpu, mask);
142	local_flush_icache_all();
143
144	/*
145	 * Flush the I$ of other harts concurrently executing, and mark them as
146	 * flushed.
147	 */
148	cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu));
149	local |= cpumask_empty(&others);
150	if (mm != current->active_mm || !local)
151		sbi_remote_fence_i(others.bits);
152	else {
153		/*
154		 * It's assumed that at least one strongly ordered operation is
155		 * performed on this hart between setting a hart's cpumask bit
156		 * and scheduling this MM context on that hart.  Sending an SBI
157		 * remote message will do this, but in the case where no
158		 * messages are sent we still need to order this hart's writes
159		 * with flush_icache_deferred().
160		 */
161		smp_mb();
162	}
163
164	preempt_enable();
165}