GitHub-Mirror/Star64_linux
2
0
Fork 0
mirror of https://github.com/Fishwaldo/Star64_linux.git synced 2025-07-04 13:21:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'ipi-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip

Browse source 
* 'ipi-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip:
  s390: remove arch specific smp_send_stop()
  panic: clean up kernel/panic.c
  panic, smp: provide smp_send_stop() wrapper on UP too
  panic: decrease oops_in_progress only after having done the panic
  generic-ipi: eliminate WARN_ON()s during oops/panic
  generic-ipi: cleanups
  generic-ipi: remove CSD_FLAG_WAIT
  generic-ipi: remove kmalloc()
  generic IPI: simplify barriers and locking
This commit is contained in:
Linus Torvalds 2009-04-03 17:33:30 -07:00
commit b1dbb67911
7 changed files with 307 additions and 265 deletions
Download patch file Download diff file Expand all files Collapse all files

6
arch/s390/include/asm/smp.h
View file

@ -92,12 +92,6 @@ extern void arch_send_call_function_ipi(cpumask_t mask);
#endif #endif
#ifndef CONFIG_SMP #ifndef CONFIG_SMP
static inline void smp_send_stop(void)
{
/* Disable all interrupts/machine checks */
__load_psw_mask(psw_kernel_bits & ~PSW_MASK_MCHECK);
}
#define hard_smp_processor_id() 0 #define hard_smp_processor_id() 0
#define smp_cpu_not_running(cpu) 1 #define smp_cpu_not_running(cpu) 1
#endif #endif

2
block/blk-softirq.c
View file

@ -64,7 +64,7 @@ static int raise_blk_irq(int cpu, struct request *rq)
data->info = rq; data->info = rq;
data->flags = 0; data->flags = 0;
__smp_call_function_single(cpu, data); __smp_call_function_single(cpu, data, 0);
return 0; return 0;
} }

5
include/linux/smp.h
View file

@ -82,7 +82,8 @@ smp_call_function_mask(cpumask_t mask, void(*func)(void *info), void *info,
return 0; return 0;
} }
void __smp_call_function_single(int cpuid, struct call_single_data *data); void __smp_call_function_single(int cpuid, struct call_single_data *data,
int wait);
/* /*
* Generic and arch helpers * Generic and arch helpers
@ -121,6 +122,8 @@ extern unsigned int setup_max_cpus;
#else /* !SMP */ #else /* !SMP */
static inline void smp_send_stop(void) { }
/* /*
* These macros fold the SMP functionality into a single CPU system * These macros fold the SMP functionality into a single CPU system
*/ */

77
kernel/panic.c
View file

@ -8,19 +8,19 @@
* This function is used through-out the kernel (including mm and fs) * This function is used through-out the kernel (including mm and fs)
* to indicate a major problem. * to indicate a major problem.
*/ */
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/delay.h>
#include <linux/reboot.h>
#include <linux/notifier.h>
#include <linux/init.h>
#include <linux/sysrq.h>
#include <linux/interrupt.h>
#include <linux/nmi.h>
#include <linux/kexec.h>
#include <linux/debug_locks.h> #include <linux/debug_locks.h>
#include <linux/random.h> #include <linux/interrupt.h>
#include <linux/kallsyms.h> #include <linux/kallsyms.h>
#include <linux/notifier.h>
#include <linux/module.h>
#include <linux/random.h>
#include <linux/reboot.h>
#include <linux/delay.h>
#include <linux/kexec.h>
#include <linux/sched.h>
#include <linux/sysrq.h>
#include <linux/init.h>
#include <linux/nmi.h>
#include <linux/dmi.h> #include <linux/dmi.h>
int panic_on_oops; int panic_on_oops;
@ -52,19 +52,15 @@ EXPORT_SYMBOL(panic_blink);
* *
* This function never returns. * This function never returns.
*/ */
NORET_TYPE void panic(const char * fmt, ...) NORET_TYPE void panic(const char * fmt, ...)
{ {
long i;
static char buf[1024]; static char buf[1024];
va_list args; va_list args;
#if defined(CONFIG_S390) long i;
unsigned long caller = (unsigned long) __builtin_return_address(0);
#endif
/* /*
* It's possible to come here directly from a panic-assertion and not * It's possible to come here directly from a panic-assertion and
* have preempt disabled. Some functions called from here want * not have preempt disabled. Some functions called from here want
* preempt to be disabled. No point enabling it later though... * preempt to be disabled. No point enabling it later though...
*/ */
preempt_disable(); preempt_disable();
@ -77,7 +73,6 @@ NORET_TYPE void panic(const char * fmt, ...)
#ifdef CONFIG_DEBUG_BUGVERBOSE #ifdef CONFIG_DEBUG_BUGVERBOSE
dump_stack(); dump_stack();
#endif #endif
bust_spinlocks(0);
/* /*
* If we have crashed and we have a crash kernel loaded let it handle * If we have crashed and we have a crash kernel loaded let it handle
@ -86,14 +81,12 @@ NORET_TYPE void panic(const char * fmt, ...)
*/ */
crash_kexec(NULL); crash_kexec(NULL);
#ifdef CONFIG_SMP
/* /*
* Note smp_send_stop is the usual smp shutdown function, which * Note smp_send_stop is the usual smp shutdown function, which
* unfortunately means it may not be hardened to work in a panic * unfortunately means it may not be hardened to work in a panic
* situation. * situation.
*/ */
smp_send_stop(); smp_send_stop();
#endif
atomic_notifier_call_chain(&panic_notifier_list, 0, buf); atomic_notifier_call_chain(&panic_notifier_list, 0, buf);
@ -103,16 +96,18 @@ NORET_TYPE void panic(const char * fmt, ...)
if (panic_timeout > 0) { if (panic_timeout > 0) {
/* /*
* Delay timeout seconds before rebooting the machine. * Delay timeout seconds before rebooting the machine.
* We can't use the "normal" timers since we just panicked.. * We can't use the "normal" timers since we just panicked.
*/ */
printk(KERN_EMERG "Rebooting in %d seconds..", panic_timeout); printk(KERN_EMERG "Rebooting in %d seconds..", panic_timeout);
for (i = 0; i < panic_timeout*1000; ) { for (i = 0; i < panic_timeout*1000; ) {
touch_nmi_watchdog(); touch_nmi_watchdog();
i += panic_blink(i); i += panic_blink(i);
mdelay(1); mdelay(1);
i++; i++;
} }
/* This will not be a clean reboot, with everything /*
* This will not be a clean reboot, with everything
* shutting down. But if there is a chance of * shutting down. But if there is a chance of
* rebooting the system it will be rebooted. * rebooting the system it will be rebooted.
*/ */
@ -127,7 +122,12 @@ NORET_TYPE void panic(const char * fmt, ...)
} }
#endif #endif
#if defined(CONFIG_S390) #if defined(CONFIG_S390)
{
unsigned long caller;
caller = (unsigned long)__builtin_return_address(0);
disabled_wait(caller); disabled_wait(caller);
}
#endif #endif
local_irq_enable(); local_irq_enable();
for (i = 0; ; ) { for (i = 0; ; ) {
@ -136,6 +136,7 @@ NORET_TYPE void panic(const char * fmt, ...)
mdelay(1); mdelay(1);
i++; i++;
} }
bust_spinlocks(0);
} }
EXPORT_SYMBOL(panic); EXPORT_SYMBOL(panic);
@ -195,7 +196,8 @@ const char *print_tainted(void)
*s = 0; *s = 0;
} else } else
snprintf(buf, sizeof(buf), "Not tainted"); snprintf(buf, sizeof(buf), "Not tainted");
return(buf);
return buf;
} }
int test_taint(unsigned flag) int test_taint(unsigned flag)
@ -211,7 +213,8 @@ unsigned long get_taint(void)
void add_taint(unsigned flag) void add_taint(unsigned flag)
{ {
debug_locks = 0; /* can't trust the integrity of the kernel anymore */ /* can't trust the integrity of the kernel anymore: */
debug_locks = 0;
set_bit(flag, &tainted_mask); set_bit(flag, &tainted_mask);
} }
EXPORT_SYMBOL(add_taint); EXPORT_SYMBOL(add_taint);
@ -266,8 +269,8 @@ static void do_oops_enter_exit(void)
} }
/* /*
* Return true if the calling CPU is allowed to print oops-related info. This * Return true if the calling CPU is allowed to print oops-related info.
* is a bit racy.. * This is a bit racy..
*/ */
int oops_may_print(void) int oops_may_print(void)
{ {
@ -276,20 +279,22 @@ int oops_may_print(void)
/* /*
* Called when the architecture enters its oops handler, before it prints * Called when the architecture enters its oops handler, before it prints
* anything. If this is the first CPU to oops, and it's oopsing the first time * anything. If this is the first CPU to oops, and it's oopsing the first
* then let it proceed. * time then let it proceed.
* *
* This is all enabled by the pause_on_oops kernel boot option. We do all this * This is all enabled by the pause_on_oops kernel boot option. We do all
* to ensure that oopses don't scroll off the screen. It has the side-effect * this to ensure that oopses don't scroll off the screen. It has the
* of preventing later-oopsing CPUs from mucking up the display, too. * side-effect of preventing later-oopsing CPUs from mucking up the display,
* too.
* *
* It turns out that the CPU which is allowed to print ends up pausing for the * It turns out that the CPU which is allowed to print ends up pausing for
* right duration, whereas all the other CPUs pause for twice as long: once in * the right duration, whereas all the other CPUs pause for twice as long:
* oops_enter(), once in oops_exit(). * once in oops_enter(), once in oops_exit().
*/ */
void oops_enter(void) void oops_enter(void)
{ {
debug_locks_off(); /* can't trust the integrity of the kernel anymore */ /* can't trust the integrity of the kernel anymore: */
debug_locks_off();
do_oops_enter_exit(); do_oops_enter_exit();
} }

2
kernel/sched.c
View file

@ -1110,7 +1110,7 @@ static void hrtick_start(struct rq *rq, u64 delay)
if (rq == this_rq()) { if (rq == this_rq()) {
hrtimer_restart(timer); hrtimer_restart(timer);
} else if (!rq->hrtick_csd_pending) { } else if (!rq->hrtick_csd_pending) {
__smp_call_function_single(cpu_of(rq), &rq->hrtick_csd); __smp_call_function_single(cpu_of(rq), &rq->hrtick_csd, 0);
rq->hrtick_csd_pending = 1; rq->hrtick_csd_pending = 1;
} }
} }

384
kernel/smp.c
View file

@ -2,31 +2,36 @@
* Generic helpers for smp ipi calls * Generic helpers for smp ipi calls
* *
* (C) Jens Axboe <jens.axboe@oracle.com> 2008 * (C) Jens Axboe <jens.axboe@oracle.com> 2008
*
*/ */
#include <linux/init.h>
#include <linux/module.h>
#include <linux/percpu.h>
#include <linux/rcupdate.h> #include <linux/rcupdate.h>
#include <linux/rculist.h> #include <linux/rculist.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/percpu.h>
#include <linux/init.h>
#include <linux/smp.h> #include <linux/smp.h>
#include <linux/cpu.h>
static DEFINE_PER_CPU(struct call_single_queue, call_single_queue); static DEFINE_PER_CPU(struct call_single_queue, call_single_queue);
static LIST_HEAD(call_function_queue);
__cacheline_aligned_in_smp DEFINE_SPINLOCK(call_function_lock); static struct {
struct list_head queue;
spinlock_t lock;
} call_function __cacheline_aligned_in_smp =
{
.queue = LIST_HEAD_INIT(call_function.queue),
.lock = __SPIN_LOCK_UNLOCKED(call_function.lock),
};
enum { enum {
CSD_FLAG_WAIT = 0x01, CSD_FLAG_LOCK = 0x01,
CSD_FLAG_ALLOC = 0x02,
CSD_FLAG_LOCK = 0x04,
}; };
struct call_function_data { struct call_function_data {
struct call_single_data csd; struct call_single_data csd;
spinlock_t lock; spinlock_t lock;
unsigned int refs; unsigned int refs;
struct rcu_head rcu_head; cpumask_var_t cpumask;
unsigned long cpumask_bits[];
}; };
struct call_single_queue { struct call_single_queue {
@ -34,8 +39,45 @@ struct call_single_queue {
spinlock_t lock; spinlock_t lock;
}; };
static DEFINE_PER_CPU(struct call_function_data, cfd_data) = {
.lock = __SPIN_LOCK_UNLOCKED(cfd_data.lock),
};
static int
hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu)
{
long cpu = (long)hcpu;
struct call_function_data *cfd = &per_cpu(cfd_data, cpu);
switch (action) {
case CPU_UP_PREPARE:
case CPU_UP_PREPARE_FROZEN:
if (!alloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL,
cpu_to_node(cpu)))
return NOTIFY_BAD;
break;
#ifdef CONFIG_CPU_HOTPLUG
case CPU_UP_CANCELED:
case CPU_UP_CANCELED_FROZEN:
case CPU_DEAD:
case CPU_DEAD_FROZEN:
free_cpumask_var(cfd->cpumask);
break;
#endif
};
return NOTIFY_OK;
}
static struct notifier_block __cpuinitdata hotplug_cfd_notifier = {
.notifier_call = hotplug_cfd,
};
static int __cpuinit init_call_single_data(void) static int __cpuinit init_call_single_data(void)
{ {
void *cpu = (void *)(long)smp_processor_id();
int i; int i;
for_each_possible_cpu(i) { for_each_possible_cpu(i) {
@ -44,29 +86,63 @@ static int __cpuinit init_call_single_data(void)
spin_lock_init(&q->lock); spin_lock_init(&q->lock);
INIT_LIST_HEAD(&q->list); INIT_LIST_HEAD(&q->list);
} }
hotplug_cfd(&hotplug_cfd_notifier, CPU_UP_PREPARE, cpu);
register_cpu_notifier(&hotplug_cfd_notifier);
return 0; return 0;
} }
early_initcall(init_call_single_data); early_initcall(init_call_single_data);
static void csd_flag_wait(struct call_single_data *data) /*
* csd_lock/csd_unlock used to serialize access to per-cpu csd resources
*
* For non-synchronous ipi calls the csd can still be in use by the
* previous function call. For multi-cpu calls its even more interesting
* as we'll have to ensure no other cpu is observing our csd.
*/
static void csd_lock_wait(struct call_single_data *data)
{ {
/* Wait for response */ while (data->flags & CSD_FLAG_LOCK)
do {
if (!(data->flags & CSD_FLAG_WAIT))
break;
cpu_relax(); cpu_relax();
} while (1); }
static void csd_lock(struct call_single_data *data)
{
csd_lock_wait(data);
data->flags = CSD_FLAG_LOCK;
/*
* prevent CPU from reordering the above assignment
* to ->flags with any subsequent assignments to other
* fields of the specified call_single_data structure:
*/
smp_mb();
}
static void csd_unlock(struct call_single_data *data)
{
WARN_ON(!(data->flags & CSD_FLAG_LOCK));
/*
* ensure we're all done before releasing data:
*/
smp_mb();
data->flags &= ~CSD_FLAG_LOCK;
} }
/* /*
* Insert a previously allocated call_single_data element for execution * Insert a previously allocated call_single_data element
* on the given CPU. data must already have ->func, ->info, and ->flags set. * for execution on the given CPU. data must already have
* ->func, ->info, and ->flags set.
*/ */
static void generic_exec_single(int cpu, struct call_single_data *data) static
void generic_exec_single(int cpu, struct call_single_data *data, int wait)
{ {
struct call_single_queue *dst = &per_cpu(call_single_queue, cpu); struct call_single_queue *dst = &per_cpu(call_single_queue, cpu);
int wait = data->flags & CSD_FLAG_WAIT, ipi;
unsigned long flags; unsigned long flags;
int ipi;
spin_lock_irqsave(&dst->lock, flags); spin_lock_irqsave(&dst->lock, flags);
ipi = list_empty(&dst->list); ipi = list_empty(&dst->list);
@ -74,24 +150,21 @@ static void generic_exec_single(int cpu, struct call_single_data *data)
spin_unlock_irqrestore(&dst->lock, flags); spin_unlock_irqrestore(&dst->lock, flags);
/* /*
* Make the list addition visible before sending the ipi. * The list addition should be visible before sending the IPI
* handler locks the list to pull the entry off it because of
* normal cache coherency rules implied by spinlocks.
*
* If IPIs can go out of order to the cache coherency protocol
* in an architecture, sufficient synchronisation should be added
* to arch code to make it appear to obey cache coherency WRT
* locking and barrier primitives. Generic code isn't really
* equipped to do the right thing...
*/ */
smp_mb();
if (ipi) if (ipi)
arch_send_call_function_single_ipi(cpu); arch_send_call_function_single_ipi(cpu);
if (wait) if (wait)
csd_flag_wait(data); csd_lock_wait(data);
}
static void rcu_free_call_data(struct rcu_head *head)
{
struct call_function_data *data;
data = container_of(head, struct call_function_data, rcu_head);
kfree(data);
} }
/* /*
@ -104,64 +177,58 @@ void generic_smp_call_function_interrupt(void)
int cpu = get_cpu(); int cpu = get_cpu();
/* /*
* It's ok to use list_for_each_rcu() here even though we may delete * Ensure entry is visible on call_function_queue after we have
* 'pos', since list_del_rcu() doesn't clear ->next * entered the IPI. See comment in smp_call_function_many.
* If we don't have this, then we may miss an entry on the list
* and never get another IPI to process it.
*/ */
rcu_read_lock(); smp_mb();
list_for_each_entry_rcu(data, &call_function_queue, csd.list) {
/*
* It's ok to use list_for_each_rcu() here even though we may
* delete 'pos', since list_del_rcu() doesn't clear ->next
*/
list_for_each_entry_rcu(data, &call_function.queue, csd.list) {
int refs; int refs;
if (!cpumask_test_cpu(cpu, to_cpumask(data->cpumask_bits))) spin_lock(&data->lock);
if (!cpumask_test_cpu(cpu, data->cpumask)) {
spin_unlock(&data->lock);
continue; continue;
}
cpumask_clear_cpu(cpu, data->cpumask);
spin_unlock(&data->lock);
data->csd.func(data->csd.info); data->csd.func(data->csd.info);
spin_lock(&data->lock); spin_lock(&data->lock);
cpumask_clear_cpu(cpu, to_cpumask(data->cpumask_bits));
WARN_ON(data->refs == 0); WARN_ON(data->refs == 0);
data->refs--; refs = --data->refs;
refs = data->refs; if (!refs) {
spin_lock(&call_function.lock);
list_del_rcu(&data->csd.list);
spin_unlock(&call_function.lock);
}
spin_unlock(&data->lock); spin_unlock(&data->lock);
if (refs) if (refs)
continue; continue;
spin_lock(&call_function_lock); csd_unlock(&data->csd);
list_del_rcu(&data->csd.list);
spin_unlock(&call_function_lock);
if (data->csd.flags & CSD_FLAG_WAIT) {
/*
* serialize stores to data with the flag clear
* and wakeup
*/
smp_wmb();
data->csd.flags &= ~CSD_FLAG_WAIT;
} }
if (data->csd.flags & CSD_FLAG_ALLOC)
call_rcu(&data->rcu_head, rcu_free_call_data);
}
rcu_read_unlock();
put_cpu(); put_cpu();
} }
/* /*
* Invoked by arch to handle an IPI for call function single. Must be called * Invoked by arch to handle an IPI for call function single. Must be
* from the arch with interrupts disabled. * called from the arch with interrupts disabled.
*/ */
void generic_smp_call_function_single_interrupt(void) void generic_smp_call_function_single_interrupt(void)
{ {
struct call_single_queue *q = &__get_cpu_var(call_single_queue); struct call_single_queue *q = &__get_cpu_var(call_single_queue);
LIST_HEAD(list);
/*
* Need to see other stores to list head for checking whether
* list is empty without holding q->lock
*/
smp_read_barrier_depends();
while (!list_empty(&q->list)) {
unsigned int data_flags; unsigned int data_flags;
LIST_HEAD(list);
spin_lock(&q->lock); spin_lock(&q->lock);
list_replace_init(&q->list, &list); list_replace_init(&q->list, &list);
@ -170,33 +237,23 @@ void generic_smp_call_function_single_interrupt(void)
while (!list_empty(&list)) { while (!list_empty(&list)) {
struct call_single_data *data; struct call_single_data *data;
data = list_entry(list.next, struct call_single_data, data = list_entry(list.next, struct call_single_data, list);
list);
list_del(&data->list); list_del(&data->list);
/* /*
* 'data' can be invalid after this call if * 'data' can be invalid after this call if flags == 0
* flags == 0 (when called through * (when called through generic_exec_single()),
* generic_exec_single(), so save them away before * so save them away before making the call:
* making the call.
*/ */
data_flags = data->flags; data_flags = data->flags;
data->func(data->info); data->func(data->info);
if (data_flags & CSD_FLAG_WAIT) {
smp_wmb();
data->flags &= ~CSD_FLAG_WAIT;
} else if (data_flags & CSD_FLAG_LOCK) {
smp_wmb();
data->flags &= ~CSD_FLAG_LOCK;
} else if (data_flags & CSD_FLAG_ALLOC)
kfree(data);
}
/* /*
* See comment on outer loop * Unlocked CSDs are valid through generic_exec_single():
*/ */
smp_read_barrier_depends(); if (data_flags & CSD_FLAG_LOCK)
csd_unlock(data);
} }
} }
@ -215,65 +272,45 @@ static DEFINE_PER_CPU(struct call_single_data, csd_data);
int smp_call_function_single(int cpu, void (*func) (void *info), void *info, int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
int wait) int wait)
{ {
struct call_single_data d; struct call_single_data d = {
.flags = 0,
};
unsigned long flags; unsigned long flags;
/* prevent preemption and reschedule on another processor, int this_cpu;
as well as CPU removal */
int me = get_cpu();
int err = 0; int err = 0;
/* Can deadlock when called with interrupts disabled */ /*
WARN_ON(irqs_disabled()); * prevent preemption and reschedule on another processor,
* as well as CPU removal
*/
this_cpu = get_cpu();
if (cpu == me) { /* Can deadlock when called with interrupts disabled */
WARN_ON_ONCE(irqs_disabled() && !oops_in_progress);
if (cpu == this_cpu) {
local_irq_save(flags); local_irq_save(flags);
func(info); func(info);
local_irq_restore(flags); local_irq_restore(flags);
} else if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) {
struct call_single_data *data;
if (!wait) {
/*
* We are calling a function on a single CPU
* and we are not going to wait for it to finish.
* We first try to allocate the data, but if we
* fail, we fall back to use a per cpu data to pass
* the information to that CPU. Since all callers
* of this code will use the same data, we must
* synchronize the callers to prevent a new caller
* from corrupting the data before the callee
* can access it.
*
* The CSD_FLAG_LOCK is used to let us know when
* the IPI handler is done with the data.
* The first caller will set it, and the callee
* will clear it. The next caller must wait for
* it to clear before we set it again. This
* will make sure the callee is done with the
* data before a new caller will use it.
*/
data = kmalloc(sizeof(*data), GFP_ATOMIC);
if (data)
data->flags = CSD_FLAG_ALLOC;
else {
data = &per_cpu(csd_data, me);
while (data->flags & CSD_FLAG_LOCK)
cpu_relax();
data->flags = CSD_FLAG_LOCK;
}
} else { } else {
data = &d; if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) {
data->flags = CSD_FLAG_WAIT; struct call_single_data *data = &d;
}
if (!wait)
data = &__get_cpu_var(csd_data);
csd_lock(data);
data->func = func; data->func = func;
data->info = info; data->info = info;
generic_exec_single(cpu, data); generic_exec_single(cpu, data, wait);
} else { } else {
err = -ENXIO; /* CPU not online */ err = -ENXIO; /* CPU not online */
} }
}
put_cpu(); put_cpu();
return err; return err;
} }
EXPORT_SYMBOL(smp_call_function_single); EXPORT_SYMBOL(smp_call_function_single);
@ -283,20 +320,23 @@ EXPORT_SYMBOL(smp_call_function_single);
* @cpu: The CPU to run on. * @cpu: The CPU to run on.
* @data: Pre-allocated and setup data structure * @data: Pre-allocated and setup data structure
* *
* Like smp_call_function_single(), but allow caller to pass in a pre-allocated * Like smp_call_function_single(), but allow caller to pass in a
* data structure. Useful for embedding @data inside other structures, for * pre-allocated data structure. Useful for embedding @data inside
* instance. * other structures, for instance.
*
*/ */
void __smp_call_function_single(int cpu, struct call_single_data *data) void __smp_call_function_single(int cpu, struct call_single_data *data,
int wait)
{ {
/* Can deadlock when called with interrupts disabled */ csd_lock(data);
WARN_ON((data->flags & CSD_FLAG_WAIT) && irqs_disabled());
generic_exec_single(cpu, data); /* Can deadlock when called with interrupts disabled */
WARN_ON_ONCE(wait && irqs_disabled() && !oops_in_progress);
generic_exec_single(cpu, data, wait);
} }
/* FIXME: Shim for archs using old arch_send_call_function_ipi API. */ /* Deprecated: shim for archs using old arch_send_call_function_ipi API. */
#ifndef arch_send_call_function_ipi_mask #ifndef arch_send_call_function_ipi_mask
# define arch_send_call_function_ipi_mask(maskp) \ # define arch_send_call_function_ipi_mask(maskp) \
arch_send_call_function_ipi(*(maskp)) arch_send_call_function_ipi(*(maskp))
@ -307,7 +347,8 @@ void __smp_call_function_single(int cpu, struct call_single_data *data)
* @mask: The set of cpus to run on (only runs on online subset). * @mask: The set of cpus to run on (only runs on online subset).
* @func: The function to run. This must be fast and non-blocking. * @func: The function to run. This must be fast and non-blocking.
* @info: An arbitrary pointer to pass to the function. * @info: An arbitrary pointer to pass to the function.
* @wait: If true, wait (atomically) until function has completed on other CPUs. * @wait: If true, wait (atomically) until function has completed
* on other CPUs.
* *
* If @wait is true, then returns once @func has returned. Note that @wait * If @wait is true, then returns once @func has returned. Note that @wait
* will be implicitly turned on in case of allocation failures, since * will be implicitly turned on in case of allocation failures, since
@ -318,27 +359,27 @@ void __smp_call_function_single(int cpu, struct call_single_data *data)
* must be disabled when calling this function. * must be disabled when calling this function.
*/ */
void smp_call_function_many(const struct cpumask *mask, void smp_call_function_many(const struct cpumask *mask,
void (*func)(void *), void *info, void (*func)(void *), void *info, bool wait)
bool wait)
{ {
struct call_function_data *data; struct call_function_data *data;
unsigned long flags; unsigned long flags;
int cpu, next_cpu; int cpu, next_cpu, this_cpu = smp_processor_id();
/* Can deadlock when called with interrupts disabled */ /* Can deadlock when called with interrupts disabled */
WARN_ON(irqs_disabled()); WARN_ON_ONCE(irqs_disabled() && !oops_in_progress);
/* So, what's a CPU they want? Ignoring this one. */ /* So, what's a CPU they want? Ignoring this one. */
cpu = cpumask_first_and(mask, cpu_online_mask); cpu = cpumask_first_and(mask, cpu_online_mask);
if (cpu == smp_processor_id()) if (cpu == this_cpu)
cpu = cpumask_next_and(cpu, mask, cpu_online_mask); cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
/* No online cpus? We're done. */ /* No online cpus? We're done. */
if (cpu >= nr_cpu_ids) if (cpu >= nr_cpu_ids)
return; return;
/* Do we have another CPU which isn't us? */ /* Do we have another CPU which isn't us? */
next_cpu = cpumask_next_and(cpu, mask, cpu_online_mask); next_cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
if (next_cpu == smp_processor_id()) if (next_cpu == this_cpu)
next_cpu = cpumask_next_and(next_cpu, mask, cpu_online_mask); next_cpu = cpumask_next_and(next_cpu, mask, cpu_online_mask);
/* Fastpath: do that cpu by itself. */ /* Fastpath: do that cpu by itself. */
@ -347,43 +388,40 @@ void smp_call_function_many(const struct cpumask *mask,
return; return;
} }
data = kmalloc(sizeof(*data) + cpumask_size(), GFP_ATOMIC); data = &__get_cpu_var(cfd_data);
if (unlikely(!data)) { csd_lock(&data->csd);
/* Slow path. */
for_each_online_cpu(cpu) {
if (cpu == smp_processor_id())
continue;
if (cpumask_test_cpu(cpu, mask))
smp_call_function_single(cpu, func, info, wait);
}
return;
}
spin_lock_init(&data->lock); spin_lock_irqsave(&data->lock, flags);
data->csd.flags = CSD_FLAG_ALLOC;
if (wait)
data->csd.flags |= CSD_FLAG_WAIT;
data->csd.func = func; data->csd.func = func;
data->csd.info = info; data->csd.info = info;
cpumask_and(to_cpumask(data->cpumask_bits), mask, cpu_online_mask); cpumask_and(data->cpumask, mask, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), to_cpumask(data->cpumask_bits)); cpumask_clear_cpu(this_cpu, data->cpumask);
data->refs = cpumask_weight(to_cpumask(data->cpumask_bits)); data->refs = cpumask_weight(data->cpumask);
spin_lock_irqsave(&call_function_lock, flags); spin_lock(&call_function.lock);
list_add_tail_rcu(&data->csd.list, &call_function_queue); /*
spin_unlock_irqrestore(&call_function_lock, flags); * Place entry at the _HEAD_ of the list, so that any cpu still
* observing the entry in generic_smp_call_function_interrupt()
* will not miss any other list entries:
*/
list_add_rcu(&data->csd.list, &call_function.queue);
spin_unlock(&call_function.lock);
spin_unlock_irqrestore(&data->lock, flags);
/* /*
* Make the list addition visible before sending the ipi. * Make the list addition visible before sending the ipi.
* (IPIs must obey or appear to obey normal Linux cache
* coherency rules -- see comment in generic_exec_single).
*/ */
smp_mb(); smp_mb();
/* Send a message to all CPUs in the map */ /* Send a message to all CPUs in the map */
arch_send_call_function_ipi_mask(to_cpumask(data->cpumask_bits)); arch_send_call_function_ipi_mask(data->cpumask);
/* optionally wait for the CPUs to complete */ /* Optionally wait for the CPUs to complete */
if (wait) if (wait)
csd_flag_wait(&data->csd); csd_lock_wait(&data->csd);
} }
EXPORT_SYMBOL(smp_call_function_many); EXPORT_SYMBOL(smp_call_function_many);
@ -391,7 +429,8 @@ EXPORT_SYMBOL(smp_call_function_many);
* smp_call_function(): Run a function on all other CPUs. * smp_call_function(): Run a function on all other CPUs.
* @func: The function to run. This must be fast and non-blocking. * @func: The function to run. This must be fast and non-blocking.
* @info: An arbitrary pointer to pass to the function. * @info: An arbitrary pointer to pass to the function.
* @wait: If true, wait (atomically) until function has completed on other CPUs. * @wait: If true, wait (atomically) until function has completed
* on other CPUs.
* *
* Returns 0. * Returns 0.
* *
@ -407,26 +446,27 @@ int smp_call_function(void (*func)(void *), void *info, int wait)
preempt_disable(); preempt_disable();
smp_call_function_many(cpu_online_mask, func, info, wait); smp_call_function_many(cpu_online_mask, func, info, wait);
preempt_enable(); preempt_enable();
return 0; return 0;
} }
EXPORT_SYMBOL(smp_call_function); EXPORT_SYMBOL(smp_call_function);
void ipi_call_lock(void) void ipi_call_lock(void)
{ {
spin_lock(&call_function_lock); spin_lock(&call_function.lock);
} }
void ipi_call_unlock(void) void ipi_call_unlock(void)
{ {
spin_unlock(&call_function_lock); spin_unlock(&call_function.lock);
} }
void ipi_call_lock_irq(void) void ipi_call_lock_irq(void)
{ {
spin_lock_irq(&call_function_lock); spin_lock_irq(&call_function.lock);
} }
void ipi_call_unlock_irq(void) void ipi_call_unlock_irq(void)
{ {
spin_unlock_irq(&call_function_lock); spin_unlock_irq(&call_function.lock);
} }

2
kernel/softirq.c
View file

@ -496,7 +496,7 @@ static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softir
cp->flags = 0; cp->flags = 0;
cp->priv = softirq; cp->priv = softirq;
__smp_call_function_single(cpu, cp); __smp_call_function_single(cpu, cp, 0);
return 0; return 0;
} }
return 1; return 1;