diff --git a/drivers/char/random.c b/drivers/char/random.c index acb8e7c218d1..06c4b06fcea0 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -260,6 +260,7 @@ #include #include #include +#include #include #include @@ -1712,6 +1713,7 @@ int random_int_secret_init(void) static DEFINE_PER_CPU(__u32 [MD5_DIGEST_WORDS], get_random_int_hash) __aligned(sizeof(unsigned long)); +static DEFINE_LOCAL_IRQ_LOCK(hash_entropy_int_lock); /* * Get a random word for internal kernel use only. Similar to urandom but @@ -1727,12 +1729,12 @@ unsigned int get_random_int(void) if (arch_get_random_int(&ret)) return ret; - hash = get_cpu_var(get_random_int_hash); + hash = get_locked_var(hash_entropy_int_lock, get_random_int_hash); hash[0] += current->pid + jiffies + random_get_entropy(); md5_transform(hash, random_int_secret); ret = hash[0]; - put_cpu_var(get_random_int_hash); + put_locked_var(hash_entropy_int_lock, get_random_int_hash); return ret; } @@ -1749,12 +1751,12 @@ unsigned long get_random_long(void) if (arch_get_random_long(&ret)) return ret; - hash = get_cpu_var(get_random_int_hash); + hash = get_locked_var(hash_entropy_int_lock, get_random_int_hash); hash[0] += current->pid + jiffies + random_get_entropy(); md5_transform(hash, random_int_secret); ret = *(unsigned long *)hash; - put_cpu_var(get_random_int_hash); + put_locked_var(hash_entropy_int_lock, get_random_int_hash); return ret; } diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 6a97d9e930c8..c206ed033542 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -433,7 +433,7 @@ static void release_stripe(struct stripe_head *sh) md_wakeup_thread(conf->mddev->thread); return; slow_path: - local_irq_save(flags); + local_irq_save_nort(flags); /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */ if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) { INIT_LIST_HEAD(&list); @@ -442,7 +442,7 @@ slow_path: spin_unlock(&conf->device_lock); release_inactive_stripe_list(conf, &list, hash); } - local_irq_restore(flags); + local_irq_restore_nort(flags); } static inline void remove_hash(struct stripe_head *sh) diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c index b24a2541a9ba..662c027417ff 100644 --- a/fs/ext4/page-io.c +++ b/fs/ext4/page-io.c @@ -87,8 +87,7 @@ static void ext4_finish_bio(struct bio *bio) * We check all buffers in the page under BH_Uptodate_Lock * to avoid races with other end io clearing async_write flags */ - local_irq_save(flags); - bit_spin_lock(BH_Uptodate_Lock, &head->b_state); + flags = bh_uptodate_lock_irqsave(head); do { if (bh_offset(bh) < bio_start || bh_offset(bh) + bh->b_size > bio_end) { @@ -100,8 +99,7 @@ static void ext4_finish_bio(struct bio *bio) if (error) buffer_io_error(bh); } while ((bh = bh->b_this_page) != head); - bit_spin_unlock(BH_Uptodate_Lock, &head->b_state); - local_irq_restore(flags); + bh_uptodate_unlock_irqrestore(head, flags); if (!under_io) end_page_writeback(page); } diff --git a/fs/jbd2/checkpoint.c b/fs/jbd2/checkpoint.c index 78c1545a3fab..8c44654ce274 100644 --- a/fs/jbd2/checkpoint.c +++ b/fs/jbd2/checkpoint.c @@ -116,8 +116,6 @@ void __jbd2_log_wait_for_space(journal_t *journal) nblocks = jbd2_space_needed(journal); while (jbd2_log_space_left(journal) < nblocks) { write_unlock(&journal->j_state_lock); - if (current->plug) - io_schedule(); mutex_lock(&journal->j_checkpoint_mutex); /* diff --git a/include/linux/locallock.h b/include/linux/locallock.h index 015271ff8ec8..930ec941c2c0 100644 --- a/include/linux/locallock.h +++ b/include/linux/locallock.h @@ -77,6 +77,9 @@ static inline int __local_trylock(struct local_irq_lock *lv) lv->owner = current; lv->nestcnt = 1; return 1; + } else if (lv->owner == current) { + lv->nestcnt++; + return 1; } return 0; } @@ -250,6 +253,12 @@ static inline int __local_unlock_irqrestore(struct local_irq_lock *lv, static inline void local_irq_lock_init(int lvar) { } +#define local_trylock(lvar) \ + ({ \ + preempt_disable(); \ + 1; \ + }) + #define local_lock(lvar) preempt_disable() #define local_unlock(lvar) preempt_enable() #define local_lock_irq(lvar) local_irq_disable() diff --git a/kernel/cpu_pm.c b/kernel/cpu_pm.c index 9656a3c36503..9da42f83ee03 100644 --- a/kernel/cpu_pm.c +++ b/kernel/cpu_pm.c @@ -22,14 +22,13 @@ #include #include -static DEFINE_RWLOCK(cpu_pm_notifier_lock); -static RAW_NOTIFIER_HEAD(cpu_pm_notifier_chain); +static ATOMIC_NOTIFIER_HEAD(cpu_pm_notifier_chain); static int cpu_pm_notify(enum cpu_pm_event event, int nr_to_call, int *nr_calls) { int ret; - ret = __raw_notifier_call_chain(&cpu_pm_notifier_chain, event, NULL, + ret = __atomic_notifier_call_chain(&cpu_pm_notifier_chain, event, NULL, nr_to_call, nr_calls); return notifier_to_errno(ret); @@ -47,14 +46,7 @@ static int cpu_pm_notify(enum cpu_pm_event event, int nr_to_call, int *nr_calls) */ int cpu_pm_register_notifier(struct notifier_block *nb) { - unsigned long flags; - int ret; - - write_lock_irqsave(&cpu_pm_notifier_lock, flags); - ret = raw_notifier_chain_register(&cpu_pm_notifier_chain, nb); - write_unlock_irqrestore(&cpu_pm_notifier_lock, flags); - - return ret; + return atomic_notifier_chain_register(&cpu_pm_notifier_chain, nb); } EXPORT_SYMBOL_GPL(cpu_pm_register_notifier); @@ -69,14 +61,7 @@ EXPORT_SYMBOL_GPL(cpu_pm_register_notifier); */ int cpu_pm_unregister_notifier(struct notifier_block *nb) { - unsigned long flags; - int ret; - - write_lock_irqsave(&cpu_pm_notifier_lock, flags); - ret = raw_notifier_chain_unregister(&cpu_pm_notifier_chain, nb); - write_unlock_irqrestore(&cpu_pm_notifier_lock, flags); - - return ret; + return atomic_notifier_chain_unregister(&cpu_pm_notifier_chain, nb); } EXPORT_SYMBOL_GPL(cpu_pm_unregister_notifier); @@ -100,7 +85,6 @@ int cpu_pm_enter(void) int nr_calls; int ret = 0; - read_lock(&cpu_pm_notifier_lock); ret = cpu_pm_notify(CPU_PM_ENTER, -1, &nr_calls); if (ret) /* @@ -108,7 +92,6 @@ int cpu_pm_enter(void) * PM entry who are notified earlier to prepare for it. */ cpu_pm_notify(CPU_PM_ENTER_FAILED, nr_calls - 1, NULL); - read_unlock(&cpu_pm_notifier_lock); return ret; } @@ -128,13 +111,7 @@ EXPORT_SYMBOL_GPL(cpu_pm_enter); */ int cpu_pm_exit(void) { - int ret; - - read_lock(&cpu_pm_notifier_lock); - ret = cpu_pm_notify(CPU_PM_EXIT, -1, NULL); - read_unlock(&cpu_pm_notifier_lock); - - return ret; + return cpu_pm_notify(CPU_PM_EXIT, -1, NULL); } EXPORT_SYMBOL_GPL(cpu_pm_exit); @@ -159,7 +136,6 @@ int cpu_cluster_pm_enter(void) int nr_calls; int ret = 0; - read_lock(&cpu_pm_notifier_lock); ret = cpu_pm_notify(CPU_CLUSTER_PM_ENTER, -1, &nr_calls); if (ret) /* @@ -167,7 +143,6 @@ int cpu_cluster_pm_enter(void) * PM entry who are notified earlier to prepare for it. */ cpu_pm_notify(CPU_CLUSTER_PM_ENTER_FAILED, nr_calls - 1, NULL); - read_unlock(&cpu_pm_notifier_lock); return ret; } @@ -190,13 +165,7 @@ EXPORT_SYMBOL_GPL(cpu_cluster_pm_enter); */ int cpu_cluster_pm_exit(void) { - int ret; - - read_lock(&cpu_pm_notifier_lock); - ret = cpu_pm_notify(CPU_CLUSTER_PM_EXIT, -1, NULL); - read_unlock(&cpu_pm_notifier_lock); - - return ret; + return cpu_pm_notify(CPU_CLUSTER_PM_EXIT, -1, NULL); } EXPORT_SYMBOL_GPL(cpu_cluster_pm_exit); diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c index 8d950b4521fc..d58780b59583 100644 --- a/kernel/locking/rtmutex.c +++ b/kernel/locking/rtmutex.c @@ -22,6 +22,7 @@ #include #include #include +#include #include "rtmutex_common.h" @@ -1532,18 +1533,13 @@ __rt_mutex_slowlock(struct rt_mutex *lock, int state, if (try_to_take_rt_mutex(lock, current, waiter)) break; - /* - * TASK_INTERRUPTIBLE checks for signals and - * timeout. Ignored otherwise. - */ - if (unlikely(state == TASK_INTERRUPTIBLE)) { - /* Signal pending? */ - if (signal_pending(current)) - ret = -EINTR; - if (timeout && !timeout->task) - ret = -ETIMEDOUT; - if (ret) - break; + if (timeout && !timeout->task) { + ret = -ETIMEDOUT; + break; + } + if (signal_pending_state(state, current)) { + ret = -EINTR; + break; } if (ww_ctx && ww_ctx->acquired > 0) { @@ -1848,9 +1844,19 @@ rt_mutex_fastlock(struct rt_mutex *lock, int state, if (likely(rt_mutex_cmpxchg(lock, NULL, current))) { rt_mutex_deadlock_account_lock(lock, current); return 0; - } else - return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK, - ww_ctx); + } + + /* + * If rt_mutex blocks, the function sched_submit_work will not call + * blk_schedule_flush_plug (because tsk_is_pi_blocked would be true). + * We must call blk_schedule_flush_plug here, if we don't call it, + * a deadlock in device mapper may happen. + */ + if (unlikely(blk_needs_flush_plug(current))) + blk_schedule_flush_plug(current); + + return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK, + ww_ctx); } static inline int @@ -1867,8 +1873,12 @@ rt_mutex_timed_fastlock(struct rt_mutex *lock, int state, likely(rt_mutex_cmpxchg(lock, NULL, current))) { rt_mutex_deadlock_account_lock(lock, current); return 0; - } else - return slowfn(lock, state, timeout, chwalk, ww_ctx); + } + + if (unlikely(blk_needs_flush_plug(current))) + blk_schedule_flush_plug(current); + + return slowfn(lock, state, timeout, chwalk, ww_ctx); } static inline int diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 2532e4f2d075..b606f31e5034 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1923,7 +1923,7 @@ EXPORT_SYMBOL(wake_up_process); */ int wake_up_lock_sleeper(struct task_struct *p) { - return try_to_wake_up(p, TASK_ALL, WF_LOCK_SLEEPER); + return try_to_wake_up(p, TASK_UNINTERRUPTIBLE, WF_LOCK_SLEEPER); } int wake_up_state(struct task_struct *p, unsigned int state) diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 2c1597d2bd6d..1e7c0c9b0e77 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1928,7 +1928,7 @@ static void init_hrtimers_cpu(int cpu) #ifdef CONFIG_HOTPLUG_CPU -static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, +static int migrate_hrtimer_list(struct hrtimer_clock_base *old_base, struct hrtimer_clock_base *new_base) { struct hrtimer *timer; @@ -1959,12 +1959,21 @@ static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, /* Clear the migration state bit */ timer->state &= ~HRTIMER_STATE_MIGRATE; } +#ifdef CONFIG_PREEMPT_RT_BASE + list_splice_tail(&old_base->expired, &new_base->expired); + /* + * Tell the caller to raise HRTIMER_SOFTIRQ. We can't safely + * acquire ktimersoftd->pi_lock while the base lock is held. + */ + return !list_empty(&new_base->expired); +#endif + return 0; } static void migrate_hrtimers(int scpu) { struct hrtimer_cpu_base *old_base, *new_base; - int i; + int i, raise = 0; BUG_ON(cpu_online(scpu)); tick_cancel_sched_timer(scpu); @@ -1980,13 +1989,16 @@ static void migrate_hrtimers(int scpu) raw_spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { - migrate_hrtimer_list(&old_base->clock_base[i], - &new_base->clock_base[i]); + raise |= migrate_hrtimer_list(&old_base->clock_base[i], + &new_base->clock_base[i]); } raw_spin_unlock(&old_base->lock); raw_spin_unlock(&new_base->lock); + if (raise) + raise_softirq_irqoff(HRTIMER_SOFTIRQ); + /* Check, if we got expired work to do */ __hrtimer_peek_ahead_timers(); local_irq_enable(); diff --git a/localversion-rt b/localversion-rt index f5eca15b92de..8de253240c16 100644 --- a/localversion-rt +++ b/localversion-rt @@ -1 +1 @@ --rt97 +-rt98 diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c index 76b11e3e2b95..434eef536092 100644 --- a/net/ipv4/icmp.c +++ b/net/ipv4/icmp.c @@ -218,7 +218,11 @@ static inline struct sock *icmp_xmit_lock(struct net *net) local_bh_disable(); - local_lock(icmp_sk_lock); + if (!local_trylock(icmp_sk_lock)) { + local_bh_enable(); + return NULL; + } + sk = icmp_sk(net); if (unlikely(!spin_trylock(&sk->sk_lock.slock))) {