diff --git a/include/linux/spinlock_rt.h b/include/linux/spinlock_rt.h index 06183876ee968f..b3c504bb98520c 100644 --- a/include/linux/spinlock_rt.h +++ b/include/linux/spinlock_rt.h @@ -22,6 +22,7 @@ extern void __lockfunc rt_spin_lock(spinlock_t *lock); extern unsigned long __lockfunc rt_spin_lock_trace_flags(spinlock_t *lock); extern void __lockfunc rt_spin_lock_nested(spinlock_t *lock, int subclass); extern void __lockfunc rt_spin_unlock(spinlock_t *lock); +extern void __lockfunc rt_spin_unlock_after_trylock_in_irq(spinlock_t *lock); extern void __lockfunc rt_spin_unlock_wait(spinlock_t *lock); extern int __lockfunc rt_spin_trylock_irqsave(spinlock_t *lock, unsigned long *flags); extern int __lockfunc rt_spin_trylock_bh(spinlock_t *lock); diff --git a/kernel/rtmutex.c b/kernel/rtmutex.c index 5d766346267494..0a64a9016960f1 100644 --- a/kernel/rtmutex.c +++ b/kernel/rtmutex.c @@ -802,10 +802,8 @@ static void noinline __sched rt_spin_lock_slowlock(struct rt_mutex *lock) /* * Slow path to release a rt_mutex spin_lock style */ -static void noinline __sched rt_spin_lock_slowunlock(struct rt_mutex *lock) +static void __sched __rt_spin_lock_slowunlock(struct rt_mutex *lock) { - raw_spin_lock(&lock->wait_lock); - debug_rt_mutex_unlock(lock); rt_mutex_deadlock_account_unlock(current); @@ -824,6 +822,23 @@ static void noinline __sched rt_spin_lock_slowunlock(struct rt_mutex *lock) rt_mutex_adjust_prio(current); } +static void noinline __sched rt_spin_lock_slowunlock(struct rt_mutex *lock) +{ + raw_spin_lock(&lock->wait_lock); + __rt_spin_lock_slowunlock(lock); +} + +static void noinline __sched rt_spin_lock_slowunlock_hirq(struct rt_mutex *lock) +{ + int ret; + + do { + ret = raw_spin_trylock(&lock->wait_lock); + } while (!ret); + + __rt_spin_lock_slowunlock(lock); +} + void __lockfunc rt_spin_lock(spinlock_t *lock) { rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock); @@ -854,6 +869,13 @@ void __lockfunc rt_spin_unlock(spinlock_t *lock) } EXPORT_SYMBOL(rt_spin_unlock); +void __lockfunc rt_spin_unlock_after_trylock_in_irq(spinlock_t *lock) +{ + /* NOTE: we always pass in '1' for nested, for simplicity */ + spin_release(&lock->dep_map, 1, _RET_IP_); + rt_spin_lock_fastunlock(&lock->lock, rt_spin_lock_slowunlock_hirq); +} + void __lockfunc __rt_spin_unlock(struct rt_mutex *lock) { rt_spin_lock_fastunlock(lock, rt_spin_lock_slowunlock); @@ -1057,7 +1079,8 @@ rt_mutex_slowtrylock(struct rt_mutex *lock) { int ret = 0; - raw_spin_lock(&lock->wait_lock); + if (!raw_spin_trylock(&lock->wait_lock)) + return ret; init_lists(lock); if (likely(rt_mutex_owner(lock) != current)) { diff --git a/kernel/timer.c b/kernel/timer.c index f53f5924670e60..48652cc03ebf91 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -1400,7 +1400,7 @@ unsigned long get_next_timer_interrupt(unsigned long now) expires = base->next_timer; } #ifdef CONFIG_PREEMPT_RT_FULL - rt_spin_unlock(&base->lock); + rt_spin_unlock_after_trylock_in_irq(&base->lock); #else spin_unlock(&base->lock); #endif