Add Requeue-PI support for x86 arch.
This commit is contained in:
Dinakar Guniguntala
Ulrich Drepper
parent
51ddd2c01e
commit
75956694f3
@ -1,3 +1,14 @@
|
||||
2009-12-01 Dinakar Guniguntala <dino@in.ibm.com>
|
||||
|
||||
* sysdeps/unix/sysv/linux/i386/i486/lowlevellock.h: Define
|
||||
FUTEX_WAIT_REQUEUE_PI and FUTEX_CMP_REQUEUE_PI.
|
||||
* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S: If mutex
|
||||
is a non robust PI mutex, then use FUTEX_CMP_REQUEUE_PI.
|
||||
* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S: Likewise.
|
||||
* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S: If mutex
|
||||
is a non robust PI mutex, then use FUTEX_WAIT_REQUEUE_PI.
|
||||
* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Likewise.
|
||||
|
||||
2009-12-12 Ulrich Drepper <drepper@redhat.com>
|
||||
|
||||
* sysdeps/unix/sysv/linux/i386/i486/sem_timedwait.S (sem_timedwait):
|
||||
|
||||
@ -91,12 +91,17 @@ __pthread_cond_broadcast:
|
||||
8: cmpl $-1, %edi
|
||||
je 9f
|
||||
|
||||
/* XXX: The kernel so far doesn't support requeue to PI futex. */
|
||||
/* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same
|
||||
type of futex (private resp. shared). */
|
||||
testl $(PI_BIT | PS_BIT), MUTEX_KIND(%edi)
|
||||
/* Do not use requeue for pshared condvars. */
|
||||
testl $PS_BIT, MUTEX_KIND(%edi)
|
||||
jne 9f
|
||||
|
||||
/* Requeue to a non-robust PI mutex if the PI bit is set and
|
||||
the robust bit is not set. */
|
||||
movl MUTEX_KIND(%edi), %eax
|
||||
andl $(ROBUST_BIT|PI_BIT), %eax
|
||||
cmpl $PI_BIT, %eax
|
||||
je 81f
|
||||
|
||||
/* Wake up all threads. */
|
||||
#ifdef __ASSUME_PRIVATE_FUTEX
|
||||
movl $(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %ecx
|
||||
@ -138,6 +143,23 @@ __pthread_cond_broadcast:
|
||||
|
||||
cfi_restore_state
|
||||
|
||||
81: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
|
||||
movl $SYS_futex, %eax
|
||||
movl $0x7fffffff, %esi
|
||||
movl $1, %edx
|
||||
/* Get the address of the futex involved. */
|
||||
# if MUTEX_FUTEX != 0
|
||||
addl $MUTEX_FUTEX, %edi
|
||||
# endif
|
||||
int $0x80
|
||||
|
||||
/* For any kind of error, which mainly is EAGAIN, we try again
|
||||
with WAKE. The general test also covers running on old
|
||||
kernels. */
|
||||
cmpl $0xfffff001, %eax
|
||||
jb 6b
|
||||
jmp 9f
|
||||
|
||||
/* Initial locking failed. */
|
||||
1:
|
||||
#if cond_lock == 0
|
||||
|
||||
@ -22,6 +22,7 @@
|
||||
#include <lowlevellock.h>
|
||||
#include <lowlevelcond.h>
|
||||
#include <kernel-features.h>
|
||||
#include <pthread-pi-defines.h>
|
||||
#include <pthread-errnos.h>
|
||||
|
||||
|
||||
@ -86,7 +87,17 @@ __pthread_cond_signal:
|
||||
#endif
|
||||
cmpl $-1, dep_mutex-cond_futex(%ebx)
|
||||
sete %cl
|
||||
subl $1, %ecx
|
||||
je 8f
|
||||
|
||||
movl dep_mutex-cond_futex(%ebx), %edx
|
||||
/* Requeue to a non-robust PI mutex if the PI bit is set and
|
||||
the robust bit is not set. */
|
||||
movl MUTEX_KIND(%edx), %eax
|
||||
andl $(ROBUST_BIT|PI_BIT), %eax
|
||||
cmpl $PI_BIT, %eax
|
||||
je 9f
|
||||
|
||||
8: subl $1, %ecx
|
||||
#ifdef __ASSUME_PRIVATE_FUTEX
|
||||
andl $FUTEX_PRIVATE_FLAG, %ecx
|
||||
#else
|
||||
@ -124,8 +135,34 @@ __pthread_cond_signal:
|
||||
|
||||
cfi_restore_state
|
||||
|
||||
7: /* %ecx should be either FUTEX_WAKE_OP or
|
||||
FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall. */
|
||||
9: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
|
||||
movl $SYS_futex, %eax
|
||||
movl $1, %edx
|
||||
xorl %esi, %esi
|
||||
movl dep_mutex-cond_futex(%ebx), %edi
|
||||
movl (%ebx), %ebp
|
||||
/* FIXME: Until Ingo fixes 4G/4G vDSO, 6 arg syscalls are broken for
|
||||
sysenter.
|
||||
ENTER_KERNEL */
|
||||
int $0x80
|
||||
popl %ebp
|
||||
popl %esi
|
||||
|
||||
leal -cond_futex(%ebx), %edi
|
||||
|
||||
/* For any kind of error, we try again with WAKE.
|
||||
The general test also covers running on old kernels. */
|
||||
cmpl $-4095, %eax
|
||||
jb 4f
|
||||
|
||||
7:
|
||||
#ifdef __ASSUME_PRIVATE_FUTEX
|
||||
andl $FUTEX_PRIVATE_FLAG, %ecx
|
||||
#else
|
||||
andl %gs:PRIVATE_FUTEX, %ecx
|
||||
#endif
|
||||
orl $FUTEX_WAKE, %ecx
|
||||
|
||||
xorl $(FUTEX_WAKE ^ FUTEX_WAKE_OP), %ecx
|
||||
movl $SYS_futex, %eax
|
||||
/* %edx should be 1 already from $FUTEX_WAKE_OP syscall.
|
||||
|
||||
@ -22,6 +22,7 @@
|
||||
#include <lowlevellock.h>
|
||||
#include <lowlevelcond.h>
|
||||
#include <pthread-errnos.h>
|
||||
#include <pthread-pi-defines.h>
|
||||
#include <kernel-features.h>
|
||||
|
||||
|
||||
@ -95,7 +96,7 @@ __pthread_cond_timedwait:
|
||||
addl $1, cond_futex(%ebx)
|
||||
addl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
|
||||
|
||||
#define FRAME_SIZE 24
|
||||
#define FRAME_SIZE 32
|
||||
subl $FRAME_SIZE, %esp
|
||||
cfi_adjust_cfa_offset(FRAME_SIZE)
|
||||
|
||||
@ -107,8 +108,10 @@ __pthread_cond_timedwait:
|
||||
movl %edx, 16(%esp)
|
||||
movl %eax, 20(%esp)
|
||||
|
||||
/* Reset the pi-requeued flag. */
|
||||
8: movl $0, 24(%esp)
|
||||
/* Get the current time. */
|
||||
8: movl %ebx, %edx
|
||||
movl %ebx, %edx
|
||||
#ifdef __NR_clock_gettime
|
||||
/* Get the clock number. */
|
||||
movl cond_nwaiters(%ebx), %ebx
|
||||
@ -158,6 +161,7 @@ __pthread_cond_timedwait:
|
||||
movl %edx, 8(%esp)
|
||||
|
||||
movl cond_futex(%ebx), %edi
|
||||
movl %edi, 28(%esp)
|
||||
|
||||
/* Unlock. */
|
||||
LOCK
|
||||
@ -172,13 +176,50 @@ __pthread_cond_timedwait:
|
||||
4: call __pthread_enable_asynccancel
|
||||
movl %eax, (%esp)
|
||||
|
||||
leal 4(%esp), %esi
|
||||
#if FUTEX_PRIVATE_FLAG > 255
|
||||
xorl %ecx, %ecx
|
||||
#endif
|
||||
cmpl $-1, dep_mutex(%ebx)
|
||||
sete %cl
|
||||
subl $1, %ecx
|
||||
je 40f
|
||||
|
||||
movl dep_mutex(%ebx), %edi
|
||||
/* Requeue to a non-robust PI mutex if the PI bit is set and
|
||||
the robust bit is not set. */
|
||||
movl MUTEX_KIND(%edi), %eax
|
||||
andl $(ROBUST_BIT|PI_BIT), %eax
|
||||
cmpl $PI_BIT, %eax
|
||||
jne 40f
|
||||
|
||||
movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
|
||||
/* The following only works like this because we only support
|
||||
two clocks, represented using a single bit. */
|
||||
testl $1, cond_nwaiters(%ebx)
|
||||
/* XXX Need to implement using sete instead of a jump. */
|
||||
jne 42f
|
||||
orl $FUTEX_CLOCK_REALTIME, %ecx
|
||||
|
||||
/* Requeue-PI uses absolute timeout */
|
||||
42: leal (%ebp), %esi
|
||||
movl 28(%esp), %edx
|
||||
addl $cond_futex, %ebx
|
||||
movl $SYS_futex, %eax
|
||||
ENTER_KERNEL
|
||||
subl $cond_futex, %ebx
|
||||
movl %eax, %esi
|
||||
/* Set the pi-requeued flag only if the kernel has returned 0. The
|
||||
kernel does not hold the mutex on ETIMEDOUT or any other error. */
|
||||
cmpl $0, %eax
|
||||
sete 24(%esp)
|
||||
je 41f
|
||||
|
||||
/* Normal and PI futexes dont mix. Use normal futex functions only
|
||||
if the kernel does not support the PI futex functions. */
|
||||
cmpl $-ENOSYS, %eax
|
||||
jne 41f
|
||||
xorl %ecx, %ecx
|
||||
|
||||
40: subl $1, %ecx
|
||||
#ifdef __ASSUME_PRIVATE_FUTEX
|
||||
andl $FUTEX_PRIVATE_FLAG, %ecx
|
||||
#else
|
||||
@ -187,7 +228,8 @@ __pthread_cond_timedwait:
|
||||
#if FUTEX_WAIT != 0
|
||||
addl $FUTEX_WAIT, %ecx
|
||||
#endif
|
||||
movl %edi, %edx
|
||||
leal 4(%esp), %esi
|
||||
movl 28(%esp), %edx
|
||||
addl $cond_futex, %ebx
|
||||
.Ladd_cond_futex:
|
||||
movl $SYS_futex, %eax
|
||||
@ -196,7 +238,7 @@ __pthread_cond_timedwait:
|
||||
.Lsub_cond_futex:
|
||||
movl %eax, %esi
|
||||
|
||||
movl (%esp), %eax
|
||||
41: movl (%esp), %eax
|
||||
call __pthread_disable_asynccancel
|
||||
.LcleanupEND:
|
||||
|
||||
@ -284,10 +326,16 @@ __pthread_cond_timedwait:
|
||||
#endif
|
||||
jne 10f
|
||||
|
||||
11: xorl %eax, %eax
|
||||
/* With requeue_pi, the mutex lock is held in the kernel. */
|
||||
movl 24(%esp), %ecx
|
||||
testl %ecx, %ecx
|
||||
jnz 26f
|
||||
|
||||
/* Remove cancellation handler. */
|
||||
11: movl 24+FRAME_SIZE(%esp), %eax
|
||||
movl 24+FRAME_SIZE(%esp), %eax
|
||||
call __pthread_mutex_cond_lock
|
||||
addl $FRAME_SIZE, %esp
|
||||
26: addl $FRAME_SIZE, %esp
|
||||
cfi_adjust_cfa_offset(-FRAME_SIZE);
|
||||
|
||||
/* We return the result of the mutex_lock operation if it failed. */
|
||||
@ -317,6 +365,9 @@ __pthread_cond_timedwait:
|
||||
|
||||
cfi_restore_state
|
||||
|
||||
27: call __pthread_mutex_cond_lock_adjust
|
||||
jmp 26b
|
||||
|
||||
/* Initial locking failed. */
|
||||
1:
|
||||
#if cond_lock == 0
|
||||
|
||||
@ -22,6 +22,8 @@
|
||||
#include <lowlevellock.h>
|
||||
#include <lowlevelcond.h>
|
||||
#include <tcb-offsets.h>
|
||||
#include <pthread-errnos.h>
|
||||
#include <pthread-pi-defines.h>
|
||||
#include <kernel-features.h>
|
||||
|
||||
|
||||
@ -43,6 +45,9 @@ __pthread_cond_wait:
|
||||
cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
|
||||
#endif
|
||||
|
||||
pushl %ebp
|
||||
cfi_adjust_cfa_offset(4)
|
||||
cfi_rel_offset(%ebp, 0)
|
||||
pushl %edi
|
||||
cfi_adjust_cfa_offset(4)
|
||||
cfi_rel_offset(%edi, 0)
|
||||
@ -55,7 +60,7 @@ __pthread_cond_wait:
|
||||
cfi_remember_state
|
||||
|
||||
xorl %esi, %esi
|
||||
movl 16(%esp), %ebx
|
||||
movl 20(%esp), %ebx
|
||||
|
||||
/* Get internal lock. */
|
||||
movl $1, %edx
|
||||
@ -71,7 +76,7 @@ __pthread_cond_wait:
|
||||
/* Store the reference to the mutex. If there is already a
|
||||
different value in there this is a bad user bug. */
|
||||
2: cmpl $-1, dep_mutex(%ebx)
|
||||
movl 20(%esp), %eax
|
||||
movl 24(%esp), %eax
|
||||
je 15f
|
||||
movl %eax, dep_mutex(%ebx)
|
||||
|
||||
@ -87,7 +92,7 @@ __pthread_cond_wait:
|
||||
addl $1, cond_futex(%ebx)
|
||||
addl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
|
||||
|
||||
#define FRAME_SIZE 16
|
||||
#define FRAME_SIZE 20
|
||||
subl $FRAME_SIZE, %esp
|
||||
cfi_adjust_cfa_offset(FRAME_SIZE)
|
||||
|
||||
@ -99,8 +104,10 @@ __pthread_cond_wait:
|
||||
movl %edx, 8(%esp)
|
||||
movl %eax, 12(%esp)
|
||||
|
||||
8: movl cond_futex(%ebx), %edi
|
||||
|
||||
/* Reset the pi-requeued flag. */
|
||||
8: movl $0, 16(%esp)
|
||||
movl cond_futex(%ebx), %ebp
|
||||
|
||||
/* Unlock. */
|
||||
LOCK
|
||||
#if cond_lock == 0
|
||||
@ -114,12 +121,39 @@ __pthread_cond_wait:
|
||||
4: call __pthread_enable_asynccancel
|
||||
movl %eax, (%esp)
|
||||
|
||||
#if FUTEX_PRIVATE_FLAG > 255
|
||||
xorl %ecx, %ecx
|
||||
#endif
|
||||
cmpl $-1, dep_mutex(%ebx)
|
||||
sete %cl
|
||||
subl $1, %ecx
|
||||
je 18f
|
||||
|
||||
movl dep_mutex(%ebx), %edi
|
||||
/* Requeue to a non-robust PI mutex if the PI bit is set and
|
||||
the robust bit is not set. */
|
||||
movl MUTEX_KIND(%edi), %eax
|
||||
andl $(ROBUST_BIT|PI_BIT), %eax
|
||||
cmpl $PI_BIT, %eax
|
||||
jne 18f
|
||||
|
||||
movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
|
||||
movl %ebp, %edx
|
||||
xorl %esi, %esi
|
||||
addl $cond_futex, %ebx
|
||||
movl $SYS_futex, %eax
|
||||
ENTER_KERNEL
|
||||
subl $cond_futex, %ebx
|
||||
/* Set the pi-requeued flag only if the kernel has returned 0. The
|
||||
kernel does not hold the mutex on error. */
|
||||
cmpl $0, %eax
|
||||
sete 16(%esp)
|
||||
je 19f
|
||||
|
||||
/* Normal and PI futexes dont mix. Use normal futex functions only
|
||||
if the kernel does not support the PI futex functions. */
|
||||
cmpl $-ENOSYS, %eax
|
||||
jne 19f
|
||||
xorl %ecx, %ecx
|
||||
|
||||
18: subl $1, %ecx
|
||||
#ifdef __ASSUME_PRIVATE_FUTEX
|
||||
andl $FUTEX_PRIVATE_FLAG, %ecx
|
||||
#else
|
||||
@ -128,7 +162,7 @@ __pthread_cond_wait:
|
||||
#if FUTEX_WAIT != 0
|
||||
addl $FUTEX_WAIT, %ecx
|
||||
#endif
|
||||
movl %edi, %edx
|
||||
movl %ebp, %edx
|
||||
addl $cond_futex, %ebx
|
||||
.Ladd_cond_futex:
|
||||
movl $SYS_futex, %eax
|
||||
@ -136,7 +170,7 @@ __pthread_cond_wait:
|
||||
subl $cond_futex, %ebx
|
||||
.Lsub_cond_futex:
|
||||
|
||||
movl (%esp), %eax
|
||||
19: movl (%esp), %eax
|
||||
call __pthread_disable_asynccancel
|
||||
.LcleanupEND:
|
||||
|
||||
@ -212,9 +246,15 @@ __pthread_cond_wait:
|
||||
#endif
|
||||
jne 10f
|
||||
|
||||
11: movl 20+FRAME_SIZE(%esp), %eax
|
||||
/* With requeue_pi, the mutex lock is held in the kernel. */
|
||||
11: xorl %eax, %eax
|
||||
movl 16(%esp), %ecx
|
||||
testl %ecx, %ecx
|
||||
jnz 20f
|
||||
|
||||
movl 24+FRAME_SIZE(%esp), %eax
|
||||
call __pthread_mutex_cond_lock
|
||||
addl $FRAME_SIZE, %esp
|
||||
20: addl $FRAME_SIZE, %esp
|
||||
cfi_adjust_cfa_offset(-FRAME_SIZE);
|
||||
|
||||
14: popl %ebx
|
||||
@ -226,12 +266,19 @@ __pthread_cond_wait:
|
||||
popl %edi
|
||||
cfi_adjust_cfa_offset(-4)
|
||||
cfi_restore(%edi)
|
||||
popl %ebp
|
||||
cfi_adjust_cfa_offset(-4)
|
||||
cfi_restore(%ebp)
|
||||
|
||||
/* We return the result of the mutex_lock operation. */
|
||||
ret
|
||||
|
||||
cfi_restore_state
|
||||
|
||||
21: call __pthread_mutex_cond_lock_adjust
|
||||
xorl %eax, %eax
|
||||
jmp 20b
|
||||
|
||||
/* Initial locking failed. */
|
||||
1:
|
||||
#if cond_lock == 0
|
||||
@ -484,7 +531,7 @@ __condvar_w_cleanup:
|
||||
movl $0x7fffffff, %edx
|
||||
ENTER_KERNEL
|
||||
|
||||
5: movl 20+FRAME_SIZE(%esp), %eax
|
||||
5: movl 24+FRAME_SIZE(%esp), %eax
|
||||
call __pthread_mutex_cond_lock
|
||||
|
||||
movl %esi, (%esp)
|
||||
|
||||
@ -54,6 +54,8 @@
|
||||
#define FUTEX_TRYLOCK_PI 8
|
||||
#define FUTEX_WAIT_BITSET 9
|
||||
#define FUTEX_WAKE_BITSET 10
|
||||
#define FUTEX_WAIT_REQUEUE_PI 11
|
||||
#define FUTEX_CMP_REQUEUE_PI 12
|
||||
#define FUTEX_PRIVATE_FLAG 128
|
||||
#define FUTEX_CLOCK_REALTIME 256
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user