system/glibc
3
0
Fork 0
Code Issues Projects Releases Wiki Activity

pthread_once: Add fast path and remove x86 variants.

Browse Source
This commit is contained in:
Torvald Riegel 2014-10-19 21:59:26 +02:00
parent 63668b7084
commit f50277c19d
4 changed files with 30 additions and 378 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
ChangeLog
View File

@ -1,3 +1,11 @@
2014-10-20 Torvald Riegel <triegel@redhat.com>
[BZ #15215]
* nptl/pthread_once.c (__pthread_once): Split out fast path to ...
(__pthread_once_slow): ... here.
* sysdeps/unix/sysv/linux/i386/pthread_once.S: Remove file.
* sysdeps/unix/sysv/linux/x86_64/pthread_once.S: Remove file.
2014-10-20 Torvald Riegel <triegel@redhat.com> 2014-10-20 Torvald Riegel <triegel@redhat.com>
[BZ #15215] [BZ #15215]

29
nptl/pthread_once.c
View File

@ -58,11 +58,13 @@ clear_once_control (void *arg)
initialization is interrupted, we then fork 2^30 times (30 bits of initialization is interrupted, we then fork 2^30 times (30 bits of
once_control are used for the fork generation), and try to initialize once_control are used for the fork generation), and try to initialize
again, we can deadlock because we can't distinguish the in-progress and again, we can deadlock because we can't distinguish the in-progress and
interrupted cases anymore. */ interrupted cases anymore.
int XXX: We split out this slow path because current compilers do not generate
__pthread_once (once_control, init_routine) as efficient code when the fast path in __pthread_once below is not in a
pthread_once_t *once_control; separate function. */
void (*init_routine) (void); static int
__attribute__ ((noinline))
__pthread_once_slow (pthread_once_t *once_control, void (*init_routine) (void))
{ {
while (1) while (1)
{ {
@ -72,7 +74,7 @@ __pthread_once (once_control, init_routine)
signals that initialization has finished, we need to see any signals that initialization has finished, we need to see any
data modifications done during initialization. */ data modifications done during initialization. */
val = *once_control; val = *once_control;
atomic_read_barrier(); atomic_read_barrier ();
do do
{ {
/* Check if the initialization has already been done. */ /* Check if the initialization has already been done. */
@ -120,7 +122,7 @@ __pthread_once (once_control, init_routine)
/* Mark *once_control as having finished the initialization. We need /* Mark *once_control as having finished the initialization. We need
release memory order here because we need to synchronize with other release memory order here because we need to synchronize with other
threads that want to use the initialized data. */ threads that want to use the initialized data. */
atomic_write_barrier(); atomic_write_barrier ();
*once_control = __PTHREAD_ONCE_DONE; *once_control = __PTHREAD_ONCE_DONE;
/* Wake up all other threads. */ /* Wake up all other threads. */
@ -130,5 +132,18 @@ __pthread_once (once_control, init_routine)
return 0; return 0;
} }
int
__pthread_once (pthread_once_t *once_control, void (*init_routine) (void))
{
/* Fast path. See __pthread_once_slow. */
int val;
val = *once_control;
atomic_read_barrier ();
if (__glibc_likely ((val & __PTHREAD_ONCE_DONE) != 0))
return 0;
else
return __pthread_once_slow (once_control, init_routine);
}
weak_alias (__pthread_once, pthread_once) weak_alias (__pthread_once, pthread_once)
hidden_def (__pthread_once) hidden_def (__pthread_once)

178
sysdeps/unix/sysv/linux/i386/pthread_once.S
View File

@ -1,178 +0,0 @@
/* Copyright (C) 2002-2014 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#include <unwindbuf.h>
#include <sysdep.h>
#include <kernel-features.h>
#include <lowlevellock.h>
.comm __fork_generation, 4, 4
.text
.globl __pthread_once
.type __pthread_once,@function
.align 16
cfi_startproc
__pthread_once:
movl 4(%esp), %ecx
testl $2, (%ecx)
jz 1f
xorl %eax, %eax
ret
1: pushl %ebx
cfi_adjust_cfa_offset (4)
cfi_rel_offset (3, 0)
pushl %esi
cfi_adjust_cfa_offset (4)
cfi_rel_offset (6, 0)
movl %ecx, %ebx
xorl %esi, %esi
/* Not yet initialized or initialization in progress.
Get the fork generation counter now. */
6: movl (%ebx), %eax
#ifdef PIC
LOAD_PIC_REG(cx)
#endif
5: movl %eax, %edx
testl $2, %eax
jnz 4f
andl $3, %edx
#ifdef PIC
orl __fork_generation@GOTOFF(%ecx), %edx
#else
orl __fork_generation, %edx
#endif
orl $1, %edx
LOCK
cmpxchgl %edx, (%ebx)
jnz 5b
/* Check whether another thread already runs the initializer. */
testl $1, %eax
jz 3f /* No -> do it. */
/* Check whether the initializer execution was interrupted
by a fork. */
xorl %edx, %eax
testl $0xfffffffc, %eax
jnz 3f /* Different for generation -> run initializer. */
/* Somebody else got here first. Wait. */
#ifdef __ASSUME_PRIVATE_FUTEX
movl $FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %ecx
#else
# if FUTEX_WAIT == 0
movl %gs:PRIVATE_FUTEX, %ecx
# else
movl $FUTEX_WAIT, %ecx
orl %gs:PRIVATE_FUTEX, %ecx
# endif
#endif
movl $SYS_futex, %eax
ENTER_KERNEL
jmp 6b
3: /* Call the initializer function after setting up the
cancellation handler. Note that it is not possible here
to use the unwind-based cleanup handling. This would require
that the user-provided function and all the code it calls
is compiled with exceptions. Unfortunately this cannot be
guaranteed. */
subl $UNWINDBUFSIZE+8, %esp
cfi_adjust_cfa_offset (UNWINDBUFSIZE+8)
movl %ecx, %ebx /* PIC register value. */
leal 8+UWJMPBUF(%esp), %eax
movl $0, 4(%esp)
movl %eax, (%esp)
call __sigsetjmp@PLT
testl %eax, %eax
jne 7f
leal 8(%esp), %eax
call HIDDEN_JUMPTARGET(__pthread_register_cancel)
/* Call the user-provided initialization function. */
call *24+UNWINDBUFSIZE(%esp)
/* Pop the cleanup handler. */
leal 8(%esp), %eax
call HIDDEN_JUMPTARGET(__pthread_unregister_cancel)
addl $UNWINDBUFSIZE+8, %esp
cfi_adjust_cfa_offset (-UNWINDBUFSIZE-8)
/* Sucessful run of the initializer. Signal that we are done. */
movl 12(%esp), %ebx
LOCK
addl $1, (%ebx)
/* Wake up all other threads. */
movl $0x7fffffff, %edx
#ifdef __ASSUME_PRIVATE_FUTEX
movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %ecx
#else
movl $FUTEX_WAKE, %ecx
orl %gs:PRIVATE_FUTEX, %ecx
#endif
movl $SYS_futex, %eax
ENTER_KERNEL
4: popl %esi
cfi_adjust_cfa_offset (-4)
cfi_restore (6)
popl %ebx
cfi_adjust_cfa_offset (-4)
cfi_restore (3)
xorl %eax, %eax
ret
7: /* __sigsetjmp returned for the second time. */
movl 20+UNWINDBUFSIZE(%esp), %ebx
cfi_adjust_cfa_offset (UNWINDBUFSIZE+16)
cfi_offset (3, -8)
cfi_offset (6, -12)
movl $0, (%ebx)
movl $0x7fffffff, %edx
#ifdef __ASSUME_PRIVATE_FUTEX
movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %ecx
#else
movl $FUTEX_WAKE, %ecx
orl %gs:PRIVATE_FUTEX, %ecx
#endif
movl $SYS_futex, %eax
ENTER_KERNEL
leal 8(%esp), %eax
call HIDDEN_JUMPTARGET (__pthread_unwind_next)
/* NOTREACHED */
hlt
cfi_endproc
.size __pthread_once,.-__pthread_once
hidden_def (__pthread_once)
strong_alias (__pthread_once, pthread_once)

193
sysdeps/unix/sysv/linux/x86_64/pthread_once.S
View File

@ -1,193 +0,0 @@
/* Copyright (C) 2002-2014 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#include <sysdep.h>
#include <kernel-features.h>
#include <tcb-offsets.h>
#include <lowlevellock.h>
.comm __fork_generation, 4, 4
.text
.globl __pthread_once
.type __pthread_once,@function
.align 16
__pthread_once:
.LSTARTCODE:
cfi_startproc
#ifdef SHARED
cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect,
DW.ref.__gcc_personality_v0)
cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART)
#else
cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0)
cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
#endif
testl $2, (%rdi)
jz 1f
xorl %eax, %eax
retq
/* Preserve the function pointer. */
1: pushq %rsi
cfi_adjust_cfa_offset(8)
xorq %r10, %r10
/* Not yet initialized or initialization in progress.
Get the fork generation counter now. */
6: movl (%rdi), %eax
5: movl %eax, %edx
testl $2, %eax
jnz 4f
andl $3, %edx
orl __fork_generation(%rip), %edx
orl $1, %edx
LOCK
cmpxchgl %edx, (%rdi)
jnz 5b
/* Check whether another thread already runs the initializer. */
testl $1, %eax
jz 3f /* No -> do it. */
/* Check whether the initializer execution was interrupted
by a fork. */
xorl %edx, %eax
testl $0xfffffffc, %eax
jnz 3f /* Different for generation -> run initializer. */
/* Somebody else got here first. Wait. */
#ifdef __ASSUME_PRIVATE_FUTEX
movl $FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %esi
#else
# if FUTEX_WAIT == 0
movl %fs:PRIVATE_FUTEX, %esi
# else
movl $FUTEX_WAIT, %esi
orl %fs:PRIVATE_FUTEX, %esi
# endif
#endif
movl $SYS_futex, %eax
syscall
jmp 6b
/* Preserve the pointer to the control variable. */
3: pushq %rdi
cfi_adjust_cfa_offset(8)
pushq %rdi
cfi_adjust_cfa_offset(8)
.LcleanupSTART:
callq *16(%rsp)
.LcleanupEND:
/* Get the control variable address back. */
popq %rdi
cfi_adjust_cfa_offset(-8)
/* Sucessful run of the initializer. Signal that we are done. */
LOCK
incl (%rdi)
addq $8, %rsp
cfi_adjust_cfa_offset(-8)
/* Wake up all other threads. */
movl $0x7fffffff, %edx
#ifdef __ASSUME_PRIVATE_FUTEX
movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi
#else
movl $FUTEX_WAKE, %esi
orl %fs:PRIVATE_FUTEX, %esi
#endif
movl $SYS_futex, %eax
syscall
4: addq $8, %rsp
cfi_adjust_cfa_offset(-8)
xorl %eax, %eax
retq
.size __pthread_once,.-__pthread_once
hidden_def (__pthread_once)
strong_alias (__pthread_once, pthread_once)
.type clear_once_control,@function
.align 16
clear_once_control:
cfi_adjust_cfa_offset(3 * 8)
movq (%rsp), %rdi
movq %rax, %r8
movl $0, (%rdi)
movl $0x7fffffff, %edx
#ifdef __ASSUME_PRIVATE_FUTEX
movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi
#else
movl $FUTEX_WAKE, %esi
orl %fs:PRIVATE_FUTEX, %esi
#endif
movl $SYS_futex, %eax
syscall
movq %r8, %rdi
.LcallUR:
call _Unwind_Resume@PLT
hlt
.LENDCODE:
cfi_endproc
.size clear_once_control,.-clear_once_control
.section .gcc_except_table,"a",@progbits
.LexceptSTART:
.byte DW_EH_PE_omit # @LPStart format
.byte DW_EH_PE_omit # @TType format
.byte DW_EH_PE_uleb128 # call-site format
.uleb128 .Lcstend-.Lcstbegin
.Lcstbegin:
.uleb128 .LcleanupSTART-.LSTARTCODE
.uleb128 .LcleanupEND-.LcleanupSTART
.uleb128 clear_once_control-.LSTARTCODE
.uleb128 0
.uleb128 .LcallUR-.LSTARTCODE
.uleb128 .LENDCODE-.LcallUR
.uleb128 0
.uleb128 0
.Lcstend:
#ifdef SHARED
.hidden DW.ref.__gcc_personality_v0
.weak DW.ref.__gcc_personality_v0
.section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits
.align LP_SIZE
.type DW.ref.__gcc_personality_v0, @object
.size DW.ref.__gcc_personality_v0, LP_SIZE
DW.ref.__gcc_personality_v0:
ASM_ADDR __gcc_personality_v0
#endif