system/glibc
3
0
Fork 0
Code Issues Projects Releases Wiki Activity

pthread_once: Add fast path and remove x86 variants.

Browse Source
This commit is contained in:
Torvald Riegel 2014-10-19 21:59:26 +02:00
parent 63668b7084
commit f50277c19d
4 changed files with 30 additions and 378 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
ChangeLog
View File

@ -1,3 +1,11 @@
2014-10-20 Torvald Riegel <triegel@redhat.com>
[BZ #15215]
* nptl/pthread_once.c (__pthread_once): Split out fast path to ...
(__pthread_once_slow): ... here.
* sysdeps/unix/sysv/linux/i386/pthread_once.S: Remove file.
* sysdeps/unix/sysv/linux/x86_64/pthread_once.S: Remove file.
2014-10-20 Torvald Riegel <triegel@redhat.com>
[BZ #15215]

29
nptl/pthread_once.c
View File

@ -58,11 +58,13 @@ clear_once_control (void *arg)
initialization is interrupted, we then fork 2^30 times (30 bits of
once_control are used for the fork generation), and try to initialize
again, we can deadlock because we can't distinguish the in-progress and
interrupted cases anymore. */
int
__pthread_once (once_control, init_routine)
pthread_once_t *once_control;
void (*init_routine) (void);
interrupted cases anymore.
XXX: We split out this slow path because current compilers do not generate
as efficient code when the fast path in __pthread_once below is not in a
separate function. */
static int
__attribute__ ((noinline))
__pthread_once_slow (pthread_once_t *once_control, void (*init_routine) (void))
{
while (1)
{
@ -72,7 +74,7 @@ __pthread_once (once_control, init_routine)
signals that initialization has finished, we need to see any
data modifications done during initialization. */
val = *once_control;
atomic_read_barrier();
atomic_read_barrier ();
do
{
/* Check if the initialization has already been done. */
@ -120,7 +122,7 @@ __pthread_once (once_control, init_routine)
/* Mark *once_control as having finished the initialization. We need
release memory order here because we need to synchronize with other
threads that want to use the initialized data. */
atomic_write_barrier();
atomic_write_barrier ();
*once_control = __PTHREAD_ONCE_DONE;
/* Wake up all other threads. */
@ -130,5 +132,18 @@ __pthread_once (once_control, init_routine)
return 0;
}
int
__pthread_once (pthread_once_t *once_control, void (*init_routine) (void))
{
/* Fast path. See __pthread_once_slow. */
int val;
val = *once_control;
atomic_read_barrier ();
if (__glibc_likely ((val & __PTHREAD_ONCE_DONE) != 0))
return 0;
else
return __pthread_once_slow (once_control, init_routine);
}
weak_alias (__pthread_once, pthread_once)
hidden_def (__pthread_once)

178
sysdeps/unix/sysv/linux/i386/pthread_once.S
View File

@ -1,178 +0,0 @@
/* Copyright (C) 2002-2014 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#include <unwindbuf.h>
#include <sysdep.h>
#include <kernel-features.h>
#include <lowlevellock.h>
.comm __fork_generation, 4, 4
.text
.globl __pthread_once
.type __pthread_once,@function
.align 16
cfi_startproc
__pthread_once:
movl 4(%esp), %ecx
testl $2, (%ecx)
jz 1f
xorl %eax, %eax
ret
1: pushl %ebx
cfi_adjust_cfa_offset (4)
cfi_rel_offset (3, 0)
pushl %esi
cfi_adjust_cfa_offset (4)
cfi_rel_offset (6, 0)
movl %ecx, %ebx
xorl %esi, %esi
/* Not yet initialized or initialization in progress.
Get the fork generation counter now. */
6: movl (%ebx), %eax
#ifdef PIC
LOAD_PIC_REG(cx)
#endif
5: movl %eax, %edx
testl $2, %eax
jnz 4f
andl $3, %edx
#ifdef PIC
orl __fork_generation@GOTOFF(%ecx), %edx
#else
orl __fork_generation, %edx
#endif
orl $1, %edx
LOCK
cmpxchgl %edx, (%ebx)
jnz 5b
/* Check whether another thread already runs the initializer. */
testl $1, %eax
jz 3f /* No -> do it. */
/* Check whether the initializer execution was interrupted
by a fork. */
xorl %edx, %eax
testl $0xfffffffc, %eax
jnz 3f /* Different for generation -> run initializer. */
/* Somebody else got here first. Wait. */
#ifdef __ASSUME_PRIVATE_FUTEX
movl $FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %ecx
#else
# if FUTEX_WAIT == 0
movl %gs:PRIVATE_FUTEX, %ecx
# else
movl $FUTEX_WAIT, %ecx
orl %gs:PRIVATE_FUTEX, %ecx
# endif
#endif
movl $SYS_futex, %eax
ENTER_KERNEL
jmp 6b
3: /* Call the initializer function after setting up the
cancellation handler. Note that it is not possible here
to use the unwind-based cleanup handling. This would require
that the user-provided function and all the code it calls
is compiled with exceptions. Unfortunately this cannot be
guaranteed. */
subl $UNWINDBUFSIZE+8, %esp
cfi_adjust_cfa_offset (UNWINDBUFSIZE+8)
movl %ecx, %ebx /* PIC register value. */
leal 8+UWJMPBUF(%esp), %eax
movl $0, 4(%esp)
movl %eax, (%esp)
call __sigsetjmp@PLT
testl %eax, %eax
jne 7f
leal 8(%esp), %eax
call HIDDEN_JUMPTARGET(__pthread_register_cancel)
/* Call the user-provided initialization function. */
call *24+UNWINDBUFSIZE(%esp)
/* Pop the cleanup handler. */
leal 8(%esp), %eax
call HIDDEN_JUMPTARGET(__pthread_unregister_cancel)
addl $UNWINDBUFSIZE+8, %esp
cfi_adjust_cfa_offset (-UNWINDBUFSIZE-8)
/* Sucessful run of the initializer. Signal that we are done. */
movl 12(%esp), %ebx
LOCK
addl $1, (%ebx)
/* Wake up all other threads. */
movl $0x7fffffff, %edx
#ifdef __ASSUME_PRIVATE_FUTEX
movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %ecx
#else
movl $FUTEX_WAKE, %ecx
orl %gs:PRIVATE_FUTEX, %ecx
#endif
movl $SYS_futex, %eax
ENTER_KERNEL
4: popl %esi
cfi_adjust_cfa_offset (-4)
cfi_restore (6)
popl %ebx
cfi_adjust_cfa_offset (-4)
cfi_restore (3)
xorl %eax, %eax
ret
7: /* __sigsetjmp returned for the second time. */
movl 20+UNWINDBUFSIZE(%esp), %ebx
cfi_adjust_cfa_offset (UNWINDBUFSIZE+16)
cfi_offset (3, -8)
cfi_offset (6, -12)
movl $0, (%ebx)
movl $0x7fffffff, %edx
#ifdef __ASSUME_PRIVATE_FUTEX
movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %ecx
#else
movl $FUTEX_WAKE, %ecx
orl %gs:PRIVATE_FUTEX, %ecx
#endif
movl $SYS_futex, %eax
ENTER_KERNEL
leal 8(%esp), %eax
call HIDDEN_JUMPTARGET (__pthread_unwind_next)
/* NOTREACHED */
hlt
cfi_endproc
.size __pthread_once,.-__pthread_once
hidden_def (__pthread_once)
strong_alias (__pthread_once, pthread_once)

193
sysdeps/unix/sysv/linux/x86_64/pthread_once.S
View File

@ -1,193 +0,0 @@
/* Copyright (C) 2002-2014 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#include <sysdep.h>
#include <kernel-features.h>
#include <tcb-offsets.h>
#include <lowlevellock.h>
.comm __fork_generation, 4, 4
.text
.globl __pthread_once
.type __pthread_once,@function
.align 16
__pthread_once:
.LSTARTCODE:
cfi_startproc
#ifdef SHARED
cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect,
DW.ref.__gcc_personality_v0)
cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART)
#else
cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0)
cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
#endif
testl $2, (%rdi)
jz 1f
xorl %eax, %eax
retq
/* Preserve the function pointer. */
1: pushq %rsi
cfi_adjust_cfa_offset(8)
xorq %r10, %r10
/* Not yet initialized or initialization in progress.
Get the fork generation counter now. */
6: movl (%rdi), %eax
5: movl %eax, %edx
testl $2, %eax
jnz 4f
andl $3, %edx
orl __fork_generation(%rip), %edx
orl $1, %edx
LOCK
cmpxchgl %edx, (%rdi)
jnz 5b
/* Check whether another thread already runs the initializer. */
testl $1, %eax
jz 3f /* No -> do it. */
/* Check whether the initializer execution was interrupted
by a fork. */
xorl %edx, %eax
testl $0xfffffffc, %eax
jnz 3f /* Different for generation -> run initializer. */
/* Somebody else got here first. Wait. */
#ifdef __ASSUME_PRIVATE_FUTEX
movl $FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %esi
#else
# if FUTEX_WAIT == 0
movl %fs:PRIVATE_FUTEX, %esi
# else
movl $FUTEX_WAIT, %esi
orl %fs:PRIVATE_FUTEX, %esi
# endif
#endif
movl $SYS_futex, %eax
syscall
jmp 6b
/* Preserve the pointer to the control variable. */
3: pushq %rdi
cfi_adjust_cfa_offset(8)
pushq %rdi
cfi_adjust_cfa_offset(8)
.LcleanupSTART:
callq *16(%rsp)
.LcleanupEND:
/* Get the control variable address back. */
popq %rdi
cfi_adjust_cfa_offset(-8)
/* Sucessful run of the initializer. Signal that we are done. */
LOCK
incl (%rdi)
addq $8, %rsp
cfi_adjust_cfa_offset(-8)
/* Wake up all other threads. */
movl $0x7fffffff, %edx
#ifdef __ASSUME_PRIVATE_FUTEX
movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi
#else
movl $FUTEX_WAKE, %esi
orl %fs:PRIVATE_FUTEX, %esi
#endif
movl $SYS_futex, %eax
syscall
4: addq $8, %rsp
cfi_adjust_cfa_offset(-8)
xorl %eax, %eax
retq
.size __pthread_once,.-__pthread_once
hidden_def (__pthread_once)
strong_alias (__pthread_once, pthread_once)
.type clear_once_control,@function
.align 16
clear_once_control:
cfi_adjust_cfa_offset(3 * 8)
movq (%rsp), %rdi
movq %rax, %r8
movl $0, (%rdi)
movl $0x7fffffff, %edx
#ifdef __ASSUME_PRIVATE_FUTEX
movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi
#else
movl $FUTEX_WAKE, %esi
orl %fs:PRIVATE_FUTEX, %esi
#endif
movl $SYS_futex, %eax
syscall
movq %r8, %rdi
.LcallUR:
call _Unwind_Resume@PLT
hlt
.LENDCODE:
cfi_endproc
.size clear_once_control,.-clear_once_control
.section .gcc_except_table,"a",@progbits
.LexceptSTART:
.byte DW_EH_PE_omit # @LPStart format
.byte DW_EH_PE_omit # @TType format
.byte DW_EH_PE_uleb128 # call-site format
.uleb128 .Lcstend-.Lcstbegin
.Lcstbegin:
.uleb128 .LcleanupSTART-.LSTARTCODE
.uleb128 .LcleanupEND-.LcleanupSTART
.uleb128 clear_once_control-.LSTARTCODE
.uleb128 0
.uleb128 .LcallUR-.LSTARTCODE
.uleb128 .LENDCODE-.LcallUR
.uleb128 0
.uleb128 0
.Lcstend:
#ifdef SHARED
.hidden DW.ref.__gcc_personality_v0
.weak DW.ref.__gcc_personality_v0
.section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits
.align LP_SIZE
.type DW.ref.__gcc_personality_v0, @object
.size DW.ref.__gcc_personality_v0, LP_SIZE
DW.ref.__gcc_personality_v0:
ASM_ADDR __gcc_personality_v0
#endif