[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[commits] r8685 - in /fsf/trunk/libc/nptl: ./ sysdeps/unix/sysv/linux/x86_64/
- To: commits@xxxxxxxxxx
- Subject: [commits] r8685 - in /fsf/trunk/libc/nptl: ./ sysdeps/unix/sysv/linux/x86_64/
- From: eglibc@xxxxxxxxxx
- Date: Mon, 20 Jul 2009 07:07:37 -0000
Author: eglibc
Date: Mon Jul 20 00:07:37 2009
New Revision: 8685
Log:
Import glibc-mainline for 2009-07-20
Modified:
fsf/trunk/libc/nptl/ChangeLog
fsf/trunk/libc/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
fsf/trunk/libc/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
fsf/trunk/libc/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
fsf/trunk/libc/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
fsf/trunk/libc/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
Modified: fsf/trunk/libc/nptl/ChangeLog
==============================================================================
--- fsf/trunk/libc/nptl/ChangeLog (original)
+++ fsf/trunk/libc/nptl/ChangeLog Mon Jul 20 00:07:37 2009
@@ -1,3 +1,17 @@
+2009-07-19 Ulrich Drepper <drepper@xxxxxxxxxx>
+
+ * sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Define
+ FUTEX_WAIT_REQUEUE_PI and FUTEX_CMP_REQUEUE_PI.
+ * sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S: If mutex
+ is a PI mutex, then use FUTEX_CMP_REQUEUE_PI.
+ * sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S: Likewise.
+ * sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S: If mutex
+ is a PI mutex, then use FUTEX_WAIT_REQUEUE_PI.
+ * sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S: Likewise.
+
+ * sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
+ (__pthread_cond_timedwait): Make more robust.
+
2009-07-18 Ulrich Drepper <drepper@xxxxxxxxxx>
* sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S
Modified: fsf/trunk/libc/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
==============================================================================
--- fsf/trunk/libc/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h (original)
+++ fsf/trunk/libc/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h Mon Jul 20 00:07:37 2009
@@ -54,6 +54,8 @@
#define FUTEX_TRYLOCK_PI 8
#define FUTEX_WAIT_BITSET 9
#define FUTEX_WAKE_BITSET 10
+#define FUTEX_WAIT_REQUEUE_PI 11
+#define FUTEX_CMP_REQUEUE_PI 12
#define FUTEX_PRIVATE_FLAG 128
#define FUTEX_CLOCK_REALTIME 256
Modified: fsf/trunk/libc/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
==============================================================================
--- fsf/trunk/libc/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S (original)
+++ fsf/trunk/libc/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S Mon Jul 20 00:07:37 2009
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007
+/* Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2009
Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@xxxxxxxxxx>, 2002.
@@ -70,11 +70,13 @@
8: cmpq $-1, %r8
je 9f
- /* XXX: The kernel so far doesn't support requeue to PI futex. */
- /* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same
- type of futex (private resp. shared). */
- testl $(PI_BIT | PS_BIT), MUTEX_KIND(%r8)
+ /* Do not use requeue for pshared condvars. */
+ testl $PS_BIT, MUTEX_KIND(%r8)
jne 9f
+
+ /* Requeue to a PI mutex if the PI bit is set. */
+ testl $PI_BIT, MUTEX_KIND(%r8)
+ jne 81f
/* Wake up all threads. */
#ifdef __ASSUME_PRIVATE_FUTEX
@@ -96,6 +98,20 @@
10: xorl %eax, %eax
retq
+
+ /* Wake up all threads. */
+81: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi
+ movl $SYS_futex, %eax
+ movl $1, %edx
+ movl $0x7fffffff, %r10d
+ syscall
+
+ /* For any kind of error, which mainly is EAGAIN, we try again
+ with WAKE. The general test also covers running on old
+ kernels. */
+ cmpq $-4095, %rax
+ jb 10b
+ jmp 9f
.align 16
/* Unlock. */
Modified: fsf/trunk/libc/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
==============================================================================
--- fsf/trunk/libc/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S (original)
+++ fsf/trunk/libc/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S Mon Jul 20 00:07:37 2009
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2005, 2007 Free Software Foundation, Inc.
+/* Copyright (C) 2002-2005, 2007, 2009 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@xxxxxxxxxx>, 2002.
@@ -21,6 +21,7 @@
#include <shlib-compat.h>
#include <lowlevellock.h>
#include <lowlevelcond.h>
+#include <pthread-pi-defines.h>
#include <kernel-features.h>
#include <pthread-errnos.h>
@@ -56,19 +57,23 @@
/* Wake up one thread. */
cmpq $-1, dep_mutex(%r8)
+ movl $FUTEX_WAKE_OP, %esi
movl $1, %edx
+ movl $SYS_futex, %eax
+ je 8f
+
+ /* Get the address of the mutex used. */
+ movq dep_mutex(%r8), %rcx
+ testl $PI_BIT, MUTEX_KIND(%rcx)
+ jne 9f
+
#ifdef __ASSUME_PRIVATE_FUTEX
- movl $FUTEX_WAKE_OP, %eax
movl $(FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG), %esi
- cmove %eax, %esi
#else
- movl $0, %eax
- movl %fs:PRIVATE_FUTEX, %esi
- cmove %eax, %esi
- orl $FUTEX_WAKE_OP, %esi
+ orl %fs:PRIVATE_FUTEX, %esi
#endif
- movl $1, %r10d
- movl $SYS_futex, %eax
+
+8: movl $1, %r10d
#if cond_lock != 0
addq $cond_lock, %r8
#endif
@@ -85,9 +90,27 @@
xorl %eax, %eax
retq
-7: /* %esi should be either FUTEX_WAKE_OP or
- FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall. */
- xorl $(FUTEX_WAKE ^ FUTEX_WAKE_OP), %esi
+ /* Wake up one thread and requeue none in the PI Mutex case. */
+9: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi
+ movq %rcx, %r8
+ xorq %r10, %r10
+ movl (%rdi), %r9d // XXX Can this be right?
+ syscall
+
+ leaq -cond_futex(%rdi), %r8
+
+ /* For any kind of error, we try again with WAKE.
+ The general test also covers running on old kernels. */
+ cmpq $-4095, %rax
+ jb 4f
+
+7:
+#ifdef __ASSUME_PRIVATE_FUTEX
+ andl $FUTEX_PRIVATE_FLAG, %esi
+#else
+ andl %fs:PRIVATE_FUTEX, %esi
+#endif
+ orl $FUTEX_WAKE, %esi
movl $SYS_futex, %eax
/* %rdx should be 1 already from $FUTEX_WAKE_OP syscall.
movl $1, %edx */
Modified: fsf/trunk/libc/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
==============================================================================
--- fsf/trunk/libc/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S (original)
+++ fsf/trunk/libc/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S Mon Jul 20 00:07:37 2009
@@ -21,6 +21,7 @@
#include <shlib-compat.h>
#include <lowlevellock.h>
#include <lowlevelcond.h>
+#include <pthread-pi-defines.h>
#include <pthread-errnos.h>
#include <kernel-features.h>
@@ -58,6 +59,9 @@
pushq %r14
cfi_adjust_cfa_offset(8)
cfi_rel_offset(%r14, 0)
+ pushq %r15
+ cfi_adjust_cfa_offset(8)
+ cfi_rel_offset(%r15, 0)
#ifdef __ASSUME_FUTEX_CLOCK_REALTIME
# define FRAME_SIZE 32
#else
@@ -153,15 +157,48 @@
.LcleanupSTART1:
34: callq __pthread_enable_asynccancel
movl %eax, (%rsp)
+ movq 8(%rsp), %rdi
movq %r13, %r10
cmpq $-1, dep_mutex(%rdi)
movl $FUTEX_WAIT_BITSET, %eax
movl $(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG), %esi
cmove %eax, %esi
+ je 60f
+
+ movq dep_mutex(%rdi), %r8
+ /* Requeue to a PI mutex if the PI bit is set. */
+ testl $PI_BIT, MUTEX_KIND(%r8)
+ je 60f
+
+ movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi
+ xorl %eax, %eax
/* The following only works like this because we only support
two clocks, represented using a single bit. */
+ testl $1, cond_nwaiters(%rdi)
+ movl $FUTEX_CLOCK_REALTIME, %edx
+ cmove %edx, %eax
+ orl %eax, %esi
+ movq %r12, %rdx
+ addq $cond_futex, %rdi
+ movl $SYS_futex, %eax
+ syscall
+
+ movl $1, %r15d
+#ifdef __ASSUME_REQUEUE_PI
+ jmp 62f
+#else
+ cmpq $-4095, %rax
+ jnae 62f
+
+ movl $(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG), %esi
+ subq $cond_futex, %rdi
+#endif
+
+60: xorl %r15d, %r15d
xorl %eax, %eax
+ /* The following only works like this because we only support
+ two clocks, represented using a single bit. */
testl $1, cond_nwaiters(%rdi)
movl $FUTEX_CLOCK_REALTIME, %edx
movl $0xffffffff, %r9d
@@ -171,7 +208,7 @@
addq $cond_futex, %rdi
movl $SYS_futex, %eax
syscall
- movq %rax, %r14
+62: movq %rax, %r14
movl (%rsp), %edi
callq __pthread_disable_asynccancel
@@ -252,14 +289,23 @@
#endif
jne 40f
-41: movq 16(%rsp), %rdi
+ /* If requeue_pi is used the kernel performs the locking of the
+ mutex. */
+41: xorl %eax, %eax
+ testl %r15d, %r15d
+ jnz 63f
+
+ movq 16(%rsp), %rdi
callq __pthread_mutex_cond_lock
- testq %rax, %rax
+63: testq %rax, %rax
cmoveq %r14, %rax
48: addq $FRAME_SIZE, %rsp
cfi_adjust_cfa_offset(-FRAME_SIZE)
+ popq %r15
+ cfi_adjust_cfa_offset(-8)
+ cfi_restore(%r15)
popq %r14
cfi_adjust_cfa_offset(-8)
cfi_restore(%r14)
@@ -273,10 +319,11 @@
retq
/* Initial locking failed. */
-31: cfi_adjust_cfa_offset(3 * 8 + FRAME_SIZE)
- cfi_rel_offset(%r12, FRAME_SIZE + 16)
- cfi_rel_offset(%r13, FRAME_SIZE + 8)
- cfi_rel_offset(%r14, FRAME_SIZE)
+31: cfi_adjust_cfa_offset(4 * 8 + FRAME_SIZE)
+ cfi_rel_offset(%r12, FRAME_SIZE + 24)
+ cfi_rel_offset(%r13, FRAME_SIZE + 16)
+ cfi_rel_offset(%r14, FRAME_SIZE + 8)
+ cfi_rel_offset(%r15, FRAME_SIZE)
#if cond_lock != 0
addq $cond_lock, %rdi
#endif
@@ -352,6 +399,8 @@
#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
.Lreltmo:
+ xorl %r15d, %r15d
+
/* Get internal lock. */
movl $1, %esi
xorl %eax, %eax
@@ -456,6 +505,7 @@
.LcleanupSTART2:
4: callq __pthread_enable_asynccancel
movl %eax, (%rsp)
+ movq 8(%rsp), %rdi
leaq 32(%rsp), %r10
cmpq $-1, dep_mutex(%rdi)
@@ -714,9 +764,10 @@
callq __pthread_mutex_cond_lock
movq 24(%rsp), %rdi
- movq FRAME_SIZE(%rsp), %r14
- movq FRAME_SIZE+8(%rsp), %r13
- movq FRAME_SIZE+16(%rsp), %r12
+ movq FRAME_SIZE(%rsp), %r15
+ movq FRAME_SIZE+8(%rsp), %r14
+ movq FRAME_SIZE+16(%rsp), %r13
+ movq FRAME_SIZE+24(%rsp), %r12
.LcallUR:
call _Unwind_Resume@PLT
hlt
Modified: fsf/trunk/libc/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
==============================================================================
--- fsf/trunk/libc/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S (original)
+++ fsf/trunk/libc/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S Mon Jul 20 00:07:37 2009
@@ -22,6 +22,7 @@
#include <lowlevellock.h>
#include <lowlevelcond.h>
#include <tcb-offsets.h>
+#include <pthread-pi-defines.h>
#include <kernel-features.h>
@@ -47,6 +48,9 @@
pushq %r12
cfi_adjust_cfa_offset(8)
cfi_rel_offset(%r12, 0)
+ pushq %r13
+ cfi_adjust_cfa_offset(8)
+ cfi_rel_offset(%r13, 0)
#define FRAME_SIZE 32
subq $FRAME_SIZE, %rsp
cfi_adjust_cfa_offset(FRAME_SIZE)
@@ -124,24 +128,48 @@
movq 8(%rsp), %rdi
xorq %r10, %r10
movq %r12, %rdx
- addq $cond_futex-cond_lock, %rdi
+ // XXX reverse + lea
+ addq $cond_futex, %rdi
cmpq $-1, dep_mutex-cond_futex(%rdi)
#ifdef __ASSUME_PRIVATE_FUTEX
movl $FUTEX_WAIT, %eax
movl $(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi
cmove %eax, %esi
#else
- movl $FUTEX_WAIT, %eax
+ movl $0, %eax
movl %fs:PRIVATE_FUTEX, %esi
cmove %eax, %esi
# if FUTEX_WAIT != 0
+# error "cc destroyed by following orl"
orl $FUTEX_WAIT, %esi
# endif
#endif
+ je 60f
+
+ movq dep_mutex-cond_futex(%rdi), %r8
+ /* Requeue to a PI mutex if the PI bit is set. */
+ testl $PI_BIT, MUTEX_KIND(%r8)
+ je 60f
+
+ movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi
movl $SYS_futex, %eax
syscall
- movl (%rsp), %edi
+ movl $1, %r13d
+#ifdef __ASSUME_REQUEUE_PI
+ jmp 62f
+#else
+ cmpq $-4095, %rax
+ jnae 62f
+
+ movl $(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi
+#endif
+
+60: xorl %r13d, %r13d
+ movl $SYS_futex, %eax
+ syscall
+
+62: movl (%rsp), %edi
callq __pthread_disable_asynccancel
.LcleanupEND:
@@ -209,11 +237,21 @@
#endif
jne 10f
-11: movq 16(%rsp), %rdi
+ /* If requeue_pi is used the kernel performs the locking of the
+ mutex. */
+11: xorl %eax, %eax
+ testl %r13d, %r13d
+ jnz 14f
+
+ movq 16(%rsp), %rdi
callq __pthread_mutex_cond_lock
+
14: addq $FRAME_SIZE, %rsp
cfi_adjust_cfa_offset(-FRAME_SIZE)
+ popq %r13
+ cfi_adjust_cfa_offset(-8)
+ cfi_restore(%r13)
popq %r12
cfi_adjust_cfa_offset(-8)
cfi_restore(%r12)
@@ -223,8 +261,9 @@
/* Initial locking failed. */
1:
- cfi_adjust_cfa_offset(8 + FRAME_SIZE)
- cfi_rel_offset(%r12, FRAME_SIZE)
+ cfi_adjust_cfa_offset(16 + FRAME_SIZE)
+ cfi_rel_offset(%r12, FRAME_SIZE + 8)
+ cfi_rel_offset(%r13, FRAME_SIZE)
#if cond_lock != 0
addq $cond_lock, %rdi
#endif
@@ -308,9 +347,11 @@
__condvar_cleanup1:
/* Stack frame:
- rsp + 40
+ rsp + 48
+--------------------------+
- rsp + 32 | %r12 |
+ rsp + 40 | %r12 |
+ +--------------------------+
+ rsp + 32 | %r13 |
+--------------------------+
rsp + 24 | unused |
+--------------------------+
@@ -431,7 +472,8 @@
callq __pthread_mutex_cond_lock
movq 24(%rsp), %rdi
- movq 32(%rsp), %r12
+ movq 40(%rsp), %r12
+ movq 32(%rsp), %r13
.LcallUR:
call _Unwind_Resume@PLT
hlt