Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'release' of git://git.kernel.org/pub/scm/linux/kernel/git/aegl/linux-2.6
* 'release' of git://git.kernel.org/pub/scm/linux/kernel/git/aegl/linux-2.6:
[IA64] implement ticket locks for Itanium
+91
-217
+1
-3
arch/ia64/Kconfig
+1
-3
arch/ia64/Kconfig
+88
-85
arch/ia64/include/asm/spinlock.h
+88
-85
arch/ia64/include/asm/spinlock.h
···
19
19
20
20
#define __raw_spin_lock_init(x) ((x)->lock = 0)
21
21
22
-
#ifdef ASM_SUPPORTED
23
22
/*
24
-
* Try to get the lock. If we fail to get the lock, make a non-standard call to
25
-
* ia64_spinlock_contention(). We do not use a normal call because that would force all
26
-
* callers of __raw_spin_lock() to be non-leaf routines. Instead, ia64_spinlock_contention() is
27
-
* carefully coded to touch only those registers that __raw_spin_lock() marks "clobbered".
23
+
* Ticket locks are conceptually two parts, one indicating the current head of
24
+
* the queue, and the other indicating the current tail. The lock is acquired
25
+
* by atomically noting the tail and incrementing it by one (thus adding
26
+
* ourself to the queue and noting our position), then waiting until the head
27
+
* becomes equal to the the initial value of the tail.
28
+
*
29
+
* 63 32 31 0
30
+
* +----------------------------------------------------+
31
+
* | next_ticket_number | now_serving |
32
+
* +----------------------------------------------------+
28
33
*/
29
34
30
-
#define IA64_SPINLOCK_CLOBBERS "ar.ccv", "ar.pfs", "p14", "p15", "r27", "r28", "r29", "r30", "b6", "memory"
35
+
#define TICKET_SHIFT 32
31
36
32
-
static inline void
33
-
__raw_spin_lock_flags (raw_spinlock_t *lock, unsigned long flags)
37
+
static __always_inline void __ticket_spin_lock(raw_spinlock_t *lock)
34
38
{
35
-
register volatile unsigned int *ptr asm ("r31") = &lock->lock;
39
+
int *p = (int *)&lock->lock, turn, now_serving;
36
40
37
-
#if (__GNUC__ == 3 && __GNUC_MINOR__ < 3)
38
-
# ifdef CONFIG_ITANIUM
39
-
/* don't use brl on Itanium... */
40
-
asm volatile ("{\n\t"
41
-
" mov ar.ccv = r0\n\t"
42
-
" mov r28 = ip\n\t"
43
-
" mov r30 = 1;;\n\t"
44
-
"}\n\t"
45
-
"cmpxchg4.acq r30 = [%1], r30, ar.ccv\n\t"
46
-
"movl r29 = ia64_spinlock_contention_pre3_4;;\n\t"
47
-
"cmp4.ne p14, p0 = r30, r0\n\t"
48
-
"mov b6 = r29;;\n\t"
49
-
"mov r27=%2\n\t"
50
-
"(p14) br.cond.spnt.many b6"
51
-
: "=r"(ptr) : "r"(ptr), "r" (flags) : IA64_SPINLOCK_CLOBBERS);
52
-
# else
53
-
asm volatile ("{\n\t"
54
-
" mov ar.ccv = r0\n\t"
55
-
" mov r28 = ip\n\t"
56
-
" mov r30 = 1;;\n\t"
57
-
"}\n\t"
58
-
"cmpxchg4.acq r30 = [%1], r30, ar.ccv;;\n\t"
59
-
"cmp4.ne p14, p0 = r30, r0\n\t"
60
-
"mov r27=%2\n\t"
61
-
"(p14) brl.cond.spnt.many ia64_spinlock_contention_pre3_4;;"
62
-
: "=r"(ptr) : "r"(ptr), "r" (flags) : IA64_SPINLOCK_CLOBBERS);
63
-
# endif /* CONFIG_MCKINLEY */
64
-
#else
65
-
# ifdef CONFIG_ITANIUM
66
-
/* don't use brl on Itanium... */
67
-
/* mis-declare, so we get the entry-point, not it's function descriptor: */
68
-
asm volatile ("mov r30 = 1\n\t"
69
-
"mov r27=%2\n\t"
70
-
"mov ar.ccv = r0;;\n\t"
71
-
"cmpxchg4.acq r30 = [%0], r30, ar.ccv\n\t"
72
-
"movl r29 = ia64_spinlock_contention;;\n\t"
73
-
"cmp4.ne p14, p0 = r30, r0\n\t"
74
-
"mov b6 = r29;;\n\t"
75
-
"(p14) br.call.spnt.many b6 = b6"
76
-
: "=r"(ptr) : "r"(ptr), "r" (flags) : IA64_SPINLOCK_CLOBBERS);
77
-
# else
78
-
asm volatile ("mov r30 = 1\n\t"
79
-
"mov r27=%2\n\t"
80
-
"mov ar.ccv = r0;;\n\t"
81
-
"cmpxchg4.acq r30 = [%0], r30, ar.ccv;;\n\t"
82
-
"cmp4.ne p14, p0 = r30, r0\n\t"
83
-
"(p14) brl.call.spnt.many b6=ia64_spinlock_contention;;"
84
-
: "=r"(ptr) : "r"(ptr), "r" (flags) : IA64_SPINLOCK_CLOBBERS);
85
-
# endif /* CONFIG_MCKINLEY */
86
-
#endif
41
+
now_serving = *p;
42
+
turn = ia64_fetchadd(1, p+1, acq);
43
+
44
+
if (turn == now_serving)
45
+
return;
46
+
47
+
do {
48
+
cpu_relax();
49
+
} while (ACCESS_ONCE(*p) != turn);
87
50
}
88
51
89
-
#define __raw_spin_lock(lock) __raw_spin_lock_flags(lock, 0)
52
+
static __always_inline int __ticket_spin_trylock(raw_spinlock_t *lock)
53
+
{
54
+
long tmp = ACCESS_ONCE(lock->lock), try;
90
55
91
-
/* Unlock by doing an ordered store and releasing the cacheline with nta */
92
-
static inline void __raw_spin_unlock(raw_spinlock_t *x) {
93
-
barrier();
94
-
asm volatile ("st4.rel.nta [%0] = r0\n\t" :: "r"(x));
56
+
if (!(((tmp >> TICKET_SHIFT) ^ tmp) & ((1L << TICKET_SHIFT) - 1))) {
57
+
try = tmp + (1L << TICKET_SHIFT);
58
+
59
+
return ia64_cmpxchg(acq, &lock->lock, tmp, try, sizeof (tmp)) == tmp;
60
+
}
61
+
return 0;
95
62
}
96
63
97
-
#else /* !ASM_SUPPORTED */
98
-
#define __raw_spin_lock_flags(lock, flags) __raw_spin_lock(lock)
99
-
# define __raw_spin_lock(x) \
100
-
do { \
101
-
__u32 *ia64_spinlock_ptr = (__u32 *) (x); \
102
-
__u64 ia64_spinlock_val; \
103
-
ia64_spinlock_val = ia64_cmpxchg4_acq(ia64_spinlock_ptr, 1, 0); \
104
-
if (unlikely(ia64_spinlock_val)) { \
105
-
do { \
106
-
while (*ia64_spinlock_ptr) \
107
-
ia64_barrier(); \
108
-
ia64_spinlock_val = ia64_cmpxchg4_acq(ia64_spinlock_ptr, 1, 0); \
109
-
} while (ia64_spinlock_val); \
110
-
} \
111
-
} while (0)
112
-
#define __raw_spin_unlock(x) do { barrier(); ((raw_spinlock_t *) x)->lock = 0; } while (0)
113
-
#endif /* !ASM_SUPPORTED */
64
+
static __always_inline void __ticket_spin_unlock(raw_spinlock_t *lock)
65
+
{
66
+
int *p = (int *)&lock->lock;
114
67
115
-
#define __raw_spin_is_locked(x) ((x)->lock != 0)
116
-
#define __raw_spin_trylock(x) (cmpxchg_acq(&(x)->lock, 0, 1) == 0)
117
-
#define __raw_spin_unlock_wait(lock) \
118
-
do { while (__raw_spin_is_locked(lock)) cpu_relax(); } while (0)
68
+
(void)ia64_fetchadd(1, p, rel);
69
+
}
70
+
71
+
static inline int __ticket_spin_is_locked(raw_spinlock_t *lock)
72
+
{
73
+
long tmp = ACCESS_ONCE(lock->lock);
74
+
75
+
return !!(((tmp >> TICKET_SHIFT) ^ tmp) & ((1L << TICKET_SHIFT) - 1));
76
+
}
77
+
78
+
static inline int __ticket_spin_is_contended(raw_spinlock_t *lock)
79
+
{
80
+
long tmp = ACCESS_ONCE(lock->lock);
81
+
82
+
return (((tmp >> TICKET_SHIFT) - tmp) & ((1L << TICKET_SHIFT) - 1)) > 1;
83
+
}
84
+
85
+
static inline int __raw_spin_is_locked(raw_spinlock_t *lock)
86
+
{
87
+
return __ticket_spin_is_locked(lock);
88
+
}
89
+
90
+
static inline int __raw_spin_is_contended(raw_spinlock_t *lock)
91
+
{
92
+
return __ticket_spin_is_contended(lock);
93
+
}
94
+
#define __raw_spin_is_contended __raw_spin_is_contended
95
+
96
+
static __always_inline void __raw_spin_lock(raw_spinlock_t *lock)
97
+
{
98
+
__ticket_spin_lock(lock);
99
+
}
100
+
101
+
static __always_inline int __raw_spin_trylock(raw_spinlock_t *lock)
102
+
{
103
+
return __ticket_spin_trylock(lock);
104
+
}
105
+
106
+
static __always_inline void __raw_spin_unlock(raw_spinlock_t *lock)
107
+
{
108
+
__ticket_spin_unlock(lock);
109
+
}
110
+
111
+
static __always_inline void __raw_spin_lock_flags(raw_spinlock_t *lock,
112
+
unsigned long flags)
113
+
{
114
+
__raw_spin_lock(lock);
115
+
}
116
+
117
+
static inline void __raw_spin_unlock_wait(raw_spinlock_t *lock)
118
+
{
119
+
while (__raw_spin_is_locked(lock))
120
+
cpu_relax();
121
+
}
119
122
120
123
#define __raw_read_can_lock(rw) (*(volatile int *)(rw) >= 0)
121
124
#define __raw_write_can_lock(rw) (*(volatile int *)(rw) == 0)
+1
-1
arch/ia64/include/asm/spinlock_types.h
+1
-1
arch/ia64/include/asm/spinlock_types.h
-89
arch/ia64/kernel/head.S
-89
arch/ia64/kernel/head.S
···
1130
1130
#endif /* CONFIG_IA64_BRL_EMU */
1131
1131
1132
1132
#ifdef CONFIG_SMP
1133
-
/*
1134
-
* This routine handles spinlock contention. It uses a non-standard calling
1135
-
* convention to avoid converting leaf routines into interior routines. Because
1136
-
* of this special convention, there are several restrictions:
1137
-
*
1138
-
* - do not use gp relative variables, this code is called from the kernel
1139
-
* and from modules, r1 is undefined.
1140
-
* - do not use stacked registers, the caller owns them.
1141
-
* - do not use the scratch stack space, the caller owns it.
1142
-
* - do not use any registers other than the ones listed below
1143
-
*
1144
-
* Inputs:
1145
-
* ar.pfs - saved CFM of caller
1146
-
* ar.ccv - 0 (and available for use)
1147
-
* r27 - flags from spin_lock_irqsave or 0. Must be preserved.
1148
-
* r28 - available for use.
1149
-
* r29 - available for use.
1150
-
* r30 - available for use.
1151
-
* r31 - address of lock, available for use.
1152
-
* b6 - return address
1153
-
* p14 - available for use.
1154
-
* p15 - used to track flag status.
1155
-
*
1156
-
* If you patch this code to use more registers, do not forget to update
1157
-
* the clobber lists for spin_lock() in arch/ia64/include/asm/spinlock.h.
1158
-
*/
1159
-
1160
-
#if (__GNUC__ == 3 && __GNUC_MINOR__ < 3)
1161
-
1162
-
GLOBAL_ENTRY(ia64_spinlock_contention_pre3_4)
1163
-
.prologue
1164
-
.save ar.pfs, r0 // this code effectively has a zero frame size
1165
-
.save rp, r28
1166
-
.body
1167
-
nop 0
1168
-
tbit.nz p15,p0=r27,IA64_PSR_I_BIT
1169
-
.restore sp // pop existing prologue after next insn
1170
-
mov b6 = r28
1171
-
.prologue
1172
-
.save ar.pfs, r0
1173
-
.altrp b6
1174
-
.body
1175
-
;;
1176
-
(p15) ssm psr.i // reenable interrupts if they were on
1177
-
// DavidM says that srlz.d is slow and is not required in this case
1178
-
.wait:
1179
-
// exponential backoff, kdb, lockmeter etc. go in here
1180
-
hint @pause
1181
-
ld4 r30=[r31] // don't use ld4.bias; if it's contended, we won't write the word
1182
-
nop 0
1183
-
;;
1184
-
cmp4.ne p14,p0=r30,r0
1185
-
(p14) br.cond.sptk.few .wait
1186
-
(p15) rsm psr.i // disable interrupts if we reenabled them
1187
-
br.cond.sptk.few b6 // lock is now free, try to acquire
1188
-
.global ia64_spinlock_contention_pre3_4_end // for kernprof
1189
-
ia64_spinlock_contention_pre3_4_end:
1190
-
END(ia64_spinlock_contention_pre3_4)
1191
-
1192
-
#else
1193
-
1194
-
GLOBAL_ENTRY(ia64_spinlock_contention)
1195
-
.prologue
1196
-
.altrp b6
1197
-
.body
1198
-
tbit.nz p15,p0=r27,IA64_PSR_I_BIT
1199
-
;;
1200
-
.wait:
1201
-
(p15) ssm psr.i // reenable interrupts if they were on
1202
-
// DavidM says that srlz.d is slow and is not required in this case
1203
-
.wait2:
1204
-
// exponential backoff, kdb, lockmeter etc. go in here
1205
-
hint @pause
1206
-
ld4 r30=[r31] // don't use ld4.bias; if it's contended, we won't write the word
1207
-
;;
1208
-
cmp4.ne p14,p0=r30,r0
1209
-
mov r30 = 1
1210
-
(p14) br.cond.sptk.few .wait2
1211
-
(p15) rsm psr.i // disable interrupts if we reenabled them
1212
-
;;
1213
-
cmpxchg4.acq r30=[r31], r30, ar.ccv
1214
-
;;
1215
-
cmp4.ne p14,p0=r0,r30
1216
-
(p14) br.cond.sptk.few .wait
1217
-
1218
-
br.ret.sptk.many b6 // lock is now taken
1219
-
END(ia64_spinlock_contention)
1220
-
1221
-
#endif
1222
1133
1223
1134
#ifdef CONFIG_HOTPLUG_CPU
1224
1135
GLOBAL_ENTRY(ia64_jump_to_sal)
-20
arch/ia64/kernel/ia64_ksyms.c
-20
arch/ia64/kernel/ia64_ksyms.c
···
84
84
#include <asm/unwind.h>
85
85
EXPORT_SYMBOL(unw_init_running);
86
86
87
-
#ifdef ASM_SUPPORTED
88
-
# ifdef CONFIG_SMP
89
-
# if (__GNUC__ == 3 && __GNUC_MINOR__ < 3)
90
-
/*
91
-
* This is not a normal routine and we don't want a function descriptor for it, so we use
92
-
* a fake declaration here.
93
-
*/
94
-
extern char ia64_spinlock_contention_pre3_4;
95
-
EXPORT_SYMBOL(ia64_spinlock_contention_pre3_4);
96
-
# else
97
-
/*
98
-
* This is not a normal routine and we don't want a function descriptor for it, so we use
99
-
* a fake declaration here.
100
-
*/
101
-
extern char ia64_spinlock_contention;
102
-
EXPORT_SYMBOL(ia64_spinlock_contention);
103
-
# endif
104
-
# endif
105
-
#endif
106
-
107
87
#if defined(CONFIG_IA64_ESI) || defined(CONFIG_IA64_ESI_MODULE)
108
88
extern void esi_call_phys (void);
109
89
EXPORT_SYMBOL_GPL(esi_call_phys);
+1
-19
arch/ia64/oprofile/backtrace.c
+1
-19
arch/ia64/oprofile/backtrace.c
···
32
32
u64 *prev_pfs_loc; /* state for WAR for old spinlock ool code */
33
33
} ia64_backtrace_t;
34
34
35
-
#if (__GNUC__ == 3 && __GNUC_MINOR__ < 3)
36
-
/*
37
-
* Returns non-zero if the PC is in the spinlock contention out-of-line code
38
-
* with non-standard calling sequence (on older compilers).
39
-
*/
40
-
static __inline__ int in_old_ool_spinlock_code(unsigned long pc)
41
-
{
42
-
extern const char ia64_spinlock_contention_pre3_4[] __attribute__ ((weak));
43
-
extern const char ia64_spinlock_contention_pre3_4_end[] __attribute__ ((weak));
44
-
unsigned long sc_start = (unsigned long)ia64_spinlock_contention_pre3_4;
45
-
unsigned long sc_end = (unsigned long)ia64_spinlock_contention_pre3_4_end;
46
-
return (sc_start && sc_end && pc >= sc_start && pc < sc_end);
47
-
}
48
-
#else
49
-
/* Newer spinlock code does a proper br.call and works fine with the unwinder */
50
-
#define in_old_ool_spinlock_code(pc) 0
51
-
#endif
52
-
53
35
/* Returns non-zero if the PC is in the Interrupt Vector Table */
54
36
static __inline__ int in_ivt_code(unsigned long pc)
55
37
{
···
62
80
*/
63
81
if (bt->prev_pfs_loc && bt->regs && bt->frame.pfs_loc == bt->prev_pfs_loc)
64
82
bt->frame.pfs_loc = &bt->regs->ar_pfs;
65
-
bt->prev_pfs_loc = (in_old_ool_spinlock_code(bt->frame.ip) ? bt->frame.pfs_loc : NULL);
83
+
bt->prev_pfs_loc = NULL;
66
84
67
85
return unw_unwind(&bt->frame) == 0;
68
86
}