diff options
Diffstat (limited to 'firmware/kernel/mrsw_lock.c')
-rw-r--r-- | firmware/kernel/mrsw_lock.c | 103 |
1 files changed, 52 insertions, 51 deletions
diff --git a/firmware/kernel/mrsw_lock.c b/firmware/kernel/mrsw_lock.c index 45c8801b74..b683f63d5f 100644 --- a/firmware/kernel/mrsw_lock.c +++ b/firmware/kernel/mrsw_lock.c | |||
@@ -19,7 +19,8 @@ | |||
19 | * | 19 | * |
20 | ****************************************************************************/ | 20 | ****************************************************************************/ |
21 | #include "kernel-internal.h" | 21 | #include "kernel-internal.h" |
22 | #include "mrsw-lock.h" | 22 | #include <string.h> |
23 | #include "mrsw_lock.h" | ||
23 | 24 | ||
24 | #ifdef HAVE_PRIORITY_SCHEDULING | 25 | #ifdef HAVE_PRIORITY_SCHEDULING |
25 | 26 | ||
@@ -34,13 +35,14 @@ mrsw_reader_claim(struct mrsw_lock *mrsw, struct thread_entry *current, | |||
34 | 35 | ||
35 | static FORCE_INLINE void | 36 | static FORCE_INLINE void |
36 | mrsw_reader_relinquish(struct mrsw_lock *mrsw, struct thread_entry *current, | 37 | mrsw_reader_relinquish(struct mrsw_lock *mrsw, struct thread_entry *current, |
37 | int count, unsigned int slotnum) | 38 | struct thread_entry *first, int count, |
39 | unsigned int slotnum) | ||
38 | { | 40 | { |
39 | /* If no writer is queued or has ownership then noone is queued; | 41 | /* If no writer is queued or has ownership then noone is queued; |
40 | if a writer owns it, then the reader would be blocked instead. | 42 | if a writer owns it, then the reader would be blocked instead. |
41 | Therefore, if the queue has threads, then the next after the | 43 | Therefore, if the queue has threads, then the next after the |
42 | owning readers is a writer and this is not the last reader. */ | 44 | owning readers is a writer and this is not the last reader. */ |
43 | if (mrsw->queue) | 45 | if (first) |
44 | corelock_lock(&mrsw->splay.cl); | 46 | corelock_lock(&mrsw->splay.cl); |
45 | 47 | ||
46 | threadbit_clear_bit(&mrsw->splay.mask, slotnum); | 48 | threadbit_clear_bit(&mrsw->splay.mask, slotnum); |
@@ -61,10 +63,10 @@ mrsw_reader_relinquish(struct mrsw_lock *mrsw, struct thread_entry *current, | |||
61 | threadbit_popcount(&mrsw->splay.mask)); | 63 | threadbit_popcount(&mrsw->splay.mask)); |
62 | /* switch owner to sole remaining reader */ | 64 | /* switch owner to sole remaining reader */ |
63 | slotnum = threadbit_ffs(&mrsw->splay.mask); | 65 | slotnum = threadbit_ffs(&mrsw->splay.mask); |
64 | mrsw->splay.blocker.thread = thread_id_entry(slotnum); | 66 | mrsw->splay.blocker.thread = __thread_slot_entry(slotnum); |
65 | } | 67 | } |
66 | 68 | ||
67 | if (mrsw->queue) | 69 | if (first) |
68 | { | 70 | { |
69 | priority_disinherit(current, &mrsw->splay.blocker); | 71 | priority_disinherit(current, &mrsw->splay.blocker); |
70 | corelock_unlock(&mrsw->splay.cl); | 72 | corelock_unlock(&mrsw->splay.cl); |
@@ -72,23 +74,25 @@ mrsw_reader_relinquish(struct mrsw_lock *mrsw, struct thread_entry *current, | |||
72 | } | 74 | } |
73 | 75 | ||
74 | static FORCE_INLINE unsigned int | 76 | static FORCE_INLINE unsigned int |
75 | mrsw_reader_wakeup_writer(struct mrsw_lock *mrsw, unsigned int slotnum) | 77 | mrsw_reader_wakeup_writer(struct mrsw_lock *mrsw, struct thread_entry *thread, |
78 | unsigned int slotnum) | ||
76 | { | 79 | { |
77 | threadbit_clear_bit(&mrsw->splay.mask, slotnum); | 80 | threadbit_clear_bit(&mrsw->splay.mask, slotnum); |
78 | return wakeup_thread(&mrsw->queue, WAKEUP_TRANSFER); | 81 | return wakeup_thread(thread, WAKEUP_TRANSFER); |
79 | } | 82 | } |
80 | 83 | ||
81 | static FORCE_INLINE unsigned int | 84 | static FORCE_INLINE unsigned int |
82 | mrsw_writer_wakeup_writer(struct mrsw_lock *mrsw) | 85 | mrsw_writer_wakeup_writer(struct mrsw_lock *mrsw, struct thread_entry *thread) |
83 | { | 86 | { |
84 | return wakeup_thread(&mrsw->queue, WAKEUP_TRANSFER); | 87 | return wakeup_thread(thread, WAKEUP_TRANSFER); |
88 | (void)mrsw; | ||
85 | } | 89 | } |
86 | 90 | ||
87 | static FORCE_INLINE unsigned int | 91 | static FORCE_INLINE unsigned int |
88 | mrsw_writer_wakeup_readers(struct mrsw_lock *mrsw) | 92 | mrsw_writer_wakeup_readers(struct mrsw_lock *mrsw, struct thread_entry *first) |
89 | { | 93 | { |
90 | unsigned int result = wakeup_thread(&mrsw->queue, WAKEUP_TRANSFER_MULTI); | 94 | unsigned int result = wakeup_thread(first, WAKEUP_TRANSFER_MULTI); |
91 | mrsw->count = thread_self_entry()->retval; | 95 | mrsw->count = __running_self_entry()->retval; |
92 | return result; | 96 | return result; |
93 | } | 97 | } |
94 | 98 | ||
@@ -97,32 +101,36 @@ mrsw_writer_wakeup_readers(struct mrsw_lock *mrsw) | |||
97 | #define mrsw_reader_claim(mrsw, current, count, slotnum) \ | 101 | #define mrsw_reader_claim(mrsw, current, count, slotnum) \ |
98 | do {} while (0) | 102 | do {} while (0) |
99 | 103 | ||
100 | #define mrsw_reader_relinquish(mrsw, current, count, slotnum) \ | 104 | #define mrsw_reader_relinquish(mrsw, current, first, count, slotnum) \ |
101 | do {} while (0) | 105 | do {} while (0) |
102 | 106 | ||
103 | static FORCE_INLINE unsigned int | 107 | static FORCE_INLINE unsigned int |
104 | mrsw_reader_wakeup_writer(struct mrsw_lock *mrsw) | 108 | mrsw_reader_wakeup_writer(struct mrsw_lock *mrsw, struct thread_entry *thread) |
105 | { | 109 | { |
106 | mrsw->splay.blocker.thread = mrsw->queue; | 110 | mrsw->splay.blocker.thread = thread; |
107 | return wakeup_thread(&mrsw->queue); | 111 | return wakeup_thread(thread); |
108 | } | 112 | } |
109 | 113 | ||
110 | static FORCE_INLINE unsigned int | 114 | static FORCE_INLINE unsigned int |
111 | mrsw_writer_wakeup_writer(struct mrsw_lock *mrsw) | 115 | mrsw_writer_wakeup_writer(struct mrsw_lock *mrsw, struct thread_entry *thread) |
112 | { | 116 | { |
113 | mrsw->splay.blocker.thread = mrsw->queue; | 117 | mrsw->splay.blocker.thread = thread; |
114 | return wakeup_thread(&mrsw->queue); | 118 | return wakeup_thread(thread); |
115 | } | 119 | } |
116 | 120 | ||
117 | static FORCE_INLINE unsigned int | 121 | static FORCE_INLINE unsigned int |
118 | mrsw_writer_wakeup_readers(struct mrsw_lock *mrsw) | 122 | mrsw_writer_wakeup_readers(struct mrsw_lock *mrsw, struct thread_entry *first) |
119 | { | 123 | { |
120 | mrsw->splay.blocker.thread = NULL; | 124 | mrsw->splay.blocker.thread = NULL; |
121 | int count = 0; | 125 | int count = 1; |
122 | 126 | ||
123 | while (mrsw->queue && mrsw->queue->retval != 0) | 127 | while (1) |
124 | { | 128 | { |
125 | wakeup_thread(&mrsw->queue); | 129 | wakeup_thread(first); |
130 | |||
131 | if (!(first = WQ_THREAD_FIRST(&mrsw->queue)) || first->retval == 0) | ||
132 | break; | ||
133 | |||
126 | count++; | 134 | count++; |
127 | } | 135 | } |
128 | 136 | ||
@@ -138,14 +146,11 @@ mrsw_writer_wakeup_readers(struct mrsw_lock *mrsw) | |||
138 | void mrsw_init(struct mrsw_lock *mrsw) | 146 | void mrsw_init(struct mrsw_lock *mrsw) |
139 | { | 147 | { |
140 | mrsw->count = 0; | 148 | mrsw->count = 0; |
141 | mrsw->queue = NULL; | 149 | wait_queue_init(&mrsw->queue); |
142 | mrsw->splay.blocker.thread = NULL; | 150 | blocker_splay_init(&mrsw->splay); |
143 | #ifdef HAVE_PRIORITY_SCHEDULING | 151 | #ifdef HAVE_PRIORITY_SCHEDULING |
144 | mrsw->splay.blocker.priority = PRIORITY_IDLE; | ||
145 | threadbit_clear(&mrsw->splay.mask); | ||
146 | corelock_init(&mrsw->splay.cl); | ||
147 | memset(mrsw->rdrecursion, 0, sizeof (mrsw->rdrecursion)); | 152 | memset(mrsw->rdrecursion, 0, sizeof (mrsw->rdrecursion)); |
148 | #endif /* HAVE_PRIORITY_SCHEDULING */ | 153 | #endif |
149 | corelock_init(&mrsw->cl); | 154 | corelock_init(&mrsw->cl); |
150 | } | 155 | } |
151 | 156 | ||
@@ -154,7 +159,7 @@ void mrsw_init(struct mrsw_lock *mrsw) | |||
154 | * access recursively. The current writer is ignored and gets access. */ | 159 | * access recursively. The current writer is ignored and gets access. */ |
155 | void mrsw_read_acquire(struct mrsw_lock *mrsw) | 160 | void mrsw_read_acquire(struct mrsw_lock *mrsw) |
156 | { | 161 | { |
157 | struct thread_entry *current = thread_self_entry(); | 162 | struct thread_entry *current = __running_self_entry(); |
158 | 163 | ||
159 | if (current == mrsw->splay.blocker.thread IF_PRIO( && mrsw->count < 0 )) | 164 | if (current == mrsw->splay.blocker.thread IF_PRIO( && mrsw->count < 0 )) |
160 | return; /* Read request while holding write access; pass */ | 165 | return; /* Read request while holding write access; pass */ |
@@ -178,7 +183,7 @@ void mrsw_read_acquire(struct mrsw_lock *mrsw) | |||
178 | 183 | ||
179 | int count = mrsw->count; | 184 | int count = mrsw->count; |
180 | 185 | ||
181 | if (LIKELY(count >= 0 && !mrsw->queue)) | 186 | if (LIKELY(count >= 0 && mrsw->queue.head == NULL)) |
182 | { | 187 | { |
183 | /* Lock open to readers: | 188 | /* Lock open to readers: |
184 | IFN_PRIO, mrsw->count tracks reader recursion */ | 189 | IFN_PRIO, mrsw->count tracks reader recursion */ |
@@ -189,13 +194,10 @@ void mrsw_read_acquire(struct mrsw_lock *mrsw) | |||
189 | } | 194 | } |
190 | 195 | ||
191 | /* A writer owns it or is waiting; block... */ | 196 | /* A writer owns it or is waiting; block... */ |
192 | IF_COP( current->obj_cl = &mrsw->cl; ) | ||
193 | IF_PRIO( current->blocker = &mrsw->splay.blocker; ) | ||
194 | current->bqp = &mrsw->queue; | ||
195 | current->retval = 1; /* indicate multi-wake candidate */ | 197 | current->retval = 1; /* indicate multi-wake candidate */ |
196 | 198 | ||
197 | disable_irq(); | 199 | disable_irq(); |
198 | block_thread(current, TIMEOUT_BLOCK); | 200 | block_thread(current, TIMEOUT_BLOCK, &mrsw->queue, &mrsw->splay.blocker); |
199 | 201 | ||
200 | corelock_unlock(&mrsw->cl); | 202 | corelock_unlock(&mrsw->cl); |
201 | 203 | ||
@@ -207,7 +209,7 @@ void mrsw_read_acquire(struct mrsw_lock *mrsw) | |||
207 | * leave opens up access to writer threads. The current writer is ignored. */ | 209 | * leave opens up access to writer threads. The current writer is ignored. */ |
208 | void mrsw_read_release(struct mrsw_lock *mrsw) | 210 | void mrsw_read_release(struct mrsw_lock *mrsw) |
209 | { | 211 | { |
210 | struct thread_entry *current = thread_self_entry(); | 212 | struct thread_entry *current = __running_self_entry(); |
211 | 213 | ||
212 | if (current == mrsw->splay.blocker.thread IF_PRIO( && mrsw->count < 0 )) | 214 | if (current == mrsw->splay.blocker.thread IF_PRIO( && mrsw->count < 0 )) |
213 | return; /* Read release while holding write access; ignore */ | 215 | return; /* Read release while holding write access; ignore */ |
@@ -237,17 +239,18 @@ void mrsw_read_release(struct mrsw_lock *mrsw) | |||
237 | unsigned int result = THREAD_NONE; | 239 | unsigned int result = THREAD_NONE; |
238 | const int oldlevel = disable_irq_save(); | 240 | const int oldlevel = disable_irq_save(); |
239 | 241 | ||
240 | if (--count == 0 && mrsw->queue) | 242 | struct thread_entry *thread = WQ_THREAD_FIRST(&mrsw->queue); |
243 | if (--count == 0 && thread != NULL) | ||
241 | { | 244 | { |
242 | /* No readers remain and a writer is waiting */ | 245 | /* No readers remain and a writer is waiting */ |
243 | mrsw->count = -1; | 246 | mrsw->count = -1; |
244 | result = mrsw_reader_wakeup_writer(mrsw IF_PRIO(, slotnum)); | 247 | result = mrsw_reader_wakeup_writer(mrsw, thread IF_PRIO(, slotnum)); |
245 | } | 248 | } |
246 | else | 249 | else |
247 | { | 250 | { |
248 | /* Giving up readership; we may be the last, or not */ | 251 | /* Giving up readership; we may be the last, or not */ |
249 | mrsw->count = count; | 252 | mrsw->count = count; |
250 | mrsw_reader_relinquish(mrsw, current, count, slotnum); | 253 | mrsw_reader_relinquish(mrsw, current, thread, count, slotnum); |
251 | } | 254 | } |
252 | 255 | ||
253 | restore_irq(oldlevel); | 256 | restore_irq(oldlevel); |
@@ -265,7 +268,7 @@ void mrsw_read_release(struct mrsw_lock *mrsw) | |||
265 | * safely call recursively. */ | 268 | * safely call recursively. */ |
266 | void mrsw_write_acquire(struct mrsw_lock *mrsw) | 269 | void mrsw_write_acquire(struct mrsw_lock *mrsw) |
267 | { | 270 | { |
268 | struct thread_entry *current = thread_self_entry(); | 271 | struct thread_entry *current = __running_self_entry(); |
269 | 272 | ||
270 | if (current == mrsw->splay.blocker.thread) | 273 | if (current == mrsw->splay.blocker.thread) |
271 | { | 274 | { |
@@ -288,13 +291,10 @@ void mrsw_write_acquire(struct mrsw_lock *mrsw) | |||
288 | } | 291 | } |
289 | 292 | ||
290 | /* Readers present or a writer owns it - block... */ | 293 | /* Readers present or a writer owns it - block... */ |
291 | IF_COP( current->obj_cl = &mrsw->cl; ) | ||
292 | IF_PRIO( current->blocker = &mrsw->splay.blocker; ) | ||
293 | current->bqp = &mrsw->queue; | ||
294 | current->retval = 0; /* indicate single-wake candidate */ | 294 | current->retval = 0; /* indicate single-wake candidate */ |
295 | 295 | ||
296 | disable_irq(); | 296 | disable_irq(); |
297 | block_thread(current, TIMEOUT_BLOCK); | 297 | block_thread(current, TIMEOUT_BLOCK, &mrsw->queue, &mrsw->splay.blocker); |
298 | 298 | ||
299 | corelock_unlock(&mrsw->cl); | 299 | corelock_unlock(&mrsw->cl); |
300 | 300 | ||
@@ -305,9 +305,9 @@ void mrsw_write_acquire(struct mrsw_lock *mrsw) | |||
305 | /* Release writer thread lock and open the lock to readers and writers */ | 305 | /* Release writer thread lock and open the lock to readers and writers */ |
306 | void mrsw_write_release(struct mrsw_lock *mrsw) | 306 | void mrsw_write_release(struct mrsw_lock *mrsw) |
307 | { | 307 | { |
308 | KERNEL_ASSERT(thread_self_entry() == mrsw->splay.blocker.thread, | 308 | KERNEL_ASSERT(__running_self_entry() == mrsw->splay.blocker.thread, |
309 | "mrsw_write_release->wrong thread (%s != %s)\n", | 309 | "mrsw_write_release->wrong thread (%s != %s)\n", |
310 | thread_self_entry()->name, | 310 | __running_self_entry()->name, |
311 | mrsw->splay.blocker.thread->name); | 311 | mrsw->splay.blocker.thread->name); |
312 | 312 | ||
313 | int count = mrsw->count; | 313 | int count = mrsw->count; |
@@ -323,15 +323,16 @@ void mrsw_write_release(struct mrsw_lock *mrsw) | |||
323 | corelock_lock(&mrsw->cl); | 323 | corelock_lock(&mrsw->cl); |
324 | const int oldlevel = disable_irq_save(); | 324 | const int oldlevel = disable_irq_save(); |
325 | 325 | ||
326 | if (mrsw->queue == NULL) /* 'count' becomes zero */ | 326 | struct thread_entry *thread = WQ_THREAD_FIRST(&mrsw->queue); |
327 | if (thread == NULL) /* 'count' becomes zero */ | ||
327 | { | 328 | { |
328 | mrsw->splay.blocker.thread = NULL; | 329 | mrsw->splay.blocker.thread = NULL; |
329 | mrsw->count = 0; | 330 | mrsw->count = 0; |
330 | } | 331 | } |
331 | else if (mrsw->queue->retval == 0) /* 'count' stays -1 */ | 332 | else if (thread->retval == 0) /* 'count' stays -1 */ |
332 | result = mrsw_writer_wakeup_writer(mrsw); | 333 | result = mrsw_writer_wakeup_writer(mrsw, thread); |
333 | else /* 'count' becomes # of readers */ | 334 | else /* 'count' becomes # of readers */ |
334 | result = mrsw_writer_wakeup_readers(mrsw); | 335 | result = mrsw_writer_wakeup_readers(mrsw, thread); |
335 | 336 | ||
336 | restore_irq(oldlevel); | 337 | restore_irq(oldlevel); |
337 | corelock_unlock(&mrsw->cl); | 338 | corelock_unlock(&mrsw->cl); |