2 * RCU segmented callback lists, function definitions
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, you can access it online at
16 * http://www.gnu.org/licenses/gpl-2.0.html.
18 * Copyright IBM Corporation, 2017
20 * Authors: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
23 #include <linux/types.h>
24 #include <linux/kernel.h>
25 #include <linux/interrupt.h>
27 #include "rcu_segcblist.h"
29 /* Initialize simple callback list. */
30 void rcu_cblist_init(struct rcu_cblist *rclp)
33 rclp->tail = &rclp->head;
39 * Debug function to actually count the number of callbacks.
40 * If the number exceeds the limit specified, return -1.
42 long rcu_cblist_count_cbs(struct rcu_cblist *rclp, long lim)
45 struct rcu_head **rhpp = &rclp->head;
52 rhpp = &(*rhpp)->next;
57 * Dequeue the oldest rcu_head structure from the specified callback
58 * list. This function assumes that the callback is non-lazy, but
59 * the caller can later invoke rcu_cblist_dequeued_lazy() if it
60 * finds otherwise (and if it cares about laziness). This allows
61 * different users to have different ways of determining laziness.
63 struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp)
71 rclp->head = rhp->next;
73 rclp->tail = &rclp->head;
78 * Initialize an rcu_segcblist structure.
80 void rcu_segcblist_init(struct rcu_segcblist *rsclp)
84 BUILD_BUG_ON(RCU_NEXT_TAIL + 1 != ARRAY_SIZE(rsclp->gp_seq));
85 BUILD_BUG_ON(ARRAY_SIZE(rsclp->tails) != ARRAY_SIZE(rsclp->gp_seq));
87 for (i = 0; i < RCU_CBLIST_NSEGS; i++)
88 rsclp->tails[i] = &rsclp->head;
94 * Disable the specified rcu_segcblist structure, so that callbacks can
95 * no longer be posted to it. This structure must be empty.
97 void rcu_segcblist_disable(struct rcu_segcblist *rsclp)
99 WARN_ON_ONCE(!rcu_segcblist_empty(rsclp));
100 WARN_ON_ONCE(rcu_segcblist_n_cbs(rsclp));
101 WARN_ON_ONCE(rcu_segcblist_n_lazy_cbs(rsclp));
102 rsclp->tails[RCU_NEXT_TAIL] = NULL;
106 * Is the specified segment of the specified rcu_segcblist structure
107 * empty of callbacks?
109 bool rcu_segcblist_segempty(struct rcu_segcblist *rsclp, int seg)
111 if (seg == RCU_DONE_TAIL)
112 return &rsclp->head == rsclp->tails[RCU_DONE_TAIL];
113 return rsclp->tails[seg - 1] == rsclp->tails[seg];
117 * Does the specified rcu_segcblist structure contain callbacks that
118 * are ready to be invoked?
120 bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp)
122 return rcu_segcblist_is_enabled(rsclp) &&
123 &rsclp->head != rsclp->tails[RCU_DONE_TAIL];
127 * Does the specified rcu_segcblist structure contain callbacks that
128 * are still pending, that is, not yet ready to be invoked?
130 bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp)
132 return rcu_segcblist_is_enabled(rsclp) &&
133 !rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL);
137 * Dequeue and return the first ready-to-invoke callback. If there
138 * are no ready-to-invoke callbacks, return NULL. Disables interrupts
139 * to avoid interference. Does not protect from interference from other
142 struct rcu_head *rcu_segcblist_dequeue(struct rcu_segcblist *rsclp)
146 struct rcu_head *rhp;
148 local_irq_save(flags);
149 if (!rcu_segcblist_ready_cbs(rsclp)) {
150 local_irq_restore(flags);
155 rsclp->head = rhp->next;
156 for (i = RCU_DONE_TAIL; i < RCU_CBLIST_NSEGS; i++) {
157 if (rsclp->tails[i] != &rhp->next)
159 rsclp->tails[i] = &rsclp->head;
161 smp_mb(); /* Dequeue before decrement for rcu_barrier(). */
162 WRITE_ONCE(rsclp->len, rsclp->len - 1);
163 local_irq_restore(flags);
168 * Account for the fact that a previously dequeued callback turned out
169 * to be marked as lazy.
171 void rcu_segcblist_dequeued_lazy(struct rcu_segcblist *rsclp)
175 local_irq_save(flags);
177 local_irq_restore(flags);
181 * Return a pointer to the first callback in the specified rcu_segcblist
182 * structure. This is useful for diagnostics.
184 struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp)
186 if (rcu_segcblist_is_enabled(rsclp))
192 * Return a pointer to the first pending callback in the specified
193 * rcu_segcblist structure. This is useful just after posting a given
194 * callback -- if that callback is the first pending callback, then
195 * you cannot rely on someone else having already started up the required
198 struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp)
200 if (rcu_segcblist_is_enabled(rsclp))
201 return *rsclp->tails[RCU_DONE_TAIL];
206 * Does the specified rcu_segcblist structure contain callbacks that
207 * have not yet been processed beyond having been posted, that is,
208 * does it contain callbacks in its last segment?
210 bool rcu_segcblist_new_cbs(struct rcu_segcblist *rsclp)
212 return rcu_segcblist_is_enabled(rsclp) &&
213 !rcu_segcblist_restempty(rsclp, RCU_NEXT_READY_TAIL);
217 * Enqueue the specified callback onto the specified rcu_segcblist
218 * structure, updating accounting as needed. Note that the ->len
219 * field may be accessed locklessly, hence the WRITE_ONCE().
220 * The ->len field is used by rcu_barrier() and friends to determine
221 * if it must post a callback on this structure, and it is OK
222 * for rcu_barrier() to sometimes post callbacks needlessly, but
223 * absolutely not OK for it to ever miss posting a callback.
225 void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,
226 struct rcu_head *rhp, bool lazy)
228 WRITE_ONCE(rsclp->len, rsclp->len + 1); /* ->len sampled locklessly. */
231 smp_mb(); /* Ensure counts are updated before callback is enqueued. */
233 *rsclp->tails[RCU_NEXT_TAIL] = rhp;
234 rsclp->tails[RCU_NEXT_TAIL] = &rhp->next;
238 * Entrain the specified callback onto the specified rcu_segcblist at
239 * the end of the last non-empty segment. If the entire rcu_segcblist
240 * is empty, make no change, but return false.
242 * This is intended for use by rcu_barrier()-like primitives, -not-
243 * for normal grace-period use. IMPORTANT: The callback you enqueue
244 * will wait for all prior callbacks, NOT necessarily for a grace
245 * period. You have been warned.
247 bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
248 struct rcu_head *rhp, bool lazy)
252 if (rcu_segcblist_n_cbs(rsclp) == 0)
254 WRITE_ONCE(rsclp->len, rsclp->len + 1);
257 smp_mb(); /* Ensure counts are updated before callback is entrained. */
259 for (i = RCU_NEXT_TAIL; i > RCU_DONE_TAIL; i--)
260 if (rsclp->tails[i] != rsclp->tails[i - 1])
262 *rsclp->tails[i] = rhp;
263 for (; i <= RCU_NEXT_TAIL; i++)
264 rsclp->tails[i] = &rhp->next;
269 * Extract only the counts from the specified rcu_segcblist structure,
270 * and place them in the specified rcu_cblist structure. This function
271 * supports both callback orphaning and invocation, hence the separation
272 * of counts and callbacks. (Callbacks ready for invocation must be
273 * orphaned and adopted separately from pending callbacks, but counts
274 * apply to all callbacks. Locking must be used to make sure that
275 * both orphaned-callbacks lists are consistent.)
277 void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp,
278 struct rcu_cblist *rclp)
280 rclp->len_lazy += rsclp->len_lazy;
281 rclp->len += rsclp->len;
283 WRITE_ONCE(rsclp->len, 0); /* ->len sampled locklessly. */
287 * Extract only those callbacks ready to be invoked from the specified
288 * rcu_segcblist structure and place them in the specified rcu_cblist
291 void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp,
292 struct rcu_cblist *rclp)
296 if (!rcu_segcblist_ready_cbs(rsclp))
297 return; /* Nothing to do. */
298 *rclp->tail = rsclp->head;
299 rsclp->head = *rsclp->tails[RCU_DONE_TAIL];
300 *rsclp->tails[RCU_DONE_TAIL] = NULL;
301 rclp->tail = rsclp->tails[RCU_DONE_TAIL];
302 for (i = RCU_CBLIST_NSEGS - 1; i >= RCU_DONE_TAIL; i--)
303 if (rsclp->tails[i] == rsclp->tails[RCU_DONE_TAIL])
304 rsclp->tails[i] = &rsclp->head;
308 * Extract only those callbacks still pending (not yet ready to be
309 * invoked) from the specified rcu_segcblist structure and place them in
310 * the specified rcu_cblist structure. Note that this loses information
311 * about any callbacks that might have been partway done waiting for
312 * their grace period. Too bad! They will have to start over.
314 void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp,
315 struct rcu_cblist *rclp)
319 if (!rcu_segcblist_pend_cbs(rsclp))
320 return; /* Nothing to do. */
321 *rclp->tail = *rsclp->tails[RCU_DONE_TAIL];
322 rclp->tail = rsclp->tails[RCU_NEXT_TAIL];
323 *rsclp->tails[RCU_DONE_TAIL] = NULL;
324 for (i = RCU_DONE_TAIL + 1; i < RCU_CBLIST_NSEGS; i++)
325 rsclp->tails[i] = rsclp->tails[RCU_DONE_TAIL];
329 * Insert counts from the specified rcu_cblist structure in the
330 * specified rcu_segcblist structure.
332 void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp,
333 struct rcu_cblist *rclp)
335 rsclp->len_lazy += rclp->len_lazy;
336 /* ->len sampled locklessly. */
337 WRITE_ONCE(rsclp->len, rsclp->len + rclp->len);
343 * Move callbacks from the specified rcu_cblist to the beginning of the
344 * done-callbacks segment of the specified rcu_segcblist.
346 void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp,
347 struct rcu_cblist *rclp)
352 return; /* No callbacks to move. */
353 *rclp->tail = rsclp->head;
354 rsclp->head = rclp->head;
355 for (i = RCU_DONE_TAIL; i < RCU_CBLIST_NSEGS; i++)
356 if (&rsclp->head == rsclp->tails[i])
357 rsclp->tails[i] = rclp->tail;
361 rclp->tail = &rclp->head;
365 * Move callbacks from the specified rcu_cblist to the end of the
366 * new-callbacks segment of the specified rcu_segcblist.
368 void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp,
369 struct rcu_cblist *rclp)
372 return; /* Nothing to do. */
373 *rsclp->tails[RCU_NEXT_TAIL] = rclp->head;
374 rsclp->tails[RCU_NEXT_TAIL] = rclp->tail;
376 rclp->tail = &rclp->head;
380 * Advance the callbacks in the specified rcu_segcblist structure based
381 * on the current value passed in for the grace-period counter.
383 void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq)
387 WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp));
388 if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL))
392 * Find all callbacks whose ->gp_seq numbers indicate that they
393 * are ready to invoke, and put them into the RCU_DONE_TAIL segment.
395 for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) {
396 if (ULONG_CMP_LT(seq, rsclp->gp_seq[i]))
398 rsclp->tails[RCU_DONE_TAIL] = rsclp->tails[i];
401 /* If no callbacks moved, nothing more need be done. */
402 if (i == RCU_WAIT_TAIL)
405 /* Clean up tail pointers that might have been misordered above. */
406 for (j = RCU_WAIT_TAIL; j < i; j++)
407 rsclp->tails[j] = rsclp->tails[RCU_DONE_TAIL];
410 * Callbacks moved, so clean up the misordered ->tails[] pointers
411 * that now point into the middle of the list of ready-to-invoke
412 * callbacks. The overall effect is to copy down the later pointers
413 * into the gap that was created by the now-ready segments.
415 for (j = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++, j++) {
416 if (rsclp->tails[j] == rsclp->tails[RCU_NEXT_TAIL])
417 break; /* No more callbacks. */
418 rsclp->tails[j] = rsclp->tails[i];
419 rsclp->gp_seq[j] = rsclp->gp_seq[i];
424 * "Accelerate" callbacks based on more-accurate grace-period information.
425 * The reason for this is that RCU does not synchronize the beginnings and
426 * ends of grace periods, and that callbacks are posted locally. This in
427 * turn means that the callbacks must be labelled conservatively early
428 * on, as getting exact information would degrade both performance and
429 * scalability. When more accurate grace-period information becomes
430 * available, previously posted callbacks can be "accelerated", marking
431 * them to complete at the end of the earlier grace period.
433 * This function operates on an rcu_segcblist structure, and also the
434 * grace-period sequence number seq at which new callbacks would become
435 * ready to invoke. Returns true if there are callbacks that won't be
436 * ready to invoke until seq, false otherwise.
438 bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq)
442 WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp));
443 if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL))
447 * Find the segment preceding the oldest segment of callbacks
448 * whose ->gp_seq[] completion is at or after that passed in via
449 * "seq", skipping any empty segments. This oldest segment, along
450 * with any later segments, can be merged in with any newly arrived
451 * callbacks in the RCU_NEXT_TAIL segment, and assigned "seq"
452 * as their ->gp_seq[] grace-period completion sequence number.
454 for (i = RCU_NEXT_READY_TAIL; i > RCU_DONE_TAIL; i--)
455 if (rsclp->tails[i] != rsclp->tails[i - 1] &&
456 ULONG_CMP_LT(rsclp->gp_seq[i], seq))
460 * If all the segments contain callbacks that correspond to
461 * earlier grace-period sequence numbers than "seq", leave.
462 * Assuming that the rcu_segcblist structure has enough
463 * segments in its arrays, this can only happen if some of
464 * the non-done segments contain callbacks that really are
465 * ready to invoke. This situation will get straightened
466 * out by the next call to rcu_segcblist_advance().
468 * Also advance to the oldest segment of callbacks whose
469 * ->gp_seq[] completion is at or after that passed in via "seq",
470 * skipping any empty segments.
472 if (++i >= RCU_NEXT_TAIL)
476 * Merge all later callbacks, including newly arrived callbacks,
477 * into the segment located by the for-loop above. Assign "seq"
478 * as the ->gp_seq[] value in order to correctly handle the case
479 * where there were no pending callbacks in the rcu_segcblist
480 * structure other than in the RCU_NEXT_TAIL segment.
482 for (; i < RCU_NEXT_TAIL; i++) {
483 rsclp->tails[i] = rsclp->tails[RCU_NEXT_TAIL];
484 rsclp->gp_seq[i] = seq;
490 * Scan the specified rcu_segcblist structure for callbacks that need
491 * a grace period later than the one specified by "seq". We don't look
492 * at the RCU_DONE_TAIL or RCU_NEXT_TAIL segments because they don't
493 * have a grace-period sequence number.
495 bool rcu_segcblist_future_gp_needed(struct rcu_segcblist *rsclp,
500 for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++)
501 if (rsclp->tails[i - 1] != rsclp->tails[i] &&
502 ULONG_CMP_LT(seq, rsclp->gp_seq[i]))