1/* SPDX-License-Identifier: GPL-2.0+ */
  2/*
  3 * RCU segmented callback lists, internal-to-rcu header file
  4 *
  5 * Copyright IBM Corporation, 2017
  6 *
  7 * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
  8 */
  9
 10#include <linux/rcu_segcblist.h>
 11
 12/* Return number of callbacks in the specified callback list. */
 13static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp)
 14{
 15	return READ_ONCE(rclp->len);
 16}
 17
 18void rcu_cblist_init(struct rcu_cblist *rclp);
 19void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp);
 20void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp,
 21			      struct rcu_cblist *srclp,
 22			      struct rcu_head *rhp);
 23struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp);
 24
 25/*
 26 * Is the specified rcu_segcblist structure empty?
 27 *
 28 * But careful!  The fact that the ->head field is NULL does not
 29 * necessarily imply that there are no callbacks associated with
 30 * this structure.  When callbacks are being invoked, they are
 31 * removed as a group.  If callback invocation must be preempted,
 32 * the remaining callbacks will be added back to the list.  Either
 33 * way, the counts are updated later.
 34 *
 35 * So it is often the case that rcu_segcblist_n_cbs() should be used
 36 * instead.
 37 */
 38static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp)
 39{
 40	return !READ_ONCE(rsclp->head);
 41}
 42
 43/* Return number of callbacks in segmented callback list. */
 44static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp)
 45{
 46#ifdef CONFIG_RCU_NOCB_CPU
 47	return atomic_long_read(&rsclp->len);
 48#else
 49	return READ_ONCE(rsclp->len);
 50#endif
 51}
 52
 53/*
 54 * Is the specified rcu_segcblist enabled, for example, not corresponding
 55 * to an offline CPU?
 56 */
 57static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp)
 58{
 59	return rsclp->enabled;
 60}
 61
 62/* Is the specified rcu_segcblist offloaded?  */
 63static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp)
 64{
 65	return rsclp->offloaded;
 66}
 67
 68/*
 69 * Are all segments following the specified segment of the specified
 70 * rcu_segcblist structure empty of callbacks?  (The specified
 71 * segment might well contain callbacks.)
 72 */
 73static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg)
 74{
 75	return !READ_ONCE(*READ_ONCE(rsclp->tails[seg]));
 76}
 77
 78void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp);
 79void rcu_segcblist_init(struct rcu_segcblist *rsclp);
 80void rcu_segcblist_disable(struct rcu_segcblist *rsclp);
 81void rcu_segcblist_offload(struct rcu_segcblist *rsclp);
 82bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp);
 83bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp);
 84struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp);
 85struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp);
 86bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp);
 87void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,
 88			   struct rcu_head *rhp);
 89bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
 90			   struct rcu_head *rhp);
 91void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp,
 92				 struct rcu_cblist *rclp);
 93void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp,
 94				    struct rcu_cblist *rclp);
 95void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp,
 96				    struct rcu_cblist *rclp);
 97void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp,
 98				struct rcu_cblist *rclp);
 99void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp,
100				   struct rcu_cblist *rclp);
101void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp,
102				   struct rcu_cblist *rclp);
103void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq);
104bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq);
105void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp,
106			 struct rcu_segcblist *src_rsclp);