Linux Audio

Check our new training course

Loading...
v4.17
  1/*
  2 * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc.
  3 * Copyright (c) 2008 Dave Chinner
  4 * All Rights Reserved.
  5 *
  6 * This program is free software; you can redistribute it and/or
  7 * modify it under the terms of the GNU General Public License as
  8 * published by the Free Software Foundation.
  9 *
 10 * This program is distributed in the hope that it would be useful,
 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13 * GNU General Public License for more details.
 14 *
 15 * You should have received a copy of the GNU General Public License
 16 * along with this program; if not, write the Free Software Foundation,
 17 * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 18 */
 19#include "xfs.h"
 20#include "xfs_fs.h"
 21#include "xfs_format.h"
 22#include "xfs_log_format.h"
 23#include "xfs_trans_resv.h"
 
 
 24#include "xfs_mount.h"
 25#include "xfs_trans.h"
 26#include "xfs_trans_priv.h"
 27#include "xfs_trace.h"
 28#include "xfs_errortag.h"
 29#include "xfs_error.h"
 30#include "xfs_log.h"
 31
 32#ifdef DEBUG
 33/*
 34 * Check that the list is sorted as it should be.
 35 */
 36STATIC void
 37xfs_ail_check(
 38	struct xfs_ail	*ailp,
 39	xfs_log_item_t	*lip)
 40{
 41	xfs_log_item_t	*prev_lip;
 42
 43	if (list_empty(&ailp->ail_head))
 44		return;
 45
 46	/*
 47	 * Check the next and previous entries are valid.
 48	 */
 49	ASSERT((lip->li_flags & XFS_LI_IN_AIL) != 0);
 50	prev_lip = list_entry(lip->li_ail.prev, xfs_log_item_t, li_ail);
 51	if (&prev_lip->li_ail != &ailp->ail_head)
 52		ASSERT(XFS_LSN_CMP(prev_lip->li_lsn, lip->li_lsn) <= 0);
 53
 54	prev_lip = list_entry(lip->li_ail.next, xfs_log_item_t, li_ail);
 55	if (&prev_lip->li_ail != &ailp->ail_head)
 56		ASSERT(XFS_LSN_CMP(prev_lip->li_lsn, lip->li_lsn) >= 0);
 57
 58
 59}
 60#else /* !DEBUG */
 61#define	xfs_ail_check(a,l)
 62#endif /* DEBUG */
 63
 64/*
 65 * Return a pointer to the last item in the AIL.  If the AIL is empty, then
 66 * return NULL.
 67 */
 68static xfs_log_item_t *
 69xfs_ail_max(
 70	struct xfs_ail  *ailp)
 71{
 72	if (list_empty(&ailp->ail_head))
 73		return NULL;
 74
 75	return list_entry(ailp->ail_head.prev, xfs_log_item_t, li_ail);
 76}
 77
 78/*
 79 * Return a pointer to the item which follows the given item in the AIL.  If
 80 * the given item is the last item in the list, then return NULL.
 81 */
 82static xfs_log_item_t *
 83xfs_ail_next(
 84	struct xfs_ail  *ailp,
 85	xfs_log_item_t  *lip)
 86{
 87	if (lip->li_ail.next == &ailp->ail_head)
 88		return NULL;
 89
 90	return list_first_entry(&lip->li_ail, xfs_log_item_t, li_ail);
 91}
 92
 93/*
 94 * This is called by the log manager code to determine the LSN of the tail of
 95 * the log.  This is exactly the LSN of the first item in the AIL.  If the AIL
 96 * is empty, then this function returns 0.
 97 *
 98 * We need the AIL lock in order to get a coherent read of the lsn of the last
 99 * item in the AIL.
100 */
101xfs_lsn_t
102xfs_ail_min_lsn(
103	struct xfs_ail	*ailp)
104{
105	xfs_lsn_t	lsn = 0;
106	xfs_log_item_t	*lip;
107
108	spin_lock(&ailp->ail_lock);
109	lip = xfs_ail_min(ailp);
110	if (lip)
111		lsn = lip->li_lsn;
112	spin_unlock(&ailp->ail_lock);
113
114	return lsn;
115}
116
117/*
118 * Return the maximum lsn held in the AIL, or zero if the AIL is empty.
119 */
120static xfs_lsn_t
121xfs_ail_max_lsn(
122	struct xfs_ail  *ailp)
123{
124	xfs_lsn_t       lsn = 0;
125	xfs_log_item_t  *lip;
126
127	spin_lock(&ailp->ail_lock);
128	lip = xfs_ail_max(ailp);
129	if (lip)
130		lsn = lip->li_lsn;
131	spin_unlock(&ailp->ail_lock);
132
133	return lsn;
134}
135
136/*
137 * The cursor keeps track of where our current traversal is up to by tracking
138 * the next item in the list for us. However, for this to be safe, removing an
139 * object from the AIL needs to invalidate any cursor that points to it. hence
140 * the traversal cursor needs to be linked to the struct xfs_ail so that
141 * deletion can search all the active cursors for invalidation.
142 */
143STATIC void
144xfs_trans_ail_cursor_init(
145	struct xfs_ail		*ailp,
146	struct xfs_ail_cursor	*cur)
147{
148	cur->item = NULL;
149	list_add_tail(&cur->list, &ailp->ail_cursors);
150}
151
152/*
153 * Get the next item in the traversal and advance the cursor.  If the cursor
154 * was invalidated (indicated by a lip of 1), restart the traversal.
155 */
156struct xfs_log_item *
157xfs_trans_ail_cursor_next(
158	struct xfs_ail		*ailp,
159	struct xfs_ail_cursor	*cur)
160{
161	struct xfs_log_item	*lip = cur->item;
162
163	if ((uintptr_t)lip & 1)
164		lip = xfs_ail_min(ailp);
165	if (lip)
166		cur->item = xfs_ail_next(ailp, lip);
167	return lip;
168}
169
170/*
171 * When the traversal is complete, we need to remove the cursor from the list
172 * of traversing cursors.
173 */
174void
175xfs_trans_ail_cursor_done(
 
176	struct xfs_ail_cursor	*cur)
177{
178	cur->item = NULL;
179	list_del_init(&cur->list);
180}
181
182/*
183 * Invalidate any cursor that is pointing to this item. This is called when an
184 * item is removed from the AIL. Any cursor pointing to this object is now
185 * invalid and the traversal needs to be terminated so it doesn't reference a
186 * freed object. We set the low bit of the cursor item pointer so we can
187 * distinguish between an invalidation and the end of the list when getting the
188 * next item from the cursor.
189 */
190STATIC void
191xfs_trans_ail_cursor_clear(
192	struct xfs_ail		*ailp,
193	struct xfs_log_item	*lip)
194{
195	struct xfs_ail_cursor	*cur;
196
197	list_for_each_entry(cur, &ailp->ail_cursors, list) {
198		if (cur->item == lip)
199			cur->item = (struct xfs_log_item *)
200					((uintptr_t)cur->item | 1);
201	}
202}
203
204/*
205 * Find the first item in the AIL with the given @lsn by searching in ascending
206 * LSN order and initialise the cursor to point to the next item for a
207 * ascending traversal.  Pass a @lsn of zero to initialise the cursor to the
208 * first item in the AIL. Returns NULL if the list is empty.
209 */
210xfs_log_item_t *
211xfs_trans_ail_cursor_first(
212	struct xfs_ail		*ailp,
213	struct xfs_ail_cursor	*cur,
214	xfs_lsn_t		lsn)
215{
216	xfs_log_item_t		*lip;
217
218	xfs_trans_ail_cursor_init(ailp, cur);
219
220	if (lsn == 0) {
221		lip = xfs_ail_min(ailp);
222		goto out;
223	}
224
225	list_for_each_entry(lip, &ailp->ail_head, li_ail) {
226		if (XFS_LSN_CMP(lip->li_lsn, lsn) >= 0)
227			goto out;
228	}
229	return NULL;
230
231out:
232	if (lip)
233		cur->item = xfs_ail_next(ailp, lip);
234	return lip;
235}
236
237static struct xfs_log_item *
238__xfs_trans_ail_cursor_last(
239	struct xfs_ail		*ailp,
240	xfs_lsn_t		lsn)
241{
242	xfs_log_item_t		*lip;
243
244	list_for_each_entry_reverse(lip, &ailp->ail_head, li_ail) {
245		if (XFS_LSN_CMP(lip->li_lsn, lsn) <= 0)
246			return lip;
247	}
248	return NULL;
249}
250
251/*
252 * Find the last item in the AIL with the given @lsn by searching in descending
253 * LSN order and initialise the cursor to point to that item.  If there is no
254 * item with the value of @lsn, then it sets the cursor to the last item with an
255 * LSN lower than @lsn.  Returns NULL if the list is empty.
256 */
257struct xfs_log_item *
258xfs_trans_ail_cursor_last(
259	struct xfs_ail		*ailp,
260	struct xfs_ail_cursor	*cur,
261	xfs_lsn_t		lsn)
262{
263	xfs_trans_ail_cursor_init(ailp, cur);
264	cur->item = __xfs_trans_ail_cursor_last(ailp, lsn);
265	return cur->item;
266}
267
268/*
269 * Splice the log item list into the AIL at the given LSN. We splice to the
270 * tail of the given LSN to maintain insert order for push traversals. The
271 * cursor is optional, allowing repeated updates to the same LSN to avoid
272 * repeated traversals.  This should not be called with an empty list.
273 */
274static void
275xfs_ail_splice(
276	struct xfs_ail		*ailp,
277	struct xfs_ail_cursor	*cur,
278	struct list_head	*list,
279	xfs_lsn_t		lsn)
280{
281	struct xfs_log_item	*lip;
282
283	ASSERT(!list_empty(list));
284
285	/*
286	 * Use the cursor to determine the insertion point if one is
287	 * provided.  If not, or if the one we got is not valid,
288	 * find the place in the AIL where the items belong.
289	 */
290	lip = cur ? cur->item : NULL;
291	if (!lip || (uintptr_t)lip & 1)
292		lip = __xfs_trans_ail_cursor_last(ailp, lsn);
293
294	/*
295	 * If a cursor is provided, we know we're processing the AIL
296	 * in lsn order, and future items to be spliced in will
297	 * follow the last one being inserted now.  Update the
298	 * cursor to point to that last item, now while we have a
299	 * reliable pointer to it.
300	 */
301	if (cur)
302		cur->item = list_entry(list->prev, struct xfs_log_item, li_ail);
303
304	/*
305	 * Finally perform the splice.  Unless the AIL was empty,
306	 * lip points to the item in the AIL _after_ which the new
307	 * items should go.  If lip is null the AIL was empty, so
308	 * the new items go at the head of the AIL.
309	 */
310	if (lip)
311		list_splice(list, &lip->li_ail);
312	else
313		list_splice(list, &ailp->ail_head);
314}
315
316/*
317 * Delete the given item from the AIL.  Return a pointer to the item.
318 */
319static void
320xfs_ail_delete(
321	struct xfs_ail  *ailp,
322	xfs_log_item_t  *lip)
323{
324	xfs_ail_check(ailp, lip);
325	list_del(&lip->li_ail);
326	xfs_trans_ail_cursor_clear(ailp, lip);
327}
328
329static inline uint
330xfsaild_push_item(
331	struct xfs_ail		*ailp,
332	struct xfs_log_item	*lip)
333{
334	/*
335	 * If log item pinning is enabled, skip the push and track the item as
336	 * pinned. This can help induce head-behind-tail conditions.
337	 */
338	if (XFS_TEST_ERROR(false, ailp->ail_mount, XFS_ERRTAG_LOG_ITEM_PIN))
339		return XFS_ITEM_PINNED;
340
341	return lip->li_ops->iop_push(lip, &ailp->ail_buf_list);
342}
343
344static long
345xfsaild_push(
346	struct xfs_ail		*ailp)
347{
348	xfs_mount_t		*mp = ailp->ail_mount;
349	struct xfs_ail_cursor	cur;
350	xfs_log_item_t		*lip;
351	xfs_lsn_t		lsn;
352	xfs_lsn_t		target;
353	long			tout;
354	int			stuck = 0;
355	int			flushing = 0;
356	int			count = 0;
357
358	/*
359	 * If we encountered pinned items or did not finish writing out all
360	 * buffers the last time we ran, force the log first and wait for it
361	 * before pushing again.
362	 */
363	if (ailp->ail_log_flush && ailp->ail_last_pushed_lsn == 0 &&
364	    (!list_empty_careful(&ailp->ail_buf_list) ||
365	     xfs_ail_min_lsn(ailp))) {
366		ailp->ail_log_flush = 0;
367
368		XFS_STATS_INC(mp, xs_push_ail_flush);
369		xfs_log_force(mp, XFS_LOG_SYNC);
370	}
371
372	spin_lock(&ailp->ail_lock);
373
374	/* barrier matches the ail_target update in xfs_ail_push() */
375	smp_rmb();
376	target = ailp->ail_target;
377	ailp->ail_target_prev = target;
378
379	lip = xfs_trans_ail_cursor_first(ailp, &cur, ailp->ail_last_pushed_lsn);
380	if (!lip) {
381		/*
382		 * If the AIL is empty or our push has reached the end we are
383		 * done now.
384		 */
385		xfs_trans_ail_cursor_done(&cur);
386		spin_unlock(&ailp->ail_lock);
387		goto out_done;
388	}
389
390	XFS_STATS_INC(mp, xs_push_ail);
391
392	lsn = lip->li_lsn;
393	while ((XFS_LSN_CMP(lip->li_lsn, target) <= 0)) {
394		int	lock_result;
395
396		/*
397		 * Note that iop_push may unlock and reacquire the AIL lock.  We
398		 * rely on the AIL cursor implementation to be able to deal with
399		 * the dropped lock.
400		 */
401		lock_result = xfsaild_push_item(ailp, lip);
402		switch (lock_result) {
403		case XFS_ITEM_SUCCESS:
404			XFS_STATS_INC(mp, xs_push_ail_success);
405			trace_xfs_ail_push(lip);
406
407			ailp->ail_last_pushed_lsn = lsn;
408			break;
409
410		case XFS_ITEM_FLUSHING:
411			/*
412			 * The item or its backing buffer is already beeing
413			 * flushed.  The typical reason for that is that an
414			 * inode buffer is locked because we already pushed the
415			 * updates to it as part of inode clustering.
416			 *
417			 * We do not want to to stop flushing just because lots
418			 * of items are already beeing flushed, but we need to
419			 * re-try the flushing relatively soon if most of the
420			 * AIL is beeing flushed.
421			 */
422			XFS_STATS_INC(mp, xs_push_ail_flushing);
423			trace_xfs_ail_flushing(lip);
424
425			flushing++;
426			ailp->ail_last_pushed_lsn = lsn;
427			break;
428
429		case XFS_ITEM_PINNED:
430			XFS_STATS_INC(mp, xs_push_ail_pinned);
431			trace_xfs_ail_pinned(lip);
432
433			stuck++;
434			ailp->ail_log_flush++;
435			break;
436		case XFS_ITEM_LOCKED:
437			XFS_STATS_INC(mp, xs_push_ail_locked);
438			trace_xfs_ail_locked(lip);
439
440			stuck++;
441			break;
442		default:
443			ASSERT(0);
444			break;
445		}
446
447		count++;
448
449		/*
450		 * Are there too many items we can't do anything with?
451		 *
452		 * If we we are skipping too many items because we can't flush
453		 * them or they are already being flushed, we back off and
454		 * given them time to complete whatever operation is being
455		 * done. i.e. remove pressure from the AIL while we can't make
456		 * progress so traversals don't slow down further inserts and
457		 * removals to/from the AIL.
458		 *
459		 * The value of 100 is an arbitrary magic number based on
460		 * observation.
461		 */
462		if (stuck > 100)
463			break;
464
465		lip = xfs_trans_ail_cursor_next(ailp, &cur);
466		if (lip == NULL)
467			break;
468		lsn = lip->li_lsn;
469	}
470	xfs_trans_ail_cursor_done(&cur);
471	spin_unlock(&ailp->ail_lock);
472
473	if (xfs_buf_delwri_submit_nowait(&ailp->ail_buf_list))
474		ailp->ail_log_flush++;
475
476	if (!count || XFS_LSN_CMP(lsn, target) >= 0) {
477out_done:
478		/*
479		 * We reached the target or the AIL is empty, so wait a bit
480		 * longer for I/O to complete and remove pushed items from the
481		 * AIL before we start the next scan from the start of the AIL.
482		 */
483		tout = 50;
484		ailp->ail_last_pushed_lsn = 0;
485	} else if (((stuck + flushing) * 100) / count > 90) {
486		/*
487		 * Either there is a lot of contention on the AIL or we are
488		 * stuck due to operations in progress. "Stuck" in this case
489		 * is defined as >90% of the items we tried to push were stuck.
490		 *
491		 * Backoff a bit more to allow some I/O to complete before
492		 * restarting from the start of the AIL. This prevents us from
493		 * spinning on the same items, and if they are pinned will all
494		 * the restart to issue a log force to unpin the stuck items.
495		 */
496		tout = 20;
497		ailp->ail_last_pushed_lsn = 0;
498	} else {
499		/*
500		 * Assume we have more work to do in a short while.
501		 */
502		tout = 10;
503	}
504
505	return tout;
506}
507
508static int
509xfsaild(
510	void		*data)
511{
512	struct xfs_ail	*ailp = data;
513	long		tout = 0;	/* milliseconds */
514
515	current->flags |= PF_MEMALLOC;
516	set_freezable();
517
518	while (1) {
519		if (tout && tout <= 20)
520			set_current_state(TASK_KILLABLE);
521		else
522			set_current_state(TASK_INTERRUPTIBLE);
523
524		/*
525		 * Check kthread_should_stop() after we set the task state
526		 * to guarantee that we either see the stop bit and exit or
527		 * the task state is reset to runnable such that it's not
528		 * scheduled out indefinitely and detects the stop bit at
529		 * next iteration.
530		 *
531		 * A memory barrier is included in above task state set to
532		 * serialize again kthread_stop().
533		 */
534		if (kthread_should_stop()) {
535			__set_current_state(TASK_RUNNING);
536			break;
537		}
538
539		spin_lock(&ailp->ail_lock);
540
541		/*
542		 * Idle if the AIL is empty and we are not racing with a target
543		 * update. We check the AIL after we set the task to a sleep
544		 * state to guarantee that we either catch an ail_target update
545		 * or that a wake_up resets the state to TASK_RUNNING.
546		 * Otherwise, we run the risk of sleeping indefinitely.
547		 *
548		 * The barrier matches the ail_target update in xfs_ail_push().
549		 */
550		smp_rmb();
551		if (!xfs_ail_min(ailp) &&
552		    ailp->ail_target == ailp->ail_target_prev) {
553			spin_unlock(&ailp->ail_lock);
554			freezable_schedule();
555			tout = 0;
556			continue;
557		}
558		spin_unlock(&ailp->ail_lock);
559
560		if (tout)
561			freezable_schedule_timeout(msecs_to_jiffies(tout));
562
563		__set_current_state(TASK_RUNNING);
564
565		try_to_freeze();
566
567		tout = xfsaild_push(ailp);
568	}
569
570	return 0;
571}
572
573/*
574 * This routine is called to move the tail of the AIL forward.  It does this by
575 * trying to flush items in the AIL whose lsns are below the given
576 * threshold_lsn.
577 *
578 * The push is run asynchronously in a workqueue, which means the caller needs
579 * to handle waiting on the async flush for space to become available.
580 * We don't want to interrupt any push that is in progress, hence we only queue
581 * work if we set the pushing bit approriately.
582 *
583 * We do this unlocked - we only need to know whether there is anything in the
584 * AIL at the time we are called. We don't need to access the contents of
585 * any of the objects, so the lock is not needed.
586 */
587void
588xfs_ail_push(
589	struct xfs_ail	*ailp,
590	xfs_lsn_t	threshold_lsn)
591{
592	xfs_log_item_t	*lip;
593
594	lip = xfs_ail_min(ailp);
595	if (!lip || XFS_FORCED_SHUTDOWN(ailp->ail_mount) ||
596	    XFS_LSN_CMP(threshold_lsn, ailp->ail_target) <= 0)
597		return;
598
599	/*
600	 * Ensure that the new target is noticed in push code before it clears
601	 * the XFS_AIL_PUSHING_BIT.
602	 */
603	smp_wmb();
604	xfs_trans_ail_copy_lsn(ailp, &ailp->ail_target, &threshold_lsn);
605	smp_wmb();
606
607	wake_up_process(ailp->ail_task);
608}
609
610/*
611 * Push out all items in the AIL immediately
612 */
613void
614xfs_ail_push_all(
615	struct xfs_ail  *ailp)
616{
617	xfs_lsn_t       threshold_lsn = xfs_ail_max_lsn(ailp);
618
619	if (threshold_lsn)
620		xfs_ail_push(ailp, threshold_lsn);
621}
622
623/*
624 * Push out all items in the AIL immediately and wait until the AIL is empty.
625 */
626void
627xfs_ail_push_all_sync(
628	struct xfs_ail  *ailp)
629{
630	struct xfs_log_item	*lip;
631	DEFINE_WAIT(wait);
632
633	spin_lock(&ailp->ail_lock);
634	while ((lip = xfs_ail_max(ailp)) != NULL) {
635		prepare_to_wait(&ailp->ail_empty, &wait, TASK_UNINTERRUPTIBLE);
636		ailp->ail_target = lip->li_lsn;
637		wake_up_process(ailp->ail_task);
638		spin_unlock(&ailp->ail_lock);
639		schedule();
640		spin_lock(&ailp->ail_lock);
641	}
642	spin_unlock(&ailp->ail_lock);
643
644	finish_wait(&ailp->ail_empty, &wait);
645}
646
647/*
648 * xfs_trans_ail_update - bulk AIL insertion operation.
649 *
650 * @xfs_trans_ail_update takes an array of log items that all need to be
651 * positioned at the same LSN in the AIL. If an item is not in the AIL, it will
652 * be added.  Otherwise, it will be repositioned  by removing it and re-adding
653 * it to the AIL. If we move the first item in the AIL, update the log tail to
654 * match the new minimum LSN in the AIL.
655 *
656 * This function takes the AIL lock once to execute the update operations on
657 * all the items in the array, and as such should not be called with the AIL
658 * lock held. As a result, once we have the AIL lock, we need to check each log
659 * item LSN to confirm it needs to be moved forward in the AIL.
660 *
661 * To optimise the insert operation, we delete all the items from the AIL in
662 * the first pass, moving them into a temporary list, then splice the temporary
663 * list into the correct position in the AIL. This avoids needing to do an
664 * insert operation on every item.
665 *
666 * This function must be called with the AIL lock held.  The lock is dropped
667 * before returning.
668 */
669void
670xfs_trans_ail_update_bulk(
671	struct xfs_ail		*ailp,
672	struct xfs_ail_cursor	*cur,
673	struct xfs_log_item	**log_items,
674	int			nr_items,
675	xfs_lsn_t		lsn) __releases(ailp->ail_lock)
676{
677	xfs_log_item_t		*mlip;
678	int			mlip_changed = 0;
679	int			i;
680	LIST_HEAD(tmp);
681
682	ASSERT(nr_items > 0);		/* Not required, but true. */
683	mlip = xfs_ail_min(ailp);
684
685	for (i = 0; i < nr_items; i++) {
686		struct xfs_log_item *lip = log_items[i];
687		if (lip->li_flags & XFS_LI_IN_AIL) {
688			/* check if we really need to move the item */
689			if (XFS_LSN_CMP(lsn, lip->li_lsn) <= 0)
690				continue;
691
692			trace_xfs_ail_move(lip, lip->li_lsn, lsn);
693			xfs_ail_delete(ailp, lip);
694			if (mlip == lip)
695				mlip_changed = 1;
696		} else {
697			lip->li_flags |= XFS_LI_IN_AIL;
698			trace_xfs_ail_insert(lip, 0, lsn);
699		}
700		lip->li_lsn = lsn;
701		list_add(&lip->li_ail, &tmp);
702	}
703
704	if (!list_empty(&tmp))
705		xfs_ail_splice(ailp, cur, &tmp, lsn);
706
707	if (mlip_changed) {
708		if (!XFS_FORCED_SHUTDOWN(ailp->ail_mount))
709			xlog_assign_tail_lsn_locked(ailp->ail_mount);
710		spin_unlock(&ailp->ail_lock);
711
712		xfs_log_space_wake(ailp->ail_mount);
713	} else {
714		spin_unlock(&ailp->ail_lock);
715	}
716}
717
718bool
719xfs_ail_delete_one(
720	struct xfs_ail		*ailp,
721	struct xfs_log_item	*lip)
722{
723	struct xfs_log_item	*mlip = xfs_ail_min(ailp);
724
725	trace_xfs_ail_delete(lip, mlip->li_lsn, lip->li_lsn);
726	xfs_ail_delete(ailp, lip);
727	xfs_clear_li_failed(lip);
728	lip->li_flags &= ~XFS_LI_IN_AIL;
729	lip->li_lsn = 0;
730
731	return mlip == lip;
732}
733
734/**
735 * Remove a log items from the AIL
736 *
737 * @xfs_trans_ail_delete_bulk takes an array of log items that all need to
738 * removed from the AIL. The caller is already holding the AIL lock, and done
739 * all the checks necessary to ensure the items passed in via @log_items are
740 * ready for deletion. This includes checking that the items are in the AIL.
741 *
742 * For each log item to be removed, unlink it  from the AIL, clear the IN_AIL
743 * flag from the item and reset the item's lsn to 0. If we remove the first
744 * item in the AIL, update the log tail to match the new minimum LSN in the
745 * AIL.
746 *
747 * This function will not drop the AIL lock until all items are removed from
748 * the AIL to minimise the amount of lock traffic on the AIL. This does not
749 * greatly increase the AIL hold time, but does significantly reduce the amount
750 * of traffic on the lock, especially during IO completion.
751 *
752 * This function must be called with the AIL lock held.  The lock is dropped
753 * before returning.
754 */
755void
756xfs_trans_ail_delete(
757	struct xfs_ail		*ailp,
758	struct xfs_log_item	*lip,
759	int			shutdown_type) __releases(ailp->ail_lock)
 
760{
761	struct xfs_mount	*mp = ailp->ail_mount;
762	bool			mlip_changed;
 
763
764	if (!(lip->li_flags & XFS_LI_IN_AIL)) {
765		spin_unlock(&ailp->ail_lock);
766		if (!XFS_FORCED_SHUTDOWN(mp)) {
767			xfs_alert_tag(mp, XFS_PTAG_AILDELETE,
768	"%s: attempting to delete a log item that is not in the AIL",
769					__func__);
770			xfs_force_shutdown(mp, shutdown_type);
 
 
 
 
 
 
 
 
771		}
772		return;
 
 
 
 
 
 
773	}
774
775	mlip_changed = xfs_ail_delete_one(ailp, lip);
776	if (mlip_changed) {
777		if (!XFS_FORCED_SHUTDOWN(mp))
778			xlog_assign_tail_lsn_locked(mp);
779		if (list_empty(&ailp->ail_head))
780			wake_up_all(&ailp->ail_empty);
781	}
782
783	spin_unlock(&ailp->ail_lock);
784	if (mlip_changed)
785		xfs_log_space_wake(ailp->ail_mount);
 
786}
787
788int
789xfs_trans_ail_init(
790	xfs_mount_t	*mp)
791{
792	struct xfs_ail	*ailp;
793
794	ailp = kmem_zalloc(sizeof(struct xfs_ail), KM_MAYFAIL);
795	if (!ailp)
796		return -ENOMEM;
797
798	ailp->ail_mount = mp;
799	INIT_LIST_HEAD(&ailp->ail_head);
800	INIT_LIST_HEAD(&ailp->ail_cursors);
801	spin_lock_init(&ailp->ail_lock);
802	INIT_LIST_HEAD(&ailp->ail_buf_list);
803	init_waitqueue_head(&ailp->ail_empty);
804
805	ailp->ail_task = kthread_run(xfsaild, ailp, "xfsaild/%s",
806			ailp->ail_mount->m_fsname);
807	if (IS_ERR(ailp->ail_task))
808		goto out_free_ailp;
809
810	mp->m_ail = ailp;
811	return 0;
812
813out_free_ailp:
814	kmem_free(ailp);
815	return -ENOMEM;
816}
817
818void
819xfs_trans_ail_destroy(
820	xfs_mount_t	*mp)
821{
822	struct xfs_ail	*ailp = mp->m_ail;
823
824	kthread_stop(ailp->ail_task);
825	kmem_free(ailp);
826}
v3.15
  1/*
  2 * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc.
  3 * Copyright (c) 2008 Dave Chinner
  4 * All Rights Reserved.
  5 *
  6 * This program is free software; you can redistribute it and/or
  7 * modify it under the terms of the GNU General Public License as
  8 * published by the Free Software Foundation.
  9 *
 10 * This program is distributed in the hope that it would be useful,
 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13 * GNU General Public License for more details.
 14 *
 15 * You should have received a copy of the GNU General Public License
 16 * along with this program; if not, write the Free Software Foundation,
 17 * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 18 */
 19#include "xfs.h"
 20#include "xfs_fs.h"
 
 21#include "xfs_log_format.h"
 22#include "xfs_trans_resv.h"
 23#include "xfs_sb.h"
 24#include "xfs_ag.h"
 25#include "xfs_mount.h"
 26#include "xfs_trans.h"
 27#include "xfs_trans_priv.h"
 28#include "xfs_trace.h"
 
 29#include "xfs_error.h"
 30#include "xfs_log.h"
 31
 32#ifdef DEBUG
 33/*
 34 * Check that the list is sorted as it should be.
 35 */
 36STATIC void
 37xfs_ail_check(
 38	struct xfs_ail	*ailp,
 39	xfs_log_item_t	*lip)
 40{
 41	xfs_log_item_t	*prev_lip;
 42
 43	if (list_empty(&ailp->xa_ail))
 44		return;
 45
 46	/*
 47	 * Check the next and previous entries are valid.
 48	 */
 49	ASSERT((lip->li_flags & XFS_LI_IN_AIL) != 0);
 50	prev_lip = list_entry(lip->li_ail.prev, xfs_log_item_t, li_ail);
 51	if (&prev_lip->li_ail != &ailp->xa_ail)
 52		ASSERT(XFS_LSN_CMP(prev_lip->li_lsn, lip->li_lsn) <= 0);
 53
 54	prev_lip = list_entry(lip->li_ail.next, xfs_log_item_t, li_ail);
 55	if (&prev_lip->li_ail != &ailp->xa_ail)
 56		ASSERT(XFS_LSN_CMP(prev_lip->li_lsn, lip->li_lsn) >= 0);
 57
 58
 59}
 60#else /* !DEBUG */
 61#define	xfs_ail_check(a,l)
 62#endif /* DEBUG */
 63
 64/*
 65 * Return a pointer to the last item in the AIL.  If the AIL is empty, then
 66 * return NULL.
 67 */
 68static xfs_log_item_t *
 69xfs_ail_max(
 70	struct xfs_ail  *ailp)
 71{
 72	if (list_empty(&ailp->xa_ail))
 73		return NULL;
 74
 75	return list_entry(ailp->xa_ail.prev, xfs_log_item_t, li_ail);
 76}
 77
 78/*
 79 * Return a pointer to the item which follows the given item in the AIL.  If
 80 * the given item is the last item in the list, then return NULL.
 81 */
 82static xfs_log_item_t *
 83xfs_ail_next(
 84	struct xfs_ail  *ailp,
 85	xfs_log_item_t  *lip)
 86{
 87	if (lip->li_ail.next == &ailp->xa_ail)
 88		return NULL;
 89
 90	return list_first_entry(&lip->li_ail, xfs_log_item_t, li_ail);
 91}
 92
 93/*
 94 * This is called by the log manager code to determine the LSN of the tail of
 95 * the log.  This is exactly the LSN of the first item in the AIL.  If the AIL
 96 * is empty, then this function returns 0.
 97 *
 98 * We need the AIL lock in order to get a coherent read of the lsn of the last
 99 * item in the AIL.
100 */
101xfs_lsn_t
102xfs_ail_min_lsn(
103	struct xfs_ail	*ailp)
104{
105	xfs_lsn_t	lsn = 0;
106	xfs_log_item_t	*lip;
107
108	spin_lock(&ailp->xa_lock);
109	lip = xfs_ail_min(ailp);
110	if (lip)
111		lsn = lip->li_lsn;
112	spin_unlock(&ailp->xa_lock);
113
114	return lsn;
115}
116
117/*
118 * Return the maximum lsn held in the AIL, or zero if the AIL is empty.
119 */
120static xfs_lsn_t
121xfs_ail_max_lsn(
122	struct xfs_ail  *ailp)
123{
124	xfs_lsn_t       lsn = 0;
125	xfs_log_item_t  *lip;
126
127	spin_lock(&ailp->xa_lock);
128	lip = xfs_ail_max(ailp);
129	if (lip)
130		lsn = lip->li_lsn;
131	spin_unlock(&ailp->xa_lock);
132
133	return lsn;
134}
135
136/*
137 * The cursor keeps track of where our current traversal is up to by tracking
138 * the next item in the list for us. However, for this to be safe, removing an
139 * object from the AIL needs to invalidate any cursor that points to it. hence
140 * the traversal cursor needs to be linked to the struct xfs_ail so that
141 * deletion can search all the active cursors for invalidation.
142 */
143STATIC void
144xfs_trans_ail_cursor_init(
145	struct xfs_ail		*ailp,
146	struct xfs_ail_cursor	*cur)
147{
148	cur->item = NULL;
149	list_add_tail(&cur->list, &ailp->xa_cursors);
150}
151
152/*
153 * Get the next item in the traversal and advance the cursor.  If the cursor
154 * was invalidated (indicated by a lip of 1), restart the traversal.
155 */
156struct xfs_log_item *
157xfs_trans_ail_cursor_next(
158	struct xfs_ail		*ailp,
159	struct xfs_ail_cursor	*cur)
160{
161	struct xfs_log_item	*lip = cur->item;
162
163	if ((__psint_t)lip & 1)
164		lip = xfs_ail_min(ailp);
165	if (lip)
166		cur->item = xfs_ail_next(ailp, lip);
167	return lip;
168}
169
170/*
171 * When the traversal is complete, we need to remove the cursor from the list
172 * of traversing cursors.
173 */
174void
175xfs_trans_ail_cursor_done(
176	struct xfs_ail		*ailp,
177	struct xfs_ail_cursor	*cur)
178{
179	cur->item = NULL;
180	list_del_init(&cur->list);
181}
182
183/*
184 * Invalidate any cursor that is pointing to this item. This is called when an
185 * item is removed from the AIL. Any cursor pointing to this object is now
186 * invalid and the traversal needs to be terminated so it doesn't reference a
187 * freed object. We set the low bit of the cursor item pointer so we can
188 * distinguish between an invalidation and the end of the list when getting the
189 * next item from the cursor.
190 */
191STATIC void
192xfs_trans_ail_cursor_clear(
193	struct xfs_ail		*ailp,
194	struct xfs_log_item	*lip)
195{
196	struct xfs_ail_cursor	*cur;
197
198	list_for_each_entry(cur, &ailp->xa_cursors, list) {
199		if (cur->item == lip)
200			cur->item = (struct xfs_log_item *)
201					((__psint_t)cur->item | 1);
202	}
203}
204
205/*
206 * Find the first item in the AIL with the given @lsn by searching in ascending
207 * LSN order and initialise the cursor to point to the next item for a
208 * ascending traversal.  Pass a @lsn of zero to initialise the cursor to the
209 * first item in the AIL. Returns NULL if the list is empty.
210 */
211xfs_log_item_t *
212xfs_trans_ail_cursor_first(
213	struct xfs_ail		*ailp,
214	struct xfs_ail_cursor	*cur,
215	xfs_lsn_t		lsn)
216{
217	xfs_log_item_t		*lip;
218
219	xfs_trans_ail_cursor_init(ailp, cur);
220
221	if (lsn == 0) {
222		lip = xfs_ail_min(ailp);
223		goto out;
224	}
225
226	list_for_each_entry(lip, &ailp->xa_ail, li_ail) {
227		if (XFS_LSN_CMP(lip->li_lsn, lsn) >= 0)
228			goto out;
229	}
230	return NULL;
231
232out:
233	if (lip)
234		cur->item = xfs_ail_next(ailp, lip);
235	return lip;
236}
237
238static struct xfs_log_item *
239__xfs_trans_ail_cursor_last(
240	struct xfs_ail		*ailp,
241	xfs_lsn_t		lsn)
242{
243	xfs_log_item_t		*lip;
244
245	list_for_each_entry_reverse(lip, &ailp->xa_ail, li_ail) {
246		if (XFS_LSN_CMP(lip->li_lsn, lsn) <= 0)
247			return lip;
248	}
249	return NULL;
250}
251
252/*
253 * Find the last item in the AIL with the given @lsn by searching in descending
254 * LSN order and initialise the cursor to point to that item.  If there is no
255 * item with the value of @lsn, then it sets the cursor to the last item with an
256 * LSN lower than @lsn.  Returns NULL if the list is empty.
257 */
258struct xfs_log_item *
259xfs_trans_ail_cursor_last(
260	struct xfs_ail		*ailp,
261	struct xfs_ail_cursor	*cur,
262	xfs_lsn_t		lsn)
263{
264	xfs_trans_ail_cursor_init(ailp, cur);
265	cur->item = __xfs_trans_ail_cursor_last(ailp, lsn);
266	return cur->item;
267}
268
269/*
270 * Splice the log item list into the AIL at the given LSN. We splice to the
271 * tail of the given LSN to maintain insert order for push traversals. The
272 * cursor is optional, allowing repeated updates to the same LSN to avoid
273 * repeated traversals.  This should not be called with an empty list.
274 */
275static void
276xfs_ail_splice(
277	struct xfs_ail		*ailp,
278	struct xfs_ail_cursor	*cur,
279	struct list_head	*list,
280	xfs_lsn_t		lsn)
281{
282	struct xfs_log_item	*lip;
283
284	ASSERT(!list_empty(list));
285
286	/*
287	 * Use the cursor to determine the insertion point if one is
288	 * provided.  If not, or if the one we got is not valid,
289	 * find the place in the AIL where the items belong.
290	 */
291	lip = cur ? cur->item : NULL;
292	if (!lip || (__psint_t) lip & 1)
293		lip = __xfs_trans_ail_cursor_last(ailp, lsn);
294
295	/*
296	 * If a cursor is provided, we know we're processing the AIL
297	 * in lsn order, and future items to be spliced in will
298	 * follow the last one being inserted now.  Update the
299	 * cursor to point to that last item, now while we have a
300	 * reliable pointer to it.
301	 */
302	if (cur)
303		cur->item = list_entry(list->prev, struct xfs_log_item, li_ail);
304
305	/*
306	 * Finally perform the splice.  Unless the AIL was empty,
307	 * lip points to the item in the AIL _after_ which the new
308	 * items should go.  If lip is null the AIL was empty, so
309	 * the new items go at the head of the AIL.
310	 */
311	if (lip)
312		list_splice(list, &lip->li_ail);
313	else
314		list_splice(list, &ailp->xa_ail);
315}
316
317/*
318 * Delete the given item from the AIL.  Return a pointer to the item.
319 */
320static void
321xfs_ail_delete(
322	struct xfs_ail  *ailp,
323	xfs_log_item_t  *lip)
324{
325	xfs_ail_check(ailp, lip);
326	list_del(&lip->li_ail);
327	xfs_trans_ail_cursor_clear(ailp, lip);
328}
329
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
330static long
331xfsaild_push(
332	struct xfs_ail		*ailp)
333{
334	xfs_mount_t		*mp = ailp->xa_mount;
335	struct xfs_ail_cursor	cur;
336	xfs_log_item_t		*lip;
337	xfs_lsn_t		lsn;
338	xfs_lsn_t		target;
339	long			tout;
340	int			stuck = 0;
341	int			flushing = 0;
342	int			count = 0;
343
344	/*
345	 * If we encountered pinned items or did not finish writing out all
346	 * buffers the last time we ran, force the log first and wait for it
347	 * before pushing again.
348	 */
349	if (ailp->xa_log_flush && ailp->xa_last_pushed_lsn == 0 &&
350	    (!list_empty_careful(&ailp->xa_buf_list) ||
351	     xfs_ail_min_lsn(ailp))) {
352		ailp->xa_log_flush = 0;
353
354		XFS_STATS_INC(xs_push_ail_flush);
355		xfs_log_force(mp, XFS_LOG_SYNC);
356	}
357
358	spin_lock(&ailp->xa_lock);
359
360	/* barrier matches the xa_target update in xfs_ail_push() */
361	smp_rmb();
362	target = ailp->xa_target;
363	ailp->xa_target_prev = target;
364
365	lip = xfs_trans_ail_cursor_first(ailp, &cur, ailp->xa_last_pushed_lsn);
366	if (!lip) {
367		/*
368		 * If the AIL is empty or our push has reached the end we are
369		 * done now.
370		 */
371		xfs_trans_ail_cursor_done(ailp, &cur);
372		spin_unlock(&ailp->xa_lock);
373		goto out_done;
374	}
375
376	XFS_STATS_INC(xs_push_ail);
377
378	lsn = lip->li_lsn;
379	while ((XFS_LSN_CMP(lip->li_lsn, target) <= 0)) {
380		int	lock_result;
381
382		/*
383		 * Note that iop_push may unlock and reacquire the AIL lock.  We
384		 * rely on the AIL cursor implementation to be able to deal with
385		 * the dropped lock.
386		 */
387		lock_result = lip->li_ops->iop_push(lip, &ailp->xa_buf_list);
388		switch (lock_result) {
389		case XFS_ITEM_SUCCESS:
390			XFS_STATS_INC(xs_push_ail_success);
391			trace_xfs_ail_push(lip);
392
393			ailp->xa_last_pushed_lsn = lsn;
394			break;
395
396		case XFS_ITEM_FLUSHING:
397			/*
398			 * The item or its backing buffer is already beeing
399			 * flushed.  The typical reason for that is that an
400			 * inode buffer is locked because we already pushed the
401			 * updates to it as part of inode clustering.
402			 *
403			 * We do not want to to stop flushing just because lots
404			 * of items are already beeing flushed, but we need to
405			 * re-try the flushing relatively soon if most of the
406			 * AIL is beeing flushed.
407			 */
408			XFS_STATS_INC(xs_push_ail_flushing);
409			trace_xfs_ail_flushing(lip);
410
411			flushing++;
412			ailp->xa_last_pushed_lsn = lsn;
413			break;
414
415		case XFS_ITEM_PINNED:
416			XFS_STATS_INC(xs_push_ail_pinned);
417			trace_xfs_ail_pinned(lip);
418
419			stuck++;
420			ailp->xa_log_flush++;
421			break;
422		case XFS_ITEM_LOCKED:
423			XFS_STATS_INC(xs_push_ail_locked);
424			trace_xfs_ail_locked(lip);
425
426			stuck++;
427			break;
428		default:
429			ASSERT(0);
430			break;
431		}
432
433		count++;
434
435		/*
436		 * Are there too many items we can't do anything with?
437		 *
438		 * If we we are skipping too many items because we can't flush
439		 * them or they are already being flushed, we back off and
440		 * given them time to complete whatever operation is being
441		 * done. i.e. remove pressure from the AIL while we can't make
442		 * progress so traversals don't slow down further inserts and
443		 * removals to/from the AIL.
444		 *
445		 * The value of 100 is an arbitrary magic number based on
446		 * observation.
447		 */
448		if (stuck > 100)
449			break;
450
451		lip = xfs_trans_ail_cursor_next(ailp, &cur);
452		if (lip == NULL)
453			break;
454		lsn = lip->li_lsn;
455	}
456	xfs_trans_ail_cursor_done(ailp, &cur);
457	spin_unlock(&ailp->xa_lock);
458
459	if (xfs_buf_delwri_submit_nowait(&ailp->xa_buf_list))
460		ailp->xa_log_flush++;
461
462	if (!count || XFS_LSN_CMP(lsn, target) >= 0) {
463out_done:
464		/*
465		 * We reached the target or the AIL is empty, so wait a bit
466		 * longer for I/O to complete and remove pushed items from the
467		 * AIL before we start the next scan from the start of the AIL.
468		 */
469		tout = 50;
470		ailp->xa_last_pushed_lsn = 0;
471	} else if (((stuck + flushing) * 100) / count > 90) {
472		/*
473		 * Either there is a lot of contention on the AIL or we are
474		 * stuck due to operations in progress. "Stuck" in this case
475		 * is defined as >90% of the items we tried to push were stuck.
476		 *
477		 * Backoff a bit more to allow some I/O to complete before
478		 * restarting from the start of the AIL. This prevents us from
479		 * spinning on the same items, and if they are pinned will all
480		 * the restart to issue a log force to unpin the stuck items.
481		 */
482		tout = 20;
483		ailp->xa_last_pushed_lsn = 0;
484	} else {
485		/*
486		 * Assume we have more work to do in a short while.
487		 */
488		tout = 10;
489	}
490
491	return tout;
492}
493
494static int
495xfsaild(
496	void		*data)
497{
498	struct xfs_ail	*ailp = data;
499	long		tout = 0;	/* milliseconds */
500
501	current->flags |= PF_MEMALLOC;
 
502
503	while (!kthread_should_stop()) {
504		if (tout && tout <= 20)
505			__set_current_state(TASK_KILLABLE);
506		else
507			__set_current_state(TASK_INTERRUPTIBLE);
508
509		spin_lock(&ailp->xa_lock);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
510
511		/*
512		 * Idle if the AIL is empty and we are not racing with a target
513		 * update. We check the AIL after we set the task to a sleep
514		 * state to guarantee that we either catch an xa_target update
515		 * or that a wake_up resets the state to TASK_RUNNING.
516		 * Otherwise, we run the risk of sleeping indefinitely.
517		 *
518		 * The barrier matches the xa_target update in xfs_ail_push().
519		 */
520		smp_rmb();
521		if (!xfs_ail_min(ailp) &&
522		    ailp->xa_target == ailp->xa_target_prev) {
523			spin_unlock(&ailp->xa_lock);
524			schedule();
525			tout = 0;
526			continue;
527		}
528		spin_unlock(&ailp->xa_lock);
529
530		if (tout)
531			schedule_timeout(msecs_to_jiffies(tout));
532
533		__set_current_state(TASK_RUNNING);
534
535		try_to_freeze();
536
537		tout = xfsaild_push(ailp);
538	}
539
540	return 0;
541}
542
543/*
544 * This routine is called to move the tail of the AIL forward.  It does this by
545 * trying to flush items in the AIL whose lsns are below the given
546 * threshold_lsn.
547 *
548 * The push is run asynchronously in a workqueue, which means the caller needs
549 * to handle waiting on the async flush for space to become available.
550 * We don't want to interrupt any push that is in progress, hence we only queue
551 * work if we set the pushing bit approriately.
552 *
553 * We do this unlocked - we only need to know whether there is anything in the
554 * AIL at the time we are called. We don't need to access the contents of
555 * any of the objects, so the lock is not needed.
556 */
557void
558xfs_ail_push(
559	struct xfs_ail	*ailp,
560	xfs_lsn_t	threshold_lsn)
561{
562	xfs_log_item_t	*lip;
563
564	lip = xfs_ail_min(ailp);
565	if (!lip || XFS_FORCED_SHUTDOWN(ailp->xa_mount) ||
566	    XFS_LSN_CMP(threshold_lsn, ailp->xa_target) <= 0)
567		return;
568
569	/*
570	 * Ensure that the new target is noticed in push code before it clears
571	 * the XFS_AIL_PUSHING_BIT.
572	 */
573	smp_wmb();
574	xfs_trans_ail_copy_lsn(ailp, &ailp->xa_target, &threshold_lsn);
575	smp_wmb();
576
577	wake_up_process(ailp->xa_task);
578}
579
580/*
581 * Push out all items in the AIL immediately
582 */
583void
584xfs_ail_push_all(
585	struct xfs_ail  *ailp)
586{
587	xfs_lsn_t       threshold_lsn = xfs_ail_max_lsn(ailp);
588
589	if (threshold_lsn)
590		xfs_ail_push(ailp, threshold_lsn);
591}
592
593/*
594 * Push out all items in the AIL immediately and wait until the AIL is empty.
595 */
596void
597xfs_ail_push_all_sync(
598	struct xfs_ail  *ailp)
599{
600	struct xfs_log_item	*lip;
601	DEFINE_WAIT(wait);
602
603	spin_lock(&ailp->xa_lock);
604	while ((lip = xfs_ail_max(ailp)) != NULL) {
605		prepare_to_wait(&ailp->xa_empty, &wait, TASK_UNINTERRUPTIBLE);
606		ailp->xa_target = lip->li_lsn;
607		wake_up_process(ailp->xa_task);
608		spin_unlock(&ailp->xa_lock);
609		schedule();
610		spin_lock(&ailp->xa_lock);
611	}
612	spin_unlock(&ailp->xa_lock);
613
614	finish_wait(&ailp->xa_empty, &wait);
615}
616
617/*
618 * xfs_trans_ail_update - bulk AIL insertion operation.
619 *
620 * @xfs_trans_ail_update takes an array of log items that all need to be
621 * positioned at the same LSN in the AIL. If an item is not in the AIL, it will
622 * be added.  Otherwise, it will be repositioned  by removing it and re-adding
623 * it to the AIL. If we move the first item in the AIL, update the log tail to
624 * match the new minimum LSN in the AIL.
625 *
626 * This function takes the AIL lock once to execute the update operations on
627 * all the items in the array, and as such should not be called with the AIL
628 * lock held. As a result, once we have the AIL lock, we need to check each log
629 * item LSN to confirm it needs to be moved forward in the AIL.
630 *
631 * To optimise the insert operation, we delete all the items from the AIL in
632 * the first pass, moving them into a temporary list, then splice the temporary
633 * list into the correct position in the AIL. This avoids needing to do an
634 * insert operation on every item.
635 *
636 * This function must be called with the AIL lock held.  The lock is dropped
637 * before returning.
638 */
639void
640xfs_trans_ail_update_bulk(
641	struct xfs_ail		*ailp,
642	struct xfs_ail_cursor	*cur,
643	struct xfs_log_item	**log_items,
644	int			nr_items,
645	xfs_lsn_t		lsn) __releases(ailp->xa_lock)
646{
647	xfs_log_item_t		*mlip;
648	int			mlip_changed = 0;
649	int			i;
650	LIST_HEAD(tmp);
651
652	ASSERT(nr_items > 0);		/* Not required, but true. */
653	mlip = xfs_ail_min(ailp);
654
655	for (i = 0; i < nr_items; i++) {
656		struct xfs_log_item *lip = log_items[i];
657		if (lip->li_flags & XFS_LI_IN_AIL) {
658			/* check if we really need to move the item */
659			if (XFS_LSN_CMP(lsn, lip->li_lsn) <= 0)
660				continue;
661
662			trace_xfs_ail_move(lip, lip->li_lsn, lsn);
663			xfs_ail_delete(ailp, lip);
664			if (mlip == lip)
665				mlip_changed = 1;
666		} else {
667			lip->li_flags |= XFS_LI_IN_AIL;
668			trace_xfs_ail_insert(lip, 0, lsn);
669		}
670		lip->li_lsn = lsn;
671		list_add(&lip->li_ail, &tmp);
672	}
673
674	if (!list_empty(&tmp))
675		xfs_ail_splice(ailp, cur, &tmp, lsn);
676
677	if (mlip_changed) {
678		if (!XFS_FORCED_SHUTDOWN(ailp->xa_mount))
679			xlog_assign_tail_lsn_locked(ailp->xa_mount);
680		spin_unlock(&ailp->xa_lock);
681
682		xfs_log_space_wake(ailp->xa_mount);
683	} else {
684		spin_unlock(&ailp->xa_lock);
685	}
686}
687
688/*
689 * xfs_trans_ail_delete_bulk - remove multiple log items from the AIL
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
690 *
691 * @xfs_trans_ail_delete_bulk takes an array of log items that all need to
692 * removed from the AIL. The caller is already holding the AIL lock, and done
693 * all the checks necessary to ensure the items passed in via @log_items are
694 * ready for deletion. This includes checking that the items are in the AIL.
695 *
696 * For each log item to be removed, unlink it  from the AIL, clear the IN_AIL
697 * flag from the item and reset the item's lsn to 0. If we remove the first
698 * item in the AIL, update the log tail to match the new minimum LSN in the
699 * AIL.
700 *
701 * This function will not drop the AIL lock until all items are removed from
702 * the AIL to minimise the amount of lock traffic on the AIL. This does not
703 * greatly increase the AIL hold time, but does significantly reduce the amount
704 * of traffic on the lock, especially during IO completion.
705 *
706 * This function must be called with the AIL lock held.  The lock is dropped
707 * before returning.
708 */
709void
710xfs_trans_ail_delete_bulk(
711	struct xfs_ail		*ailp,
712	struct xfs_log_item	**log_items,
713	int			nr_items,
714	int			shutdown_type) __releases(ailp->xa_lock)
715{
716	xfs_log_item_t		*mlip;
717	int			mlip_changed = 0;
718	int			i;
719
720	mlip = xfs_ail_min(ailp);
721
722	for (i = 0; i < nr_items; i++) {
723		struct xfs_log_item *lip = log_items[i];
724		if (!(lip->li_flags & XFS_LI_IN_AIL)) {
725			struct xfs_mount	*mp = ailp->xa_mount;
726
727			spin_unlock(&ailp->xa_lock);
728			if (!XFS_FORCED_SHUTDOWN(mp)) {
729				xfs_alert_tag(mp, XFS_PTAG_AILDELETE,
730		"%s: attempting to delete a log item that is not in the AIL",
731						__func__);
732				xfs_force_shutdown(mp, shutdown_type);
733			}
734			return;
735		}
736
737		trace_xfs_ail_delete(lip, mlip->li_lsn, lip->li_lsn);
738		xfs_ail_delete(ailp, lip);
739		lip->li_flags &= ~XFS_LI_IN_AIL;
740		lip->li_lsn = 0;
741		if (mlip == lip)
742			mlip_changed = 1;
743	}
744
 
745	if (mlip_changed) {
746		if (!XFS_FORCED_SHUTDOWN(ailp->xa_mount))
747			xlog_assign_tail_lsn_locked(ailp->xa_mount);
748		if (list_empty(&ailp->xa_ail))
749			wake_up_all(&ailp->xa_empty);
750		spin_unlock(&ailp->xa_lock);
751
752		xfs_log_space_wake(ailp->xa_mount);
753	} else {
754		spin_unlock(&ailp->xa_lock);
755	}
756}
757
758int
759xfs_trans_ail_init(
760	xfs_mount_t	*mp)
761{
762	struct xfs_ail	*ailp;
763
764	ailp = kmem_zalloc(sizeof(struct xfs_ail), KM_MAYFAIL);
765	if (!ailp)
766		return ENOMEM;
767
768	ailp->xa_mount = mp;
769	INIT_LIST_HEAD(&ailp->xa_ail);
770	INIT_LIST_HEAD(&ailp->xa_cursors);
771	spin_lock_init(&ailp->xa_lock);
772	INIT_LIST_HEAD(&ailp->xa_buf_list);
773	init_waitqueue_head(&ailp->xa_empty);
774
775	ailp->xa_task = kthread_run(xfsaild, ailp, "xfsaild/%s",
776			ailp->xa_mount->m_fsname);
777	if (IS_ERR(ailp->xa_task))
778		goto out_free_ailp;
779
780	mp->m_ail = ailp;
781	return 0;
782
783out_free_ailp:
784	kmem_free(ailp);
785	return ENOMEM;
786}
787
788void
789xfs_trans_ail_destroy(
790	xfs_mount_t	*mp)
791{
792	struct xfs_ail	*ailp = mp->m_ail;
793
794	kthread_stop(ailp->xa_task);
795	kmem_free(ailp);
796}