Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* netfs cookie management
3 *
4 * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 *
7 * See Documentation/filesystems/caching/netfs-api.rst for more information on
8 * the netfs API.
9 */
10
11#define FSCACHE_DEBUG_LEVEL COOKIE
12#include <linux/module.h>
13#include <linux/slab.h>
14#include "internal.h"
15
16struct kmem_cache *fscache_cookie_jar;
17
18static void fscache_cookie_lru_timed_out(struct timer_list *timer);
19static void fscache_cookie_lru_worker(struct work_struct *work);
20static void fscache_cookie_worker(struct work_struct *work);
21static void fscache_unhash_cookie(struct fscache_cookie *cookie);
22static void fscache_perform_invalidation(struct fscache_cookie *cookie);
23
24#define fscache_cookie_hash_shift 15
25static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift];
26static LIST_HEAD(fscache_cookies);
27static DEFINE_RWLOCK(fscache_cookies_lock);
28static LIST_HEAD(fscache_cookie_lru);
29static DEFINE_SPINLOCK(fscache_cookie_lru_lock);
30DEFINE_TIMER(fscache_cookie_lru_timer, fscache_cookie_lru_timed_out);
31static DECLARE_WORK(fscache_cookie_lru_work, fscache_cookie_lru_worker);
32static const char fscache_cookie_states[FSCACHE_COOKIE_STATE__NR] = "-LCAIFUWRD";
33static unsigned int fscache_lru_cookie_timeout = 10 * HZ;
34
35void fscache_print_cookie(struct fscache_cookie *cookie, char prefix)
36{
37 const u8 *k;
38
39 pr_err("%c-cookie c=%08x [fl=%lx na=%u nA=%u s=%c]\n",
40 prefix,
41 cookie->debug_id,
42 cookie->flags,
43 atomic_read(&cookie->n_active),
44 atomic_read(&cookie->n_accesses),
45 fscache_cookie_states[cookie->state]);
46 pr_err("%c-cookie V=%08x [%s]\n",
47 prefix,
48 cookie->volume->debug_id,
49 cookie->volume->key);
50
51 k = (cookie->key_len <= sizeof(cookie->inline_key)) ?
52 cookie->inline_key : cookie->key;
53 pr_err("%c-key=[%u] '%*phN'\n", prefix, cookie->key_len, cookie->key_len, k);
54}
55
56static void fscache_free_cookie(struct fscache_cookie *cookie)
57{
58 if (WARN_ON_ONCE(!list_empty(&cookie->commit_link))) {
59 spin_lock(&fscache_cookie_lru_lock);
60 list_del_init(&cookie->commit_link);
61 spin_unlock(&fscache_cookie_lru_lock);
62 fscache_stat_d(&fscache_n_cookies_lru);
63 fscache_stat(&fscache_n_cookies_lru_removed);
64 }
65
66 if (WARN_ON_ONCE(test_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags))) {
67 fscache_print_cookie(cookie, 'F');
68 return;
69 }
70
71 write_lock(&fscache_cookies_lock);
72 list_del(&cookie->proc_link);
73 write_unlock(&fscache_cookies_lock);
74 if (cookie->aux_len > sizeof(cookie->inline_aux))
75 kfree(cookie->aux);
76 if (cookie->key_len > sizeof(cookie->inline_key))
77 kfree(cookie->key);
78 fscache_stat_d(&fscache_n_cookies);
79 kmem_cache_free(fscache_cookie_jar, cookie);
80}
81
82static void __fscache_queue_cookie(struct fscache_cookie *cookie)
83{
84 if (!queue_work(fscache_wq, &cookie->work))
85 fscache_put_cookie(cookie, fscache_cookie_put_over_queued);
86}
87
88static void fscache_queue_cookie(struct fscache_cookie *cookie,
89 enum fscache_cookie_trace where)
90{
91 fscache_get_cookie(cookie, where);
92 __fscache_queue_cookie(cookie);
93}
94
95/*
96 * Initialise the access gate on a cookie by setting a flag to prevent the
97 * state machine from being queued when the access counter transitions to 0.
98 * We're only interested in this when we withdraw caching services from the
99 * cookie.
100 */
101static void fscache_init_access_gate(struct fscache_cookie *cookie)
102{
103 int n_accesses;
104
105 n_accesses = atomic_read(&cookie->n_accesses);
106 trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref),
107 n_accesses, fscache_access_cache_pin);
108 set_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags);
109}
110
111/**
112 * fscache_end_cookie_access - Unpin a cache at the end of an access.
113 * @cookie: A data file cookie
114 * @why: An indication of the circumstances of the access for tracing
115 *
116 * Unpin a cache cookie after we've accessed it and bring a deferred
117 * relinquishment or withdrawal state into effect.
118 *
119 * The @why indicator is provided for tracing purposes.
120 */
121void fscache_end_cookie_access(struct fscache_cookie *cookie,
122 enum fscache_access_trace why)
123{
124 int n_accesses;
125
126 smp_mb__before_atomic();
127 n_accesses = atomic_dec_return(&cookie->n_accesses);
128 trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref),
129 n_accesses, why);
130 if (n_accesses == 0 &&
131 !test_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags))
132 fscache_queue_cookie(cookie, fscache_cookie_get_end_access);
133}
134EXPORT_SYMBOL(fscache_end_cookie_access);
135
136/*
137 * Pin the cache behind a cookie so that we can access it.
138 */
139static void __fscache_begin_cookie_access(struct fscache_cookie *cookie,
140 enum fscache_access_trace why)
141{
142 int n_accesses;
143
144 n_accesses = atomic_inc_return(&cookie->n_accesses);
145 smp_mb__after_atomic(); /* (Future) read state after is-caching.
146 * Reread n_accesses after is-caching
147 */
148 trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref),
149 n_accesses, why);
150}
151
152/**
153 * fscache_begin_cookie_access - Pin a cache so data can be accessed
154 * @cookie: A data file cookie
155 * @why: An indication of the circumstances of the access for tracing
156 *
157 * Attempt to pin the cache to prevent it from going away whilst we're
158 * accessing data and returns true if successful. This works as follows:
159 *
160 * (1) If the cookie is not being cached (ie. FSCACHE_COOKIE_IS_CACHING is not
161 * set), we return false to indicate access was not permitted.
162 *
163 * (2) If the cookie is being cached, we increment its n_accesses count and
164 * then recheck the IS_CACHING flag, ending the access if it got cleared.
165 *
166 * (3) When we end the access, we decrement the cookie's n_accesses and wake
167 * up the any waiters if it reaches 0.
168 *
169 * (4) Whilst the cookie is actively being cached, its n_accesses is kept
170 * artificially incremented to prevent wakeups from happening.
171 *
172 * (5) When the cache is taken offline or if the cookie is culled, the flag is
173 * cleared to prevent new accesses, the cookie's n_accesses is decremented
174 * and we wait for it to become 0.
175 *
176 * The @why indicator are merely provided for tracing purposes.
177 */
178bool fscache_begin_cookie_access(struct fscache_cookie *cookie,
179 enum fscache_access_trace why)
180{
181 if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags))
182 return false;
183 __fscache_begin_cookie_access(cookie, why);
184 if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags) ||
185 !fscache_cache_is_live(cookie->volume->cache)) {
186 fscache_end_cookie_access(cookie, fscache_access_unlive);
187 return false;
188 }
189 return true;
190}
191
192static inline void wake_up_cookie_state(struct fscache_cookie *cookie)
193{
194 /* Use a barrier to ensure that waiters see the state variable
195 * change, as spin_unlock doesn't guarantee a barrier.
196 *
197 * See comments over wake_up_bit() and waitqueue_active().
198 */
199 smp_mb();
200 wake_up_var(&cookie->state);
201}
202
203/*
204 * Change the state a cookie is at and wake up anyone waiting for that. Impose
205 * an ordering between the stuff stored in the cookie and the state member.
206 * Paired with fscache_cookie_state().
207 */
208static void __fscache_set_cookie_state(struct fscache_cookie *cookie,
209 enum fscache_cookie_state state)
210{
211 smp_store_release(&cookie->state, state);
212}
213
214static void fscache_set_cookie_state(struct fscache_cookie *cookie,
215 enum fscache_cookie_state state)
216{
217 spin_lock(&cookie->lock);
218 __fscache_set_cookie_state(cookie, state);
219 spin_unlock(&cookie->lock);
220 wake_up_cookie_state(cookie);
221}
222
223/**
224 * fscache_cookie_lookup_negative - Note negative lookup
225 * @cookie: The cookie that was being looked up
226 *
227 * Note that some part of the metadata path in the cache doesn't exist and so
228 * we can release any waiting readers in the certain knowledge that there's
229 * nothing for them to actually read.
230 *
231 * This function uses no locking and must only be called from the state machine.
232 */
233void fscache_cookie_lookup_negative(struct fscache_cookie *cookie)
234{
235 set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);
236 fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_CREATING);
237}
238EXPORT_SYMBOL(fscache_cookie_lookup_negative);
239
240/**
241 * fscache_resume_after_invalidation - Allow I/O to resume after invalidation
242 * @cookie: The cookie that was invalidated
243 *
244 * Tell fscache that invalidation is sufficiently complete that I/O can be
245 * allowed again.
246 */
247void fscache_resume_after_invalidation(struct fscache_cookie *cookie)
248{
249 fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE);
250}
251EXPORT_SYMBOL(fscache_resume_after_invalidation);
252
253/**
254 * fscache_caching_failed - Report that a failure stopped caching on a cookie
255 * @cookie: The cookie that was affected
256 *
257 * Tell fscache that caching on a cookie needs to be stopped due to some sort
258 * of failure.
259 *
260 * This function uses no locking and must only be called from the state machine.
261 */
262void fscache_caching_failed(struct fscache_cookie *cookie)
263{
264 clear_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags);
265 fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_FAILED);
266 trace_fscache_cookie(cookie->debug_id, refcount_read(&cookie->ref),
267 fscache_cookie_failed);
268}
269EXPORT_SYMBOL(fscache_caching_failed);
270
271/*
272 * Set the index key in a cookie. The cookie struct has space for a 16-byte
273 * key plus length and hash, but if that's not big enough, it's instead a
274 * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then
275 * the key data.
276 */
277static int fscache_set_key(struct fscache_cookie *cookie,
278 const void *index_key, size_t index_key_len)
279{
280 void *buf;
281 size_t buf_size;
282
283 buf_size = round_up(index_key_len, sizeof(__le32));
284
285 if (index_key_len > sizeof(cookie->inline_key)) {
286 buf = kzalloc(buf_size, GFP_KERNEL);
287 if (!buf)
288 return -ENOMEM;
289 cookie->key = buf;
290 } else {
291 buf = cookie->inline_key;
292 }
293
294 memcpy(buf, index_key, index_key_len);
295 cookie->key_hash = fscache_hash(cookie->volume->key_hash,
296 buf, buf_size);
297 return 0;
298}
299
300static bool fscache_cookie_same(const struct fscache_cookie *a,
301 const struct fscache_cookie *b)
302{
303 const void *ka, *kb;
304
305 if (a->key_hash != b->key_hash ||
306 a->volume != b->volume ||
307 a->key_len != b->key_len)
308 return false;
309
310 if (a->key_len <= sizeof(a->inline_key)) {
311 ka = &a->inline_key;
312 kb = &b->inline_key;
313 } else {
314 ka = a->key;
315 kb = b->key;
316 }
317 return memcmp(ka, kb, a->key_len) == 0;
318}
319
320static atomic_t fscache_cookie_debug_id = ATOMIC_INIT(1);
321
322/*
323 * Allocate a cookie.
324 */
325static struct fscache_cookie *fscache_alloc_cookie(
326 struct fscache_volume *volume,
327 u8 advice,
328 const void *index_key, size_t index_key_len,
329 const void *aux_data, size_t aux_data_len,
330 loff_t object_size)
331{
332 struct fscache_cookie *cookie;
333
334 /* allocate and initialise a cookie */
335 cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL);
336 if (!cookie)
337 return NULL;
338 fscache_stat(&fscache_n_cookies);
339
340 cookie->volume = volume;
341 cookie->advice = advice;
342 cookie->key_len = index_key_len;
343 cookie->aux_len = aux_data_len;
344 cookie->object_size = object_size;
345 if (object_size == 0)
346 __set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);
347
348 if (fscache_set_key(cookie, index_key, index_key_len) < 0)
349 goto nomem;
350
351 if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
352 memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
353 } else {
354 cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
355 if (!cookie->aux)
356 goto nomem;
357 }
358
359 refcount_set(&cookie->ref, 1);
360 cookie->debug_id = atomic_inc_return(&fscache_cookie_debug_id);
361 spin_lock_init(&cookie->lock);
362 INIT_LIST_HEAD(&cookie->commit_link);
363 INIT_WORK(&cookie->work, fscache_cookie_worker);
364 __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT);
365
366 write_lock(&fscache_cookies_lock);
367 list_add_tail(&cookie->proc_link, &fscache_cookies);
368 write_unlock(&fscache_cookies_lock);
369 fscache_see_cookie(cookie, fscache_cookie_new_acquire);
370 return cookie;
371
372nomem:
373 fscache_free_cookie(cookie);
374 return NULL;
375}
376
377static inline bool fscache_cookie_is_dropped(struct fscache_cookie *cookie)
378{
379 return READ_ONCE(cookie->state) == FSCACHE_COOKIE_STATE_DROPPED;
380}
381
382static void fscache_wait_on_collision(struct fscache_cookie *candidate,
383 struct fscache_cookie *wait_for)
384{
385 enum fscache_cookie_state *statep = &wait_for->state;
386
387 wait_var_event_timeout(statep, fscache_cookie_is_dropped(wait_for),
388 20 * HZ);
389 if (!fscache_cookie_is_dropped(wait_for)) {
390 pr_notice("Potential collision c=%08x old: c=%08x",
391 candidate->debug_id, wait_for->debug_id);
392 wait_var_event(statep, fscache_cookie_is_dropped(wait_for));
393 }
394}
395
396/*
397 * Attempt to insert the new cookie into the hash. If there's a collision, we
398 * wait for the old cookie to complete if it's being relinquished and an error
399 * otherwise.
400 */
401static bool fscache_hash_cookie(struct fscache_cookie *candidate)
402{
403 struct fscache_cookie *cursor, *wait_for = NULL;
404 struct hlist_bl_head *h;
405 struct hlist_bl_node *p;
406 unsigned int bucket;
407
408 bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
409 h = &fscache_cookie_hash[bucket];
410
411 hlist_bl_lock(h);
412 hlist_bl_for_each_entry(cursor, p, h, hash_link) {
413 if (fscache_cookie_same(candidate, cursor)) {
414 if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cursor->flags))
415 goto collision;
416 wait_for = fscache_get_cookie(cursor,
417 fscache_cookie_get_hash_collision);
418 break;
419 }
420 }
421
422 fscache_get_volume(candidate->volume, fscache_volume_get_cookie);
423 atomic_inc(&candidate->volume->n_cookies);
424 hlist_bl_add_head(&candidate->hash_link, h);
425 set_bit(FSCACHE_COOKIE_IS_HASHED, &candidate->flags);
426 hlist_bl_unlock(h);
427
428 if (wait_for) {
429 fscache_wait_on_collision(candidate, wait_for);
430 fscache_put_cookie(wait_for, fscache_cookie_put_hash_collision);
431 }
432 return true;
433
434collision:
435 trace_fscache_cookie(cursor->debug_id, refcount_read(&cursor->ref),
436 fscache_cookie_collision);
437 pr_err("Duplicate cookie detected\n");
438 fscache_print_cookie(cursor, 'O');
439 fscache_print_cookie(candidate, 'N');
440 hlist_bl_unlock(h);
441 return false;
442}
443
444/*
445 * Request a cookie to represent a data storage object within a volume.
446 *
447 * We never let on to the netfs about errors. We may set a negative cookie
448 * pointer, but that's okay
449 */
450struct fscache_cookie *__fscache_acquire_cookie(
451 struct fscache_volume *volume,
452 u8 advice,
453 const void *index_key, size_t index_key_len,
454 const void *aux_data, size_t aux_data_len,
455 loff_t object_size)
456{
457 struct fscache_cookie *cookie;
458
459 _enter("V=%x", volume->debug_id);
460
461 if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255)
462 return NULL;
463 if (!aux_data || !aux_data_len) {
464 aux_data = NULL;
465 aux_data_len = 0;
466 }
467
468 fscache_stat(&fscache_n_acquires);
469
470 cookie = fscache_alloc_cookie(volume, advice,
471 index_key, index_key_len,
472 aux_data, aux_data_len,
473 object_size);
474 if (!cookie) {
475 fscache_stat(&fscache_n_acquires_oom);
476 return NULL;
477 }
478
479 if (!fscache_hash_cookie(cookie)) {
480 fscache_see_cookie(cookie, fscache_cookie_discard);
481 fscache_free_cookie(cookie);
482 return NULL;
483 }
484
485 trace_fscache_acquire(cookie);
486 fscache_stat(&fscache_n_acquires_ok);
487 _leave(" = c=%08x", cookie->debug_id);
488 return cookie;
489}
490EXPORT_SYMBOL(__fscache_acquire_cookie);
491
492/*
493 * Prepare a cache object to be written to.
494 */
495static void fscache_prepare_to_write(struct fscache_cookie *cookie)
496{
497 cookie->volume->cache->ops->prepare_to_write(cookie);
498}
499
500/*
501 * Look up a cookie in the cache.
502 */
503static void fscache_perform_lookup(struct fscache_cookie *cookie)
504{
505 enum fscache_access_trace trace = fscache_access_lookup_cookie_end_failed;
506 bool need_withdraw = false;
507
508 _enter("");
509
510 if (!cookie->volume->cache_priv) {
511 fscache_create_volume(cookie->volume, true);
512 if (!cookie->volume->cache_priv) {
513 fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT);
514 goto out;
515 }
516 }
517
518 if (!cookie->volume->cache->ops->lookup_cookie(cookie)) {
519 if (cookie->state != FSCACHE_COOKIE_STATE_FAILED)
520 fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT);
521 need_withdraw = true;
522 _leave(" [fail]");
523 goto out;
524 }
525
526 fscache_see_cookie(cookie, fscache_cookie_see_active);
527 spin_lock(&cookie->lock);
528 if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags))
529 __fscache_set_cookie_state(cookie,
530 FSCACHE_COOKIE_STATE_INVALIDATING);
531 else
532 __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE);
533 spin_unlock(&cookie->lock);
534 wake_up_cookie_state(cookie);
535 trace = fscache_access_lookup_cookie_end;
536
537out:
538 fscache_end_cookie_access(cookie, trace);
539 if (need_withdraw)
540 fscache_withdraw_cookie(cookie);
541 fscache_end_volume_access(cookie->volume, cookie, trace);
542}
543
544/*
545 * Begin the process of looking up a cookie. We offload the actual process to
546 * a worker thread.
547 */
548static bool fscache_begin_lookup(struct fscache_cookie *cookie, bool will_modify)
549{
550 if (will_modify) {
551 set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags);
552 set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags);
553 }
554 if (!fscache_begin_volume_access(cookie->volume, cookie,
555 fscache_access_lookup_cookie))
556 return false;
557
558 __fscache_begin_cookie_access(cookie, fscache_access_lookup_cookie);
559 __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_LOOKING_UP);
560 set_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags);
561 set_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags);
562 return true;
563}
564
565/*
566 * Start using the cookie for I/O. This prevents the backing object from being
567 * reaped by VM pressure.
568 */
569void __fscache_use_cookie(struct fscache_cookie *cookie, bool will_modify)
570{
571 enum fscache_cookie_state state;
572 bool queue = false;
573 int n_active;
574
575 _enter("c=%08x", cookie->debug_id);
576
577 if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags),
578 "Trying to use relinquished cookie\n"))
579 return;
580
581 spin_lock(&cookie->lock);
582
583 n_active = atomic_inc_return(&cookie->n_active);
584 trace_fscache_active(cookie->debug_id, refcount_read(&cookie->ref),
585 n_active, atomic_read(&cookie->n_accesses),
586 will_modify ?
587 fscache_active_use_modify : fscache_active_use);
588
589again:
590 state = fscache_cookie_state(cookie);
591 switch (state) {
592 case FSCACHE_COOKIE_STATE_QUIESCENT:
593 queue = fscache_begin_lookup(cookie, will_modify);
594 break;
595
596 case FSCACHE_COOKIE_STATE_LOOKING_UP:
597 case FSCACHE_COOKIE_STATE_CREATING:
598 if (will_modify)
599 set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags);
600 break;
601 case FSCACHE_COOKIE_STATE_ACTIVE:
602 case FSCACHE_COOKIE_STATE_INVALIDATING:
603 if (will_modify &&
604 !test_and_set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags)) {
605 set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags);
606 queue = true;
607 }
608 /*
609 * We could race with cookie_lru which may set LRU_DISCARD bit
610 * but has yet to run the cookie state machine. If this happens
611 * and another thread tries to use the cookie, clear LRU_DISCARD
612 * so we don't end up withdrawing the cookie while in use.
613 */
614 if (test_and_clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags))
615 fscache_see_cookie(cookie, fscache_cookie_see_lru_discard_clear);
616 break;
617
618 case FSCACHE_COOKIE_STATE_FAILED:
619 case FSCACHE_COOKIE_STATE_WITHDRAWING:
620 break;
621
622 case FSCACHE_COOKIE_STATE_LRU_DISCARDING:
623 spin_unlock(&cookie->lock);
624 wait_var_event(&cookie->state,
625 fscache_cookie_state(cookie) !=
626 FSCACHE_COOKIE_STATE_LRU_DISCARDING);
627 spin_lock(&cookie->lock);
628 goto again;
629
630 case FSCACHE_COOKIE_STATE_DROPPED:
631 case FSCACHE_COOKIE_STATE_RELINQUISHING:
632 WARN(1, "Can't use cookie in state %u\n", state);
633 break;
634 }
635
636 spin_unlock(&cookie->lock);
637 if (queue)
638 fscache_queue_cookie(cookie, fscache_cookie_get_use_work);
639 _leave("");
640}
641EXPORT_SYMBOL(__fscache_use_cookie);
642
643static void fscache_unuse_cookie_locked(struct fscache_cookie *cookie)
644{
645 clear_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags);
646 if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags))
647 return;
648
649 cookie->unused_at = jiffies;
650 spin_lock(&fscache_cookie_lru_lock);
651 if (list_empty(&cookie->commit_link)) {
652 fscache_get_cookie(cookie, fscache_cookie_get_lru);
653 fscache_stat(&fscache_n_cookies_lru);
654 }
655 list_move_tail(&cookie->commit_link, &fscache_cookie_lru);
656
657 spin_unlock(&fscache_cookie_lru_lock);
658 timer_reduce(&fscache_cookie_lru_timer,
659 jiffies + fscache_lru_cookie_timeout);
660}
661
662/*
663 * Stop using the cookie for I/O.
664 */
665void __fscache_unuse_cookie(struct fscache_cookie *cookie,
666 const void *aux_data, const loff_t *object_size)
667{
668 unsigned int debug_id = cookie->debug_id;
669 unsigned int r = refcount_read(&cookie->ref);
670 unsigned int a = atomic_read(&cookie->n_accesses);
671 unsigned int c;
672
673 if (aux_data || object_size)
674 __fscache_update_cookie(cookie, aux_data, object_size);
675
676 /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
677 c = atomic_fetch_add_unless(&cookie->n_active, -1, 1);
678 if (c != 1) {
679 trace_fscache_active(debug_id, r, c - 1, a, fscache_active_unuse);
680 return;
681 }
682
683 spin_lock(&cookie->lock);
684 r = refcount_read(&cookie->ref);
685 a = atomic_read(&cookie->n_accesses);
686 c = atomic_dec_return(&cookie->n_active);
687 trace_fscache_active(debug_id, r, c, a, fscache_active_unuse);
688 if (c == 0)
689 fscache_unuse_cookie_locked(cookie);
690 spin_unlock(&cookie->lock);
691}
692EXPORT_SYMBOL(__fscache_unuse_cookie);
693
694/*
695 * Perform work upon the cookie, such as committing its cache state,
696 * relinquishing it or withdrawing the backing cache. We're protected from the
697 * cache going away under us as object withdrawal must come through this
698 * non-reentrant work item.
699 */
700static void fscache_cookie_state_machine(struct fscache_cookie *cookie)
701{
702 enum fscache_cookie_state state;
703 bool wake = false;
704
705 _enter("c=%x", cookie->debug_id);
706
707again:
708 spin_lock(&cookie->lock);
709again_locked:
710 state = cookie->state;
711 switch (state) {
712 case FSCACHE_COOKIE_STATE_QUIESCENT:
713 /* The QUIESCENT state is jumped to the LOOKING_UP state by
714 * fscache_use_cookie().
715 */
716
717 if (atomic_read(&cookie->n_accesses) == 0 &&
718 test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) {
719 __fscache_set_cookie_state(cookie,
720 FSCACHE_COOKIE_STATE_RELINQUISHING);
721 wake = true;
722 goto again_locked;
723 }
724 break;
725
726 case FSCACHE_COOKIE_STATE_LOOKING_UP:
727 spin_unlock(&cookie->lock);
728 fscache_init_access_gate(cookie);
729 fscache_perform_lookup(cookie);
730 goto again;
731
732 case FSCACHE_COOKIE_STATE_INVALIDATING:
733 spin_unlock(&cookie->lock);
734 fscache_perform_invalidation(cookie);
735 goto again;
736
737 case FSCACHE_COOKIE_STATE_ACTIVE:
738 if (test_and_clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags)) {
739 spin_unlock(&cookie->lock);
740 fscache_prepare_to_write(cookie);
741 spin_lock(&cookie->lock);
742 }
743 if (test_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags)) {
744 __fscache_set_cookie_state(cookie,
745 FSCACHE_COOKIE_STATE_LRU_DISCARDING);
746 wake = true;
747 goto again_locked;
748 }
749 fallthrough;
750
751 case FSCACHE_COOKIE_STATE_FAILED:
752 if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags))
753 fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end);
754
755 if (atomic_read(&cookie->n_accesses) != 0)
756 break;
757 if (test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) {
758 __fscache_set_cookie_state(cookie,
759 FSCACHE_COOKIE_STATE_RELINQUISHING);
760 wake = true;
761 goto again_locked;
762 }
763 if (test_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags)) {
764 __fscache_set_cookie_state(cookie,
765 FSCACHE_COOKIE_STATE_WITHDRAWING);
766 wake = true;
767 goto again_locked;
768 }
769 break;
770
771 case FSCACHE_COOKIE_STATE_LRU_DISCARDING:
772 case FSCACHE_COOKIE_STATE_RELINQUISHING:
773 case FSCACHE_COOKIE_STATE_WITHDRAWING:
774 if (cookie->cache_priv) {
775 spin_unlock(&cookie->lock);
776 cookie->volume->cache->ops->withdraw_cookie(cookie);
777 spin_lock(&cookie->lock);
778 }
779
780 if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags))
781 fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end);
782
783 switch (state) {
784 case FSCACHE_COOKIE_STATE_RELINQUISHING:
785 fscache_see_cookie(cookie, fscache_cookie_see_relinquish);
786 fscache_unhash_cookie(cookie);
787 __fscache_set_cookie_state(cookie,
788 FSCACHE_COOKIE_STATE_DROPPED);
789 wake = true;
790 goto out;
791 case FSCACHE_COOKIE_STATE_LRU_DISCARDING:
792 fscache_see_cookie(cookie, fscache_cookie_see_lru_discard);
793 break;
794 case FSCACHE_COOKIE_STATE_WITHDRAWING:
795 fscache_see_cookie(cookie, fscache_cookie_see_withdraw);
796 break;
797 default:
798 BUG();
799 }
800
801 clear_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags);
802 clear_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags);
803 clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags);
804 clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags);
805 set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);
806 __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT);
807 wake = true;
808 goto again_locked;
809
810 case FSCACHE_COOKIE_STATE_DROPPED:
811 break;
812
813 default:
814 WARN_ONCE(1, "Cookie %x in unexpected state %u\n",
815 cookie->debug_id, state);
816 break;
817 }
818
819out:
820 spin_unlock(&cookie->lock);
821 if (wake)
822 wake_up_cookie_state(cookie);
823 _leave("");
824}
825
826static void fscache_cookie_worker(struct work_struct *work)
827{
828 struct fscache_cookie *cookie = container_of(work, struct fscache_cookie, work);
829
830 fscache_see_cookie(cookie, fscache_cookie_see_work);
831 fscache_cookie_state_machine(cookie);
832 fscache_put_cookie(cookie, fscache_cookie_put_work);
833}
834
835/*
836 * Wait for the object to become inactive. The cookie's work item will be
837 * scheduled when someone transitions n_accesses to 0 - but if someone's
838 * already done that, schedule it anyway.
839 */
840static void __fscache_withdraw_cookie(struct fscache_cookie *cookie)
841{
842 int n_accesses;
843 bool unpinned;
844
845 unpinned = test_and_clear_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags);
846
847 /* Need to read the access count after unpinning */
848 n_accesses = atomic_read(&cookie->n_accesses);
849 if (unpinned)
850 trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref),
851 n_accesses, fscache_access_cache_unpin);
852 if (n_accesses == 0)
853 fscache_queue_cookie(cookie, fscache_cookie_get_end_access);
854}
855
856static void fscache_cookie_lru_do_one(struct fscache_cookie *cookie)
857{
858 fscache_see_cookie(cookie, fscache_cookie_see_lru_do_one);
859
860 spin_lock(&cookie->lock);
861 if (cookie->state != FSCACHE_COOKIE_STATE_ACTIVE ||
862 time_before(jiffies, cookie->unused_at + fscache_lru_cookie_timeout) ||
863 atomic_read(&cookie->n_active) > 0) {
864 spin_unlock(&cookie->lock);
865 fscache_stat(&fscache_n_cookies_lru_removed);
866 } else {
867 set_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags);
868 spin_unlock(&cookie->lock);
869 fscache_stat(&fscache_n_cookies_lru_expired);
870 _debug("lru c=%x", cookie->debug_id);
871 __fscache_withdraw_cookie(cookie);
872 }
873
874 fscache_put_cookie(cookie, fscache_cookie_put_lru);
875}
876
877static void fscache_cookie_lru_worker(struct work_struct *work)
878{
879 struct fscache_cookie *cookie;
880 unsigned long unused_at;
881
882 spin_lock(&fscache_cookie_lru_lock);
883
884 while (!list_empty(&fscache_cookie_lru)) {
885 cookie = list_first_entry(&fscache_cookie_lru,
886 struct fscache_cookie, commit_link);
887 unused_at = cookie->unused_at + fscache_lru_cookie_timeout;
888 if (time_before(jiffies, unused_at)) {
889 timer_reduce(&fscache_cookie_lru_timer, unused_at);
890 break;
891 }
892
893 list_del_init(&cookie->commit_link);
894 fscache_stat_d(&fscache_n_cookies_lru);
895 spin_unlock(&fscache_cookie_lru_lock);
896 fscache_cookie_lru_do_one(cookie);
897 spin_lock(&fscache_cookie_lru_lock);
898 }
899
900 spin_unlock(&fscache_cookie_lru_lock);
901}
902
903static void fscache_cookie_lru_timed_out(struct timer_list *timer)
904{
905 queue_work(fscache_wq, &fscache_cookie_lru_work);
906}
907
908static void fscache_cookie_drop_from_lru(struct fscache_cookie *cookie)
909{
910 bool need_put = false;
911
912 if (!list_empty(&cookie->commit_link)) {
913 spin_lock(&fscache_cookie_lru_lock);
914 if (!list_empty(&cookie->commit_link)) {
915 list_del_init(&cookie->commit_link);
916 fscache_stat_d(&fscache_n_cookies_lru);
917 fscache_stat(&fscache_n_cookies_lru_dropped);
918 need_put = true;
919 }
920 spin_unlock(&fscache_cookie_lru_lock);
921 if (need_put)
922 fscache_put_cookie(cookie, fscache_cookie_put_lru);
923 }
924}
925
926/*
927 * Remove a cookie from the hash table.
928 */
929static void fscache_unhash_cookie(struct fscache_cookie *cookie)
930{
931 struct hlist_bl_head *h;
932 unsigned int bucket;
933
934 bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
935 h = &fscache_cookie_hash[bucket];
936
937 hlist_bl_lock(h);
938 hlist_bl_del(&cookie->hash_link);
939 clear_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags);
940 hlist_bl_unlock(h);
941 fscache_stat(&fscache_n_relinquishes_dropped);
942}
943
944static void fscache_drop_withdraw_cookie(struct fscache_cookie *cookie)
945{
946 fscache_cookie_drop_from_lru(cookie);
947 __fscache_withdraw_cookie(cookie);
948}
949
950/**
951 * fscache_withdraw_cookie - Mark a cookie for withdrawal
952 * @cookie: The cookie to be withdrawn.
953 *
954 * Allow the cache backend to withdraw the backing for a cookie for its own
955 * reasons, even if that cookie is in active use.
956 */
957void fscache_withdraw_cookie(struct fscache_cookie *cookie)
958{
959 set_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags);
960 fscache_drop_withdraw_cookie(cookie);
961}
962EXPORT_SYMBOL(fscache_withdraw_cookie);
963
964/*
965 * Allow the netfs to release a cookie back to the cache.
966 * - the object will be marked as recyclable on disk if retire is true
967 */
968void __fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire)
969{
970 fscache_stat(&fscache_n_relinquishes);
971 if (retire)
972 fscache_stat(&fscache_n_relinquishes_retire);
973
974 _enter("c=%08x{%d},%d",
975 cookie->debug_id, atomic_read(&cookie->n_active), retire);
976
977 if (WARN(test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags),
978 "Cookie c=%x already relinquished\n", cookie->debug_id))
979 return;
980
981 if (retire)
982 set_bit(FSCACHE_COOKIE_RETIRED, &cookie->flags);
983 trace_fscache_relinquish(cookie, retire);
984
985 ASSERTCMP(atomic_read(&cookie->n_active), ==, 0);
986 ASSERTCMP(atomic_read(&cookie->volume->n_cookies), >, 0);
987 atomic_dec(&cookie->volume->n_cookies);
988
989 if (test_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags)) {
990 set_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags);
991 fscache_drop_withdraw_cookie(cookie);
992 } else {
993 fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_DROPPED);
994 fscache_unhash_cookie(cookie);
995 }
996 fscache_put_cookie(cookie, fscache_cookie_put_relinquish);
997}
998EXPORT_SYMBOL(__fscache_relinquish_cookie);
999
1000/*
1001 * Drop a reference to a cookie.
1002 */
1003void fscache_put_cookie(struct fscache_cookie *cookie,
1004 enum fscache_cookie_trace where)
1005{
1006 struct fscache_volume *volume = cookie->volume;
1007 unsigned int cookie_debug_id = cookie->debug_id;
1008 bool zero;
1009 int ref;
1010
1011 zero = __refcount_dec_and_test(&cookie->ref, &ref);
1012 trace_fscache_cookie(cookie_debug_id, ref - 1, where);
1013 if (zero) {
1014 fscache_free_cookie(cookie);
1015 fscache_put_volume(volume, fscache_volume_put_cookie);
1016 }
1017}
1018EXPORT_SYMBOL(fscache_put_cookie);
1019
1020/*
1021 * Get a reference to a cookie.
1022 */
1023struct fscache_cookie *fscache_get_cookie(struct fscache_cookie *cookie,
1024 enum fscache_cookie_trace where)
1025{
1026 int ref;
1027
1028 __refcount_inc(&cookie->ref, &ref);
1029 trace_fscache_cookie(cookie->debug_id, ref + 1, where);
1030 return cookie;
1031}
1032EXPORT_SYMBOL(fscache_get_cookie);
1033
1034/*
1035 * Ask the cache to effect invalidation of a cookie.
1036 */
1037static void fscache_perform_invalidation(struct fscache_cookie *cookie)
1038{
1039 if (!cookie->volume->cache->ops->invalidate_cookie(cookie))
1040 fscache_caching_failed(cookie);
1041 fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end);
1042}
1043
1044/*
1045 * Invalidate an object.
1046 */
1047void __fscache_invalidate(struct fscache_cookie *cookie,
1048 const void *aux_data, loff_t new_size,
1049 unsigned int flags)
1050{
1051 bool is_caching;
1052
1053 _enter("c=%x", cookie->debug_id);
1054
1055 fscache_stat(&fscache_n_invalidates);
1056
1057 if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags),
1058 "Trying to invalidate relinquished cookie\n"))
1059 return;
1060
1061 if ((flags & FSCACHE_INVAL_DIO_WRITE) &&
1062 test_and_set_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags))
1063 return;
1064
1065 spin_lock(&cookie->lock);
1066 set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags);
1067 fscache_update_aux(cookie, aux_data, &new_size);
1068 cookie->inval_counter++;
1069 trace_fscache_invalidate(cookie, new_size);
1070
1071 switch (cookie->state) {
1072 case FSCACHE_COOKIE_STATE_INVALIDATING: /* is_still_valid will catch it */
1073 default:
1074 spin_unlock(&cookie->lock);
1075 _leave(" [no %u]", cookie->state);
1076 return;
1077
1078 case FSCACHE_COOKIE_STATE_LOOKING_UP:
1079 if (!test_and_set_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags))
1080 __fscache_begin_cookie_access(cookie, fscache_access_invalidate_cookie);
1081 fallthrough;
1082 case FSCACHE_COOKIE_STATE_CREATING:
1083 spin_unlock(&cookie->lock);
1084 _leave(" [look %x]", cookie->inval_counter);
1085 return;
1086
1087 case FSCACHE_COOKIE_STATE_ACTIVE:
1088 is_caching = fscache_begin_cookie_access(
1089 cookie, fscache_access_invalidate_cookie);
1090 if (is_caching)
1091 __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_INVALIDATING);
1092 spin_unlock(&cookie->lock);
1093 wake_up_cookie_state(cookie);
1094
1095 if (is_caching)
1096 fscache_queue_cookie(cookie, fscache_cookie_get_inval_work);
1097 _leave(" [inv]");
1098 return;
1099 }
1100}
1101EXPORT_SYMBOL(__fscache_invalidate);
1102
1103#ifdef CONFIG_PROC_FS
1104/*
1105 * Generate a list of extant cookies in /proc/fs/fscache/cookies
1106 */
1107static int fscache_cookies_seq_show(struct seq_file *m, void *v)
1108{
1109 struct fscache_cookie *cookie;
1110 unsigned int keylen = 0, auxlen = 0;
1111 u8 *p;
1112
1113 if (v == &fscache_cookies) {
1114 seq_puts(m,
1115 "COOKIE VOLUME REF ACT ACC S FL DEF \n"
1116 "======== ======== === === === = == ================\n"
1117 );
1118 return 0;
1119 }
1120
1121 cookie = list_entry(v, struct fscache_cookie, proc_link);
1122
1123 seq_printf(m,
1124 "%08x %08x %3d %3d %3d %c %02lx",
1125 cookie->debug_id,
1126 cookie->volume->debug_id,
1127 refcount_read(&cookie->ref),
1128 atomic_read(&cookie->n_active),
1129 atomic_read(&cookie->n_accesses),
1130 fscache_cookie_states[cookie->state],
1131 cookie->flags);
1132
1133 keylen = cookie->key_len;
1134 auxlen = cookie->aux_len;
1135
1136 if (keylen > 0 || auxlen > 0) {
1137 seq_puts(m, " ");
1138 p = keylen <= sizeof(cookie->inline_key) ?
1139 cookie->inline_key : cookie->key;
1140 for (; keylen > 0; keylen--)
1141 seq_printf(m, "%02x", *p++);
1142 if (auxlen > 0) {
1143 seq_puts(m, ", ");
1144 p = auxlen <= sizeof(cookie->inline_aux) ?
1145 cookie->inline_aux : cookie->aux;
1146 for (; auxlen > 0; auxlen--)
1147 seq_printf(m, "%02x", *p++);
1148 }
1149 }
1150
1151 seq_puts(m, "\n");
1152 return 0;
1153}
1154
1155static void *fscache_cookies_seq_start(struct seq_file *m, loff_t *_pos)
1156 __acquires(fscache_cookies_lock)
1157{
1158 read_lock(&fscache_cookies_lock);
1159 return seq_list_start_head(&fscache_cookies, *_pos);
1160}
1161
1162static void *fscache_cookies_seq_next(struct seq_file *m, void *v, loff_t *_pos)
1163{
1164 return seq_list_next(v, &fscache_cookies, _pos);
1165}
1166
1167static void fscache_cookies_seq_stop(struct seq_file *m, void *v)
1168 __releases(rcu)
1169{
1170 read_unlock(&fscache_cookies_lock);
1171}
1172
1173
1174const struct seq_operations fscache_cookies_seq_ops = {
1175 .start = fscache_cookies_seq_start,
1176 .next = fscache_cookies_seq_next,
1177 .stop = fscache_cookies_seq_stop,
1178 .show = fscache_cookies_seq_show,
1179};
1180#endif
1/* netfs cookie management
2 *
3 * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved.
4 * Written by David Howells (dhowells@redhat.com)
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 * See Documentation/filesystems/caching/netfs-api.txt for more information on
12 * the netfs API.
13 */
14
15#define FSCACHE_DEBUG_LEVEL COOKIE
16#include <linux/module.h>
17#include <linux/slab.h>
18#include "internal.h"
19
20struct kmem_cache *fscache_cookie_jar;
21
22static atomic_t fscache_object_debug_id = ATOMIC_INIT(0);
23
24static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie);
25static int fscache_alloc_object(struct fscache_cache *cache,
26 struct fscache_cookie *cookie);
27static int fscache_attach_object(struct fscache_cookie *cookie,
28 struct fscache_object *object);
29
30/*
31 * initialise an cookie jar slab element prior to any use
32 */
33void fscache_cookie_init_once(void *_cookie)
34{
35 struct fscache_cookie *cookie = _cookie;
36
37 memset(cookie, 0, sizeof(*cookie));
38 spin_lock_init(&cookie->lock);
39 spin_lock_init(&cookie->stores_lock);
40 INIT_HLIST_HEAD(&cookie->backing_objects);
41}
42
43/*
44 * request a cookie to represent an object (index, datafile, xattr, etc)
45 * - parent specifies the parent object
46 * - the top level index cookie for each netfs is stored in the fscache_netfs
47 * struct upon registration
48 * - def points to the definition
49 * - the netfs_data will be passed to the functions pointed to in *def
50 * - all attached caches will be searched to see if they contain this object
51 * - index objects aren't stored on disk until there's a dependent file that
52 * needs storing
53 * - other objects are stored in a selected cache immediately, and all the
54 * indices forming the path to it are instantiated if necessary
55 * - we never let on to the netfs about errors
56 * - we may set a negative cookie pointer, but that's okay
57 */
58struct fscache_cookie *__fscache_acquire_cookie(
59 struct fscache_cookie *parent,
60 const struct fscache_cookie_def *def,
61 void *netfs_data)
62{
63 struct fscache_cookie *cookie;
64
65 BUG_ON(!def);
66
67 _enter("{%s},{%s},%p",
68 parent ? (char *) parent->def->name : "<no-parent>",
69 def->name, netfs_data);
70
71 fscache_stat(&fscache_n_acquires);
72
73 /* if there's no parent cookie, then we don't create one here either */
74 if (!parent) {
75 fscache_stat(&fscache_n_acquires_null);
76 _leave(" [no parent]");
77 return NULL;
78 }
79
80 /* validate the definition */
81 BUG_ON(!def->get_key);
82 BUG_ON(!def->name[0]);
83
84 BUG_ON(def->type == FSCACHE_COOKIE_TYPE_INDEX &&
85 parent->def->type != FSCACHE_COOKIE_TYPE_INDEX);
86
87 /* allocate and initialise a cookie */
88 cookie = kmem_cache_alloc(fscache_cookie_jar, GFP_KERNEL);
89 if (!cookie) {
90 fscache_stat(&fscache_n_acquires_oom);
91 _leave(" [ENOMEM]");
92 return NULL;
93 }
94
95 atomic_set(&cookie->usage, 1);
96 atomic_set(&cookie->n_children, 0);
97
98 atomic_inc(&parent->usage);
99 atomic_inc(&parent->n_children);
100
101 cookie->def = def;
102 cookie->parent = parent;
103 cookie->netfs_data = netfs_data;
104 cookie->flags = 0;
105
106 /* radix tree insertion won't use the preallocation pool unless it's
107 * told it may not wait */
108 INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_WAIT);
109
110 switch (cookie->def->type) {
111 case FSCACHE_COOKIE_TYPE_INDEX:
112 fscache_stat(&fscache_n_cookie_index);
113 break;
114 case FSCACHE_COOKIE_TYPE_DATAFILE:
115 fscache_stat(&fscache_n_cookie_data);
116 break;
117 default:
118 fscache_stat(&fscache_n_cookie_special);
119 break;
120 }
121
122 /* if the object is an index then we need do nothing more here - we
123 * create indices on disk when we need them as an index may exist in
124 * multiple caches */
125 if (cookie->def->type != FSCACHE_COOKIE_TYPE_INDEX) {
126 if (fscache_acquire_non_index_cookie(cookie) < 0) {
127 atomic_dec(&parent->n_children);
128 __fscache_cookie_put(cookie);
129 fscache_stat(&fscache_n_acquires_nobufs);
130 _leave(" = NULL");
131 return NULL;
132 }
133 }
134
135 fscache_stat(&fscache_n_acquires_ok);
136 _leave(" = %p", cookie);
137 return cookie;
138}
139EXPORT_SYMBOL(__fscache_acquire_cookie);
140
141/*
142 * acquire a non-index cookie
143 * - this must make sure the index chain is instantiated and instantiate the
144 * object representation too
145 */
146static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie)
147{
148 struct fscache_object *object;
149 struct fscache_cache *cache;
150 uint64_t i_size;
151 int ret;
152
153 _enter("");
154
155 cookie->flags = 1 << FSCACHE_COOKIE_UNAVAILABLE;
156
157 /* now we need to see whether the backing objects for this cookie yet
158 * exist, if not there'll be nothing to search */
159 down_read(&fscache_addremove_sem);
160
161 if (list_empty(&fscache_cache_list)) {
162 up_read(&fscache_addremove_sem);
163 _leave(" = 0 [no caches]");
164 return 0;
165 }
166
167 /* select a cache in which to store the object */
168 cache = fscache_select_cache_for_object(cookie->parent);
169 if (!cache) {
170 up_read(&fscache_addremove_sem);
171 fscache_stat(&fscache_n_acquires_no_cache);
172 _leave(" = -ENOMEDIUM [no cache]");
173 return -ENOMEDIUM;
174 }
175
176 _debug("cache %s", cache->tag->name);
177
178 cookie->flags =
179 (1 << FSCACHE_COOKIE_LOOKING_UP) |
180 (1 << FSCACHE_COOKIE_CREATING) |
181 (1 << FSCACHE_COOKIE_NO_DATA_YET);
182
183 /* ask the cache to allocate objects for this cookie and its parent
184 * chain */
185 ret = fscache_alloc_object(cache, cookie);
186 if (ret < 0) {
187 up_read(&fscache_addremove_sem);
188 _leave(" = %d", ret);
189 return ret;
190 }
191
192 /* pass on how big the object we're caching is supposed to be */
193 cookie->def->get_attr(cookie->netfs_data, &i_size);
194
195 spin_lock(&cookie->lock);
196 if (hlist_empty(&cookie->backing_objects)) {
197 spin_unlock(&cookie->lock);
198 goto unavailable;
199 }
200
201 object = hlist_entry(cookie->backing_objects.first,
202 struct fscache_object, cookie_link);
203
204 fscache_set_store_limit(object, i_size);
205
206 /* initiate the process of looking up all the objects in the chain
207 * (done by fscache_initialise_object()) */
208 fscache_enqueue_object(object);
209
210 spin_unlock(&cookie->lock);
211
212 /* we may be required to wait for lookup to complete at this point */
213 if (!fscache_defer_lookup) {
214 _debug("non-deferred lookup %p", &cookie->flags);
215 wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
216 fscache_wait_bit, TASK_UNINTERRUPTIBLE);
217 _debug("complete");
218 if (test_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags))
219 goto unavailable;
220 }
221
222 up_read(&fscache_addremove_sem);
223 _leave(" = 0 [deferred]");
224 return 0;
225
226unavailable:
227 up_read(&fscache_addremove_sem);
228 _leave(" = -ENOBUFS");
229 return -ENOBUFS;
230}
231
232/*
233 * recursively allocate cache object records for a cookie/cache combination
234 * - caller must be holding the addremove sem
235 */
236static int fscache_alloc_object(struct fscache_cache *cache,
237 struct fscache_cookie *cookie)
238{
239 struct fscache_object *object;
240 struct hlist_node *_n;
241 int ret;
242
243 _enter("%p,%p{%s}", cache, cookie, cookie->def->name);
244
245 spin_lock(&cookie->lock);
246 hlist_for_each_entry(object, _n, &cookie->backing_objects,
247 cookie_link) {
248 if (object->cache == cache)
249 goto object_already_extant;
250 }
251 spin_unlock(&cookie->lock);
252
253 /* ask the cache to allocate an object (we may end up with duplicate
254 * objects at this stage, but we sort that out later) */
255 fscache_stat(&fscache_n_cop_alloc_object);
256 object = cache->ops->alloc_object(cache, cookie);
257 fscache_stat_d(&fscache_n_cop_alloc_object);
258 if (IS_ERR(object)) {
259 fscache_stat(&fscache_n_object_no_alloc);
260 ret = PTR_ERR(object);
261 goto error;
262 }
263
264 fscache_stat(&fscache_n_object_alloc);
265
266 object->debug_id = atomic_inc_return(&fscache_object_debug_id);
267
268 _debug("ALLOC OBJ%x: %s {%lx}",
269 object->debug_id, cookie->def->name, object->events);
270
271 ret = fscache_alloc_object(cache, cookie->parent);
272 if (ret < 0)
273 goto error_put;
274
275 /* only attach if we managed to allocate all we needed, otherwise
276 * discard the object we just allocated and instead use the one
277 * attached to the cookie */
278 if (fscache_attach_object(cookie, object) < 0) {
279 fscache_stat(&fscache_n_cop_put_object);
280 cache->ops->put_object(object);
281 fscache_stat_d(&fscache_n_cop_put_object);
282 }
283
284 _leave(" = 0");
285 return 0;
286
287object_already_extant:
288 ret = -ENOBUFS;
289 if (object->state >= FSCACHE_OBJECT_DYING) {
290 spin_unlock(&cookie->lock);
291 goto error;
292 }
293 spin_unlock(&cookie->lock);
294 _leave(" = 0 [found]");
295 return 0;
296
297error_put:
298 fscache_stat(&fscache_n_cop_put_object);
299 cache->ops->put_object(object);
300 fscache_stat_d(&fscache_n_cop_put_object);
301error:
302 _leave(" = %d", ret);
303 return ret;
304}
305
306/*
307 * attach a cache object to a cookie
308 */
309static int fscache_attach_object(struct fscache_cookie *cookie,
310 struct fscache_object *object)
311{
312 struct fscache_object *p;
313 struct fscache_cache *cache = object->cache;
314 struct hlist_node *_n;
315 int ret;
316
317 _enter("{%s},{OBJ%x}", cookie->def->name, object->debug_id);
318
319 spin_lock(&cookie->lock);
320
321 /* there may be multiple initial creations of this object, but we only
322 * want one */
323 ret = -EEXIST;
324 hlist_for_each_entry(p, _n, &cookie->backing_objects, cookie_link) {
325 if (p->cache == object->cache) {
326 if (p->state >= FSCACHE_OBJECT_DYING)
327 ret = -ENOBUFS;
328 goto cant_attach_object;
329 }
330 }
331
332 /* pin the parent object */
333 spin_lock_nested(&cookie->parent->lock, 1);
334 hlist_for_each_entry(p, _n, &cookie->parent->backing_objects,
335 cookie_link) {
336 if (p->cache == object->cache) {
337 if (p->state >= FSCACHE_OBJECT_DYING) {
338 ret = -ENOBUFS;
339 spin_unlock(&cookie->parent->lock);
340 goto cant_attach_object;
341 }
342 object->parent = p;
343 spin_lock(&p->lock);
344 p->n_children++;
345 spin_unlock(&p->lock);
346 break;
347 }
348 }
349 spin_unlock(&cookie->parent->lock);
350
351 /* attach to the cache's object list */
352 if (list_empty(&object->cache_link)) {
353 spin_lock(&cache->object_list_lock);
354 list_add(&object->cache_link, &cache->object_list);
355 spin_unlock(&cache->object_list_lock);
356 }
357
358 /* attach to the cookie */
359 object->cookie = cookie;
360 atomic_inc(&cookie->usage);
361 hlist_add_head(&object->cookie_link, &cookie->backing_objects);
362
363 fscache_objlist_add(object);
364 ret = 0;
365
366cant_attach_object:
367 spin_unlock(&cookie->lock);
368 _leave(" = %d", ret);
369 return ret;
370}
371
372/*
373 * update the index entries backing a cookie
374 */
375void __fscache_update_cookie(struct fscache_cookie *cookie)
376{
377 struct fscache_object *object;
378 struct hlist_node *_p;
379
380 fscache_stat(&fscache_n_updates);
381
382 if (!cookie) {
383 fscache_stat(&fscache_n_updates_null);
384 _leave(" [no cookie]");
385 return;
386 }
387
388 _enter("{%s}", cookie->def->name);
389
390 BUG_ON(!cookie->def->get_aux);
391
392 spin_lock(&cookie->lock);
393
394 /* update the index entry on disk in each cache backing this cookie */
395 hlist_for_each_entry(object, _p,
396 &cookie->backing_objects, cookie_link) {
397 fscache_raise_event(object, FSCACHE_OBJECT_EV_UPDATE);
398 }
399
400 spin_unlock(&cookie->lock);
401 _leave("");
402}
403EXPORT_SYMBOL(__fscache_update_cookie);
404
405/*
406 * release a cookie back to the cache
407 * - the object will be marked as recyclable on disk if retire is true
408 * - all dependents of this cookie must have already been unregistered
409 * (indices/files/pages)
410 */
411void __fscache_relinquish_cookie(struct fscache_cookie *cookie, int retire)
412{
413 struct fscache_cache *cache;
414 struct fscache_object *object;
415 unsigned long event;
416
417 fscache_stat(&fscache_n_relinquishes);
418 if (retire)
419 fscache_stat(&fscache_n_relinquishes_retire);
420
421 if (!cookie) {
422 fscache_stat(&fscache_n_relinquishes_null);
423 _leave(" [no cookie]");
424 return;
425 }
426
427 _enter("%p{%s,%p},%d",
428 cookie, cookie->def->name, cookie->netfs_data, retire);
429
430 if (atomic_read(&cookie->n_children) != 0) {
431 printk(KERN_ERR "FS-Cache: Cookie '%s' still has children\n",
432 cookie->def->name);
433 BUG();
434 }
435
436 /* wait for the cookie to finish being instantiated (or to fail) */
437 if (test_bit(FSCACHE_COOKIE_CREATING, &cookie->flags)) {
438 fscache_stat(&fscache_n_relinquishes_waitcrt);
439 wait_on_bit(&cookie->flags, FSCACHE_COOKIE_CREATING,
440 fscache_wait_bit, TASK_UNINTERRUPTIBLE);
441 }
442
443 event = retire ? FSCACHE_OBJECT_EV_RETIRE : FSCACHE_OBJECT_EV_RELEASE;
444
445 spin_lock(&cookie->lock);
446
447 /* break links with all the active objects */
448 while (!hlist_empty(&cookie->backing_objects)) {
449 object = hlist_entry(cookie->backing_objects.first,
450 struct fscache_object,
451 cookie_link);
452
453 _debug("RELEASE OBJ%x", object->debug_id);
454
455 /* detach each cache object from the object cookie */
456 spin_lock(&object->lock);
457 hlist_del_init(&object->cookie_link);
458
459 cache = object->cache;
460 object->cookie = NULL;
461 fscache_raise_event(object, event);
462 spin_unlock(&object->lock);
463
464 if (atomic_dec_and_test(&cookie->usage))
465 /* the cookie refcount shouldn't be reduced to 0 yet */
466 BUG();
467 }
468
469 /* detach pointers back to the netfs */
470 cookie->netfs_data = NULL;
471 cookie->def = NULL;
472
473 spin_unlock(&cookie->lock);
474
475 if (cookie->parent) {
476 ASSERTCMP(atomic_read(&cookie->parent->usage), >, 0);
477 ASSERTCMP(atomic_read(&cookie->parent->n_children), >, 0);
478 atomic_dec(&cookie->parent->n_children);
479 }
480
481 /* finally dispose of the cookie */
482 ASSERTCMP(atomic_read(&cookie->usage), >, 0);
483 fscache_cookie_put(cookie);
484
485 _leave("");
486}
487EXPORT_SYMBOL(__fscache_relinquish_cookie);
488
489/*
490 * destroy a cookie
491 */
492void __fscache_cookie_put(struct fscache_cookie *cookie)
493{
494 struct fscache_cookie *parent;
495
496 _enter("%p", cookie);
497
498 for (;;) {
499 _debug("FREE COOKIE %p", cookie);
500 parent = cookie->parent;
501 BUG_ON(!hlist_empty(&cookie->backing_objects));
502 kmem_cache_free(fscache_cookie_jar, cookie);
503
504 if (!parent)
505 break;
506
507 cookie = parent;
508 BUG_ON(atomic_read(&cookie->usage) <= 0);
509 if (!atomic_dec_and_test(&cookie->usage))
510 break;
511 }
512
513 _leave("");
514}