Loading...
1/* netfs cookie management
2 *
3 * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved.
4 * Written by David Howells (dhowells@redhat.com)
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 * See Documentation/filesystems/caching/netfs-api.txt for more information on
12 * the netfs API.
13 */
14
15#define FSCACHE_DEBUG_LEVEL COOKIE
16#include <linux/module.h>
17#include <linux/slab.h>
18#include "internal.h"
19
20struct kmem_cache *fscache_cookie_jar;
21
22static atomic_t fscache_object_debug_id = ATOMIC_INIT(0);
23
24static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie);
25static int fscache_alloc_object(struct fscache_cache *cache,
26 struct fscache_cookie *cookie);
27static int fscache_attach_object(struct fscache_cookie *cookie,
28 struct fscache_object *object);
29
30/*
31 * initialise an cookie jar slab element prior to any use
32 */
33void fscache_cookie_init_once(void *_cookie)
34{
35 struct fscache_cookie *cookie = _cookie;
36
37 memset(cookie, 0, sizeof(*cookie));
38 spin_lock_init(&cookie->lock);
39 spin_lock_init(&cookie->stores_lock);
40 INIT_HLIST_HEAD(&cookie->backing_objects);
41}
42
43/*
44 * request a cookie to represent an object (index, datafile, xattr, etc)
45 * - parent specifies the parent object
46 * - the top level index cookie for each netfs is stored in the fscache_netfs
47 * struct upon registration
48 * - def points to the definition
49 * - the netfs_data will be passed to the functions pointed to in *def
50 * - all attached caches will be searched to see if they contain this object
51 * - index objects aren't stored on disk until there's a dependent file that
52 * needs storing
53 * - other objects are stored in a selected cache immediately, and all the
54 * indices forming the path to it are instantiated if necessary
55 * - we never let on to the netfs about errors
56 * - we may set a negative cookie pointer, but that's okay
57 */
58struct fscache_cookie *__fscache_acquire_cookie(
59 struct fscache_cookie *parent,
60 const struct fscache_cookie_def *def,
61 void *netfs_data)
62{
63 struct fscache_cookie *cookie;
64
65 BUG_ON(!def);
66
67 _enter("{%s},{%s},%p",
68 parent ? (char *) parent->def->name : "<no-parent>",
69 def->name, netfs_data);
70
71 fscache_stat(&fscache_n_acquires);
72
73 /* if there's no parent cookie, then we don't create one here either */
74 if (!parent) {
75 fscache_stat(&fscache_n_acquires_null);
76 _leave(" [no parent]");
77 return NULL;
78 }
79
80 /* validate the definition */
81 BUG_ON(!def->get_key);
82 BUG_ON(!def->name[0]);
83
84 BUG_ON(def->type == FSCACHE_COOKIE_TYPE_INDEX &&
85 parent->def->type != FSCACHE_COOKIE_TYPE_INDEX);
86
87 /* allocate and initialise a cookie */
88 cookie = kmem_cache_alloc(fscache_cookie_jar, GFP_KERNEL);
89 if (!cookie) {
90 fscache_stat(&fscache_n_acquires_oom);
91 _leave(" [ENOMEM]");
92 return NULL;
93 }
94
95 atomic_set(&cookie->usage, 1);
96 atomic_set(&cookie->n_children, 0);
97
98 atomic_inc(&parent->usage);
99 atomic_inc(&parent->n_children);
100
101 cookie->def = def;
102 cookie->parent = parent;
103 cookie->netfs_data = netfs_data;
104 cookie->flags = 0;
105
106 /* radix tree insertion won't use the preallocation pool unless it's
107 * told it may not wait */
108 INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_WAIT);
109
110 switch (cookie->def->type) {
111 case FSCACHE_COOKIE_TYPE_INDEX:
112 fscache_stat(&fscache_n_cookie_index);
113 break;
114 case FSCACHE_COOKIE_TYPE_DATAFILE:
115 fscache_stat(&fscache_n_cookie_data);
116 break;
117 default:
118 fscache_stat(&fscache_n_cookie_special);
119 break;
120 }
121
122 /* if the object is an index then we need do nothing more here - we
123 * create indices on disk when we need them as an index may exist in
124 * multiple caches */
125 if (cookie->def->type != FSCACHE_COOKIE_TYPE_INDEX) {
126 if (fscache_acquire_non_index_cookie(cookie) < 0) {
127 atomic_dec(&parent->n_children);
128 __fscache_cookie_put(cookie);
129 fscache_stat(&fscache_n_acquires_nobufs);
130 _leave(" = NULL");
131 return NULL;
132 }
133 }
134
135 fscache_stat(&fscache_n_acquires_ok);
136 _leave(" = %p", cookie);
137 return cookie;
138}
139EXPORT_SYMBOL(__fscache_acquire_cookie);
140
141/*
142 * acquire a non-index cookie
143 * - this must make sure the index chain is instantiated and instantiate the
144 * object representation too
145 */
146static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie)
147{
148 struct fscache_object *object;
149 struct fscache_cache *cache;
150 uint64_t i_size;
151 int ret;
152
153 _enter("");
154
155 cookie->flags = 1 << FSCACHE_COOKIE_UNAVAILABLE;
156
157 /* now we need to see whether the backing objects for this cookie yet
158 * exist, if not there'll be nothing to search */
159 down_read(&fscache_addremove_sem);
160
161 if (list_empty(&fscache_cache_list)) {
162 up_read(&fscache_addremove_sem);
163 _leave(" = 0 [no caches]");
164 return 0;
165 }
166
167 /* select a cache in which to store the object */
168 cache = fscache_select_cache_for_object(cookie->parent);
169 if (!cache) {
170 up_read(&fscache_addremove_sem);
171 fscache_stat(&fscache_n_acquires_no_cache);
172 _leave(" = -ENOMEDIUM [no cache]");
173 return -ENOMEDIUM;
174 }
175
176 _debug("cache %s", cache->tag->name);
177
178 cookie->flags =
179 (1 << FSCACHE_COOKIE_LOOKING_UP) |
180 (1 << FSCACHE_COOKIE_CREATING) |
181 (1 << FSCACHE_COOKIE_NO_DATA_YET);
182
183 /* ask the cache to allocate objects for this cookie and its parent
184 * chain */
185 ret = fscache_alloc_object(cache, cookie);
186 if (ret < 0) {
187 up_read(&fscache_addremove_sem);
188 _leave(" = %d", ret);
189 return ret;
190 }
191
192 /* pass on how big the object we're caching is supposed to be */
193 cookie->def->get_attr(cookie->netfs_data, &i_size);
194
195 spin_lock(&cookie->lock);
196 if (hlist_empty(&cookie->backing_objects)) {
197 spin_unlock(&cookie->lock);
198 goto unavailable;
199 }
200
201 object = hlist_entry(cookie->backing_objects.first,
202 struct fscache_object, cookie_link);
203
204 fscache_set_store_limit(object, i_size);
205
206 /* initiate the process of looking up all the objects in the chain
207 * (done by fscache_initialise_object()) */
208 fscache_enqueue_object(object);
209
210 spin_unlock(&cookie->lock);
211
212 /* we may be required to wait for lookup to complete at this point */
213 if (!fscache_defer_lookup) {
214 _debug("non-deferred lookup %p", &cookie->flags);
215 wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
216 fscache_wait_bit, TASK_UNINTERRUPTIBLE);
217 _debug("complete");
218 if (test_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags))
219 goto unavailable;
220 }
221
222 up_read(&fscache_addremove_sem);
223 _leave(" = 0 [deferred]");
224 return 0;
225
226unavailable:
227 up_read(&fscache_addremove_sem);
228 _leave(" = -ENOBUFS");
229 return -ENOBUFS;
230}
231
232/*
233 * recursively allocate cache object records for a cookie/cache combination
234 * - caller must be holding the addremove sem
235 */
236static int fscache_alloc_object(struct fscache_cache *cache,
237 struct fscache_cookie *cookie)
238{
239 struct fscache_object *object;
240 struct hlist_node *_n;
241 int ret;
242
243 _enter("%p,%p{%s}", cache, cookie, cookie->def->name);
244
245 spin_lock(&cookie->lock);
246 hlist_for_each_entry(object, _n, &cookie->backing_objects,
247 cookie_link) {
248 if (object->cache == cache)
249 goto object_already_extant;
250 }
251 spin_unlock(&cookie->lock);
252
253 /* ask the cache to allocate an object (we may end up with duplicate
254 * objects at this stage, but we sort that out later) */
255 fscache_stat(&fscache_n_cop_alloc_object);
256 object = cache->ops->alloc_object(cache, cookie);
257 fscache_stat_d(&fscache_n_cop_alloc_object);
258 if (IS_ERR(object)) {
259 fscache_stat(&fscache_n_object_no_alloc);
260 ret = PTR_ERR(object);
261 goto error;
262 }
263
264 fscache_stat(&fscache_n_object_alloc);
265
266 object->debug_id = atomic_inc_return(&fscache_object_debug_id);
267
268 _debug("ALLOC OBJ%x: %s {%lx}",
269 object->debug_id, cookie->def->name, object->events);
270
271 ret = fscache_alloc_object(cache, cookie->parent);
272 if (ret < 0)
273 goto error_put;
274
275 /* only attach if we managed to allocate all we needed, otherwise
276 * discard the object we just allocated and instead use the one
277 * attached to the cookie */
278 if (fscache_attach_object(cookie, object) < 0) {
279 fscache_stat(&fscache_n_cop_put_object);
280 cache->ops->put_object(object);
281 fscache_stat_d(&fscache_n_cop_put_object);
282 }
283
284 _leave(" = 0");
285 return 0;
286
287object_already_extant:
288 ret = -ENOBUFS;
289 if (object->state >= FSCACHE_OBJECT_DYING) {
290 spin_unlock(&cookie->lock);
291 goto error;
292 }
293 spin_unlock(&cookie->lock);
294 _leave(" = 0 [found]");
295 return 0;
296
297error_put:
298 fscache_stat(&fscache_n_cop_put_object);
299 cache->ops->put_object(object);
300 fscache_stat_d(&fscache_n_cop_put_object);
301error:
302 _leave(" = %d", ret);
303 return ret;
304}
305
306/*
307 * attach a cache object to a cookie
308 */
309static int fscache_attach_object(struct fscache_cookie *cookie,
310 struct fscache_object *object)
311{
312 struct fscache_object *p;
313 struct fscache_cache *cache = object->cache;
314 struct hlist_node *_n;
315 int ret;
316
317 _enter("{%s},{OBJ%x}", cookie->def->name, object->debug_id);
318
319 spin_lock(&cookie->lock);
320
321 /* there may be multiple initial creations of this object, but we only
322 * want one */
323 ret = -EEXIST;
324 hlist_for_each_entry(p, _n, &cookie->backing_objects, cookie_link) {
325 if (p->cache == object->cache) {
326 if (p->state >= FSCACHE_OBJECT_DYING)
327 ret = -ENOBUFS;
328 goto cant_attach_object;
329 }
330 }
331
332 /* pin the parent object */
333 spin_lock_nested(&cookie->parent->lock, 1);
334 hlist_for_each_entry(p, _n, &cookie->parent->backing_objects,
335 cookie_link) {
336 if (p->cache == object->cache) {
337 if (p->state >= FSCACHE_OBJECT_DYING) {
338 ret = -ENOBUFS;
339 spin_unlock(&cookie->parent->lock);
340 goto cant_attach_object;
341 }
342 object->parent = p;
343 spin_lock(&p->lock);
344 p->n_children++;
345 spin_unlock(&p->lock);
346 break;
347 }
348 }
349 spin_unlock(&cookie->parent->lock);
350
351 /* attach to the cache's object list */
352 if (list_empty(&object->cache_link)) {
353 spin_lock(&cache->object_list_lock);
354 list_add(&object->cache_link, &cache->object_list);
355 spin_unlock(&cache->object_list_lock);
356 }
357
358 /* attach to the cookie */
359 object->cookie = cookie;
360 atomic_inc(&cookie->usage);
361 hlist_add_head(&object->cookie_link, &cookie->backing_objects);
362
363 fscache_objlist_add(object);
364 ret = 0;
365
366cant_attach_object:
367 spin_unlock(&cookie->lock);
368 _leave(" = %d", ret);
369 return ret;
370}
371
372/*
373 * update the index entries backing a cookie
374 */
375void __fscache_update_cookie(struct fscache_cookie *cookie)
376{
377 struct fscache_object *object;
378 struct hlist_node *_p;
379
380 fscache_stat(&fscache_n_updates);
381
382 if (!cookie) {
383 fscache_stat(&fscache_n_updates_null);
384 _leave(" [no cookie]");
385 return;
386 }
387
388 _enter("{%s}", cookie->def->name);
389
390 BUG_ON(!cookie->def->get_aux);
391
392 spin_lock(&cookie->lock);
393
394 /* update the index entry on disk in each cache backing this cookie */
395 hlist_for_each_entry(object, _p,
396 &cookie->backing_objects, cookie_link) {
397 fscache_raise_event(object, FSCACHE_OBJECT_EV_UPDATE);
398 }
399
400 spin_unlock(&cookie->lock);
401 _leave("");
402}
403EXPORT_SYMBOL(__fscache_update_cookie);
404
405/*
406 * release a cookie back to the cache
407 * - the object will be marked as recyclable on disk if retire is true
408 * - all dependents of this cookie must have already been unregistered
409 * (indices/files/pages)
410 */
411void __fscache_relinquish_cookie(struct fscache_cookie *cookie, int retire)
412{
413 struct fscache_cache *cache;
414 struct fscache_object *object;
415 unsigned long event;
416
417 fscache_stat(&fscache_n_relinquishes);
418 if (retire)
419 fscache_stat(&fscache_n_relinquishes_retire);
420
421 if (!cookie) {
422 fscache_stat(&fscache_n_relinquishes_null);
423 _leave(" [no cookie]");
424 return;
425 }
426
427 _enter("%p{%s,%p},%d",
428 cookie, cookie->def->name, cookie->netfs_data, retire);
429
430 if (atomic_read(&cookie->n_children) != 0) {
431 printk(KERN_ERR "FS-Cache: Cookie '%s' still has children\n",
432 cookie->def->name);
433 BUG();
434 }
435
436 /* wait for the cookie to finish being instantiated (or to fail) */
437 if (test_bit(FSCACHE_COOKIE_CREATING, &cookie->flags)) {
438 fscache_stat(&fscache_n_relinquishes_waitcrt);
439 wait_on_bit(&cookie->flags, FSCACHE_COOKIE_CREATING,
440 fscache_wait_bit, TASK_UNINTERRUPTIBLE);
441 }
442
443 event = retire ? FSCACHE_OBJECT_EV_RETIRE : FSCACHE_OBJECT_EV_RELEASE;
444
445 spin_lock(&cookie->lock);
446
447 /* break links with all the active objects */
448 while (!hlist_empty(&cookie->backing_objects)) {
449 object = hlist_entry(cookie->backing_objects.first,
450 struct fscache_object,
451 cookie_link);
452
453 _debug("RELEASE OBJ%x", object->debug_id);
454
455 /* detach each cache object from the object cookie */
456 spin_lock(&object->lock);
457 hlist_del_init(&object->cookie_link);
458
459 cache = object->cache;
460 object->cookie = NULL;
461 fscache_raise_event(object, event);
462 spin_unlock(&object->lock);
463
464 if (atomic_dec_and_test(&cookie->usage))
465 /* the cookie refcount shouldn't be reduced to 0 yet */
466 BUG();
467 }
468
469 /* detach pointers back to the netfs */
470 cookie->netfs_data = NULL;
471 cookie->def = NULL;
472
473 spin_unlock(&cookie->lock);
474
475 if (cookie->parent) {
476 ASSERTCMP(atomic_read(&cookie->parent->usage), >, 0);
477 ASSERTCMP(atomic_read(&cookie->parent->n_children), >, 0);
478 atomic_dec(&cookie->parent->n_children);
479 }
480
481 /* finally dispose of the cookie */
482 ASSERTCMP(atomic_read(&cookie->usage), >, 0);
483 fscache_cookie_put(cookie);
484
485 _leave("");
486}
487EXPORT_SYMBOL(__fscache_relinquish_cookie);
488
489/*
490 * destroy a cookie
491 */
492void __fscache_cookie_put(struct fscache_cookie *cookie)
493{
494 struct fscache_cookie *parent;
495
496 _enter("%p", cookie);
497
498 for (;;) {
499 _debug("FREE COOKIE %p", cookie);
500 parent = cookie->parent;
501 BUG_ON(!hlist_empty(&cookie->backing_objects));
502 kmem_cache_free(fscache_cookie_jar, cookie);
503
504 if (!parent)
505 break;
506
507 cookie = parent;
508 BUG_ON(atomic_read(&cookie->usage) <= 0);
509 if (!atomic_dec_and_test(&cookie->usage))
510 break;
511 }
512
513 _leave("");
514}
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* netfs cookie management
3 *
4 * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 *
7 * See Documentation/filesystems/caching/netfs-api.txt for more information on
8 * the netfs API.
9 */
10
11#define FSCACHE_DEBUG_LEVEL COOKIE
12#include <linux/module.h>
13#include <linux/slab.h>
14#include "internal.h"
15
16struct kmem_cache *fscache_cookie_jar;
17
18static atomic_t fscache_object_debug_id = ATOMIC_INIT(0);
19
20#define fscache_cookie_hash_shift 15
21static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift];
22
23static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
24 loff_t object_size);
25static int fscache_alloc_object(struct fscache_cache *cache,
26 struct fscache_cookie *cookie);
27static int fscache_attach_object(struct fscache_cookie *cookie,
28 struct fscache_object *object);
29
30static void fscache_print_cookie(struct fscache_cookie *cookie, char prefix)
31{
32 struct hlist_node *object;
33 const u8 *k;
34 unsigned loop;
35
36 pr_err("%c-cookie c=%p [p=%p fl=%lx nc=%u na=%u]\n",
37 prefix, cookie, cookie->parent, cookie->flags,
38 atomic_read(&cookie->n_children),
39 atomic_read(&cookie->n_active));
40 pr_err("%c-cookie d=%p n=%p\n",
41 prefix, cookie->def, cookie->netfs_data);
42
43 object = READ_ONCE(cookie->backing_objects.first);
44 if (object)
45 pr_err("%c-cookie o=%p\n",
46 prefix, hlist_entry(object, struct fscache_object, cookie_link));
47
48 pr_err("%c-key=[%u] '", prefix, cookie->key_len);
49 k = (cookie->key_len <= sizeof(cookie->inline_key)) ?
50 cookie->inline_key : cookie->key;
51 for (loop = 0; loop < cookie->key_len; loop++)
52 pr_cont("%02x", k[loop]);
53 pr_cont("'\n");
54}
55
56void fscache_free_cookie(struct fscache_cookie *cookie)
57{
58 if (cookie) {
59 BUG_ON(!hlist_empty(&cookie->backing_objects));
60 if (cookie->aux_len > sizeof(cookie->inline_aux))
61 kfree(cookie->aux);
62 if (cookie->key_len > sizeof(cookie->inline_key))
63 kfree(cookie->key);
64 kmem_cache_free(fscache_cookie_jar, cookie);
65 }
66}
67
68/*
69 * Set the index key in a cookie. The cookie struct has space for a 16-byte
70 * key plus length and hash, but if that's not big enough, it's instead a
71 * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then
72 * the key data.
73 */
74static int fscache_set_key(struct fscache_cookie *cookie,
75 const void *index_key, size_t index_key_len)
76{
77 unsigned long long h;
78 u32 *buf;
79 int bufs;
80 int i;
81
82 bufs = DIV_ROUND_UP(index_key_len, sizeof(*buf));
83
84 if (index_key_len > sizeof(cookie->inline_key)) {
85 buf = kcalloc(bufs, sizeof(*buf), GFP_KERNEL);
86 if (!buf)
87 return -ENOMEM;
88 cookie->key = buf;
89 } else {
90 buf = (u32 *)cookie->inline_key;
91 }
92
93 memcpy(buf, index_key, index_key_len);
94
95 /* Calculate a hash and combine this with the length in the first word
96 * or first half word
97 */
98 h = (unsigned long)cookie->parent;
99 h += index_key_len + cookie->type;
100
101 for (i = 0; i < bufs; i++)
102 h += buf[i];
103
104 cookie->key_hash = h ^ (h >> 32);
105 return 0;
106}
107
108static long fscache_compare_cookie(const struct fscache_cookie *a,
109 const struct fscache_cookie *b)
110{
111 const void *ka, *kb;
112
113 if (a->key_hash != b->key_hash)
114 return (long)a->key_hash - (long)b->key_hash;
115 if (a->parent != b->parent)
116 return (long)a->parent - (long)b->parent;
117 if (a->key_len != b->key_len)
118 return (long)a->key_len - (long)b->key_len;
119 if (a->type != b->type)
120 return (long)a->type - (long)b->type;
121
122 if (a->key_len <= sizeof(a->inline_key)) {
123 ka = &a->inline_key;
124 kb = &b->inline_key;
125 } else {
126 ka = a->key;
127 kb = b->key;
128 }
129 return memcmp(ka, kb, a->key_len);
130}
131
132/*
133 * Allocate a cookie.
134 */
135struct fscache_cookie *fscache_alloc_cookie(
136 struct fscache_cookie *parent,
137 const struct fscache_cookie_def *def,
138 const void *index_key, size_t index_key_len,
139 const void *aux_data, size_t aux_data_len,
140 void *netfs_data,
141 loff_t object_size)
142{
143 struct fscache_cookie *cookie;
144
145 /* allocate and initialise a cookie */
146 cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL);
147 if (!cookie)
148 return NULL;
149
150 cookie->key_len = index_key_len;
151 cookie->aux_len = aux_data_len;
152
153 if (fscache_set_key(cookie, index_key, index_key_len) < 0)
154 goto nomem;
155
156 if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
157 memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
158 } else {
159 cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
160 if (!cookie->aux)
161 goto nomem;
162 }
163
164 atomic_set(&cookie->usage, 1);
165 atomic_set(&cookie->n_children, 0);
166
167 /* We keep the active count elevated until relinquishment to prevent an
168 * attempt to wake up every time the object operations queue quiesces.
169 */
170 atomic_set(&cookie->n_active, 1);
171
172 cookie->def = def;
173 cookie->parent = parent;
174 cookie->netfs_data = netfs_data;
175 cookie->flags = (1 << FSCACHE_COOKIE_NO_DATA_YET);
176 cookie->type = def->type;
177 spin_lock_init(&cookie->lock);
178 spin_lock_init(&cookie->stores_lock);
179 INIT_HLIST_HEAD(&cookie->backing_objects);
180
181 /* radix tree insertion won't use the preallocation pool unless it's
182 * told it may not wait */
183 INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
184 return cookie;
185
186nomem:
187 fscache_free_cookie(cookie);
188 return NULL;
189}
190
191/*
192 * Attempt to insert the new cookie into the hash. If there's a collision, we
193 * return the old cookie if it's not in use and an error otherwise.
194 */
195struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *candidate)
196{
197 struct fscache_cookie *cursor;
198 struct hlist_bl_head *h;
199 struct hlist_bl_node *p;
200 unsigned int bucket;
201
202 bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
203 h = &fscache_cookie_hash[bucket];
204
205 hlist_bl_lock(h);
206 hlist_bl_for_each_entry(cursor, p, h, hash_link) {
207 if (fscache_compare_cookie(candidate, cursor) == 0)
208 goto collision;
209 }
210
211 __set_bit(FSCACHE_COOKIE_ACQUIRED, &candidate->flags);
212 fscache_cookie_get(candidate->parent, fscache_cookie_get_acquire_parent);
213 atomic_inc(&candidate->parent->n_children);
214 hlist_bl_add_head(&candidate->hash_link, h);
215 hlist_bl_unlock(h);
216 return candidate;
217
218collision:
219 if (test_and_set_bit(FSCACHE_COOKIE_ACQUIRED, &cursor->flags)) {
220 trace_fscache_cookie(cursor, fscache_cookie_collision,
221 atomic_read(&cursor->usage));
222 pr_err("Duplicate cookie detected\n");
223 fscache_print_cookie(cursor, 'O');
224 fscache_print_cookie(candidate, 'N');
225 hlist_bl_unlock(h);
226 return NULL;
227 }
228
229 fscache_cookie_get(cursor, fscache_cookie_get_reacquire);
230 hlist_bl_unlock(h);
231 return cursor;
232}
233
234/*
235 * request a cookie to represent an object (index, datafile, xattr, etc)
236 * - parent specifies the parent object
237 * - the top level index cookie for each netfs is stored in the fscache_netfs
238 * struct upon registration
239 * - def points to the definition
240 * - the netfs_data will be passed to the functions pointed to in *def
241 * - all attached caches will be searched to see if they contain this object
242 * - index objects aren't stored on disk until there's a dependent file that
243 * needs storing
244 * - other objects are stored in a selected cache immediately, and all the
245 * indices forming the path to it are instantiated if necessary
246 * - we never let on to the netfs about errors
247 * - we may set a negative cookie pointer, but that's okay
248 */
249struct fscache_cookie *__fscache_acquire_cookie(
250 struct fscache_cookie *parent,
251 const struct fscache_cookie_def *def,
252 const void *index_key, size_t index_key_len,
253 const void *aux_data, size_t aux_data_len,
254 void *netfs_data,
255 loff_t object_size,
256 bool enable)
257{
258 struct fscache_cookie *candidate, *cookie;
259
260 BUG_ON(!def);
261
262 _enter("{%s},{%s},%p,%u",
263 parent ? (char *) parent->def->name : "<no-parent>",
264 def->name, netfs_data, enable);
265
266 if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255)
267 return NULL;
268 if (!aux_data || !aux_data_len) {
269 aux_data = NULL;
270 aux_data_len = 0;
271 }
272
273 fscache_stat(&fscache_n_acquires);
274
275 /* if there's no parent cookie, then we don't create one here either */
276 if (!parent) {
277 fscache_stat(&fscache_n_acquires_null);
278 _leave(" [no parent]");
279 return NULL;
280 }
281
282 /* validate the definition */
283 BUG_ON(!def->name[0]);
284
285 BUG_ON(def->type == FSCACHE_COOKIE_TYPE_INDEX &&
286 parent->type != FSCACHE_COOKIE_TYPE_INDEX);
287
288 candidate = fscache_alloc_cookie(parent, def,
289 index_key, index_key_len,
290 aux_data, aux_data_len,
291 netfs_data, object_size);
292 if (!candidate) {
293 fscache_stat(&fscache_n_acquires_oom);
294 _leave(" [ENOMEM]");
295 return NULL;
296 }
297
298 cookie = fscache_hash_cookie(candidate);
299 if (!cookie) {
300 trace_fscache_cookie(candidate, fscache_cookie_discard, 1);
301 goto out;
302 }
303
304 if (cookie == candidate)
305 candidate = NULL;
306
307 switch (cookie->type) {
308 case FSCACHE_COOKIE_TYPE_INDEX:
309 fscache_stat(&fscache_n_cookie_index);
310 break;
311 case FSCACHE_COOKIE_TYPE_DATAFILE:
312 fscache_stat(&fscache_n_cookie_data);
313 break;
314 default:
315 fscache_stat(&fscache_n_cookie_special);
316 break;
317 }
318
319 trace_fscache_acquire(cookie);
320
321 if (enable) {
322 /* if the object is an index then we need do nothing more here
323 * - we create indices on disk when we need them as an index
324 * may exist in multiple caches */
325 if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
326 if (fscache_acquire_non_index_cookie(cookie, object_size) == 0) {
327 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
328 } else {
329 atomic_dec(&parent->n_children);
330 fscache_cookie_put(cookie,
331 fscache_cookie_put_acquire_nobufs);
332 fscache_stat(&fscache_n_acquires_nobufs);
333 _leave(" = NULL");
334 return NULL;
335 }
336 } else {
337 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
338 }
339 }
340
341 fscache_stat(&fscache_n_acquires_ok);
342
343out:
344 fscache_free_cookie(candidate);
345 return cookie;
346}
347EXPORT_SYMBOL(__fscache_acquire_cookie);
348
349/*
350 * Enable a cookie to permit it to accept new operations.
351 */
352void __fscache_enable_cookie(struct fscache_cookie *cookie,
353 const void *aux_data,
354 loff_t object_size,
355 bool (*can_enable)(void *data),
356 void *data)
357{
358 _enter("%p", cookie);
359
360 trace_fscache_enable(cookie);
361
362 wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
363 TASK_UNINTERRUPTIBLE);
364
365 fscache_update_aux(cookie, aux_data);
366
367 if (test_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
368 goto out_unlock;
369
370 if (can_enable && !can_enable(data)) {
371 /* The netfs decided it didn't want to enable after all */
372 } else if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
373 /* Wait for outstanding disablement to complete */
374 __fscache_wait_on_invalidate(cookie);
375
376 if (fscache_acquire_non_index_cookie(cookie, object_size) == 0)
377 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
378 } else {
379 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
380 }
381
382out_unlock:
383 clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
384 wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
385}
386EXPORT_SYMBOL(__fscache_enable_cookie);
387
388/*
389 * acquire a non-index cookie
390 * - this must make sure the index chain is instantiated and instantiate the
391 * object representation too
392 */
393static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
394 loff_t object_size)
395{
396 struct fscache_object *object;
397 struct fscache_cache *cache;
398 int ret;
399
400 _enter("");
401
402 set_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags);
403
404 /* now we need to see whether the backing objects for this cookie yet
405 * exist, if not there'll be nothing to search */
406 down_read(&fscache_addremove_sem);
407
408 if (list_empty(&fscache_cache_list)) {
409 up_read(&fscache_addremove_sem);
410 _leave(" = 0 [no caches]");
411 return 0;
412 }
413
414 /* select a cache in which to store the object */
415 cache = fscache_select_cache_for_object(cookie->parent);
416 if (!cache) {
417 up_read(&fscache_addremove_sem);
418 fscache_stat(&fscache_n_acquires_no_cache);
419 _leave(" = -ENOMEDIUM [no cache]");
420 return -ENOMEDIUM;
421 }
422
423 _debug("cache %s", cache->tag->name);
424
425 set_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags);
426
427 /* ask the cache to allocate objects for this cookie and its parent
428 * chain */
429 ret = fscache_alloc_object(cache, cookie);
430 if (ret < 0) {
431 up_read(&fscache_addremove_sem);
432 _leave(" = %d", ret);
433 return ret;
434 }
435
436 spin_lock(&cookie->lock);
437 if (hlist_empty(&cookie->backing_objects)) {
438 spin_unlock(&cookie->lock);
439 goto unavailable;
440 }
441
442 object = hlist_entry(cookie->backing_objects.first,
443 struct fscache_object, cookie_link);
444
445 fscache_set_store_limit(object, object_size);
446
447 /* initiate the process of looking up all the objects in the chain
448 * (done by fscache_initialise_object()) */
449 fscache_raise_event(object, FSCACHE_OBJECT_EV_NEW_CHILD);
450
451 spin_unlock(&cookie->lock);
452
453 /* we may be required to wait for lookup to complete at this point */
454 if (!fscache_defer_lookup) {
455 _debug("non-deferred lookup %p", &cookie->flags);
456 wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
457 TASK_UNINTERRUPTIBLE);
458 _debug("complete");
459 if (test_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags))
460 goto unavailable;
461 }
462
463 up_read(&fscache_addremove_sem);
464 _leave(" = 0 [deferred]");
465 return 0;
466
467unavailable:
468 up_read(&fscache_addremove_sem);
469 _leave(" = -ENOBUFS");
470 return -ENOBUFS;
471}
472
473/*
474 * recursively allocate cache object records for a cookie/cache combination
475 * - caller must be holding the addremove sem
476 */
477static int fscache_alloc_object(struct fscache_cache *cache,
478 struct fscache_cookie *cookie)
479{
480 struct fscache_object *object;
481 int ret;
482
483 _enter("%p,%p{%s}", cache, cookie, cookie->def->name);
484
485 spin_lock(&cookie->lock);
486 hlist_for_each_entry(object, &cookie->backing_objects,
487 cookie_link) {
488 if (object->cache == cache)
489 goto object_already_extant;
490 }
491 spin_unlock(&cookie->lock);
492
493 /* ask the cache to allocate an object (we may end up with duplicate
494 * objects at this stage, but we sort that out later) */
495 fscache_stat(&fscache_n_cop_alloc_object);
496 object = cache->ops->alloc_object(cache, cookie);
497 fscache_stat_d(&fscache_n_cop_alloc_object);
498 if (IS_ERR(object)) {
499 fscache_stat(&fscache_n_object_no_alloc);
500 ret = PTR_ERR(object);
501 goto error;
502 }
503
504 ASSERTCMP(object->cookie, ==, cookie);
505 fscache_stat(&fscache_n_object_alloc);
506
507 object->debug_id = atomic_inc_return(&fscache_object_debug_id);
508
509 _debug("ALLOC OBJ%x: %s {%lx}",
510 object->debug_id, cookie->def->name, object->events);
511
512 ret = fscache_alloc_object(cache, cookie->parent);
513 if (ret < 0)
514 goto error_put;
515
516 /* only attach if we managed to allocate all we needed, otherwise
517 * discard the object we just allocated and instead use the one
518 * attached to the cookie */
519 if (fscache_attach_object(cookie, object) < 0) {
520 fscache_stat(&fscache_n_cop_put_object);
521 cache->ops->put_object(object, fscache_obj_put_attach_fail);
522 fscache_stat_d(&fscache_n_cop_put_object);
523 }
524
525 _leave(" = 0");
526 return 0;
527
528object_already_extant:
529 ret = -ENOBUFS;
530 if (fscache_object_is_dying(object) ||
531 fscache_cache_is_broken(object)) {
532 spin_unlock(&cookie->lock);
533 goto error;
534 }
535 spin_unlock(&cookie->lock);
536 _leave(" = 0 [found]");
537 return 0;
538
539error_put:
540 fscache_stat(&fscache_n_cop_put_object);
541 cache->ops->put_object(object, fscache_obj_put_alloc_fail);
542 fscache_stat_d(&fscache_n_cop_put_object);
543error:
544 _leave(" = %d", ret);
545 return ret;
546}
547
548/*
549 * attach a cache object to a cookie
550 */
551static int fscache_attach_object(struct fscache_cookie *cookie,
552 struct fscache_object *object)
553{
554 struct fscache_object *p;
555 struct fscache_cache *cache = object->cache;
556 int ret;
557
558 _enter("{%s},{OBJ%x}", cookie->def->name, object->debug_id);
559
560 ASSERTCMP(object->cookie, ==, cookie);
561
562 spin_lock(&cookie->lock);
563
564 /* there may be multiple initial creations of this object, but we only
565 * want one */
566 ret = -EEXIST;
567 hlist_for_each_entry(p, &cookie->backing_objects, cookie_link) {
568 if (p->cache == object->cache) {
569 if (fscache_object_is_dying(p))
570 ret = -ENOBUFS;
571 goto cant_attach_object;
572 }
573 }
574
575 /* pin the parent object */
576 spin_lock_nested(&cookie->parent->lock, 1);
577 hlist_for_each_entry(p, &cookie->parent->backing_objects,
578 cookie_link) {
579 if (p->cache == object->cache) {
580 if (fscache_object_is_dying(p)) {
581 ret = -ENOBUFS;
582 spin_unlock(&cookie->parent->lock);
583 goto cant_attach_object;
584 }
585 object->parent = p;
586 spin_lock(&p->lock);
587 p->n_children++;
588 spin_unlock(&p->lock);
589 break;
590 }
591 }
592 spin_unlock(&cookie->parent->lock);
593
594 /* attach to the cache's object list */
595 if (list_empty(&object->cache_link)) {
596 spin_lock(&cache->object_list_lock);
597 list_add(&object->cache_link, &cache->object_list);
598 spin_unlock(&cache->object_list_lock);
599 }
600
601 /* Attach to the cookie. The object already has a ref on it. */
602 hlist_add_head(&object->cookie_link, &cookie->backing_objects);
603
604 fscache_objlist_add(object);
605 ret = 0;
606
607cant_attach_object:
608 spin_unlock(&cookie->lock);
609 _leave(" = %d", ret);
610 return ret;
611}
612
613/*
614 * Invalidate an object. Callable with spinlocks held.
615 */
616void __fscache_invalidate(struct fscache_cookie *cookie)
617{
618 struct fscache_object *object;
619
620 _enter("{%s}", cookie->def->name);
621
622 fscache_stat(&fscache_n_invalidates);
623
624 /* Only permit invalidation of data files. Invalidating an index will
625 * require the caller to release all its attachments to the tree rooted
626 * there, and if it's doing that, it may as well just retire the
627 * cookie.
628 */
629 ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
630
631 /* If there's an object, we tell the object state machine to handle the
632 * invalidation on our behalf, otherwise there's nothing to do.
633 */
634 if (!hlist_empty(&cookie->backing_objects)) {
635 spin_lock(&cookie->lock);
636
637 if (fscache_cookie_enabled(cookie) &&
638 !hlist_empty(&cookie->backing_objects) &&
639 !test_and_set_bit(FSCACHE_COOKIE_INVALIDATING,
640 &cookie->flags)) {
641 object = hlist_entry(cookie->backing_objects.first,
642 struct fscache_object,
643 cookie_link);
644 if (fscache_object_is_live(object))
645 fscache_raise_event(
646 object, FSCACHE_OBJECT_EV_INVALIDATE);
647 }
648
649 spin_unlock(&cookie->lock);
650 }
651
652 _leave("");
653}
654EXPORT_SYMBOL(__fscache_invalidate);
655
656/*
657 * Wait for object invalidation to complete.
658 */
659void __fscache_wait_on_invalidate(struct fscache_cookie *cookie)
660{
661 _enter("%p", cookie);
662
663 wait_on_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING,
664 TASK_UNINTERRUPTIBLE);
665
666 _leave("");
667}
668EXPORT_SYMBOL(__fscache_wait_on_invalidate);
669
670/*
671 * update the index entries backing a cookie
672 */
673void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data)
674{
675 struct fscache_object *object;
676
677 fscache_stat(&fscache_n_updates);
678
679 if (!cookie) {
680 fscache_stat(&fscache_n_updates_null);
681 _leave(" [no cookie]");
682 return;
683 }
684
685 _enter("{%s}", cookie->def->name);
686
687 spin_lock(&cookie->lock);
688
689 fscache_update_aux(cookie, aux_data);
690
691 if (fscache_cookie_enabled(cookie)) {
692 /* update the index entry on disk in each cache backing this
693 * cookie.
694 */
695 hlist_for_each_entry(object,
696 &cookie->backing_objects, cookie_link) {
697 fscache_raise_event(object, FSCACHE_OBJECT_EV_UPDATE);
698 }
699 }
700
701 spin_unlock(&cookie->lock);
702 _leave("");
703}
704EXPORT_SYMBOL(__fscache_update_cookie);
705
706/*
707 * Disable a cookie to stop it from accepting new requests from the netfs.
708 */
709void __fscache_disable_cookie(struct fscache_cookie *cookie,
710 const void *aux_data,
711 bool invalidate)
712{
713 struct fscache_object *object;
714 bool awaken = false;
715
716 _enter("%p,%u", cookie, invalidate);
717
718 trace_fscache_disable(cookie);
719
720 ASSERTCMP(atomic_read(&cookie->n_active), >, 0);
721
722 if (atomic_read(&cookie->n_children) != 0) {
723 pr_err("Cookie '%s' still has children\n",
724 cookie->def->name);
725 BUG();
726 }
727
728 wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
729 TASK_UNINTERRUPTIBLE);
730
731 fscache_update_aux(cookie, aux_data);
732
733 if (!test_and_clear_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
734 goto out_unlock_enable;
735
736 /* If the cookie is being invalidated, wait for that to complete first
737 * so that we can reuse the flag.
738 */
739 __fscache_wait_on_invalidate(cookie);
740
741 /* Dispose of the backing objects */
742 set_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags);
743
744 spin_lock(&cookie->lock);
745 if (!hlist_empty(&cookie->backing_objects)) {
746 hlist_for_each_entry(object, &cookie->backing_objects, cookie_link) {
747 if (invalidate)
748 set_bit(FSCACHE_OBJECT_RETIRED, &object->flags);
749 clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
750 fscache_raise_event(object, FSCACHE_OBJECT_EV_KILL);
751 }
752 } else {
753 if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags))
754 awaken = true;
755 }
756 spin_unlock(&cookie->lock);
757 if (awaken)
758 wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING);
759
760 /* Wait for cessation of activity requiring access to the netfs (when
761 * n_active reaches 0). This makes sure outstanding reads and writes
762 * have completed.
763 */
764 if (!atomic_dec_and_test(&cookie->n_active)) {
765 wait_var_event(&cookie->n_active,
766 !atomic_read(&cookie->n_active));
767 }
768
769 /* Make sure any pending writes are cancelled. */
770 if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX)
771 fscache_invalidate_writes(cookie);
772
773 /* Reset the cookie state if it wasn't relinquished */
774 if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags)) {
775 atomic_inc(&cookie->n_active);
776 set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
777 }
778
779out_unlock_enable:
780 clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
781 wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
782 _leave("");
783}
784EXPORT_SYMBOL(__fscache_disable_cookie);
785
786/*
787 * release a cookie back to the cache
788 * - the object will be marked as recyclable on disk if retire is true
789 * - all dependents of this cookie must have already been unregistered
790 * (indices/files/pages)
791 */
792void __fscache_relinquish_cookie(struct fscache_cookie *cookie,
793 const void *aux_data,
794 bool retire)
795{
796 fscache_stat(&fscache_n_relinquishes);
797 if (retire)
798 fscache_stat(&fscache_n_relinquishes_retire);
799
800 if (!cookie) {
801 fscache_stat(&fscache_n_relinquishes_null);
802 _leave(" [no cookie]");
803 return;
804 }
805
806 _enter("%p{%s,%p,%d},%d",
807 cookie, cookie->def->name, cookie->netfs_data,
808 atomic_read(&cookie->n_active), retire);
809
810 trace_fscache_relinquish(cookie, retire);
811
812 /* No further netfs-accessing operations on this cookie permitted */
813 if (test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags))
814 BUG();
815
816 __fscache_disable_cookie(cookie, aux_data, retire);
817
818 /* Clear pointers back to the netfs */
819 cookie->netfs_data = NULL;
820 cookie->def = NULL;
821 BUG_ON(!radix_tree_empty(&cookie->stores));
822
823 if (cookie->parent) {
824 ASSERTCMP(atomic_read(&cookie->parent->usage), >, 0);
825 ASSERTCMP(atomic_read(&cookie->parent->n_children), >, 0);
826 atomic_dec(&cookie->parent->n_children);
827 }
828
829 /* Dispose of the netfs's link to the cookie */
830 ASSERTCMP(atomic_read(&cookie->usage), >, 0);
831 fscache_cookie_put(cookie, fscache_cookie_put_relinquish);
832
833 _leave("");
834}
835EXPORT_SYMBOL(__fscache_relinquish_cookie);
836
837/*
838 * Remove a cookie from the hash table.
839 */
840static void fscache_unhash_cookie(struct fscache_cookie *cookie)
841{
842 struct hlist_bl_head *h;
843 unsigned int bucket;
844
845 bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
846 h = &fscache_cookie_hash[bucket];
847
848 hlist_bl_lock(h);
849 hlist_bl_del(&cookie->hash_link);
850 hlist_bl_unlock(h);
851}
852
853/*
854 * Drop a reference to a cookie.
855 */
856void fscache_cookie_put(struct fscache_cookie *cookie,
857 enum fscache_cookie_trace where)
858{
859 struct fscache_cookie *parent;
860 int usage;
861
862 _enter("%p", cookie);
863
864 do {
865 usage = atomic_dec_return(&cookie->usage);
866 trace_fscache_cookie(cookie, where, usage);
867
868 if (usage > 0)
869 return;
870 BUG_ON(usage < 0);
871
872 parent = cookie->parent;
873 fscache_unhash_cookie(cookie);
874 fscache_free_cookie(cookie);
875
876 cookie = parent;
877 where = fscache_cookie_put_parent;
878 } while (cookie);
879
880 _leave("");
881}
882
883/*
884 * check the consistency between the netfs inode and the backing cache
885 *
886 * NOTE: it only serves no-index type
887 */
888int __fscache_check_consistency(struct fscache_cookie *cookie,
889 const void *aux_data)
890{
891 struct fscache_operation *op;
892 struct fscache_object *object;
893 bool wake_cookie = false;
894 int ret;
895
896 _enter("%p,", cookie);
897
898 ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
899
900 if (fscache_wait_for_deferred_lookup(cookie) < 0)
901 return -ERESTARTSYS;
902
903 if (hlist_empty(&cookie->backing_objects))
904 return 0;
905
906 op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY);
907 if (!op)
908 return -ENOMEM;
909
910 fscache_operation_init(cookie, op, NULL, NULL, NULL);
911 op->flags = FSCACHE_OP_MYTHREAD |
912 (1 << FSCACHE_OP_WAITING) |
913 (1 << FSCACHE_OP_UNUSE_COOKIE);
914 trace_fscache_page_op(cookie, NULL, op, fscache_page_op_check_consistency);
915
916 spin_lock(&cookie->lock);
917
918 fscache_update_aux(cookie, aux_data);
919
920 if (!fscache_cookie_enabled(cookie) ||
921 hlist_empty(&cookie->backing_objects))
922 goto inconsistent;
923 object = hlist_entry(cookie->backing_objects.first,
924 struct fscache_object, cookie_link);
925 if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
926 goto inconsistent;
927
928 op->debug_id = atomic_inc_return(&fscache_op_debug_id);
929
930 __fscache_use_cookie(cookie);
931 if (fscache_submit_op(object, op) < 0)
932 goto submit_failed;
933
934 /* the work queue now carries its own ref on the object */
935 spin_unlock(&cookie->lock);
936
937 ret = fscache_wait_for_operation_activation(object, op, NULL, NULL);
938 if (ret == 0) {
939 /* ask the cache to honour the operation */
940 ret = object->cache->ops->check_consistency(op);
941 fscache_op_complete(op, false);
942 } else if (ret == -ENOBUFS) {
943 ret = 0;
944 }
945
946 fscache_put_operation(op);
947 _leave(" = %d", ret);
948 return ret;
949
950submit_failed:
951 wake_cookie = __fscache_unuse_cookie(cookie);
952inconsistent:
953 spin_unlock(&cookie->lock);
954 if (wake_cookie)
955 __fscache_wake_unused_cookie(cookie);
956 kfree(op);
957 _leave(" = -ESTALE");
958 return -ESTALE;
959}
960EXPORT_SYMBOL(__fscache_check_consistency);