Loading...
1// SPDX-License-Identifier: GPL-2.0
2
3/*
4 * Copyright 2019, 2020 Amazon.com, Inc. or its affiliates. All rights reserved.
5 *
6 * User extended attribute client side cache functions.
7 *
8 * Author: Frank van der Linden <fllinden@amazon.com>
9 */
10#include <linux/errno.h>
11#include <linux/nfs_fs.h>
12#include <linux/hashtable.h>
13#include <linux/refcount.h>
14#include <uapi/linux/xattr.h>
15
16#include "nfs4_fs.h"
17#include "internal.h"
18
19/*
20 * User extended attributes client side caching is implemented by having
21 * a cache structure attached to NFS inodes. This structure is allocated
22 * when needed, and freed when the cache is zapped.
23 *
24 * The cache structure contains as hash table of entries, and a pointer
25 * to a special-cased entry for the listxattr cache.
26 *
27 * Accessing and allocating / freeing the caches is done via reference
28 * counting. The cache entries use a similar refcounting scheme.
29 *
30 * This makes freeing a cache, both from the shrinker and from the
31 * zap cache path, easy. It also means that, in current use cases,
32 * the large majority of inodes will not waste any memory, as they
33 * will never have any user extended attributes assigned to them.
34 *
35 * Attribute entries are hashed in to a simple hash table. They are
36 * also part of an LRU.
37 *
38 * There are three shrinkers.
39 *
40 * Two shrinkers deal with the cache entries themselves: one for
41 * large entries (> PAGE_SIZE), and one for smaller entries. The
42 * shrinker for the larger entries works more aggressively than
43 * those for the smaller entries.
44 *
45 * The other shrinker frees the cache structures themselves.
46 */
47
48/*
49 * 64 buckets is a good default. There is likely no reasonable
50 * workload that uses more than even 64 user extended attributes.
51 * You can certainly add a lot more - but you get what you ask for
52 * in those circumstances.
53 */
54#define NFS4_XATTR_HASH_SIZE 64
55
56#define NFSDBG_FACILITY NFSDBG_XATTRCACHE
57
58struct nfs4_xattr_cache;
59struct nfs4_xattr_entry;
60
61struct nfs4_xattr_bucket {
62 spinlock_t lock;
63 struct hlist_head hlist;
64 struct nfs4_xattr_cache *cache;
65 bool draining;
66};
67
68struct nfs4_xattr_cache {
69 struct kref ref;
70 spinlock_t hash_lock; /* protects hashtable and lru */
71 struct nfs4_xattr_bucket buckets[NFS4_XATTR_HASH_SIZE];
72 struct list_head lru;
73 struct list_head dispose;
74 atomic_long_t nent;
75 spinlock_t listxattr_lock;
76 struct inode *inode;
77 struct nfs4_xattr_entry *listxattr;
78};
79
80struct nfs4_xattr_entry {
81 struct kref ref;
82 struct hlist_node hnode;
83 struct list_head lru;
84 struct list_head dispose;
85 char *xattr_name;
86 void *xattr_value;
87 size_t xattr_size;
88 struct nfs4_xattr_bucket *bucket;
89 uint32_t flags;
90};
91
92#define NFS4_XATTR_ENTRY_EXTVAL 0x0001
93
94/*
95 * LRU list of NFS inodes that have xattr caches.
96 */
97static struct list_lru nfs4_xattr_cache_lru;
98static struct list_lru nfs4_xattr_entry_lru;
99static struct list_lru nfs4_xattr_large_entry_lru;
100
101static struct kmem_cache *nfs4_xattr_cache_cachep;
102
103/*
104 * Hashing helper functions.
105 */
106static void
107nfs4_xattr_hash_init(struct nfs4_xattr_cache *cache)
108{
109 unsigned int i;
110
111 for (i = 0; i < NFS4_XATTR_HASH_SIZE; i++) {
112 INIT_HLIST_HEAD(&cache->buckets[i].hlist);
113 spin_lock_init(&cache->buckets[i].lock);
114 cache->buckets[i].cache = cache;
115 cache->buckets[i].draining = false;
116 }
117}
118
119/*
120 * Locking order:
121 * 1. inode i_lock or bucket lock
122 * 2. list_lru lock (taken by list_lru_* functions)
123 */
124
125/*
126 * Wrapper functions to add a cache entry to the right LRU.
127 */
128static bool
129nfs4_xattr_entry_lru_add(struct nfs4_xattr_entry *entry)
130{
131 struct list_lru *lru;
132
133 lru = (entry->flags & NFS4_XATTR_ENTRY_EXTVAL) ?
134 &nfs4_xattr_large_entry_lru : &nfs4_xattr_entry_lru;
135
136 return list_lru_add(lru, &entry->lru);
137}
138
139static bool
140nfs4_xattr_entry_lru_del(struct nfs4_xattr_entry *entry)
141{
142 struct list_lru *lru;
143
144 lru = (entry->flags & NFS4_XATTR_ENTRY_EXTVAL) ?
145 &nfs4_xattr_large_entry_lru : &nfs4_xattr_entry_lru;
146
147 return list_lru_del(lru, &entry->lru);
148}
149
150/*
151 * This function allocates cache entries. They are the normal
152 * extended attribute name/value pairs, but may also be a listxattr
153 * cache. Those allocations use the same entry so that they can be
154 * treated as one by the memory shrinker.
155 *
156 * xattr cache entries are allocated together with names. If the
157 * value fits in to one page with the entry structure and the name,
158 * it will also be part of the same allocation (kmalloc). This is
159 * expected to be the vast majority of cases. Larger allocations
160 * have a value pointer that is allocated separately by kvmalloc.
161 *
162 * Parameters:
163 *
164 * @name: Name of the extended attribute. NULL for listxattr cache
165 * entry.
166 * @value: Value of attribute, or listxattr cache. NULL if the
167 * value is to be copied from pages instead.
168 * @pages: Pages to copy the value from, if not NULL. Passed in to
169 * make it easier to copy the value after an RPC, even if
170 * the value will not be passed up to application (e.g.
171 * for a 'query' getxattr with NULL buffer).
172 * @len: Length of the value. Can be 0 for zero-length attribues.
173 * @value and @pages will be NULL if @len is 0.
174 */
175static struct nfs4_xattr_entry *
176nfs4_xattr_alloc_entry(const char *name, const void *value,
177 struct page **pages, size_t len)
178{
179 struct nfs4_xattr_entry *entry;
180 void *valp;
181 char *namep;
182 size_t alloclen, slen;
183 char *buf;
184 uint32_t flags;
185
186 BUILD_BUG_ON(sizeof(struct nfs4_xattr_entry) +
187 XATTR_NAME_MAX + 1 > PAGE_SIZE);
188
189 alloclen = sizeof(struct nfs4_xattr_entry);
190 if (name != NULL) {
191 slen = strlen(name) + 1;
192 alloclen += slen;
193 } else
194 slen = 0;
195
196 if (alloclen + len <= PAGE_SIZE) {
197 alloclen += len;
198 flags = 0;
199 } else {
200 flags = NFS4_XATTR_ENTRY_EXTVAL;
201 }
202
203 buf = kmalloc(alloclen, GFP_KERNEL_ACCOUNT | GFP_NOFS);
204 if (buf == NULL)
205 return NULL;
206 entry = (struct nfs4_xattr_entry *)buf;
207
208 if (name != NULL) {
209 namep = buf + sizeof(struct nfs4_xattr_entry);
210 memcpy(namep, name, slen);
211 } else {
212 namep = NULL;
213 }
214
215
216 if (flags & NFS4_XATTR_ENTRY_EXTVAL) {
217 valp = kvmalloc(len, GFP_KERNEL_ACCOUNT | GFP_NOFS);
218 if (valp == NULL) {
219 kfree(buf);
220 return NULL;
221 }
222 } else if (len != 0) {
223 valp = buf + sizeof(struct nfs4_xattr_entry) + slen;
224 } else
225 valp = NULL;
226
227 if (valp != NULL) {
228 if (value != NULL)
229 memcpy(valp, value, len);
230 else
231 _copy_from_pages(valp, pages, 0, len);
232 }
233
234 entry->flags = flags;
235 entry->xattr_value = valp;
236 kref_init(&entry->ref);
237 entry->xattr_name = namep;
238 entry->xattr_size = len;
239 entry->bucket = NULL;
240 INIT_LIST_HEAD(&entry->lru);
241 INIT_LIST_HEAD(&entry->dispose);
242 INIT_HLIST_NODE(&entry->hnode);
243
244 return entry;
245}
246
247static void
248nfs4_xattr_free_entry(struct nfs4_xattr_entry *entry)
249{
250 if (entry->flags & NFS4_XATTR_ENTRY_EXTVAL)
251 kvfree(entry->xattr_value);
252 kfree(entry);
253}
254
255static void
256nfs4_xattr_free_entry_cb(struct kref *kref)
257{
258 struct nfs4_xattr_entry *entry;
259
260 entry = container_of(kref, struct nfs4_xattr_entry, ref);
261
262 if (WARN_ON(!list_empty(&entry->lru)))
263 return;
264
265 nfs4_xattr_free_entry(entry);
266}
267
268static void
269nfs4_xattr_free_cache_cb(struct kref *kref)
270{
271 struct nfs4_xattr_cache *cache;
272 int i;
273
274 cache = container_of(kref, struct nfs4_xattr_cache, ref);
275
276 for (i = 0; i < NFS4_XATTR_HASH_SIZE; i++) {
277 if (WARN_ON(!hlist_empty(&cache->buckets[i].hlist)))
278 return;
279 cache->buckets[i].draining = false;
280 }
281
282 cache->listxattr = NULL;
283
284 kmem_cache_free(nfs4_xattr_cache_cachep, cache);
285
286}
287
288static struct nfs4_xattr_cache *
289nfs4_xattr_alloc_cache(void)
290{
291 struct nfs4_xattr_cache *cache;
292
293 cache = kmem_cache_alloc(nfs4_xattr_cache_cachep,
294 GFP_KERNEL_ACCOUNT | GFP_NOFS);
295 if (cache == NULL)
296 return NULL;
297
298 kref_init(&cache->ref);
299 atomic_long_set(&cache->nent, 0);
300
301 return cache;
302}
303
304/*
305 * Set the listxattr cache, which is a special-cased cache entry.
306 * The special value ERR_PTR(-ESTALE) is used to indicate that
307 * the cache is being drained - this prevents a new listxattr
308 * cache from being added to what is now a stale cache.
309 */
310static int
311nfs4_xattr_set_listcache(struct nfs4_xattr_cache *cache,
312 struct nfs4_xattr_entry *new)
313{
314 struct nfs4_xattr_entry *old;
315 int ret = 1;
316
317 spin_lock(&cache->listxattr_lock);
318
319 old = cache->listxattr;
320
321 if (old == ERR_PTR(-ESTALE)) {
322 ret = 0;
323 goto out;
324 }
325
326 cache->listxattr = new;
327 if (new != NULL && new != ERR_PTR(-ESTALE))
328 nfs4_xattr_entry_lru_add(new);
329
330 if (old != NULL) {
331 nfs4_xattr_entry_lru_del(old);
332 kref_put(&old->ref, nfs4_xattr_free_entry_cb);
333 }
334out:
335 spin_unlock(&cache->listxattr_lock);
336
337 return ret;
338}
339
340/*
341 * Unlink a cache from its parent inode, clearing out an invalid
342 * cache. Must be called with i_lock held.
343 */
344static struct nfs4_xattr_cache *
345nfs4_xattr_cache_unlink(struct inode *inode)
346{
347 struct nfs_inode *nfsi;
348 struct nfs4_xattr_cache *oldcache;
349
350 nfsi = NFS_I(inode);
351
352 oldcache = nfsi->xattr_cache;
353 if (oldcache != NULL) {
354 list_lru_del(&nfs4_xattr_cache_lru, &oldcache->lru);
355 oldcache->inode = NULL;
356 }
357 nfsi->xattr_cache = NULL;
358 nfsi->cache_validity &= ~NFS_INO_INVALID_XATTR;
359
360 return oldcache;
361
362}
363
364/*
365 * Discard a cache. Called by get_cache() if there was an old,
366 * invalid cache. Can also be called from a shrinker callback.
367 *
368 * The cache is dead, it has already been unlinked from its inode,
369 * and no longer appears on the cache LRU list.
370 *
371 * Mark all buckets as draining, so that no new entries are added. This
372 * could still happen in the unlikely, but possible case that another
373 * thread had grabbed a reference before it was unlinked from the inode,
374 * and is still holding it for an add operation.
375 *
376 * Remove all entries from the LRU lists, so that there is no longer
377 * any way to 'find' this cache. Then, remove the entries from the hash
378 * table.
379 *
380 * At that point, the cache will remain empty and can be freed when the final
381 * reference drops, which is very likely the kref_put at the end of
382 * this function, or the one called immediately afterwards in the
383 * shrinker callback.
384 */
385static void
386nfs4_xattr_discard_cache(struct nfs4_xattr_cache *cache)
387{
388 unsigned int i;
389 struct nfs4_xattr_entry *entry;
390 struct nfs4_xattr_bucket *bucket;
391 struct hlist_node *n;
392
393 nfs4_xattr_set_listcache(cache, ERR_PTR(-ESTALE));
394
395 for (i = 0; i < NFS4_XATTR_HASH_SIZE; i++) {
396 bucket = &cache->buckets[i];
397
398 spin_lock(&bucket->lock);
399 bucket->draining = true;
400 hlist_for_each_entry_safe(entry, n, &bucket->hlist, hnode) {
401 nfs4_xattr_entry_lru_del(entry);
402 hlist_del_init(&entry->hnode);
403 kref_put(&entry->ref, nfs4_xattr_free_entry_cb);
404 }
405 spin_unlock(&bucket->lock);
406 }
407
408 atomic_long_set(&cache->nent, 0);
409
410 kref_put(&cache->ref, nfs4_xattr_free_cache_cb);
411}
412
413/*
414 * Get a referenced copy of the cache structure. Avoid doing allocs
415 * while holding i_lock. Which means that we do some optimistic allocation,
416 * and might have to free the result in rare cases.
417 *
418 * This function only checks the NFS_INO_INVALID_XATTR cache validity bit
419 * and acts accordingly, replacing the cache when needed. For the read case
420 * (!add), this means that the caller must make sure that the cache
421 * is valid before caling this function. getxattr and listxattr call
422 * revalidate_inode to do this. The attribute cache timeout (for the
423 * non-delegated case) is expected to be dealt with in the revalidate
424 * call.
425 */
426
427static struct nfs4_xattr_cache *
428nfs4_xattr_get_cache(struct inode *inode, int add)
429{
430 struct nfs_inode *nfsi;
431 struct nfs4_xattr_cache *cache, *oldcache, *newcache;
432
433 nfsi = NFS_I(inode);
434
435 cache = oldcache = NULL;
436
437 spin_lock(&inode->i_lock);
438
439 if (nfsi->cache_validity & NFS_INO_INVALID_XATTR)
440 oldcache = nfs4_xattr_cache_unlink(inode);
441 else
442 cache = nfsi->xattr_cache;
443
444 if (cache != NULL)
445 kref_get(&cache->ref);
446
447 spin_unlock(&inode->i_lock);
448
449 if (add && cache == NULL) {
450 newcache = NULL;
451
452 cache = nfs4_xattr_alloc_cache();
453 if (cache == NULL)
454 goto out;
455
456 spin_lock(&inode->i_lock);
457 if (nfsi->cache_validity & NFS_INO_INVALID_XATTR) {
458 /*
459 * The cache was invalidated again. Give up,
460 * since what we want to enter is now likely
461 * outdated anyway.
462 */
463 spin_unlock(&inode->i_lock);
464 kref_put(&cache->ref, nfs4_xattr_free_cache_cb);
465 cache = NULL;
466 goto out;
467 }
468
469 /*
470 * Check if someone beat us to it.
471 */
472 if (nfsi->xattr_cache != NULL) {
473 newcache = nfsi->xattr_cache;
474 kref_get(&newcache->ref);
475 } else {
476 kref_get(&cache->ref);
477 nfsi->xattr_cache = cache;
478 cache->inode = inode;
479 list_lru_add(&nfs4_xattr_cache_lru, &cache->lru);
480 }
481
482 spin_unlock(&inode->i_lock);
483
484 /*
485 * If there was a race, throw away the cache we just
486 * allocated, and use the new one allocated by someone
487 * else.
488 */
489 if (newcache != NULL) {
490 kref_put(&cache->ref, nfs4_xattr_free_cache_cb);
491 cache = newcache;
492 }
493 }
494
495out:
496 /*
497 * Discard the now orphaned old cache.
498 */
499 if (oldcache != NULL)
500 nfs4_xattr_discard_cache(oldcache);
501
502 return cache;
503}
504
505static inline struct nfs4_xattr_bucket *
506nfs4_xattr_hash_bucket(struct nfs4_xattr_cache *cache, const char *name)
507{
508 return &cache->buckets[jhash(name, strlen(name), 0) &
509 (ARRAY_SIZE(cache->buckets) - 1)];
510}
511
512static struct nfs4_xattr_entry *
513nfs4_xattr_get_entry(struct nfs4_xattr_bucket *bucket, const char *name)
514{
515 struct nfs4_xattr_entry *entry;
516
517 entry = NULL;
518
519 hlist_for_each_entry(entry, &bucket->hlist, hnode) {
520 if (!strcmp(entry->xattr_name, name))
521 break;
522 }
523
524 return entry;
525}
526
527static int
528nfs4_xattr_hash_add(struct nfs4_xattr_cache *cache,
529 struct nfs4_xattr_entry *entry)
530{
531 struct nfs4_xattr_bucket *bucket;
532 struct nfs4_xattr_entry *oldentry = NULL;
533 int ret = 1;
534
535 bucket = nfs4_xattr_hash_bucket(cache, entry->xattr_name);
536 entry->bucket = bucket;
537
538 spin_lock(&bucket->lock);
539
540 if (bucket->draining) {
541 ret = 0;
542 goto out;
543 }
544
545 oldentry = nfs4_xattr_get_entry(bucket, entry->xattr_name);
546 if (oldentry != NULL) {
547 hlist_del_init(&oldentry->hnode);
548 nfs4_xattr_entry_lru_del(oldentry);
549 } else {
550 atomic_long_inc(&cache->nent);
551 }
552
553 hlist_add_head(&entry->hnode, &bucket->hlist);
554 nfs4_xattr_entry_lru_add(entry);
555
556out:
557 spin_unlock(&bucket->lock);
558
559 if (oldentry != NULL)
560 kref_put(&oldentry->ref, nfs4_xattr_free_entry_cb);
561
562 return ret;
563}
564
565static void
566nfs4_xattr_hash_remove(struct nfs4_xattr_cache *cache, const char *name)
567{
568 struct nfs4_xattr_bucket *bucket;
569 struct nfs4_xattr_entry *entry;
570
571 bucket = nfs4_xattr_hash_bucket(cache, name);
572
573 spin_lock(&bucket->lock);
574
575 entry = nfs4_xattr_get_entry(bucket, name);
576 if (entry != NULL) {
577 hlist_del_init(&entry->hnode);
578 nfs4_xattr_entry_lru_del(entry);
579 atomic_long_dec(&cache->nent);
580 }
581
582 spin_unlock(&bucket->lock);
583
584 if (entry != NULL)
585 kref_put(&entry->ref, nfs4_xattr_free_entry_cb);
586}
587
588static struct nfs4_xattr_entry *
589nfs4_xattr_hash_find(struct nfs4_xattr_cache *cache, const char *name)
590{
591 struct nfs4_xattr_bucket *bucket;
592 struct nfs4_xattr_entry *entry;
593
594 bucket = nfs4_xattr_hash_bucket(cache, name);
595
596 spin_lock(&bucket->lock);
597
598 entry = nfs4_xattr_get_entry(bucket, name);
599 if (entry != NULL)
600 kref_get(&entry->ref);
601
602 spin_unlock(&bucket->lock);
603
604 return entry;
605}
606
607/*
608 * Entry point to retrieve an entry from the cache.
609 */
610ssize_t nfs4_xattr_cache_get(struct inode *inode, const char *name, char *buf,
611 ssize_t buflen)
612{
613 struct nfs4_xattr_cache *cache;
614 struct nfs4_xattr_entry *entry;
615 ssize_t ret;
616
617 cache = nfs4_xattr_get_cache(inode, 0);
618 if (cache == NULL)
619 return -ENOENT;
620
621 ret = 0;
622 entry = nfs4_xattr_hash_find(cache, name);
623
624 if (entry != NULL) {
625 dprintk("%s: cache hit '%s', len %lu\n", __func__,
626 entry->xattr_name, (unsigned long)entry->xattr_size);
627 if (buflen == 0) {
628 /* Length probe only */
629 ret = entry->xattr_size;
630 } else if (buflen < entry->xattr_size)
631 ret = -ERANGE;
632 else {
633 memcpy(buf, entry->xattr_value, entry->xattr_size);
634 ret = entry->xattr_size;
635 }
636 kref_put(&entry->ref, nfs4_xattr_free_entry_cb);
637 } else {
638 dprintk("%s: cache miss '%s'\n", __func__, name);
639 ret = -ENOENT;
640 }
641
642 kref_put(&cache->ref, nfs4_xattr_free_cache_cb);
643
644 return ret;
645}
646
647/*
648 * Retrieve a cached list of xattrs from the cache.
649 */
650ssize_t nfs4_xattr_cache_list(struct inode *inode, char *buf, ssize_t buflen)
651{
652 struct nfs4_xattr_cache *cache;
653 struct nfs4_xattr_entry *entry;
654 ssize_t ret;
655
656 cache = nfs4_xattr_get_cache(inode, 0);
657 if (cache == NULL)
658 return -ENOENT;
659
660 spin_lock(&cache->listxattr_lock);
661
662 entry = cache->listxattr;
663
664 if (entry != NULL && entry != ERR_PTR(-ESTALE)) {
665 if (buflen == 0) {
666 /* Length probe only */
667 ret = entry->xattr_size;
668 } else if (entry->xattr_size > buflen)
669 ret = -ERANGE;
670 else {
671 memcpy(buf, entry->xattr_value, entry->xattr_size);
672 ret = entry->xattr_size;
673 }
674 } else {
675 ret = -ENOENT;
676 }
677
678 spin_unlock(&cache->listxattr_lock);
679
680 kref_put(&cache->ref, nfs4_xattr_free_cache_cb);
681
682 return ret;
683}
684
685/*
686 * Add an xattr to the cache.
687 *
688 * This also invalidates the xattr list cache.
689 */
690void nfs4_xattr_cache_add(struct inode *inode, const char *name,
691 const char *buf, struct page **pages, ssize_t buflen)
692{
693 struct nfs4_xattr_cache *cache;
694 struct nfs4_xattr_entry *entry;
695
696 dprintk("%s: add '%s' len %lu\n", __func__,
697 name, (unsigned long)buflen);
698
699 cache = nfs4_xattr_get_cache(inode, 1);
700 if (cache == NULL)
701 return;
702
703 entry = nfs4_xattr_alloc_entry(name, buf, pages, buflen);
704 if (entry == NULL)
705 goto out;
706
707 (void)nfs4_xattr_set_listcache(cache, NULL);
708
709 if (!nfs4_xattr_hash_add(cache, entry))
710 kref_put(&entry->ref, nfs4_xattr_free_entry_cb);
711
712out:
713 kref_put(&cache->ref, nfs4_xattr_free_cache_cb);
714}
715
716
717/*
718 * Remove an xattr from the cache.
719 *
720 * This also invalidates the xattr list cache.
721 */
722void nfs4_xattr_cache_remove(struct inode *inode, const char *name)
723{
724 struct nfs4_xattr_cache *cache;
725
726 dprintk("%s: remove '%s'\n", __func__, name);
727
728 cache = nfs4_xattr_get_cache(inode, 0);
729 if (cache == NULL)
730 return;
731
732 (void)nfs4_xattr_set_listcache(cache, NULL);
733 nfs4_xattr_hash_remove(cache, name);
734
735 kref_put(&cache->ref, nfs4_xattr_free_cache_cb);
736}
737
738/*
739 * Cache listxattr output, replacing any possible old one.
740 */
741void nfs4_xattr_cache_set_list(struct inode *inode, const char *buf,
742 ssize_t buflen)
743{
744 struct nfs4_xattr_cache *cache;
745 struct nfs4_xattr_entry *entry;
746
747 cache = nfs4_xattr_get_cache(inode, 1);
748 if (cache == NULL)
749 return;
750
751 entry = nfs4_xattr_alloc_entry(NULL, buf, NULL, buflen);
752 if (entry == NULL)
753 goto out;
754
755 /*
756 * This is just there to be able to get to bucket->cache,
757 * which is obviously the same for all buckets, so just
758 * use bucket 0.
759 */
760 entry->bucket = &cache->buckets[0];
761
762 if (!nfs4_xattr_set_listcache(cache, entry))
763 kref_put(&entry->ref, nfs4_xattr_free_entry_cb);
764
765out:
766 kref_put(&cache->ref, nfs4_xattr_free_cache_cb);
767}
768
769/*
770 * Zap the entire cache. Called when an inode is evicted.
771 */
772void nfs4_xattr_cache_zap(struct inode *inode)
773{
774 struct nfs4_xattr_cache *oldcache;
775
776 spin_lock(&inode->i_lock);
777 oldcache = nfs4_xattr_cache_unlink(inode);
778 spin_unlock(&inode->i_lock);
779
780 if (oldcache)
781 nfs4_xattr_discard_cache(oldcache);
782}
783
784/*
785 * The entry LRU is shrunk more aggressively than the cache LRU,
786 * by settings @seeks to 1.
787 *
788 * Cache structures are freed only when they've become empty, after
789 * pruning all but one entry.
790 */
791
792static unsigned long nfs4_xattr_cache_count(struct shrinker *shrink,
793 struct shrink_control *sc);
794static unsigned long nfs4_xattr_entry_count(struct shrinker *shrink,
795 struct shrink_control *sc);
796static unsigned long nfs4_xattr_cache_scan(struct shrinker *shrink,
797 struct shrink_control *sc);
798static unsigned long nfs4_xattr_entry_scan(struct shrinker *shrink,
799 struct shrink_control *sc);
800
801static struct shrinker nfs4_xattr_cache_shrinker = {
802 .count_objects = nfs4_xattr_cache_count,
803 .scan_objects = nfs4_xattr_cache_scan,
804 .seeks = DEFAULT_SEEKS,
805 .flags = SHRINKER_MEMCG_AWARE,
806};
807
808static struct shrinker nfs4_xattr_entry_shrinker = {
809 .count_objects = nfs4_xattr_entry_count,
810 .scan_objects = nfs4_xattr_entry_scan,
811 .seeks = DEFAULT_SEEKS,
812 .batch = 512,
813 .flags = SHRINKER_MEMCG_AWARE,
814};
815
816static struct shrinker nfs4_xattr_large_entry_shrinker = {
817 .count_objects = nfs4_xattr_entry_count,
818 .scan_objects = nfs4_xattr_entry_scan,
819 .seeks = 1,
820 .batch = 512,
821 .flags = SHRINKER_MEMCG_AWARE,
822};
823
824static enum lru_status
825cache_lru_isolate(struct list_head *item,
826 struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
827{
828 struct list_head *dispose = arg;
829 struct inode *inode;
830 struct nfs4_xattr_cache *cache = container_of(item,
831 struct nfs4_xattr_cache, lru);
832
833 if (atomic_long_read(&cache->nent) > 1)
834 return LRU_SKIP;
835
836 /*
837 * If a cache structure is on the LRU list, we know that
838 * its inode is valid. Try to lock it to break the link.
839 * Since we're inverting the lock order here, only try.
840 */
841 inode = cache->inode;
842
843 if (!spin_trylock(&inode->i_lock))
844 return LRU_SKIP;
845
846 kref_get(&cache->ref);
847
848 cache->inode = NULL;
849 NFS_I(inode)->xattr_cache = NULL;
850 NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_XATTR;
851 list_lru_isolate(lru, &cache->lru);
852
853 spin_unlock(&inode->i_lock);
854
855 list_add_tail(&cache->dispose, dispose);
856 return LRU_REMOVED;
857}
858
859static unsigned long
860nfs4_xattr_cache_scan(struct shrinker *shrink, struct shrink_control *sc)
861{
862 LIST_HEAD(dispose);
863 unsigned long freed;
864 struct nfs4_xattr_cache *cache;
865
866 freed = list_lru_shrink_walk(&nfs4_xattr_cache_lru, sc,
867 cache_lru_isolate, &dispose);
868 while (!list_empty(&dispose)) {
869 cache = list_first_entry(&dispose, struct nfs4_xattr_cache,
870 dispose);
871 list_del_init(&cache->dispose);
872 nfs4_xattr_discard_cache(cache);
873 kref_put(&cache->ref, nfs4_xattr_free_cache_cb);
874 }
875
876 return freed;
877}
878
879
880static unsigned long
881nfs4_xattr_cache_count(struct shrinker *shrink, struct shrink_control *sc)
882{
883 unsigned long count;
884
885 count = list_lru_count(&nfs4_xattr_cache_lru);
886 return vfs_pressure_ratio(count);
887}
888
889static enum lru_status
890entry_lru_isolate(struct list_head *item,
891 struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
892{
893 struct list_head *dispose = arg;
894 struct nfs4_xattr_bucket *bucket;
895 struct nfs4_xattr_cache *cache;
896 struct nfs4_xattr_entry *entry = container_of(item,
897 struct nfs4_xattr_entry, lru);
898
899 bucket = entry->bucket;
900 cache = bucket->cache;
901
902 /*
903 * Unhook the entry from its parent (either a cache bucket
904 * or a cache structure if it's a listxattr buf), so that
905 * it's no longer found. Then add it to the isolate list,
906 * to be freed later.
907 *
908 * In both cases, we're reverting lock order, so use
909 * trylock and skip the entry if we can't get the lock.
910 */
911 if (entry->xattr_name != NULL) {
912 /* Regular cache entry */
913 if (!spin_trylock(&bucket->lock))
914 return LRU_SKIP;
915
916 kref_get(&entry->ref);
917
918 hlist_del_init(&entry->hnode);
919 atomic_long_dec(&cache->nent);
920 list_lru_isolate(lru, &entry->lru);
921
922 spin_unlock(&bucket->lock);
923 } else {
924 /* Listxattr cache entry */
925 if (!spin_trylock(&cache->listxattr_lock))
926 return LRU_SKIP;
927
928 kref_get(&entry->ref);
929
930 cache->listxattr = NULL;
931 list_lru_isolate(lru, &entry->lru);
932
933 spin_unlock(&cache->listxattr_lock);
934 }
935
936 list_add_tail(&entry->dispose, dispose);
937 return LRU_REMOVED;
938}
939
940static unsigned long
941nfs4_xattr_entry_scan(struct shrinker *shrink, struct shrink_control *sc)
942{
943 LIST_HEAD(dispose);
944 unsigned long freed;
945 struct nfs4_xattr_entry *entry;
946 struct list_lru *lru;
947
948 lru = (shrink == &nfs4_xattr_large_entry_shrinker) ?
949 &nfs4_xattr_large_entry_lru : &nfs4_xattr_entry_lru;
950
951 freed = list_lru_shrink_walk(lru, sc, entry_lru_isolate, &dispose);
952
953 while (!list_empty(&dispose)) {
954 entry = list_first_entry(&dispose, struct nfs4_xattr_entry,
955 dispose);
956 list_del_init(&entry->dispose);
957
958 /*
959 * Drop two references: the one that we just grabbed
960 * in entry_lru_isolate, and the one that was set
961 * when the entry was first allocated.
962 */
963 kref_put(&entry->ref, nfs4_xattr_free_entry_cb);
964 kref_put(&entry->ref, nfs4_xattr_free_entry_cb);
965 }
966
967 return freed;
968}
969
970static unsigned long
971nfs4_xattr_entry_count(struct shrinker *shrink, struct shrink_control *sc)
972{
973 unsigned long count;
974 struct list_lru *lru;
975
976 lru = (shrink == &nfs4_xattr_large_entry_shrinker) ?
977 &nfs4_xattr_large_entry_lru : &nfs4_xattr_entry_lru;
978
979 count = list_lru_count(lru);
980 return vfs_pressure_ratio(count);
981}
982
983
984static void nfs4_xattr_cache_init_once(void *p)
985{
986 struct nfs4_xattr_cache *cache = (struct nfs4_xattr_cache *)p;
987
988 spin_lock_init(&cache->listxattr_lock);
989 atomic_long_set(&cache->nent, 0);
990 nfs4_xattr_hash_init(cache);
991 cache->listxattr = NULL;
992 INIT_LIST_HEAD(&cache->lru);
993 INIT_LIST_HEAD(&cache->dispose);
994}
995
996int __init nfs4_xattr_cache_init(void)
997{
998 int ret = 0;
999
1000 nfs4_xattr_cache_cachep = kmem_cache_create("nfs4_xattr_cache_cache",
1001 sizeof(struct nfs4_xattr_cache), 0,
1002 (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD|SLAB_ACCOUNT),
1003 nfs4_xattr_cache_init_once);
1004 if (nfs4_xattr_cache_cachep == NULL)
1005 return -ENOMEM;
1006
1007 ret = list_lru_init_memcg(&nfs4_xattr_large_entry_lru,
1008 &nfs4_xattr_large_entry_shrinker);
1009 if (ret)
1010 goto out4;
1011
1012 ret = list_lru_init_memcg(&nfs4_xattr_entry_lru,
1013 &nfs4_xattr_entry_shrinker);
1014 if (ret)
1015 goto out3;
1016
1017 ret = list_lru_init_memcg(&nfs4_xattr_cache_lru,
1018 &nfs4_xattr_cache_shrinker);
1019 if (ret)
1020 goto out2;
1021
1022 ret = register_shrinker(&nfs4_xattr_cache_shrinker);
1023 if (ret)
1024 goto out1;
1025
1026 ret = register_shrinker(&nfs4_xattr_entry_shrinker);
1027 if (ret)
1028 goto out;
1029
1030 ret = register_shrinker(&nfs4_xattr_large_entry_shrinker);
1031 if (!ret)
1032 return 0;
1033
1034 unregister_shrinker(&nfs4_xattr_entry_shrinker);
1035out:
1036 unregister_shrinker(&nfs4_xattr_cache_shrinker);
1037out1:
1038 list_lru_destroy(&nfs4_xattr_cache_lru);
1039out2:
1040 list_lru_destroy(&nfs4_xattr_entry_lru);
1041out3:
1042 list_lru_destroy(&nfs4_xattr_large_entry_lru);
1043out4:
1044 kmem_cache_destroy(nfs4_xattr_cache_cachep);
1045
1046 return ret;
1047}
1048
1049void nfs4_xattr_cache_exit(void)
1050{
1051 unregister_shrinker(&nfs4_xattr_entry_shrinker);
1052 unregister_shrinker(&nfs4_xattr_cache_shrinker);
1053 list_lru_destroy(&nfs4_xattr_entry_lru);
1054 list_lru_destroy(&nfs4_xattr_cache_lru);
1055 kmem_cache_destroy(nfs4_xattr_cache_cachep);
1056}
1// SPDX-License-Identifier: GPL-2.0
2
3/*
4 * Copyright 2019, 2020 Amazon.com, Inc. or its affiliates. All rights reserved.
5 *
6 * User extended attribute client side cache functions.
7 *
8 * Author: Frank van der Linden <fllinden@amazon.com>
9 */
10#include <linux/errno.h>
11#include <linux/nfs_fs.h>
12#include <linux/hashtable.h>
13#include <linux/refcount.h>
14#include <uapi/linux/xattr.h>
15
16#include "nfs4_fs.h"
17#include "internal.h"
18
19/*
20 * User extended attributes client side caching is implemented by having
21 * a cache structure attached to NFS inodes. This structure is allocated
22 * when needed, and freed when the cache is zapped.
23 *
24 * The cache structure contains as hash table of entries, and a pointer
25 * to a special-cased entry for the listxattr cache.
26 *
27 * Accessing and allocating / freeing the caches is done via reference
28 * counting. The cache entries use a similar refcounting scheme.
29 *
30 * This makes freeing a cache, both from the shrinker and from the
31 * zap cache path, easy. It also means that, in current use cases,
32 * the large majority of inodes will not waste any memory, as they
33 * will never have any user extended attributes assigned to them.
34 *
35 * Attribute entries are hashed in to a simple hash table. They are
36 * also part of an LRU.
37 *
38 * There are three shrinkers.
39 *
40 * Two shrinkers deal with the cache entries themselves: one for
41 * large entries (> PAGE_SIZE), and one for smaller entries. The
42 * shrinker for the larger entries works more aggressively than
43 * those for the smaller entries.
44 *
45 * The other shrinker frees the cache structures themselves.
46 */
47
48/*
49 * 64 buckets is a good default. There is likely no reasonable
50 * workload that uses more than even 64 user extended attributes.
51 * You can certainly add a lot more - but you get what you ask for
52 * in those circumstances.
53 */
54#define NFS4_XATTR_HASH_SIZE 64
55
56#define NFSDBG_FACILITY NFSDBG_XATTRCACHE
57
58struct nfs4_xattr_cache;
59struct nfs4_xattr_entry;
60
61struct nfs4_xattr_bucket {
62 spinlock_t lock;
63 struct hlist_head hlist;
64 struct nfs4_xattr_cache *cache;
65 bool draining;
66};
67
68struct nfs4_xattr_cache {
69 struct kref ref;
70 struct nfs4_xattr_bucket buckets[NFS4_XATTR_HASH_SIZE];
71 struct list_head lru;
72 struct list_head dispose;
73 atomic_long_t nent;
74 spinlock_t listxattr_lock;
75 struct inode *inode;
76 struct nfs4_xattr_entry *listxattr;
77};
78
79struct nfs4_xattr_entry {
80 struct kref ref;
81 struct hlist_node hnode;
82 struct list_head lru;
83 struct list_head dispose;
84 char *xattr_name;
85 void *xattr_value;
86 size_t xattr_size;
87 struct nfs4_xattr_bucket *bucket;
88 uint32_t flags;
89};
90
91#define NFS4_XATTR_ENTRY_EXTVAL 0x0001
92
93/*
94 * LRU list of NFS inodes that have xattr caches.
95 */
96static struct list_lru nfs4_xattr_cache_lru;
97static struct list_lru nfs4_xattr_entry_lru;
98static struct list_lru nfs4_xattr_large_entry_lru;
99
100static struct kmem_cache *nfs4_xattr_cache_cachep;
101
102/*
103 * Hashing helper functions.
104 */
105static void
106nfs4_xattr_hash_init(struct nfs4_xattr_cache *cache)
107{
108 unsigned int i;
109
110 for (i = 0; i < NFS4_XATTR_HASH_SIZE; i++) {
111 INIT_HLIST_HEAD(&cache->buckets[i].hlist);
112 spin_lock_init(&cache->buckets[i].lock);
113 cache->buckets[i].cache = cache;
114 cache->buckets[i].draining = false;
115 }
116}
117
118/*
119 * Locking order:
120 * 1. inode i_lock or bucket lock
121 * 2. list_lru lock (taken by list_lru_* functions)
122 */
123
124/*
125 * Wrapper functions to add a cache entry to the right LRU.
126 */
127static bool
128nfs4_xattr_entry_lru_add(struct nfs4_xattr_entry *entry)
129{
130 struct list_lru *lru;
131
132 lru = (entry->flags & NFS4_XATTR_ENTRY_EXTVAL) ?
133 &nfs4_xattr_large_entry_lru : &nfs4_xattr_entry_lru;
134
135 return list_lru_add(lru, &entry->lru);
136}
137
138static bool
139nfs4_xattr_entry_lru_del(struct nfs4_xattr_entry *entry)
140{
141 struct list_lru *lru;
142
143 lru = (entry->flags & NFS4_XATTR_ENTRY_EXTVAL) ?
144 &nfs4_xattr_large_entry_lru : &nfs4_xattr_entry_lru;
145
146 return list_lru_del(lru, &entry->lru);
147}
148
149/*
150 * This function allocates cache entries. They are the normal
151 * extended attribute name/value pairs, but may also be a listxattr
152 * cache. Those allocations use the same entry so that they can be
153 * treated as one by the memory shrinker.
154 *
155 * xattr cache entries are allocated together with names. If the
156 * value fits in to one page with the entry structure and the name,
157 * it will also be part of the same allocation (kmalloc). This is
158 * expected to be the vast majority of cases. Larger allocations
159 * have a value pointer that is allocated separately by kvmalloc.
160 *
161 * Parameters:
162 *
163 * @name: Name of the extended attribute. NULL for listxattr cache
164 * entry.
165 * @value: Value of attribute, or listxattr cache. NULL if the
166 * value is to be copied from pages instead.
167 * @pages: Pages to copy the value from, if not NULL. Passed in to
168 * make it easier to copy the value after an RPC, even if
169 * the value will not be passed up to application (e.g.
170 * for a 'query' getxattr with NULL buffer).
171 * @len: Length of the value. Can be 0 for zero-length attributes.
172 * @value and @pages will be NULL if @len is 0.
173 */
174static struct nfs4_xattr_entry *
175nfs4_xattr_alloc_entry(const char *name, const void *value,
176 struct page **pages, size_t len)
177{
178 struct nfs4_xattr_entry *entry;
179 void *valp;
180 char *namep;
181 size_t alloclen, slen;
182 char *buf;
183 uint32_t flags;
184
185 BUILD_BUG_ON(sizeof(struct nfs4_xattr_entry) +
186 XATTR_NAME_MAX + 1 > PAGE_SIZE);
187
188 alloclen = sizeof(struct nfs4_xattr_entry);
189 if (name != NULL) {
190 slen = strlen(name) + 1;
191 alloclen += slen;
192 } else
193 slen = 0;
194
195 if (alloclen + len <= PAGE_SIZE) {
196 alloclen += len;
197 flags = 0;
198 } else {
199 flags = NFS4_XATTR_ENTRY_EXTVAL;
200 }
201
202 buf = kmalloc(alloclen, GFP_KERNEL_ACCOUNT | GFP_NOFS);
203 if (buf == NULL)
204 return NULL;
205 entry = (struct nfs4_xattr_entry *)buf;
206
207 if (name != NULL) {
208 namep = buf + sizeof(struct nfs4_xattr_entry);
209 memcpy(namep, name, slen);
210 } else {
211 namep = NULL;
212 }
213
214
215 if (flags & NFS4_XATTR_ENTRY_EXTVAL) {
216 valp = kvmalloc(len, GFP_KERNEL_ACCOUNT | GFP_NOFS);
217 if (valp == NULL) {
218 kfree(buf);
219 return NULL;
220 }
221 } else if (len != 0) {
222 valp = buf + sizeof(struct nfs4_xattr_entry) + slen;
223 } else
224 valp = NULL;
225
226 if (valp != NULL) {
227 if (value != NULL)
228 memcpy(valp, value, len);
229 else
230 _copy_from_pages(valp, pages, 0, len);
231 }
232
233 entry->flags = flags;
234 entry->xattr_value = valp;
235 kref_init(&entry->ref);
236 entry->xattr_name = namep;
237 entry->xattr_size = len;
238 entry->bucket = NULL;
239 INIT_LIST_HEAD(&entry->lru);
240 INIT_LIST_HEAD(&entry->dispose);
241 INIT_HLIST_NODE(&entry->hnode);
242
243 return entry;
244}
245
246static void
247nfs4_xattr_free_entry(struct nfs4_xattr_entry *entry)
248{
249 if (entry->flags & NFS4_XATTR_ENTRY_EXTVAL)
250 kvfree(entry->xattr_value);
251 kfree(entry);
252}
253
254static void
255nfs4_xattr_free_entry_cb(struct kref *kref)
256{
257 struct nfs4_xattr_entry *entry;
258
259 entry = container_of(kref, struct nfs4_xattr_entry, ref);
260
261 if (WARN_ON(!list_empty(&entry->lru)))
262 return;
263
264 nfs4_xattr_free_entry(entry);
265}
266
267static void
268nfs4_xattr_free_cache_cb(struct kref *kref)
269{
270 struct nfs4_xattr_cache *cache;
271 int i;
272
273 cache = container_of(kref, struct nfs4_xattr_cache, ref);
274
275 for (i = 0; i < NFS4_XATTR_HASH_SIZE; i++) {
276 if (WARN_ON(!hlist_empty(&cache->buckets[i].hlist)))
277 return;
278 cache->buckets[i].draining = false;
279 }
280
281 cache->listxattr = NULL;
282
283 kmem_cache_free(nfs4_xattr_cache_cachep, cache);
284
285}
286
287static struct nfs4_xattr_cache *
288nfs4_xattr_alloc_cache(void)
289{
290 struct nfs4_xattr_cache *cache;
291
292 cache = kmem_cache_alloc(nfs4_xattr_cache_cachep,
293 GFP_KERNEL_ACCOUNT | GFP_NOFS);
294 if (cache == NULL)
295 return NULL;
296
297 kref_init(&cache->ref);
298 atomic_long_set(&cache->nent, 0);
299
300 return cache;
301}
302
303/*
304 * Set the listxattr cache, which is a special-cased cache entry.
305 * The special value ERR_PTR(-ESTALE) is used to indicate that
306 * the cache is being drained - this prevents a new listxattr
307 * cache from being added to what is now a stale cache.
308 */
309static int
310nfs4_xattr_set_listcache(struct nfs4_xattr_cache *cache,
311 struct nfs4_xattr_entry *new)
312{
313 struct nfs4_xattr_entry *old;
314 int ret = 1;
315
316 spin_lock(&cache->listxattr_lock);
317
318 old = cache->listxattr;
319
320 if (old == ERR_PTR(-ESTALE)) {
321 ret = 0;
322 goto out;
323 }
324
325 cache->listxattr = new;
326 if (new != NULL && new != ERR_PTR(-ESTALE))
327 nfs4_xattr_entry_lru_add(new);
328
329 if (old != NULL) {
330 nfs4_xattr_entry_lru_del(old);
331 kref_put(&old->ref, nfs4_xattr_free_entry_cb);
332 }
333out:
334 spin_unlock(&cache->listxattr_lock);
335
336 return ret;
337}
338
339/*
340 * Unlink a cache from its parent inode, clearing out an invalid
341 * cache. Must be called with i_lock held.
342 */
343static struct nfs4_xattr_cache *
344nfs4_xattr_cache_unlink(struct inode *inode)
345{
346 struct nfs_inode *nfsi;
347 struct nfs4_xattr_cache *oldcache;
348
349 nfsi = NFS_I(inode);
350
351 oldcache = nfsi->xattr_cache;
352 if (oldcache != NULL) {
353 list_lru_del(&nfs4_xattr_cache_lru, &oldcache->lru);
354 oldcache->inode = NULL;
355 }
356 nfsi->xattr_cache = NULL;
357 nfsi->cache_validity &= ~NFS_INO_INVALID_XATTR;
358
359 return oldcache;
360
361}
362
363/*
364 * Discard a cache. Called by get_cache() if there was an old,
365 * invalid cache. Can also be called from a shrinker callback.
366 *
367 * The cache is dead, it has already been unlinked from its inode,
368 * and no longer appears on the cache LRU list.
369 *
370 * Mark all buckets as draining, so that no new entries are added. This
371 * could still happen in the unlikely, but possible case that another
372 * thread had grabbed a reference before it was unlinked from the inode,
373 * and is still holding it for an add operation.
374 *
375 * Remove all entries from the LRU lists, so that there is no longer
376 * any way to 'find' this cache. Then, remove the entries from the hash
377 * table.
378 *
379 * At that point, the cache will remain empty and can be freed when the final
380 * reference drops, which is very likely the kref_put at the end of
381 * this function, or the one called immediately afterwards in the
382 * shrinker callback.
383 */
384static void
385nfs4_xattr_discard_cache(struct nfs4_xattr_cache *cache)
386{
387 unsigned int i;
388 struct nfs4_xattr_entry *entry;
389 struct nfs4_xattr_bucket *bucket;
390 struct hlist_node *n;
391
392 nfs4_xattr_set_listcache(cache, ERR_PTR(-ESTALE));
393
394 for (i = 0; i < NFS4_XATTR_HASH_SIZE; i++) {
395 bucket = &cache->buckets[i];
396
397 spin_lock(&bucket->lock);
398 bucket->draining = true;
399 hlist_for_each_entry_safe(entry, n, &bucket->hlist, hnode) {
400 nfs4_xattr_entry_lru_del(entry);
401 hlist_del_init(&entry->hnode);
402 kref_put(&entry->ref, nfs4_xattr_free_entry_cb);
403 }
404 spin_unlock(&bucket->lock);
405 }
406
407 atomic_long_set(&cache->nent, 0);
408
409 kref_put(&cache->ref, nfs4_xattr_free_cache_cb);
410}
411
412/*
413 * Get a referenced copy of the cache structure. Avoid doing allocs
414 * while holding i_lock. Which means that we do some optimistic allocation,
415 * and might have to free the result in rare cases.
416 *
417 * This function only checks the NFS_INO_INVALID_XATTR cache validity bit
418 * and acts accordingly, replacing the cache when needed. For the read case
419 * (!add), this means that the caller must make sure that the cache
420 * is valid before caling this function. getxattr and listxattr call
421 * revalidate_inode to do this. The attribute cache timeout (for the
422 * non-delegated case) is expected to be dealt with in the revalidate
423 * call.
424 */
425
426static struct nfs4_xattr_cache *
427nfs4_xattr_get_cache(struct inode *inode, int add)
428{
429 struct nfs_inode *nfsi;
430 struct nfs4_xattr_cache *cache, *oldcache, *newcache;
431
432 nfsi = NFS_I(inode);
433
434 cache = oldcache = NULL;
435
436 spin_lock(&inode->i_lock);
437
438 if (nfsi->cache_validity & NFS_INO_INVALID_XATTR)
439 oldcache = nfs4_xattr_cache_unlink(inode);
440 else
441 cache = nfsi->xattr_cache;
442
443 if (cache != NULL)
444 kref_get(&cache->ref);
445
446 spin_unlock(&inode->i_lock);
447
448 if (add && cache == NULL) {
449 newcache = NULL;
450
451 cache = nfs4_xattr_alloc_cache();
452 if (cache == NULL)
453 goto out;
454
455 spin_lock(&inode->i_lock);
456 if (nfsi->cache_validity & NFS_INO_INVALID_XATTR) {
457 /*
458 * The cache was invalidated again. Give up,
459 * since what we want to enter is now likely
460 * outdated anyway.
461 */
462 spin_unlock(&inode->i_lock);
463 kref_put(&cache->ref, nfs4_xattr_free_cache_cb);
464 cache = NULL;
465 goto out;
466 }
467
468 /*
469 * Check if someone beat us to it.
470 */
471 if (nfsi->xattr_cache != NULL) {
472 newcache = nfsi->xattr_cache;
473 kref_get(&newcache->ref);
474 } else {
475 kref_get(&cache->ref);
476 nfsi->xattr_cache = cache;
477 cache->inode = inode;
478 list_lru_add(&nfs4_xattr_cache_lru, &cache->lru);
479 }
480
481 spin_unlock(&inode->i_lock);
482
483 /*
484 * If there was a race, throw away the cache we just
485 * allocated, and use the new one allocated by someone
486 * else.
487 */
488 if (newcache != NULL) {
489 kref_put(&cache->ref, nfs4_xattr_free_cache_cb);
490 cache = newcache;
491 }
492 }
493
494out:
495 /*
496 * Discard the now orphaned old cache.
497 */
498 if (oldcache != NULL)
499 nfs4_xattr_discard_cache(oldcache);
500
501 return cache;
502}
503
504static inline struct nfs4_xattr_bucket *
505nfs4_xattr_hash_bucket(struct nfs4_xattr_cache *cache, const char *name)
506{
507 return &cache->buckets[jhash(name, strlen(name), 0) &
508 (ARRAY_SIZE(cache->buckets) - 1)];
509}
510
511static struct nfs4_xattr_entry *
512nfs4_xattr_get_entry(struct nfs4_xattr_bucket *bucket, const char *name)
513{
514 struct nfs4_xattr_entry *entry;
515
516 entry = NULL;
517
518 hlist_for_each_entry(entry, &bucket->hlist, hnode) {
519 if (!strcmp(entry->xattr_name, name))
520 break;
521 }
522
523 return entry;
524}
525
526static int
527nfs4_xattr_hash_add(struct nfs4_xattr_cache *cache,
528 struct nfs4_xattr_entry *entry)
529{
530 struct nfs4_xattr_bucket *bucket;
531 struct nfs4_xattr_entry *oldentry = NULL;
532 int ret = 1;
533
534 bucket = nfs4_xattr_hash_bucket(cache, entry->xattr_name);
535 entry->bucket = bucket;
536
537 spin_lock(&bucket->lock);
538
539 if (bucket->draining) {
540 ret = 0;
541 goto out;
542 }
543
544 oldentry = nfs4_xattr_get_entry(bucket, entry->xattr_name);
545 if (oldentry != NULL) {
546 hlist_del_init(&oldentry->hnode);
547 nfs4_xattr_entry_lru_del(oldentry);
548 } else {
549 atomic_long_inc(&cache->nent);
550 }
551
552 hlist_add_head(&entry->hnode, &bucket->hlist);
553 nfs4_xattr_entry_lru_add(entry);
554
555out:
556 spin_unlock(&bucket->lock);
557
558 if (oldentry != NULL)
559 kref_put(&oldentry->ref, nfs4_xattr_free_entry_cb);
560
561 return ret;
562}
563
564static void
565nfs4_xattr_hash_remove(struct nfs4_xattr_cache *cache, const char *name)
566{
567 struct nfs4_xattr_bucket *bucket;
568 struct nfs4_xattr_entry *entry;
569
570 bucket = nfs4_xattr_hash_bucket(cache, name);
571
572 spin_lock(&bucket->lock);
573
574 entry = nfs4_xattr_get_entry(bucket, name);
575 if (entry != NULL) {
576 hlist_del_init(&entry->hnode);
577 nfs4_xattr_entry_lru_del(entry);
578 atomic_long_dec(&cache->nent);
579 }
580
581 spin_unlock(&bucket->lock);
582
583 if (entry != NULL)
584 kref_put(&entry->ref, nfs4_xattr_free_entry_cb);
585}
586
587static struct nfs4_xattr_entry *
588nfs4_xattr_hash_find(struct nfs4_xattr_cache *cache, const char *name)
589{
590 struct nfs4_xattr_bucket *bucket;
591 struct nfs4_xattr_entry *entry;
592
593 bucket = nfs4_xattr_hash_bucket(cache, name);
594
595 spin_lock(&bucket->lock);
596
597 entry = nfs4_xattr_get_entry(bucket, name);
598 if (entry != NULL)
599 kref_get(&entry->ref);
600
601 spin_unlock(&bucket->lock);
602
603 return entry;
604}
605
606/*
607 * Entry point to retrieve an entry from the cache.
608 */
609ssize_t nfs4_xattr_cache_get(struct inode *inode, const char *name, char *buf,
610 ssize_t buflen)
611{
612 struct nfs4_xattr_cache *cache;
613 struct nfs4_xattr_entry *entry;
614 ssize_t ret;
615
616 cache = nfs4_xattr_get_cache(inode, 0);
617 if (cache == NULL)
618 return -ENOENT;
619
620 ret = 0;
621 entry = nfs4_xattr_hash_find(cache, name);
622
623 if (entry != NULL) {
624 dprintk("%s: cache hit '%s', len %lu\n", __func__,
625 entry->xattr_name, (unsigned long)entry->xattr_size);
626 if (buflen == 0) {
627 /* Length probe only */
628 ret = entry->xattr_size;
629 } else if (buflen < entry->xattr_size)
630 ret = -ERANGE;
631 else {
632 memcpy(buf, entry->xattr_value, entry->xattr_size);
633 ret = entry->xattr_size;
634 }
635 kref_put(&entry->ref, nfs4_xattr_free_entry_cb);
636 } else {
637 dprintk("%s: cache miss '%s'\n", __func__, name);
638 ret = -ENOENT;
639 }
640
641 kref_put(&cache->ref, nfs4_xattr_free_cache_cb);
642
643 return ret;
644}
645
646/*
647 * Retrieve a cached list of xattrs from the cache.
648 */
649ssize_t nfs4_xattr_cache_list(struct inode *inode, char *buf, ssize_t buflen)
650{
651 struct nfs4_xattr_cache *cache;
652 struct nfs4_xattr_entry *entry;
653 ssize_t ret;
654
655 cache = nfs4_xattr_get_cache(inode, 0);
656 if (cache == NULL)
657 return -ENOENT;
658
659 spin_lock(&cache->listxattr_lock);
660
661 entry = cache->listxattr;
662
663 if (entry != NULL && entry != ERR_PTR(-ESTALE)) {
664 if (buflen == 0) {
665 /* Length probe only */
666 ret = entry->xattr_size;
667 } else if (entry->xattr_size > buflen)
668 ret = -ERANGE;
669 else {
670 memcpy(buf, entry->xattr_value, entry->xattr_size);
671 ret = entry->xattr_size;
672 }
673 } else {
674 ret = -ENOENT;
675 }
676
677 spin_unlock(&cache->listxattr_lock);
678
679 kref_put(&cache->ref, nfs4_xattr_free_cache_cb);
680
681 return ret;
682}
683
684/*
685 * Add an xattr to the cache.
686 *
687 * This also invalidates the xattr list cache.
688 */
689void nfs4_xattr_cache_add(struct inode *inode, const char *name,
690 const char *buf, struct page **pages, ssize_t buflen)
691{
692 struct nfs4_xattr_cache *cache;
693 struct nfs4_xattr_entry *entry;
694
695 dprintk("%s: add '%s' len %lu\n", __func__,
696 name, (unsigned long)buflen);
697
698 cache = nfs4_xattr_get_cache(inode, 1);
699 if (cache == NULL)
700 return;
701
702 entry = nfs4_xattr_alloc_entry(name, buf, pages, buflen);
703 if (entry == NULL)
704 goto out;
705
706 (void)nfs4_xattr_set_listcache(cache, NULL);
707
708 if (!nfs4_xattr_hash_add(cache, entry))
709 kref_put(&entry->ref, nfs4_xattr_free_entry_cb);
710
711out:
712 kref_put(&cache->ref, nfs4_xattr_free_cache_cb);
713}
714
715
716/*
717 * Remove an xattr from the cache.
718 *
719 * This also invalidates the xattr list cache.
720 */
721void nfs4_xattr_cache_remove(struct inode *inode, const char *name)
722{
723 struct nfs4_xattr_cache *cache;
724
725 dprintk("%s: remove '%s'\n", __func__, name);
726
727 cache = nfs4_xattr_get_cache(inode, 0);
728 if (cache == NULL)
729 return;
730
731 (void)nfs4_xattr_set_listcache(cache, NULL);
732 nfs4_xattr_hash_remove(cache, name);
733
734 kref_put(&cache->ref, nfs4_xattr_free_cache_cb);
735}
736
737/*
738 * Cache listxattr output, replacing any possible old one.
739 */
740void nfs4_xattr_cache_set_list(struct inode *inode, const char *buf,
741 ssize_t buflen)
742{
743 struct nfs4_xattr_cache *cache;
744 struct nfs4_xattr_entry *entry;
745
746 cache = nfs4_xattr_get_cache(inode, 1);
747 if (cache == NULL)
748 return;
749
750 entry = nfs4_xattr_alloc_entry(NULL, buf, NULL, buflen);
751 if (entry == NULL)
752 goto out;
753
754 /*
755 * This is just there to be able to get to bucket->cache,
756 * which is obviously the same for all buckets, so just
757 * use bucket 0.
758 */
759 entry->bucket = &cache->buckets[0];
760
761 if (!nfs4_xattr_set_listcache(cache, entry))
762 kref_put(&entry->ref, nfs4_xattr_free_entry_cb);
763
764out:
765 kref_put(&cache->ref, nfs4_xattr_free_cache_cb);
766}
767
768/*
769 * Zap the entire cache. Called when an inode is evicted.
770 */
771void nfs4_xattr_cache_zap(struct inode *inode)
772{
773 struct nfs4_xattr_cache *oldcache;
774
775 spin_lock(&inode->i_lock);
776 oldcache = nfs4_xattr_cache_unlink(inode);
777 spin_unlock(&inode->i_lock);
778
779 if (oldcache)
780 nfs4_xattr_discard_cache(oldcache);
781}
782
783/*
784 * The entry LRU is shrunk more aggressively than the cache LRU,
785 * by settings @seeks to 1.
786 *
787 * Cache structures are freed only when they've become empty, after
788 * pruning all but one entry.
789 */
790
791static unsigned long nfs4_xattr_cache_count(struct shrinker *shrink,
792 struct shrink_control *sc);
793static unsigned long nfs4_xattr_entry_count(struct shrinker *shrink,
794 struct shrink_control *sc);
795static unsigned long nfs4_xattr_cache_scan(struct shrinker *shrink,
796 struct shrink_control *sc);
797static unsigned long nfs4_xattr_entry_scan(struct shrinker *shrink,
798 struct shrink_control *sc);
799
800static struct shrinker nfs4_xattr_cache_shrinker = {
801 .count_objects = nfs4_xattr_cache_count,
802 .scan_objects = nfs4_xattr_cache_scan,
803 .seeks = DEFAULT_SEEKS,
804 .flags = SHRINKER_MEMCG_AWARE,
805};
806
807static struct shrinker nfs4_xattr_entry_shrinker = {
808 .count_objects = nfs4_xattr_entry_count,
809 .scan_objects = nfs4_xattr_entry_scan,
810 .seeks = DEFAULT_SEEKS,
811 .batch = 512,
812 .flags = SHRINKER_MEMCG_AWARE,
813};
814
815static struct shrinker nfs4_xattr_large_entry_shrinker = {
816 .count_objects = nfs4_xattr_entry_count,
817 .scan_objects = nfs4_xattr_entry_scan,
818 .seeks = 1,
819 .batch = 512,
820 .flags = SHRINKER_MEMCG_AWARE,
821};
822
823static enum lru_status
824cache_lru_isolate(struct list_head *item,
825 struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
826{
827 struct list_head *dispose = arg;
828 struct inode *inode;
829 struct nfs4_xattr_cache *cache = container_of(item,
830 struct nfs4_xattr_cache, lru);
831
832 if (atomic_long_read(&cache->nent) > 1)
833 return LRU_SKIP;
834
835 /*
836 * If a cache structure is on the LRU list, we know that
837 * its inode is valid. Try to lock it to break the link.
838 * Since we're inverting the lock order here, only try.
839 */
840 inode = cache->inode;
841
842 if (!spin_trylock(&inode->i_lock))
843 return LRU_SKIP;
844
845 kref_get(&cache->ref);
846
847 cache->inode = NULL;
848 NFS_I(inode)->xattr_cache = NULL;
849 NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_XATTR;
850 list_lru_isolate(lru, &cache->lru);
851
852 spin_unlock(&inode->i_lock);
853
854 list_add_tail(&cache->dispose, dispose);
855 return LRU_REMOVED;
856}
857
858static unsigned long
859nfs4_xattr_cache_scan(struct shrinker *shrink, struct shrink_control *sc)
860{
861 LIST_HEAD(dispose);
862 unsigned long freed;
863 struct nfs4_xattr_cache *cache;
864
865 freed = list_lru_shrink_walk(&nfs4_xattr_cache_lru, sc,
866 cache_lru_isolate, &dispose);
867 while (!list_empty(&dispose)) {
868 cache = list_first_entry(&dispose, struct nfs4_xattr_cache,
869 dispose);
870 list_del_init(&cache->dispose);
871 nfs4_xattr_discard_cache(cache);
872 kref_put(&cache->ref, nfs4_xattr_free_cache_cb);
873 }
874
875 return freed;
876}
877
878
879static unsigned long
880nfs4_xattr_cache_count(struct shrinker *shrink, struct shrink_control *sc)
881{
882 unsigned long count;
883
884 count = list_lru_shrink_count(&nfs4_xattr_cache_lru, sc);
885 return vfs_pressure_ratio(count);
886}
887
888static enum lru_status
889entry_lru_isolate(struct list_head *item,
890 struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
891{
892 struct list_head *dispose = arg;
893 struct nfs4_xattr_bucket *bucket;
894 struct nfs4_xattr_cache *cache;
895 struct nfs4_xattr_entry *entry = container_of(item,
896 struct nfs4_xattr_entry, lru);
897
898 bucket = entry->bucket;
899 cache = bucket->cache;
900
901 /*
902 * Unhook the entry from its parent (either a cache bucket
903 * or a cache structure if it's a listxattr buf), so that
904 * it's no longer found. Then add it to the isolate list,
905 * to be freed later.
906 *
907 * In both cases, we're reverting lock order, so use
908 * trylock and skip the entry if we can't get the lock.
909 */
910 if (entry->xattr_name != NULL) {
911 /* Regular cache entry */
912 if (!spin_trylock(&bucket->lock))
913 return LRU_SKIP;
914
915 kref_get(&entry->ref);
916
917 hlist_del_init(&entry->hnode);
918 atomic_long_dec(&cache->nent);
919 list_lru_isolate(lru, &entry->lru);
920
921 spin_unlock(&bucket->lock);
922 } else {
923 /* Listxattr cache entry */
924 if (!spin_trylock(&cache->listxattr_lock))
925 return LRU_SKIP;
926
927 kref_get(&entry->ref);
928
929 cache->listxattr = NULL;
930 list_lru_isolate(lru, &entry->lru);
931
932 spin_unlock(&cache->listxattr_lock);
933 }
934
935 list_add_tail(&entry->dispose, dispose);
936 return LRU_REMOVED;
937}
938
939static unsigned long
940nfs4_xattr_entry_scan(struct shrinker *shrink, struct shrink_control *sc)
941{
942 LIST_HEAD(dispose);
943 unsigned long freed;
944 struct nfs4_xattr_entry *entry;
945 struct list_lru *lru;
946
947 lru = (shrink == &nfs4_xattr_large_entry_shrinker) ?
948 &nfs4_xattr_large_entry_lru : &nfs4_xattr_entry_lru;
949
950 freed = list_lru_shrink_walk(lru, sc, entry_lru_isolate, &dispose);
951
952 while (!list_empty(&dispose)) {
953 entry = list_first_entry(&dispose, struct nfs4_xattr_entry,
954 dispose);
955 list_del_init(&entry->dispose);
956
957 /*
958 * Drop two references: the one that we just grabbed
959 * in entry_lru_isolate, and the one that was set
960 * when the entry was first allocated.
961 */
962 kref_put(&entry->ref, nfs4_xattr_free_entry_cb);
963 kref_put(&entry->ref, nfs4_xattr_free_entry_cb);
964 }
965
966 return freed;
967}
968
969static unsigned long
970nfs4_xattr_entry_count(struct shrinker *shrink, struct shrink_control *sc)
971{
972 unsigned long count;
973 struct list_lru *lru;
974
975 lru = (shrink == &nfs4_xattr_large_entry_shrinker) ?
976 &nfs4_xattr_large_entry_lru : &nfs4_xattr_entry_lru;
977
978 count = list_lru_shrink_count(lru, sc);
979 return vfs_pressure_ratio(count);
980}
981
982
983static void nfs4_xattr_cache_init_once(void *p)
984{
985 struct nfs4_xattr_cache *cache = (struct nfs4_xattr_cache *)p;
986
987 spin_lock_init(&cache->listxattr_lock);
988 atomic_long_set(&cache->nent, 0);
989 nfs4_xattr_hash_init(cache);
990 cache->listxattr = NULL;
991 INIT_LIST_HEAD(&cache->lru);
992 INIT_LIST_HEAD(&cache->dispose);
993}
994
995int __init nfs4_xattr_cache_init(void)
996{
997 int ret = 0;
998
999 nfs4_xattr_cache_cachep = kmem_cache_create("nfs4_xattr_cache_cache",
1000 sizeof(struct nfs4_xattr_cache), 0,
1001 (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD|SLAB_ACCOUNT),
1002 nfs4_xattr_cache_init_once);
1003 if (nfs4_xattr_cache_cachep == NULL)
1004 return -ENOMEM;
1005
1006 ret = list_lru_init_memcg(&nfs4_xattr_large_entry_lru,
1007 &nfs4_xattr_large_entry_shrinker);
1008 if (ret)
1009 goto out4;
1010
1011 ret = list_lru_init_memcg(&nfs4_xattr_entry_lru,
1012 &nfs4_xattr_entry_shrinker);
1013 if (ret)
1014 goto out3;
1015
1016 ret = list_lru_init_memcg(&nfs4_xattr_cache_lru,
1017 &nfs4_xattr_cache_shrinker);
1018 if (ret)
1019 goto out2;
1020
1021 ret = register_shrinker(&nfs4_xattr_cache_shrinker);
1022 if (ret)
1023 goto out1;
1024
1025 ret = register_shrinker(&nfs4_xattr_entry_shrinker);
1026 if (ret)
1027 goto out;
1028
1029 ret = register_shrinker(&nfs4_xattr_large_entry_shrinker);
1030 if (!ret)
1031 return 0;
1032
1033 unregister_shrinker(&nfs4_xattr_entry_shrinker);
1034out:
1035 unregister_shrinker(&nfs4_xattr_cache_shrinker);
1036out1:
1037 list_lru_destroy(&nfs4_xattr_cache_lru);
1038out2:
1039 list_lru_destroy(&nfs4_xattr_entry_lru);
1040out3:
1041 list_lru_destroy(&nfs4_xattr_large_entry_lru);
1042out4:
1043 kmem_cache_destroy(nfs4_xattr_cache_cachep);
1044
1045 return ret;
1046}
1047
1048void nfs4_xattr_cache_exit(void)
1049{
1050 unregister_shrinker(&nfs4_xattr_large_entry_shrinker);
1051 unregister_shrinker(&nfs4_xattr_entry_shrinker);
1052 unregister_shrinker(&nfs4_xattr_cache_shrinker);
1053 list_lru_destroy(&nfs4_xattr_large_entry_lru);
1054 list_lru_destroy(&nfs4_xattr_entry_lru);
1055 list_lru_destroy(&nfs4_xattr_cache_lru);
1056 kmem_cache_destroy(nfs4_xattr_cache_cachep);
1057}