Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * net/core/dst_cache.c - dst entry cache
4 *
5 * Copyright (c) 2016 Paolo Abeni <pabeni@redhat.com>
6 */
7
8#include <linux/kernel.h>
9#include <linux/percpu.h>
10#include <net/dst_cache.h>
11#include <net/route.h>
12#if IS_ENABLED(CONFIG_IPV6)
13#include <net/ip6_fib.h>
14#endif
15#include <uapi/linux/in.h>
16
17struct dst_cache_pcpu {
18 unsigned long refresh_ts;
19 struct dst_entry *dst;
20 u32 cookie;
21 union {
22 struct in_addr in_saddr;
23 struct in6_addr in6_saddr;
24 };
25};
26
27static void dst_cache_per_cpu_dst_set(struct dst_cache_pcpu *dst_cache,
28 struct dst_entry *dst, u32 cookie)
29{
30 DEBUG_NET_WARN_ON_ONCE(!in_softirq());
31 dst_release(dst_cache->dst);
32 if (dst)
33 dst_hold(dst);
34
35 dst_cache->cookie = cookie;
36 dst_cache->dst = dst;
37}
38
39static struct dst_entry *dst_cache_per_cpu_get(struct dst_cache *dst_cache,
40 struct dst_cache_pcpu *idst)
41{
42 struct dst_entry *dst;
43
44 DEBUG_NET_WARN_ON_ONCE(!in_softirq());
45 dst = idst->dst;
46 if (!dst)
47 goto fail;
48
49 /* the cache already hold a dst reference; it can't go away */
50 dst_hold(dst);
51
52 if (unlikely(!time_after(idst->refresh_ts,
53 READ_ONCE(dst_cache->reset_ts)) ||
54 (dst->obsolete && !dst->ops->check(dst, idst->cookie)))) {
55 dst_cache_per_cpu_dst_set(idst, NULL, 0);
56 dst_release(dst);
57 goto fail;
58 }
59 return dst;
60
61fail:
62 idst->refresh_ts = jiffies;
63 return NULL;
64}
65
66struct dst_entry *dst_cache_get(struct dst_cache *dst_cache)
67{
68 if (!dst_cache->cache)
69 return NULL;
70
71 return dst_cache_per_cpu_get(dst_cache, this_cpu_ptr(dst_cache->cache));
72}
73EXPORT_SYMBOL_GPL(dst_cache_get);
74
75struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr)
76{
77 struct dst_cache_pcpu *idst;
78 struct dst_entry *dst;
79
80 if (!dst_cache->cache)
81 return NULL;
82
83 idst = this_cpu_ptr(dst_cache->cache);
84 dst = dst_cache_per_cpu_get(dst_cache, idst);
85 if (!dst)
86 return NULL;
87
88 *saddr = idst->in_saddr.s_addr;
89 return dst_rtable(dst);
90}
91EXPORT_SYMBOL_GPL(dst_cache_get_ip4);
92
93void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst,
94 __be32 saddr)
95{
96 struct dst_cache_pcpu *idst;
97
98 if (!dst_cache->cache)
99 return;
100
101 idst = this_cpu_ptr(dst_cache->cache);
102 dst_cache_per_cpu_dst_set(idst, dst, 0);
103 idst->in_saddr.s_addr = saddr;
104}
105EXPORT_SYMBOL_GPL(dst_cache_set_ip4);
106
107#if IS_ENABLED(CONFIG_IPV6)
108void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst,
109 const struct in6_addr *saddr)
110{
111 struct dst_cache_pcpu *idst;
112
113 if (!dst_cache->cache)
114 return;
115
116 idst = this_cpu_ptr(dst_cache->cache);
117 dst_cache_per_cpu_dst_set(idst, dst,
118 rt6_get_cookie(dst_rt6_info(dst)));
119 idst->in6_saddr = *saddr;
120}
121EXPORT_SYMBOL_GPL(dst_cache_set_ip6);
122
123struct dst_entry *dst_cache_get_ip6(struct dst_cache *dst_cache,
124 struct in6_addr *saddr)
125{
126 struct dst_cache_pcpu *idst;
127 struct dst_entry *dst;
128
129 if (!dst_cache->cache)
130 return NULL;
131
132 idst = this_cpu_ptr(dst_cache->cache);
133 dst = dst_cache_per_cpu_get(dst_cache, idst);
134 if (!dst)
135 return NULL;
136
137 *saddr = idst->in6_saddr;
138 return dst;
139}
140EXPORT_SYMBOL_GPL(dst_cache_get_ip6);
141#endif
142
143int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp)
144{
145 dst_cache->cache = alloc_percpu_gfp(struct dst_cache_pcpu,
146 gfp | __GFP_ZERO);
147 if (!dst_cache->cache)
148 return -ENOMEM;
149
150 dst_cache_reset(dst_cache);
151 return 0;
152}
153EXPORT_SYMBOL_GPL(dst_cache_init);
154
155void dst_cache_destroy(struct dst_cache *dst_cache)
156{
157 int i;
158
159 if (!dst_cache->cache)
160 return;
161
162 for_each_possible_cpu(i)
163 dst_release(per_cpu_ptr(dst_cache->cache, i)->dst);
164
165 free_percpu(dst_cache->cache);
166}
167EXPORT_SYMBOL_GPL(dst_cache_destroy);
168
169void dst_cache_reset_now(struct dst_cache *dst_cache)
170{
171 int i;
172
173 if (!dst_cache->cache)
174 return;
175
176 dst_cache_reset(dst_cache);
177 for_each_possible_cpu(i) {
178 struct dst_cache_pcpu *idst = per_cpu_ptr(dst_cache->cache, i);
179 struct dst_entry *dst = idst->dst;
180
181 idst->cookie = 0;
182 idst->dst = NULL;
183 dst_release(dst);
184 }
185}
186EXPORT_SYMBOL_GPL(dst_cache_reset_now);
1/*
2 * net/core/dst_cache.c - dst entry cache
3 *
4 * Copyright (c) 2016 Paolo Abeni <pabeni@redhat.com>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 */
11
12#include <linux/kernel.h>
13#include <linux/percpu.h>
14#include <net/dst_cache.h>
15#include <net/route.h>
16#if IS_ENABLED(CONFIG_IPV6)
17#include <net/ip6_fib.h>
18#endif
19#include <uapi/linux/in.h>
20
21struct dst_cache_pcpu {
22 unsigned long refresh_ts;
23 struct dst_entry *dst;
24 u32 cookie;
25 union {
26 struct in_addr in_saddr;
27 struct in6_addr in6_saddr;
28 };
29};
30
31static void dst_cache_per_cpu_dst_set(struct dst_cache_pcpu *dst_cache,
32 struct dst_entry *dst, u32 cookie)
33{
34 dst_release(dst_cache->dst);
35 if (dst)
36 dst_hold(dst);
37
38 dst_cache->cookie = cookie;
39 dst_cache->dst = dst;
40}
41
42static struct dst_entry *dst_cache_per_cpu_get(struct dst_cache *dst_cache,
43 struct dst_cache_pcpu *idst)
44{
45 struct dst_entry *dst;
46
47 dst = idst->dst;
48 if (!dst)
49 goto fail;
50
51 /* the cache already hold a dst reference; it can't go away */
52 dst_hold(dst);
53
54 if (unlikely(!time_after(idst->refresh_ts, dst_cache->reset_ts) ||
55 (dst->obsolete && !dst->ops->check(dst, idst->cookie)))) {
56 dst_cache_per_cpu_dst_set(idst, NULL, 0);
57 dst_release(dst);
58 goto fail;
59 }
60 return dst;
61
62fail:
63 idst->refresh_ts = jiffies;
64 return NULL;
65}
66
67struct dst_entry *dst_cache_get(struct dst_cache *dst_cache)
68{
69 if (!dst_cache->cache)
70 return NULL;
71
72 return dst_cache_per_cpu_get(dst_cache, this_cpu_ptr(dst_cache->cache));
73}
74EXPORT_SYMBOL_GPL(dst_cache_get);
75
76struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr)
77{
78 struct dst_cache_pcpu *idst;
79 struct dst_entry *dst;
80
81 if (!dst_cache->cache)
82 return NULL;
83
84 idst = this_cpu_ptr(dst_cache->cache);
85 dst = dst_cache_per_cpu_get(dst_cache, idst);
86 if (!dst)
87 return NULL;
88
89 *saddr = idst->in_saddr.s_addr;
90 return container_of(dst, struct rtable, dst);
91}
92EXPORT_SYMBOL_GPL(dst_cache_get_ip4);
93
94void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst,
95 __be32 saddr)
96{
97 struct dst_cache_pcpu *idst;
98
99 if (!dst_cache->cache)
100 return;
101
102 idst = this_cpu_ptr(dst_cache->cache);
103 dst_cache_per_cpu_dst_set(idst, dst, 0);
104 idst->in_saddr.s_addr = saddr;
105}
106EXPORT_SYMBOL_GPL(dst_cache_set_ip4);
107
108#if IS_ENABLED(CONFIG_IPV6)
109void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst,
110 const struct in6_addr *addr)
111{
112 struct dst_cache_pcpu *idst;
113
114 if (!dst_cache->cache)
115 return;
116
117 idst = this_cpu_ptr(dst_cache->cache);
118 dst_cache_per_cpu_dst_set(this_cpu_ptr(dst_cache->cache), dst,
119 rt6_get_cookie((struct rt6_info *)dst));
120 idst->in6_saddr = *addr;
121}
122EXPORT_SYMBOL_GPL(dst_cache_set_ip6);
123
124struct dst_entry *dst_cache_get_ip6(struct dst_cache *dst_cache,
125 struct in6_addr *saddr)
126{
127 struct dst_cache_pcpu *idst;
128 struct dst_entry *dst;
129
130 if (!dst_cache->cache)
131 return NULL;
132
133 idst = this_cpu_ptr(dst_cache->cache);
134 dst = dst_cache_per_cpu_get(dst_cache, idst);
135 if (!dst)
136 return NULL;
137
138 *saddr = idst->in6_saddr;
139 return dst;
140}
141EXPORT_SYMBOL_GPL(dst_cache_get_ip6);
142#endif
143
144int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp)
145{
146 dst_cache->cache = alloc_percpu_gfp(struct dst_cache_pcpu,
147 gfp | __GFP_ZERO);
148 if (!dst_cache->cache)
149 return -ENOMEM;
150
151 dst_cache_reset(dst_cache);
152 return 0;
153}
154EXPORT_SYMBOL_GPL(dst_cache_init);
155
156void dst_cache_destroy(struct dst_cache *dst_cache)
157{
158 int i;
159
160 if (!dst_cache->cache)
161 return;
162
163 for_each_possible_cpu(i)
164 dst_release(per_cpu_ptr(dst_cache->cache, i)->dst);
165
166 free_percpu(dst_cache->cache);
167}
168EXPORT_SYMBOL_GPL(dst_cache_destroy);