1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 * net/core/dst_cache.c - dst entry cache
  4 *
  5 * Copyright (c) 2016 Paolo Abeni <pabeni@redhat.com>
  6 */
  7
  8#include <linux/kernel.h>
  9#include <linux/percpu.h>
 10#include <net/dst_cache.h>
 11#include <net/route.h>
 12#if IS_ENABLED(CONFIG_IPV6)
 13#include <net/ip6_fib.h>
 14#endif
 15#include <uapi/linux/in.h>
 16
 17struct dst_cache_pcpu {
 18	unsigned long refresh_ts;
 19	struct dst_entry *dst;
 20	u32 cookie;
 21	union {
 22		struct in_addr in_saddr;
 23		struct in6_addr in6_saddr;
 24	};
 25};
 26
 27static void dst_cache_per_cpu_dst_set(struct dst_cache_pcpu *dst_cache,
 28				      struct dst_entry *dst, u32 cookie)
 29{
 30	DEBUG_NET_WARN_ON_ONCE(!in_softirq());
 31	dst_release(dst_cache->dst);
 32	if (dst)
 33		dst_hold(dst);
 34
 35	dst_cache->cookie = cookie;
 36	dst_cache->dst = dst;
 37}
 38
 39static struct dst_entry *dst_cache_per_cpu_get(struct dst_cache *dst_cache,
 40					       struct dst_cache_pcpu *idst)
 41{
 42	struct dst_entry *dst;
 43
 44	DEBUG_NET_WARN_ON_ONCE(!in_softirq());
 45	dst = idst->dst;
 46	if (!dst)
 47		goto fail;
 48
 49	/* the cache already hold a dst reference; it can't go away */
 50	dst_hold(dst);
 51
 52	if (unlikely(!time_after(idst->refresh_ts,
 53				 READ_ONCE(dst_cache->reset_ts)) ||
 54		     (dst->obsolete && !dst->ops->check(dst, idst->cookie)))) {
 55		dst_cache_per_cpu_dst_set(idst, NULL, 0);
 56		dst_release(dst);
 57		goto fail;
 58	}
 59	return dst;
 60
 61fail:
 62	idst->refresh_ts = jiffies;
 63	return NULL;
 64}
 65
 66struct dst_entry *dst_cache_get(struct dst_cache *dst_cache)
 67{
 68	if (!dst_cache->cache)
 69		return NULL;
 70
 71	return dst_cache_per_cpu_get(dst_cache, this_cpu_ptr(dst_cache->cache));
 72}
 73EXPORT_SYMBOL_GPL(dst_cache_get);
 74
 75struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr)
 76{
 77	struct dst_cache_pcpu *idst;
 78	struct dst_entry *dst;
 79
 80	if (!dst_cache->cache)
 81		return NULL;
 82
 83	idst = this_cpu_ptr(dst_cache->cache);
 84	dst = dst_cache_per_cpu_get(dst_cache, idst);
 85	if (!dst)
 86		return NULL;
 87
 88	*saddr = idst->in_saddr.s_addr;
 89	return dst_rtable(dst);
 90}
 91EXPORT_SYMBOL_GPL(dst_cache_get_ip4);
 92
 93void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst,
 94		       __be32 saddr)
 95{
 96	struct dst_cache_pcpu *idst;
 97
 98	if (!dst_cache->cache)
 99		return;
100
101	idst = this_cpu_ptr(dst_cache->cache);
102	dst_cache_per_cpu_dst_set(idst, dst, 0);
103	idst->in_saddr.s_addr = saddr;
104}
105EXPORT_SYMBOL_GPL(dst_cache_set_ip4);
106
107#if IS_ENABLED(CONFIG_IPV6)
108void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst,
109		       const struct in6_addr *saddr)
110{
111	struct dst_cache_pcpu *idst;
112
113	if (!dst_cache->cache)
114		return;
115
116	idst = this_cpu_ptr(dst_cache->cache);
117	dst_cache_per_cpu_dst_set(idst, dst,
118				  rt6_get_cookie(dst_rt6_info(dst)));
119	idst->in6_saddr = *saddr;
120}
121EXPORT_SYMBOL_GPL(dst_cache_set_ip6);
122
123struct dst_entry *dst_cache_get_ip6(struct dst_cache *dst_cache,
124				    struct in6_addr *saddr)
125{
126	struct dst_cache_pcpu *idst;
127	struct dst_entry *dst;
128
129	if (!dst_cache->cache)
130		return NULL;
131
132	idst = this_cpu_ptr(dst_cache->cache);
133	dst = dst_cache_per_cpu_get(dst_cache, idst);
134	if (!dst)
135		return NULL;
136
137	*saddr = idst->in6_saddr;
138	return dst;
139}
140EXPORT_SYMBOL_GPL(dst_cache_get_ip6);
141#endif
142
143int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp)
144{
145	dst_cache->cache = alloc_percpu_gfp(struct dst_cache_pcpu,
146					    gfp | __GFP_ZERO);
147	if (!dst_cache->cache)
148		return -ENOMEM;
149
150	dst_cache_reset(dst_cache);
151	return 0;
152}
153EXPORT_SYMBOL_GPL(dst_cache_init);
154
155void dst_cache_destroy(struct dst_cache *dst_cache)
156{
157	int i;
158
159	if (!dst_cache->cache)
160		return;
161
162	for_each_possible_cpu(i)
163		dst_release(per_cpu_ptr(dst_cache->cache, i)->dst);
164
165	free_percpu(dst_cache->cache);
166}
167EXPORT_SYMBOL_GPL(dst_cache_destroy);
168
169void dst_cache_reset_now(struct dst_cache *dst_cache)
170{
171	int i;
172
173	if (!dst_cache->cache)
174		return;
175
176	dst_cache_reset(dst_cache);
177	for_each_possible_cpu(i) {
178		struct dst_cache_pcpu *idst = per_cpu_ptr(dst_cache->cache, i);
179		struct dst_entry *dst = idst->dst;
180
181		idst->cookie = 0;
182		idst->dst = NULL;
183		dst_release(dst);
184	}
185}
186EXPORT_SYMBOL_GPL(dst_cache_reset_now);