1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * INET		An implementation of the TCP/IP protocol suite for the LINUX
   4 *		operating system.  INET is implemented using the  BSD Socket
   5 *		interface as the means of communication with the user level.
   6 *
   7 *		IPv4 Forwarding Information Base: semantics.
   8 *
   9 * Authors:	Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 
 
 
 
 
  10 */
  11
  12#include <linux/uaccess.h>
  13#include <linux/bitops.h>
  14#include <linux/types.h>
  15#include <linux/kernel.h>
  16#include <linux/jiffies.h>
  17#include <linux/mm.h>
  18#include <linux/string.h>
  19#include <linux/socket.h>
  20#include <linux/sockios.h>
  21#include <linux/errno.h>
  22#include <linux/in.h>
  23#include <linux/inet.h>
  24#include <linux/inetdevice.h>
  25#include <linux/netdevice.h>
  26#include <linux/if_arp.h>
  27#include <linux/proc_fs.h>
  28#include <linux/skbuff.h>
  29#include <linux/init.h>
  30#include <linux/slab.h>
  31#include <linux/netlink.h>
  32
  33#include <net/arp.h>
  34#include <net/ip.h>
  35#include <net/protocol.h>
  36#include <net/route.h>
  37#include <net/tcp.h>
  38#include <net/sock.h>
  39#include <net/ip_fib.h>
  40#include <net/ip6_fib.h>
  41#include <net/nexthop.h>
  42#include <net/netlink.h>
  43#include <net/rtnh.h>
  44#include <net/lwtunnel.h>
  45#include <net/fib_notifier.h>
  46#include <net/addrconf.h>
  47
  48#include "fib_lookup.h"
  49
  50static DEFINE_SPINLOCK(fib_info_lock);
  51static struct hlist_head *fib_info_hash;
  52static struct hlist_head *fib_info_laddrhash;
  53static unsigned int fib_info_hash_size;
  54static unsigned int fib_info_cnt;
  55
  56#define DEVINDEX_HASHBITS 8
  57#define DEVINDEX_HASHSIZE (1U << DEVINDEX_HASHBITS)
  58static struct hlist_head fib_info_devhash[DEVINDEX_HASHSIZE];
  59
  60/* for_nexthops and change_nexthops only used when nexthop object
  61 * is not set in a fib_info. The logic within can reference fib_nh.
  62 */
  63#ifdef CONFIG_IP_ROUTE_MULTIPATH
  64
  65#define for_nexthops(fi) {						\
  66	int nhsel; const struct fib_nh *nh;				\
  67	for (nhsel = 0, nh = (fi)->fib_nh;				\
  68	     nhsel < fib_info_num_path((fi));				\
  69	     nh++, nhsel++)
  70
  71#define change_nexthops(fi) {						\
  72	int nhsel; struct fib_nh *nexthop_nh;				\
  73	for (nhsel = 0,	nexthop_nh = (struct fib_nh *)((fi)->fib_nh);	\
  74	     nhsel < fib_info_num_path((fi));				\
  75	     nexthop_nh++, nhsel++)
  76
  77#else /* CONFIG_IP_ROUTE_MULTIPATH */
  78
  79/* Hope, that gcc will optimize it to get rid of dummy loop */
  80
  81#define for_nexthops(fi) {						\
  82	int nhsel; const struct fib_nh *nh = (fi)->fib_nh;		\
  83	for (nhsel = 0; nhsel < 1; nhsel++)
  84
  85#define change_nexthops(fi) {						\
  86	int nhsel;							\
  87	struct fib_nh *nexthop_nh = (struct fib_nh *)((fi)->fib_nh);	\
  88	for (nhsel = 0; nhsel < 1; nhsel++)
  89
  90#endif /* CONFIG_IP_ROUTE_MULTIPATH */
  91
  92#define endfor_nexthops(fi) }
  93
  94
  95const struct fib_prop fib_props[RTN_MAX + 1] = {
  96	[RTN_UNSPEC] = {
  97		.error	= 0,
  98		.scope	= RT_SCOPE_NOWHERE,
  99	},
 100	[RTN_UNICAST] = {
 101		.error	= 0,
 102		.scope	= RT_SCOPE_UNIVERSE,
 103	},
 104	[RTN_LOCAL] = {
 105		.error	= 0,
 106		.scope	= RT_SCOPE_HOST,
 107	},
 108	[RTN_BROADCAST] = {
 109		.error	= 0,
 110		.scope	= RT_SCOPE_LINK,
 111	},
 112	[RTN_ANYCAST] = {
 113		.error	= 0,
 114		.scope	= RT_SCOPE_LINK,
 115	},
 116	[RTN_MULTICAST] = {
 117		.error	= 0,
 118		.scope	= RT_SCOPE_UNIVERSE,
 119	},
 120	[RTN_BLACKHOLE] = {
 121		.error	= -EINVAL,
 122		.scope	= RT_SCOPE_UNIVERSE,
 123	},
 124	[RTN_UNREACHABLE] = {
 125		.error	= -EHOSTUNREACH,
 126		.scope	= RT_SCOPE_UNIVERSE,
 127	},
 128	[RTN_PROHIBIT] = {
 129		.error	= -EACCES,
 130		.scope	= RT_SCOPE_UNIVERSE,
 131	},
 132	[RTN_THROW] = {
 133		.error	= -EAGAIN,
 134		.scope	= RT_SCOPE_UNIVERSE,
 135	},
 136	[RTN_NAT] = {
 137		.error	= -EINVAL,
 138		.scope	= RT_SCOPE_NOWHERE,
 139	},
 140	[RTN_XRESOLVE] = {
 141		.error	= -EINVAL,
 142		.scope	= RT_SCOPE_NOWHERE,
 143	},
 144};
 145
 146static void rt_fibinfo_free(struct rtable __rcu **rtp)
 147{
 148	struct rtable *rt = rcu_dereference_protected(*rtp, 1);
 149
 150	if (!rt)
 151		return;
 152
 153	/* Not even needed : RCU_INIT_POINTER(*rtp, NULL);
 154	 * because we waited an RCU grace period before calling
 155	 * free_fib_info_rcu()
 156	 */
 157
 158	dst_dev_put(&rt->dst);
 159	dst_release_immediate(&rt->dst);
 160}
 161
 162static void free_nh_exceptions(struct fib_nh_common *nhc)
 163{
 164	struct fnhe_hash_bucket *hash;
 165	int i;
 166
 167	hash = rcu_dereference_protected(nhc->nhc_exceptions, 1);
 168	if (!hash)
 169		return;
 170	for (i = 0; i < FNHE_HASH_SIZE; i++) {
 171		struct fib_nh_exception *fnhe;
 172
 173		fnhe = rcu_dereference_protected(hash[i].chain, 1);
 174		while (fnhe) {
 175			struct fib_nh_exception *next;
 176
 177			next = rcu_dereference_protected(fnhe->fnhe_next, 1);
 178
 179			rt_fibinfo_free(&fnhe->fnhe_rth_input);
 180			rt_fibinfo_free(&fnhe->fnhe_rth_output);
 181
 182			kfree(fnhe);
 183
 184			fnhe = next;
 185		}
 186	}
 187	kfree(hash);
 188}
 189
 190static void rt_fibinfo_free_cpus(struct rtable __rcu * __percpu *rtp)
 191{
 192	int cpu;
 193
 194	if (!rtp)
 195		return;
 196
 197	for_each_possible_cpu(cpu) {
 198		struct rtable *rt;
 199
 200		rt = rcu_dereference_protected(*per_cpu_ptr(rtp, cpu), 1);
 201		if (rt) {
 202			dst_dev_put(&rt->dst);
 203			dst_release_immediate(&rt->dst);
 204		}
 205	}
 206	free_percpu(rtp);
 207}
 208
 209void fib_nh_common_release(struct fib_nh_common *nhc)
 210{
 211	if (nhc->nhc_dev)
 212		dev_put(nhc->nhc_dev);
 213
 214	lwtstate_put(nhc->nhc_lwtstate);
 215	rt_fibinfo_free_cpus(nhc->nhc_pcpu_rth_output);
 216	rt_fibinfo_free(&nhc->nhc_rth_input);
 217	free_nh_exceptions(nhc);
 218}
 219EXPORT_SYMBOL_GPL(fib_nh_common_release);
 220
 221void fib_nh_release(struct net *net, struct fib_nh *fib_nh)
 222{
 223#ifdef CONFIG_IP_ROUTE_CLASSID
 224	if (fib_nh->nh_tclassid)
 225		net->ipv4.fib_num_tclassid_users--;
 226#endif
 227	fib_nh_common_release(&fib_nh->nh_common);
 228}
 229
 230/* Release a nexthop info record */
 231static void free_fib_info_rcu(struct rcu_head *head)
 232{
 233	struct fib_info *fi = container_of(head, struct fib_info, rcu);
 
 234
 235	if (fi->nh) {
 236		nexthop_put(fi->nh);
 237	} else {
 238		change_nexthops(fi) {
 239			fib_nh_release(fi->fib_net, nexthop_nh);
 240		} endfor_nexthops(fi);
 241	}
 242
 243	ip_fib_metrics_put(fi->fib_metrics);
 244
 
 
 
 245	kfree(fi);
 246}
 247
 248void free_fib_info(struct fib_info *fi)
 249{
 250	if (fi->fib_dead == 0) {
 251		pr_warn("Freeing alive fib_info %p\n", fi);
 252		return;
 253	}
 254	fib_info_cnt--;
 255
 
 
 
 
 
 256	call_rcu(&fi->rcu, free_fib_info_rcu);
 257}
 258EXPORT_SYMBOL_GPL(free_fib_info);
 259
 260void fib_release_info(struct fib_info *fi)
 261{
 262	spin_lock_bh(&fib_info_lock);
 263	if (fi && --fi->fib_treeref == 0) {
 264		hlist_del(&fi->fib_hash);
 265		if (fi->fib_prefsrc)
 266			hlist_del(&fi->fib_lhash);
 267		if (fi->nh) {
 268			list_del(&fi->nh_list);
 269		} else {
 270			change_nexthops(fi) {
 271				if (!nexthop_nh->fib_nh_dev)
 272					continue;
 273				hlist_del(&nexthop_nh->nh_hash);
 274			} endfor_nexthops(fi)
 275		}
 276		fi->fib_dead = 1;
 277		fib_info_put(fi);
 278	}
 279	spin_unlock_bh(&fib_info_lock);
 280}
 281
 282static inline int nh_comp(struct fib_info *fi, struct fib_info *ofi)
 283{
 284	const struct fib_nh *onh;
 285
 286	if (fi->nh || ofi->nh)
 287		return nexthop_cmp(fi->nh, ofi->nh) ? 0 : -1;
 288
 289	if (ofi->fib_nhs == 0)
 290		return 0;
 291
 292	for_nexthops(fi) {
 293		onh = fib_info_nh(ofi, nhsel);
 294
 295		if (nh->fib_nh_oif != onh->fib_nh_oif ||
 296		    nh->fib_nh_gw_family != onh->fib_nh_gw_family ||
 297		    nh->fib_nh_scope != onh->fib_nh_scope ||
 298#ifdef CONFIG_IP_ROUTE_MULTIPATH
 299		    nh->fib_nh_weight != onh->fib_nh_weight ||
 300#endif
 301#ifdef CONFIG_IP_ROUTE_CLASSID
 302		    nh->nh_tclassid != onh->nh_tclassid ||
 303#endif
 304		    lwtunnel_cmp_encap(nh->fib_nh_lws, onh->fib_nh_lws) ||
 305		    ((nh->fib_nh_flags ^ onh->fib_nh_flags) & ~RTNH_COMPARE_MASK))
 306			return -1;
 307
 308		if (nh->fib_nh_gw_family == AF_INET &&
 309		    nh->fib_nh_gw4 != onh->fib_nh_gw4)
 310			return -1;
 311
 312		if (nh->fib_nh_gw_family == AF_INET6 &&
 313		    ipv6_addr_cmp(&nh->fib_nh_gw6, &onh->fib_nh_gw6))
 314			return -1;
 
 315	} endfor_nexthops(fi);
 316	return 0;
 317}
 318
 319static inline unsigned int fib_devindex_hashfn(unsigned int val)
 320{
 321	unsigned int mask = DEVINDEX_HASHSIZE - 1;
 322
 323	return (val ^
 324		(val >> DEVINDEX_HASHBITS) ^
 325		(val >> (DEVINDEX_HASHBITS * 2))) & mask;
 326}
 327
 328static unsigned int fib_info_hashfn_1(int init_val, u8 protocol, u8 scope,
 329				      u32 prefsrc, u32 priority)
 330{
 331	unsigned int val = init_val;
 332
 333	val ^= (protocol << 8) | scope;
 334	val ^= prefsrc;
 335	val ^= priority;
 336
 337	return val;
 338}
 339
 340static unsigned int fib_info_hashfn_result(unsigned int val)
 341{
 342	unsigned int mask = (fib_info_hash_size - 1);
 
 343
 344	return (val ^ (val >> 7) ^ (val >> 12)) & mask;
 345}
 346
 347static inline unsigned int fib_info_hashfn(struct fib_info *fi)
 348{
 349	unsigned int val;
 350
 351	val = fib_info_hashfn_1(fi->fib_nhs, fi->fib_protocol,
 352				fi->fib_scope, (__force u32)fi->fib_prefsrc,
 353				fi->fib_priority);
 354
 355	if (fi->nh) {
 356		val ^= fib_devindex_hashfn(fi->nh->id);
 357	} else {
 358		for_nexthops(fi) {
 359			val ^= fib_devindex_hashfn(nh->fib_nh_oif);
 360		} endfor_nexthops(fi)
 361	}
 362
 363	return fib_info_hashfn_result(val);
 364}
 365
 366/* no metrics, only nexthop id */
 367static struct fib_info *fib_find_info_nh(struct net *net,
 368					 const struct fib_config *cfg)
 369{
 370	struct hlist_head *head;
 371	struct fib_info *fi;
 372	unsigned int hash;
 373
 374	hash = fib_info_hashfn_1(fib_devindex_hashfn(cfg->fc_nh_id),
 375				 cfg->fc_protocol, cfg->fc_scope,
 376				 (__force u32)cfg->fc_prefsrc,
 377				 cfg->fc_priority);
 378	hash = fib_info_hashfn_result(hash);
 379	head = &fib_info_hash[hash];
 380
 381	hlist_for_each_entry(fi, head, fib_hash) {
 382		if (!net_eq(fi->fib_net, net))
 383			continue;
 384		if (!fi->nh || fi->nh->id != cfg->fc_nh_id)
 385			continue;
 386		if (cfg->fc_protocol == fi->fib_protocol &&
 387		    cfg->fc_scope == fi->fib_scope &&
 388		    cfg->fc_prefsrc == fi->fib_prefsrc &&
 389		    cfg->fc_priority == fi->fib_priority &&
 390		    cfg->fc_type == fi->fib_type &&
 391		    cfg->fc_table == fi->fib_tb_id &&
 392		    !((cfg->fc_flags ^ fi->fib_flags) & ~RTNH_COMPARE_MASK))
 393			return fi;
 394	}
 395
 396	return NULL;
 397}
 398
 399static struct fib_info *fib_find_info(struct fib_info *nfi)
 400{
 401	struct hlist_head *head;
 402	struct fib_info *fi;
 403	unsigned int hash;
 404
 405	hash = fib_info_hashfn(nfi);
 406	head = &fib_info_hash[hash];
 407
 408	hlist_for_each_entry(fi, head, fib_hash) {
 409		if (!net_eq(fi->fib_net, nfi->fib_net))
 410			continue;
 411		if (fi->fib_nhs != nfi->fib_nhs)
 412			continue;
 413		if (nfi->fib_protocol == fi->fib_protocol &&
 414		    nfi->fib_scope == fi->fib_scope &&
 415		    nfi->fib_prefsrc == fi->fib_prefsrc &&
 416		    nfi->fib_priority == fi->fib_priority &&
 417		    nfi->fib_type == fi->fib_type &&
 418		    memcmp(nfi->fib_metrics, fi->fib_metrics,
 419			   sizeof(u32) * RTAX_MAX) == 0 &&
 420		    !((nfi->fib_flags ^ fi->fib_flags) & ~RTNH_COMPARE_MASK) &&
 421		    nh_comp(fi, nfi) == 0)
 422			return fi;
 423	}
 424
 425	return NULL;
 426}
 427
 428/* Check, that the gateway is already configured.
 429 * Used only by redirect accept routine.
 430 */
 431int ip_fib_check_default(__be32 gw, struct net_device *dev)
 432{
 433	struct hlist_head *head;
 434	struct fib_nh *nh;
 435	unsigned int hash;
 436
 437	spin_lock(&fib_info_lock);
 438
 439	hash = fib_devindex_hashfn(dev->ifindex);
 440	head = &fib_info_devhash[hash];
 441	hlist_for_each_entry(nh, head, nh_hash) {
 442		if (nh->fib_nh_dev == dev &&
 443		    nh->fib_nh_gw4 == gw &&
 444		    !(nh->fib_nh_flags & RTNH_F_DEAD)) {
 445			spin_unlock(&fib_info_lock);
 446			return 0;
 447		}
 448	}
 449
 450	spin_unlock(&fib_info_lock);
 451
 452	return -1;
 453}
 454
 455static inline size_t fib_nlmsg_size(struct fib_info *fi)
 456{
 457	size_t payload = NLMSG_ALIGN(sizeof(struct rtmsg))
 458			 + nla_total_size(4) /* RTA_TABLE */
 459			 + nla_total_size(4) /* RTA_DST */
 460			 + nla_total_size(4) /* RTA_PRIORITY */
 461			 + nla_total_size(4) /* RTA_PREFSRC */
 462			 + nla_total_size(TCP_CA_NAME_MAX); /* RTAX_CC_ALGO */
 463	unsigned int nhs = fib_info_num_path(fi);
 464
 465	/* space for nested metrics */
 466	payload += nla_total_size((RTAX_MAX * nla_total_size(4)));
 467
 468	if (fi->nh)
 469		payload += nla_total_size(4); /* RTA_NH_ID */
 470
 471	if (nhs) {
 472		size_t nh_encapsize = 0;
 473		/* Also handles the special case nhs == 1 */
 474
 475		/* each nexthop is packed in an attribute */
 476		size_t nhsize = nla_total_size(sizeof(struct rtnexthop));
 477		unsigned int i;
 478
 479		/* may contain flow and gateway attribute */
 480		nhsize += 2 * nla_total_size(4);
 481
 482		/* grab encap info */
 483		for (i = 0; i < fib_info_num_path(fi); i++) {
 484			struct fib_nh_common *nhc = fib_info_nhc(fi, i);
 485
 486			if (nhc->nhc_lwtstate) {
 487				/* RTA_ENCAP_TYPE */
 488				nh_encapsize += lwtunnel_get_encap_size(
 489						nhc->nhc_lwtstate);
 490				/* RTA_ENCAP */
 491				nh_encapsize +=  nla_total_size(2);
 492			}
 493		}
 494
 495		/* all nexthops are packed in a nested attribute */
 496		payload += nla_total_size((nhs * nhsize) + nh_encapsize);
 
 497
 498	}
 499
 500	return payload;
 501}
 502
 503void rtmsg_fib(int event, __be32 key, struct fib_alias *fa,
 504	       int dst_len, u32 tb_id, const struct nl_info *info,
 505	       unsigned int nlm_flags)
 506{
 507	struct sk_buff *skb;
 508	u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0;
 509	int err = -ENOBUFS;
 510
 511	skb = nlmsg_new(fib_nlmsg_size(fa->fa_info), GFP_KERNEL);
 512	if (!skb)
 513		goto errout;
 514
 515	err = fib_dump_info(skb, info->portid, seq, event, tb_id,
 516			    fa->fa_type, key, dst_len,
 517			    fa->fa_tos, fa->fa_info, nlm_flags);
 518	if (err < 0) {
 519		/* -EMSGSIZE implies BUG in fib_nlmsg_size() */
 520		WARN_ON(err == -EMSGSIZE);
 521		kfree_skb(skb);
 522		goto errout;
 523	}
 524	rtnl_notify(skb, info->nl_net, info->portid, RTNLGRP_IPV4_ROUTE,
 525		    info->nlh, GFP_KERNEL);
 526	return;
 527errout:
 528	if (err < 0)
 529		rtnl_set_sk_err(info->nl_net, RTNLGRP_IPV4_ROUTE, err);
 530}
 531
 532static int fib_detect_death(struct fib_info *fi, int order,
 533			    struct fib_info **last_resort, int *last_idx,
 534			    int dflt)
 535{
 536	const struct fib_nh_common *nhc = fib_info_nhc(fi, 0);
 537	struct neighbour *n;
 538	int state = NUD_NONE;
 539
 540	if (likely(nhc->nhc_gw_family == AF_INET))
 541		n = neigh_lookup(&arp_tbl, &nhc->nhc_gw.ipv4, nhc->nhc_dev);
 542	else if (nhc->nhc_gw_family == AF_INET6)
 543		n = neigh_lookup(ipv6_stub->nd_tbl, &nhc->nhc_gw.ipv6,
 544				 nhc->nhc_dev);
 545	else
 546		n = NULL;
 547
 548	if (n) {
 549		state = n->nud_state;
 550		neigh_release(n);
 551	} else {
 552		return 0;
 553	}
 554	if (state == NUD_REACHABLE)
 555		return 0;
 556	if ((state & NUD_VALID) && order != dflt)
 557		return 0;
 558	if ((state & NUD_VALID) ||
 559	    (*last_idx < 0 && order > dflt && state != NUD_INCOMPLETE)) {
 560		*last_resort = fi;
 561		*last_idx = order;
 562	}
 563	return 1;
 564}
 565
 566int fib_nh_common_init(struct fib_nh_common *nhc, struct nlattr *encap,
 567		       u16 encap_type, void *cfg, gfp_t gfp_flags,
 568		       struct netlink_ext_ack *extack)
 569{
 570	int err;
 571
 572	nhc->nhc_pcpu_rth_output = alloc_percpu_gfp(struct rtable __rcu *,
 573						    gfp_flags);
 574	if (!nhc->nhc_pcpu_rth_output)
 575		return -ENOMEM;
 576
 577	if (encap) {
 578		struct lwtunnel_state *lwtstate;
 579
 580		if (encap_type == LWTUNNEL_ENCAP_NONE) {
 581			NL_SET_ERR_MSG(extack, "LWT encap type not specified");
 582			err = -EINVAL;
 583			goto lwt_failure;
 584		}
 585		err = lwtunnel_build_state(encap_type, encap, nhc->nhc_family,
 586					   cfg, &lwtstate, extack);
 587		if (err)
 588			goto lwt_failure;
 589
 590		nhc->nhc_lwtstate = lwtstate_get(lwtstate);
 591	}
 592
 593	return 0;
 594
 595lwt_failure:
 596	rt_fibinfo_free_cpus(nhc->nhc_pcpu_rth_output);
 597	nhc->nhc_pcpu_rth_output = NULL;
 598	return err;
 599}
 600EXPORT_SYMBOL_GPL(fib_nh_common_init);
 601
 602int fib_nh_init(struct net *net, struct fib_nh *nh,
 603		struct fib_config *cfg, int nh_weight,
 604		struct netlink_ext_ack *extack)
 605{
 606	int err;
 607
 608	nh->fib_nh_family = AF_INET;
 609
 610	err = fib_nh_common_init(&nh->nh_common, cfg->fc_encap,
 611				 cfg->fc_encap_type, cfg, GFP_KERNEL, extack);
 612	if (err)
 613		return err;
 614
 615	nh->fib_nh_oif = cfg->fc_oif;
 616	nh->fib_nh_gw_family = cfg->fc_gw_family;
 617	if (cfg->fc_gw_family == AF_INET)
 618		nh->fib_nh_gw4 = cfg->fc_gw4;
 619	else if (cfg->fc_gw_family == AF_INET6)
 620		nh->fib_nh_gw6 = cfg->fc_gw6;
 621
 622	nh->fib_nh_flags = cfg->fc_flags;
 623
 624#ifdef CONFIG_IP_ROUTE_CLASSID
 625	nh->nh_tclassid = cfg->fc_flow;
 626	if (nh->nh_tclassid)
 627		net->ipv4.fib_num_tclassid_users++;
 628#endif
 629#ifdef CONFIG_IP_ROUTE_MULTIPATH
 630	nh->fib_nh_weight = nh_weight;
 631#endif
 632	return 0;
 633}
 634
 635#ifdef CONFIG_IP_ROUTE_MULTIPATH
 636
 637static int fib_count_nexthops(struct rtnexthop *rtnh, int remaining,
 638			      struct netlink_ext_ack *extack)
 639{
 640	int nhs = 0;
 641
 642	while (rtnh_ok(rtnh, remaining)) {
 643		nhs++;
 644		rtnh = rtnh_next(rtnh, &remaining);
 645	}
 646
 647	/* leftover implies invalid nexthop configuration, discard it */
 648	if (remaining > 0) {
 649		NL_SET_ERR_MSG(extack,
 650			       "Invalid nexthop configuration - extra data after nexthops");
 651		nhs = 0;
 652	}
 653
 654	return nhs;
 655}
 656
 657/* only called when fib_nh is integrated into fib_info */
 658static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh,
 659		       int remaining, struct fib_config *cfg,
 660		       struct netlink_ext_ack *extack)
 661{
 662	struct net *net = fi->fib_net;
 663	struct fib_config fib_cfg;
 664	struct fib_nh *nh;
 665	int ret;
 666
 667	change_nexthops(fi) {
 668		int attrlen;
 669
 670		memset(&fib_cfg, 0, sizeof(fib_cfg));
 671
 672		if (!rtnh_ok(rtnh, remaining)) {
 673			NL_SET_ERR_MSG(extack,
 674				       "Invalid nexthop configuration - extra data after nexthop");
 675			return -EINVAL;
 676		}
 677
 678		if (rtnh->rtnh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN)) {
 679			NL_SET_ERR_MSG(extack,
 680				       "Invalid flags for nexthop - can not contain DEAD or LINKDOWN");
 681			return -EINVAL;
 682		}
 683
 684		fib_cfg.fc_flags = (cfg->fc_flags & ~0xFF) | rtnh->rtnh_flags;
 685		fib_cfg.fc_oif = rtnh->rtnh_ifindex;
 
 
 686
 687		attrlen = rtnh_attrlen(rtnh);
 688		if (attrlen > 0) {
 689			struct nlattr *nla, *nlav, *attrs = rtnh_attrs(rtnh);
 690
 691			nla = nla_find(attrs, attrlen, RTA_GATEWAY);
 692			nlav = nla_find(attrs, attrlen, RTA_VIA);
 693			if (nla && nlav) {
 694				NL_SET_ERR_MSG(extack,
 695					       "Nexthop configuration can not contain both GATEWAY and VIA");
 696				return -EINVAL;
 697			}
 
 
 698			if (nla) {
 699				fib_cfg.fc_gw4 = nla_get_in_addr(nla);
 700				if (fib_cfg.fc_gw4)
 701					fib_cfg.fc_gw_family = AF_INET;
 702			} else if (nlav) {
 703				ret = fib_gw_from_via(&fib_cfg, nlav, extack);
 
 
 
 
 
 
 
 
 
 
 
 704				if (ret)
 705					goto errout;
 
 
 706			}
 707
 708			nla = nla_find(attrs, attrlen, RTA_FLOW);
 709			if (nla)
 710				fib_cfg.fc_flow = nla_get_u32(nla);
 711
 712			fib_cfg.fc_encap = nla_find(attrs, attrlen, RTA_ENCAP);
 713			nla = nla_find(attrs, attrlen, RTA_ENCAP_TYPE);
 714			if (nla)
 715				fib_cfg.fc_encap_type = nla_get_u16(nla);
 716		}
 717
 718		ret = fib_nh_init(net, nexthop_nh, &fib_cfg,
 719				  rtnh->rtnh_hops + 1, extack);
 720		if (ret)
 721			goto errout;
 722
 723		rtnh = rtnh_next(rtnh, &remaining);
 724	} endfor_nexthops(fi);
 725
 
 
 
 726	ret = -EINVAL;
 727	nh = fib_info_nh(fi, 0);
 728	if (cfg->fc_oif && nh->fib_nh_oif != cfg->fc_oif) {
 729		NL_SET_ERR_MSG(extack,
 730			       "Nexthop device index does not match RTA_OIF");
 731		goto errout;
 732	}
 733	if (cfg->fc_gw_family) {
 734		if (cfg->fc_gw_family != nh->fib_nh_gw_family ||
 735		    (cfg->fc_gw_family == AF_INET &&
 736		     nh->fib_nh_gw4 != cfg->fc_gw4) ||
 737		    (cfg->fc_gw_family == AF_INET6 &&
 738		     ipv6_addr_cmp(&nh->fib_nh_gw6, &cfg->fc_gw6))) {
 739			NL_SET_ERR_MSG(extack,
 740				       "Nexthop gateway does not match RTA_GATEWAY or RTA_VIA");
 741			goto errout;
 742		}
 743	}
 744#ifdef CONFIG_IP_ROUTE_CLASSID
 745	if (cfg->fc_flow && nh->nh_tclassid != cfg->fc_flow) {
 746		NL_SET_ERR_MSG(extack,
 747			       "Nexthop class id does not match RTA_FLOW");
 748		goto errout;
 749	}
 750#endif
 751	ret = 0;
 752errout:
 753	return ret;
 754}
 755
 756/* only called when fib_nh is integrated into fib_info */
 757static void fib_rebalance(struct fib_info *fi)
 758{
 759	int total;
 760	int w;
 
 761
 762	if (fib_info_num_path(fi) < 2)
 763		return;
 764
 765	total = 0;
 766	for_nexthops(fi) {
 767		if (nh->fib_nh_flags & RTNH_F_DEAD)
 768			continue;
 769
 770		if (ip_ignore_linkdown(nh->fib_nh_dev) &&
 771		    nh->fib_nh_flags & RTNH_F_LINKDOWN)
 
 
 
 772			continue;
 773
 774		total += nh->fib_nh_weight;
 775	} endfor_nexthops(fi);
 776
 777	w = 0;
 778	change_nexthops(fi) {
 779		int upper_bound;
 780
 781		if (nexthop_nh->fib_nh_flags & RTNH_F_DEAD) {
 
 
 782			upper_bound = -1;
 783		} else if (ip_ignore_linkdown(nexthop_nh->fib_nh_dev) &&
 784			   nexthop_nh->fib_nh_flags & RTNH_F_LINKDOWN) {
 
 785			upper_bound = -1;
 786		} else {
 787			w += nexthop_nh->fib_nh_weight;
 788			upper_bound = DIV_ROUND_CLOSEST_ULL((u64)w << 31,
 789							    total) - 1;
 790		}
 791
 792		atomic_set(&nexthop_nh->fib_nh_upper_bound, upper_bound);
 793	} endfor_nexthops(fi);
 794}
 795#else /* CONFIG_IP_ROUTE_MULTIPATH */
 796
 797static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh,
 798		       int remaining, struct fib_config *cfg,
 799		       struct netlink_ext_ack *extack)
 800{
 801	NL_SET_ERR_MSG(extack, "Multipath support not enabled in kernel");
 802
 803	return -EINVAL;
 804}
 805
 806#define fib_rebalance(fi) do { } while (0)
 807
 808#endif /* CONFIG_IP_ROUTE_MULTIPATH */
 809
 810static int fib_encap_match(u16 encap_type,
 811			   struct nlattr *encap,
 812			   const struct fib_nh *nh,
 813			   const struct fib_config *cfg,
 814			   struct netlink_ext_ack *extack)
 815{
 816	struct lwtunnel_state *lwtstate;
 817	int ret, result = 0;
 818
 819	if (encap_type == LWTUNNEL_ENCAP_NONE)
 820		return 0;
 821
 822	ret = lwtunnel_build_state(encap_type, encap, AF_INET,
 823				   cfg, &lwtstate, extack);
 824	if (!ret) {
 825		result = lwtunnel_cmp_encap(lwtstate, nh->fib_nh_lws);
 826		lwtstate_free(lwtstate);
 827	}
 828
 829	return result;
 830}
 831
 832int fib_nh_match(struct fib_config *cfg, struct fib_info *fi,
 833		 struct netlink_ext_ack *extack)
 834{
 835#ifdef CONFIG_IP_ROUTE_MULTIPATH
 836	struct rtnexthop *rtnh;
 837	int remaining;
 838#endif
 839
 840	if (cfg->fc_priority && cfg->fc_priority != fi->fib_priority)
 841		return 1;
 842
 843	if (cfg->fc_nh_id) {
 844		if (fi->nh && cfg->fc_nh_id == fi->nh->id)
 845			return 0;
 846		return 1;
 847	}
 848
 849	if (cfg->fc_oif || cfg->fc_gw_family) {
 850		struct fib_nh *nh = fib_info_nh(fi, 0);
 851
 852		if (cfg->fc_encap) {
 853			if (fib_encap_match(cfg->fc_encap_type, cfg->fc_encap,
 854					    nh, cfg, extack))
 855				return 1;
 856		}
 857#ifdef CONFIG_IP_ROUTE_CLASSID
 858		if (cfg->fc_flow &&
 859		    cfg->fc_flow != nh->nh_tclassid)
 860			return 1;
 861#endif
 862		if ((cfg->fc_oif && cfg->fc_oif != nh->fib_nh_oif) ||
 863		    (cfg->fc_gw_family &&
 864		     cfg->fc_gw_family != nh->fib_nh_gw_family))
 865			return 1;
 866
 867		if (cfg->fc_gw_family == AF_INET &&
 868		    cfg->fc_gw4 != nh->fib_nh_gw4)
 869			return 1;
 870
 871		if (cfg->fc_gw_family == AF_INET6 &&
 872		    ipv6_addr_cmp(&cfg->fc_gw6, &nh->fib_nh_gw6))
 873			return 1;
 874
 875		return 0;
 876	}
 877
 878#ifdef CONFIG_IP_ROUTE_MULTIPATH
 879	if (!cfg->fc_mp)
 880		return 0;
 881
 882	rtnh = cfg->fc_mp;
 883	remaining = cfg->fc_mp_len;
 884
 885	for_nexthops(fi) {
 886		int attrlen;
 887
 888		if (!rtnh_ok(rtnh, remaining))
 889			return -EINVAL;
 890
 891		if (rtnh->rtnh_ifindex && rtnh->rtnh_ifindex != nh->fib_nh_oif)
 892			return 1;
 893
 894		attrlen = rtnh_attrlen(rtnh);
 895		if (attrlen > 0) {
 896			struct nlattr *nla, *nlav, *attrs = rtnh_attrs(rtnh);
 897
 898			nla = nla_find(attrs, attrlen, RTA_GATEWAY);
 899			nlav = nla_find(attrs, attrlen, RTA_VIA);
 900			if (nla && nlav) {
 901				NL_SET_ERR_MSG(extack,
 902					       "Nexthop configuration can not contain both GATEWAY and VIA");
 903				return -EINVAL;
 904			}
 905
 906			if (nla) {
 907				if (nh->fib_nh_gw_family != AF_INET ||
 908				    nla_get_in_addr(nla) != nh->fib_nh_gw4)
 909					return 1;
 910			} else if (nlav) {
 911				struct fib_config cfg2;
 912				int err;
 913
 914				err = fib_gw_from_via(&cfg2, nlav, extack);
 915				if (err)
 916					return err;
 917
 918				switch (nh->fib_nh_gw_family) {
 919				case AF_INET:
 920					if (cfg2.fc_gw_family != AF_INET ||
 921					    cfg2.fc_gw4 != nh->fib_nh_gw4)
 922						return 1;
 923					break;
 924				case AF_INET6:
 925					if (cfg2.fc_gw_family != AF_INET6 ||
 926					    ipv6_addr_cmp(&cfg2.fc_gw6,
 927							  &nh->fib_nh_gw6))
 928						return 1;
 929					break;
 930				}
 931			}
 932
 933#ifdef CONFIG_IP_ROUTE_CLASSID
 934			nla = nla_find(attrs, attrlen, RTA_FLOW);
 935			if (nla && nla_get_u32(nla) != nh->nh_tclassid)
 936				return 1;
 937#endif
 938		}
 939
 940		rtnh = rtnh_next(rtnh, &remaining);
 941	} endfor_nexthops(fi);
 942#endif
 943	return 0;
 944}
 945
 946bool fib_metrics_match(struct fib_config *cfg, struct fib_info *fi)
 947{
 948	struct nlattr *nla;
 949	int remaining;
 950
 951	if (!cfg->fc_mx)
 952		return true;
 953
 954	nla_for_each_attr(nla, cfg->fc_mx, cfg->fc_mx_len, remaining) {
 955		int type = nla_type(nla);
 956		u32 fi_val, val;
 957
 958		if (!type)
 959			continue;
 960		if (type > RTAX_MAX)
 961			return false;
 962
 963		if (type == RTAX_CC_ALGO) {
 964			char tmp[TCP_CA_NAME_MAX];
 965			bool ecn_ca = false;
 966
 967			nla_strlcpy(tmp, nla, sizeof(tmp));
 968			val = tcp_ca_get_key_by_name(fi->fib_net, tmp, &ecn_ca);
 969		} else {
 970			if (nla_len(nla) != sizeof(u32))
 971				return false;
 972			val = nla_get_u32(nla);
 973		}
 974
 975		fi_val = fi->fib_metrics->metrics[type - 1];
 976		if (type == RTAX_FEATURES)
 977			fi_val &= ~DST_FEATURE_ECN_CA;
 978
 979		if (fi_val != val)
 980			return false;
 981	}
 982
 983	return true;
 984}
 985
 986static int fib_check_nh_v6_gw(struct net *net, struct fib_nh *nh,
 987			      u32 table, struct netlink_ext_ack *extack)
 988{
 989	struct fib6_config cfg = {
 990		.fc_table = table,
 991		.fc_flags = nh->fib_nh_flags | RTF_GATEWAY,
 992		.fc_ifindex = nh->fib_nh_oif,
 993		.fc_gateway = nh->fib_nh_gw6,
 994	};
 995	struct fib6_nh fib6_nh = {};
 996	int err;
 997
 998	err = ipv6_stub->fib6_nh_init(net, &fib6_nh, &cfg, GFP_KERNEL, extack);
 999	if (!err) {
1000		nh->fib_nh_dev = fib6_nh.fib_nh_dev;
1001		dev_hold(nh->fib_nh_dev);
1002		nh->fib_nh_oif = nh->fib_nh_dev->ifindex;
1003		nh->fib_nh_scope = RT_SCOPE_LINK;
1004
1005		ipv6_stub->fib6_nh_release(&fib6_nh);
1006	}
1007
1008	return err;
1009}
1010
1011/*
1012 * Picture
1013 * -------
1014 *
1015 * Semantics of nexthop is very messy by historical reasons.
1016 * We have to take into account, that:
1017 * a) gateway can be actually local interface address,
1018 *    so that gatewayed route is direct.
1019 * b) gateway must be on-link address, possibly
1020 *    described not by an ifaddr, but also by a direct route.
1021 * c) If both gateway and interface are specified, they should not
1022 *    contradict.
1023 * d) If we use tunnel routes, gateway could be not on-link.
1024 *
1025 * Attempt to reconcile all of these (alas, self-contradictory) conditions
1026 * results in pretty ugly and hairy code with obscure logic.
1027 *
1028 * I chose to generalized it instead, so that the size
1029 * of code does not increase practically, but it becomes
1030 * much more general.
1031 * Every prefix is assigned a "scope" value: "host" is local address,
1032 * "link" is direct route,
1033 * [ ... "site" ... "interior" ... ]
1034 * and "universe" is true gateway route with global meaning.
1035 *
1036 * Every prefix refers to a set of "nexthop"s (gw, oif),
1037 * where gw must have narrower scope. This recursion stops
1038 * when gw has LOCAL scope or if "nexthop" is declared ONLINK,
1039 * which means that gw is forced to be on link.
1040 *
1041 * Code is still hairy, but now it is apparently logically
1042 * consistent and very flexible. F.e. as by-product it allows
1043 * to co-exists in peace independent exterior and interior
1044 * routing processes.
1045 *
1046 * Normally it looks as following.
1047 *
1048 * {universe prefix}  -> (gw, oif) [scope link]
1049 *		  |
1050 *		  |-> {link prefix} -> (gw, oif) [scope local]
1051 *					|
1052 *					|-> {local prefix} (terminal node)
1053 */
1054static int fib_check_nh_v4_gw(struct net *net, struct fib_nh *nh, u32 table,
1055			      u8 scope, struct netlink_ext_ack *extack)
1056{
1057	struct net_device *dev;
1058	struct fib_result res;
1059	int err = 0;
 
 
1060
1061	if (nh->fib_nh_flags & RTNH_F_ONLINK) {
1062		unsigned int addr_type;
 
1063
1064		if (scope >= RT_SCOPE_LINK) {
1065			NL_SET_ERR_MSG(extack, "Nexthop has invalid scope");
1066			return -EINVAL;
1067		}
1068		dev = __dev_get_by_index(net, nh->fib_nh_oif);
1069		if (!dev) {
1070			NL_SET_ERR_MSG(extack, "Nexthop device required for onlink");
1071			return -ENODEV;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1072		}
1073		if (!(dev->flags & IFF_UP)) {
1074			NL_SET_ERR_MSG(extack, "Nexthop device is not up");
1075			return -ENETDOWN;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1076		}
1077		addr_type = inet_addr_type_dev_table(net, dev, nh->fib_nh_gw4);
1078		if (addr_type != RTN_UNICAST) {
1079			NL_SET_ERR_MSG(extack, "Nexthop has invalid gateway");
1080			return -EINVAL;
1081		}
1082		if (!netif_carrier_ok(dev))
1083			nh->fib_nh_flags |= RTNH_F_LINKDOWN;
1084		nh->fib_nh_dev = dev;
1085		dev_hold(dev);
1086		nh->fib_nh_scope = RT_SCOPE_LINK;
1087		return 0;
1088	}
1089	rcu_read_lock();
1090	{
1091		struct fib_table *tbl = NULL;
1092		struct flowi4 fl4 = {
1093			.daddr = nh->fib_nh_gw4,
1094			.flowi4_scope = scope + 1,
1095			.flowi4_oif = nh->fib_nh_oif,
1096			.flowi4_iif = LOOPBACK_IFINDEX,
1097		};
1098
1099		/* It is not necessary, but requires a bit of thinking */
1100		if (fl4.flowi4_scope < RT_SCOPE_LINK)
1101			fl4.flowi4_scope = RT_SCOPE_LINK;
1102
1103		if (table)
1104			tbl = fib_get_table(net, table);
1105
1106		if (tbl)
1107			err = fib_table_lookup(tbl, &fl4, &res,
1108					       FIB_LOOKUP_IGNORE_LINKSTATE |
1109					       FIB_LOOKUP_NOREF);
1110
1111		/* on error or if no table given do full lookup. This
1112		 * is needed for example when nexthops are in the local
1113		 * table rather than the given table
1114		 */
1115		if (!tbl || err) {
1116			err = fib_lookup(net, &fl4, &res,
1117					 FIB_LOOKUP_IGNORE_LINKSTATE);
1118		}
 
 
 
 
 
 
1119
1120		if (err) {
1121			NL_SET_ERR_MSG(extack, "Nexthop has invalid gateway");
 
 
 
 
 
 
 
 
 
 
 
1122			goto out;
1123		}
 
 
 
 
 
 
1124	}
1125
1126	err = -EINVAL;
1127	if (res.type != RTN_UNICAST && res.type != RTN_LOCAL) {
1128		NL_SET_ERR_MSG(extack, "Nexthop has invalid gateway");
1129		goto out;
1130	}
1131	nh->fib_nh_scope = res.scope;
1132	nh->fib_nh_oif = FIB_RES_OIF(res);
1133	nh->fib_nh_dev = dev = FIB_RES_DEV(res);
1134	if (!dev) {
1135		NL_SET_ERR_MSG(extack,
1136			       "No egress device for nexthop gateway");
1137		goto out;
1138	}
1139	dev_hold(dev);
1140	if (!netif_carrier_ok(dev))
1141		nh->fib_nh_flags |= RTNH_F_LINKDOWN;
1142	err = (dev->flags & IFF_UP) ? 0 : -ENETDOWN;
1143out:
1144	rcu_read_unlock();
1145	return err;
1146}
1147
1148static int fib_check_nh_nongw(struct net *net, struct fib_nh *nh,
1149			      struct netlink_ext_ack *extack)
1150{
1151	struct in_device *in_dev;
1152	int err;
1153
1154	if (nh->fib_nh_flags & (RTNH_F_PERVASIVE | RTNH_F_ONLINK)) {
1155		NL_SET_ERR_MSG(extack,
1156			       "Invalid flags for nexthop - PERVASIVE and ONLINK can not be set");
1157		return -EINVAL;
1158	}
1159
1160	rcu_read_lock();
1161
1162	err = -ENODEV;
1163	in_dev = inetdev_by_index(net, nh->fib_nh_oif);
1164	if (!in_dev)
1165		goto out;
1166	err = -ENETDOWN;
1167	if (!(in_dev->dev->flags & IFF_UP)) {
1168		NL_SET_ERR_MSG(extack, "Device for nexthop is not up");
1169		goto out;
1170	}
1171
1172	nh->fib_nh_dev = in_dev->dev;
1173	dev_hold(nh->fib_nh_dev);
1174	nh->fib_nh_scope = RT_SCOPE_HOST;
1175	if (!netif_carrier_ok(nh->fib_nh_dev))
1176		nh->fib_nh_flags |= RTNH_F_LINKDOWN;
1177	err = 0;
1178out:
1179	rcu_read_unlock();
1180	return err;
1181}
1182
1183int fib_check_nh(struct net *net, struct fib_nh *nh, u32 table, u8 scope,
1184		 struct netlink_ext_ack *extack)
1185{
1186	int err;
1187
1188	if (nh->fib_nh_gw_family == AF_INET)
1189		err = fib_check_nh_v4_gw(net, nh, table, scope, extack);
1190	else if (nh->fib_nh_gw_family == AF_INET6)
1191		err = fib_check_nh_v6_gw(net, nh, table, extack);
1192	else
1193		err = fib_check_nh_nongw(net, nh, extack);
1194
1195	return err;
1196}
1197
1198static inline unsigned int fib_laddr_hashfn(__be32 val)
1199{
1200	unsigned int mask = (fib_info_hash_size - 1);
1201
1202	return ((__force u32)val ^
1203		((__force u32)val >> 7) ^
1204		((__force u32)val >> 14)) & mask;
1205}
1206
1207static struct hlist_head *fib_info_hash_alloc(int bytes)
1208{
1209	if (bytes <= PAGE_SIZE)
1210		return kzalloc(bytes, GFP_KERNEL);
1211	else
1212		return (struct hlist_head *)
1213			__get_free_pages(GFP_KERNEL | __GFP_ZERO,
1214					 get_order(bytes));
1215}
1216
1217static void fib_info_hash_free(struct hlist_head *hash, int bytes)
1218{
1219	if (!hash)
1220		return;
1221
1222	if (bytes <= PAGE_SIZE)
1223		kfree(hash);
1224	else
1225		free_pages((unsigned long) hash, get_order(bytes));
1226}
1227
1228static void fib_info_hash_move(struct hlist_head *new_info_hash,
1229			       struct hlist_head *new_laddrhash,
1230			       unsigned int new_size)
1231{
1232	struct hlist_head *old_info_hash, *old_laddrhash;
1233	unsigned int old_size = fib_info_hash_size;
1234	unsigned int i, bytes;
1235
1236	spin_lock_bh(&fib_info_lock);
1237	old_info_hash = fib_info_hash;
1238	old_laddrhash = fib_info_laddrhash;
1239	fib_info_hash_size = new_size;
1240
1241	for (i = 0; i < old_size; i++) {
1242		struct hlist_head *head = &fib_info_hash[i];
1243		struct hlist_node *n;
1244		struct fib_info *fi;
1245
1246		hlist_for_each_entry_safe(fi, n, head, fib_hash) {
1247			struct hlist_head *dest;
1248			unsigned int new_hash;
1249
1250			new_hash = fib_info_hashfn(fi);
1251			dest = &new_info_hash[new_hash];
1252			hlist_add_head(&fi->fib_hash, dest);
1253		}
1254	}
1255	fib_info_hash = new_info_hash;
1256
1257	for (i = 0; i < old_size; i++) {
1258		struct hlist_head *lhead = &fib_info_laddrhash[i];
1259		struct hlist_node *n;
1260		struct fib_info *fi;
1261
1262		hlist_for_each_entry_safe(fi, n, lhead, fib_lhash) {
1263			struct hlist_head *ldest;
1264			unsigned int new_hash;
1265
1266			new_hash = fib_laddr_hashfn(fi->fib_prefsrc);
1267			ldest = &new_laddrhash[new_hash];
1268			hlist_add_head(&fi->fib_lhash, ldest);
1269		}
1270	}
1271	fib_info_laddrhash = new_laddrhash;
1272
1273	spin_unlock_bh(&fib_info_lock);
1274
1275	bytes = old_size * sizeof(struct hlist_head *);
1276	fib_info_hash_free(old_info_hash, bytes);
1277	fib_info_hash_free(old_laddrhash, bytes);
1278}
1279
1280__be32 fib_info_update_nhc_saddr(struct net *net, struct fib_nh_common *nhc,
1281				 unsigned char scope)
1282{
1283	struct fib_nh *nh;
1284
1285	if (nhc->nhc_family != AF_INET)
1286		return inet_select_addr(nhc->nhc_dev, 0, scope);
1287
1288	nh = container_of(nhc, struct fib_nh, nh_common);
1289	nh->nh_saddr = inet_select_addr(nh->fib_nh_dev, nh->fib_nh_gw4, scope);
1290	nh->nh_saddr_genid = atomic_read(&net->ipv4.dev_addr_genid);
1291
1292	return nh->nh_saddr;
1293}
1294
1295__be32 fib_result_prefsrc(struct net *net, struct fib_result *res)
1296{
1297	struct fib_nh_common *nhc = res->nhc;
1298
1299	if (res->fi->fib_prefsrc)
1300		return res->fi->fib_prefsrc;
1301
1302	if (nhc->nhc_family == AF_INET) {
1303		struct fib_nh *nh;
1304
1305		nh = container_of(nhc, struct fib_nh, nh_common);
1306		if (nh->nh_saddr_genid == atomic_read(&net->ipv4.dev_addr_genid))
1307			return nh->nh_saddr;
1308	}
1309
1310	return fib_info_update_nhc_saddr(net, nhc, res->fi->fib_scope);
1311}
1312
1313static bool fib_valid_prefsrc(struct fib_config *cfg, __be32 fib_prefsrc)
1314{
1315	if (cfg->fc_type != RTN_LOCAL || !cfg->fc_dst ||
1316	    fib_prefsrc != cfg->fc_dst) {
1317		u32 tb_id = cfg->fc_table;
1318		int rc;
1319
1320		if (tb_id == RT_TABLE_MAIN)
1321			tb_id = RT_TABLE_LOCAL;
1322
1323		rc = inet_addr_type_table(cfg->fc_nlinfo.nl_net,
1324					  fib_prefsrc, tb_id);
1325
1326		if (rc != RTN_LOCAL && tb_id != RT_TABLE_LOCAL) {
1327			rc = inet_addr_type_table(cfg->fc_nlinfo.nl_net,
1328						  fib_prefsrc, RT_TABLE_LOCAL);
1329		}
1330
1331		if (rc != RTN_LOCAL)
1332			return false;
1333	}
1334	return true;
1335}
1336
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1337struct fib_info *fib_create_info(struct fib_config *cfg,
1338				 struct netlink_ext_ack *extack)
1339{
1340	int err;
1341	struct fib_info *fi = NULL;
1342	struct nexthop *nh = NULL;
1343	struct fib_info *ofi;
1344	int nhs = 1;
1345	struct net *net = cfg->fc_nlinfo.nl_net;
1346
1347	if (cfg->fc_type > RTN_MAX)
1348		goto err_inval;
1349
1350	/* Fast check to catch the most weird cases */
1351	if (fib_props[cfg->fc_type].scope > cfg->fc_scope) {
1352		NL_SET_ERR_MSG(extack, "Invalid scope");
1353		goto err_inval;
1354	}
1355
1356	if (cfg->fc_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN)) {
1357		NL_SET_ERR_MSG(extack,
1358			       "Invalid rtm_flags - can not contain DEAD or LINKDOWN");
1359		goto err_inval;
1360	}
1361
1362	if (cfg->fc_nh_id) {
1363		if (!cfg->fc_mx) {
1364			fi = fib_find_info_nh(net, cfg);
1365			if (fi) {
1366				fi->fib_treeref++;
1367				return fi;
1368			}
1369		}
1370
1371		nh = nexthop_find_by_id(net, cfg->fc_nh_id);
1372		if (!nh) {
1373			NL_SET_ERR_MSG(extack, "Nexthop id does not exist");
1374			goto err_inval;
1375		}
1376		nhs = 0;
1377	}
1378
1379#ifdef CONFIG_IP_ROUTE_MULTIPATH
1380	if (cfg->fc_mp) {
1381		nhs = fib_count_nexthops(cfg->fc_mp, cfg->fc_mp_len, extack);
1382		if (nhs == 0)
1383			goto err_inval;
1384	}
1385#endif
1386
1387	err = -ENOBUFS;
1388	if (fib_info_cnt >= fib_info_hash_size) {
1389		unsigned int new_size = fib_info_hash_size << 1;
1390		struct hlist_head *new_info_hash;
1391		struct hlist_head *new_laddrhash;
1392		unsigned int bytes;
1393
1394		if (!new_size)
1395			new_size = 16;
1396		bytes = new_size * sizeof(struct hlist_head *);
1397		new_info_hash = fib_info_hash_alloc(bytes);
1398		new_laddrhash = fib_info_hash_alloc(bytes);
1399		if (!new_info_hash || !new_laddrhash) {
1400			fib_info_hash_free(new_info_hash, bytes);
1401			fib_info_hash_free(new_laddrhash, bytes);
1402		} else
1403			fib_info_hash_move(new_info_hash, new_laddrhash, new_size);
1404
1405		if (!fib_info_hash_size)
1406			goto failure;
1407	}
1408
1409	fi = kzalloc(struct_size(fi, fib_nh, nhs), GFP_KERNEL);
1410	if (!fi)
1411		goto failure;
1412	fi->fib_metrics = ip_fib_metrics_init(fi->fib_net, cfg->fc_mx,
1413					      cfg->fc_mx_len, extack);
1414	if (IS_ERR(fi->fib_metrics)) {
1415		err = PTR_ERR(fi->fib_metrics);
1416		kfree(fi);
1417		return ERR_PTR(err);
 
 
 
1418	}
1419
1420	fib_info_cnt++;
1421	fi->fib_net = net;
1422	fi->fib_protocol = cfg->fc_protocol;
1423	fi->fib_scope = cfg->fc_scope;
1424	fi->fib_flags = cfg->fc_flags;
1425	fi->fib_priority = cfg->fc_priority;
1426	fi->fib_prefsrc = cfg->fc_prefsrc;
1427	fi->fib_type = cfg->fc_type;
1428	fi->fib_tb_id = cfg->fc_table;
1429
1430	fi->fib_nhs = nhs;
1431	if (nh) {
1432		if (!nexthop_get(nh)) {
1433			NL_SET_ERR_MSG(extack, "Nexthop has been deleted");
1434			err = -EINVAL;
1435		} else {
1436			err = 0;
1437			fi->nh = nh;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1438		}
 
 
 
 
 
 
 
 
 
 
 
 
1439	} else {
1440		change_nexthops(fi) {
1441			nexthop_nh->nh_parent = fi;
1442		} endfor_nexthops(fi)
1443
1444		if (cfg->fc_mp)
1445			err = fib_get_nhs(fi, cfg->fc_mp, cfg->fc_mp_len, cfg,
1446					  extack);
1447		else
1448			err = fib_nh_init(net, fi->fib_nh, cfg, 1, extack);
1449	}
1450
1451	if (err != 0)
1452		goto failure;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1453
1454	if (fib_props[cfg->fc_type].error) {
1455		if (cfg->fc_gw_family || cfg->fc_oif || cfg->fc_mp) {
1456			NL_SET_ERR_MSG(extack,
1457				       "Gateway, device and multipath can not be specified for this route type");
1458			goto err_inval;
1459		}
1460		goto link_it;
1461	} else {
1462		switch (cfg->fc_type) {
1463		case RTN_UNICAST:
1464		case RTN_LOCAL:
1465		case RTN_BROADCAST:
1466		case RTN_ANYCAST:
1467		case RTN_MULTICAST:
1468			break;
1469		default:
1470			NL_SET_ERR_MSG(extack, "Invalid route type");
1471			goto err_inval;
1472		}
1473	}
1474
1475	if (cfg->fc_scope > RT_SCOPE_HOST) {
1476		NL_SET_ERR_MSG(extack, "Invalid scope");
1477		goto err_inval;
1478	}
1479
1480	if (fi->nh) {
1481		err = fib_check_nexthop(fi->nh, cfg->fc_scope, extack);
1482		if (err)
1483			goto failure;
1484	} else if (cfg->fc_scope == RT_SCOPE_HOST) {
1485		struct fib_nh *nh = fi->fib_nh;
1486
1487		/* Local address is added. */
1488		if (nhs != 1) {
1489			NL_SET_ERR_MSG(extack,
1490				       "Route with host scope can not have multiple nexthops");
1491			goto err_inval;
1492		}
1493		if (nh->fib_nh_gw_family) {
1494			NL_SET_ERR_MSG(extack,
1495				       "Route with host scope can not have a gateway");
1496			goto err_inval;
1497		}
1498		nh->fib_nh_scope = RT_SCOPE_NOWHERE;
1499		nh->fib_nh_dev = dev_get_by_index(net, nh->fib_nh_oif);
1500		err = -ENODEV;
1501		if (!nh->fib_nh_dev)
1502			goto failure;
1503	} else {
1504		int linkdown = 0;
1505
1506		change_nexthops(fi) {
1507			err = fib_check_nh(cfg->fc_nlinfo.nl_net, nexthop_nh,
1508					   cfg->fc_table, cfg->fc_scope,
1509					   extack);
1510			if (err != 0)
1511				goto failure;
1512			if (nexthop_nh->fib_nh_flags & RTNH_F_LINKDOWN)
1513				linkdown++;
1514		} endfor_nexthops(fi)
1515		if (linkdown == fi->fib_nhs)
1516			fi->fib_flags |= RTNH_F_LINKDOWN;
1517	}
1518
1519	if (fi->fib_prefsrc && !fib_valid_prefsrc(cfg, fi->fib_prefsrc)) {
1520		NL_SET_ERR_MSG(extack, "Invalid prefsrc address");
1521		goto err_inval;
1522	}
1523
1524	if (!fi->nh) {
1525		change_nexthops(fi) {
1526			fib_info_update_nhc_saddr(net, &nexthop_nh->nh_common,
1527						  fi->fib_scope);
1528			if (nexthop_nh->fib_nh_gw_family == AF_INET6)
1529				fi->fib_nh_is_v6 = true;
1530		} endfor_nexthops(fi)
1531
1532		fib_rebalance(fi);
1533	}
1534
1535link_it:
1536	ofi = fib_find_info(fi);
1537	if (ofi) {
1538		fi->fib_dead = 1;
1539		free_fib_info(fi);
1540		ofi->fib_treeref++;
1541		return ofi;
1542	}
1543
1544	fi->fib_treeref++;
1545	refcount_set(&fi->fib_clntref, 1);
1546	spin_lock_bh(&fib_info_lock);
1547	hlist_add_head(&fi->fib_hash,
1548		       &fib_info_hash[fib_info_hashfn(fi)]);
1549	if (fi->fib_prefsrc) {
1550		struct hlist_head *head;
1551
1552		head = &fib_info_laddrhash[fib_laddr_hashfn(fi->fib_prefsrc)];
1553		hlist_add_head(&fi->fib_lhash, head);
1554	}
1555	if (fi->nh) {
1556		list_add(&fi->nh_list, &nh->fi_list);
1557	} else {
1558		change_nexthops(fi) {
1559			struct hlist_head *head;
1560			unsigned int hash;
1561
1562			if (!nexthop_nh->fib_nh_dev)
1563				continue;
1564			hash = fib_devindex_hashfn(nexthop_nh->fib_nh_dev->ifindex);
1565			head = &fib_info_devhash[hash];
1566			hlist_add_head(&nexthop_nh->nh_hash, head);
1567		} endfor_nexthops(fi)
1568	}
1569	spin_unlock_bh(&fib_info_lock);
1570	return fi;
1571
1572err_inval:
1573	err = -EINVAL;
1574
1575failure:
1576	if (fi) {
1577		fi->fib_dead = 1;
1578		free_fib_info(fi);
1579	}
1580
1581	return ERR_PTR(err);
1582}
1583
1584int fib_nexthop_info(struct sk_buff *skb, const struct fib_nh_common *nhc,
1585		     u8 rt_family, unsigned char *flags, bool skip_oif)
1586{
1587	if (nhc->nhc_flags & RTNH_F_DEAD)
1588		*flags |= RTNH_F_DEAD;
1589
1590	if (nhc->nhc_flags & RTNH_F_LINKDOWN) {
1591		*flags |= RTNH_F_LINKDOWN;
1592
1593		rcu_read_lock();
1594		switch (nhc->nhc_family) {
1595		case AF_INET:
1596			if (ip_ignore_linkdown(nhc->nhc_dev))
1597				*flags |= RTNH_F_DEAD;
1598			break;
1599		case AF_INET6:
1600			if (ip6_ignore_linkdown(nhc->nhc_dev))
1601				*flags |= RTNH_F_DEAD;
1602			break;
1603		}
1604		rcu_read_unlock();
1605	}
1606
1607	switch (nhc->nhc_gw_family) {
1608	case AF_INET:
1609		if (nla_put_in_addr(skb, RTA_GATEWAY, nhc->nhc_gw.ipv4))
1610			goto nla_put_failure;
1611		break;
1612	case AF_INET6:
1613		/* if gateway family does not match nexthop family
1614		 * gateway is encoded as RTA_VIA
1615		 */
1616		if (rt_family != nhc->nhc_gw_family) {
1617			int alen = sizeof(struct in6_addr);
1618			struct nlattr *nla;
1619			struct rtvia *via;
1620
1621			nla = nla_reserve(skb, RTA_VIA, alen + 2);
1622			if (!nla)
1623				goto nla_put_failure;
1624
1625			via = nla_data(nla);
1626			via->rtvia_family = AF_INET6;
1627			memcpy(via->rtvia_addr, &nhc->nhc_gw.ipv6, alen);
1628		} else if (nla_put_in6_addr(skb, RTA_GATEWAY,
1629					    &nhc->nhc_gw.ipv6) < 0) {
1630			goto nla_put_failure;
1631		}
1632		break;
1633	}
1634
1635	*flags |= (nhc->nhc_flags & RTNH_F_ONLINK);
1636	if (nhc->nhc_flags & RTNH_F_OFFLOAD)
1637		*flags |= RTNH_F_OFFLOAD;
1638
1639	if (!skip_oif && nhc->nhc_dev &&
1640	    nla_put_u32(skb, RTA_OIF, nhc->nhc_dev->ifindex))
1641		goto nla_put_failure;
1642
1643	if (nhc->nhc_lwtstate &&
1644	    lwtunnel_fill_encap(skb, nhc->nhc_lwtstate,
1645				RTA_ENCAP, RTA_ENCAP_TYPE) < 0)
1646		goto nla_put_failure;
1647
1648	return 0;
1649
1650nla_put_failure:
1651	return -EMSGSIZE;
1652}
1653EXPORT_SYMBOL_GPL(fib_nexthop_info);
1654
1655#if IS_ENABLED(CONFIG_IP_ROUTE_MULTIPATH) || IS_ENABLED(CONFIG_IPV6)
1656int fib_add_nexthop(struct sk_buff *skb, const struct fib_nh_common *nhc,
1657		    int nh_weight, u8 rt_family)
1658{
1659	const struct net_device *dev = nhc->nhc_dev;
1660	struct rtnexthop *rtnh;
1661	unsigned char flags = 0;
1662
1663	rtnh = nla_reserve_nohdr(skb, sizeof(*rtnh));
1664	if (!rtnh)
1665		goto nla_put_failure;
1666
1667	rtnh->rtnh_hops = nh_weight - 1;
1668	rtnh->rtnh_ifindex = dev ? dev->ifindex : 0;
1669
1670	if (fib_nexthop_info(skb, nhc, rt_family, &flags, true) < 0)
1671		goto nla_put_failure;
1672
1673	rtnh->rtnh_flags = flags;
1674
1675	/* length of rtnetlink header + attributes */
1676	rtnh->rtnh_len = nlmsg_get_pos(skb) - (void *)rtnh;
1677
1678	return 0;
1679
1680nla_put_failure:
1681	return -EMSGSIZE;
1682}
1683EXPORT_SYMBOL_GPL(fib_add_nexthop);
1684#endif
1685
1686#ifdef CONFIG_IP_ROUTE_MULTIPATH
1687static int fib_add_multipath(struct sk_buff *skb, struct fib_info *fi)
1688{
1689	struct nlattr *mp;
1690
1691	mp = nla_nest_start_noflag(skb, RTA_MULTIPATH);
1692	if (!mp)
1693		goto nla_put_failure;
1694
1695	if (unlikely(fi->nh)) {
1696		if (nexthop_mpath_fill_node(skb, fi->nh, AF_INET) < 0)
1697			goto nla_put_failure;
1698		goto mp_end;
1699	}
1700
1701	for_nexthops(fi) {
1702		if (fib_add_nexthop(skb, &nh->nh_common, nh->fib_nh_weight,
1703				    AF_INET) < 0)
1704			goto nla_put_failure;
1705#ifdef CONFIG_IP_ROUTE_CLASSID
1706		if (nh->nh_tclassid &&
1707		    nla_put_u32(skb, RTA_FLOW, nh->nh_tclassid))
1708			goto nla_put_failure;
1709#endif
1710	} endfor_nexthops(fi);
1711
1712mp_end:
1713	nla_nest_end(skb, mp);
1714
1715	return 0;
1716
1717nla_put_failure:
1718	return -EMSGSIZE;
1719}
1720#else
1721static int fib_add_multipath(struct sk_buff *skb, struct fib_info *fi)
1722{
1723	return 0;
1724}
1725#endif
1726
1727int fib_dump_info(struct sk_buff *skb, u32 portid, u32 seq, int event,
1728		  u32 tb_id, u8 type, __be32 dst, int dst_len, u8 tos,
1729		  struct fib_info *fi, unsigned int flags)
1730{
1731	unsigned int nhs = fib_info_num_path(fi);
1732	struct nlmsghdr *nlh;
1733	struct rtmsg *rtm;
1734
1735	nlh = nlmsg_put(skb, portid, seq, event, sizeof(*rtm), flags);
1736	if (!nlh)
1737		return -EMSGSIZE;
1738
1739	rtm = nlmsg_data(nlh);
1740	rtm->rtm_family = AF_INET;
1741	rtm->rtm_dst_len = dst_len;
1742	rtm->rtm_src_len = 0;
1743	rtm->rtm_tos = tos;
1744	if (tb_id < 256)
1745		rtm->rtm_table = tb_id;
1746	else
1747		rtm->rtm_table = RT_TABLE_COMPAT;
1748	if (nla_put_u32(skb, RTA_TABLE, tb_id))
1749		goto nla_put_failure;
1750	rtm->rtm_type = type;
1751	rtm->rtm_flags = fi->fib_flags;
1752	rtm->rtm_scope = fi->fib_scope;
1753	rtm->rtm_protocol = fi->fib_protocol;
1754
1755	if (rtm->rtm_dst_len &&
1756	    nla_put_in_addr(skb, RTA_DST, dst))
1757		goto nla_put_failure;
1758	if (fi->fib_priority &&
1759	    nla_put_u32(skb, RTA_PRIORITY, fi->fib_priority))
1760		goto nla_put_failure;
1761	if (rtnetlink_put_metrics(skb, fi->fib_metrics->metrics) < 0)
1762		goto nla_put_failure;
1763
1764	if (fi->fib_prefsrc &&
1765	    nla_put_in_addr(skb, RTA_PREFSRC, fi->fib_prefsrc))
1766		goto nla_put_failure;
 
 
 
 
 
 
 
 
 
1767
1768	if (fi->nh) {
1769		if (nla_put_u32(skb, RTA_NH_ID, fi->nh->id))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1770			goto nla_put_failure;
1771		if (nexthop_is_blackhole(fi->nh))
1772			rtm->rtm_type = RTN_BLACKHOLE;
1773	}
 
 
 
 
1774
1775	if (nhs == 1) {
1776		const struct fib_nh_common *nhc = fib_info_nhc(fi, 0);
1777		unsigned char flags = 0;
1778
1779		if (fib_nexthop_info(skb, nhc, AF_INET, &flags, false) < 0)
1780			goto nla_put_failure;
1781
1782		rtm->rtm_flags = flags;
1783#ifdef CONFIG_IP_ROUTE_CLASSID
1784		if (nhc->nhc_family == AF_INET) {
1785			struct fib_nh *nh;
1786
1787			nh = container_of(nhc, struct fib_nh, nh_common);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1788			if (nh->nh_tclassid &&
1789			    nla_put_u32(skb, RTA_FLOW, nh->nh_tclassid))
1790				goto nla_put_failure;
1791		}
1792#endif
1793	} else {
1794		if (fib_add_multipath(skb, fi) < 0)
1795			goto nla_put_failure;
1796	}
 
 
 
1797
 
 
 
1798	nlmsg_end(skb, nlh);
1799	return 0;
1800
1801nla_put_failure:
1802	nlmsg_cancel(skb, nlh);
1803	return -EMSGSIZE;
1804}
1805
1806/*
1807 * Update FIB if:
1808 * - local address disappeared -> we must delete all the entries
1809 *   referring to it.
1810 * - device went down -> we must shutdown all nexthops going via it.
1811 */
1812int fib_sync_down_addr(struct net_device *dev, __be32 local)
1813{
1814	int ret = 0;
1815	unsigned int hash = fib_laddr_hashfn(local);
1816	struct hlist_head *head = &fib_info_laddrhash[hash];
1817	int tb_id = l3mdev_fib_table(dev) ? : RT_TABLE_MAIN;
1818	struct net *net = dev_net(dev);
 
1819	struct fib_info *fi;
1820
1821	if (!fib_info_laddrhash || local == 0)
1822		return 0;
1823
1824	hlist_for_each_entry(fi, head, fib_lhash) {
1825		if (!net_eq(fi->fib_net, net) ||
1826		    fi->fib_tb_id != tb_id)
1827			continue;
1828		if (fi->fib_prefsrc == local) {
1829			fi->fib_flags |= RTNH_F_DEAD;
1830			ret++;
1831		}
1832	}
1833	return ret;
1834}
1835
1836static int call_fib_nh_notifiers(struct fib_nh *nh,
1837				 enum fib_event_type event_type)
1838{
1839	bool ignore_link_down = ip_ignore_linkdown(nh->fib_nh_dev);
1840	struct fib_nh_notifier_info info = {
1841		.fib_nh = nh,
1842	};
1843
1844	switch (event_type) {
1845	case FIB_EVENT_NH_ADD:
1846		if (nh->fib_nh_flags & RTNH_F_DEAD)
1847			break;
1848		if (ignore_link_down && nh->fib_nh_flags & RTNH_F_LINKDOWN)
 
1849			break;
1850		return call_fib4_notifiers(dev_net(nh->fib_nh_dev), event_type,
1851					   &info.info);
1852	case FIB_EVENT_NH_DEL:
1853		if ((ignore_link_down && nh->fib_nh_flags & RTNH_F_LINKDOWN) ||
1854		    (nh->fib_nh_flags & RTNH_F_DEAD))
1855			return call_fib4_notifiers(dev_net(nh->fib_nh_dev),
 
1856						   event_type, &info.info);
1857	default:
1858		break;
1859	}
1860
1861	return NOTIFY_DONE;
1862}
1863
1864/* Update the PMTU of exceptions when:
1865 * - the new MTU of the first hop becomes smaller than the PMTU
1866 * - the old MTU was the same as the PMTU, and it limited discovery of
1867 *   larger MTUs on the path. With that limit raised, we can now
1868 *   discover larger MTUs
1869 * A special case is locked exceptions, for which the PMTU is smaller
1870 * than the minimal accepted PMTU:
1871 * - if the new MTU is greater than the PMTU, don't make any change
1872 * - otherwise, unlock and set PMTU
1873 */
1874void fib_nhc_update_mtu(struct fib_nh_common *nhc, u32 new, u32 orig)
1875{
1876	struct fnhe_hash_bucket *bucket;
1877	int i;
1878
1879	bucket = rcu_dereference_protected(nhc->nhc_exceptions, 1);
1880	if (!bucket)
1881		return;
1882
1883	for (i = 0; i < FNHE_HASH_SIZE; i++) {
1884		struct fib_nh_exception *fnhe;
1885
1886		for (fnhe = rcu_dereference_protected(bucket[i].chain, 1);
1887		     fnhe;
1888		     fnhe = rcu_dereference_protected(fnhe->fnhe_next, 1)) {
1889			if (fnhe->fnhe_mtu_locked) {
1890				if (new <= fnhe->fnhe_pmtu) {
1891					fnhe->fnhe_pmtu = new;
1892					fnhe->fnhe_mtu_locked = false;
1893				}
1894			} else if (new < fnhe->fnhe_pmtu ||
1895				   orig == fnhe->fnhe_pmtu) {
1896				fnhe->fnhe_pmtu = new;
1897			}
1898		}
1899	}
1900}
1901
1902void fib_sync_mtu(struct net_device *dev, u32 orig_mtu)
1903{
1904	unsigned int hash = fib_devindex_hashfn(dev->ifindex);
1905	struct hlist_head *head = &fib_info_devhash[hash];
1906	struct fib_nh *nh;
1907
1908	hlist_for_each_entry(nh, head, nh_hash) {
1909		if (nh->fib_nh_dev == dev)
1910			fib_nhc_update_mtu(&nh->nh_common, dev->mtu, orig_mtu);
1911	}
1912}
1913
1914/* Event              force Flags           Description
1915 * NETDEV_CHANGE      0     LINKDOWN        Carrier OFF, not for scope host
1916 * NETDEV_DOWN        0     LINKDOWN|DEAD   Link down, not for scope host
1917 * NETDEV_DOWN        1     LINKDOWN|DEAD   Last address removed
1918 * NETDEV_UNREGISTER  1     LINKDOWN|DEAD   Device removed
1919 *
1920 * only used when fib_nh is built into fib_info
1921 */
1922int fib_sync_down_dev(struct net_device *dev, unsigned long event, bool force)
1923{
1924	int ret = 0;
1925	int scope = RT_SCOPE_NOWHERE;
1926	struct fib_info *prev_fi = NULL;
1927	unsigned int hash = fib_devindex_hashfn(dev->ifindex);
1928	struct hlist_head *head = &fib_info_devhash[hash];
1929	struct fib_nh *nh;
1930
1931	if (force)
1932		scope = -1;
1933
1934	hlist_for_each_entry(nh, head, nh_hash) {
1935		struct fib_info *fi = nh->nh_parent;
1936		int dead;
1937
1938		BUG_ON(!fi->fib_nhs);
1939		if (nh->fib_nh_dev != dev || fi == prev_fi)
1940			continue;
1941		prev_fi = fi;
1942		dead = 0;
1943		change_nexthops(fi) {
1944			if (nexthop_nh->fib_nh_flags & RTNH_F_DEAD)
1945				dead++;
1946			else if (nexthop_nh->fib_nh_dev == dev &&
1947				 nexthop_nh->fib_nh_scope != scope) {
1948				switch (event) {
1949				case NETDEV_DOWN:
1950				case NETDEV_UNREGISTER:
1951					nexthop_nh->fib_nh_flags |= RTNH_F_DEAD;
1952					/* fall through */
1953				case NETDEV_CHANGE:
1954					nexthop_nh->fib_nh_flags |= RTNH_F_LINKDOWN;
1955					break;
1956				}
1957				call_fib_nh_notifiers(nexthop_nh,
1958						      FIB_EVENT_NH_DEL);
1959				dead++;
1960			}
1961#ifdef CONFIG_IP_ROUTE_MULTIPATH
1962			if (event == NETDEV_UNREGISTER &&
1963			    nexthop_nh->fib_nh_dev == dev) {
1964				dead = fi->fib_nhs;
1965				break;
1966			}
1967#endif
1968		} endfor_nexthops(fi)
1969		if (dead == fi->fib_nhs) {
1970			switch (event) {
1971			case NETDEV_DOWN:
1972			case NETDEV_UNREGISTER:
1973				fi->fib_flags |= RTNH_F_DEAD;
1974				/* fall through */
1975			case NETDEV_CHANGE:
1976				fi->fib_flags |= RTNH_F_LINKDOWN;
1977				break;
1978			}
1979			ret++;
1980		}
1981
1982		fib_rebalance(fi);
1983	}
1984
1985	return ret;
1986}
1987
1988/* Must be invoked inside of an RCU protected region.  */
1989static void fib_select_default(const struct flowi4 *flp, struct fib_result *res)
1990{
1991	struct fib_info *fi = NULL, *last_resort = NULL;
1992	struct hlist_head *fa_head = res->fa_head;
1993	struct fib_table *tb = res->table;
1994	u8 slen = 32 - res->prefixlen;
1995	int order = -1, last_idx = -1;
1996	struct fib_alias *fa, *fa1 = NULL;
1997	u32 last_prio = res->fi->fib_priority;
1998	u8 last_tos = 0;
1999
2000	hlist_for_each_entry_rcu(fa, fa_head, fa_list) {
2001		struct fib_info *next_fi = fa->fa_info;
2002		struct fib_nh *nh;
2003
2004		if (fa->fa_slen != slen)
2005			continue;
2006		if (fa->fa_tos && fa->fa_tos != flp->flowi4_tos)
2007			continue;
2008		if (fa->tb_id != tb->tb_id)
2009			continue;
2010		if (next_fi->fib_priority > last_prio &&
2011		    fa->fa_tos == last_tos) {
2012			if (last_tos)
2013				continue;
2014			break;
2015		}
2016		if (next_fi->fib_flags & RTNH_F_DEAD)
2017			continue;
2018		last_tos = fa->fa_tos;
2019		last_prio = next_fi->fib_priority;
2020
2021		if (next_fi->fib_scope != res->scope ||
2022		    fa->fa_type != RTN_UNICAST)
2023			continue;
2024
2025		nh = fib_info_nh(next_fi, 0);
2026		if (!nh->fib_nh_gw4 || nh->fib_nh_scope != RT_SCOPE_LINK)
2027			continue;
2028
2029		fib_alias_accessed(fa);
2030
2031		if (!fi) {
2032			if (next_fi != res->fi)
2033				break;
2034			fa1 = fa;
2035		} else if (!fib_detect_death(fi, order, &last_resort,
2036					     &last_idx, fa1->fa_default)) {
2037			fib_result_assign(res, fi);
2038			fa1->fa_default = order;
2039			goto out;
2040		}
2041		fi = next_fi;
2042		order++;
2043	}
2044
2045	if (order <= 0 || !fi) {
2046		if (fa1)
2047			fa1->fa_default = -1;
2048		goto out;
2049	}
2050
2051	if (!fib_detect_death(fi, order, &last_resort, &last_idx,
2052			      fa1->fa_default)) {
2053		fib_result_assign(res, fi);
2054		fa1->fa_default = order;
2055		goto out;
2056	}
2057
2058	if (last_idx >= 0)
2059		fib_result_assign(res, last_resort);
2060	fa1->fa_default = last_idx;
2061out:
2062	return;
2063}
2064
2065/*
2066 * Dead device goes up. We wake up dead nexthops.
2067 * It takes sense only on multipath routes.
2068 *
2069 * only used when fib_nh is built into fib_info
2070 */
2071int fib_sync_up(struct net_device *dev, unsigned char nh_flags)
2072{
2073	struct fib_info *prev_fi;
2074	unsigned int hash;
2075	struct hlist_head *head;
2076	struct fib_nh *nh;
2077	int ret;
2078
2079	if (!(dev->flags & IFF_UP))
2080		return 0;
2081
2082	if (nh_flags & RTNH_F_DEAD) {
2083		unsigned int flags = dev_get_flags(dev);
2084
2085		if (flags & (IFF_RUNNING | IFF_LOWER_UP))
2086			nh_flags |= RTNH_F_LINKDOWN;
2087	}
2088
2089	prev_fi = NULL;
2090	hash = fib_devindex_hashfn(dev->ifindex);
2091	head = &fib_info_devhash[hash];
2092	ret = 0;
2093
2094	hlist_for_each_entry(nh, head, nh_hash) {
2095		struct fib_info *fi = nh->nh_parent;
2096		int alive;
2097
2098		BUG_ON(!fi->fib_nhs);
2099		if (nh->fib_nh_dev != dev || fi == prev_fi)
2100			continue;
2101
2102		prev_fi = fi;
2103		alive = 0;
2104		change_nexthops(fi) {
2105			if (!(nexthop_nh->fib_nh_flags & nh_flags)) {
2106				alive++;
2107				continue;
2108			}
2109			if (!nexthop_nh->fib_nh_dev ||
2110			    !(nexthop_nh->fib_nh_dev->flags & IFF_UP))
2111				continue;
2112			if (nexthop_nh->fib_nh_dev != dev ||
2113			    !__in_dev_get_rtnl(dev))
2114				continue;
2115			alive++;
2116			nexthop_nh->fib_nh_flags &= ~nh_flags;
2117			call_fib_nh_notifiers(nexthop_nh, FIB_EVENT_NH_ADD);
2118		} endfor_nexthops(fi)
2119
2120		if (alive > 0) {
2121			fi->fib_flags &= ~nh_flags;
2122			ret++;
2123		}
2124
2125		fib_rebalance(fi);
2126	}
2127
2128	return ret;
2129}
2130
2131#ifdef CONFIG_IP_ROUTE_MULTIPATH
2132static bool fib_good_nh(const struct fib_nh *nh)
2133{
2134	int state = NUD_REACHABLE;
2135
2136	if (nh->fib_nh_scope == RT_SCOPE_LINK) {
2137		struct neighbour *n;
2138
2139		rcu_read_lock_bh();
2140
2141		if (likely(nh->fib_nh_gw_family == AF_INET))
2142			n = __ipv4_neigh_lookup_noref(nh->fib_nh_dev,
2143						   (__force u32)nh->fib_nh_gw4);
2144		else if (nh->fib_nh_gw_family == AF_INET6)
2145			n = __ipv6_neigh_lookup_noref_stub(nh->fib_nh_dev,
2146							   &nh->fib_nh_gw6);
2147		else
2148			n = NULL;
2149		if (n)
2150			state = n->nud_state;
2151
2152		rcu_read_unlock_bh();
2153	}
2154
2155	return !!(state & NUD_VALID);
2156}
2157
2158void fib_select_multipath(struct fib_result *res, int hash)
2159{
2160	struct fib_info *fi = res->fi;
2161	struct net *net = fi->fib_net;
2162	bool first = false;
2163
2164	if (unlikely(res->fi->nh)) {
2165		nexthop_path_fib_result(res, hash);
2166		return;
2167	}
2168
2169	change_nexthops(fi) {
2170		if (net->ipv4.sysctl_fib_multipath_use_neigh) {
2171			if (!fib_good_nh(nexthop_nh))
2172				continue;
2173			if (!first) {
2174				res->nh_sel = nhsel;
2175				res->nhc = &nexthop_nh->nh_common;
2176				first = true;
2177			}
2178		}
2179
2180		if (hash > atomic_read(&nexthop_nh->fib_nh_upper_bound))
2181			continue;
2182
2183		res->nh_sel = nhsel;
2184		res->nhc = &nexthop_nh->nh_common;
2185		return;
2186	} endfor_nexthops(fi);
2187}
2188#endif
2189
2190void fib_select_path(struct net *net, struct fib_result *res,
2191		     struct flowi4 *fl4, const struct sk_buff *skb)
2192{
2193	if (fl4->flowi4_oif && !(fl4->flowi4_flags & FLOWI_FLAG_SKIP_NH_OIF))
2194		goto check_saddr;
2195
2196#ifdef CONFIG_IP_ROUTE_MULTIPATH
2197	if (fib_info_num_path(res->fi) > 1) {
2198		int h = fib_multipath_hash(net, fl4, skb, NULL);
2199
2200		fib_select_multipath(res, h);
2201	}
2202	else
2203#endif
2204	if (!res->prefixlen &&
2205	    res->table->tb_num_default > 1 &&
2206	    res->type == RTN_UNICAST)
2207		fib_select_default(fl4, res);
2208
2209check_saddr:
2210	if (!fl4->saddr)
2211		fl4->saddr = fib_result_prefsrc(net, res);
2212}