1// SPDX-License-Identifier: GPL-2.0
   2/* Generic nexthop implementation
   3 *
   4 * Copyright (c) 2017-19 Cumulus Networks
   5 * Copyright (c) 2017-19 David Ahern <dsa@cumulusnetworks.com>
   6 */
   7
   8#include <linux/nexthop.h>
   9#include <linux/rtnetlink.h>
  10#include <linux/slab.h>
  11#include <linux/vmalloc.h>
  12#include <net/arp.h>
  13#include <net/ipv6_stubs.h>
  14#include <net/lwtunnel.h>
  15#include <net/ndisc.h>
  16#include <net/nexthop.h>
  17#include <net/route.h>
  18#include <net/sock.h>
  19
  20#define NH_RES_DEFAULT_IDLE_TIMER	(120 * HZ)
  21#define NH_RES_DEFAULT_UNBALANCED_TIMER	0	/* No forced rebalancing. */
  22
  23static void remove_nexthop(struct net *net, struct nexthop *nh,
  24			   struct nl_info *nlinfo);
  25
  26#define NH_DEV_HASHBITS  8
  27#define NH_DEV_HASHSIZE (1U << NH_DEV_HASHBITS)
  28
  29static const struct nla_policy rtm_nh_policy_new[] = {
  30	[NHA_ID]		= { .type = NLA_U32 },
  31	[NHA_GROUP]		= { .type = NLA_BINARY },
  32	[NHA_GROUP_TYPE]	= { .type = NLA_U16 },
  33	[NHA_BLACKHOLE]		= { .type = NLA_FLAG },
  34	[NHA_OIF]		= { .type = NLA_U32 },
  35	[NHA_GATEWAY]		= { .type = NLA_BINARY },
  36	[NHA_ENCAP_TYPE]	= { .type = NLA_U16 },
  37	[NHA_ENCAP]		= { .type = NLA_NESTED },
  38	[NHA_FDB]		= { .type = NLA_FLAG },
  39	[NHA_RES_GROUP]		= { .type = NLA_NESTED },
  40};
  41
  42static const struct nla_policy rtm_nh_policy_get[] = {
  43	[NHA_ID]		= { .type = NLA_U32 },
  44};
  45
  46static const struct nla_policy rtm_nh_policy_dump[] = {
  47	[NHA_OIF]		= { .type = NLA_U32 },
  48	[NHA_GROUPS]		= { .type = NLA_FLAG },
  49	[NHA_MASTER]		= { .type = NLA_U32 },
  50	[NHA_FDB]		= { .type = NLA_FLAG },
  51};
  52
  53static const struct nla_policy rtm_nh_res_policy_new[] = {
  54	[NHA_RES_GROUP_BUCKETS]			= { .type = NLA_U16 },
  55	[NHA_RES_GROUP_IDLE_TIMER]		= { .type = NLA_U32 },
  56	[NHA_RES_GROUP_UNBALANCED_TIMER]	= { .type = NLA_U32 },
  57};
  58
  59static const struct nla_policy rtm_nh_policy_dump_bucket[] = {
  60	[NHA_ID]		= { .type = NLA_U32 },
  61	[NHA_OIF]		= { .type = NLA_U32 },
  62	[NHA_MASTER]		= { .type = NLA_U32 },
  63	[NHA_RES_BUCKET]	= { .type = NLA_NESTED },
  64};
  65
  66static const struct nla_policy rtm_nh_res_bucket_policy_dump[] = {
  67	[NHA_RES_BUCKET_NH_ID]	= { .type = NLA_U32 },
  68};
  69
  70static const struct nla_policy rtm_nh_policy_get_bucket[] = {
  71	[NHA_ID]		= { .type = NLA_U32 },
  72	[NHA_RES_BUCKET]	= { .type = NLA_NESTED },
  73};
  74
  75static const struct nla_policy rtm_nh_res_bucket_policy_get[] = {
  76	[NHA_RES_BUCKET_INDEX]	= { .type = NLA_U16 },
  77};
  78
  79static bool nexthop_notifiers_is_empty(struct net *net)
  80{
  81	return !net->nexthop.notifier_chain.head;
  82}
  83
  84static void
  85__nh_notifier_single_info_init(struct nh_notifier_single_info *nh_info,
  86			       const struct nh_info *nhi)
  87{
  88	nh_info->dev = nhi->fib_nhc.nhc_dev;
  89	nh_info->gw_family = nhi->fib_nhc.nhc_gw_family;
  90	if (nh_info->gw_family == AF_INET)
  91		nh_info->ipv4 = nhi->fib_nhc.nhc_gw.ipv4;
  92	else if (nh_info->gw_family == AF_INET6)
  93		nh_info->ipv6 = nhi->fib_nhc.nhc_gw.ipv6;
  94
  95	nh_info->is_reject = nhi->reject_nh;
  96	nh_info->is_fdb = nhi->fdb_nh;
  97	nh_info->has_encap = !!nhi->fib_nhc.nhc_lwtstate;
  98}
  99
 100static int nh_notifier_single_info_init(struct nh_notifier_info *info,
 101					const struct nexthop *nh)
 102{
 103	struct nh_info *nhi = rtnl_dereference(nh->nh_info);
 104
 105	info->type = NH_NOTIFIER_INFO_TYPE_SINGLE;
 106	info->nh = kzalloc(sizeof(*info->nh), GFP_KERNEL);
 107	if (!info->nh)
 108		return -ENOMEM;
 109
 110	__nh_notifier_single_info_init(info->nh, nhi);
 111
 112	return 0;
 113}
 114
 115static void nh_notifier_single_info_fini(struct nh_notifier_info *info)
 116{
 117	kfree(info->nh);
 118}
 119
 120static int nh_notifier_mpath_info_init(struct nh_notifier_info *info,
 121				       struct nh_group *nhg)
 122{
 123	u16 num_nh = nhg->num_nh;
 124	int i;
 125
 126	info->type = NH_NOTIFIER_INFO_TYPE_GRP;
 127	info->nh_grp = kzalloc(struct_size(info->nh_grp, nh_entries, num_nh),
 128			       GFP_KERNEL);
 129	if (!info->nh_grp)
 130		return -ENOMEM;
 131
 132	info->nh_grp->num_nh = num_nh;
 133	info->nh_grp->is_fdb = nhg->fdb_nh;
 134
 135	for (i = 0; i < num_nh; i++) {
 136		struct nh_grp_entry *nhge = &nhg->nh_entries[i];
 137		struct nh_info *nhi;
 138
 139		nhi = rtnl_dereference(nhge->nh->nh_info);
 140		info->nh_grp->nh_entries[i].id = nhge->nh->id;
 141		info->nh_grp->nh_entries[i].weight = nhge->weight;
 142		__nh_notifier_single_info_init(&info->nh_grp->nh_entries[i].nh,
 143					       nhi);
 144	}
 145
 146	return 0;
 147}
 148
 149static int nh_notifier_res_table_info_init(struct nh_notifier_info *info,
 150					   struct nh_group *nhg)
 151{
 152	struct nh_res_table *res_table = rtnl_dereference(nhg->res_table);
 153	u16 num_nh_buckets = res_table->num_nh_buckets;
 154	unsigned long size;
 155	u16 i;
 156
 157	info->type = NH_NOTIFIER_INFO_TYPE_RES_TABLE;
 158	size = struct_size(info->nh_res_table, nhs, num_nh_buckets);
 159	info->nh_res_table = __vmalloc(size, GFP_KERNEL | __GFP_ZERO |
 160				       __GFP_NOWARN);
 161	if (!info->nh_res_table)
 162		return -ENOMEM;
 163
 164	info->nh_res_table->num_nh_buckets = num_nh_buckets;
 165
 166	for (i = 0; i < num_nh_buckets; i++) {
 167		struct nh_res_bucket *bucket = &res_table->nh_buckets[i];
 168		struct nh_grp_entry *nhge;
 169		struct nh_info *nhi;
 170
 171		nhge = rtnl_dereference(bucket->nh_entry);
 172		nhi = rtnl_dereference(nhge->nh->nh_info);
 173		__nh_notifier_single_info_init(&info->nh_res_table->nhs[i],
 174					       nhi);
 175	}
 176
 177	return 0;
 178}
 179
 180static int nh_notifier_grp_info_init(struct nh_notifier_info *info,
 181				     const struct nexthop *nh)
 182{
 183	struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
 184
 185	if (nhg->hash_threshold)
 186		return nh_notifier_mpath_info_init(info, nhg);
 187	else if (nhg->resilient)
 188		return nh_notifier_res_table_info_init(info, nhg);
 189	return -EINVAL;
 190}
 191
 192static void nh_notifier_grp_info_fini(struct nh_notifier_info *info,
 193				      const struct nexthop *nh)
 194{
 195	struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
 196
 197	if (nhg->hash_threshold)
 198		kfree(info->nh_grp);
 199	else if (nhg->resilient)
 200		vfree(info->nh_res_table);
 201}
 202
 203static int nh_notifier_info_init(struct nh_notifier_info *info,
 204				 const struct nexthop *nh)
 205{
 206	info->id = nh->id;
 207
 208	if (nh->is_group)
 209		return nh_notifier_grp_info_init(info, nh);
 210	else
 211		return nh_notifier_single_info_init(info, nh);
 212}
 213
 214static void nh_notifier_info_fini(struct nh_notifier_info *info,
 215				  const struct nexthop *nh)
 216{
 217	if (nh->is_group)
 218		nh_notifier_grp_info_fini(info, nh);
 219	else
 220		nh_notifier_single_info_fini(info);
 221}
 222
 223static int call_nexthop_notifiers(struct net *net,
 224				  enum nexthop_event_type event_type,
 225				  struct nexthop *nh,
 226				  struct netlink_ext_ack *extack)
 227{
 228	struct nh_notifier_info info = {
 229		.net = net,
 230		.extack = extack,
 231	};
 232	int err;
 233
 234	ASSERT_RTNL();
 235
 236	if (nexthop_notifiers_is_empty(net))
 237		return 0;
 238
 239	err = nh_notifier_info_init(&info, nh);
 240	if (err) {
 241		NL_SET_ERR_MSG(extack, "Failed to initialize nexthop notifier info");
 242		return err;
 243	}
 244
 245	err = blocking_notifier_call_chain(&net->nexthop.notifier_chain,
 246					   event_type, &info);
 247	nh_notifier_info_fini(&info, nh);
 248
 249	return notifier_to_errno(err);
 250}
 251
 252static int
 253nh_notifier_res_bucket_idle_timer_get(const struct nh_notifier_info *info,
 254				      bool force, unsigned int *p_idle_timer_ms)
 255{
 256	struct nh_res_table *res_table;
 257	struct nh_group *nhg;
 258	struct nexthop *nh;
 259	int err = 0;
 260
 261	/* When 'force' is false, nexthop bucket replacement is performed
 262	 * because the bucket was deemed to be idle. In this case, capable
 263	 * listeners can choose to perform an atomic replacement: The bucket is
 264	 * only replaced if it is inactive. However, if the idle timer interval
 265	 * is smaller than the interval in which a listener is querying
 266	 * buckets' activity from the device, then atomic replacement should
 267	 * not be tried. Pass the idle timer value to listeners, so that they
 268	 * could determine which type of replacement to perform.
 269	 */
 270	if (force) {
 271		*p_idle_timer_ms = 0;
 272		return 0;
 273	}
 274
 275	rcu_read_lock();
 276
 277	nh = nexthop_find_by_id(info->net, info->id);
 278	if (!nh) {
 279		err = -EINVAL;
 280		goto out;
 281	}
 282
 283	nhg = rcu_dereference(nh->nh_grp);
 284	res_table = rcu_dereference(nhg->res_table);
 285	*p_idle_timer_ms = jiffies_to_msecs(res_table->idle_timer);
 286
 287out:
 288	rcu_read_unlock();
 289
 290	return err;
 291}
 292
 293static int nh_notifier_res_bucket_info_init(struct nh_notifier_info *info,
 294					    u16 bucket_index, bool force,
 295					    struct nh_info *oldi,
 296					    struct nh_info *newi)
 297{
 298	unsigned int idle_timer_ms;
 299	int err;
 300
 301	err = nh_notifier_res_bucket_idle_timer_get(info, force,
 302						    &idle_timer_ms);
 303	if (err)
 304		return err;
 305
 306	info->type = NH_NOTIFIER_INFO_TYPE_RES_BUCKET;
 307	info->nh_res_bucket = kzalloc(sizeof(*info->nh_res_bucket),
 308				      GFP_KERNEL);
 309	if (!info->nh_res_bucket)
 310		return -ENOMEM;
 311
 312	info->nh_res_bucket->bucket_index = bucket_index;
 313	info->nh_res_bucket->idle_timer_ms = idle_timer_ms;
 314	info->nh_res_bucket->force = force;
 315	__nh_notifier_single_info_init(&info->nh_res_bucket->old_nh, oldi);
 316	__nh_notifier_single_info_init(&info->nh_res_bucket->new_nh, newi);
 317	return 0;
 318}
 319
 320static void nh_notifier_res_bucket_info_fini(struct nh_notifier_info *info)
 321{
 322	kfree(info->nh_res_bucket);
 323}
 324
 325static int __call_nexthop_res_bucket_notifiers(struct net *net, u32 nhg_id,
 326					       u16 bucket_index, bool force,
 327					       struct nh_info *oldi,
 328					       struct nh_info *newi,
 329					       struct netlink_ext_ack *extack)
 330{
 331	struct nh_notifier_info info = {
 332		.net = net,
 333		.extack = extack,
 334		.id = nhg_id,
 335	};
 336	int err;
 337
 338	if (nexthop_notifiers_is_empty(net))
 339		return 0;
 340
 341	err = nh_notifier_res_bucket_info_init(&info, bucket_index, force,
 342					       oldi, newi);
 343	if (err)
 344		return err;
 345
 346	err = blocking_notifier_call_chain(&net->nexthop.notifier_chain,
 347					   NEXTHOP_EVENT_BUCKET_REPLACE, &info);
 348	nh_notifier_res_bucket_info_fini(&info);
 349
 350	return notifier_to_errno(err);
 351}
 352
 353/* There are three users of RES_TABLE, and NHs etc. referenced from there:
 354 *
 355 * 1) a collection of callbacks for NH maintenance. This operates under
 356 *    RTNL,
 357 * 2) the delayed work that gradually balances the resilient table,
 358 * 3) and nexthop_select_path(), operating under RCU.
 359 *
 360 * Both the delayed work and the RTNL block are writers, and need to
 361 * maintain mutual exclusion. Since there are only two and well-known
 362 * writers for each table, the RTNL code can make sure it has exclusive
 363 * access thus:
 364 *
 365 * - Have the DW operate without locking;
 366 * - synchronously cancel the DW;
 367 * - do the writing;
 368 * - if the write was not actually a delete, call upkeep, which schedules
 369 *   DW again if necessary.
 370 *
 371 * The functions that are always called from the RTNL context use
 372 * rtnl_dereference(). The functions that can also be called from the DW do
 373 * a raw dereference and rely on the above mutual exclusion scheme.
 374 */
 375#define nh_res_dereference(p) (rcu_dereference_raw(p))
 376
 377static int call_nexthop_res_bucket_notifiers(struct net *net, u32 nhg_id,
 378					     u16 bucket_index, bool force,
 379					     struct nexthop *old_nh,
 380					     struct nexthop *new_nh,
 381					     struct netlink_ext_ack *extack)
 382{
 383	struct nh_info *oldi = nh_res_dereference(old_nh->nh_info);
 384	struct nh_info *newi = nh_res_dereference(new_nh->nh_info);
 385
 386	return __call_nexthop_res_bucket_notifiers(net, nhg_id, bucket_index,
 387						   force, oldi, newi, extack);
 388}
 389
 390static int call_nexthop_res_table_notifiers(struct net *net, struct nexthop *nh,
 391					    struct netlink_ext_ack *extack)
 392{
 393	struct nh_notifier_info info = {
 394		.net = net,
 395		.extack = extack,
 396	};
 397	struct nh_group *nhg;
 398	int err;
 399
 400	ASSERT_RTNL();
 401
 402	if (nexthop_notifiers_is_empty(net))
 403		return 0;
 404
 405	/* At this point, the nexthop buckets are still not populated. Only
 406	 * emit a notification with the logical nexthops, so that a listener
 407	 * could potentially veto it in case of unsupported configuration.
 408	 */
 409	nhg = rtnl_dereference(nh->nh_grp);
 410	err = nh_notifier_mpath_info_init(&info, nhg);
 411	if (err) {
 412		NL_SET_ERR_MSG(extack, "Failed to initialize nexthop notifier info");
 413		return err;
 414	}
 415
 416	err = blocking_notifier_call_chain(&net->nexthop.notifier_chain,
 417					   NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE,
 418					   &info);
 419	kfree(info.nh_grp);
 420
 421	return notifier_to_errno(err);
 422}
 423
 424static int call_nexthop_notifier(struct notifier_block *nb, struct net *net,
 425				 enum nexthop_event_type event_type,
 426				 struct nexthop *nh,
 427				 struct netlink_ext_ack *extack)
 428{
 429	struct nh_notifier_info info = {
 430		.net = net,
 431		.extack = extack,
 432	};
 433	int err;
 434
 435	err = nh_notifier_info_init(&info, nh);
 436	if (err)
 437		return err;
 438
 439	err = nb->notifier_call(nb, event_type, &info);
 440	nh_notifier_info_fini(&info, nh);
 441
 442	return notifier_to_errno(err);
 443}
 444
 445static unsigned int nh_dev_hashfn(unsigned int val)
 446{
 447	unsigned int mask = NH_DEV_HASHSIZE - 1;
 448
 449	return (val ^
 450		(val >> NH_DEV_HASHBITS) ^
 451		(val >> (NH_DEV_HASHBITS * 2))) & mask;
 452}
 453
 454static void nexthop_devhash_add(struct net *net, struct nh_info *nhi)
 455{
 456	struct net_device *dev = nhi->fib_nhc.nhc_dev;
 457	struct hlist_head *head;
 458	unsigned int hash;
 459
 460	WARN_ON(!dev);
 461
 462	hash = nh_dev_hashfn(dev->ifindex);
 463	head = &net->nexthop.devhash[hash];
 464	hlist_add_head(&nhi->dev_hash, head);
 465}
 466
 467static void nexthop_free_group(struct nexthop *nh)
 468{
 469	struct nh_group *nhg;
 470	int i;
 471
 472	nhg = rcu_dereference_raw(nh->nh_grp);
 473	for (i = 0; i < nhg->num_nh; ++i) {
 474		struct nh_grp_entry *nhge = &nhg->nh_entries[i];
 475
 476		WARN_ON(!list_empty(&nhge->nh_list));
 477		nexthop_put(nhge->nh);
 478	}
 479
 480	WARN_ON(nhg->spare == nhg);
 481
 482	if (nhg->resilient)
 483		vfree(rcu_dereference_raw(nhg->res_table));
 484
 485	kfree(nhg->spare);
 486	kfree(nhg);
 487}
 488
 489static void nexthop_free_single(struct nexthop *nh)
 490{
 491	struct nh_info *nhi;
 492
 493	nhi = rcu_dereference_raw(nh->nh_info);
 494	switch (nhi->family) {
 495	case AF_INET:
 496		fib_nh_release(nh->net, &nhi->fib_nh);
 497		break;
 498	case AF_INET6:
 499		ipv6_stub->fib6_nh_release(&nhi->fib6_nh);
 500		break;
 501	}
 502	kfree(nhi);
 503}
 504
 505void nexthop_free_rcu(struct rcu_head *head)
 506{
 507	struct nexthop *nh = container_of(head, struct nexthop, rcu);
 508
 509	if (nh->is_group)
 510		nexthop_free_group(nh);
 511	else
 512		nexthop_free_single(nh);
 513
 514	kfree(nh);
 515}
 516EXPORT_SYMBOL_GPL(nexthop_free_rcu);
 517
 518static struct nexthop *nexthop_alloc(void)
 519{
 520	struct nexthop *nh;
 521
 522	nh = kzalloc(sizeof(struct nexthop), GFP_KERNEL);
 523	if (nh) {
 524		INIT_LIST_HEAD(&nh->fi_list);
 525		INIT_LIST_HEAD(&nh->f6i_list);
 526		INIT_LIST_HEAD(&nh->grp_list);
 527		INIT_LIST_HEAD(&nh->fdb_list);
 528	}
 529	return nh;
 530}
 531
 532static struct nh_group *nexthop_grp_alloc(u16 num_nh)
 533{
 534	struct nh_group *nhg;
 535
 536	nhg = kzalloc(struct_size(nhg, nh_entries, num_nh), GFP_KERNEL);
 537	if (nhg)
 538		nhg->num_nh = num_nh;
 539
 540	return nhg;
 541}
 542
 543static void nh_res_table_upkeep_dw(struct work_struct *work);
 544
 545static struct nh_res_table *
 546nexthop_res_table_alloc(struct net *net, u32 nhg_id, struct nh_config *cfg)
 547{
 548	const u16 num_nh_buckets = cfg->nh_grp_res_num_buckets;
 549	struct nh_res_table *res_table;
 550	unsigned long size;
 551
 552	size = struct_size(res_table, nh_buckets, num_nh_buckets);
 553	res_table = __vmalloc(size, GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN);
 554	if (!res_table)
 555		return NULL;
 556
 557	res_table->net = net;
 558	res_table->nhg_id = nhg_id;
 559	INIT_DELAYED_WORK(&res_table->upkeep_dw, &nh_res_table_upkeep_dw);
 560	INIT_LIST_HEAD(&res_table->uw_nh_entries);
 561	res_table->idle_timer = cfg->nh_grp_res_idle_timer;
 562	res_table->unbalanced_timer = cfg->nh_grp_res_unbalanced_timer;
 563	res_table->num_nh_buckets = num_nh_buckets;
 564	return res_table;
 565}
 566
 567static void nh_base_seq_inc(struct net *net)
 568{
 569	while (++net->nexthop.seq == 0)
 570		;
 571}
 572
 573/* no reference taken; rcu lock or rtnl must be held */
 574struct nexthop *nexthop_find_by_id(struct net *net, u32 id)
 575{
 576	struct rb_node **pp, *parent = NULL, *next;
 577
 578	pp = &net->nexthop.rb_root.rb_node;
 579	while (1) {
 580		struct nexthop *nh;
 581
 582		next = rcu_dereference_raw(*pp);
 583		if (!next)
 584			break;
 585		parent = next;
 586
 587		nh = rb_entry(parent, struct nexthop, rb_node);
 588		if (id < nh->id)
 589			pp = &next->rb_left;
 590		else if (id > nh->id)
 591			pp = &next->rb_right;
 592		else
 593			return nh;
 594	}
 595	return NULL;
 596}
 597EXPORT_SYMBOL_GPL(nexthop_find_by_id);
 598
 599/* used for auto id allocation; called with rtnl held */
 600static u32 nh_find_unused_id(struct net *net)
 601{
 602	u32 id_start = net->nexthop.last_id_allocated;
 603
 604	while (1) {
 605		net->nexthop.last_id_allocated++;
 606		if (net->nexthop.last_id_allocated == id_start)
 607			break;
 608
 609		if (!nexthop_find_by_id(net, net->nexthop.last_id_allocated))
 610			return net->nexthop.last_id_allocated;
 611	}
 612	return 0;
 613}
 614
 615static void nh_res_time_set_deadline(unsigned long next_time,
 616				     unsigned long *deadline)
 617{
 618	if (time_before(next_time, *deadline))
 619		*deadline = next_time;
 620}
 621
 622static clock_t nh_res_table_unbalanced_time(struct nh_res_table *res_table)
 623{
 624	if (list_empty(&res_table->uw_nh_entries))
 625		return 0;
 626	return jiffies_delta_to_clock_t(jiffies - res_table->unbalanced_since);
 627}
 628
 629static int nla_put_nh_group_res(struct sk_buff *skb, struct nh_group *nhg)
 630{
 631	struct nh_res_table *res_table = rtnl_dereference(nhg->res_table);
 632	struct nlattr *nest;
 633
 634	nest = nla_nest_start(skb, NHA_RES_GROUP);
 635	if (!nest)
 636		return -EMSGSIZE;
 637
 638	if (nla_put_u16(skb, NHA_RES_GROUP_BUCKETS,
 639			res_table->num_nh_buckets) ||
 640	    nla_put_u32(skb, NHA_RES_GROUP_IDLE_TIMER,
 641			jiffies_to_clock_t(res_table->idle_timer)) ||
 642	    nla_put_u32(skb, NHA_RES_GROUP_UNBALANCED_TIMER,
 643			jiffies_to_clock_t(res_table->unbalanced_timer)) ||
 644	    nla_put_u64_64bit(skb, NHA_RES_GROUP_UNBALANCED_TIME,
 645			      nh_res_table_unbalanced_time(res_table),
 646			      NHA_RES_GROUP_PAD))
 647		goto nla_put_failure;
 648
 649	nla_nest_end(skb, nest);
 650	return 0;
 651
 652nla_put_failure:
 653	nla_nest_cancel(skb, nest);
 654	return -EMSGSIZE;
 655}
 656
 657static int nla_put_nh_group(struct sk_buff *skb, struct nh_group *nhg)
 658{
 659	struct nexthop_grp *p;
 660	size_t len = nhg->num_nh * sizeof(*p);
 661	struct nlattr *nla;
 662	u16 group_type = 0;
 663	int i;
 664
 665	if (nhg->hash_threshold)
 666		group_type = NEXTHOP_GRP_TYPE_MPATH;
 667	else if (nhg->resilient)
 668		group_type = NEXTHOP_GRP_TYPE_RES;
 669
 670	if (nla_put_u16(skb, NHA_GROUP_TYPE, group_type))
 671		goto nla_put_failure;
 672
 673	nla = nla_reserve(skb, NHA_GROUP, len);
 674	if (!nla)
 675		goto nla_put_failure;
 676
 677	p = nla_data(nla);
 678	for (i = 0; i < nhg->num_nh; ++i) {
 679		p->id = nhg->nh_entries[i].nh->id;
 680		p->weight = nhg->nh_entries[i].weight - 1;
 681		p += 1;
 682	}
 683
 684	if (nhg->resilient && nla_put_nh_group_res(skb, nhg))
 685		goto nla_put_failure;
 686
 687	return 0;
 688
 689nla_put_failure:
 690	return -EMSGSIZE;
 691}
 692
 693static int nh_fill_node(struct sk_buff *skb, struct nexthop *nh,
 694			int event, u32 portid, u32 seq, unsigned int nlflags)
 695{
 696	struct fib6_nh *fib6_nh;
 697	struct fib_nh *fib_nh;
 698	struct nlmsghdr *nlh;
 699	struct nh_info *nhi;
 700	struct nhmsg *nhm;
 701
 702	nlh = nlmsg_put(skb, portid, seq, event, sizeof(*nhm), nlflags);
 703	if (!nlh)
 704		return -EMSGSIZE;
 705
 706	nhm = nlmsg_data(nlh);
 707	nhm->nh_family = AF_UNSPEC;
 708	nhm->nh_flags = nh->nh_flags;
 709	nhm->nh_protocol = nh->protocol;
 710	nhm->nh_scope = 0;
 711	nhm->resvd = 0;
 712
 713	if (nla_put_u32(skb, NHA_ID, nh->id))
 714		goto nla_put_failure;
 715
 716	if (nh->is_group) {
 717		struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
 718
 719		if (nhg->fdb_nh && nla_put_flag(skb, NHA_FDB))
 720			goto nla_put_failure;
 721		if (nla_put_nh_group(skb, nhg))
 722			goto nla_put_failure;
 723		goto out;
 724	}
 725
 726	nhi = rtnl_dereference(nh->nh_info);
 727	nhm->nh_family = nhi->family;
 728	if (nhi->reject_nh) {
 729		if (nla_put_flag(skb, NHA_BLACKHOLE))
 730			goto nla_put_failure;
 731		goto out;
 732	} else if (nhi->fdb_nh) {
 733		if (nla_put_flag(skb, NHA_FDB))
 734			goto nla_put_failure;
 735	} else {
 736		const struct net_device *dev;
 737
 738		dev = nhi->fib_nhc.nhc_dev;
 739		if (dev && nla_put_u32(skb, NHA_OIF, dev->ifindex))
 740			goto nla_put_failure;
 741	}
 742
 743	nhm->nh_scope = nhi->fib_nhc.nhc_scope;
 744	switch (nhi->family) {
 745	case AF_INET:
 746		fib_nh = &nhi->fib_nh;
 747		if (fib_nh->fib_nh_gw_family &&
 748		    nla_put_be32(skb, NHA_GATEWAY, fib_nh->fib_nh_gw4))
 749			goto nla_put_failure;
 750		break;
 751
 752	case AF_INET6:
 753		fib6_nh = &nhi->fib6_nh;
 754		if (fib6_nh->fib_nh_gw_family &&
 755		    nla_put_in6_addr(skb, NHA_GATEWAY, &fib6_nh->fib_nh_gw6))
 756			goto nla_put_failure;
 757		break;
 758	}
 759
 760	if (nhi->fib_nhc.nhc_lwtstate &&
 761	    lwtunnel_fill_encap(skb, nhi->fib_nhc.nhc_lwtstate,
 762				NHA_ENCAP, NHA_ENCAP_TYPE) < 0)
 763		goto nla_put_failure;
 764
 765out:
 766	nlmsg_end(skb, nlh);
 767	return 0;
 768
 769nla_put_failure:
 770	nlmsg_cancel(skb, nlh);
 771	return -EMSGSIZE;
 772}
 773
 774static size_t nh_nlmsg_size_grp_res(struct nh_group *nhg)
 775{
 776	return nla_total_size(0) +	/* NHA_RES_GROUP */
 777		nla_total_size(2) +	/* NHA_RES_GROUP_BUCKETS */
 778		nla_total_size(4) +	/* NHA_RES_GROUP_IDLE_TIMER */
 779		nla_total_size(4) +	/* NHA_RES_GROUP_UNBALANCED_TIMER */
 780		nla_total_size_64bit(8);/* NHA_RES_GROUP_UNBALANCED_TIME */
 781}
 782
 783static size_t nh_nlmsg_size_grp(struct nexthop *nh)
 784{
 785	struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
 786	size_t sz = sizeof(struct nexthop_grp) * nhg->num_nh;
 787	size_t tot = nla_total_size(sz) +
 788		nla_total_size(2); /* NHA_GROUP_TYPE */
 789
 790	if (nhg->resilient)
 791		tot += nh_nlmsg_size_grp_res(nhg);
 792
 793	return tot;
 794}
 795
 796static size_t nh_nlmsg_size_single(struct nexthop *nh)
 797{
 798	struct nh_info *nhi = rtnl_dereference(nh->nh_info);
 799	size_t sz;
 800
 801	/* covers NHA_BLACKHOLE since NHA_OIF and BLACKHOLE
 802	 * are mutually exclusive
 803	 */
 804	sz = nla_total_size(4);  /* NHA_OIF */
 805
 806	switch (nhi->family) {
 807	case AF_INET:
 808		if (nhi->fib_nh.fib_nh_gw_family)
 809			sz += nla_total_size(4);  /* NHA_GATEWAY */
 810		break;
 811
 812	case AF_INET6:
 813		/* NHA_GATEWAY */
 814		if (nhi->fib6_nh.fib_nh_gw_family)
 815			sz += nla_total_size(sizeof(const struct in6_addr));
 816		break;
 817	}
 818
 819	if (nhi->fib_nhc.nhc_lwtstate) {
 820		sz += lwtunnel_get_encap_size(nhi->fib_nhc.nhc_lwtstate);
 821		sz += nla_total_size(2);  /* NHA_ENCAP_TYPE */
 822	}
 823
 824	return sz;
 825}
 826
 827static size_t nh_nlmsg_size(struct nexthop *nh)
 828{
 829	size_t sz = NLMSG_ALIGN(sizeof(struct nhmsg));
 830
 831	sz += nla_total_size(4); /* NHA_ID */
 832
 833	if (nh->is_group)
 834		sz += nh_nlmsg_size_grp(nh);
 835	else
 836		sz += nh_nlmsg_size_single(nh);
 837
 838	return sz;
 839}
 840
 841static void nexthop_notify(int event, struct nexthop *nh, struct nl_info *info)
 842{
 843	unsigned int nlflags = info->nlh ? info->nlh->nlmsg_flags : 0;
 844	u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0;
 845	struct sk_buff *skb;
 846	int err = -ENOBUFS;
 847
 848	skb = nlmsg_new(nh_nlmsg_size(nh), gfp_any());
 849	if (!skb)
 850		goto errout;
 851
 852	err = nh_fill_node(skb, nh, event, info->portid, seq, nlflags);
 853	if (err < 0) {
 854		/* -EMSGSIZE implies BUG in nh_nlmsg_size() */
 855		WARN_ON(err == -EMSGSIZE);
 856		kfree_skb(skb);
 857		goto errout;
 858	}
 859
 860	rtnl_notify(skb, info->nl_net, info->portid, RTNLGRP_NEXTHOP,
 861		    info->nlh, gfp_any());
 862	return;
 863errout:
 864	if (err < 0)
 865		rtnl_set_sk_err(info->nl_net, RTNLGRP_NEXTHOP, err);
 866}
 867
 868static unsigned long nh_res_bucket_used_time(const struct nh_res_bucket *bucket)
 869{
 870	return (unsigned long)atomic_long_read(&bucket->used_time);
 871}
 872
 873static unsigned long
 874nh_res_bucket_idle_point(const struct nh_res_table *res_table,
 875			 const struct nh_res_bucket *bucket,
 876			 unsigned long now)
 877{
 878	unsigned long time = nh_res_bucket_used_time(bucket);
 879
 880	/* Bucket was not used since it was migrated. The idle time is now. */
 881	if (time == bucket->migrated_time)
 882		return now;
 883
 884	return time + res_table->idle_timer;
 885}
 886
 887static unsigned long
 888nh_res_table_unb_point(const struct nh_res_table *res_table)
 889{
 890	return res_table->unbalanced_since + res_table->unbalanced_timer;
 891}
 892
 893static void nh_res_bucket_set_idle(const struct nh_res_table *res_table,
 894				   struct nh_res_bucket *bucket)
 895{
 896	unsigned long now = jiffies;
 897
 898	atomic_long_set(&bucket->used_time, (long)now);
 899	bucket->migrated_time = now;
 900}
 901
 902static void nh_res_bucket_set_busy(struct nh_res_bucket *bucket)
 903{
 904	atomic_long_set(&bucket->used_time, (long)jiffies);
 905}
 906
 907static clock_t nh_res_bucket_idle_time(const struct nh_res_bucket *bucket)
 908{
 909	unsigned long used_time = nh_res_bucket_used_time(bucket);
 910
 911	return jiffies_delta_to_clock_t(jiffies - used_time);
 912}
 913
 914static int nh_fill_res_bucket(struct sk_buff *skb, struct nexthop *nh,
 915			      struct nh_res_bucket *bucket, u16 bucket_index,
 916			      int event, u32 portid, u32 seq,
 917			      unsigned int nlflags,
 918			      struct netlink_ext_ack *extack)
 919{
 920	struct nh_grp_entry *nhge = nh_res_dereference(bucket->nh_entry);
 921	struct nlmsghdr *nlh;
 922	struct nlattr *nest;
 923	struct nhmsg *nhm;
 924
 925	nlh = nlmsg_put(skb, portid, seq, event, sizeof(*nhm), nlflags);
 926	if (!nlh)
 927		return -EMSGSIZE;
 928
 929	nhm = nlmsg_data(nlh);
 930	nhm->nh_family = AF_UNSPEC;
 931	nhm->nh_flags = bucket->nh_flags;
 932	nhm->nh_protocol = nh->protocol;
 933	nhm->nh_scope = 0;
 934	nhm->resvd = 0;
 935
 936	if (nla_put_u32(skb, NHA_ID, nh->id))
 937		goto nla_put_failure;
 938
 939	nest = nla_nest_start(skb, NHA_RES_BUCKET);
 940	if (!nest)
 941		goto nla_put_failure;
 942
 943	if (nla_put_u16(skb, NHA_RES_BUCKET_INDEX, bucket_index) ||
 944	    nla_put_u32(skb, NHA_RES_BUCKET_NH_ID, nhge->nh->id) ||
 945	    nla_put_u64_64bit(skb, NHA_RES_BUCKET_IDLE_TIME,
 946			      nh_res_bucket_idle_time(bucket),
 947			      NHA_RES_BUCKET_PAD))
 948		goto nla_put_failure_nest;
 949
 950	nla_nest_end(skb, nest);
 951	nlmsg_end(skb, nlh);
 952	return 0;
 953
 954nla_put_failure_nest:
 955	nla_nest_cancel(skb, nest);
 956nla_put_failure:
 957	nlmsg_cancel(skb, nlh);
 958	return -EMSGSIZE;
 959}
 960
 961static void nexthop_bucket_notify(struct nh_res_table *res_table,
 962				  u16 bucket_index)
 963{
 964	struct nh_res_bucket *bucket = &res_table->nh_buckets[bucket_index];
 965	struct nh_grp_entry *nhge = nh_res_dereference(bucket->nh_entry);
 966	struct nexthop *nh = nhge->nh_parent;
 967	struct sk_buff *skb;
 968	int err = -ENOBUFS;
 969
 970	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
 971	if (!skb)
 972		goto errout;
 973
 974	err = nh_fill_res_bucket(skb, nh, bucket, bucket_index,
 975				 RTM_NEWNEXTHOPBUCKET, 0, 0, NLM_F_REPLACE,
 976				 NULL);
 977	if (err < 0) {
 978		kfree_skb(skb);
 979		goto errout;
 980	}
 981
 982	rtnl_notify(skb, nh->net, 0, RTNLGRP_NEXTHOP, NULL, GFP_KERNEL);
 983	return;
 984errout:
 985	if (err < 0)
 986		rtnl_set_sk_err(nh->net, RTNLGRP_NEXTHOP, err);
 987}
 988
 989static bool valid_group_nh(struct nexthop *nh, unsigned int npaths,
 990			   bool *is_fdb, struct netlink_ext_ack *extack)
 991{
 992	if (nh->is_group) {
 993		struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
 994
 995		/* Nesting groups within groups is not supported. */
 996		if (nhg->hash_threshold) {
 997			NL_SET_ERR_MSG(extack,
 998				       "Hash-threshold group can not be a nexthop within a group");
 999			return false;
1000		}
1001		if (nhg->resilient) {
1002			NL_SET_ERR_MSG(extack,
1003				       "Resilient group can not be a nexthop within a group");
1004			return false;
1005		}
1006		*is_fdb = nhg->fdb_nh;
1007	} else {
1008		struct nh_info *nhi = rtnl_dereference(nh->nh_info);
1009
1010		if (nhi->reject_nh && npaths > 1) {
1011			NL_SET_ERR_MSG(extack,
1012				       "Blackhole nexthop can not be used in a group with more than 1 path");
1013			return false;
1014		}
1015		*is_fdb = nhi->fdb_nh;
1016	}
1017
1018	return true;
1019}
1020
1021static int nh_check_attr_fdb_group(struct nexthop *nh, u8 *nh_family,
1022				   struct netlink_ext_ack *extack)
1023{
1024	struct nh_info *nhi;
1025
1026	nhi = rtnl_dereference(nh->nh_info);
1027
1028	if (!nhi->fdb_nh) {
1029		NL_SET_ERR_MSG(extack, "FDB nexthop group can only have fdb nexthops");
1030		return -EINVAL;
1031	}
1032
1033	if (*nh_family == AF_UNSPEC) {
1034		*nh_family = nhi->family;
1035	} else if (*nh_family != nhi->family) {
1036		NL_SET_ERR_MSG(extack, "FDB nexthop group cannot have mixed family nexthops");
1037		return -EINVAL;
1038	}
1039
1040	return 0;
1041}
1042
1043static int nh_check_attr_group(struct net *net,
1044			       struct nlattr *tb[], size_t tb_size,
1045			       u16 nh_grp_type, struct netlink_ext_ack *extack)
1046{
1047	unsigned int len = nla_len(tb[NHA_GROUP]);
1048	u8 nh_family = AF_UNSPEC;
1049	struct nexthop_grp *nhg;
1050	unsigned int i, j;
1051	u8 nhg_fdb = 0;
1052
1053	if (!len || len & (sizeof(struct nexthop_grp) - 1)) {
1054		NL_SET_ERR_MSG(extack,
1055			       "Invalid length for nexthop group attribute");
1056		return -EINVAL;
1057	}
1058
1059	/* convert len to number of nexthop ids */
1060	len /= sizeof(*nhg);
1061
1062	nhg = nla_data(tb[NHA_GROUP]);
1063	for (i = 0; i < len; ++i) {
1064		if (nhg[i].resvd1 || nhg[i].resvd2) {
1065			NL_SET_ERR_MSG(extack, "Reserved fields in nexthop_grp must be 0");
1066			return -EINVAL;
1067		}
1068		if (nhg[i].weight > 254) {
1069			NL_SET_ERR_MSG(extack, "Invalid value for weight");
1070			return -EINVAL;
1071		}
1072		for (j = i + 1; j < len; ++j) {
1073			if (nhg[i].id == nhg[j].id) {
1074				NL_SET_ERR_MSG(extack, "Nexthop id can not be used twice in a group");
1075				return -EINVAL;
1076			}
1077		}
1078	}
1079
1080	if (tb[NHA_FDB])
1081		nhg_fdb = 1;
1082	nhg = nla_data(tb[NHA_GROUP]);
1083	for (i = 0; i < len; ++i) {
1084		struct nexthop *nh;
1085		bool is_fdb_nh;
1086
1087		nh = nexthop_find_by_id(net, nhg[i].id);
1088		if (!nh) {
1089			NL_SET_ERR_MSG(extack, "Invalid nexthop id");
1090			return -EINVAL;
1091		}
1092		if (!valid_group_nh(nh, len, &is_fdb_nh, extack))
1093			return -EINVAL;
1094
1095		if (nhg_fdb && nh_check_attr_fdb_group(nh, &nh_family, extack))
1096			return -EINVAL;
1097
1098		if (!nhg_fdb && is_fdb_nh) {
1099			NL_SET_ERR_MSG(extack, "Non FDB nexthop group cannot have fdb nexthops");
1100			return -EINVAL;
1101		}
1102	}
1103	for (i = NHA_GROUP_TYPE + 1; i < tb_size; ++i) {
1104		if (!tb[i])
1105			continue;
1106		switch (i) {
1107		case NHA_FDB:
1108			continue;
1109		case NHA_RES_GROUP:
1110			if (nh_grp_type == NEXTHOP_GRP_TYPE_RES)
1111				continue;
1112			break;
1113		}
1114		NL_SET_ERR_MSG(extack,
1115			       "No other attributes can be set in nexthop groups");
1116		return -EINVAL;
1117	}
1118
1119	return 0;
1120}
1121
1122static bool ipv6_good_nh(const struct fib6_nh *nh)
1123{
1124	int state = NUD_REACHABLE;
1125	struct neighbour *n;
1126
1127	rcu_read_lock();
1128
1129	n = __ipv6_neigh_lookup_noref_stub(nh->fib_nh_dev, &nh->fib_nh_gw6);
1130	if (n)
1131		state = READ_ONCE(n->nud_state);
1132
1133	rcu_read_unlock();
1134
1135	return !!(state & NUD_VALID);
1136}
1137
1138static bool ipv4_good_nh(const struct fib_nh *nh)
1139{
1140	int state = NUD_REACHABLE;
1141	struct neighbour *n;
1142
1143	rcu_read_lock();
1144
1145	n = __ipv4_neigh_lookup_noref(nh->fib_nh_dev,
1146				      (__force u32)nh->fib_nh_gw4);
1147	if (n)
1148		state = READ_ONCE(n->nud_state);
1149
1150	rcu_read_unlock();
1151
1152	return !!(state & NUD_VALID);
1153}
1154
1155static bool nexthop_is_good_nh(const struct nexthop *nh)
1156{
1157	struct nh_info *nhi = rcu_dereference(nh->nh_info);
1158
1159	switch (nhi->family) {
1160	case AF_INET:
1161		return ipv4_good_nh(&nhi->fib_nh);
1162	case AF_INET6:
1163		return ipv6_good_nh(&nhi->fib6_nh);
1164	}
1165
1166	return false;
1167}
1168
1169static struct nexthop *nexthop_select_path_fdb(struct nh_group *nhg, int hash)
1170{
1171	int i;
1172
1173	for (i = 0; i < nhg->num_nh; i++) {
1174		struct nh_grp_entry *nhge = &nhg->nh_entries[i];
1175
1176		if (hash > atomic_read(&nhge->hthr.upper_bound))
1177			continue;
1178
1179		return nhge->nh;
1180	}
1181
1182	WARN_ON_ONCE(1);
1183	return NULL;
1184}
1185
1186static struct nexthop *nexthop_select_path_hthr(struct nh_group *nhg, int hash)
1187{
1188	struct nexthop *rc = NULL;
1189	int i;
1190
1191	if (nhg->fdb_nh)
1192		return nexthop_select_path_fdb(nhg, hash);
1193
1194	for (i = 0; i < nhg->num_nh; ++i) {
1195		struct nh_grp_entry *nhge = &nhg->nh_entries[i];
1196
1197		/* nexthops always check if it is good and does
1198		 * not rely on a sysctl for this behavior
1199		 */
1200		if (!nexthop_is_good_nh(nhge->nh))
1201			continue;
1202
1203		if (!rc)
1204			rc = nhge->nh;
1205
1206		if (hash > atomic_read(&nhge->hthr.upper_bound))
1207			continue;
1208
1209		return nhge->nh;
1210	}
1211
1212	return rc ? : nhg->nh_entries[0].nh;
1213}
1214
1215static struct nexthop *nexthop_select_path_res(struct nh_group *nhg, int hash)
1216{
1217	struct nh_res_table *res_table = rcu_dereference(nhg->res_table);
1218	u16 bucket_index = hash % res_table->num_nh_buckets;
1219	struct nh_res_bucket *bucket;
1220	struct nh_grp_entry *nhge;
1221
1222	/* nexthop_select_path() is expected to return a non-NULL value, so
1223	 * skip protocol validation and just hand out whatever there is.
1224	 */
1225	bucket = &res_table->nh_buckets[bucket_index];
1226	nh_res_bucket_set_busy(bucket);
1227	nhge = rcu_dereference(bucket->nh_entry);
1228	return nhge->nh;
1229}
1230
1231struct nexthop *nexthop_select_path(struct nexthop *nh, int hash)
1232{
1233	struct nh_group *nhg;
1234
1235	if (!nh->is_group)
1236		return nh;
1237
1238	nhg = rcu_dereference(nh->nh_grp);
1239	if (nhg->hash_threshold)
1240		return nexthop_select_path_hthr(nhg, hash);
1241	else if (nhg->resilient)
1242		return nexthop_select_path_res(nhg, hash);
1243
1244	/* Unreachable. */
1245	return NULL;
1246}
1247EXPORT_SYMBOL_GPL(nexthop_select_path);
1248
1249int nexthop_for_each_fib6_nh(struct nexthop *nh,
1250			     int (*cb)(struct fib6_nh *nh, void *arg),
1251			     void *arg)
1252{
1253	struct nh_info *nhi;
1254	int err;
1255
1256	if (nh->is_group) {
1257		struct nh_group *nhg;
1258		int i;
1259
1260		nhg = rcu_dereference_rtnl(nh->nh_grp);
1261		for (i = 0; i < nhg->num_nh; i++) {
1262			struct nh_grp_entry *nhge = &nhg->nh_entries[i];
1263
1264			nhi = rcu_dereference_rtnl(nhge->nh->nh_info);
1265			err = cb(&nhi->fib6_nh, arg);
1266			if (err)
1267				return err;
1268		}
1269	} else {
1270		nhi = rcu_dereference_rtnl(nh->nh_info);
1271		err = cb(&nhi->fib6_nh, arg);
1272		if (err)
1273			return err;
1274	}
1275
1276	return 0;
1277}
1278EXPORT_SYMBOL_GPL(nexthop_for_each_fib6_nh);
1279
1280static int check_src_addr(const struct in6_addr *saddr,
1281			  struct netlink_ext_ack *extack)
1282{
1283	if (!ipv6_addr_any(saddr)) {
1284		NL_SET_ERR_MSG(extack, "IPv6 routes using source address can not use nexthop objects");
1285		return -EINVAL;
1286	}
1287	return 0;
1288}
1289
1290int fib6_check_nexthop(struct nexthop *nh, struct fib6_config *cfg,
1291		       struct netlink_ext_ack *extack)
1292{
1293	struct nh_info *nhi;
1294	bool is_fdb_nh;
1295
1296	/* fib6_src is unique to a fib6_info and limits the ability to cache
1297	 * routes in fib6_nh within a nexthop that is potentially shared
1298	 * across multiple fib entries. If the config wants to use source
1299	 * routing it can not use nexthop objects. mlxsw also does not allow
1300	 * fib6_src on routes.
1301	 */
1302	if (cfg && check_src_addr(&cfg->fc_src, extack) < 0)
1303		return -EINVAL;
1304
1305	if (nh->is_group) {
1306		struct nh_group *nhg;
1307
1308		nhg = rtnl_dereference(nh->nh_grp);
1309		if (nhg->has_v4)
1310			goto no_v4_nh;
1311		is_fdb_nh = nhg->fdb_nh;
1312	} else {
1313		nhi = rtnl_dereference(nh->nh_info);
1314		if (nhi->family == AF_INET)
1315			goto no_v4_nh;
1316		is_fdb_nh = nhi->fdb_nh;
1317	}
1318
1319	if (is_fdb_nh) {
1320		NL_SET_ERR_MSG(extack, "Route cannot point to a fdb nexthop");
1321		return -EINVAL;
1322	}
1323
1324	return 0;
1325no_v4_nh:
1326	NL_SET_ERR_MSG(extack, "IPv6 routes can not use an IPv4 nexthop");
1327	return -EINVAL;
1328}
1329EXPORT_SYMBOL_GPL(fib6_check_nexthop);
1330
1331/* if existing nexthop has ipv6 routes linked to it, need
1332 * to verify this new spec works with ipv6
1333 */
1334static int fib6_check_nh_list(struct nexthop *old, struct nexthop *new,
1335			      struct netlink_ext_ack *extack)
1336{
1337	struct fib6_info *f6i;
1338
1339	if (list_empty(&old->f6i_list))
1340		return 0;
1341
1342	list_for_each_entry(f6i, &old->f6i_list, nh_list) {
1343		if (check_src_addr(&f6i->fib6_src.addr, extack) < 0)
1344			return -EINVAL;
1345	}
1346
1347	return fib6_check_nexthop(new, NULL, extack);
1348}
1349
1350static int nexthop_check_scope(struct nh_info *nhi, u8 scope,
1351			       struct netlink_ext_ack *extack)
1352{
1353	if (scope == RT_SCOPE_HOST && nhi->fib_nhc.nhc_gw_family) {
1354		NL_SET_ERR_MSG(extack,
1355			       "Route with host scope can not have a gateway");
1356		return -EINVAL;
1357	}
1358
1359	if (nhi->fib_nhc.nhc_flags & RTNH_F_ONLINK && scope >= RT_SCOPE_LINK) {
1360		NL_SET_ERR_MSG(extack, "Scope mismatch with nexthop");
1361		return -EINVAL;
1362	}
1363
1364	return 0;
1365}
1366
1367/* Invoked by fib add code to verify nexthop by id is ok with
1368 * config for prefix; parts of fib_check_nh not done when nexthop
1369 * object is used.
1370 */
1371int fib_check_nexthop(struct nexthop *nh, u8 scope,
1372		      struct netlink_ext_ack *extack)
1373{
1374	struct nh_info *nhi;
1375	int err = 0;
1376
1377	if (nh->is_group) {
1378		struct nh_group *nhg;
1379
1380		nhg = rtnl_dereference(nh->nh_grp);
1381		if (nhg->fdb_nh) {
1382			NL_SET_ERR_MSG(extack, "Route cannot point to a fdb nexthop");
1383			err = -EINVAL;
1384			goto out;
1385		}
1386
1387		if (scope == RT_SCOPE_HOST) {
1388			NL_SET_ERR_MSG(extack, "Route with host scope can not have multiple nexthops");
1389			err = -EINVAL;
1390			goto out;
1391		}
1392
1393		/* all nexthops in a group have the same scope */
1394		nhi = rtnl_dereference(nhg->nh_entries[0].nh->nh_info);
1395		err = nexthop_check_scope(nhi, scope, extack);
1396	} else {
1397		nhi = rtnl_dereference(nh->nh_info);
1398		if (nhi->fdb_nh) {
1399			NL_SET_ERR_MSG(extack, "Route cannot point to a fdb nexthop");
1400			err = -EINVAL;
1401			goto out;
1402		}
1403		err = nexthop_check_scope(nhi, scope, extack);
1404	}
1405
1406out:
1407	return err;
1408}
1409
1410static int fib_check_nh_list(struct nexthop *old, struct nexthop *new,
1411			     struct netlink_ext_ack *extack)
1412{
1413	struct fib_info *fi;
1414
1415	list_for_each_entry(fi, &old->fi_list, nh_list) {
1416		int err;
1417
1418		err = fib_check_nexthop(new, fi->fib_scope, extack);
1419		if (err)
1420			return err;
1421	}
1422	return 0;
1423}
1424
1425static bool nh_res_nhge_is_balanced(const struct nh_grp_entry *nhge)
1426{
1427	return nhge->res.count_buckets == nhge->res.wants_buckets;
1428}
1429
1430static bool nh_res_nhge_is_ow(const struct nh_grp_entry *nhge)
1431{
1432	return nhge->res.count_buckets > nhge->res.wants_buckets;
1433}
1434
1435static bool nh_res_nhge_is_uw(const struct nh_grp_entry *nhge)
1436{
1437	return nhge->res.count_buckets < nhge->res.wants_buckets;
1438}
1439
1440static bool nh_res_table_is_balanced(const struct nh_res_table *res_table)
1441{
1442	return list_empty(&res_table->uw_nh_entries);
1443}
1444
1445static void nh_res_bucket_unset_nh(struct nh_res_bucket *bucket)
1446{
1447	struct nh_grp_entry *nhge;
1448
1449	if (bucket->occupied) {
1450		nhge = nh_res_dereference(bucket->nh_entry);
1451		nhge->res.count_buckets--;
1452		bucket->occupied = false;
1453	}
1454}
1455
1456static void nh_res_bucket_set_nh(struct nh_res_bucket *bucket,
1457				 struct nh_grp_entry *nhge)
1458{
1459	nh_res_bucket_unset_nh(bucket);
1460
1461	bucket->occupied = true;
1462	rcu_assign_pointer(bucket->nh_entry, nhge);
1463	nhge->res.count_buckets++;
1464}
1465
1466static bool nh_res_bucket_should_migrate(struct nh_res_table *res_table,
1467					 struct nh_res_bucket *bucket,
1468					 unsigned long *deadline, bool *force)
1469{
1470	unsigned long now = jiffies;
1471	struct nh_grp_entry *nhge;
1472	unsigned long idle_point;
1473
1474	if (!bucket->occupied) {
1475		/* The bucket is not occupied, its NHGE pointer is either
1476		 * NULL or obsolete. We _have to_ migrate: set force.
1477		 */
1478		*force = true;
1479		return true;
1480	}
1481
1482	nhge = nh_res_dereference(bucket->nh_entry);
1483
1484	/* If the bucket is populated by an underweight or balanced
1485	 * nexthop, do not migrate.
1486	 */
1487	if (!nh_res_nhge_is_ow(nhge))
1488		return false;
1489
1490	/* At this point we know that the bucket is populated with an
1491	 * overweight nexthop. It needs to be migrated to a new nexthop if
1492	 * the idle timer of unbalanced timer expired.
1493	 */
1494
1495	idle_point = nh_res_bucket_idle_point(res_table, bucket, now);
1496	if (time_after_eq(now, idle_point)) {
1497		/* The bucket is idle. We _can_ migrate: unset force. */
1498		*force = false;
1499		return true;
1500	}
1501
1502	/* Unbalanced timer of 0 means "never force". */
1503	if (res_table->unbalanced_timer) {
1504		unsigned long unb_point;
1505
1506		unb_point = nh_res_table_unb_point(res_table);
1507		if (time_after(now, unb_point)) {
1508			/* The bucket is not idle, but the unbalanced timer
1509			 * expired. We _can_ migrate, but set force anyway,
1510			 * so that drivers know to ignore activity reports
1511			 * from the HW.
1512			 */
1513			*force = true;
1514			return true;
1515		}
1516
1517		nh_res_time_set_deadline(unb_point, deadline);
1518	}
1519
1520	nh_res_time_set_deadline(idle_point, deadline);
1521	return false;
1522}
1523
1524static bool nh_res_bucket_migrate(struct nh_res_table *res_table,
1525				  u16 bucket_index, bool notify,
1526				  bool notify_nl, bool force)
1527{
1528	struct nh_res_bucket *bucket = &res_table->nh_buckets[bucket_index];
1529	struct nh_grp_entry *new_nhge;
1530	struct netlink_ext_ack extack;
1531	int err;
1532
1533	new_nhge = list_first_entry_or_null(&res_table->uw_nh_entries,
1534					    struct nh_grp_entry,
1535					    res.uw_nh_entry);
1536	if (WARN_ON_ONCE(!new_nhge))
1537		/* If this function is called, "bucket" is either not
1538		 * occupied, or it belongs to a next hop that is
1539		 * overweight. In either case, there ought to be a
1540		 * corresponding underweight next hop.
1541		 */
1542		return false;
1543
1544	if (notify) {
1545		struct nh_grp_entry *old_nhge;
1546
1547		old_nhge = nh_res_dereference(bucket->nh_entry);
1548		err = call_nexthop_res_bucket_notifiers(res_table->net,
1549							res_table->nhg_id,
1550							bucket_index, force,
1551							old_nhge->nh,
1552							new_nhge->nh, &extack);
1553		if (err) {
1554			pr_err_ratelimited("%s\n", extack._msg);
1555			if (!force)
1556				return false;
1557			/* It is not possible to veto a forced replacement, so
1558			 * just clear the hardware flags from the nexthop
1559			 * bucket to indicate to user space that this bucket is
1560			 * not correctly populated in hardware.
1561			 */
1562			bucket->nh_flags &= ~(RTNH_F_OFFLOAD | RTNH_F_TRAP);
1563		}
1564	}
1565
1566	nh_res_bucket_set_nh(bucket, new_nhge);
1567	nh_res_bucket_set_idle(res_table, bucket);
1568
1569	if (notify_nl)
1570		nexthop_bucket_notify(res_table, bucket_index);
1571
1572	if (nh_res_nhge_is_balanced(new_nhge))
1573		list_del(&new_nhge->res.uw_nh_entry);
1574	return true;
1575}
1576
1577#define NH_RES_UPKEEP_DW_MINIMUM_INTERVAL (HZ / 2)
1578
1579static void nh_res_table_upkeep(struct nh_res_table *res_table,
1580				bool notify, bool notify_nl)
1581{
1582	unsigned long now = jiffies;
1583	unsigned long deadline;
1584	u16 i;
1585
1586	/* Deadline is the next time that upkeep should be run. It is the
1587	 * earliest time at which one of the buckets might be migrated.
1588	 * Start at the most pessimistic estimate: either unbalanced_timer
1589	 * from now, or if there is none, idle_timer from now. For each
1590	 * encountered time point, call nh_res_time_set_deadline() to
1591	 * refine the estimate.
1592	 */
1593	if (res_table->unbalanced_timer)
1594		deadline = now + res_table->unbalanced_timer;
1595	else
1596		deadline = now + res_table->idle_timer;
1597
1598	for (i = 0; i < res_table->num_nh_buckets; i++) {
1599		struct nh_res_bucket *bucket = &res_table->nh_buckets[i];
1600		bool force;
1601
1602		if (nh_res_bucket_should_migrate(res_table, bucket,
1603						 &deadline, &force)) {
1604			if (!nh_res_bucket_migrate(res_table, i, notify,
1605						   notify_nl, force)) {
1606				unsigned long idle_point;
1607
1608				/* A driver can override the migration
1609				 * decision if the HW reports that the
1610				 * bucket is actually not idle. Therefore
1611				 * remark the bucket as busy again and
1612				 * update the deadline.
1613				 */
1614				nh_res_bucket_set_busy(bucket);
1615				idle_point = nh_res_bucket_idle_point(res_table,
1616								      bucket,
1617								      now);
1618				nh_res_time_set_deadline(idle_point, &deadline);
1619			}
1620		}
1621	}
1622
1623	/* If the group is still unbalanced, schedule the next upkeep to
1624	 * either the deadline computed above, or the minimum deadline,
1625	 * whichever comes later.
1626	 */
1627	if (!nh_res_table_is_balanced(res_table)) {
1628		unsigned long now = jiffies;
1629		unsigned long min_deadline;
1630
1631		min_deadline = now + NH_RES_UPKEEP_DW_MINIMUM_INTERVAL;
1632		if (time_before(deadline, min_deadline))
1633			deadline = min_deadline;
1634
1635		queue_delayed_work(system_power_efficient_wq,
1636				   &res_table->upkeep_dw, deadline - now);
1637	}
1638}
1639
1640static void nh_res_table_upkeep_dw(struct work_struct *work)
1641{
1642	struct delayed_work *dw = to_delayed_work(work);
1643	struct nh_res_table *res_table;
1644
1645	res_table = container_of(dw, struct nh_res_table, upkeep_dw);
1646	nh_res_table_upkeep(res_table, true, true);
1647}
1648
1649static void nh_res_table_cancel_upkeep(struct nh_res_table *res_table)
1650{
1651	cancel_delayed_work_sync(&res_table->upkeep_dw);
1652}
1653
1654static void nh_res_group_rebalance(struct nh_group *nhg,
1655				   struct nh_res_table *res_table)
1656{
1657	int prev_upper_bound = 0;
1658	int total = 0;
1659	int w = 0;
1660	int i;
1661
1662	INIT_LIST_HEAD(&res_table->uw_nh_entries);
1663
1664	for (i = 0; i < nhg->num_nh; ++i)
1665		total += nhg->nh_entries[i].weight;
1666
1667	for (i = 0; i < nhg->num_nh; ++i) {
1668		struct nh_grp_entry *nhge = &nhg->nh_entries[i];
1669		int upper_bound;
1670
1671		w += nhge->weight;
1672		upper_bound = DIV_ROUND_CLOSEST(res_table->num_nh_buckets * w,
1673						total);
1674		nhge->res.wants_buckets = upper_bound - prev_upper_bound;
1675		prev_upper_bound = upper_bound;
1676
1677		if (nh_res_nhge_is_uw(nhge)) {
1678			if (list_empty(&res_table->uw_nh_entries))
1679				res_table->unbalanced_since = jiffies;
1680			list_add(&nhge->res.uw_nh_entry,
1681				 &res_table->uw_nh_entries);
1682		}
1683	}
1684}
1685
1686/* Migrate buckets in res_table so that they reference NHGE's from NHG with
1687 * the right NH ID. Set those buckets that do not have a corresponding NHGE
1688 * entry in NHG as not occupied.
1689 */
1690static void nh_res_table_migrate_buckets(struct nh_res_table *res_table,
1691					 struct nh_group *nhg)
1692{
1693	u16 i;
1694
1695	for (i = 0; i < res_table->num_nh_buckets; i++) {
1696		struct nh_res_bucket *bucket = &res_table->nh_buckets[i];
1697		u32 id = rtnl_dereference(bucket->nh_entry)->nh->id;
1698		bool found = false;
1699		int j;
1700
1701		for (j = 0; j < nhg->num_nh; j++) {
1702			struct nh_grp_entry *nhge = &nhg->nh_entries[j];
1703
1704			if (nhge->nh->id == id) {
1705				nh_res_bucket_set_nh(bucket, nhge);
1706				found = true;
1707				break;
1708			}
1709		}
1710
1711		if (!found)
1712			nh_res_bucket_unset_nh(bucket);
1713	}
1714}
1715
1716static void replace_nexthop_grp_res(struct nh_group *oldg,
1717				    struct nh_group *newg)
1718{
1719	/* For NH group replacement, the new NHG might only have a stub
1720	 * hash table with 0 buckets, because the number of buckets was not
1721	 * specified. For NH removal, oldg and newg both reference the same
1722	 * res_table. So in any case, in the following, we want to work
1723	 * with oldg->res_table.
1724	 */
1725	struct nh_res_table *old_res_table = rtnl_dereference(oldg->res_table);
1726	unsigned long prev_unbalanced_since = old_res_table->unbalanced_since;
1727	bool prev_has_uw = !list_empty(&old_res_table->uw_nh_entries);
1728
1729	nh_res_table_cancel_upkeep(old_res_table);
1730	nh_res_table_migrate_buckets(old_res_table, newg);
1731	nh_res_group_rebalance(newg, old_res_table);
1732	if (prev_has_uw && !list_empty(&old_res_table->uw_nh_entries))
1733		old_res_table->unbalanced_since = prev_unbalanced_since;
1734	nh_res_table_upkeep(old_res_table, true, false);
1735}
1736
1737static void nh_hthr_group_rebalance(struct nh_group *nhg)
1738{
1739	int total = 0;
1740	int w = 0;
1741	int i;
1742
1743	for (i = 0; i < nhg->num_nh; ++i)
1744		total += nhg->nh_entries[i].weight;
1745
1746	for (i = 0; i < nhg->num_nh; ++i) {
1747		struct nh_grp_entry *nhge = &nhg->nh_entries[i];
1748		int upper_bound;
1749
1750		w += nhge->weight;
1751		upper_bound = DIV_ROUND_CLOSEST_ULL((u64)w << 31, total) - 1;
1752		atomic_set(&nhge->hthr.upper_bound, upper_bound);
1753	}
1754}
1755
1756static void remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge,
1757				struct nl_info *nlinfo)
1758{
1759	struct nh_grp_entry *nhges, *new_nhges;
1760	struct nexthop *nhp = nhge->nh_parent;
1761	struct netlink_ext_ack extack;
1762	struct nexthop *nh = nhge->nh;
1763	struct nh_group *nhg, *newg;
1764	int i, j, err;
1765
1766	WARN_ON(!nh);
1767
1768	nhg = rtnl_dereference(nhp->nh_grp);
1769	newg = nhg->spare;
1770
1771	/* last entry, keep it visible and remove the parent */
1772	if (nhg->num_nh == 1) {
1773		remove_nexthop(net, nhp, nlinfo);
1774		return;
1775	}
1776
1777	newg->has_v4 = false;
1778	newg->is_multipath = nhg->is_multipath;
1779	newg->hash_threshold = nhg->hash_threshold;
1780	newg->resilient = nhg->resilient;
1781	newg->fdb_nh = nhg->fdb_nh;
1782	newg->num_nh = nhg->num_nh;
1783
1784	/* copy old entries to new except the one getting removed */
1785	nhges = nhg->nh_entries;
1786	new_nhges = newg->nh_entries;
1787	for (i = 0, j = 0; i < nhg->num_nh; ++i) {
1788		struct nh_info *nhi;
1789
1790		/* current nexthop getting removed */
1791		if (nhg->nh_entries[i].nh == nh) {
1792			newg->num_nh--;
1793			continue;
1794		}
1795
1796		nhi = rtnl_dereference(nhges[i].nh->nh_info);
1797		if (nhi->family == AF_INET)
1798			newg->has_v4 = true;
1799
1800		list_del(&nhges[i].nh_list);
1801		new_nhges[j].nh_parent = nhges[i].nh_parent;
1802		new_nhges[j].nh = nhges[i].nh;
1803		new_nhges[j].weight = nhges[i].weight;
1804		list_add(&new_nhges[j].nh_list, &new_nhges[j].nh->grp_list);
1805		j++;
1806	}
1807
1808	if (newg->hash_threshold)
1809		nh_hthr_group_rebalance(newg);
1810	else if (newg->resilient)
1811		replace_nexthop_grp_res(nhg, newg);
1812
1813	rcu_assign_pointer(nhp->nh_grp, newg);
1814
1815	list_del(&nhge->nh_list);
1816	nexthop_put(nhge->nh);
1817
1818	/* Removal of a NH from a resilient group is notified through
1819	 * bucket notifications.
1820	 */
1821	if (newg->hash_threshold) {
1822		err = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, nhp,
1823					     &extack);
1824		if (err)
1825			pr_err("%s\n", extack._msg);
1826	}
1827
1828	if (nlinfo)
1829		nexthop_notify(RTM_NEWNEXTHOP, nhp, nlinfo);
1830}
1831
1832static void remove_nexthop_from_groups(struct net *net, struct nexthop *nh,
1833				       struct nl_info *nlinfo)
1834{
1835	struct nh_grp_entry *nhge, *tmp;
1836
1837	list_for_each_entry_safe(nhge, tmp, &nh->grp_list, nh_list)
1838		remove_nh_grp_entry(net, nhge, nlinfo);
1839
1840	/* make sure all see the newly published array before releasing rtnl */
1841	synchronize_net();
1842}
1843
1844static void remove_nexthop_group(struct nexthop *nh, struct nl_info *nlinfo)
1845{
1846	struct nh_group *nhg = rcu_dereference_rtnl(nh->nh_grp);
1847	struct nh_res_table *res_table;
1848	int i, num_nh = nhg->num_nh;
1849
1850	for (i = 0; i < num_nh; ++i) {
1851		struct nh_grp_entry *nhge = &nhg->nh_entries[i];
1852
1853		if (WARN_ON(!nhge->nh))
1854			continue;
1855
1856		list_del_init(&nhge->nh_list);
1857	}
1858
1859	if (nhg->resilient) {
1860		res_table = rtnl_dereference(nhg->res_table);
1861		nh_res_table_cancel_upkeep(res_table);
1862	}
1863}
1864
1865/* not called for nexthop replace */
1866static void __remove_nexthop_fib(struct net *net, struct nexthop *nh)
1867{
1868	struct fib6_info *f6i, *tmp;
1869	bool do_flush = false;
1870	struct fib_info *fi;
1871
1872	list_for_each_entry(fi, &nh->fi_list, nh_list) {
1873		fi->fib_flags |= RTNH_F_DEAD;
1874		do_flush = true;
1875	}
1876	if (do_flush)
1877		fib_flush(net);
1878
1879	/* ip6_del_rt removes the entry from this list hence the _safe */
1880	list_for_each_entry_safe(f6i, tmp, &nh->f6i_list, nh_list) {
1881		/* __ip6_del_rt does a release, so do a hold here */
1882		fib6_info_hold(f6i);
1883		ipv6_stub->ip6_del_rt(net, f6i,
1884				      !READ_ONCE(net->ipv4.sysctl_nexthop_compat_mode));
1885	}
1886}
1887
1888static void __remove_nexthop(struct net *net, struct nexthop *nh,
1889			     struct nl_info *nlinfo)
1890{
1891	__remove_nexthop_fib(net, nh);
1892
1893	if (nh->is_group) {
1894		remove_nexthop_group(nh, nlinfo);
1895	} else {
1896		struct nh_info *nhi;
1897
1898		nhi = rtnl_dereference(nh->nh_info);
1899		if (nhi->fib_nhc.nhc_dev)
1900			hlist_del(&nhi->dev_hash);
1901
1902		remove_nexthop_from_groups(net, nh, nlinfo);
1903	}
1904}
1905
1906static void remove_nexthop(struct net *net, struct nexthop *nh,
1907			   struct nl_info *nlinfo)
1908{
1909	call_nexthop_notifiers(net, NEXTHOP_EVENT_DEL, nh, NULL);
1910
1911	/* remove from the tree */
1912	rb_erase(&nh->rb_node, &net->nexthop.rb_root);
1913
1914	if (nlinfo)
1915		nexthop_notify(RTM_DELNEXTHOP, nh, nlinfo);
1916
1917	__remove_nexthop(net, nh, nlinfo);
1918	nh_base_seq_inc(net);
1919
1920	nexthop_put(nh);
1921}
1922
1923/* if any FIB entries reference this nexthop, any dst entries
1924 * need to be regenerated
1925 */
1926static void nh_rt_cache_flush(struct net *net, struct nexthop *nh,
1927			      struct nexthop *replaced_nh)
1928{
1929	struct fib6_info *f6i;
1930	struct nh_group *nhg;
1931	int i;
1932
1933	if (!list_empty(&nh->fi_list))
1934		rt_cache_flush(net);
1935
1936	list_for_each_entry(f6i, &nh->f6i_list, nh_list)
1937		ipv6_stub->fib6_update_sernum(net, f6i);
1938
1939	/* if an IPv6 group was replaced, we have to release all old
1940	 * dsts to make sure all refcounts are released
1941	 */
1942	if (!replaced_nh->is_group)
1943		return;
1944
1945	nhg = rtnl_dereference(replaced_nh->nh_grp);
1946	for (i = 0; i < nhg->num_nh; i++) {
1947		struct nh_grp_entry *nhge = &nhg->nh_entries[i];
1948		struct nh_info *nhi = rtnl_dereference(nhge->nh->nh_info);
1949
1950		if (nhi->family == AF_INET6)
1951			ipv6_stub->fib6_nh_release_dsts(&nhi->fib6_nh);
1952	}
1953}
1954
1955static int replace_nexthop_grp(struct net *net, struct nexthop *old,
1956			       struct nexthop *new, const struct nh_config *cfg,
1957			       struct netlink_ext_ack *extack)
1958{
1959	struct nh_res_table *tmp_table = NULL;
1960	struct nh_res_table *new_res_table;
1961	struct nh_res_table *old_res_table;
1962	struct nh_group *oldg, *newg;
1963	int i, err;
1964
1965	if (!new->is_group) {
1966		NL_SET_ERR_MSG(extack, "Can not replace a nexthop group with a nexthop.");
1967		return -EINVAL;
1968	}
1969
1970	oldg = rtnl_dereference(old->nh_grp);
1971	newg = rtnl_dereference(new->nh_grp);
1972
1973	if (newg->hash_threshold != oldg->hash_threshold) {
1974		NL_SET_ERR_MSG(extack, "Can not replace a nexthop group with one of a different type.");
1975		return -EINVAL;
1976	}
1977
1978	if (newg->hash_threshold) {
1979		err = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, new,
1980					     extack);
1981		if (err)
1982			return err;
1983	} else if (newg->resilient) {
1984		new_res_table = rtnl_dereference(newg->res_table);
1985		old_res_table = rtnl_dereference(oldg->res_table);
1986
1987		/* Accept if num_nh_buckets was not given, but if it was
1988		 * given, demand that the value be correct.
1989		 */
1990		if (cfg->nh_grp_res_has_num_buckets &&
1991		    cfg->nh_grp_res_num_buckets !=
1992		    old_res_table->num_nh_buckets) {
1993			NL_SET_ERR_MSG(extack, "Can not change number of buckets of a resilient nexthop group.");
1994			return -EINVAL;
1995		}
1996
1997		/* Emit a pre-replace notification so that listeners could veto
1998		 * a potentially unsupported configuration. Otherwise,
1999		 * individual bucket replacement notifications would need to be
2000		 * vetoed, which is something that should only happen if the
2001		 * bucket is currently active.
2002		 */
2003		err = call_nexthop_res_table_notifiers(net, new, extack);
2004		if (err)
2005			return err;
2006
2007		if (cfg->nh_grp_res_has_idle_timer)
2008			old_res_table->idle_timer = cfg->nh_grp_res_idle_timer;
2009		if (cfg->nh_grp_res_has_unbalanced_timer)
2010			old_res_table->unbalanced_timer =
2011				cfg->nh_grp_res_unbalanced_timer;
2012
2013		replace_nexthop_grp_res(oldg, newg);
2014
2015		tmp_table = new_res_table;
2016		rcu_assign_pointer(newg->res_table, old_res_table);
2017		rcu_assign_pointer(newg->spare->res_table, old_res_table);
2018	}
2019
2020	/* update parents - used by nexthop code for cleanup */
2021	for (i = 0; i < newg->num_nh; i++)
2022		newg->nh_entries[i].nh_parent = old;
2023
2024	rcu_assign_pointer(old->nh_grp, newg);
2025
2026	/* Make sure concurrent readers are not using 'oldg' anymore. */
2027	synchronize_net();
2028
2029	if (newg->resilient) {
2030		rcu_assign_pointer(oldg->res_table, tmp_table);
2031		rcu_assign_pointer(oldg->spare->res_table, tmp_table);
2032	}
2033
2034	for (i = 0; i < oldg->num_nh; i++)
2035		oldg->nh_entries[i].nh_parent = new;
2036
2037	rcu_assign_pointer(new->nh_grp, oldg);
2038
2039	return 0;
2040}
2041
2042static void nh_group_v4_update(struct nh_group *nhg)
2043{
2044	struct nh_grp_entry *nhges;
2045	bool has_v4 = false;
2046	int i;
2047
2048	nhges = nhg->nh_entries;
2049	for (i = 0; i < nhg->num_nh; i++) {
2050		struct nh_info *nhi;
2051
2052		nhi = rtnl_dereference(nhges[i].nh->nh_info);
2053		if (nhi->family == AF_INET)
2054			has_v4 = true;
2055	}
2056	nhg->has_v4 = has_v4;
2057}
2058
2059static int replace_nexthop_single_notify_res(struct net *net,
2060					     struct nh_res_table *res_table,
2061					     struct nexthop *old,
2062					     struct nh_info *oldi,
2063					     struct nh_info *newi,
2064					     struct netlink_ext_ack *extack)
2065{
2066	u32 nhg_id = res_table->nhg_id;
2067	int err;
2068	u16 i;
2069
2070	for (i = 0; i < res_table->num_nh_buckets; i++) {
2071		struct nh_res_bucket *bucket = &res_table->nh_buckets[i];
2072		struct nh_grp_entry *nhge;
2073
2074		nhge = rtnl_dereference(bucket->nh_entry);
2075		if (nhge->nh == old) {
2076			err = __call_nexthop_res_bucket_notifiers(net, nhg_id,
2077								  i, true,
2078								  oldi, newi,
2079								  extack);
2080			if (err)
2081				goto err_notify;
2082		}
2083	}
2084
2085	return 0;
2086
2087err_notify:
2088	while (i-- > 0) {
2089		struct nh_res_bucket *bucket = &res_table->nh_buckets[i];
2090		struct nh_grp_entry *nhge;
2091
2092		nhge = rtnl_dereference(bucket->nh_entry);
2093		if (nhge->nh == old)
2094			__call_nexthop_res_bucket_notifiers(net, nhg_id, i,
2095							    true, newi, oldi,
2096							    extack);
2097	}
2098	return err;
2099}
2100
2101static int replace_nexthop_single_notify(struct net *net,
2102					 struct nexthop *group_nh,
2103					 struct nexthop *old,
2104					 struct nh_info *oldi,
2105					 struct nh_info *newi,
2106					 struct netlink_ext_ack *extack)
2107{
2108	struct nh_group *nhg = rtnl_dereference(group_nh->nh_grp);
2109	struct nh_res_table *res_table;
2110
2111	if (nhg->hash_threshold) {
2112		return call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE,
2113					      group_nh, extack);
2114	} else if (nhg->resilient) {
2115		res_table = rtnl_dereference(nhg->res_table);
2116		return replace_nexthop_single_notify_res(net, res_table,
2117							 old, oldi, newi,
2118							 extack);
2119	}
2120
2121	return -EINVAL;
2122}
2123
2124static int replace_nexthop_single(struct net *net, struct nexthop *old,
2125				  struct nexthop *new,
2126				  struct netlink_ext_ack *extack)
2127{
2128	u8 old_protocol, old_nh_flags;
2129	struct nh_info *oldi, *newi;
2130	struct nh_grp_entry *nhge;
2131	int err;
2132
2133	if (new->is_group) {
2134		NL_SET_ERR_MSG(extack, "Can not replace a nexthop with a nexthop group.");
2135		return -EINVAL;
2136	}
2137
2138	err = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, new, extack);
2139	if (err)
2140		return err;
2141
2142	/* Hardware flags were set on 'old' as 'new' is not in the red-black
2143	 * tree. Therefore, inherit the flags from 'old' to 'new'.
2144	 */
2145	new->nh_flags |= old->nh_flags & (RTNH_F_OFFLOAD | RTNH_F_TRAP);
2146
2147	oldi = rtnl_dereference(old->nh_info);
2148	newi = rtnl_dereference(new->nh_info);
2149
2150	newi->nh_parent = old;
2151	oldi->nh_parent = new;
2152
2153	old_protocol = old->protocol;
2154	old_nh_flags = old->nh_flags;
2155
2156	old->protocol = new->protocol;
2157	old->nh_flags = new->nh_flags;
2158
2159	rcu_assign_pointer(old->nh_info, newi);
2160	rcu_assign_pointer(new->nh_info, oldi);
2161
2162	/* Send a replace notification for all the groups using the nexthop. */
2163	list_for_each_entry(nhge, &old->grp_list, nh_list) {
2164		struct nexthop *nhp = nhge->nh_parent;
2165
2166		err = replace_nexthop_single_notify(net, nhp, old, oldi, newi,
2167						    extack);
2168		if (err)
2169			goto err_notify;
2170	}
2171
2172	/* When replacing an IPv4 nexthop with an IPv6 nexthop, potentially
2173	 * update IPv4 indication in all the groups using the nexthop.
2174	 */
2175	if (oldi->family == AF_INET && newi->family == AF_INET6) {
2176		list_for_each_entry(nhge, &old->grp_list, nh_list) {
2177			struct nexthop *nhp = nhge->nh_parent;
2178			struct nh_group *nhg;
2179
2180			nhg = rtnl_dereference(nhp->nh_grp);
2181			nh_group_v4_update(nhg);
2182		}
2183	}
2184
2185	return 0;
2186
2187err_notify:
2188	rcu_assign_pointer(new->nh_info, newi);
2189	rcu_assign_pointer(old->nh_info, oldi);
2190	old->nh_flags = old_nh_flags;
2191	old->protocol = old_protocol;
2192	oldi->nh_parent = old;
2193	newi->nh_parent = new;
2194	list_for_each_entry_continue_reverse(nhge, &old->grp_list, nh_list) {
2195		struct nexthop *nhp = nhge->nh_parent;
2196
2197		replace_nexthop_single_notify(net, nhp, old, newi, oldi, NULL);
2198	}
2199	call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, old, extack);
2200	return err;
2201}
2202
2203static void __nexthop_replace_notify(struct net *net, struct nexthop *nh,
2204				     struct nl_info *info)
2205{
2206	struct fib6_info *f6i;
2207
2208	if (!list_empty(&nh->fi_list)) {
2209		struct fib_info *fi;
2210
2211		/* expectation is a few fib_info per nexthop and then
2212		 * a lot of routes per fib_info. So mark the fib_info
2213		 * and then walk the fib tables once
2214		 */
2215		list_for_each_entry(fi, &nh->fi_list, nh_list)
2216			fi->nh_updated = true;
2217
2218		fib_info_notify_update(net, info);
2219
2220		list_for_each_entry(fi, &nh->fi_list, nh_list)
2221			fi->nh_updated = false;
2222	}
2223
2224	list_for_each_entry(f6i, &nh->f6i_list, nh_list)
2225		ipv6_stub->fib6_rt_update(net, f6i, info);
2226}
2227
2228/* send RTM_NEWROUTE with REPLACE flag set for all FIB entries
2229 * linked to this nexthop and for all groups that the nexthop
2230 * is a member of
2231 */
2232static void nexthop_replace_notify(struct net *net, struct nexthop *nh,
2233				   struct nl_info *info)
2234{
2235	struct nh_grp_entry *nhge;
2236
2237	__nexthop_replace_notify(net, nh, info);
2238
2239	list_for_each_entry(nhge, &nh->grp_list, nh_list)
2240		__nexthop_replace_notify(net, nhge->nh_parent, info);
2241}
2242
2243static int replace_nexthop(struct net *net, struct nexthop *old,
2244			   struct nexthop *new, const struct nh_config *cfg,
2245			   struct netlink_ext_ack *extack)
2246{
2247	bool new_is_reject = false;
2248	struct nh_grp_entry *nhge;
2249	int err;
2250
2251	/* check that existing FIB entries are ok with the
2252	 * new nexthop definition
2253	 */
2254	err = fib_check_nh_list(old, new, extack);
2255	if (err)
2256		return err;
2257
2258	err = fib6_check_nh_list(old, new, extack);
2259	if (err)
2260		return err;
2261
2262	if (!new->is_group) {
2263		struct nh_info *nhi = rtnl_dereference(new->nh_info);
2264
2265		new_is_reject = nhi->reject_nh;
2266	}
2267
2268	list_for_each_entry(nhge, &old->grp_list, nh_list) {
2269		/* if new nexthop is a blackhole, any groups using this
2270		 * nexthop cannot have more than 1 path
2271		 */
2272		if (new_is_reject &&
2273		    nexthop_num_path(nhge->nh_parent) > 1) {
2274			NL_SET_ERR_MSG(extack, "Blackhole nexthop can not be a member of a group with more than one path");
2275			return -EINVAL;
2276		}
2277
2278		err = fib_check_nh_list(nhge->nh_parent, new, extack);
2279		if (err)
2280			return err;
2281
2282		err = fib6_check_nh_list(nhge->nh_parent, new, extack);
2283		if (err)
2284			return err;
2285	}
2286
2287	if (old->is_group)
2288		err = replace_nexthop_grp(net, old, new, cfg, extack);
2289	else
2290		err = replace_nexthop_single(net, old, new, extack);
2291
2292	if (!err) {
2293		nh_rt_cache_flush(net, old, new);
2294
2295		__remove_nexthop(net, new, NULL);
2296		nexthop_put(new);
2297	}
2298
2299	return err;
2300}
2301
2302/* called with rtnl_lock held */
2303static int insert_nexthop(struct net *net, struct nexthop *new_nh,
2304			  struct nh_config *cfg, struct netlink_ext_ack *extack)
2305{
2306	struct rb_node **pp, *parent = NULL, *next;
2307	struct rb_root *root = &net->nexthop.rb_root;
2308	bool replace = !!(cfg->nlflags & NLM_F_REPLACE);
2309	bool create = !!(cfg->nlflags & NLM_F_CREATE);
2310	u32 new_id = new_nh->id;
2311	int replace_notify = 0;
2312	int rc = -EEXIST;
2313
2314	pp = &root->rb_node;
2315	while (1) {
2316		struct nexthop *nh;
2317
2318		next = *pp;
2319		if (!next)
2320			break;
2321
2322		parent = next;
2323
2324		nh = rb_entry(parent, struct nexthop, rb_node);
2325		if (new_id < nh->id) {
2326			pp = &next->rb_left;
2327		} else if (new_id > nh->id) {
2328			pp = &next->rb_right;
2329		} else if (replace) {
2330			rc = replace_nexthop(net, nh, new_nh, cfg, extack);
2331			if (!rc) {
2332				new_nh = nh; /* send notification with old nh */
2333				replace_notify = 1;
2334			}
2335			goto out;
2336		} else {
2337			/* id already exists and not a replace */
2338			goto out;
2339		}
2340	}
2341
2342	if (replace && !create) {
2343		NL_SET_ERR_MSG(extack, "Replace specified without create and no entry exists");
2344		rc = -ENOENT;
2345		goto out;
2346	}
2347
2348	if (new_nh->is_group) {
2349		struct nh_group *nhg = rtnl_dereference(new_nh->nh_grp);
2350		struct nh_res_table *res_table;
2351
2352		if (nhg->resilient) {
2353			res_table = rtnl_dereference(nhg->res_table);
2354
2355			/* Not passing the number of buckets is OK when
2356			 * replacing, but not when creating a new group.
2357			 */
2358			if (!cfg->nh_grp_res_has_num_buckets) {
2359				NL_SET_ERR_MSG(extack, "Number of buckets not specified for nexthop group insertion");
2360				rc = -EINVAL;
2361				goto out;
2362			}
2363
2364			nh_res_group_rebalance(nhg, res_table);
2365
2366			/* Do not send bucket notifications, we do full
2367			 * notification below.
2368			 */
2369			nh_res_table_upkeep(res_table, false, false);
2370		}
2371	}
2372
2373	rb_link_node_rcu(&new_nh->rb_node, parent, pp);
2374	rb_insert_color(&new_nh->rb_node, root);
2375
2376	/* The initial insertion is a full notification for hash-threshold as
2377	 * well as resilient groups.
2378	 */
2379	rc = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, new_nh, extack);
2380	if (rc)
2381		rb_erase(&new_nh->rb_node, &net->nexthop.rb_root);
2382
2383out:
2384	if (!rc) {
2385		nh_base_seq_inc(net);
2386		nexthop_notify(RTM_NEWNEXTHOP, new_nh, &cfg->nlinfo);
2387		if (replace_notify &&
2388		    READ_ONCE(net->ipv4.sysctl_nexthop_compat_mode))
2389			nexthop_replace_notify(net, new_nh, &cfg->nlinfo);
2390	}
2391
2392	return rc;
2393}
2394
2395/* rtnl */
2396/* remove all nexthops tied to a device being deleted */
2397static void nexthop_flush_dev(struct net_device *dev, unsigned long event)
2398{
2399	unsigned int hash = nh_dev_hashfn(dev->ifindex);
2400	struct net *net = dev_net(dev);
2401	struct hlist_head *head = &net->nexthop.devhash[hash];
2402	struct hlist_node *n;
2403	struct nh_info *nhi;
2404
2405	hlist_for_each_entry_safe(nhi, n, head, dev_hash) {
2406		if (nhi->fib_nhc.nhc_dev != dev)
2407			continue;
2408
2409		if (nhi->reject_nh &&
2410		    (event == NETDEV_DOWN || event == NETDEV_CHANGE))
2411			continue;
2412
2413		remove_nexthop(net, nhi->nh_parent, NULL);
2414	}
2415}
2416
2417/* rtnl; called when net namespace is deleted */
2418static void flush_all_nexthops(struct net *net)
2419{
2420	struct rb_root *root = &net->nexthop.rb_root;
2421	struct rb_node *node;
2422	struct nexthop *nh;
2423
2424	while ((node = rb_first(root))) {
2425		nh = rb_entry(node, struct nexthop, rb_node);
2426		remove_nexthop(net, nh, NULL);
2427		cond_resched();
2428	}
2429}
2430
2431static struct nexthop *nexthop_create_group(struct net *net,
2432					    struct nh_config *cfg)
2433{
2434	struct nlattr *grps_attr = cfg->nh_grp;
2435	struct nexthop_grp *entry = nla_data(grps_attr);
2436	u16 num_nh = nla_len(grps_attr) / sizeof(*entry);
2437	struct nh_group *nhg;
2438	struct nexthop *nh;
2439	int err;
2440	int i;
2441
2442	if (WARN_ON(!num_nh))
2443		return ERR_PTR(-EINVAL);
2444
2445	nh = nexthop_alloc();
2446	if (!nh)
2447		return ERR_PTR(-ENOMEM);
2448
2449	nh->is_group = 1;
2450
2451	nhg = nexthop_grp_alloc(num_nh);
2452	if (!nhg) {
2453		kfree(nh);
2454		return ERR_PTR(-ENOMEM);
2455	}
2456
2457	/* spare group used for removals */
2458	nhg->spare = nexthop_grp_alloc(num_nh);
2459	if (!nhg->spare) {
2460		kfree(nhg);
2461		kfree(nh);
2462		return ERR_PTR(-ENOMEM);
2463	}
2464	nhg->spare->spare = nhg;
2465
2466	for (i = 0; i < nhg->num_nh; ++i) {
2467		struct nexthop *nhe;
2468		struct nh_info *nhi;
2469
2470		nhe = nexthop_find_by_id(net, entry[i].id);
2471		if (!nexthop_get(nhe)) {
2472			err = -ENOENT;
2473			goto out_no_nh;
2474		}
2475
2476		nhi = rtnl_dereference(nhe->nh_info);
2477		if (nhi->family == AF_INET)
2478			nhg->has_v4 = true;
2479
2480		nhg->nh_entries[i].nh = nhe;
2481		nhg->nh_entries[i].weight = entry[i].weight + 1;
2482		list_add(&nhg->nh_entries[i].nh_list, &nhe->grp_list);
2483		nhg->nh_entries[i].nh_parent = nh;
2484	}
2485
2486	if (cfg->nh_grp_type == NEXTHOP_GRP_TYPE_MPATH) {
2487		nhg->hash_threshold = 1;
2488		nhg->is_multipath = true;
2489	} else if (cfg->nh_grp_type == NEXTHOP_GRP_TYPE_RES) {
2490		struct nh_res_table *res_table;
2491
2492		res_table = nexthop_res_table_alloc(net, cfg->nh_id, cfg);
2493		if (!res_table) {
2494			err = -ENOMEM;
2495			goto out_no_nh;
2496		}
2497
2498		rcu_assign_pointer(nhg->spare->res_table, res_table);
2499		rcu_assign_pointer(nhg->res_table, res_table);
2500		nhg->resilient = true;
2501		nhg->is_multipath = true;
2502	}
2503
2504	WARN_ON_ONCE(nhg->hash_threshold + nhg->resilient != 1);
2505
2506	if (nhg->hash_threshold)
2507		nh_hthr_group_rebalance(nhg);
2508
2509	if (cfg->nh_fdb)
2510		nhg->fdb_nh = 1;
2511
2512	rcu_assign_pointer(nh->nh_grp, nhg);
2513
2514	return nh;
2515
2516out_no_nh:
2517	for (i--; i >= 0; --i) {
2518		list_del(&nhg->nh_entries[i].nh_list);
2519		nexthop_put(nhg->nh_entries[i].nh);
2520	}
2521
2522	kfree(nhg->spare);
2523	kfree(nhg);
2524	kfree(nh);
2525
2526	return ERR_PTR(err);
2527}
2528
2529static int nh_create_ipv4(struct net *net, struct nexthop *nh,
2530			  struct nh_info *nhi, struct nh_config *cfg,
2531			  struct netlink_ext_ack *extack)
2532{
2533	struct fib_nh *fib_nh = &nhi->fib_nh;
2534	struct fib_config fib_cfg = {
2535		.fc_oif   = cfg->nh_ifindex,
2536		.fc_gw4   = cfg->gw.ipv4,
2537		.fc_gw_family = cfg->gw.ipv4 ? AF_INET : 0,
2538		.fc_flags = cfg->nh_flags,
2539		.fc_nlinfo = cfg->nlinfo,
2540		.fc_encap = cfg->nh_encap,
2541		.fc_encap_type = cfg->nh_encap_type,
2542	};
2543	u32 tb_id = (cfg->dev ? l3mdev_fib_table(cfg->dev) : RT_TABLE_MAIN);
2544	int err;
2545
2546	err = fib_nh_init(net, fib_nh, &fib_cfg, 1, extack);
2547	if (err) {
2548		fib_nh_release(net, fib_nh);
2549		goto out;
2550	}
2551
2552	if (nhi->fdb_nh)
2553		goto out;
2554
2555	/* sets nh_dev if successful */
2556	err = fib_check_nh(net, fib_nh, tb_id, 0, extack);
2557	if (!err) {
2558		nh->nh_flags = fib_nh->fib_nh_flags;
2559		fib_info_update_nhc_saddr(net, &fib_nh->nh_common,
2560					  !fib_nh->fib_nh_scope ? 0 : fib_nh->fib_nh_scope - 1);
2561	} else {
2562		fib_nh_release(net, fib_nh);
2563	}
2564out:
2565	return err;
2566}
2567
2568static int nh_create_ipv6(struct net *net,  struct nexthop *nh,
2569			  struct nh_info *nhi, struct nh_config *cfg,
2570			  struct netlink_ext_ack *extack)
2571{
2572	struct fib6_nh *fib6_nh = &nhi->fib6_nh;
2573	struct fib6_config fib6_cfg = {
2574		.fc_table = l3mdev_fib_table(cfg->dev),
2575		.fc_ifindex = cfg->nh_ifindex,
2576		.fc_gateway = cfg->gw.ipv6,
2577		.fc_flags = cfg->nh_flags,
2578		.fc_nlinfo = cfg->nlinfo,
2579		.fc_encap = cfg->nh_encap,
2580		.fc_encap_type = cfg->nh_encap_type,
2581		.fc_is_fdb = cfg->nh_fdb,
2582	};
2583	int err;
2584
2585	if (!ipv6_addr_any(&cfg->gw.ipv6))
2586		fib6_cfg.fc_flags |= RTF_GATEWAY;
2587
2588	/* sets nh_dev if successful */
2589	err = ipv6_stub->fib6_nh_init(net, fib6_nh, &fib6_cfg, GFP_KERNEL,
2590				      extack);
2591	if (err) {
2592		/* IPv6 is not enabled, don't call fib6_nh_release */
2593		if (err == -EAFNOSUPPORT)
2594			goto out;
2595		ipv6_stub->fib6_nh_release(fib6_nh);
2596	} else {
2597		nh->nh_flags = fib6_nh->fib_nh_flags;
2598	}
2599out:
2600	return err;
2601}
2602
2603static struct nexthop *nexthop_create(struct net *net, struct nh_config *cfg,
2604				      struct netlink_ext_ack *extack)
2605{
2606	struct nh_info *nhi;
2607	struct nexthop *nh;
2608	int err = 0;
2609
2610	nh = nexthop_alloc();
2611	if (!nh)
2612		return ERR_PTR(-ENOMEM);
2613
2614	nhi = kzalloc(sizeof(*nhi), GFP_KERNEL);
2615	if (!nhi) {
2616		kfree(nh);
2617		return ERR_PTR(-ENOMEM);
2618	}
2619
2620	nh->nh_flags = cfg->nh_flags;
2621	nh->net = net;
2622
2623	nhi->nh_parent = nh;
2624	nhi->family = cfg->nh_family;
2625	nhi->fib_nhc.nhc_scope = RT_SCOPE_LINK;
2626
2627	if (cfg->nh_fdb)
2628		nhi->fdb_nh = 1;
2629
2630	if (cfg->nh_blackhole) {
2631		nhi->reject_nh = 1;
2632		cfg->nh_ifindex = net->loopback_dev->ifindex;
2633	}
2634
2635	switch (cfg->nh_family) {
2636	case AF_INET:
2637		err = nh_create_ipv4(net, nh, nhi, cfg, extack);
2638		break;
2639	case AF_INET6:
2640		err = nh_create_ipv6(net, nh, nhi, cfg, extack);
2641		break;
2642	}
2643
2644	if (err) {
2645		kfree(nhi);
2646		kfree(nh);
2647		return ERR_PTR(err);
2648	}
2649
2650	/* add the entry to the device based hash */
2651	if (!nhi->fdb_nh)
2652		nexthop_devhash_add(net, nhi);
2653
2654	rcu_assign_pointer(nh->nh_info, nhi);
2655
2656	return nh;
2657}
2658
2659/* called with rtnl lock held */
2660static struct nexthop *nexthop_add(struct net *net, struct nh_config *cfg,
2661				   struct netlink_ext_ack *extack)
2662{
2663	struct nexthop *nh;
2664	int err;
2665
2666	if (cfg->nlflags & NLM_F_REPLACE && !cfg->nh_id) {
2667		NL_SET_ERR_MSG(extack, "Replace requires nexthop id");
2668		return ERR_PTR(-EINVAL);
2669	}
2670
2671	if (!cfg->nh_id) {
2672		cfg->nh_id = nh_find_unused_id(net);
2673		if (!cfg->nh_id) {
2674			NL_SET_ERR_MSG(extack, "No unused id");
2675			return ERR_PTR(-EINVAL);
2676		}
2677	}
2678
2679	if (cfg->nh_grp)
2680		nh = nexthop_create_group(net, cfg);
2681	else
2682		nh = nexthop_create(net, cfg, extack);
2683
2684	if (IS_ERR(nh))
2685		return nh;
2686
2687	refcount_set(&nh->refcnt, 1);
2688	nh->id = cfg->nh_id;
2689	nh->protocol = cfg->nh_protocol;
2690	nh->net = net;
2691
2692	err = insert_nexthop(net, nh, cfg, extack);
2693	if (err) {
2694		__remove_nexthop(net, nh, NULL);
2695		nexthop_put(nh);
2696		nh = ERR_PTR(err);
2697	}
2698
2699	return nh;
2700}
2701
2702static int rtm_nh_get_timer(struct nlattr *attr, unsigned long fallback,
2703			    unsigned long *timer_p, bool *has_p,
2704			    struct netlink_ext_ack *extack)
2705{
2706	unsigned long timer;
2707	u32 value;
2708
2709	if (!attr) {
2710		*timer_p = fallback;
2711		*has_p = false;
2712		return 0;
2713	}
2714
2715	value = nla_get_u32(attr);
2716	timer = clock_t_to_jiffies(value);
2717	if (timer == ~0UL) {
2718		NL_SET_ERR_MSG(extack, "Timer value too large");
2719		return -EINVAL;
2720	}
2721
2722	*timer_p = timer;
2723	*has_p = true;
2724	return 0;
2725}
2726
2727static int rtm_to_nh_config_grp_res(struct nlattr *res, struct nh_config *cfg,
2728				    struct netlink_ext_ack *extack)
2729{
2730	struct nlattr *tb[ARRAY_SIZE(rtm_nh_res_policy_new)] = {};
2731	int err;
2732
2733	if (res) {
2734		err = nla_parse_nested(tb,
2735				       ARRAY_SIZE(rtm_nh_res_policy_new) - 1,
2736				       res, rtm_nh_res_policy_new, extack);
2737		if (err < 0)
2738			return err;
2739	}
2740
2741	if (tb[NHA_RES_GROUP_BUCKETS]) {
2742		cfg->nh_grp_res_num_buckets =
2743			nla_get_u16(tb[NHA_RES_GROUP_BUCKETS]);
2744		cfg->nh_grp_res_has_num_buckets = true;
2745		if (!cfg->nh_grp_res_num_buckets) {
2746			NL_SET_ERR_MSG(extack, "Number of buckets needs to be non-0");
2747			return -EINVAL;
2748		}
2749	}
2750
2751	err = rtm_nh_get_timer(tb[NHA_RES_GROUP_IDLE_TIMER],
2752			       NH_RES_DEFAULT_IDLE_TIMER,
2753			       &cfg->nh_grp_res_idle_timer,
2754			       &cfg->nh_grp_res_has_idle_timer,
2755			       extack);
2756	if (err)
2757		return err;
2758
2759	return rtm_nh_get_timer(tb[NHA_RES_GROUP_UNBALANCED_TIMER],
2760				NH_RES_DEFAULT_UNBALANCED_TIMER,
2761				&cfg->nh_grp_res_unbalanced_timer,
2762				&cfg->nh_grp_res_has_unbalanced_timer,
2763				extack);
2764}
2765
2766static int rtm_to_nh_config(struct net *net, struct sk_buff *skb,
2767			    struct nlmsghdr *nlh, struct nh_config *cfg,
2768			    struct netlink_ext_ack *extack)
2769{
2770	struct nhmsg *nhm = nlmsg_data(nlh);
2771	struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_new)];
2772	int err;
2773
2774	err = nlmsg_parse(nlh, sizeof(*nhm), tb,
2775			  ARRAY_SIZE(rtm_nh_policy_new) - 1,
2776			  rtm_nh_policy_new, extack);
2777	if (err < 0)
2778		return err;
2779
2780	err = -EINVAL;
2781	if (nhm->resvd || nhm->nh_scope) {
2782		NL_SET_ERR_MSG(extack, "Invalid values in ancillary header");
2783		goto out;
2784	}
2785	if (nhm->nh_flags & ~NEXTHOP_VALID_USER_FLAGS) {
2786		NL_SET_ERR_MSG(extack, "Invalid nexthop flags in ancillary header");
2787		goto out;
2788	}
2789
2790	switch (nhm->nh_family) {
2791	case AF_INET:
2792	case AF_INET6:
2793		break;
2794	case AF_UNSPEC:
2795		if (tb[NHA_GROUP])
2796			break;
2797		fallthrough;
2798	default:
2799		NL_SET_ERR_MSG(extack, "Invalid address family");
2800		goto out;
2801	}
2802
2803	memset(cfg, 0, sizeof(*cfg));
2804	cfg->nlflags = nlh->nlmsg_flags;
2805	cfg->nlinfo.portid = NETLINK_CB(skb).portid;
2806	cfg->nlinfo.nlh = nlh;
2807	cfg->nlinfo.nl_net = net;
2808
2809	cfg->nh_family = nhm->nh_family;
2810	cfg->nh_protocol = nhm->nh_protocol;
2811	cfg->nh_flags = nhm->nh_flags;
2812
2813	if (tb[NHA_ID])
2814		cfg->nh_id = nla_get_u32(tb[NHA_ID]);
2815
2816	if (tb[NHA_FDB]) {
2817		if (tb[NHA_OIF] || tb[NHA_BLACKHOLE] ||
2818		    tb[NHA_ENCAP]   || tb[NHA_ENCAP_TYPE]) {
2819			NL_SET_ERR_MSG(extack, "Fdb attribute can not be used with encap, oif or blackhole");
2820			goto out;
2821		}
2822		if (nhm->nh_flags) {
2823			NL_SET_ERR_MSG(extack, "Unsupported nexthop flags in ancillary header");
2824			goto out;
2825		}
2826		cfg->nh_fdb = nla_get_flag(tb[NHA_FDB]);
2827	}
2828
2829	if (tb[NHA_GROUP]) {
2830		if (nhm->nh_family != AF_UNSPEC) {
2831			NL_SET_ERR_MSG(extack, "Invalid family for group");
2832			goto out;
2833		}
2834		cfg->nh_grp = tb[NHA_GROUP];
2835
2836		cfg->nh_grp_type = NEXTHOP_GRP_TYPE_MPATH;
2837		if (tb[NHA_GROUP_TYPE])
2838			cfg->nh_grp_type = nla_get_u16(tb[NHA_GROUP_TYPE]);
2839
2840		if (cfg->nh_grp_type > NEXTHOP_GRP_TYPE_MAX) {
2841			NL_SET_ERR_MSG(extack, "Invalid group type");
2842			goto out;
2843		}
2844		err = nh_check_attr_group(net, tb, ARRAY_SIZE(tb),
2845					  cfg->nh_grp_type, extack);
2846		if (err)
2847			goto out;
2848
2849		if (cfg->nh_grp_type == NEXTHOP_GRP_TYPE_RES)
2850			err = rtm_to_nh_config_grp_res(tb[NHA_RES_GROUP],
2851						       cfg, extack);
2852
2853		/* no other attributes should be set */
2854		goto out;
2855	}
2856
2857	if (tb[NHA_BLACKHOLE]) {
2858		if (tb[NHA_GATEWAY] || tb[NHA_OIF] ||
2859		    tb[NHA_ENCAP]   || tb[NHA_ENCAP_TYPE] || tb[NHA_FDB]) {
2860			NL_SET_ERR_MSG(extack, "Blackhole attribute can not be used with gateway, oif, encap or fdb");
2861			goto out;
2862		}
2863
2864		cfg->nh_blackhole = 1;
2865		err = 0;
2866		goto out;
2867	}
2868
2869	if (!cfg->nh_fdb && !tb[NHA_OIF]) {
2870		NL_SET_ERR_MSG(extack, "Device attribute required for non-blackhole and non-fdb nexthops");
2871		goto out;
2872	}
2873
2874	if (!cfg->nh_fdb && tb[NHA_OIF]) {
2875		cfg->nh_ifindex = nla_get_u32(tb[NHA_OIF]);
2876		if (cfg->nh_ifindex)
2877			cfg->dev = __dev_get_by_index(net, cfg->nh_ifindex);
2878
2879		if (!cfg->dev) {
2880			NL_SET_ERR_MSG(extack, "Invalid device index");
2881			goto out;
2882		} else if (!(cfg->dev->flags & IFF_UP)) {
2883			NL_SET_ERR_MSG(extack, "Nexthop device is not up");
2884			err = -ENETDOWN;
2885			goto out;
2886		} else if (!netif_carrier_ok(cfg->dev)) {
2887			NL_SET_ERR_MSG(extack, "Carrier for nexthop device is down");
2888			err = -ENETDOWN;
2889			goto out;
2890		}
2891	}
2892
2893	err = -EINVAL;
2894	if (tb[NHA_GATEWAY]) {
2895		struct nlattr *gwa = tb[NHA_GATEWAY];
2896
2897		switch (cfg->nh_family) {
2898		case AF_INET:
2899			if (nla_len(gwa) != sizeof(u32)) {
2900				NL_SET_ERR_MSG(extack, "Invalid gateway");
2901				goto out;
2902			}
2903			cfg->gw.ipv4 = nla_get_be32(gwa);
2904			break;
2905		case AF_INET6:
2906			if (nla_len(gwa) != sizeof(struct in6_addr)) {
2907				NL_SET_ERR_MSG(extack, "Invalid gateway");
2908				goto out;
2909			}
2910			cfg->gw.ipv6 = nla_get_in6_addr(gwa);
2911			break;
2912		default:
2913			NL_SET_ERR_MSG(extack,
2914				       "Unknown address family for gateway");
2915			goto out;
2916		}
2917	} else {
2918		/* device only nexthop (no gateway) */
2919		if (cfg->nh_flags & RTNH_F_ONLINK) {
2920			NL_SET_ERR_MSG(extack,
2921				       "ONLINK flag can not be set for nexthop without a gateway");
2922			goto out;
2923		}
2924	}
2925
2926	if (tb[NHA_ENCAP]) {
2927		cfg->nh_encap = tb[NHA_ENCAP];
2928
2929		if (!tb[NHA_ENCAP_TYPE]) {
2930			NL_SET_ERR_MSG(extack, "LWT encapsulation type is missing");
2931			goto out;
2932		}
2933
2934		cfg->nh_encap_type = nla_get_u16(tb[NHA_ENCAP_TYPE]);
2935		err = lwtunnel_valid_encap_type(cfg->nh_encap_type, extack);
2936		if (err < 0)
2937			goto out;
2938
2939	} else if (tb[NHA_ENCAP_TYPE]) {
2940		NL_SET_ERR_MSG(extack, "LWT encapsulation attribute is missing");
2941		goto out;
2942	}
2943
2944
2945	err = 0;
2946out:
2947	return err;
2948}
2949
2950/* rtnl */
2951static int rtm_new_nexthop(struct sk_buff *skb, struct nlmsghdr *nlh,
2952			   struct netlink_ext_ack *extack)
2953{
2954	struct net *net = sock_net(skb->sk);
2955	struct nh_config cfg;
2956	struct nexthop *nh;
2957	int err;
2958
2959	err = rtm_to_nh_config(net, skb, nlh, &cfg, extack);
2960	if (!err) {
2961		nh = nexthop_add(net, &cfg, extack);
2962		if (IS_ERR(nh))
2963			err = PTR_ERR(nh);
2964	}
2965
2966	return err;
2967}
2968
2969static int __nh_valid_get_del_req(const struct nlmsghdr *nlh,
2970				  struct nlattr **tb, u32 *id,
2971				  struct netlink_ext_ack *extack)
2972{
2973	struct nhmsg *nhm = nlmsg_data(nlh);
2974
2975	if (nhm->nh_protocol || nhm->resvd || nhm->nh_scope || nhm->nh_flags) {
2976		NL_SET_ERR_MSG(extack, "Invalid values in header");
2977		return -EINVAL;
2978	}
2979
2980	if (!tb[NHA_ID]) {
2981		NL_SET_ERR_MSG(extack, "Nexthop id is missing");
2982		return -EINVAL;
2983	}
2984
2985	*id = nla_get_u32(tb[NHA_ID]);
2986	if (!(*id)) {
2987		NL_SET_ERR_MSG(extack, "Invalid nexthop id");
2988		return -EINVAL;
2989	}
2990
2991	return 0;
2992}
2993
2994static int nh_valid_get_del_req(const struct nlmsghdr *nlh, u32 *id,
2995				struct netlink_ext_ack *extack)
2996{
2997	struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_get)];
2998	int err;
2999
3000	err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
3001			  ARRAY_SIZE(rtm_nh_policy_get) - 1,
3002			  rtm_nh_policy_get, extack);
3003	if (err < 0)
3004		return err;
3005
3006	return __nh_valid_get_del_req(nlh, tb, id, extack);
3007}
3008
3009/* rtnl */
3010static int rtm_del_nexthop(struct sk_buff *skb, struct nlmsghdr *nlh,
3011			   struct netlink_ext_ack *extack)
3012{
3013	struct net *net = sock_net(skb->sk);
3014	struct nl_info nlinfo = {
3015		.nlh = nlh,
3016		.nl_net = net,
3017		.portid = NETLINK_CB(skb).portid,
3018	};
3019	struct nexthop *nh;
3020	int err;
3021	u32 id;
3022
3023	err = nh_valid_get_del_req(nlh, &id, extack);
3024	if (err)
3025		return err;
3026
3027	nh = nexthop_find_by_id(net, id);
3028	if (!nh)
3029		return -ENOENT;
3030
3031	remove_nexthop(net, nh, &nlinfo);
3032
3033	return 0;
3034}
3035
3036/* rtnl */
3037static int rtm_get_nexthop(struct sk_buff *in_skb, struct nlmsghdr *nlh,
3038			   struct netlink_ext_ack *extack)
3039{
3040	struct net *net = sock_net(in_skb->sk);
3041	struct sk_buff *skb = NULL;
3042	struct nexthop *nh;
3043	int err;
3044	u32 id;
3045
3046	err = nh_valid_get_del_req(nlh, &id, extack);
3047	if (err)
3048		return err;
3049
3050	err = -ENOBUFS;
3051	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
3052	if (!skb)
3053		goto out;
3054
3055	err = -ENOENT;
3056	nh = nexthop_find_by_id(net, id);
3057	if (!nh)
3058		goto errout_free;
3059
3060	err = nh_fill_node(skb, nh, RTM_NEWNEXTHOP, NETLINK_CB(in_skb).portid,
3061			   nlh->nlmsg_seq, 0);
3062	if (err < 0) {
3063		WARN_ON(err == -EMSGSIZE);
3064		goto errout_free;
3065	}
3066
3067	err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
3068out:
3069	return err;
3070errout_free:
3071	kfree_skb(skb);
3072	goto out;
3073}
3074
3075struct nh_dump_filter {
3076	u32 nh_id;
3077	int dev_idx;
3078	int master_idx;
3079	bool group_filter;
3080	bool fdb_filter;
3081	u32 res_bucket_nh_id;
3082};
3083
3084static bool nh_dump_filtered(struct nexthop *nh,
3085			     struct nh_dump_filter *filter, u8 family)
3086{
3087	const struct net_device *dev;
3088	const struct nh_info *nhi;
3089
3090	if (filter->group_filter && !nh->is_group)
3091		return true;
3092
3093	if (!filter->dev_idx && !filter->master_idx && !family)
3094		return false;
3095
3096	if (nh->is_group)
3097		return true;
3098
3099	nhi = rtnl_dereference(nh->nh_info);
3100	if (family && nhi->family != family)
3101		return true;
3102
3103	dev = nhi->fib_nhc.nhc_dev;
3104	if (filter->dev_idx && (!dev || dev->ifindex != filter->dev_idx))
3105		return true;
3106
3107	if (filter->master_idx) {
3108		struct net_device *master;
3109
3110		if (!dev)
3111			return true;
3112
3113		master = netdev_master_upper_dev_get((struct net_device *)dev);
3114		if (!master || master->ifindex != filter->master_idx)
3115			return true;
3116	}
3117
3118	return false;
3119}
3120
3121static int __nh_valid_dump_req(const struct nlmsghdr *nlh, struct nlattr **tb,
3122			       struct nh_dump_filter *filter,
3123			       struct netlink_ext_ack *extack)
3124{
3125	struct nhmsg *nhm;
3126	u32 idx;
3127
3128	if (tb[NHA_OIF]) {
3129		idx = nla_get_u32(tb[NHA_OIF]);
3130		if (idx > INT_MAX) {
3131			NL_SET_ERR_MSG(extack, "Invalid device index");
3132			return -EINVAL;
3133		}
3134		filter->dev_idx = idx;
3135	}
3136	if (tb[NHA_MASTER]) {
3137		idx = nla_get_u32(tb[NHA_MASTER]);
3138		if (idx > INT_MAX) {
3139			NL_SET_ERR_MSG(extack, "Invalid master device index");
3140			return -EINVAL;
3141		}
3142		filter->master_idx = idx;
3143	}
3144	filter->group_filter = nla_get_flag(tb[NHA_GROUPS]);
3145	filter->fdb_filter = nla_get_flag(tb[NHA_FDB]);
3146
3147	nhm = nlmsg_data(nlh);
3148	if (nhm->nh_protocol || nhm->resvd || nhm->nh_scope || nhm->nh_flags) {
3149		NL_SET_ERR_MSG(extack, "Invalid values in header for nexthop dump request");
3150		return -EINVAL;
3151	}
3152
3153	return 0;
3154}
3155
3156static int nh_valid_dump_req(const struct nlmsghdr *nlh,
3157			     struct nh_dump_filter *filter,
3158			     struct netlink_callback *cb)
3159{
3160	struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_dump)];
3161	int err;
3162
3163	err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
3164			  ARRAY_SIZE(rtm_nh_policy_dump) - 1,
3165			  rtm_nh_policy_dump, cb->extack);
3166	if (err < 0)
3167		return err;
3168
3169	return __nh_valid_dump_req(nlh, tb, filter, cb->extack);
3170}
3171
3172struct rtm_dump_nh_ctx {
3173	u32 idx;
3174};
3175
3176static struct rtm_dump_nh_ctx *
3177rtm_dump_nh_ctx(struct netlink_callback *cb)
3178{
3179	struct rtm_dump_nh_ctx *ctx = (void *)cb->ctx;
3180
3181	BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));
3182	return ctx;
3183}
3184
3185static int rtm_dump_walk_nexthops(struct sk_buff *skb,
3186				  struct netlink_callback *cb,
3187				  struct rb_root *root,
3188				  struct rtm_dump_nh_ctx *ctx,
3189				  int (*nh_cb)(struct sk_buff *skb,
3190					       struct netlink_callback *cb,
3191					       struct nexthop *nh, void *data),
3192				  void *data)
3193{
3194	struct rb_node *node;
3195	int s_idx;
3196	int err;
3197
3198	s_idx = ctx->idx;
3199	for (node = rb_first(root); node; node = rb_next(node)) {
3200		struct nexthop *nh;
3201
3202		nh = rb_entry(node, struct nexthop, rb_node);
3203		if (nh->id < s_idx)
3204			continue;
3205
3206		ctx->idx = nh->id;
3207		err = nh_cb(skb, cb, nh, data);
3208		if (err)
3209			return err;
3210	}
3211
3212	return 0;
3213}
3214
3215static int rtm_dump_nexthop_cb(struct sk_buff *skb, struct netlink_callback *cb,
3216			       struct nexthop *nh, void *data)
3217{
3218	struct nhmsg *nhm = nlmsg_data(cb->nlh);
3219	struct nh_dump_filter *filter = data;
3220
3221	if (nh_dump_filtered(nh, filter, nhm->nh_family))
3222		return 0;
3223
3224	return nh_fill_node(skb, nh, RTM_NEWNEXTHOP,
3225			    NETLINK_CB(cb->skb).portid,
3226			    cb->nlh->nlmsg_seq, NLM_F_MULTI);
3227}
3228
3229/* rtnl */
3230static int rtm_dump_nexthop(struct sk_buff *skb, struct netlink_callback *cb)
3231{
3232	struct rtm_dump_nh_ctx *ctx = rtm_dump_nh_ctx(cb);
3233	struct net *net = sock_net(skb->sk);
3234	struct rb_root *root = &net->nexthop.rb_root;
3235	struct nh_dump_filter filter = {};
3236	int err;
3237
3238	err = nh_valid_dump_req(cb->nlh, &filter, cb);
3239	if (err < 0)
3240		return err;
3241
3242	err = rtm_dump_walk_nexthops(skb, cb, root, ctx,
3243				     &rtm_dump_nexthop_cb, &filter);
3244	if (err < 0) {
3245		if (likely(skb->len))
3246			err = skb->len;
3247	}
3248
3249	cb->seq = net->nexthop.seq;
3250	nl_dump_check_consistent(cb, nlmsg_hdr(skb));
3251	return err;
3252}
3253
3254static struct nexthop *
3255nexthop_find_group_resilient(struct net *net, u32 id,
3256			     struct netlink_ext_ack *extack)
3257{
3258	struct nh_group *nhg;
3259	struct nexthop *nh;
3260
3261	nh = nexthop_find_by_id(net, id);
3262	if (!nh)
3263		return ERR_PTR(-ENOENT);
3264
3265	if (!nh->is_group) {
3266		NL_SET_ERR_MSG(extack, "Not a nexthop group");
3267		return ERR_PTR(-EINVAL);
3268	}
3269
3270	nhg = rtnl_dereference(nh->nh_grp);
3271	if (!nhg->resilient) {
3272		NL_SET_ERR_MSG(extack, "Nexthop group not of type resilient");
3273		return ERR_PTR(-EINVAL);
3274	}
3275
3276	return nh;
3277}
3278
3279static int nh_valid_dump_nhid(struct nlattr *attr, u32 *nh_id_p,
3280			      struct netlink_ext_ack *extack)
3281{
3282	u32 idx;
3283
3284	if (attr) {
3285		idx = nla_get_u32(attr);
3286		if (!idx) {
3287			NL_SET_ERR_MSG(extack, "Invalid nexthop id");
3288			return -EINVAL;
3289		}
3290		*nh_id_p = idx;
3291	} else {
3292		*nh_id_p = 0;
3293	}
3294
3295	return 0;
3296}
3297
3298static int nh_valid_dump_bucket_req(const struct nlmsghdr *nlh,
3299				    struct nh_dump_filter *filter,
3300				    struct netlink_callback *cb)
3301{
3302	struct nlattr *res_tb[ARRAY_SIZE(rtm_nh_res_bucket_policy_dump)];
3303	struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_dump_bucket)];
3304	int err;
3305
3306	err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
3307			  ARRAY_SIZE(rtm_nh_policy_dump_bucket) - 1,
3308			  rtm_nh_policy_dump_bucket, NULL);
3309	if (err < 0)
3310		return err;
3311
3312	err = nh_valid_dump_nhid(tb[NHA_ID], &filter->nh_id, cb->extack);
3313	if (err)
3314		return err;
3315
3316	if (tb[NHA_RES_BUCKET]) {
3317		size_t max = ARRAY_SIZE(rtm_nh_res_bucket_policy_dump) - 1;
3318
3319		err = nla_parse_nested(res_tb, max,
3320				       tb[NHA_RES_BUCKET],
3321				       rtm_nh_res_bucket_policy_dump,
3322				       cb->extack);
3323		if (err < 0)
3324			return err;
3325
3326		err = nh_valid_dump_nhid(res_tb[NHA_RES_BUCKET_NH_ID],
3327					 &filter->res_bucket_nh_id,
3328					 cb->extack);
3329		if (err)
3330			return err;
3331	}
3332
3333	return __nh_valid_dump_req(nlh, tb, filter, cb->extack);
3334}
3335
3336struct rtm_dump_res_bucket_ctx {
3337	struct rtm_dump_nh_ctx nh;
3338	u16 bucket_index;
3339};
3340
3341static struct rtm_dump_res_bucket_ctx *
3342rtm_dump_res_bucket_ctx(struct netlink_callback *cb)
3343{
3344	struct rtm_dump_res_bucket_ctx *ctx = (void *)cb->ctx;
3345
3346	BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));
3347	return ctx;
3348}
3349
3350struct rtm_dump_nexthop_bucket_data {
3351	struct rtm_dump_res_bucket_ctx *ctx;
3352	struct nh_dump_filter filter;
3353};
3354
3355static int rtm_dump_nexthop_bucket_nh(struct sk_buff *skb,
3356				      struct netlink_callback *cb,
3357				      struct nexthop *nh,
3358				      struct rtm_dump_nexthop_bucket_data *dd)
3359{
3360	u32 portid = NETLINK_CB(cb->skb).portid;
3361	struct nhmsg *nhm = nlmsg_data(cb->nlh);
3362	struct nh_res_table *res_table;
3363	struct nh_group *nhg;
3364	u16 bucket_index;
3365	int err;
3366
3367	nhg = rtnl_dereference(nh->nh_grp);
3368	res_table = rtnl_dereference(nhg->res_table);
3369	for (bucket_index = dd->ctx->bucket_index;
3370	     bucket_index < res_table->num_nh_buckets;
3371	     bucket_index++) {
3372		struct nh_res_bucket *bucket;
3373		struct nh_grp_entry *nhge;
3374
3375		bucket = &res_table->nh_buckets[bucket_index];
3376		nhge = rtnl_dereference(bucket->nh_entry);
3377		if (nh_dump_filtered(nhge->nh, &dd->filter, nhm->nh_family))
3378			continue;
3379
3380		if (dd->filter.res_bucket_nh_id &&
3381		    dd->filter.res_bucket_nh_id != nhge->nh->id)
3382			continue;
3383
3384		dd->ctx->bucket_index = bucket_index;
3385		err = nh_fill_res_bucket(skb, nh, bucket, bucket_index,
3386					 RTM_NEWNEXTHOPBUCKET, portid,
3387					 cb->nlh->nlmsg_seq, NLM_F_MULTI,
3388					 cb->extack);
3389		if (err)
3390			return err;
3391	}
3392
3393	dd->ctx->bucket_index = 0;
3394
3395	return 0;
3396}
3397
3398static int rtm_dump_nexthop_bucket_cb(struct sk_buff *skb,
3399				      struct netlink_callback *cb,
3400				      struct nexthop *nh, void *data)
3401{
3402	struct rtm_dump_nexthop_bucket_data *dd = data;
3403	struct nh_group *nhg;
3404
3405	if (!nh->is_group)
3406		return 0;
3407
3408	nhg = rtnl_dereference(nh->nh_grp);
3409	if (!nhg->resilient)
3410		return 0;
3411
3412	return rtm_dump_nexthop_bucket_nh(skb, cb, nh, dd);
3413}
3414
3415/* rtnl */
3416static int rtm_dump_nexthop_bucket(struct sk_buff *skb,
3417				   struct netlink_callback *cb)
3418{
3419	struct rtm_dump_res_bucket_ctx *ctx = rtm_dump_res_bucket_ctx(cb);
3420	struct rtm_dump_nexthop_bucket_data dd = { .ctx = ctx };
3421	struct net *net = sock_net(skb->sk);
3422	struct nexthop *nh;
3423	int err;
3424
3425	err = nh_valid_dump_bucket_req(cb->nlh, &dd.filter, cb);
3426	if (err)
3427		return err;
3428
3429	if (dd.filter.nh_id) {
3430		nh = nexthop_find_group_resilient(net, dd.filter.nh_id,
3431						  cb->extack);
3432		if (IS_ERR(nh))
3433			return PTR_ERR(nh);
3434		err = rtm_dump_nexthop_bucket_nh(skb, cb, nh, &dd);
3435	} else {
3436		struct rb_root *root = &net->nexthop.rb_root;
3437
3438		err = rtm_dump_walk_nexthops(skb, cb, root, &ctx->nh,
3439					     &rtm_dump_nexthop_bucket_cb, &dd);
3440	}
3441
3442	if (err < 0) {
3443		if (likely(skb->len))
3444			err = skb->len;
3445	}
3446
3447	cb->seq = net->nexthop.seq;
3448	nl_dump_check_consistent(cb, nlmsg_hdr(skb));
3449	return err;
3450}
3451
3452static int nh_valid_get_bucket_req_res_bucket(struct nlattr *res,
3453					      u16 *bucket_index,
3454					      struct netlink_ext_ack *extack)
3455{
3456	struct nlattr *tb[ARRAY_SIZE(rtm_nh_res_bucket_policy_get)];
3457	int err;
3458
3459	err = nla_parse_nested(tb, ARRAY_SIZE(rtm_nh_res_bucket_policy_get) - 1,
3460			       res, rtm_nh_res_bucket_policy_get, extack);
3461	if (err < 0)
3462		return err;
3463
3464	if (!tb[NHA_RES_BUCKET_INDEX]) {
3465		NL_SET_ERR_MSG(extack, "Bucket index is missing");
3466		return -EINVAL;
3467	}
3468
3469	*bucket_index = nla_get_u16(tb[NHA_RES_BUCKET_INDEX]);
3470	return 0;
3471}
3472
3473static int nh_valid_get_bucket_req(const struct nlmsghdr *nlh,
3474				   u32 *id, u16 *bucket_index,
3475				   struct netlink_ext_ack *extack)
3476{
3477	struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_get_bucket)];
3478	int err;
3479
3480	err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
3481			  ARRAY_SIZE(rtm_nh_policy_get_bucket) - 1,
3482			  rtm_nh_policy_get_bucket, extack);
3483	if (err < 0)
3484		return err;
3485
3486	err = __nh_valid_get_del_req(nlh, tb, id, extack);
3487	if (err)
3488		return err;
3489
3490	if (!tb[NHA_RES_BUCKET]) {
3491		NL_SET_ERR_MSG(extack, "Bucket information is missing");
3492		return -EINVAL;
3493	}
3494
3495	err = nh_valid_get_bucket_req_res_bucket(tb[NHA_RES_BUCKET],
3496						 bucket_index, extack);
3497	if (err)
3498		return err;
3499
3500	return 0;
3501}
3502
3503/* rtnl */
3504static int rtm_get_nexthop_bucket(struct sk_buff *in_skb, struct nlmsghdr *nlh,
3505				  struct netlink_ext_ack *extack)
3506{
3507	struct net *net = sock_net(in_skb->sk);
3508	struct nh_res_table *res_table;
3509	struct sk_buff *skb = NULL;
3510	struct nh_group *nhg;
3511	struct nexthop *nh;
3512	u16 bucket_index;
3513	int err;
3514	u32 id;
3515
3516	err = nh_valid_get_bucket_req(nlh, &id, &bucket_index, extack);
3517	if (err)
3518		return err;
3519
3520	nh = nexthop_find_group_resilient(net, id, extack);
3521	if (IS_ERR(nh))
3522		return PTR_ERR(nh);
3523
3524	nhg = rtnl_dereference(nh->nh_grp);
3525	res_table = rtnl_dereference(nhg->res_table);
3526	if (bucket_index >= res_table->num_nh_buckets) {
3527		NL_SET_ERR_MSG(extack, "Bucket index out of bounds");
3528		return -ENOENT;
3529	}
3530
3531	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
3532	if (!skb)
3533		return -ENOBUFS;
3534
3535	err = nh_fill_res_bucket(skb, nh, &res_table->nh_buckets[bucket_index],
3536				 bucket_index, RTM_NEWNEXTHOPBUCKET,
3537				 NETLINK_CB(in_skb).portid, nlh->nlmsg_seq,
3538				 0, extack);
3539	if (err < 0) {
3540		WARN_ON(err == -EMSGSIZE);
3541		goto errout_free;
3542	}
3543
3544	return rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
3545
3546errout_free:
3547	kfree_skb(skb);
3548	return err;
3549}
3550
3551static void nexthop_sync_mtu(struct net_device *dev, u32 orig_mtu)
3552{
3553	unsigned int hash = nh_dev_hashfn(dev->ifindex);
3554	struct net *net = dev_net(dev);
3555	struct hlist_head *head = &net->nexthop.devhash[hash];
3556	struct hlist_node *n;
3557	struct nh_info *nhi;
3558
3559	hlist_for_each_entry_safe(nhi, n, head, dev_hash) {
3560		if (nhi->fib_nhc.nhc_dev == dev) {
3561			if (nhi->family == AF_INET)
3562				fib_nhc_update_mtu(&nhi->fib_nhc, dev->mtu,
3563						   orig_mtu);
3564		}
3565	}
3566}
3567
3568/* rtnl */
3569static int nh_netdev_event(struct notifier_block *this,
3570			   unsigned long event, void *ptr)
3571{
3572	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
3573	struct netdev_notifier_info_ext *info_ext;
3574
3575	switch (event) {
3576	case NETDEV_DOWN:
3577	case NETDEV_UNREGISTER:
3578		nexthop_flush_dev(dev, event);
3579		break;
3580	case NETDEV_CHANGE:
3581		if (!(dev_get_flags(dev) & (IFF_RUNNING | IFF_LOWER_UP)))
3582			nexthop_flush_dev(dev, event);
3583		break;
3584	case NETDEV_CHANGEMTU:
3585		info_ext = ptr;
3586		nexthop_sync_mtu(dev, info_ext->ext.mtu);
3587		rt_cache_flush(dev_net(dev));
3588		break;
3589	}
3590	return NOTIFY_DONE;
3591}
3592
3593static struct notifier_block nh_netdev_notifier = {
3594	.notifier_call = nh_netdev_event,
3595};
3596
3597static int nexthops_dump(struct net *net, struct notifier_block *nb,
3598			 enum nexthop_event_type event_type,
3599			 struct netlink_ext_ack *extack)
3600{
3601	struct rb_root *root = &net->nexthop.rb_root;
3602	struct rb_node *node;
3603	int err = 0;
3604
3605	for (node = rb_first(root); node; node = rb_next(node)) {
3606		struct nexthop *nh;
3607
3608		nh = rb_entry(node, struct nexthop, rb_node);
3609		err = call_nexthop_notifier(nb, net, event_type, nh, extack);
3610		if (err)
3611			break;
3612	}
3613
3614	return err;
3615}
3616
3617int register_nexthop_notifier(struct net *net, struct notifier_block *nb,
3618			      struct netlink_ext_ack *extack)
3619{
3620	int err;
3621
3622	rtnl_lock();
3623	err = nexthops_dump(net, nb, NEXTHOP_EVENT_REPLACE, extack);
3624	if (err)
3625		goto unlock;
3626	err = blocking_notifier_chain_register(&net->nexthop.notifier_chain,
3627					       nb);
3628unlock:
3629	rtnl_unlock();
3630	return err;
3631}
3632EXPORT_SYMBOL(register_nexthop_notifier);
3633
3634int unregister_nexthop_notifier(struct net *net, struct notifier_block *nb)
3635{
3636	int err;
3637
3638	rtnl_lock();
3639	err = blocking_notifier_chain_unregister(&net->nexthop.notifier_chain,
3640						 nb);
3641	if (err)
3642		goto unlock;
3643	nexthops_dump(net, nb, NEXTHOP_EVENT_DEL, NULL);
3644unlock:
3645	rtnl_unlock();
3646	return err;
3647}
3648EXPORT_SYMBOL(unregister_nexthop_notifier);
3649
3650void nexthop_set_hw_flags(struct net *net, u32 id, bool offload, bool trap)
3651{
3652	struct nexthop *nexthop;
3653
3654	rcu_read_lock();
3655
3656	nexthop = nexthop_find_by_id(net, id);
3657	if (!nexthop)
3658		goto out;
3659
3660	nexthop->nh_flags &= ~(RTNH_F_OFFLOAD | RTNH_F_TRAP);
3661	if (offload)
3662		nexthop->nh_flags |= RTNH_F_OFFLOAD;
3663	if (trap)
3664		nexthop->nh_flags |= RTNH_F_TRAP;
3665
3666out:
3667	rcu_read_unlock();
3668}
3669EXPORT_SYMBOL(nexthop_set_hw_flags);
3670
3671void nexthop_bucket_set_hw_flags(struct net *net, u32 id, u16 bucket_index,
3672				 bool offload, bool trap)
3673{
3674	struct nh_res_table *res_table;
3675	struct nh_res_bucket *bucket;
3676	struct nexthop *nexthop;
3677	struct nh_group *nhg;
3678
3679	rcu_read_lock();
3680
3681	nexthop = nexthop_find_by_id(net, id);
3682	if (!nexthop || !nexthop->is_group)
3683		goto out;
3684
3685	nhg = rcu_dereference(nexthop->nh_grp);
3686	if (!nhg->resilient)
3687		goto out;
3688
3689	if (bucket_index >= nhg->res_table->num_nh_buckets)
3690		goto out;
3691
3692	res_table = rcu_dereference(nhg->res_table);
3693	bucket = &res_table->nh_buckets[bucket_index];
3694	bucket->nh_flags &= ~(RTNH_F_OFFLOAD | RTNH_F_TRAP);
3695	if (offload)
3696		bucket->nh_flags |= RTNH_F_OFFLOAD;
3697	if (trap)
3698		bucket->nh_flags |= RTNH_F_TRAP;
3699
3700out:
3701	rcu_read_unlock();
3702}
3703EXPORT_SYMBOL(nexthop_bucket_set_hw_flags);
3704
3705void nexthop_res_grp_activity_update(struct net *net, u32 id, u16 num_buckets,
3706				     unsigned long *activity)
3707{
3708	struct nh_res_table *res_table;
3709	struct nexthop *nexthop;
3710	struct nh_group *nhg;
3711	u16 i;
3712
3713	rcu_read_lock();
3714
3715	nexthop = nexthop_find_by_id(net, id);
3716	if (!nexthop || !nexthop->is_group)
3717		goto out;
3718
3719	nhg = rcu_dereference(nexthop->nh_grp);
3720	if (!nhg->resilient)
3721		goto out;
3722
3723	/* Instead of silently ignoring some buckets, demand that the sizes
3724	 * be the same.
3725	 */
3726	res_table = rcu_dereference(nhg->res_table);
3727	if (num_buckets != res_table->num_nh_buckets)
3728		goto out;
3729
3730	for (i = 0; i < num_buckets; i++) {
3731		if (test_bit(i, activity))
3732			nh_res_bucket_set_busy(&res_table->nh_buckets[i]);
3733	}
3734
3735out:
3736	rcu_read_unlock();
3737}
3738EXPORT_SYMBOL(nexthop_res_grp_activity_update);
3739
3740static void __net_exit nexthop_net_exit_batch(struct list_head *net_list)
3741{
3742	struct net *net;
3743
3744	rtnl_lock();
3745	list_for_each_entry(net, net_list, exit_list) {
3746		flush_all_nexthops(net);
3747		kfree(net->nexthop.devhash);
3748	}
3749	rtnl_unlock();
3750}
3751
3752static int __net_init nexthop_net_init(struct net *net)
3753{
3754	size_t sz = sizeof(struct hlist_head) * NH_DEV_HASHSIZE;
3755
3756	net->nexthop.rb_root = RB_ROOT;
3757	net->nexthop.devhash = kzalloc(sz, GFP_KERNEL);
3758	if (!net->nexthop.devhash)
3759		return -ENOMEM;
3760	BLOCKING_INIT_NOTIFIER_HEAD(&net->nexthop.notifier_chain);
3761
3762	return 0;
3763}
3764
3765static struct pernet_operations nexthop_net_ops = {
3766	.init = nexthop_net_init,
3767	.exit_batch = nexthop_net_exit_batch,
3768};
3769
3770static int __init nexthop_init(void)
3771{
3772	register_pernet_subsys(&nexthop_net_ops);
3773
3774	register_netdevice_notifier(&nh_netdev_notifier);
3775
3776	rtnl_register(PF_UNSPEC, RTM_NEWNEXTHOP, rtm_new_nexthop, NULL, 0);
3777	rtnl_register(PF_UNSPEC, RTM_DELNEXTHOP, rtm_del_nexthop, NULL, 0);
3778	rtnl_register(PF_UNSPEC, RTM_GETNEXTHOP, rtm_get_nexthop,
3779		      rtm_dump_nexthop, 0);
3780
3781	rtnl_register(PF_INET, RTM_NEWNEXTHOP, rtm_new_nexthop, NULL, 0);
3782	rtnl_register(PF_INET, RTM_GETNEXTHOP, NULL, rtm_dump_nexthop, 0);
3783
3784	rtnl_register(PF_INET6, RTM_NEWNEXTHOP, rtm_new_nexthop, NULL, 0);
3785	rtnl_register(PF_INET6, RTM_GETNEXTHOP, NULL, rtm_dump_nexthop, 0);
3786
3787	rtnl_register(PF_UNSPEC, RTM_GETNEXTHOPBUCKET, rtm_get_nexthop_bucket,
3788		      rtm_dump_nexthop_bucket, 0);
3789
3790	return 0;
3791}
3792subsys_initcall(nexthop_init);