cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

af_mpls.c (65058B)


      1// SPDX-License-Identifier: GPL-2.0-only
      2#include <linux/types.h>
      3#include <linux/skbuff.h>
      4#include <linux/socket.h>
      5#include <linux/sysctl.h>
      6#include <linux/net.h>
      7#include <linux/module.h>
      8#include <linux/if_arp.h>
      9#include <linux/ipv6.h>
     10#include <linux/mpls.h>
     11#include <linux/netconf.h>
     12#include <linux/nospec.h>
     13#include <linux/vmalloc.h>
     14#include <linux/percpu.h>
     15#include <net/ip.h>
     16#include <net/dst.h>
     17#include <net/sock.h>
     18#include <net/arp.h>
     19#include <net/ip_fib.h>
     20#include <net/netevent.h>
     21#include <net/ip_tunnels.h>
     22#include <net/netns/generic.h>
     23#if IS_ENABLED(CONFIG_IPV6)
     24#include <net/ipv6.h>
     25#endif
     26#include <net/ipv6_stubs.h>
     27#include <net/rtnh.h>
     28#include "internal.h"
     29
     30/* max memory we will use for mpls_route */
     31#define MAX_MPLS_ROUTE_MEM	4096
     32
     33/* Maximum number of labels to look ahead at when selecting a path of
     34 * a multipath route
     35 */
     36#define MAX_MP_SELECT_LABELS 4
     37
     38#define MPLS_NEIGH_TABLE_UNSPEC (NEIGH_LINK_TABLE + 1)
     39
     40static int label_limit = (1 << 20) - 1;
     41static int ttl_max = 255;
     42
     43#if IS_ENABLED(CONFIG_NET_IP_TUNNEL)
     44static size_t ipgre_mpls_encap_hlen(struct ip_tunnel_encap *e)
     45{
     46	return sizeof(struct mpls_shim_hdr);
     47}
     48
     49static const struct ip_tunnel_encap_ops mpls_iptun_ops = {
     50	.encap_hlen	= ipgre_mpls_encap_hlen,
     51};
     52
     53static int ipgre_tunnel_encap_add_mpls_ops(void)
     54{
     55	return ip_tunnel_encap_add_ops(&mpls_iptun_ops, TUNNEL_ENCAP_MPLS);
     56}
     57
     58static void ipgre_tunnel_encap_del_mpls_ops(void)
     59{
     60	ip_tunnel_encap_del_ops(&mpls_iptun_ops, TUNNEL_ENCAP_MPLS);
     61}
     62#else
     63static int ipgre_tunnel_encap_add_mpls_ops(void)
     64{
     65	return 0;
     66}
     67
     68static void ipgre_tunnel_encap_del_mpls_ops(void)
     69{
     70}
     71#endif
     72
     73static void rtmsg_lfib(int event, u32 label, struct mpls_route *rt,
     74		       struct nlmsghdr *nlh, struct net *net, u32 portid,
     75		       unsigned int nlm_flags);
     76
     77static struct mpls_route *mpls_route_input_rcu(struct net *net, unsigned index)
     78{
     79	struct mpls_route *rt = NULL;
     80
     81	if (index < net->mpls.platform_labels) {
     82		struct mpls_route __rcu **platform_label =
     83			rcu_dereference(net->mpls.platform_label);
     84		rt = rcu_dereference(platform_label[index]);
     85	}
     86	return rt;
     87}
     88
     89bool mpls_output_possible(const struct net_device *dev)
     90{
     91	return dev && (dev->flags & IFF_UP) && netif_carrier_ok(dev);
     92}
     93EXPORT_SYMBOL_GPL(mpls_output_possible);
     94
     95static u8 *__mpls_nh_via(struct mpls_route *rt, struct mpls_nh *nh)
     96{
     97	return (u8 *)nh + rt->rt_via_offset;
     98}
     99
    100static const u8 *mpls_nh_via(const struct mpls_route *rt,
    101			     const struct mpls_nh *nh)
    102{
    103	return __mpls_nh_via((struct mpls_route *)rt, (struct mpls_nh *)nh);
    104}
    105
    106static unsigned int mpls_nh_header_size(const struct mpls_nh *nh)
    107{
    108	/* The size of the layer 2.5 labels to be added for this route */
    109	return nh->nh_labels * sizeof(struct mpls_shim_hdr);
    110}
    111
    112unsigned int mpls_dev_mtu(const struct net_device *dev)
    113{
    114	/* The amount of data the layer 2 frame can hold */
    115	return dev->mtu;
    116}
    117EXPORT_SYMBOL_GPL(mpls_dev_mtu);
    118
    119bool mpls_pkt_too_big(const struct sk_buff *skb, unsigned int mtu)
    120{
    121	if (skb->len <= mtu)
    122		return false;
    123
    124	if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu))
    125		return false;
    126
    127	return true;
    128}
    129EXPORT_SYMBOL_GPL(mpls_pkt_too_big);
    130
    131void mpls_stats_inc_outucastpkts(struct net_device *dev,
    132				 const struct sk_buff *skb)
    133{
    134	struct mpls_dev *mdev;
    135
    136	if (skb->protocol == htons(ETH_P_MPLS_UC)) {
    137		mdev = mpls_dev_get(dev);
    138		if (mdev)
    139			MPLS_INC_STATS_LEN(mdev, skb->len,
    140					   tx_packets,
    141					   tx_bytes);
    142	} else if (skb->protocol == htons(ETH_P_IP)) {
    143		IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUT, skb->len);
    144#if IS_ENABLED(CONFIG_IPV6)
    145	} else if (skb->protocol == htons(ETH_P_IPV6)) {
    146		struct inet6_dev *in6dev = __in6_dev_get(dev);
    147
    148		if (in6dev)
    149			IP6_UPD_PO_STATS(dev_net(dev), in6dev,
    150					 IPSTATS_MIB_OUT, skb->len);
    151#endif
    152	}
    153}
    154EXPORT_SYMBOL_GPL(mpls_stats_inc_outucastpkts);
    155
    156static u32 mpls_multipath_hash(struct mpls_route *rt, struct sk_buff *skb)
    157{
    158	struct mpls_entry_decoded dec;
    159	unsigned int mpls_hdr_len = 0;
    160	struct mpls_shim_hdr *hdr;
    161	bool eli_seen = false;
    162	int label_index;
    163	u32 hash = 0;
    164
    165	for (label_index = 0; label_index < MAX_MP_SELECT_LABELS;
    166	     label_index++) {
    167		mpls_hdr_len += sizeof(*hdr);
    168		if (!pskb_may_pull(skb, mpls_hdr_len))
    169			break;
    170
    171		/* Read and decode the current label */
    172		hdr = mpls_hdr(skb) + label_index;
    173		dec = mpls_entry_decode(hdr);
    174
    175		/* RFC6790 - reserved labels MUST NOT be used as keys
    176		 * for the load-balancing function
    177		 */
    178		if (likely(dec.label >= MPLS_LABEL_FIRST_UNRESERVED)) {
    179			hash = jhash_1word(dec.label, hash);
    180
    181			/* The entropy label follows the entropy label
    182			 * indicator, so this means that the entropy
    183			 * label was just added to the hash - no need to
    184			 * go any deeper either in the label stack or in the
    185			 * payload
    186			 */
    187			if (eli_seen)
    188				break;
    189		} else if (dec.label == MPLS_LABEL_ENTROPY) {
    190			eli_seen = true;
    191		}
    192
    193		if (!dec.bos)
    194			continue;
    195
    196		/* found bottom label; does skb have room for a header? */
    197		if (pskb_may_pull(skb, mpls_hdr_len + sizeof(struct iphdr))) {
    198			const struct iphdr *v4hdr;
    199
    200			v4hdr = (const struct iphdr *)(hdr + 1);
    201			if (v4hdr->version == 4) {
    202				hash = jhash_3words(ntohl(v4hdr->saddr),
    203						    ntohl(v4hdr->daddr),
    204						    v4hdr->protocol, hash);
    205			} else if (v4hdr->version == 6 &&
    206				   pskb_may_pull(skb, mpls_hdr_len +
    207						 sizeof(struct ipv6hdr))) {
    208				const struct ipv6hdr *v6hdr;
    209
    210				v6hdr = (const struct ipv6hdr *)(hdr + 1);
    211				hash = __ipv6_addr_jhash(&v6hdr->saddr, hash);
    212				hash = __ipv6_addr_jhash(&v6hdr->daddr, hash);
    213				hash = jhash_1word(v6hdr->nexthdr, hash);
    214			}
    215		}
    216
    217		break;
    218	}
    219
    220	return hash;
    221}
    222
    223static struct mpls_nh *mpls_get_nexthop(struct mpls_route *rt, u8 index)
    224{
    225	return (struct mpls_nh *)((u8 *)rt->rt_nh + index * rt->rt_nh_size);
    226}
    227
    228/* number of alive nexthops (rt->rt_nhn_alive) and the flags for
    229 * a next hop (nh->nh_flags) are modified by netdev event handlers.
    230 * Since those fields can change at any moment, use READ_ONCE to
    231 * access both.
    232 */
    233static const struct mpls_nh *mpls_select_multipath(struct mpls_route *rt,
    234						   struct sk_buff *skb)
    235{
    236	u32 hash = 0;
    237	int nh_index = 0;
    238	int n = 0;
    239	u8 alive;
    240
    241	/* No need to look further into packet if there's only
    242	 * one path
    243	 */
    244	if (rt->rt_nhn == 1)
    245		return rt->rt_nh;
    246
    247	alive = READ_ONCE(rt->rt_nhn_alive);
    248	if (alive == 0)
    249		return NULL;
    250
    251	hash = mpls_multipath_hash(rt, skb);
    252	nh_index = hash % alive;
    253	if (alive == rt->rt_nhn)
    254		goto out;
    255	for_nexthops(rt) {
    256		unsigned int nh_flags = READ_ONCE(nh->nh_flags);
    257
    258		if (nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN))
    259			continue;
    260		if (n == nh_index)
    261			return nh;
    262		n++;
    263	} endfor_nexthops(rt);
    264
    265out:
    266	return mpls_get_nexthop(rt, nh_index);
    267}
    268
    269static bool mpls_egress(struct net *net, struct mpls_route *rt,
    270			struct sk_buff *skb, struct mpls_entry_decoded dec)
    271{
    272	enum mpls_payload_type payload_type;
    273	bool success = false;
    274
    275	/* The IPv4 code below accesses through the IPv4 header
    276	 * checksum, which is 12 bytes into the packet.
    277	 * The IPv6 code below accesses through the IPv6 hop limit
    278	 * which is 8 bytes into the packet.
    279	 *
    280	 * For all supported cases there should always be at least 12
    281	 * bytes of packet data present.  The IPv4 header is 20 bytes
    282	 * without options and the IPv6 header is always 40 bytes
    283	 * long.
    284	 */
    285	if (!pskb_may_pull(skb, 12))
    286		return false;
    287
    288	payload_type = rt->rt_payload_type;
    289	if (payload_type == MPT_UNSPEC)
    290		payload_type = ip_hdr(skb)->version;
    291
    292	switch (payload_type) {
    293	case MPT_IPV4: {
    294		struct iphdr *hdr4 = ip_hdr(skb);
    295		u8 new_ttl;
    296		skb->protocol = htons(ETH_P_IP);
    297
    298		/* If propagating TTL, take the decremented TTL from
    299		 * the incoming MPLS header, otherwise decrement the
    300		 * TTL, but only if not 0 to avoid underflow.
    301		 */
    302		if (rt->rt_ttl_propagate == MPLS_TTL_PROP_ENABLED ||
    303		    (rt->rt_ttl_propagate == MPLS_TTL_PROP_DEFAULT &&
    304		     net->mpls.ip_ttl_propagate))
    305			new_ttl = dec.ttl;
    306		else
    307			new_ttl = hdr4->ttl ? hdr4->ttl - 1 : 0;
    308
    309		csum_replace2(&hdr4->check,
    310			      htons(hdr4->ttl << 8),
    311			      htons(new_ttl << 8));
    312		hdr4->ttl = new_ttl;
    313		success = true;
    314		break;
    315	}
    316	case MPT_IPV6: {
    317		struct ipv6hdr *hdr6 = ipv6_hdr(skb);
    318		skb->protocol = htons(ETH_P_IPV6);
    319
    320		/* If propagating TTL, take the decremented TTL from
    321		 * the incoming MPLS header, otherwise decrement the
    322		 * hop limit, but only if not 0 to avoid underflow.
    323		 */
    324		if (rt->rt_ttl_propagate == MPLS_TTL_PROP_ENABLED ||
    325		    (rt->rt_ttl_propagate == MPLS_TTL_PROP_DEFAULT &&
    326		     net->mpls.ip_ttl_propagate))
    327			hdr6->hop_limit = dec.ttl;
    328		else if (hdr6->hop_limit)
    329			hdr6->hop_limit = hdr6->hop_limit - 1;
    330		success = true;
    331		break;
    332	}
    333	case MPT_UNSPEC:
    334		/* Should have decided which protocol it is by now */
    335		break;
    336	}
    337
    338	return success;
    339}
    340
    341static int mpls_forward(struct sk_buff *skb, struct net_device *dev,
    342			struct packet_type *pt, struct net_device *orig_dev)
    343{
    344	struct net *net = dev_net(dev);
    345	struct mpls_shim_hdr *hdr;
    346	const struct mpls_nh *nh;
    347	struct mpls_route *rt;
    348	struct mpls_entry_decoded dec;
    349	struct net_device *out_dev;
    350	struct mpls_dev *out_mdev;
    351	struct mpls_dev *mdev;
    352	unsigned int hh_len;
    353	unsigned int new_header_size;
    354	unsigned int mtu;
    355	int err;
    356
    357	/* Careful this entire function runs inside of an rcu critical section */
    358
    359	mdev = mpls_dev_get(dev);
    360	if (!mdev)
    361		goto drop;
    362
    363	MPLS_INC_STATS_LEN(mdev, skb->len, rx_packets,
    364			   rx_bytes);
    365
    366	if (!mdev->input_enabled) {
    367		MPLS_INC_STATS(mdev, rx_dropped);
    368		goto drop;
    369	}
    370
    371	if (skb->pkt_type != PACKET_HOST)
    372		goto err;
    373
    374	if ((skb = skb_share_check(skb, GFP_ATOMIC)) == NULL)
    375		goto err;
    376
    377	if (!pskb_may_pull(skb, sizeof(*hdr)))
    378		goto err;
    379
    380	skb_dst_drop(skb);
    381
    382	/* Read and decode the label */
    383	hdr = mpls_hdr(skb);
    384	dec = mpls_entry_decode(hdr);
    385
    386	rt = mpls_route_input_rcu(net, dec.label);
    387	if (!rt) {
    388		MPLS_INC_STATS(mdev, rx_noroute);
    389		goto drop;
    390	}
    391
    392	nh = mpls_select_multipath(rt, skb);
    393	if (!nh)
    394		goto err;
    395
    396	/* Pop the label */
    397	skb_pull(skb, sizeof(*hdr));
    398	skb_reset_network_header(skb);
    399
    400	skb_orphan(skb);
    401
    402	if (skb_warn_if_lro(skb))
    403		goto err;
    404
    405	skb_forward_csum(skb);
    406
    407	/* Verify ttl is valid */
    408	if (dec.ttl <= 1)
    409		goto err;
    410
    411	/* Find the output device */
    412	out_dev = nh->nh_dev;
    413	if (!mpls_output_possible(out_dev))
    414		goto tx_err;
    415
    416	/* Verify the destination can hold the packet */
    417	new_header_size = mpls_nh_header_size(nh);
    418	mtu = mpls_dev_mtu(out_dev);
    419	if (mpls_pkt_too_big(skb, mtu - new_header_size))
    420		goto tx_err;
    421
    422	hh_len = LL_RESERVED_SPACE(out_dev);
    423	if (!out_dev->header_ops)
    424		hh_len = 0;
    425
    426	/* Ensure there is enough space for the headers in the skb */
    427	if (skb_cow(skb, hh_len + new_header_size))
    428		goto tx_err;
    429
    430	skb->dev = out_dev;
    431	skb->protocol = htons(ETH_P_MPLS_UC);
    432
    433	dec.ttl -= 1;
    434	if (unlikely(!new_header_size && dec.bos)) {
    435		/* Penultimate hop popping */
    436		if (!mpls_egress(dev_net(out_dev), rt, skb, dec))
    437			goto err;
    438	} else {
    439		bool bos;
    440		int i;
    441		skb_push(skb, new_header_size);
    442		skb_reset_network_header(skb);
    443		/* Push the new labels */
    444		hdr = mpls_hdr(skb);
    445		bos = dec.bos;
    446		for (i = nh->nh_labels - 1; i >= 0; i--) {
    447			hdr[i] = mpls_entry_encode(nh->nh_label[i],
    448						   dec.ttl, 0, bos);
    449			bos = false;
    450		}
    451	}
    452
    453	mpls_stats_inc_outucastpkts(out_dev, skb);
    454
    455	/* If via wasn't specified then send out using device address */
    456	if (nh->nh_via_table == MPLS_NEIGH_TABLE_UNSPEC)
    457		err = neigh_xmit(NEIGH_LINK_TABLE, out_dev,
    458				 out_dev->dev_addr, skb);
    459	else
    460		err = neigh_xmit(nh->nh_via_table, out_dev,
    461				 mpls_nh_via(rt, nh), skb);
    462	if (err)
    463		net_dbg_ratelimited("%s: packet transmission failed: %d\n",
    464				    __func__, err);
    465	return 0;
    466
    467tx_err:
    468	out_mdev = out_dev ? mpls_dev_get(out_dev) : NULL;
    469	if (out_mdev)
    470		MPLS_INC_STATS(out_mdev, tx_errors);
    471	goto drop;
    472err:
    473	MPLS_INC_STATS(mdev, rx_errors);
    474drop:
    475	kfree_skb(skb);
    476	return NET_RX_DROP;
    477}
    478
    479static struct packet_type mpls_packet_type __read_mostly = {
    480	.type = cpu_to_be16(ETH_P_MPLS_UC),
    481	.func = mpls_forward,
    482};
    483
    484static const struct nla_policy rtm_mpls_policy[RTA_MAX+1] = {
    485	[RTA_DST]		= { .type = NLA_U32 },
    486	[RTA_OIF]		= { .type = NLA_U32 },
    487	[RTA_TTL_PROPAGATE]	= { .type = NLA_U8 },
    488};
    489
    490struct mpls_route_config {
    491	u32			rc_protocol;
    492	u32			rc_ifindex;
    493	u8			rc_via_table;
    494	u8			rc_via_alen;
    495	u8			rc_via[MAX_VIA_ALEN];
    496	u32			rc_label;
    497	u8			rc_ttl_propagate;
    498	u8			rc_output_labels;
    499	u32			rc_output_label[MAX_NEW_LABELS];
    500	u32			rc_nlflags;
    501	enum mpls_payload_type	rc_payload_type;
    502	struct nl_info		rc_nlinfo;
    503	struct rtnexthop	*rc_mp;
    504	int			rc_mp_len;
    505};
    506
    507/* all nexthops within a route have the same size based on max
    508 * number of labels and max via length for a hop
    509 */
    510static struct mpls_route *mpls_rt_alloc(u8 num_nh, u8 max_alen, u8 max_labels)
    511{
    512	u8 nh_size = MPLS_NH_SIZE(max_labels, max_alen);
    513	struct mpls_route *rt;
    514	size_t size;
    515
    516	size = sizeof(*rt) + num_nh * nh_size;
    517	if (size > MAX_MPLS_ROUTE_MEM)
    518		return ERR_PTR(-EINVAL);
    519
    520	rt = kzalloc(size, GFP_KERNEL);
    521	if (!rt)
    522		return ERR_PTR(-ENOMEM);
    523
    524	rt->rt_nhn = num_nh;
    525	rt->rt_nhn_alive = num_nh;
    526	rt->rt_nh_size = nh_size;
    527	rt->rt_via_offset = MPLS_NH_VIA_OFF(max_labels);
    528
    529	return rt;
    530}
    531
    532static void mpls_rt_free(struct mpls_route *rt)
    533{
    534	if (rt)
    535		kfree_rcu(rt, rt_rcu);
    536}
    537
    538static void mpls_notify_route(struct net *net, unsigned index,
    539			      struct mpls_route *old, struct mpls_route *new,
    540			      const struct nl_info *info)
    541{
    542	struct nlmsghdr *nlh = info ? info->nlh : NULL;
    543	unsigned portid = info ? info->portid : 0;
    544	int event = new ? RTM_NEWROUTE : RTM_DELROUTE;
    545	struct mpls_route *rt = new ? new : old;
    546	unsigned nlm_flags = (old && new) ? NLM_F_REPLACE : 0;
    547	/* Ignore reserved labels for now */
    548	if (rt && (index >= MPLS_LABEL_FIRST_UNRESERVED))
    549		rtmsg_lfib(event, index, rt, nlh, net, portid, nlm_flags);
    550}
    551
    552static void mpls_route_update(struct net *net, unsigned index,
    553			      struct mpls_route *new,
    554			      const struct nl_info *info)
    555{
    556	struct mpls_route __rcu **platform_label;
    557	struct mpls_route *rt;
    558
    559	ASSERT_RTNL();
    560
    561	platform_label = rtnl_dereference(net->mpls.platform_label);
    562	rt = rtnl_dereference(platform_label[index]);
    563	rcu_assign_pointer(platform_label[index], new);
    564
    565	mpls_notify_route(net, index, rt, new, info);
    566
    567	/* If we removed a route free it now */
    568	mpls_rt_free(rt);
    569}
    570
    571static unsigned find_free_label(struct net *net)
    572{
    573	struct mpls_route __rcu **platform_label;
    574	size_t platform_labels;
    575	unsigned index;
    576
    577	platform_label = rtnl_dereference(net->mpls.platform_label);
    578	platform_labels = net->mpls.platform_labels;
    579	for (index = MPLS_LABEL_FIRST_UNRESERVED; index < platform_labels;
    580	     index++) {
    581		if (!rtnl_dereference(platform_label[index]))
    582			return index;
    583	}
    584	return LABEL_NOT_SPECIFIED;
    585}
    586
    587#if IS_ENABLED(CONFIG_INET)
    588static struct net_device *inet_fib_lookup_dev(struct net *net,
    589					      const void *addr)
    590{
    591	struct net_device *dev;
    592	struct rtable *rt;
    593	struct in_addr daddr;
    594
    595	memcpy(&daddr, addr, sizeof(struct in_addr));
    596	rt = ip_route_output(net, daddr.s_addr, 0, 0, 0);
    597	if (IS_ERR(rt))
    598		return ERR_CAST(rt);
    599
    600	dev = rt->dst.dev;
    601	dev_hold(dev);
    602
    603	ip_rt_put(rt);
    604
    605	return dev;
    606}
    607#else
    608static struct net_device *inet_fib_lookup_dev(struct net *net,
    609					      const void *addr)
    610{
    611	return ERR_PTR(-EAFNOSUPPORT);
    612}
    613#endif
    614
    615#if IS_ENABLED(CONFIG_IPV6)
    616static struct net_device *inet6_fib_lookup_dev(struct net *net,
    617					       const void *addr)
    618{
    619	struct net_device *dev;
    620	struct dst_entry *dst;
    621	struct flowi6 fl6;
    622
    623	if (!ipv6_stub)
    624		return ERR_PTR(-EAFNOSUPPORT);
    625
    626	memset(&fl6, 0, sizeof(fl6));
    627	memcpy(&fl6.daddr, addr, sizeof(struct in6_addr));
    628	dst = ipv6_stub->ipv6_dst_lookup_flow(net, NULL, &fl6, NULL);
    629	if (IS_ERR(dst))
    630		return ERR_CAST(dst);
    631
    632	dev = dst->dev;
    633	dev_hold(dev);
    634	dst_release(dst);
    635
    636	return dev;
    637}
    638#else
    639static struct net_device *inet6_fib_lookup_dev(struct net *net,
    640					       const void *addr)
    641{
    642	return ERR_PTR(-EAFNOSUPPORT);
    643}
    644#endif
    645
    646static struct net_device *find_outdev(struct net *net,
    647				      struct mpls_route *rt,
    648				      struct mpls_nh *nh, int oif)
    649{
    650	struct net_device *dev = NULL;
    651
    652	if (!oif) {
    653		switch (nh->nh_via_table) {
    654		case NEIGH_ARP_TABLE:
    655			dev = inet_fib_lookup_dev(net, mpls_nh_via(rt, nh));
    656			break;
    657		case NEIGH_ND_TABLE:
    658			dev = inet6_fib_lookup_dev(net, mpls_nh_via(rt, nh));
    659			break;
    660		case NEIGH_LINK_TABLE:
    661			break;
    662		}
    663	} else {
    664		dev = dev_get_by_index(net, oif);
    665	}
    666
    667	if (!dev)
    668		return ERR_PTR(-ENODEV);
    669
    670	if (IS_ERR(dev))
    671		return dev;
    672
    673	/* The caller is holding rtnl anyways, so release the dev reference */
    674	dev_put(dev);
    675
    676	return dev;
    677}
    678
    679static int mpls_nh_assign_dev(struct net *net, struct mpls_route *rt,
    680			      struct mpls_nh *nh, int oif)
    681{
    682	struct net_device *dev = NULL;
    683	int err = -ENODEV;
    684
    685	dev = find_outdev(net, rt, nh, oif);
    686	if (IS_ERR(dev)) {
    687		err = PTR_ERR(dev);
    688		dev = NULL;
    689		goto errout;
    690	}
    691
    692	/* Ensure this is a supported device */
    693	err = -EINVAL;
    694	if (!mpls_dev_get(dev))
    695		goto errout;
    696
    697	if ((nh->nh_via_table == NEIGH_LINK_TABLE) &&
    698	    (dev->addr_len != nh->nh_via_alen))
    699		goto errout;
    700
    701	nh->nh_dev = dev;
    702
    703	if (!(dev->flags & IFF_UP)) {
    704		nh->nh_flags |= RTNH_F_DEAD;
    705	} else {
    706		unsigned int flags;
    707
    708		flags = dev_get_flags(dev);
    709		if (!(flags & (IFF_RUNNING | IFF_LOWER_UP)))
    710			nh->nh_flags |= RTNH_F_LINKDOWN;
    711	}
    712
    713	return 0;
    714
    715errout:
    716	return err;
    717}
    718
    719static int nla_get_via(const struct nlattr *nla, u8 *via_alen, u8 *via_table,
    720		       u8 via_addr[], struct netlink_ext_ack *extack)
    721{
    722	struct rtvia *via = nla_data(nla);
    723	int err = -EINVAL;
    724	int alen;
    725
    726	if (nla_len(nla) < offsetof(struct rtvia, rtvia_addr)) {
    727		NL_SET_ERR_MSG_ATTR(extack, nla,
    728				    "Invalid attribute length for RTA_VIA");
    729		goto errout;
    730	}
    731	alen = nla_len(nla) -
    732			offsetof(struct rtvia, rtvia_addr);
    733	if (alen > MAX_VIA_ALEN) {
    734		NL_SET_ERR_MSG_ATTR(extack, nla,
    735				    "Invalid address length for RTA_VIA");
    736		goto errout;
    737	}
    738
    739	/* Validate the address family */
    740	switch (via->rtvia_family) {
    741	case AF_PACKET:
    742		*via_table = NEIGH_LINK_TABLE;
    743		break;
    744	case AF_INET:
    745		*via_table = NEIGH_ARP_TABLE;
    746		if (alen != 4)
    747			goto errout;
    748		break;
    749	case AF_INET6:
    750		*via_table = NEIGH_ND_TABLE;
    751		if (alen != 16)
    752			goto errout;
    753		break;
    754	default:
    755		/* Unsupported address family */
    756		goto errout;
    757	}
    758
    759	memcpy(via_addr, via->rtvia_addr, alen);
    760	*via_alen = alen;
    761	err = 0;
    762
    763errout:
    764	return err;
    765}
    766
    767static int mpls_nh_build_from_cfg(struct mpls_route_config *cfg,
    768				  struct mpls_route *rt)
    769{
    770	struct net *net = cfg->rc_nlinfo.nl_net;
    771	struct mpls_nh *nh = rt->rt_nh;
    772	int err;
    773	int i;
    774
    775	if (!nh)
    776		return -ENOMEM;
    777
    778	nh->nh_labels = cfg->rc_output_labels;
    779	for (i = 0; i < nh->nh_labels; i++)
    780		nh->nh_label[i] = cfg->rc_output_label[i];
    781
    782	nh->nh_via_table = cfg->rc_via_table;
    783	memcpy(__mpls_nh_via(rt, nh), cfg->rc_via, cfg->rc_via_alen);
    784	nh->nh_via_alen = cfg->rc_via_alen;
    785
    786	err = mpls_nh_assign_dev(net, rt, nh, cfg->rc_ifindex);
    787	if (err)
    788		goto errout;
    789
    790	if (nh->nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN))
    791		rt->rt_nhn_alive--;
    792
    793	return 0;
    794
    795errout:
    796	return err;
    797}
    798
    799static int mpls_nh_build(struct net *net, struct mpls_route *rt,
    800			 struct mpls_nh *nh, int oif, struct nlattr *via,
    801			 struct nlattr *newdst, u8 max_labels,
    802			 struct netlink_ext_ack *extack)
    803{
    804	int err = -ENOMEM;
    805
    806	if (!nh)
    807		goto errout;
    808
    809	if (newdst) {
    810		err = nla_get_labels(newdst, max_labels, &nh->nh_labels,
    811				     nh->nh_label, extack);
    812		if (err)
    813			goto errout;
    814	}
    815
    816	if (via) {
    817		err = nla_get_via(via, &nh->nh_via_alen, &nh->nh_via_table,
    818				  __mpls_nh_via(rt, nh), extack);
    819		if (err)
    820			goto errout;
    821	} else {
    822		nh->nh_via_table = MPLS_NEIGH_TABLE_UNSPEC;
    823	}
    824
    825	err = mpls_nh_assign_dev(net, rt, nh, oif);
    826	if (err)
    827		goto errout;
    828
    829	return 0;
    830
    831errout:
    832	return err;
    833}
    834
    835static u8 mpls_count_nexthops(struct rtnexthop *rtnh, int len,
    836			      u8 cfg_via_alen, u8 *max_via_alen,
    837			      u8 *max_labels)
    838{
    839	int remaining = len;
    840	u8 nhs = 0;
    841
    842	*max_via_alen = 0;
    843	*max_labels = 0;
    844
    845	while (rtnh_ok(rtnh, remaining)) {
    846		struct nlattr *nla, *attrs = rtnh_attrs(rtnh);
    847		int attrlen;
    848		u8 n_labels = 0;
    849
    850		attrlen = rtnh_attrlen(rtnh);
    851		nla = nla_find(attrs, attrlen, RTA_VIA);
    852		if (nla && nla_len(nla) >=
    853		    offsetof(struct rtvia, rtvia_addr)) {
    854			int via_alen = nla_len(nla) -
    855				offsetof(struct rtvia, rtvia_addr);
    856
    857			if (via_alen <= MAX_VIA_ALEN)
    858				*max_via_alen = max_t(u16, *max_via_alen,
    859						      via_alen);
    860		}
    861
    862		nla = nla_find(attrs, attrlen, RTA_NEWDST);
    863		if (nla &&
    864		    nla_get_labels(nla, MAX_NEW_LABELS, &n_labels,
    865				   NULL, NULL) != 0)
    866			return 0;
    867
    868		*max_labels = max_t(u8, *max_labels, n_labels);
    869
    870		/* number of nexthops is tracked by a u8.
    871		 * Check for overflow.
    872		 */
    873		if (nhs == 255)
    874			return 0;
    875		nhs++;
    876
    877		rtnh = rtnh_next(rtnh, &remaining);
    878	}
    879
    880	/* leftover implies invalid nexthop configuration, discard it */
    881	return remaining > 0 ? 0 : nhs;
    882}
    883
    884static int mpls_nh_build_multi(struct mpls_route_config *cfg,
    885			       struct mpls_route *rt, u8 max_labels,
    886			       struct netlink_ext_ack *extack)
    887{
    888	struct rtnexthop *rtnh = cfg->rc_mp;
    889	struct nlattr *nla_via, *nla_newdst;
    890	int remaining = cfg->rc_mp_len;
    891	int err = 0;
    892	u8 nhs = 0;
    893
    894	change_nexthops(rt) {
    895		int attrlen;
    896
    897		nla_via = NULL;
    898		nla_newdst = NULL;
    899
    900		err = -EINVAL;
    901		if (!rtnh_ok(rtnh, remaining))
    902			goto errout;
    903
    904		/* neither weighted multipath nor any flags
    905		 * are supported
    906		 */
    907		if (rtnh->rtnh_hops || rtnh->rtnh_flags)
    908			goto errout;
    909
    910		attrlen = rtnh_attrlen(rtnh);
    911		if (attrlen > 0) {
    912			struct nlattr *attrs = rtnh_attrs(rtnh);
    913
    914			nla_via = nla_find(attrs, attrlen, RTA_VIA);
    915			nla_newdst = nla_find(attrs, attrlen, RTA_NEWDST);
    916		}
    917
    918		err = mpls_nh_build(cfg->rc_nlinfo.nl_net, rt, nh,
    919				    rtnh->rtnh_ifindex, nla_via, nla_newdst,
    920				    max_labels, extack);
    921		if (err)
    922			goto errout;
    923
    924		if (nh->nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN))
    925			rt->rt_nhn_alive--;
    926
    927		rtnh = rtnh_next(rtnh, &remaining);
    928		nhs++;
    929	} endfor_nexthops(rt);
    930
    931	rt->rt_nhn = nhs;
    932
    933	return 0;
    934
    935errout:
    936	return err;
    937}
    938
    939static bool mpls_label_ok(struct net *net, unsigned int *index,
    940			  struct netlink_ext_ack *extack)
    941{
    942	bool is_ok = true;
    943
    944	/* Reserved labels may not be set */
    945	if (*index < MPLS_LABEL_FIRST_UNRESERVED) {
    946		NL_SET_ERR_MSG(extack,
    947			       "Invalid label - must be MPLS_LABEL_FIRST_UNRESERVED or higher");
    948		is_ok = false;
    949	}
    950
    951	/* The full 20 bit range may not be supported. */
    952	if (is_ok && *index >= net->mpls.platform_labels) {
    953		NL_SET_ERR_MSG(extack,
    954			       "Label >= configured maximum in platform_labels");
    955		is_ok = false;
    956	}
    957
    958	*index = array_index_nospec(*index, net->mpls.platform_labels);
    959	return is_ok;
    960}
    961
    962static int mpls_route_add(struct mpls_route_config *cfg,
    963			  struct netlink_ext_ack *extack)
    964{
    965	struct mpls_route __rcu **platform_label;
    966	struct net *net = cfg->rc_nlinfo.nl_net;
    967	struct mpls_route *rt, *old;
    968	int err = -EINVAL;
    969	u8 max_via_alen;
    970	unsigned index;
    971	u8 max_labels;
    972	u8 nhs;
    973
    974	index = cfg->rc_label;
    975
    976	/* If a label was not specified during insert pick one */
    977	if ((index == LABEL_NOT_SPECIFIED) &&
    978	    (cfg->rc_nlflags & NLM_F_CREATE)) {
    979		index = find_free_label(net);
    980	}
    981
    982	if (!mpls_label_ok(net, &index, extack))
    983		goto errout;
    984
    985	/* Append makes no sense with mpls */
    986	err = -EOPNOTSUPP;
    987	if (cfg->rc_nlflags & NLM_F_APPEND) {
    988		NL_SET_ERR_MSG(extack, "MPLS does not support route append");
    989		goto errout;
    990	}
    991
    992	err = -EEXIST;
    993	platform_label = rtnl_dereference(net->mpls.platform_label);
    994	old = rtnl_dereference(platform_label[index]);
    995	if ((cfg->rc_nlflags & NLM_F_EXCL) && old)
    996		goto errout;
    997
    998	err = -EEXIST;
    999	if (!(cfg->rc_nlflags & NLM_F_REPLACE) && old)
   1000		goto errout;
   1001
   1002	err = -ENOENT;
   1003	if (!(cfg->rc_nlflags & NLM_F_CREATE) && !old)
   1004		goto errout;
   1005
   1006	err = -EINVAL;
   1007	if (cfg->rc_mp) {
   1008		nhs = mpls_count_nexthops(cfg->rc_mp, cfg->rc_mp_len,
   1009					  cfg->rc_via_alen, &max_via_alen,
   1010					  &max_labels);
   1011	} else {
   1012		max_via_alen = cfg->rc_via_alen;
   1013		max_labels = cfg->rc_output_labels;
   1014		nhs = 1;
   1015	}
   1016
   1017	if (nhs == 0) {
   1018		NL_SET_ERR_MSG(extack, "Route does not contain a nexthop");
   1019		goto errout;
   1020	}
   1021
   1022	rt = mpls_rt_alloc(nhs, max_via_alen, max_labels);
   1023	if (IS_ERR(rt)) {
   1024		err = PTR_ERR(rt);
   1025		goto errout;
   1026	}
   1027
   1028	rt->rt_protocol = cfg->rc_protocol;
   1029	rt->rt_payload_type = cfg->rc_payload_type;
   1030	rt->rt_ttl_propagate = cfg->rc_ttl_propagate;
   1031
   1032	if (cfg->rc_mp)
   1033		err = mpls_nh_build_multi(cfg, rt, max_labels, extack);
   1034	else
   1035		err = mpls_nh_build_from_cfg(cfg, rt);
   1036	if (err)
   1037		goto freert;
   1038
   1039	mpls_route_update(net, index, rt, &cfg->rc_nlinfo);
   1040
   1041	return 0;
   1042
   1043freert:
   1044	mpls_rt_free(rt);
   1045errout:
   1046	return err;
   1047}
   1048
   1049static int mpls_route_del(struct mpls_route_config *cfg,
   1050			  struct netlink_ext_ack *extack)
   1051{
   1052	struct net *net = cfg->rc_nlinfo.nl_net;
   1053	unsigned index;
   1054	int err = -EINVAL;
   1055
   1056	index = cfg->rc_label;
   1057
   1058	if (!mpls_label_ok(net, &index, extack))
   1059		goto errout;
   1060
   1061	mpls_route_update(net, index, NULL, &cfg->rc_nlinfo);
   1062
   1063	err = 0;
   1064errout:
   1065	return err;
   1066}
   1067
   1068static void mpls_get_stats(struct mpls_dev *mdev,
   1069			   struct mpls_link_stats *stats)
   1070{
   1071	struct mpls_pcpu_stats *p;
   1072	int i;
   1073
   1074	memset(stats, 0, sizeof(*stats));
   1075
   1076	for_each_possible_cpu(i) {
   1077		struct mpls_link_stats local;
   1078		unsigned int start;
   1079
   1080		p = per_cpu_ptr(mdev->stats, i);
   1081		do {
   1082			start = u64_stats_fetch_begin(&p->syncp);
   1083			local = p->stats;
   1084		} while (u64_stats_fetch_retry(&p->syncp, start));
   1085
   1086		stats->rx_packets	+= local.rx_packets;
   1087		stats->rx_bytes		+= local.rx_bytes;
   1088		stats->tx_packets	+= local.tx_packets;
   1089		stats->tx_bytes		+= local.tx_bytes;
   1090		stats->rx_errors	+= local.rx_errors;
   1091		stats->tx_errors	+= local.tx_errors;
   1092		stats->rx_dropped	+= local.rx_dropped;
   1093		stats->tx_dropped	+= local.tx_dropped;
   1094		stats->rx_noroute	+= local.rx_noroute;
   1095	}
   1096}
   1097
   1098static int mpls_fill_stats_af(struct sk_buff *skb,
   1099			      const struct net_device *dev)
   1100{
   1101	struct mpls_link_stats *stats;
   1102	struct mpls_dev *mdev;
   1103	struct nlattr *nla;
   1104
   1105	mdev = mpls_dev_get(dev);
   1106	if (!mdev)
   1107		return -ENODATA;
   1108
   1109	nla = nla_reserve_64bit(skb, MPLS_STATS_LINK,
   1110				sizeof(struct mpls_link_stats),
   1111				MPLS_STATS_UNSPEC);
   1112	if (!nla)
   1113		return -EMSGSIZE;
   1114
   1115	stats = nla_data(nla);
   1116	mpls_get_stats(mdev, stats);
   1117
   1118	return 0;
   1119}
   1120
   1121static size_t mpls_get_stats_af_size(const struct net_device *dev)
   1122{
   1123	struct mpls_dev *mdev;
   1124
   1125	mdev = mpls_dev_get(dev);
   1126	if (!mdev)
   1127		return 0;
   1128
   1129	return nla_total_size_64bit(sizeof(struct mpls_link_stats));
   1130}
   1131
   1132static int mpls_netconf_fill_devconf(struct sk_buff *skb, struct mpls_dev *mdev,
   1133				     u32 portid, u32 seq, int event,
   1134				     unsigned int flags, int type)
   1135{
   1136	struct nlmsghdr  *nlh;
   1137	struct netconfmsg *ncm;
   1138	bool all = false;
   1139
   1140	nlh = nlmsg_put(skb, portid, seq, event, sizeof(struct netconfmsg),
   1141			flags);
   1142	if (!nlh)
   1143		return -EMSGSIZE;
   1144
   1145	if (type == NETCONFA_ALL)
   1146		all = true;
   1147
   1148	ncm = nlmsg_data(nlh);
   1149	ncm->ncm_family = AF_MPLS;
   1150
   1151	if (nla_put_s32(skb, NETCONFA_IFINDEX, mdev->dev->ifindex) < 0)
   1152		goto nla_put_failure;
   1153
   1154	if ((all || type == NETCONFA_INPUT) &&
   1155	    nla_put_s32(skb, NETCONFA_INPUT,
   1156			mdev->input_enabled) < 0)
   1157		goto nla_put_failure;
   1158
   1159	nlmsg_end(skb, nlh);
   1160	return 0;
   1161
   1162nla_put_failure:
   1163	nlmsg_cancel(skb, nlh);
   1164	return -EMSGSIZE;
   1165}
   1166
   1167static int mpls_netconf_msgsize_devconf(int type)
   1168{
   1169	int size = NLMSG_ALIGN(sizeof(struct netconfmsg))
   1170			+ nla_total_size(4); /* NETCONFA_IFINDEX */
   1171	bool all = false;
   1172
   1173	if (type == NETCONFA_ALL)
   1174		all = true;
   1175
   1176	if (all || type == NETCONFA_INPUT)
   1177		size += nla_total_size(4);
   1178
   1179	return size;
   1180}
   1181
   1182static void mpls_netconf_notify_devconf(struct net *net, int event,
   1183					int type, struct mpls_dev *mdev)
   1184{
   1185	struct sk_buff *skb;
   1186	int err = -ENOBUFS;
   1187
   1188	skb = nlmsg_new(mpls_netconf_msgsize_devconf(type), GFP_KERNEL);
   1189	if (!skb)
   1190		goto errout;
   1191
   1192	err = mpls_netconf_fill_devconf(skb, mdev, 0, 0, event, 0, type);
   1193	if (err < 0) {
   1194		/* -EMSGSIZE implies BUG in mpls_netconf_msgsize_devconf() */
   1195		WARN_ON(err == -EMSGSIZE);
   1196		kfree_skb(skb);
   1197		goto errout;
   1198	}
   1199
   1200	rtnl_notify(skb, net, 0, RTNLGRP_MPLS_NETCONF, NULL, GFP_KERNEL);
   1201	return;
   1202errout:
   1203	if (err < 0)
   1204		rtnl_set_sk_err(net, RTNLGRP_MPLS_NETCONF, err);
   1205}
   1206
   1207static const struct nla_policy devconf_mpls_policy[NETCONFA_MAX + 1] = {
   1208	[NETCONFA_IFINDEX]	= { .len = sizeof(int) },
   1209};
   1210
   1211static int mpls_netconf_valid_get_req(struct sk_buff *skb,
   1212				      const struct nlmsghdr *nlh,
   1213				      struct nlattr **tb,
   1214				      struct netlink_ext_ack *extack)
   1215{
   1216	int i, err;
   1217
   1218	if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(struct netconfmsg))) {
   1219		NL_SET_ERR_MSG_MOD(extack,
   1220				   "Invalid header for netconf get request");
   1221		return -EINVAL;
   1222	}
   1223
   1224	if (!netlink_strict_get_check(skb))
   1225		return nlmsg_parse_deprecated(nlh, sizeof(struct netconfmsg),
   1226					      tb, NETCONFA_MAX,
   1227					      devconf_mpls_policy, extack);
   1228
   1229	err = nlmsg_parse_deprecated_strict(nlh, sizeof(struct netconfmsg),
   1230					    tb, NETCONFA_MAX,
   1231					    devconf_mpls_policy, extack);
   1232	if (err)
   1233		return err;
   1234
   1235	for (i = 0; i <= NETCONFA_MAX; i++) {
   1236		if (!tb[i])
   1237			continue;
   1238
   1239		switch (i) {
   1240		case NETCONFA_IFINDEX:
   1241			break;
   1242		default:
   1243			NL_SET_ERR_MSG_MOD(extack, "Unsupported attribute in netconf get request");
   1244			return -EINVAL;
   1245		}
   1246	}
   1247
   1248	return 0;
   1249}
   1250
   1251static int mpls_netconf_get_devconf(struct sk_buff *in_skb,
   1252				    struct nlmsghdr *nlh,
   1253				    struct netlink_ext_ack *extack)
   1254{
   1255	struct net *net = sock_net(in_skb->sk);
   1256	struct nlattr *tb[NETCONFA_MAX + 1];
   1257	struct net_device *dev;
   1258	struct mpls_dev *mdev;
   1259	struct sk_buff *skb;
   1260	int ifindex;
   1261	int err;
   1262
   1263	err = mpls_netconf_valid_get_req(in_skb, nlh, tb, extack);
   1264	if (err < 0)
   1265		goto errout;
   1266
   1267	err = -EINVAL;
   1268	if (!tb[NETCONFA_IFINDEX])
   1269		goto errout;
   1270
   1271	ifindex = nla_get_s32(tb[NETCONFA_IFINDEX]);
   1272	dev = __dev_get_by_index(net, ifindex);
   1273	if (!dev)
   1274		goto errout;
   1275
   1276	mdev = mpls_dev_get(dev);
   1277	if (!mdev)
   1278		goto errout;
   1279
   1280	err = -ENOBUFS;
   1281	skb = nlmsg_new(mpls_netconf_msgsize_devconf(NETCONFA_ALL), GFP_KERNEL);
   1282	if (!skb)
   1283		goto errout;
   1284
   1285	err = mpls_netconf_fill_devconf(skb, mdev,
   1286					NETLINK_CB(in_skb).portid,
   1287					nlh->nlmsg_seq, RTM_NEWNETCONF, 0,
   1288					NETCONFA_ALL);
   1289	if (err < 0) {
   1290		/* -EMSGSIZE implies BUG in mpls_netconf_msgsize_devconf() */
   1291		WARN_ON(err == -EMSGSIZE);
   1292		kfree_skb(skb);
   1293		goto errout;
   1294	}
   1295	err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
   1296errout:
   1297	return err;
   1298}
   1299
   1300static int mpls_netconf_dump_devconf(struct sk_buff *skb,
   1301				     struct netlink_callback *cb)
   1302{
   1303	const struct nlmsghdr *nlh = cb->nlh;
   1304	struct net *net = sock_net(skb->sk);
   1305	struct hlist_head *head;
   1306	struct net_device *dev;
   1307	struct mpls_dev *mdev;
   1308	int idx, s_idx;
   1309	int h, s_h;
   1310
   1311	if (cb->strict_check) {
   1312		struct netlink_ext_ack *extack = cb->extack;
   1313		struct netconfmsg *ncm;
   1314
   1315		if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*ncm))) {
   1316			NL_SET_ERR_MSG_MOD(extack, "Invalid header for netconf dump request");
   1317			return -EINVAL;
   1318		}
   1319
   1320		if (nlmsg_attrlen(nlh, sizeof(*ncm))) {
   1321			NL_SET_ERR_MSG_MOD(extack, "Invalid data after header in netconf dump request");
   1322			return -EINVAL;
   1323		}
   1324	}
   1325
   1326	s_h = cb->args[0];
   1327	s_idx = idx = cb->args[1];
   1328
   1329	for (h = s_h; h < NETDEV_HASHENTRIES; h++, s_idx = 0) {
   1330		idx = 0;
   1331		head = &net->dev_index_head[h];
   1332		rcu_read_lock();
   1333		cb->seq = net->dev_base_seq;
   1334		hlist_for_each_entry_rcu(dev, head, index_hlist) {
   1335			if (idx < s_idx)
   1336				goto cont;
   1337			mdev = mpls_dev_get(dev);
   1338			if (!mdev)
   1339				goto cont;
   1340			if (mpls_netconf_fill_devconf(skb, mdev,
   1341						      NETLINK_CB(cb->skb).portid,
   1342						      nlh->nlmsg_seq,
   1343						      RTM_NEWNETCONF,
   1344						      NLM_F_MULTI,
   1345						      NETCONFA_ALL) < 0) {
   1346				rcu_read_unlock();
   1347				goto done;
   1348			}
   1349			nl_dump_check_consistent(cb, nlmsg_hdr(skb));
   1350cont:
   1351			idx++;
   1352		}
   1353		rcu_read_unlock();
   1354	}
   1355done:
   1356	cb->args[0] = h;
   1357	cb->args[1] = idx;
   1358
   1359	return skb->len;
   1360}
   1361
   1362#define MPLS_PERDEV_SYSCTL_OFFSET(field)	\
   1363	(&((struct mpls_dev *)0)->field)
   1364
   1365static int mpls_conf_proc(struct ctl_table *ctl, int write,
   1366			  void *buffer, size_t *lenp, loff_t *ppos)
   1367{
   1368	int oval = *(int *)ctl->data;
   1369	int ret = proc_dointvec(ctl, write, buffer, lenp, ppos);
   1370
   1371	if (write) {
   1372		struct mpls_dev *mdev = ctl->extra1;
   1373		int i = (int *)ctl->data - (int *)mdev;
   1374		struct net *net = ctl->extra2;
   1375		int val = *(int *)ctl->data;
   1376
   1377		if (i == offsetof(struct mpls_dev, input_enabled) &&
   1378		    val != oval) {
   1379			mpls_netconf_notify_devconf(net, RTM_NEWNETCONF,
   1380						    NETCONFA_INPUT, mdev);
   1381		}
   1382	}
   1383
   1384	return ret;
   1385}
   1386
   1387static const struct ctl_table mpls_dev_table[] = {
   1388	{
   1389		.procname	= "input",
   1390		.maxlen		= sizeof(int),
   1391		.mode		= 0644,
   1392		.proc_handler	= mpls_conf_proc,
   1393		.data		= MPLS_PERDEV_SYSCTL_OFFSET(input_enabled),
   1394	},
   1395	{ }
   1396};
   1397
   1398static int mpls_dev_sysctl_register(struct net_device *dev,
   1399				    struct mpls_dev *mdev)
   1400{
   1401	char path[sizeof("net/mpls/conf/") + IFNAMSIZ];
   1402	struct net *net = dev_net(dev);
   1403	struct ctl_table *table;
   1404	int i;
   1405
   1406	table = kmemdup(&mpls_dev_table, sizeof(mpls_dev_table), GFP_KERNEL);
   1407	if (!table)
   1408		goto out;
   1409
   1410	/* Table data contains only offsets relative to the base of
   1411	 * the mdev at this point, so make them absolute.
   1412	 */
   1413	for (i = 0; i < ARRAY_SIZE(mpls_dev_table); i++) {
   1414		table[i].data = (char *)mdev + (uintptr_t)table[i].data;
   1415		table[i].extra1 = mdev;
   1416		table[i].extra2 = net;
   1417	}
   1418
   1419	snprintf(path, sizeof(path), "net/mpls/conf/%s", dev->name);
   1420
   1421	mdev->sysctl = register_net_sysctl(net, path, table);
   1422	if (!mdev->sysctl)
   1423		goto free;
   1424
   1425	mpls_netconf_notify_devconf(net, RTM_NEWNETCONF, NETCONFA_ALL, mdev);
   1426	return 0;
   1427
   1428free:
   1429	kfree(table);
   1430out:
   1431	return -ENOBUFS;
   1432}
   1433
   1434static void mpls_dev_sysctl_unregister(struct net_device *dev,
   1435				       struct mpls_dev *mdev)
   1436{
   1437	struct net *net = dev_net(dev);
   1438	struct ctl_table *table;
   1439
   1440	table = mdev->sysctl->ctl_table_arg;
   1441	unregister_net_sysctl_table(mdev->sysctl);
   1442	kfree(table);
   1443
   1444	mpls_netconf_notify_devconf(net, RTM_DELNETCONF, 0, mdev);
   1445}
   1446
   1447static struct mpls_dev *mpls_add_dev(struct net_device *dev)
   1448{
   1449	struct mpls_dev *mdev;
   1450	int err = -ENOMEM;
   1451	int i;
   1452
   1453	ASSERT_RTNL();
   1454
   1455	mdev = kzalloc(sizeof(*mdev), GFP_KERNEL);
   1456	if (!mdev)
   1457		return ERR_PTR(err);
   1458
   1459	mdev->stats = alloc_percpu(struct mpls_pcpu_stats);
   1460	if (!mdev->stats)
   1461		goto free;
   1462
   1463	for_each_possible_cpu(i) {
   1464		struct mpls_pcpu_stats *mpls_stats;
   1465
   1466		mpls_stats = per_cpu_ptr(mdev->stats, i);
   1467		u64_stats_init(&mpls_stats->syncp);
   1468	}
   1469
   1470	mdev->dev = dev;
   1471
   1472	err = mpls_dev_sysctl_register(dev, mdev);
   1473	if (err)
   1474		goto free;
   1475
   1476	rcu_assign_pointer(dev->mpls_ptr, mdev);
   1477
   1478	return mdev;
   1479
   1480free:
   1481	free_percpu(mdev->stats);
   1482	kfree(mdev);
   1483	return ERR_PTR(err);
   1484}
   1485
   1486static void mpls_dev_destroy_rcu(struct rcu_head *head)
   1487{
   1488	struct mpls_dev *mdev = container_of(head, struct mpls_dev, rcu);
   1489
   1490	free_percpu(mdev->stats);
   1491	kfree(mdev);
   1492}
   1493
   1494static int mpls_ifdown(struct net_device *dev, int event)
   1495{
   1496	struct mpls_route __rcu **platform_label;
   1497	struct net *net = dev_net(dev);
   1498	unsigned index;
   1499
   1500	platform_label = rtnl_dereference(net->mpls.platform_label);
   1501	for (index = 0; index < net->mpls.platform_labels; index++) {
   1502		struct mpls_route *rt = rtnl_dereference(platform_label[index]);
   1503		bool nh_del = false;
   1504		u8 alive = 0;
   1505
   1506		if (!rt)
   1507			continue;
   1508
   1509		if (event == NETDEV_UNREGISTER) {
   1510			u8 deleted = 0;
   1511
   1512			for_nexthops(rt) {
   1513				if (!nh->nh_dev || nh->nh_dev == dev)
   1514					deleted++;
   1515				if (nh->nh_dev == dev)
   1516					nh_del = true;
   1517			} endfor_nexthops(rt);
   1518
   1519			/* if there are no more nexthops, delete the route */
   1520			if (deleted == rt->rt_nhn) {
   1521				mpls_route_update(net, index, NULL, NULL);
   1522				continue;
   1523			}
   1524
   1525			if (nh_del) {
   1526				size_t size = sizeof(*rt) + rt->rt_nhn *
   1527					rt->rt_nh_size;
   1528				struct mpls_route *orig = rt;
   1529
   1530				rt = kmemdup(orig, size, GFP_KERNEL);
   1531				if (!rt)
   1532					return -ENOMEM;
   1533			}
   1534		}
   1535
   1536		change_nexthops(rt) {
   1537			unsigned int nh_flags = nh->nh_flags;
   1538
   1539			if (nh->nh_dev != dev)
   1540				goto next;
   1541
   1542			switch (event) {
   1543			case NETDEV_DOWN:
   1544			case NETDEV_UNREGISTER:
   1545				nh_flags |= RTNH_F_DEAD;
   1546				fallthrough;
   1547			case NETDEV_CHANGE:
   1548				nh_flags |= RTNH_F_LINKDOWN;
   1549				break;
   1550			}
   1551			if (event == NETDEV_UNREGISTER)
   1552				nh->nh_dev = NULL;
   1553
   1554			if (nh->nh_flags != nh_flags)
   1555				WRITE_ONCE(nh->nh_flags, nh_flags);
   1556next:
   1557			if (!(nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN)))
   1558				alive++;
   1559		} endfor_nexthops(rt);
   1560
   1561		WRITE_ONCE(rt->rt_nhn_alive, alive);
   1562
   1563		if (nh_del)
   1564			mpls_route_update(net, index, rt, NULL);
   1565	}
   1566
   1567	return 0;
   1568}
   1569
   1570static void mpls_ifup(struct net_device *dev, unsigned int flags)
   1571{
   1572	struct mpls_route __rcu **platform_label;
   1573	struct net *net = dev_net(dev);
   1574	unsigned index;
   1575	u8 alive;
   1576
   1577	platform_label = rtnl_dereference(net->mpls.platform_label);
   1578	for (index = 0; index < net->mpls.platform_labels; index++) {
   1579		struct mpls_route *rt = rtnl_dereference(platform_label[index]);
   1580
   1581		if (!rt)
   1582			continue;
   1583
   1584		alive = 0;
   1585		change_nexthops(rt) {
   1586			unsigned int nh_flags = nh->nh_flags;
   1587
   1588			if (!(nh_flags & flags)) {
   1589				alive++;
   1590				continue;
   1591			}
   1592			if (nh->nh_dev != dev)
   1593				continue;
   1594			alive++;
   1595			nh_flags &= ~flags;
   1596			WRITE_ONCE(nh->nh_flags, nh_flags);
   1597		} endfor_nexthops(rt);
   1598
   1599		WRITE_ONCE(rt->rt_nhn_alive, alive);
   1600	}
   1601}
   1602
   1603static int mpls_dev_notify(struct notifier_block *this, unsigned long event,
   1604			   void *ptr)
   1605{
   1606	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
   1607	struct mpls_dev *mdev;
   1608	unsigned int flags;
   1609	int err;
   1610
   1611	if (event == NETDEV_REGISTER) {
   1612		mdev = mpls_add_dev(dev);
   1613		if (IS_ERR(mdev))
   1614			return notifier_from_errno(PTR_ERR(mdev));
   1615
   1616		return NOTIFY_OK;
   1617	}
   1618
   1619	mdev = mpls_dev_get(dev);
   1620	if (!mdev)
   1621		return NOTIFY_OK;
   1622
   1623	switch (event) {
   1624
   1625	case NETDEV_DOWN:
   1626		err = mpls_ifdown(dev, event);
   1627		if (err)
   1628			return notifier_from_errno(err);
   1629		break;
   1630	case NETDEV_UP:
   1631		flags = dev_get_flags(dev);
   1632		if (flags & (IFF_RUNNING | IFF_LOWER_UP))
   1633			mpls_ifup(dev, RTNH_F_DEAD | RTNH_F_LINKDOWN);
   1634		else
   1635			mpls_ifup(dev, RTNH_F_DEAD);
   1636		break;
   1637	case NETDEV_CHANGE:
   1638		flags = dev_get_flags(dev);
   1639		if (flags & (IFF_RUNNING | IFF_LOWER_UP)) {
   1640			mpls_ifup(dev, RTNH_F_DEAD | RTNH_F_LINKDOWN);
   1641		} else {
   1642			err = mpls_ifdown(dev, event);
   1643			if (err)
   1644				return notifier_from_errno(err);
   1645		}
   1646		break;
   1647	case NETDEV_UNREGISTER:
   1648		err = mpls_ifdown(dev, event);
   1649		if (err)
   1650			return notifier_from_errno(err);
   1651		mdev = mpls_dev_get(dev);
   1652		if (mdev) {
   1653			mpls_dev_sysctl_unregister(dev, mdev);
   1654			RCU_INIT_POINTER(dev->mpls_ptr, NULL);
   1655			call_rcu(&mdev->rcu, mpls_dev_destroy_rcu);
   1656		}
   1657		break;
   1658	case NETDEV_CHANGENAME:
   1659		mdev = mpls_dev_get(dev);
   1660		if (mdev) {
   1661			mpls_dev_sysctl_unregister(dev, mdev);
   1662			err = mpls_dev_sysctl_register(dev, mdev);
   1663			if (err)
   1664				return notifier_from_errno(err);
   1665		}
   1666		break;
   1667	}
   1668	return NOTIFY_OK;
   1669}
   1670
   1671static struct notifier_block mpls_dev_notifier = {
   1672	.notifier_call = mpls_dev_notify,
   1673};
   1674
   1675static int nla_put_via(struct sk_buff *skb,
   1676		       u8 table, const void *addr, int alen)
   1677{
   1678	static const int table_to_family[NEIGH_NR_TABLES + 1] = {
   1679		AF_INET, AF_INET6, AF_DECnet, AF_PACKET,
   1680	};
   1681	struct nlattr *nla;
   1682	struct rtvia *via;
   1683	int family = AF_UNSPEC;
   1684
   1685	nla = nla_reserve(skb, RTA_VIA, alen + 2);
   1686	if (!nla)
   1687		return -EMSGSIZE;
   1688
   1689	if (table <= NEIGH_NR_TABLES)
   1690		family = table_to_family[table];
   1691
   1692	via = nla_data(nla);
   1693	via->rtvia_family = family;
   1694	memcpy(via->rtvia_addr, addr, alen);
   1695	return 0;
   1696}
   1697
   1698int nla_put_labels(struct sk_buff *skb, int attrtype,
   1699		   u8 labels, const u32 label[])
   1700{
   1701	struct nlattr *nla;
   1702	struct mpls_shim_hdr *nla_label;
   1703	bool bos;
   1704	int i;
   1705	nla = nla_reserve(skb, attrtype, labels*4);
   1706	if (!nla)
   1707		return -EMSGSIZE;
   1708
   1709	nla_label = nla_data(nla);
   1710	bos = true;
   1711	for (i = labels - 1; i >= 0; i--) {
   1712		nla_label[i] = mpls_entry_encode(label[i], 0, 0, bos);
   1713		bos = false;
   1714	}
   1715
   1716	return 0;
   1717}
   1718EXPORT_SYMBOL_GPL(nla_put_labels);
   1719
   1720int nla_get_labels(const struct nlattr *nla, u8 max_labels, u8 *labels,
   1721		   u32 label[], struct netlink_ext_ack *extack)
   1722{
   1723	unsigned len = nla_len(nla);
   1724	struct mpls_shim_hdr *nla_label;
   1725	u8 nla_labels;
   1726	bool bos;
   1727	int i;
   1728
   1729	/* len needs to be an even multiple of 4 (the label size). Number
   1730	 * of labels is a u8 so check for overflow.
   1731	 */
   1732	if (len & 3 || len / 4 > 255) {
   1733		NL_SET_ERR_MSG_ATTR(extack, nla,
   1734				    "Invalid length for labels attribute");
   1735		return -EINVAL;
   1736	}
   1737
   1738	/* Limit the number of new labels allowed */
   1739	nla_labels = len/4;
   1740	if (nla_labels > max_labels) {
   1741		NL_SET_ERR_MSG(extack, "Too many labels");
   1742		return -EINVAL;
   1743	}
   1744
   1745	/* when label == NULL, caller wants number of labels */
   1746	if (!label)
   1747		goto out;
   1748
   1749	nla_label = nla_data(nla);
   1750	bos = true;
   1751	for (i = nla_labels - 1; i >= 0; i--, bos = false) {
   1752		struct mpls_entry_decoded dec;
   1753		dec = mpls_entry_decode(nla_label + i);
   1754
   1755		/* Ensure the bottom of stack flag is properly set
   1756		 * and ttl and tc are both clear.
   1757		 */
   1758		if (dec.ttl) {
   1759			NL_SET_ERR_MSG_ATTR(extack, nla,
   1760					    "TTL in label must be 0");
   1761			return -EINVAL;
   1762		}
   1763
   1764		if (dec.tc) {
   1765			NL_SET_ERR_MSG_ATTR(extack, nla,
   1766					    "Traffic class in label must be 0");
   1767			return -EINVAL;
   1768		}
   1769
   1770		if (dec.bos != bos) {
   1771			NL_SET_BAD_ATTR(extack, nla);
   1772			if (bos) {
   1773				NL_SET_ERR_MSG(extack,
   1774					       "BOS bit must be set in first label");
   1775			} else {
   1776				NL_SET_ERR_MSG(extack,
   1777					       "BOS bit can only be set in first label");
   1778			}
   1779			return -EINVAL;
   1780		}
   1781
   1782		switch (dec.label) {
   1783		case MPLS_LABEL_IMPLNULL:
   1784			/* RFC3032: This is a label that an LSR may
   1785			 * assign and distribute, but which never
   1786			 * actually appears in the encapsulation.
   1787			 */
   1788			NL_SET_ERR_MSG_ATTR(extack, nla,
   1789					    "Implicit NULL Label (3) can not be used in encapsulation");
   1790			return -EINVAL;
   1791		}
   1792
   1793		label[i] = dec.label;
   1794	}
   1795out:
   1796	*labels = nla_labels;
   1797	return 0;
   1798}
   1799EXPORT_SYMBOL_GPL(nla_get_labels);
   1800
   1801static int rtm_to_route_config(struct sk_buff *skb,
   1802			       struct nlmsghdr *nlh,
   1803			       struct mpls_route_config *cfg,
   1804			       struct netlink_ext_ack *extack)
   1805{
   1806	struct rtmsg *rtm;
   1807	struct nlattr *tb[RTA_MAX+1];
   1808	int index;
   1809	int err;
   1810
   1811	err = nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX,
   1812				     rtm_mpls_policy, extack);
   1813	if (err < 0)
   1814		goto errout;
   1815
   1816	err = -EINVAL;
   1817	rtm = nlmsg_data(nlh);
   1818
   1819	if (rtm->rtm_family != AF_MPLS) {
   1820		NL_SET_ERR_MSG(extack, "Invalid address family in rtmsg");
   1821		goto errout;
   1822	}
   1823	if (rtm->rtm_dst_len != 20) {
   1824		NL_SET_ERR_MSG(extack, "rtm_dst_len must be 20 for MPLS");
   1825		goto errout;
   1826	}
   1827	if (rtm->rtm_src_len != 0) {
   1828		NL_SET_ERR_MSG(extack, "rtm_src_len must be 0 for MPLS");
   1829		goto errout;
   1830	}
   1831	if (rtm->rtm_tos != 0) {
   1832		NL_SET_ERR_MSG(extack, "rtm_tos must be 0 for MPLS");
   1833		goto errout;
   1834	}
   1835	if (rtm->rtm_table != RT_TABLE_MAIN) {
   1836		NL_SET_ERR_MSG(extack,
   1837			       "MPLS only supports the main route table");
   1838		goto errout;
   1839	}
   1840	/* Any value is acceptable for rtm_protocol */
   1841
   1842	/* As mpls uses destination specific addresses
   1843	 * (or source specific address in the case of multicast)
   1844	 * all addresses have universal scope.
   1845	 */
   1846	if (rtm->rtm_scope != RT_SCOPE_UNIVERSE) {
   1847		NL_SET_ERR_MSG(extack,
   1848			       "Invalid route scope  - MPLS only supports UNIVERSE");
   1849		goto errout;
   1850	}
   1851	if (rtm->rtm_type != RTN_UNICAST) {
   1852		NL_SET_ERR_MSG(extack,
   1853			       "Invalid route type - MPLS only supports UNICAST");
   1854		goto errout;
   1855	}
   1856	if (rtm->rtm_flags != 0) {
   1857		NL_SET_ERR_MSG(extack, "rtm_flags must be 0 for MPLS");
   1858		goto errout;
   1859	}
   1860
   1861	cfg->rc_label		= LABEL_NOT_SPECIFIED;
   1862	cfg->rc_protocol	= rtm->rtm_protocol;
   1863	cfg->rc_via_table	= MPLS_NEIGH_TABLE_UNSPEC;
   1864	cfg->rc_ttl_propagate	= MPLS_TTL_PROP_DEFAULT;
   1865	cfg->rc_nlflags		= nlh->nlmsg_flags;
   1866	cfg->rc_nlinfo.portid	= NETLINK_CB(skb).portid;
   1867	cfg->rc_nlinfo.nlh	= nlh;
   1868	cfg->rc_nlinfo.nl_net	= sock_net(skb->sk);
   1869
   1870	for (index = 0; index <= RTA_MAX; index++) {
   1871		struct nlattr *nla = tb[index];
   1872		if (!nla)
   1873			continue;
   1874
   1875		switch (index) {
   1876		case RTA_OIF:
   1877			cfg->rc_ifindex = nla_get_u32(nla);
   1878			break;
   1879		case RTA_NEWDST:
   1880			if (nla_get_labels(nla, MAX_NEW_LABELS,
   1881					   &cfg->rc_output_labels,
   1882					   cfg->rc_output_label, extack))
   1883				goto errout;
   1884			break;
   1885		case RTA_DST:
   1886		{
   1887			u8 label_count;
   1888			if (nla_get_labels(nla, 1, &label_count,
   1889					   &cfg->rc_label, extack))
   1890				goto errout;
   1891
   1892			if (!mpls_label_ok(cfg->rc_nlinfo.nl_net,
   1893					   &cfg->rc_label, extack))
   1894				goto errout;
   1895			break;
   1896		}
   1897		case RTA_GATEWAY:
   1898			NL_SET_ERR_MSG(extack, "MPLS does not support RTA_GATEWAY attribute");
   1899			goto errout;
   1900		case RTA_VIA:
   1901		{
   1902			if (nla_get_via(nla, &cfg->rc_via_alen,
   1903					&cfg->rc_via_table, cfg->rc_via,
   1904					extack))
   1905				goto errout;
   1906			break;
   1907		}
   1908		case RTA_MULTIPATH:
   1909		{
   1910			cfg->rc_mp = nla_data(nla);
   1911			cfg->rc_mp_len = nla_len(nla);
   1912			break;
   1913		}
   1914		case RTA_TTL_PROPAGATE:
   1915		{
   1916			u8 ttl_propagate = nla_get_u8(nla);
   1917
   1918			if (ttl_propagate > 1) {
   1919				NL_SET_ERR_MSG_ATTR(extack, nla,
   1920						    "RTA_TTL_PROPAGATE can only be 0 or 1");
   1921				goto errout;
   1922			}
   1923			cfg->rc_ttl_propagate = ttl_propagate ?
   1924				MPLS_TTL_PROP_ENABLED :
   1925				MPLS_TTL_PROP_DISABLED;
   1926			break;
   1927		}
   1928		default:
   1929			NL_SET_ERR_MSG_ATTR(extack, nla, "Unknown attribute");
   1930			/* Unsupported attribute */
   1931			goto errout;
   1932		}
   1933	}
   1934
   1935	err = 0;
   1936errout:
   1937	return err;
   1938}
   1939
   1940static int mpls_rtm_delroute(struct sk_buff *skb, struct nlmsghdr *nlh,
   1941			     struct netlink_ext_ack *extack)
   1942{
   1943	struct mpls_route_config *cfg;
   1944	int err;
   1945
   1946	cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
   1947	if (!cfg)
   1948		return -ENOMEM;
   1949
   1950	err = rtm_to_route_config(skb, nlh, cfg, extack);
   1951	if (err < 0)
   1952		goto out;
   1953
   1954	err = mpls_route_del(cfg, extack);
   1955out:
   1956	kfree(cfg);
   1957
   1958	return err;
   1959}
   1960
   1961
   1962static int mpls_rtm_newroute(struct sk_buff *skb, struct nlmsghdr *nlh,
   1963			     struct netlink_ext_ack *extack)
   1964{
   1965	struct mpls_route_config *cfg;
   1966	int err;
   1967
   1968	cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
   1969	if (!cfg)
   1970		return -ENOMEM;
   1971
   1972	err = rtm_to_route_config(skb, nlh, cfg, extack);
   1973	if (err < 0)
   1974		goto out;
   1975
   1976	err = mpls_route_add(cfg, extack);
   1977out:
   1978	kfree(cfg);
   1979
   1980	return err;
   1981}
   1982
   1983static int mpls_dump_route(struct sk_buff *skb, u32 portid, u32 seq, int event,
   1984			   u32 label, struct mpls_route *rt, int flags)
   1985{
   1986	struct net_device *dev;
   1987	struct nlmsghdr *nlh;
   1988	struct rtmsg *rtm;
   1989
   1990	nlh = nlmsg_put(skb, portid, seq, event, sizeof(*rtm), flags);
   1991	if (nlh == NULL)
   1992		return -EMSGSIZE;
   1993
   1994	rtm = nlmsg_data(nlh);
   1995	rtm->rtm_family = AF_MPLS;
   1996	rtm->rtm_dst_len = 20;
   1997	rtm->rtm_src_len = 0;
   1998	rtm->rtm_tos = 0;
   1999	rtm->rtm_table = RT_TABLE_MAIN;
   2000	rtm->rtm_protocol = rt->rt_protocol;
   2001	rtm->rtm_scope = RT_SCOPE_UNIVERSE;
   2002	rtm->rtm_type = RTN_UNICAST;
   2003	rtm->rtm_flags = 0;
   2004
   2005	if (nla_put_labels(skb, RTA_DST, 1, &label))
   2006		goto nla_put_failure;
   2007
   2008	if (rt->rt_ttl_propagate != MPLS_TTL_PROP_DEFAULT) {
   2009		bool ttl_propagate =
   2010			rt->rt_ttl_propagate == MPLS_TTL_PROP_ENABLED;
   2011
   2012		if (nla_put_u8(skb, RTA_TTL_PROPAGATE,
   2013			       ttl_propagate))
   2014			goto nla_put_failure;
   2015	}
   2016	if (rt->rt_nhn == 1) {
   2017		const struct mpls_nh *nh = rt->rt_nh;
   2018
   2019		if (nh->nh_labels &&
   2020		    nla_put_labels(skb, RTA_NEWDST, nh->nh_labels,
   2021				   nh->nh_label))
   2022			goto nla_put_failure;
   2023		if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC &&
   2024		    nla_put_via(skb, nh->nh_via_table, mpls_nh_via(rt, nh),
   2025				nh->nh_via_alen))
   2026			goto nla_put_failure;
   2027		dev = nh->nh_dev;
   2028		if (dev && nla_put_u32(skb, RTA_OIF, dev->ifindex))
   2029			goto nla_put_failure;
   2030		if (nh->nh_flags & RTNH_F_LINKDOWN)
   2031			rtm->rtm_flags |= RTNH_F_LINKDOWN;
   2032		if (nh->nh_flags & RTNH_F_DEAD)
   2033			rtm->rtm_flags |= RTNH_F_DEAD;
   2034	} else {
   2035		struct rtnexthop *rtnh;
   2036		struct nlattr *mp;
   2037		u8 linkdown = 0;
   2038		u8 dead = 0;
   2039
   2040		mp = nla_nest_start_noflag(skb, RTA_MULTIPATH);
   2041		if (!mp)
   2042			goto nla_put_failure;
   2043
   2044		for_nexthops(rt) {
   2045			dev = nh->nh_dev;
   2046			if (!dev)
   2047				continue;
   2048
   2049			rtnh = nla_reserve_nohdr(skb, sizeof(*rtnh));
   2050			if (!rtnh)
   2051				goto nla_put_failure;
   2052
   2053			rtnh->rtnh_ifindex = dev->ifindex;
   2054			if (nh->nh_flags & RTNH_F_LINKDOWN) {
   2055				rtnh->rtnh_flags |= RTNH_F_LINKDOWN;
   2056				linkdown++;
   2057			}
   2058			if (nh->nh_flags & RTNH_F_DEAD) {
   2059				rtnh->rtnh_flags |= RTNH_F_DEAD;
   2060				dead++;
   2061			}
   2062
   2063			if (nh->nh_labels && nla_put_labels(skb, RTA_NEWDST,
   2064							    nh->nh_labels,
   2065							    nh->nh_label))
   2066				goto nla_put_failure;
   2067			if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC &&
   2068			    nla_put_via(skb, nh->nh_via_table,
   2069					mpls_nh_via(rt, nh),
   2070					nh->nh_via_alen))
   2071				goto nla_put_failure;
   2072
   2073			/* length of rtnetlink header + attributes */
   2074			rtnh->rtnh_len = nlmsg_get_pos(skb) - (void *)rtnh;
   2075		} endfor_nexthops(rt);
   2076
   2077		if (linkdown == rt->rt_nhn)
   2078			rtm->rtm_flags |= RTNH_F_LINKDOWN;
   2079		if (dead == rt->rt_nhn)
   2080			rtm->rtm_flags |= RTNH_F_DEAD;
   2081
   2082		nla_nest_end(skb, mp);
   2083	}
   2084
   2085	nlmsg_end(skb, nlh);
   2086	return 0;
   2087
   2088nla_put_failure:
   2089	nlmsg_cancel(skb, nlh);
   2090	return -EMSGSIZE;
   2091}
   2092
   2093#if IS_ENABLED(CONFIG_INET)
   2094static int mpls_valid_fib_dump_req(struct net *net, const struct nlmsghdr *nlh,
   2095				   struct fib_dump_filter *filter,
   2096				   struct netlink_callback *cb)
   2097{
   2098	return ip_valid_fib_dump_req(net, nlh, filter, cb);
   2099}
   2100#else
   2101static int mpls_valid_fib_dump_req(struct net *net, const struct nlmsghdr *nlh,
   2102				   struct fib_dump_filter *filter,
   2103				   struct netlink_callback *cb)
   2104{
   2105	struct netlink_ext_ack *extack = cb->extack;
   2106	struct nlattr *tb[RTA_MAX + 1];
   2107	struct rtmsg *rtm;
   2108	int err, i;
   2109
   2110	if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*rtm))) {
   2111		NL_SET_ERR_MSG_MOD(extack, "Invalid header for FIB dump request");
   2112		return -EINVAL;
   2113	}
   2114
   2115	rtm = nlmsg_data(nlh);
   2116	if (rtm->rtm_dst_len || rtm->rtm_src_len  || rtm->rtm_tos   ||
   2117	    rtm->rtm_table   || rtm->rtm_scope    || rtm->rtm_type  ||
   2118	    rtm->rtm_flags) {
   2119		NL_SET_ERR_MSG_MOD(extack, "Invalid values in header for FIB dump request");
   2120		return -EINVAL;
   2121	}
   2122
   2123	if (rtm->rtm_protocol) {
   2124		filter->protocol = rtm->rtm_protocol;
   2125		filter->filter_set = 1;
   2126		cb->answer_flags = NLM_F_DUMP_FILTERED;
   2127	}
   2128
   2129	err = nlmsg_parse_deprecated_strict(nlh, sizeof(*rtm), tb, RTA_MAX,
   2130					    rtm_mpls_policy, extack);
   2131	if (err < 0)
   2132		return err;
   2133
   2134	for (i = 0; i <= RTA_MAX; ++i) {
   2135		int ifindex;
   2136
   2137		if (i == RTA_OIF) {
   2138			ifindex = nla_get_u32(tb[i]);
   2139			filter->dev = __dev_get_by_index(net, ifindex);
   2140			if (!filter->dev)
   2141				return -ENODEV;
   2142			filter->filter_set = 1;
   2143		} else if (tb[i]) {
   2144			NL_SET_ERR_MSG_MOD(extack, "Unsupported attribute in dump request");
   2145			return -EINVAL;
   2146		}
   2147	}
   2148
   2149	return 0;
   2150}
   2151#endif
   2152
   2153static bool mpls_rt_uses_dev(struct mpls_route *rt,
   2154			     const struct net_device *dev)
   2155{
   2156	if (rt->rt_nhn == 1) {
   2157		struct mpls_nh *nh = rt->rt_nh;
   2158
   2159		if (nh->nh_dev == dev)
   2160			return true;
   2161	} else {
   2162		for_nexthops(rt) {
   2163			if (nh->nh_dev == dev)
   2164				return true;
   2165		} endfor_nexthops(rt);
   2166	}
   2167
   2168	return false;
   2169}
   2170
   2171static int mpls_dump_routes(struct sk_buff *skb, struct netlink_callback *cb)
   2172{
   2173	const struct nlmsghdr *nlh = cb->nlh;
   2174	struct net *net = sock_net(skb->sk);
   2175	struct mpls_route __rcu **platform_label;
   2176	struct fib_dump_filter filter = {};
   2177	unsigned int flags = NLM_F_MULTI;
   2178	size_t platform_labels;
   2179	unsigned int index;
   2180
   2181	ASSERT_RTNL();
   2182
   2183	if (cb->strict_check) {
   2184		int err;
   2185
   2186		err = mpls_valid_fib_dump_req(net, nlh, &filter, cb);
   2187		if (err < 0)
   2188			return err;
   2189
   2190		/* for MPLS, there is only 1 table with fixed type and flags.
   2191		 * If either are set in the filter then return nothing.
   2192		 */
   2193		if ((filter.table_id && filter.table_id != RT_TABLE_MAIN) ||
   2194		    (filter.rt_type && filter.rt_type != RTN_UNICAST) ||
   2195		     filter.flags)
   2196			return skb->len;
   2197	}
   2198
   2199	index = cb->args[0];
   2200	if (index < MPLS_LABEL_FIRST_UNRESERVED)
   2201		index = MPLS_LABEL_FIRST_UNRESERVED;
   2202
   2203	platform_label = rtnl_dereference(net->mpls.platform_label);
   2204	platform_labels = net->mpls.platform_labels;
   2205
   2206	if (filter.filter_set)
   2207		flags |= NLM_F_DUMP_FILTERED;
   2208
   2209	for (; index < platform_labels; index++) {
   2210		struct mpls_route *rt;
   2211
   2212		rt = rtnl_dereference(platform_label[index]);
   2213		if (!rt)
   2214			continue;
   2215
   2216		if ((filter.dev && !mpls_rt_uses_dev(rt, filter.dev)) ||
   2217		    (filter.protocol && rt->rt_protocol != filter.protocol))
   2218			continue;
   2219
   2220		if (mpls_dump_route(skb, NETLINK_CB(cb->skb).portid,
   2221				    cb->nlh->nlmsg_seq, RTM_NEWROUTE,
   2222				    index, rt, flags) < 0)
   2223			break;
   2224	}
   2225	cb->args[0] = index;
   2226
   2227	return skb->len;
   2228}
   2229
   2230static inline size_t lfib_nlmsg_size(struct mpls_route *rt)
   2231{
   2232	size_t payload =
   2233		NLMSG_ALIGN(sizeof(struct rtmsg))
   2234		+ nla_total_size(4)			/* RTA_DST */
   2235		+ nla_total_size(1);			/* RTA_TTL_PROPAGATE */
   2236
   2237	if (rt->rt_nhn == 1) {
   2238		struct mpls_nh *nh = rt->rt_nh;
   2239
   2240		if (nh->nh_dev)
   2241			payload += nla_total_size(4); /* RTA_OIF */
   2242		if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC) /* RTA_VIA */
   2243			payload += nla_total_size(2 + nh->nh_via_alen);
   2244		if (nh->nh_labels) /* RTA_NEWDST */
   2245			payload += nla_total_size(nh->nh_labels * 4);
   2246	} else {
   2247		/* each nexthop is packed in an attribute */
   2248		size_t nhsize = 0;
   2249
   2250		for_nexthops(rt) {
   2251			if (!nh->nh_dev)
   2252				continue;
   2253			nhsize += nla_total_size(sizeof(struct rtnexthop));
   2254			/* RTA_VIA */
   2255			if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC)
   2256				nhsize += nla_total_size(2 + nh->nh_via_alen);
   2257			if (nh->nh_labels)
   2258				nhsize += nla_total_size(nh->nh_labels * 4);
   2259		} endfor_nexthops(rt);
   2260		/* nested attribute */
   2261		payload += nla_total_size(nhsize);
   2262	}
   2263
   2264	return payload;
   2265}
   2266
   2267static void rtmsg_lfib(int event, u32 label, struct mpls_route *rt,
   2268		       struct nlmsghdr *nlh, struct net *net, u32 portid,
   2269		       unsigned int nlm_flags)
   2270{
   2271	struct sk_buff *skb;
   2272	u32 seq = nlh ? nlh->nlmsg_seq : 0;
   2273	int err = -ENOBUFS;
   2274
   2275	skb = nlmsg_new(lfib_nlmsg_size(rt), GFP_KERNEL);
   2276	if (skb == NULL)
   2277		goto errout;
   2278
   2279	err = mpls_dump_route(skb, portid, seq, event, label, rt, nlm_flags);
   2280	if (err < 0) {
   2281		/* -EMSGSIZE implies BUG in lfib_nlmsg_size */
   2282		WARN_ON(err == -EMSGSIZE);
   2283		kfree_skb(skb);
   2284		goto errout;
   2285	}
   2286	rtnl_notify(skb, net, portid, RTNLGRP_MPLS_ROUTE, nlh, GFP_KERNEL);
   2287
   2288	return;
   2289errout:
   2290	if (err < 0)
   2291		rtnl_set_sk_err(net, RTNLGRP_MPLS_ROUTE, err);
   2292}
   2293
   2294static int mpls_valid_getroute_req(struct sk_buff *skb,
   2295				   const struct nlmsghdr *nlh,
   2296				   struct nlattr **tb,
   2297				   struct netlink_ext_ack *extack)
   2298{
   2299	struct rtmsg *rtm;
   2300	int i, err;
   2301
   2302	if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*rtm))) {
   2303		NL_SET_ERR_MSG_MOD(extack,
   2304				   "Invalid header for get route request");
   2305		return -EINVAL;
   2306	}
   2307
   2308	if (!netlink_strict_get_check(skb))
   2309		return nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX,
   2310					      rtm_mpls_policy, extack);
   2311
   2312	rtm = nlmsg_data(nlh);
   2313	if ((rtm->rtm_dst_len && rtm->rtm_dst_len != 20) ||
   2314	    rtm->rtm_src_len || rtm->rtm_tos || rtm->rtm_table ||
   2315	    rtm->rtm_protocol || rtm->rtm_scope || rtm->rtm_type) {
   2316		NL_SET_ERR_MSG_MOD(extack, "Invalid values in header for get route request");
   2317		return -EINVAL;
   2318	}
   2319	if (rtm->rtm_flags & ~RTM_F_FIB_MATCH) {
   2320		NL_SET_ERR_MSG_MOD(extack,
   2321				   "Invalid flags for get route request");
   2322		return -EINVAL;
   2323	}
   2324
   2325	err = nlmsg_parse_deprecated_strict(nlh, sizeof(*rtm), tb, RTA_MAX,
   2326					    rtm_mpls_policy, extack);
   2327	if (err)
   2328		return err;
   2329
   2330	if ((tb[RTA_DST] || tb[RTA_NEWDST]) && !rtm->rtm_dst_len) {
   2331		NL_SET_ERR_MSG_MOD(extack, "rtm_dst_len must be 20 for MPLS");
   2332		return -EINVAL;
   2333	}
   2334
   2335	for (i = 0; i <= RTA_MAX; i++) {
   2336		if (!tb[i])
   2337			continue;
   2338
   2339		switch (i) {
   2340		case RTA_DST:
   2341		case RTA_NEWDST:
   2342			break;
   2343		default:
   2344			NL_SET_ERR_MSG_MOD(extack, "Unsupported attribute in get route request");
   2345			return -EINVAL;
   2346		}
   2347	}
   2348
   2349	return 0;
   2350}
   2351
   2352static int mpls_getroute(struct sk_buff *in_skb, struct nlmsghdr *in_nlh,
   2353			 struct netlink_ext_ack *extack)
   2354{
   2355	struct net *net = sock_net(in_skb->sk);
   2356	u32 portid = NETLINK_CB(in_skb).portid;
   2357	u32 in_label = LABEL_NOT_SPECIFIED;
   2358	struct nlattr *tb[RTA_MAX + 1];
   2359	u32 labels[MAX_NEW_LABELS];
   2360	struct mpls_shim_hdr *hdr;
   2361	unsigned int hdr_size = 0;
   2362	const struct mpls_nh *nh;
   2363	struct net_device *dev;
   2364	struct mpls_route *rt;
   2365	struct rtmsg *rtm, *r;
   2366	struct nlmsghdr *nlh;
   2367	struct sk_buff *skb;
   2368	u8 n_labels;
   2369	int err;
   2370
   2371	err = mpls_valid_getroute_req(in_skb, in_nlh, tb, extack);
   2372	if (err < 0)
   2373		goto errout;
   2374
   2375	rtm = nlmsg_data(in_nlh);
   2376
   2377	if (tb[RTA_DST]) {
   2378		u8 label_count;
   2379
   2380		if (nla_get_labels(tb[RTA_DST], 1, &label_count,
   2381				   &in_label, extack)) {
   2382			err = -EINVAL;
   2383			goto errout;
   2384		}
   2385
   2386		if (!mpls_label_ok(net, &in_label, extack)) {
   2387			err = -EINVAL;
   2388			goto errout;
   2389		}
   2390	}
   2391
   2392	rt = mpls_route_input_rcu(net, in_label);
   2393	if (!rt) {
   2394		err = -ENETUNREACH;
   2395		goto errout;
   2396	}
   2397
   2398	if (rtm->rtm_flags & RTM_F_FIB_MATCH) {
   2399		skb = nlmsg_new(lfib_nlmsg_size(rt), GFP_KERNEL);
   2400		if (!skb) {
   2401			err = -ENOBUFS;
   2402			goto errout;
   2403		}
   2404
   2405		err = mpls_dump_route(skb, portid, in_nlh->nlmsg_seq,
   2406				      RTM_NEWROUTE, in_label, rt, 0);
   2407		if (err < 0) {
   2408			/* -EMSGSIZE implies BUG in lfib_nlmsg_size */
   2409			WARN_ON(err == -EMSGSIZE);
   2410			goto errout_free;
   2411		}
   2412
   2413		return rtnl_unicast(skb, net, portid);
   2414	}
   2415
   2416	if (tb[RTA_NEWDST]) {
   2417		if (nla_get_labels(tb[RTA_NEWDST], MAX_NEW_LABELS, &n_labels,
   2418				   labels, extack) != 0) {
   2419			err = -EINVAL;
   2420			goto errout;
   2421		}
   2422
   2423		hdr_size = n_labels * sizeof(struct mpls_shim_hdr);
   2424	}
   2425
   2426	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
   2427	if (!skb) {
   2428		err = -ENOBUFS;
   2429		goto errout;
   2430	}
   2431
   2432	skb->protocol = htons(ETH_P_MPLS_UC);
   2433
   2434	if (hdr_size) {
   2435		bool bos;
   2436		int i;
   2437
   2438		if (skb_cow(skb, hdr_size)) {
   2439			err = -ENOBUFS;
   2440			goto errout_free;
   2441		}
   2442
   2443		skb_reserve(skb, hdr_size);
   2444		skb_push(skb, hdr_size);
   2445		skb_reset_network_header(skb);
   2446
   2447		/* Push new labels */
   2448		hdr = mpls_hdr(skb);
   2449		bos = true;
   2450		for (i = n_labels - 1; i >= 0; i--) {
   2451			hdr[i] = mpls_entry_encode(labels[i],
   2452						   1, 0, bos);
   2453			bos = false;
   2454		}
   2455	}
   2456
   2457	nh = mpls_select_multipath(rt, skb);
   2458	if (!nh) {
   2459		err = -ENETUNREACH;
   2460		goto errout_free;
   2461	}
   2462
   2463	if (hdr_size) {
   2464		skb_pull(skb, hdr_size);
   2465		skb_reset_network_header(skb);
   2466	}
   2467
   2468	nlh = nlmsg_put(skb, portid, in_nlh->nlmsg_seq,
   2469			RTM_NEWROUTE, sizeof(*r), 0);
   2470	if (!nlh) {
   2471		err = -EMSGSIZE;
   2472		goto errout_free;
   2473	}
   2474
   2475	r = nlmsg_data(nlh);
   2476	r->rtm_family	 = AF_MPLS;
   2477	r->rtm_dst_len	= 20;
   2478	r->rtm_src_len	= 0;
   2479	r->rtm_table	= RT_TABLE_MAIN;
   2480	r->rtm_type	= RTN_UNICAST;
   2481	r->rtm_scope	= RT_SCOPE_UNIVERSE;
   2482	r->rtm_protocol = rt->rt_protocol;
   2483	r->rtm_flags	= 0;
   2484
   2485	if (nla_put_labels(skb, RTA_DST, 1, &in_label))
   2486		goto nla_put_failure;
   2487
   2488	if (nh->nh_labels &&
   2489	    nla_put_labels(skb, RTA_NEWDST, nh->nh_labels,
   2490			   nh->nh_label))
   2491		goto nla_put_failure;
   2492
   2493	if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC &&
   2494	    nla_put_via(skb, nh->nh_via_table, mpls_nh_via(rt, nh),
   2495			nh->nh_via_alen))
   2496		goto nla_put_failure;
   2497	dev = nh->nh_dev;
   2498	if (dev && nla_put_u32(skb, RTA_OIF, dev->ifindex))
   2499		goto nla_put_failure;
   2500
   2501	nlmsg_end(skb, nlh);
   2502
   2503	err = rtnl_unicast(skb, net, portid);
   2504errout:
   2505	return err;
   2506
   2507nla_put_failure:
   2508	nlmsg_cancel(skb, nlh);
   2509	err = -EMSGSIZE;
   2510errout_free:
   2511	kfree_skb(skb);
   2512	return err;
   2513}
   2514
   2515static int resize_platform_label_table(struct net *net, size_t limit)
   2516{
   2517	size_t size = sizeof(struct mpls_route *) * limit;
   2518	size_t old_limit;
   2519	size_t cp_size;
   2520	struct mpls_route __rcu **labels = NULL, **old;
   2521	struct mpls_route *rt0 = NULL, *rt2 = NULL;
   2522	unsigned index;
   2523
   2524	if (size) {
   2525		labels = kvzalloc(size, GFP_KERNEL);
   2526		if (!labels)
   2527			goto nolabels;
   2528	}
   2529
   2530	/* In case the predefined labels need to be populated */
   2531	if (limit > MPLS_LABEL_IPV4NULL) {
   2532		struct net_device *lo = net->loopback_dev;
   2533		rt0 = mpls_rt_alloc(1, lo->addr_len, 0);
   2534		if (IS_ERR(rt0))
   2535			goto nort0;
   2536		rt0->rt_nh->nh_dev = lo;
   2537		rt0->rt_protocol = RTPROT_KERNEL;
   2538		rt0->rt_payload_type = MPT_IPV4;
   2539		rt0->rt_ttl_propagate = MPLS_TTL_PROP_DEFAULT;
   2540		rt0->rt_nh->nh_via_table = NEIGH_LINK_TABLE;
   2541		rt0->rt_nh->nh_via_alen = lo->addr_len;
   2542		memcpy(__mpls_nh_via(rt0, rt0->rt_nh), lo->dev_addr,
   2543		       lo->addr_len);
   2544	}
   2545	if (limit > MPLS_LABEL_IPV6NULL) {
   2546		struct net_device *lo = net->loopback_dev;
   2547		rt2 = mpls_rt_alloc(1, lo->addr_len, 0);
   2548		if (IS_ERR(rt2))
   2549			goto nort2;
   2550		rt2->rt_nh->nh_dev = lo;
   2551		rt2->rt_protocol = RTPROT_KERNEL;
   2552		rt2->rt_payload_type = MPT_IPV6;
   2553		rt2->rt_ttl_propagate = MPLS_TTL_PROP_DEFAULT;
   2554		rt2->rt_nh->nh_via_table = NEIGH_LINK_TABLE;
   2555		rt2->rt_nh->nh_via_alen = lo->addr_len;
   2556		memcpy(__mpls_nh_via(rt2, rt2->rt_nh), lo->dev_addr,
   2557		       lo->addr_len);
   2558	}
   2559
   2560	rtnl_lock();
   2561	/* Remember the original table */
   2562	old = rtnl_dereference(net->mpls.platform_label);
   2563	old_limit = net->mpls.platform_labels;
   2564
   2565	/* Free any labels beyond the new table */
   2566	for (index = limit; index < old_limit; index++)
   2567		mpls_route_update(net, index, NULL, NULL);
   2568
   2569	/* Copy over the old labels */
   2570	cp_size = size;
   2571	if (old_limit < limit)
   2572		cp_size = old_limit * sizeof(struct mpls_route *);
   2573
   2574	memcpy(labels, old, cp_size);
   2575
   2576	/* If needed set the predefined labels */
   2577	if ((old_limit <= MPLS_LABEL_IPV6NULL) &&
   2578	    (limit > MPLS_LABEL_IPV6NULL)) {
   2579		RCU_INIT_POINTER(labels[MPLS_LABEL_IPV6NULL], rt2);
   2580		rt2 = NULL;
   2581	}
   2582
   2583	if ((old_limit <= MPLS_LABEL_IPV4NULL) &&
   2584	    (limit > MPLS_LABEL_IPV4NULL)) {
   2585		RCU_INIT_POINTER(labels[MPLS_LABEL_IPV4NULL], rt0);
   2586		rt0 = NULL;
   2587	}
   2588
   2589	/* Update the global pointers */
   2590	net->mpls.platform_labels = limit;
   2591	rcu_assign_pointer(net->mpls.platform_label, labels);
   2592
   2593	rtnl_unlock();
   2594
   2595	mpls_rt_free(rt2);
   2596	mpls_rt_free(rt0);
   2597
   2598	if (old) {
   2599		synchronize_rcu();
   2600		kvfree(old);
   2601	}
   2602	return 0;
   2603
   2604nort2:
   2605	mpls_rt_free(rt0);
   2606nort0:
   2607	kvfree(labels);
   2608nolabels:
   2609	return -ENOMEM;
   2610}
   2611
   2612static int mpls_platform_labels(struct ctl_table *table, int write,
   2613				void *buffer, size_t *lenp, loff_t *ppos)
   2614{
   2615	struct net *net = table->data;
   2616	int platform_labels = net->mpls.platform_labels;
   2617	int ret;
   2618	struct ctl_table tmp = {
   2619		.procname	= table->procname,
   2620		.data		= &platform_labels,
   2621		.maxlen		= sizeof(int),
   2622		.mode		= table->mode,
   2623		.extra1		= SYSCTL_ZERO,
   2624		.extra2		= &label_limit,
   2625	};
   2626
   2627	ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);
   2628
   2629	if (write && ret == 0)
   2630		ret = resize_platform_label_table(net, platform_labels);
   2631
   2632	return ret;
   2633}
   2634
   2635#define MPLS_NS_SYSCTL_OFFSET(field)		\
   2636	(&((struct net *)0)->field)
   2637
   2638static const struct ctl_table mpls_table[] = {
   2639	{
   2640		.procname	= "platform_labels",
   2641		.data		= NULL,
   2642		.maxlen		= sizeof(int),
   2643		.mode		= 0644,
   2644		.proc_handler	= mpls_platform_labels,
   2645	},
   2646	{
   2647		.procname	= "ip_ttl_propagate",
   2648		.data		= MPLS_NS_SYSCTL_OFFSET(mpls.ip_ttl_propagate),
   2649		.maxlen		= sizeof(int),
   2650		.mode		= 0644,
   2651		.proc_handler	= proc_dointvec_minmax,
   2652		.extra1		= SYSCTL_ZERO,
   2653		.extra2		= SYSCTL_ONE,
   2654	},
   2655	{
   2656		.procname	= "default_ttl",
   2657		.data		= MPLS_NS_SYSCTL_OFFSET(mpls.default_ttl),
   2658		.maxlen		= sizeof(int),
   2659		.mode		= 0644,
   2660		.proc_handler	= proc_dointvec_minmax,
   2661		.extra1		= SYSCTL_ONE,
   2662		.extra2		= &ttl_max,
   2663	},
   2664	{ }
   2665};
   2666
   2667static int mpls_net_init(struct net *net)
   2668{
   2669	struct ctl_table *table;
   2670	int i;
   2671
   2672	net->mpls.platform_labels = 0;
   2673	net->mpls.platform_label = NULL;
   2674	net->mpls.ip_ttl_propagate = 1;
   2675	net->mpls.default_ttl = 255;
   2676
   2677	table = kmemdup(mpls_table, sizeof(mpls_table), GFP_KERNEL);
   2678	if (table == NULL)
   2679		return -ENOMEM;
   2680
   2681	/* Table data contains only offsets relative to the base of
   2682	 * the mdev at this point, so make them absolute.
   2683	 */
   2684	for (i = 0; i < ARRAY_SIZE(mpls_table) - 1; i++)
   2685		table[i].data = (char *)net + (uintptr_t)table[i].data;
   2686
   2687	net->mpls.ctl = register_net_sysctl(net, "net/mpls", table);
   2688	if (net->mpls.ctl == NULL) {
   2689		kfree(table);
   2690		return -ENOMEM;
   2691	}
   2692
   2693	return 0;
   2694}
   2695
   2696static void mpls_net_exit(struct net *net)
   2697{
   2698	struct mpls_route __rcu **platform_label;
   2699	size_t platform_labels;
   2700	struct ctl_table *table;
   2701	unsigned int index;
   2702
   2703	table = net->mpls.ctl->ctl_table_arg;
   2704	unregister_net_sysctl_table(net->mpls.ctl);
   2705	kfree(table);
   2706
   2707	/* An rcu grace period has passed since there was a device in
   2708	 * the network namespace (and thus the last in flight packet)
   2709	 * left this network namespace.  This is because
   2710	 * unregister_netdevice_many and netdev_run_todo has completed
   2711	 * for each network device that was in this network namespace.
   2712	 *
   2713	 * As such no additional rcu synchronization is necessary when
   2714	 * freeing the platform_label table.
   2715	 */
   2716	rtnl_lock();
   2717	platform_label = rtnl_dereference(net->mpls.platform_label);
   2718	platform_labels = net->mpls.platform_labels;
   2719	for (index = 0; index < platform_labels; index++) {
   2720		struct mpls_route *rt = rtnl_dereference(platform_label[index]);
   2721		RCU_INIT_POINTER(platform_label[index], NULL);
   2722		mpls_notify_route(net, index, rt, NULL, NULL);
   2723		mpls_rt_free(rt);
   2724	}
   2725	rtnl_unlock();
   2726
   2727	kvfree(platform_label);
   2728}
   2729
   2730static struct pernet_operations mpls_net_ops = {
   2731	.init = mpls_net_init,
   2732	.exit = mpls_net_exit,
   2733};
   2734
   2735static struct rtnl_af_ops mpls_af_ops __read_mostly = {
   2736	.family		   = AF_MPLS,
   2737	.fill_stats_af	   = mpls_fill_stats_af,
   2738	.get_stats_af_size = mpls_get_stats_af_size,
   2739};
   2740
   2741static int __init mpls_init(void)
   2742{
   2743	int err;
   2744
   2745	BUILD_BUG_ON(sizeof(struct mpls_shim_hdr) != 4);
   2746
   2747	err = register_pernet_subsys(&mpls_net_ops);
   2748	if (err)
   2749		goto out;
   2750
   2751	err = register_netdevice_notifier(&mpls_dev_notifier);
   2752	if (err)
   2753		goto out_unregister_pernet;
   2754
   2755	dev_add_pack(&mpls_packet_type);
   2756
   2757	rtnl_af_register(&mpls_af_ops);
   2758
   2759	rtnl_register_module(THIS_MODULE, PF_MPLS, RTM_NEWROUTE,
   2760			     mpls_rtm_newroute, NULL, 0);
   2761	rtnl_register_module(THIS_MODULE, PF_MPLS, RTM_DELROUTE,
   2762			     mpls_rtm_delroute, NULL, 0);
   2763	rtnl_register_module(THIS_MODULE, PF_MPLS, RTM_GETROUTE,
   2764			     mpls_getroute, mpls_dump_routes, 0);
   2765	rtnl_register_module(THIS_MODULE, PF_MPLS, RTM_GETNETCONF,
   2766			     mpls_netconf_get_devconf,
   2767			     mpls_netconf_dump_devconf, 0);
   2768	err = ipgre_tunnel_encap_add_mpls_ops();
   2769	if (err)
   2770		pr_err("Can't add mpls over gre tunnel ops\n");
   2771
   2772	err = 0;
   2773out:
   2774	return err;
   2775
   2776out_unregister_pernet:
   2777	unregister_pernet_subsys(&mpls_net_ops);
   2778	goto out;
   2779}
   2780module_init(mpls_init);
   2781
   2782static void __exit mpls_exit(void)
   2783{
   2784	rtnl_unregister_all(PF_MPLS);
   2785	rtnl_af_unregister(&mpls_af_ops);
   2786	dev_remove_pack(&mpls_packet_type);
   2787	unregister_netdevice_notifier(&mpls_dev_notifier);
   2788	unregister_pernet_subsys(&mpls_net_ops);
   2789	ipgre_tunnel_encap_del_mpls_ops();
   2790}
   2791module_exit(mpls_exit);
   2792
   2793MODULE_DESCRIPTION("MultiProtocol Label Switching");
   2794MODULE_LICENSE("GPL v2");
   2795MODULE_ALIAS_NETPROTO(PF_MPLS);