// SPDX-License-Identifier: GPL-2.0-or-later

/*

* NET3 Protocol independent device support routines.

*

* Derived from the non IP parts of dev.c 1.0.19

* Authors: Ross Biro

* Fred N. van Kempen, <[email protected]>

* Mark Evans, <[email protected]>

*

* Additional Authors:

* Florian la Roche <[email protected]>

* Alan Cox <[email protected]>

* David Hinds <[email protected]>

* Alexey Kuznetsov <[email protected]>

* Adam Sulmicki <[email protected]>

* Pekka Riikonen <[email protected]>

*

* Changes:

* D.J. Barrow : Fixed bug where dev->refcnt gets set

* to 2 if register_netdev gets called

* before net_dev_init & also removed a

* few lines of code in the process.

* Alan Cox : device private ioctl copies fields back.

* Alan Cox : Transmit queue code does relevant

* stunts to keep the queue safe.

* Alan Cox : Fixed double lock.

* Alan Cox : Fixed promisc NULL pointer trap

* ???????? : Support the full private ioctl range

* Alan Cox : Moved ioctl permission check into

* drivers

* Tim Kordas : SIOCADDMULTI/SIOCDELMULTI

* Alan Cox : 100 backlog just doesn't cut it when

* you start doing multicast video 8)

* Alan Cox : Rewrote net_bh and list manager.

* Alan Cox : Fix ETH_P_ALL echoback lengths.

* Alan Cox : Took out transmit every packet pass

* Saved a few bytes in the ioctl handler

* Alan Cox : Network driver sets packet type before

* calling netif_rx. Saves a function

* call a packet.

* Alan Cox : Hashed net_bh()

* Richard Kooijman: Timestamp fixes.

* Alan Cox : Wrong field in SIOCGIFDSTADDR

* Alan Cox : Device lock protection.

* Alan Cox : Fixed nasty side effect of device close

* changes.

* Rudi Cilibrasi : Pass the right thing to

* set_mac_address()

* Dave Miller : 32bit quantity for the device lock to

* make it work out on a Sparc.

* Bjorn Ekwall : Added KERNELD hack.

* Alan Cox : Cleaned up the backlog initialise.

* Craig Metz : SIOCGIFCONF fix if space for under

* 1 device.

* Thomas Bogendoerfer : Return ENODEV for dev_open, if there

* is no device open function.

* Andi Kleen : Fix error reporting for SIOCGIFCONF

* Michael Chastain : Fix signed/unsigned for SIOCGIFCONF

* Cyrus Durgin : Cleaned for KMOD

* Adam Sulmicki : Bug Fix : Network Device Unload

* A network device unload needs to purge

* the backlog queue.

* Paul Rusty Russell : SIOCSIFNAME

* Pekka Riikonen : Netdev boot-time settings code

* Andrew Morton : Make unregister_netdevice wait

* indefinitely on dev->refcnt

* J Hadi Salim : - Backlog queue sampling

* - netif_rx() feedback

*/

#include <linux/uaccess.h>

#include <linux/bitmap.h>

#include <linux/capability.h>

#include <linux/cpu.h>

#include <linux/types.h>

#include <linux/kernel.h>

#include <linux/hash.h>

#include <linux/slab.h>

#include <linux/sched.h>

#include <linux/sched/isolation.h>

#include <linux/sched/mm.h>

#include <linux/smpboot.h>

#include <linux/mutex.h>

#include <linux/rwsem.h>

#include <linux/string.h>

#include <linux/mm.h>

#include <linux/socket.h>

#include <linux/sockios.h>

#include <linux/errno.h>

#include <linux/interrupt.h>

#include <linux/if_ether.h>

#include <linux/netdevice.h>

#include <linux/etherdevice.h>

#include <linux/ethtool.h>

#include <linux/ethtool_netlink.h>

#include <linux/skbuff.h>

#include <linux/kthread.h>

#include <linux/bpf.h>

#include <linux/bpf_trace.h>

#include <net/net_namespace.h>

#include <net/sock.h>

#include <net/busy_poll.h>

#include <linux/rtnetlink.h>

#include <linux/stat.h>

#include <net/dsa.h>

#include <net/dst.h>

#include <net/dst_metadata.h>

#include <net/gro.h>

#include <net/netdev_queues.h>

#include <net/pkt_sched.h>

#include <net/pkt_cls.h>

#include <net/checksum.h>

#include <net/xfrm.h>

#include <net/tcx.h>

#include <linux/highmem.h>

#include <linux/init.h>

#include <linux/module.h>

#include <linux/netpoll.h>

#include <linux/rcupdate.h>

#include <linux/delay.h>

#include <net/iw_handler.h>

#include <asm/current.h>

#include <linux/audit.h>

#include <linux/dmaengine.h>

#include <linux/err.h>

#include <linux/ctype.h>

#include <linux/if_arp.h>

#include <linux/if_vlan.h>

#include <linux/ip.h>

#include <net/ip.h>

#include <net/mpls.h>

#include <linux/ipv6.h>

#include <linux/in.h>

#include <linux/jhash.h>

#include <linux/random.h>

#include <trace/events/napi.h>

#include <trace/events/net.h>

#include <trace/events/skb.h>

#include <trace/events/qdisc.h>

#include <trace/events/xdp.h>

#include <linux/inetdevice.h>

#include <linux/cpu_rmap.h>

#include <linux/static_key.h>

#include <linux/hashtable.h>

#include <linux/vmalloc.h>

#include <linux/if_macvlan.h>

#include <linux/errqueue.h>

#include <linux/hrtimer.h>

#include <linux/netfilter_netdev.h>

#include <linux/crash_dump.h>

#include <linux/sctp.h>

#include <net/udp_tunnel.h>

#include <linux/net_namespace.h>

#include <linux/indirect_call_wrapper.h>

#include <net/devlink.h>

#include <linux/pm_runtime.h>

#include <linux/prandom.h>

#include <linux/once_lite.h>

#include <net/netdev_lock.h>

#include <net/netdev_rx_queue.h>

#include <net/page_pool/types.h>

#include <net/page_pool/helpers.h>

#include <net/page_pool/memory_provider.h>

#include <net/rps.h>

#include <linux/phy_link_topology.h>

#include "dev.h"

#include "devmem.h"

#include "net-sysfs.h"

static DEFINE_SPINLOCK(ptype_lock);

struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;

static int netif_rx_internal(struct sk_buff *skb);

static int call_netdevice_notifiers_extack(unsigned long val,

struct net_device *dev,

struct netlink_ext_ack *extack);

static DEFINE_MUTEX(ifalias_mutex);

/* protects napi_hash addition/deletion and napi_gen_id */

static DEFINE_SPINLOCK(napi_hash_lock);

static unsigned int napi_gen_id = NR_CPUS;

static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);

static inline void dev_base_seq_inc(struct net *net)

{

unsigned int val = net->dev_base_seq + 1;

WRITE_ONCE(net->dev_base_seq, val ?: 1);

}

static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)

{

unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));

return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];

}

static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)

{

return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];

}

#ifndef CONFIG_PREEMPT_RT

static DEFINE_STATIC_KEY_FALSE(use_backlog_threads_key);

static int __init setup_backlog_napi_threads(char *arg)

{

static_branch_enable(&use_backlog_threads_key);

return 0;

}

early_param("thread_backlog_napi", setup_backlog_napi_threads);

static bool use_backlog_threads(void)

{

return static_branch_unlikely(&use_backlog_threads_key);

}

#else

static bool use_backlog_threads(void)

{

return true;

}

#endif

static inline void backlog_lock_irq_save(struct softnet_data *sd,

unsigned long *flags)

{

if (IS_ENABLED(CONFIG_PREEMPT_RT)) {

spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags);

} else {

local_irq_save(*flags);

if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads())

spin_lock(&sd->input_pkt_queue.lock);

}

}

static inline void backlog_lock_irq_disable(struct softnet_data *sd)

{

if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads())

spin_lock_irq(&sd->input_pkt_queue.lock);

else

local_irq_disable();

}

static inline void backlog_unlock_irq_restore(struct softnet_data *sd,

unsigned long flags)

{

if (IS_ENABLED(CONFIG_PREEMPT_RT)) {

spin_unlock_irqrestore(&sd->input_pkt_queue.lock, flags);

} else {

if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads())

spin_unlock(&sd->input_pkt_queue.lock);

local_irq_restore(flags);

}

}

static inline void backlog_unlock_irq_enable(struct softnet_data *sd)

{

if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads())

spin_unlock_irq(&sd->input_pkt_queue.lock);

else

local_irq_enable();

}

static struct netdev_name_node *netdev_name_node_alloc(struct net_device *dev,

const char *name)

{

struct netdev_name_node *name_node;

name_node = kmalloc_obj(*name_node);

if (!name_node)

return NULL;

INIT_HLIST_NODE(&name_node->hlist);

name_node->dev = dev;

name_node->name = name;

return name_node;

}

static struct netdev_name_node *

netdev_name_node_head_alloc(struct net_device *dev)

{

struct netdev_name_node *name_node;

name_node = netdev_name_node_alloc(dev, dev->name);

if (!name_node)

return NULL;

INIT_LIST_HEAD(&name_node->list);

return name_node;

}

static void netdev_name_node_free(struct netdev_name_node *name_node)

{

kfree(name_node);

}

static void netdev_name_node_add(struct net *net,

struct netdev_name_node *name_node)

{

hlist_add_head_rcu(&name_node->hlist,

dev_name_hash(net, name_node->name));

}

static void netdev_name_node_del(struct netdev_name_node *name_node)

{

hlist_del_rcu(&name_node->hlist);

}

static struct netdev_name_node *netdev_name_node_lookup(struct net *net,

const char *name)

{

struct hlist_head *head = dev_name_hash(net, name);

struct netdev_name_node *name_node;

hlist_for_each_entry(name_node, head, hlist)

if (!strcmp(name_node->name, name))

return name_node;

return NULL;

}

static struct netdev_name_node *netdev_name_node_lookup_rcu(struct net *net,

const char *name)

{

struct hlist_head *head = dev_name_hash(net, name);

struct netdev_name_node *name_node;

hlist_for_each_entry_rcu(name_node, head, hlist)

if (!strcmp(name_node->name, name))

return name_node;

return NULL;

}

bool netdev_name_in_use(struct net *net, const char *name)

{

return netdev_name_node_lookup(net, name);

}

EXPORT_SYMBOL(netdev_name_in_use);

int netdev_name_node_alt_create(struct net_device *dev, const char *name)

{

struct netdev_name_node *name_node;

struct net *net = dev_net(dev);

name_node = netdev_name_node_lookup(net, name);

if (name_node)

return -EEXIST;

name_node = netdev_name_node_alloc(dev, name);

if (!name_node)

return -ENOMEM;

netdev_name_node_add(net, name_node);

/* The node that holds dev->name acts as a head of per-device list. */

list_add_tail_rcu(&name_node->list, &dev->name_node->list);

return 0;

}

static void netdev_name_node_alt_free(struct rcu_head *head)

{

struct netdev_name_node *name_node =

container_of(head, struct netdev_name_node, rcu);

kfree(name_node->name);

netdev_name_node_free(name_node);

}

static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node)

{

netdev_name_node_del(name_node);

list_del(&name_node->list);

call_rcu(&name_node->rcu, netdev_name_node_alt_free);

}

int netdev_name_node_alt_destroy(struct net_device *dev, const char *name)

{

struct netdev_name_node *name_node;

struct net *net = dev_net(dev);

name_node = netdev_name_node_lookup(net, name);

if (!name_node)

return -ENOENT;

/* lookup might have found our primary name or a name belonging

* to another device.

*/

if (name_node == dev->name_node || name_node->dev != dev)

return -EINVAL;

__netdev_name_node_alt_destroy(name_node);

return 0;

}

static void netdev_name_node_alt_flush(struct net_device *dev)

{

struct netdev_name_node *name_node, *tmp;

list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list) {

list_del(&name_node->list);

netdev_name_node_alt_free(&name_node->rcu);

}

}

/* Device list insertion */

static void list_netdevice(struct net_device *dev)

{

struct netdev_name_node *name_node;

struct net *net = dev_net(dev);

ASSERT_RTNL();

list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);

netdev_name_node_add(net, dev->name_node);

hlist_add_head_rcu(&dev->index_hlist,

dev_index_hash(net, dev->ifindex));

netdev_for_each_altname(dev, name_node)

netdev_name_node_add(net, name_node);

/* We reserved the ifindex, this can't fail */

WARN_ON(xa_store(&net->dev_by_index, dev->ifindex, dev, GFP_KERNEL));

dev_base_seq_inc(net);

}

/* Device list removal

* caller must respect a RCU grace period before freeing/reusing dev

*/

static void unlist_netdevice(struct net_device *dev)

{

struct netdev_name_node *name_node;

struct net *net = dev_net(dev);

ASSERT_RTNL();

xa_erase(&net->dev_by_index, dev->ifindex);

netdev_for_each_altname(dev, name_node)

netdev_name_node_del(name_node);

/* Unlink dev from the device chain */

list_del_rcu(&dev->dev_list);

netdev_name_node_del(dev->name_node);

hlist_del_rcu(&dev->index_hlist);

dev_base_seq_inc(dev_net(dev));

}

/*

* Our notifier list

*/

static RAW_NOTIFIER_HEAD(netdev_chain);

/*

* Device drivers call our routines to queue packets here. We empty the

* queue in the local softnet handler.

*/

DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data) = {

.process_queue_bh_lock = INIT_LOCAL_LOCK(process_queue_bh_lock),

};

EXPORT_PER_CPU_SYMBOL(softnet_data);

/* Page_pool has a lockless array/stack to alloc/recycle pages.

* PP consumers must pay attention to run APIs in the appropriate context

* (e.g. NAPI context).

*/

DEFINE_PER_CPU(struct page_pool_bh, system_page_pool) = {

.bh_lock = INIT_LOCAL_LOCK(bh_lock),

};

#ifdef CONFIG_LOCKDEP

/*

* register_netdevice() inits txq->_xmit_lock and sets lockdep class

* according to dev->type

*/

static const unsigned short netdev_lock_type[] = {

ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,

ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,

ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,

ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,

ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,

ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,

ARPHRD_CAN, ARPHRD_MCTP,

ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,

ARPHRD_RAWHDLC, ARPHRD_RAWIP,

ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,

ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,

ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,

ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,

ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,

ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,

ARPHRD_IEEE80211_RADIOTAP,

ARPHRD_IEEE802154, ARPHRD_IEEE802154_MONITOR,

ARPHRD_PHONET, ARPHRD_PHONET_PIPE,

ARPHRD_CAIF, ARPHRD_IP6GRE, ARPHRD_NETLINK, ARPHRD_6LOWPAN,

ARPHRD_VSOCKMON,

ARPHRD_VOID, ARPHRD_NONE};

static const char *const netdev_lock_name[] = {

"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",

"_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",

"_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",

"_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",

"_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",

"_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",

"_xmit_CAN", "_xmit_MCTP",

"_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",

"_xmit_RAWHDLC", "_xmit_RAWIP",

"_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",

"_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",

"_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",

"_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",

"_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",

"_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",

"_xmit_IEEE80211_RADIOTAP",

"_xmit_IEEE802154", "_xmit_IEEE802154_MONITOR",

"_xmit_PHONET", "_xmit_PHONET_PIPE",

"_xmit_CAIF", "_xmit_IP6GRE", "_xmit_NETLINK", "_xmit_6LOWPAN",

"_xmit_VSOCKMON",

"_xmit_VOID", "_xmit_NONE"};

static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];

static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];

static inline unsigned short netdev_lock_pos(unsigned short dev_type)

{

int i;

for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)

if (netdev_lock_type[i] == dev_type)

return i;

/* the last key is used by default */

WARN_ONCE(1, "netdev_lock_pos() could not find dev_type=%u\n", dev_type);

return ARRAY_SIZE(netdev_lock_type) - 1;

}

static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,

unsigned short dev_type)

{

int i;

i = netdev_lock_pos(dev_type);

lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],

netdev_lock_name[i]);

}

static inline void netdev_set_addr_lockdep_class(struct net_device *dev)

{

int i;

i = netdev_lock_pos(dev->type);

lockdep_set_class_and_name(&dev->addr_list_lock,

&netdev_addr_lock_key[i],

netdev_lock_name[i]);

}

#else

static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,

unsigned short dev_type)

{

}

static inline void netdev_set_addr_lockdep_class(struct net_device *dev)

{

}

#endif

/*******************************************************************************

*

* Protocol management and registration routines

*

*******************************************************************************/

/*

* Add a protocol ID to the list. Now that the input handler is

* smarter we can dispense with all the messy stuff that used to be

* here.

*

* BEWARE!!! Protocol handlers, mangling input packets,

* MUST BE last in hash buckets and checking protocol handlers

* MUST start from promiscuous ptype_all chain in net_bh.

* It is true now, do not change it.

* Explanation follows: if protocol handler, mangling packet, will

* be the first on list, it is not able to sense, that packet

* is cloned and should be copied-on-write, so that it will

* change it and subsequent readers will get broken packet.

* --ANK (980803)

*/

static inline struct list_head *ptype_head(const struct packet_type *pt)

{

if (pt->type == htons(ETH_P_ALL)) {

if (!pt->af_packet_net && !pt->dev)

return NULL;

return pt->dev ? &pt->dev->ptype_all :

&pt->af_packet_net->ptype_all;

}

if (pt->dev)

return &pt->dev->ptype_specific;

return pt->af_packet_net ? &pt->af_packet_net->ptype_specific :

&ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];

}

/**

* dev_add_pack - add packet handler

* @pt: packet type declaration

*

* Add a protocol handler to the networking stack. The passed &packet_type

* is linked into kernel lists and may not be freed until it has been

* removed from the kernel lists.

*

* This call does not sleep therefore it can not

* guarantee all CPU's that are in middle of receiving packets

* will see the new packet type (until the next received packet).

*/

void dev_add_pack(struct packet_type *pt)

{

struct list_head *head = ptype_head(pt);

if (WARN_ON_ONCE(!head))

return;

spin_lock(&ptype_lock);

list_add_rcu(&pt->list, head);

spin_unlock(&ptype_lock);

}

EXPORT_SYMBOL(dev_add_pack);

/**

* __dev_remove_pack - remove packet handler

* @pt: packet type declaration

*

* Remove a protocol handler that was previously added to the kernel

* protocol handlers by dev_add_pack(). The passed &packet_type is removed

* from the kernel lists and can be freed or reused once this function

* returns.

*

* The packet type might still be in use by receivers

* and must not be freed until after all the CPU's have gone

* through a quiescent state.

*/

void __dev_remove_pack(struct packet_type *pt)

{

struct list_head *head = ptype_head(pt);

struct packet_type *pt1;

if (!head)

return;

spin_lock(&ptype_lock);

list_for_each_entry(pt1, head, list) {

if (pt == pt1) {

list_del_rcu(&pt->list);

goto out;

}

}

pr_warn("dev_remove_pack: %p not found\n", pt);

out:

spin_unlock(&ptype_lock);

}

EXPORT_SYMBOL(__dev_remove_pack);

/**

* dev_remove_pack - remove packet handler

* @pt: packet type declaration

*

* Remove a protocol handler that was previously added to the kernel

* protocol handlers by dev_add_pack(). The passed &packet_type is removed

* from the kernel lists and can be freed or reused once this function

* returns.

*

* This call sleeps to guarantee that no CPU is looking at the packet

* type after return.

*/

void dev_remove_pack(struct packet_type *pt)

{

__dev_remove_pack(pt);

synchronize_net();

}

EXPORT_SYMBOL(dev_remove_pack);

/*******************************************************************************

*

* Device Interface Subroutines

*

*******************************************************************************/

/**

* dev_get_iflink - get 'iflink' value of a interface

* @dev: targeted interface

*

* Indicates the ifindex the interface is linked to.

* Physical interfaces have the same 'ifindex' and 'iflink' values.

*/

int dev_get_iflink(const struct net_device *dev)

{

if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)

return dev->netdev_ops->ndo_get_iflink(dev);

return READ_ONCE(dev->ifindex);

}

EXPORT_SYMBOL(dev_get_iflink);

/**

* dev_fill_metadata_dst - Retrieve tunnel egress information.

* @dev: targeted interface

* @skb: The packet.

*

* For better visibility of tunnel traffic OVS needs to retrieve

* egress tunnel information for a packet. Following API allows

* user to get this info.

*/

int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)

{

struct ip_tunnel_info *info;

if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)

return -EINVAL;

info = skb_tunnel_info_unclone(skb);

if (!info)

return -ENOMEM;

if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))

return -EINVAL;

return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);

}

EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);

static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack)

{

int k = stack->num_paths++;

if (k >= NET_DEVICE_PATH_STACK_MAX)

return NULL;

return &stack->path[k];

}

int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,

struct net_device_path_stack *stack)

{

const struct net_device *last_dev;

struct net_device_path_ctx ctx = {

.dev = dev,

};

struct net_device_path *path;

int ret = 0;

memcpy(ctx.daddr, daddr, sizeof(ctx.daddr));

stack->num_paths = 0;

while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) {

last_dev = ctx.dev;

path = dev_fwd_path(stack);

if (!path)

return -1;

memset(path, 0, sizeof(struct net_device_path));

ret = ctx.dev->netdev_ops->ndo_fill_forward_path(&ctx, path);

if (ret < 0)

return -1;

if (WARN_ON_ONCE(last_dev == ctx.dev))

return -1;

}

if (!ctx.dev)

return ret;

path = dev_fwd_path(stack);

if (!path)

return -1;

path->type = DEV_PATH_ETHERNET;

path->dev = ctx.dev;

return ret;

}

EXPORT_SYMBOL_GPL(dev_fill_forward_path);

/* must be called under rcu_read_lock(), as we dont take a reference */

static struct napi_struct *napi_by_id(unsigned int napi_id)

{

unsigned int hash = napi_id % HASH_SIZE(napi_hash);

struct napi_struct *napi;

hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)

if (napi->napi_id == napi_id)

return napi;

return NULL;

}

/* must be called under rcu_read_lock(), as we dont take a reference */

static struct napi_struct *

netdev_napi_by_id(struct net *net, unsigned int napi_id)

{

struct napi_struct *napi;

napi = napi_by_id(napi_id);

if (!napi)

return NULL;

if (WARN_ON_ONCE(!napi->dev))

return NULL;

if (!net_eq(net, dev_net(napi->dev)))

return NULL;

return napi;

}

/**

* netdev_napi_by_id_lock() - find a device by NAPI ID and lock it

* @net: the applicable net namespace

* @napi_id: ID of a NAPI of a target device

*

* Find a NAPI instance with @napi_id. Lock its device.

* The device must be in %NETREG_REGISTERED state for lookup to succeed.

* netdev_unlock() must be called to release it.

*

* Return: pointer to NAPI, its device with lock held, NULL if not found.

*/

struct napi_struct *

netdev_napi_by_id_lock(struct net *net, unsigned int napi_id)

{

struct napi_struct *napi;

struct net_device *dev;

rcu_read_lock();

napi = netdev_napi_by_id(net, napi_id);

if (!napi || READ_ONCE(napi->dev->reg_state) != NETREG_REGISTERED) {

rcu_read_unlock();

return NULL;

}

dev = napi->dev;

dev_hold(dev);

rcu_read_unlock();

dev = __netdev_put_lock(dev, net);

if (!dev)

return NULL;

rcu_read_lock();

napi = netdev_napi_by_id(net, napi_id);

if (napi && napi->dev != dev)

napi = NULL;

rcu_read_unlock();

if (!napi)

netdev_unlock(dev);

return napi;

}

/**

* __dev_get_by_name - find a device by its name

* @net: the applicable net namespace

* @name: name to find

*

* Find an interface by name. Must be called under RTNL semaphore.

* If the name is found a pointer to the device is returned.

* If the name is not found then %NULL is returned. The

* reference counters are not incremented so the caller must be

* careful with locks.

*/

struct net_device *__dev_get_by_name(struct net *net, const char *name)

{

struct netdev_name_node *node_name;

node_name = netdev_name_node_lookup(net, name);

return node_name ? node_name->dev : NULL;

}

EXPORT_SYMBOL(__dev_get_by_name);

/**

* dev_get_by_name_rcu - find a device by its name

* @net: the applicable net namespace

* @name: name to find

*

* Find an interface by name.

* If the name is found a pointer to the device is returned.

* If the name is not found then %NULL is returned.

* The reference counters are not incremented so the caller must be

* careful with locks. The caller must hold RCU lock.

*/

struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)

{

struct netdev_name_node *node_name;

node_name = netdev_name_node_lookup_rcu(net, name);

return node_name ? node_name->dev : NULL;

}

EXPORT_SYMBOL(dev_get_by_name_rcu);

/* Deprecated for new users, call netdev_get_by_name() instead */

struct net_device *dev_get_by_name(struct net *net, const char *name)

{

struct net_device *dev;

rcu_read_lock();

dev = dev_get_by_name_rcu(net, name);

dev_hold(dev);

rcu_read_unlock();

return dev;

}

EXPORT_SYMBOL(dev_get_by_name);

/**

* netdev_get_by_name() - find a device by its name

* @net: the applicable net namespace

* @name: name to find

* @tracker: tracking object for the acquired reference

* @gfp: allocation flags for the tracker

*

* Find an interface by name. This can be called from any

* context and does its own locking. The returned handle has

* the usage count incremented and the caller must use netdev_put() to

* release it when it is no longer needed. %NULL is returned if no

* matching device is found.

*/

struct net_device *netdev_get_by_name(struct net *net, const char *name,

netdevice_tracker *tracker, gfp_t gfp)

{

struct net_device *dev;

dev = dev_get_by_name(net, name);

if (dev)

netdev_tracker_alloc(dev, tracker, gfp);

return dev;

}

EXPORT_SYMBOL(netdev_get_by_name);

/**

* __dev_get_by_index - find a device by its ifindex

* @net: the applicable net namespace

* @ifindex: index of device

*

* Search for an interface by index. Returns %NULL if the device

* is not found or a pointer to the device. The device has not

* had its reference counter increased so the caller must be careful

* about locking. The caller must hold the RTNL semaphore.

*/

struct net_device *__dev_get_by_index(struct net *net, int ifindex)

{

struct net_device *dev;

struct hlist_head *head = dev_index_hash(net, ifindex);

hlist_for_each_entry(dev, head, index_hlist)

if (dev->ifindex == ifindex)

return dev;

return NULL;

}

EXPORT_SYMBOL(__dev_get_by_index);

/**

* dev_get_by_index_rcu - find a device by its ifindex

* @net: the applicable net namespace

* @ifindex: index of device

*

* Search for an interface by index. Returns %NULL if the device

* is not found or a pointer to the device. The device has not

* had its reference counter increased so the caller must be careful

* about locking. The caller must hold RCU lock.

*/

struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)

{

struct net_device *dev;

struct hlist_head *head = dev_index_hash(net, ifindex);

hlist_for_each_entry_rcu(dev, head, index_hlist)

if (dev->ifindex == ifindex)

return dev;

return NULL;

}

EXPORT_SYMBOL(dev_get_by_index_rcu);

/* Deprecated for new users, call netdev_get_by_index() instead */

struct net_device *dev_get_by_index(struct net *net, int ifindex)

{

struct net_device *dev;

rcu_read_lock();