1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* INET An implementation of the TCP/IP protocol suite for the LINUX
4	* operating system. INET is implemented using the BSD Socket
5	* interface as the means of communication with the user level.
6	*
7	* PACKET - implements raw packet sockets.
8	*
9	* Authors: Ross Biro
10	* Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
11	* Alan Cox, <gw4pts@gw4pts.ampr.org>
12	*
13	* Fixes:
14	* Alan Cox : verify_area() now used correctly
15	* Alan Cox : new skbuff lists, look ma no backlogs!
16	* Alan Cox : tidied skbuff lists.
17	* Alan Cox : Now uses generic datagram routines I
18	* added. Also fixed the peek/read crash
19	* from all old Linux datagram code.
20	* Alan Cox : Uses the improved datagram code.
21	* Alan Cox : Added NULL's for socket options.
22	* Alan Cox : Re-commented the code.
23	* Alan Cox : Use new kernel side addressing
24	* Rob Janssen : Correct MTU usage.
25	* Dave Platt : Counter leaks caused by incorrect
26	* interrupt locking and some slightly
27	* dubious gcc output. Can you read
28	* compiler: it said _VOLATILE_
29	* Richard Kooijman : Timestamp fixes.
30	* Alan Cox : New buffers. Use sk->mac.raw.
31	* Alan Cox : sendmsg/recvmsg support.
32	* Alan Cox : Protocol setting support
33	* Alexey Kuznetsov : Untied from IPv4 stack.
34	* Cyrus Durgin : Fixed kerneld for kmod.
35	* Michal Ostrowski : Module initialization cleanup.
36	* Ulises Alonso : Frame number limit removal and
37	* packet_set_ring memory leak.
38	* Eric Biederman : Allow for > 8 byte hardware addresses.
39	* The convention is that longer addresses
40	* will simply extend the hardware address
41	* byte arrays at the end of sockaddr_ll
42	* and packet_mreq.
43	* Johann Baudy : Added TX RING.
44	* Chetan Loke : Implemented TPACKET_V3 block abstraction
45	* layer.
46	* Copyright (C) 2011, <lokec@ccs.neu.edu>
47	*/
48
49	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
50
51	#include <linux/ethtool.h>
52	#include <linux/filter.h>
53	#include <linux/types.h>
54	#include <linux/mm.h>
55	#include <linux/capability.h>
56	#include <linux/fcntl.h>
57	#include <linux/socket.h>
58	#include <linux/in.h>
59	#include <linux/inet.h>
60	#include <linux/netdevice.h>
61	#include <linux/if_packet.h>
62	#include <linux/wireless.h>
63	#include <linux/kernel.h>
64	#include <linux/kmod.h>
65	#include <linux/slab.h>
66	#include <linux/vmalloc.h>
67	#include <net/net_namespace.h>
68	#include <net/ip.h>
69	#include <net/protocol.h>
70	#include <linux/skbuff.h>
71	#include <net/sock.h>
72	#include <linux/errno.h>
73	#include <linux/timer.h>
74	#include <linux/uaccess.h>
75	#include <asm/ioctls.h>
76	#include <asm/page.h>
77	#include <asm/cacheflush.h>
78	#include <asm/io.h>
79	#include <linux/proc_fs.h>
80	#include <linux/seq_file.h>
81	#include <linux/poll.h>
82	#include <linux/module.h>
83	#include <linux/init.h>
84	#include <linux/mutex.h>
85	#include <linux/if_vlan.h>
86	#include <linux/virtio_net.h>
87	#include <linux/errqueue.h>
88	#include <linux/net_tstamp.h>
89	#include <linux/percpu.h>
90	#ifdef CONFIG_INET
91	#include <net/inet_common.h>
92	#endif
93	#include <linux/bpf.h>
94	#include <net/compat.h>
95	#include <linux/netfilter_netdev.h>
96
97	#include "internal.h"
98
99	/*
100	Assumptions:
101	- If the device has no dev->header_ops->create, there is no LL header
102	visible above the device. In this case, its hard_header_len should be 0.
103	The device may prepend its own header internally. In this case, its
104	needed_headroom should be set to the space needed for it to add its
105	internal header.
106	For example, a WiFi driver pretending to be an Ethernet driver should
107	set its hard_header_len to be the Ethernet header length, and set its
108	needed_headroom to be (the real WiFi header length - the fake Ethernet
109	header length).
110	- packet socket receives packets with pulled ll header,
111	so that SOCK_RAW should push it back.
112
113	On receive:
114	-----------
115
116	Incoming, dev_has_header(dev) == true
117	mac_header -> ll header
118	data -> data
119
120	Outgoing, dev_has_header(dev) == true
121	mac_header -> ll header
122	data -> ll header
123
124	Incoming, dev_has_header(dev) == false
125	mac_header -> data
126	However drivers often make it point to the ll header.
127	This is incorrect because the ll header should be invisible to us.
128	data -> data
129
130	Outgoing, dev_has_header(dev) == false
131	mac_header -> data. ll header is invisible to us.
132	data -> data
133
134	Resume
135	If dev_has_header(dev) == false we are unable to restore the ll header,
136	because it is invisible to us.
137
138
139	On transmit:
140	------------
141
142	dev_has_header(dev) == true
143	mac_header -> ll header
144	data -> ll header
145
146	dev_has_header(dev) == false (ll header is invisible to us)
147	mac_header -> data
148	data -> data
149
150	We should set network_header on output to the correct position,
151	packet classifier depends on it.
152	*/
153
154	/* Private packet socket structures. */
155
156	/* identical to struct packet_mreq except it has
157	* a longer address field.
158	*/
159	struct packet_mreq_max {
160	int mr_ifindex;
161	unsigned short mr_type;
162	unsigned short mr_alen;
163	unsigned char mr_address[MAX_ADDR_LEN];
164	};
165
166	union tpacket_uhdr {
167	struct tpacket_hdr *h1;
168	struct tpacket2_hdr *h2;
169	struct tpacket3_hdr *h3;
170	void *raw;
171	};
172
173	static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
174	int closing, int tx_ring);
175
176	#define V3_ALIGNMENT (8)
177
178	#define BLK_HDR_LEN (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))
179
180	#define BLK_PLUS_PRIV(sz_of_priv) \
181	(BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))
182
183	#define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
184	#define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
185	#define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
186	#define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
187	#define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
188	#define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
189
190	struct packet_sock;
191	static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
192	struct packet_type *pt, struct net_device *orig_dev);
193
194	static void *packet_previous_frame(struct packet_sock *po,
195	struct packet_ring_buffer *rb,
196	int status);
197	static void packet_increment_head(struct packet_ring_buffer *buff);
198	static int prb_curr_blk_in_use(struct tpacket_block_desc *);
199	static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
200	struct packet_sock *);
201	static void prb_retire_current_block(struct tpacket_kbdq_core *,
202	struct packet_sock *, unsigned int status);
203	static int prb_queue_frozen(struct tpacket_kbdq_core *);
204	static void prb_open_block(struct tpacket_kbdq_core *,
205	struct tpacket_block_desc *);
206	static enum hrtimer_restart prb_retire_rx_blk_timer_expired(struct hrtimer *);
207	static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
208	static void prb_clear_rxhash(struct tpacket_kbdq_core *,
209	struct tpacket3_hdr *);
210	static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
211	struct tpacket3_hdr *);
212	static void packet_flush_mclist(struct sock *sk);
213	static u16 packet_pick_tx_queue(struct sk_buff *skb);
214
215	struct packet_skb_cb {
216	union {
217	struct sockaddr_pkt pkt;
218	union {
219	/* Trick: alias skb original length with
220	* ll.sll_family and ll.protocol in order
221	* to save room.
222	*/
223	unsigned int origlen;
224	struct sockaddr_ll ll;
225	};
226	} sa;
227	};
228
229	#define vio_le() virtio_legacy_is_little_endian()
230
231	#define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
232
233	#define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
234	#define GET_PBLOCK_DESC(x, bid) \
235	((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
236	#define GET_CURR_PBLOCK_DESC_FROM_CORE(x) \
237	((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
238	#define GET_NEXT_PRB_BLK_NUM(x) \
239	(((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
240	((x)->kactive_blk_num+1) : 0)
241
242	static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
243	static void __fanout_link(struct sock *sk, struct packet_sock *po);
244
245	#ifdef CONFIG_NETFILTER_EGRESS
246	static noinline struct sk_buff *nf_hook_direct_egress(struct sk_buff *skb)
247	{
248	struct sk_buff *next, *head = NULL, *tail;
249	int rc;
250
251	rcu_read_lock();
252	for (; skb != NULL; skb = next) {
253	next = skb->next;
254	skb_mark_not_on_list(skb);
255
256	if (!nf_hook_egress(skb, rc: &rc, dev: skb->dev))
257	continue;
258
259	if (!head)
260	head = skb;
261	else
262	tail->next = skb;
263
264	tail = skb;
265	}
266	rcu_read_unlock();
267
268	return head;
269	}
270	#endif
271
272	static int packet_xmit(const struct packet_sock *po, struct sk_buff *skb)
273	{
274	if (!packet_sock_flag(po, flag: PACKET_SOCK_QDISC_BYPASS))
275	return dev_queue_xmit(skb);
276
277	#ifdef CONFIG_NETFILTER_EGRESS
278	if (nf_hook_egress_active()) {
279	skb = nf_hook_direct_egress(skb);
280	if (!skb)
281	return NET_XMIT_DROP;
282	}
283	#endif
284	return dev_direct_xmit(skb, queue_id: packet_pick_tx_queue(skb));
285	}
286
287	static struct net_device *packet_cached_dev_get(struct packet_sock *po)
288	{
289	struct net_device *dev;
290
291	rcu_read_lock();
292	dev = rcu_dereference(po->cached_dev);
293	dev_hold(dev);
294	rcu_read_unlock();
295
296	return dev;
297	}
298
299	static void packet_cached_dev_assign(struct packet_sock *po,
300	struct net_device *dev)
301	{
302	rcu_assign_pointer(po->cached_dev, dev);
303	}
304
305	static void packet_cached_dev_reset(struct packet_sock *po)
306	{
307	RCU_INIT_POINTER(po->cached_dev, NULL);
308	}
309
310	static u16 packet_pick_tx_queue(struct sk_buff *skb)
311	{
312	struct net_device *dev = skb->dev;
313	const struct net_device_ops *ops = dev->netdev_ops;
314	int cpu = raw_smp_processor_id();
315	u16 queue_index;
316
317	#ifdef CONFIG_XPS
318	skb->sender_cpu = cpu + 1;
319	#endif
320	skb_record_rx_queue(skb, rx_queue: cpu % dev->real_num_tx_queues);
321	if (ops->ndo_select_queue) {
322	queue_index = ops->ndo_select_queue(dev, skb, NULL);
323	queue_index = netdev_cap_txqueue(dev, queue_index);
324	} else {
325	queue_index = netdev_pick_tx(dev, skb, NULL);
326	}
327
328	return queue_index;
329	}
330
331	/* __register_prot_hook must be invoked through register_prot_hook
332	* or from a context in which asynchronous accesses to the packet
333	* socket is not possible (packet_create()).
334	*/
335	static void __register_prot_hook(struct sock *sk)
336	{
337	struct packet_sock *po = pkt_sk(sk);
338
339	if (!packet_sock_flag(po, flag: PACKET_SOCK_RUNNING)) {
340	if (po->fanout)
341	__fanout_link(sk, po);
342	else
343	dev_add_pack(pt: &po->prot_hook);
344
345	sock_hold(sk);
346	packet_sock_flag_set(po, flag: PACKET_SOCK_RUNNING, val: 1);
347	}
348	}
349
350	static void register_prot_hook(struct sock *sk)
351	{
352	lockdep_assert_held_once(&pkt_sk(sk)->bind_lock);
353	__register_prot_hook(sk);
354	}
355
356	/* If the sync parameter is true, we will temporarily drop
357	* the po->bind_lock and do a synchronize_net to make sure no
358	* asynchronous packet processing paths still refer to the elements
359	* of po->prot_hook. If the sync parameter is false, it is the
360	* callers responsibility to take care of this.
361	*/
362	static void __unregister_prot_hook(struct sock *sk, bool sync)
363	{
364	struct packet_sock *po = pkt_sk(sk);
365
366	lockdep_assert_held_once(&po->bind_lock);
367
368	packet_sock_flag_set(po, flag: PACKET_SOCK_RUNNING, val: 0);
369
370	if (po->fanout)
371	__fanout_unlink(sk, po);
372	else
373	__dev_remove_pack(pt: &po->prot_hook);
374
375	__sock_put(sk);
376
377	if (sync) {
378	spin_unlock(lock: &po->bind_lock);
379	synchronize_net();
380	spin_lock(lock: &po->bind_lock);
381	}
382	}
383
384	static void unregister_prot_hook(struct sock *sk, bool sync)
385	{
386	struct packet_sock *po = pkt_sk(sk);
387
388	if (packet_sock_flag(po, flag: PACKET_SOCK_RUNNING))
389	__unregister_prot_hook(sk, sync);
390	}
391
392	static inline struct page * __pure pgv_to_page(void *addr)
393	{
394	if (is_vmalloc_addr(x: addr))
395	return vmalloc_to_page(addr);
396	return virt_to_page(addr);
397	}
398
399	static void __packet_set_status(struct packet_sock *po, void *frame, int status)
400	{
401	union tpacket_uhdr h;
402
403	/* WRITE_ONCE() are paired with READ_ONCE() in __packet_get_status */
404
405	h.raw = frame;
406	switch (po->tp_version) {
407	case TPACKET_V1:
408	WRITE_ONCE(h.h1->tp_status, status);
409	flush_dcache_page(page: pgv_to_page(addr: &h.h1->tp_status));
410	break;
411	case TPACKET_V2:
412	WRITE_ONCE(h.h2->tp_status, status);
413	flush_dcache_page(page: pgv_to_page(addr: &h.h2->tp_status));
414	break;
415	case TPACKET_V3:
416	WRITE_ONCE(h.h3->tp_status, status);
417	flush_dcache_page(page: pgv_to_page(addr: &h.h3->tp_status));
418	break;
419	default:
420	WARN(1, "TPACKET version not supported.\n");
421	BUG();
422	}
423
424	smp_wmb();
425	}
426
427	static int __packet_get_status(const struct packet_sock *po, void *frame)
428	{
429	union tpacket_uhdr h;
430
431	smp_rmb();
432
433	/* READ_ONCE() are paired with WRITE_ONCE() in __packet_set_status */
434
435	h.raw = frame;
436	switch (po->tp_version) {
437	case TPACKET_V1:
438	flush_dcache_page(page: pgv_to_page(addr: &h.h1->tp_status));
439	return READ_ONCE(h.h1->tp_status);
440	case TPACKET_V2:
441	flush_dcache_page(page: pgv_to_page(addr: &h.h2->tp_status));
442	return READ_ONCE(h.h2->tp_status);
443	case TPACKET_V3:
444	flush_dcache_page(page: pgv_to_page(addr: &h.h3->tp_status));
445	return READ_ONCE(h.h3->tp_status);
446	default:
447	WARN(1, "TPACKET version not supported.\n");
448	BUG();
449	return 0;
450	}
451	}
452
453	static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec64 *ts,
454	unsigned int flags)
455	{
456	struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
457
458	if (shhwtstamps &&
459	(flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
460	ktime_to_timespec64_cond(kt: shhwtstamps->hwtstamp, ts))
461	return TP_STATUS_TS_RAW_HARDWARE;
462
463	if ((flags & SOF_TIMESTAMPING_SOFTWARE) &&
464	ktime_to_timespec64_cond(kt: skb_tstamp(skb), ts))
465	return TP_STATUS_TS_SOFTWARE;
466
467	return 0;
468	}
469
470	static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
471	struct sk_buff *skb)
472	{
473	union tpacket_uhdr h;
474	struct timespec64 ts;
475	__u32 ts_status;
476
477	if (!(ts_status = tpacket_get_timestamp(skb, ts: &ts, READ_ONCE(po->tp_tstamp))))
478	return 0;
479
480	h.raw = frame;
481	/*
482	* versions 1 through 3 overflow the timestamps in y2106, since they
483	* all store the seconds in a 32-bit unsigned integer.
484	* If we create a version 4, that should have a 64-bit timestamp,
485	* either 64-bit seconds + 32-bit nanoseconds, or just 64-bit
486	* nanoseconds.
487	*/
488	switch (po->tp_version) {
489	case TPACKET_V1:
490	h.h1->tp_sec = ts.tv_sec;
491	h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
492	break;
493	case TPACKET_V2:
494	h.h2->tp_sec = ts.tv_sec;
495	h.h2->tp_nsec = ts.tv_nsec;
496	break;
497	case TPACKET_V3:
498	h.h3->tp_sec = ts.tv_sec;
499	h.h3->tp_nsec = ts.tv_nsec;
500	break;
501	default:
502	WARN(1, "TPACKET version not supported.\n");
503	BUG();
504	}
505
506	/* one flush is safe, as both fields always lie on the same cacheline */
507	flush_dcache_page(page: pgv_to_page(addr: &h.h1->tp_sec));
508	smp_wmb();
509
510	return ts_status;
511	}
512
513	static void *packet_lookup_frame(const struct packet_sock *po,
514	const struct packet_ring_buffer *rb,
515	unsigned int position,
516	int status)
517	{
518	unsigned int pg_vec_pos, frame_offset;
519	union tpacket_uhdr h;
520
521	pg_vec_pos = position / rb->frames_per_block;
522	frame_offset = position % rb->frames_per_block;
523
524	h.raw = rb->pg_vec[pg_vec_pos].buffer +
525	(frame_offset * rb->frame_size);
526
527	if (status != __packet_get_status(po, frame: h.raw))
528	return NULL;
529
530	return h.raw;
531	}
532
533	static void *packet_current_frame(struct packet_sock *po,
534	struct packet_ring_buffer *rb,
535	int status)
536	{
537	return packet_lookup_frame(po, rb, position: rb->head, status);
538	}
539
540	static u16 vlan_get_tci(const struct sk_buff *skb, struct net_device *dev)
541	{
542	struct vlan_hdr vhdr, *vh;
543	unsigned int header_len;
544
545	if (!dev)
546	return 0;
547
548	/* In the SOCK_DGRAM scenario, skb data starts at the network
549	* protocol, which is after the VLAN headers. The outer VLAN
550	* header is at the hard_header_len offset in non-variable
551	* length link layer headers. If it's a VLAN device, the
552	* min_header_len should be used to exclude the VLAN header
553	* size.
554	*/
555	if (dev->min_header_len == dev->hard_header_len)
556	header_len = dev->hard_header_len;
557	else if (is_vlan_dev(dev))
558	header_len = dev->min_header_len;
559	else
560	return 0;
561
562	vh = skb_header_pointer(skb, offset: skb_mac_offset(skb) + header_len,
563	len: sizeof(vhdr), buffer: &vhdr);
564	if (unlikely(!vh))
565	return 0;
566
567	return ntohs(vh->h_vlan_TCI);
568	}
569
570	static __be16 vlan_get_protocol_dgram(const struct sk_buff *skb)
571	{
572	__be16 proto = skb->protocol;
573
574	if (unlikely(eth_type_vlan(proto)))
575	proto = __vlan_get_protocol_offset(skb, type: proto,
576	mac_offset: skb_mac_offset(skb), NULL);
577
578	return proto;
579	}
580
581	static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
582	struct sk_buff_head *rb_queue)
583	{
584	struct tpacket_kbdq_core *pkc;
585
586	pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
587	hrtimer_cancel(timer: &pkc->retire_blk_timer);
588	}
589
590	static int prb_calc_retire_blk_tmo(struct packet_sock *po,
591	int blk_size_in_bytes)
592	{
593	struct net_device *dev;
594	unsigned int mbits, div;
595	struct ethtool_link_ksettings ecmd;
596	int err;
597
598	rtnl_lock();
599	dev = __dev_get_by_index(net: sock_net(sk: &po->sk), ifindex: po->ifindex);
600	if (unlikely(!dev)) {
601	rtnl_unlock();
602	return DEFAULT_PRB_RETIRE_TOV;
603	}
604	err = __ethtool_get_link_ksettings(dev, link_ksettings: &ecmd);
605	rtnl_unlock();
606	if (err)
607	return DEFAULT_PRB_RETIRE_TOV;
608
609	/* If the link speed is so slow you don't really
610	* need to worry about perf anyways
611	*/
612	if (ecmd.base.speed < SPEED_1000 ||
613	ecmd.base.speed == SPEED_UNKNOWN)
614	return DEFAULT_PRB_RETIRE_TOV;
615
616	div = ecmd.base.speed / 1000;
617	mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
618
619	if (div)
620	mbits /= div;
621
622	if (div)
623	return mbits + 1;
624	return mbits;
625	}
626
627	static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
628	union tpacket_req_u *req_u)
629	{
630	p1->feature_req_word = req_u->req3.tp_feature_req_word;
631	}
632
633	static void init_prb_bdqc(struct packet_sock *po,
634	struct packet_ring_buffer *rb,
635	struct pgv *pg_vec,
636	union tpacket_req_u *req_u)
637	{
638	struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
639	struct tpacket_block_desc *pbd;
640
641	memset(p1, 0x0, sizeof(*p1));
642
643	p1->knxt_seq_num = 1;
644	p1->pkbdq = pg_vec;
645	pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
646	p1->pkblk_start = pg_vec[0].buffer;
647	p1->kblk_size = req_u->req3.tp_block_size;
648	p1->knum_blocks = req_u->req3.tp_block_nr;
649	p1->hdrlen = po->tp_hdrlen;
650	p1->version = po->tp_version;
651	po->stats.stats3.tp_freeze_q_cnt = 0;
652	if (req_u->req3.tp_retire_blk_tov)
653	p1->interval_ktime = ms_to_ktime(ms: req_u->req3.tp_retire_blk_tov);
654	else
655	p1->interval_ktime = ms_to_ktime(ms: prb_calc_retire_blk_tmo(po,
656	blk_size_in_bytes: req_u->req3.tp_block_size));
657	p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
658	rwlock_init(&p1->blk_fill_in_prog_lock);
659
660	p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
661	prb_init_ft_ops(p1, req_u);
662	hrtimer_setup(timer: &p1->retire_blk_timer, function: prb_retire_rx_blk_timer_expired,
663	CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL_SOFT);
664	hrtimer_start(timer: &p1->retire_blk_timer, tim: p1->interval_ktime,
665	mode: HRTIMER_MODE_REL_SOFT);
666	prb_open_block(p1, pbd);
667	}
668
669	/*
670	* With a 1MB block-size, on a 1Gbps line, it will take
671	* i) ~8 ms to fill a block + ii) memcpy etc.
672	* In this cut we are not accounting for the memcpy time.
673	*
674	* Since the tmo granularity is in msecs, it is not too expensive
675	* to refresh the timer, lets say every '8' msecs.
676	* Either the user can set the 'tmo' or we can derive it based on
677	* a) line-speed and b) block-size.
678	* prb_calc_retire_blk_tmo() calculates the tmo.
679	*/
680	static enum hrtimer_restart prb_retire_rx_blk_timer_expired(struct hrtimer *t)
681	{
682	struct packet_sock *po =
683	timer_container_of(po, t, rx_ring.prb_bdqc.retire_blk_timer);
684	struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
685	unsigned int frozen;
686	struct tpacket_block_desc *pbd;
687
688	spin_lock(lock: &po->sk.sk_receive_queue.lock);
689
690	frozen = prb_queue_frozen(pkc);
691	pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
692
693	/* We only need to plug the race when the block is partially filled.
694	* tpacket_rcv:
695	* lock(); increment BLOCK_NUM_PKTS; unlock()
696	* copy_bits() is in progress ...
697	* timer fires on other cpu:
698	* we can't retire the current block because copy_bits
699	* is in progress.
700	*
701	*/
702	if (BLOCK_NUM_PKTS(pbd)) {
703	/* Waiting for skb_copy_bits to finish... */
704	write_lock(&pkc->blk_fill_in_prog_lock);
705	write_unlock(&pkc->blk_fill_in_prog_lock);
706	}
707
708	if (!frozen) {
709	if (BLOCK_NUM_PKTS(pbd)) {
710	/* Not an empty block. Need retire the block. */
711	prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
712	prb_dispatch_next_block(pkc, po);
713	}
714	} else {
715	/* Case 1. Queue was frozen because user-space was
716	* lagging behind.
717	*/
718	if (!prb_curr_blk_in_use(pbd)) {
719	/* Case 2. queue was frozen,user-space caught up,
720	* now the link went idle && the timer fired.
721	* We don't have a block to close.So we open this
722	* block and restart the timer.
723	* opening a block thaws the queue,restarts timer
724	* Thawing/timer-refresh is a side effect.
725	*/
726	prb_open_block(pkc, pbd);
727	}
728	}
729
730	hrtimer_forward_now(timer: &pkc->retire_blk_timer, interval: pkc->interval_ktime);
731	spin_unlock(lock: &po->sk.sk_receive_queue.lock);
732	return HRTIMER_RESTART;
733	}
734
735	static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
736	struct tpacket_block_desc *pbd1, __u32 status)
737	{
738	/* Flush everything minus the block header */
739
740	#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
741	u8 *start, *end;
742
743	start = (u8 *)pbd1;
744
745	/* Skip the block header(we know header WILL fit in 4K) */
746	start += PAGE_SIZE;
747
748	end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
749	for (; start < end; start += PAGE_SIZE)
750	flush_dcache_page(pgv_to_page(start));
751
752	smp_wmb();
753	#endif
754
755	/* Now update the block status. */
756
757	BLOCK_STATUS(pbd1) = status;
758
759	/* Flush the block header */
760
761	#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
762	start = (u8 *)pbd1;
763	flush_dcache_page(pgv_to_page(start));
764
765	smp_wmb();
766	#endif
767	}
768
769	/*
770	* Side effect:
771	*
772	* 1) flush the block
773	* 2) Increment active_blk_num
774	*
775	* Note:We DONT refresh the timer on purpose.
776	* Because almost always the next block will be opened.
777	*/
778	static void prb_close_block(struct tpacket_kbdq_core *pkc1,
779	struct tpacket_block_desc *pbd1,
780	struct packet_sock *po, unsigned int stat)
781	{
782	__u32 status = TP_STATUS_USER | stat;
783
784	struct tpacket3_hdr *last_pkt;
785	struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
786	struct sock *sk = &po->sk;
787
788	if (atomic_read(v: &po->tp_drops))
789	status |= TP_STATUS_LOSING;
790
791	last_pkt = (struct tpacket3_hdr *)pkc1->prev;
792	last_pkt->tp_next_offset = 0;
793
794	/* Get the ts of the last pkt */
795	if (BLOCK_NUM_PKTS(pbd1)) {
796	h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
797	h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec;
798	} else {
799	/* Ok, we tmo'd - so get the current time.
800	*
801	* It shouldn't really happen as we don't close empty
802	* blocks. See prb_retire_rx_blk_timer_expired().
803	*/
804	struct timespec64 ts;
805	ktime_get_real_ts64(tv: &ts);
806	h1->ts_last_pkt.ts_sec = ts.tv_sec;
807	h1->ts_last_pkt.ts_nsec = ts.tv_nsec;
808	}
809
810	smp_wmb();
811
812	/* Flush the block */
813	prb_flush_block(pkc1, pbd1, status);
814
815	sk->sk_data_ready(sk);
816
817	pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
818	}
819
820	static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
821	{
822	pkc->reset_pending_on_curr_blk = 0;
823	}
824
825	/*
826	* prb_open_block is called by tpacket_rcv or timer callback.
827	*
828	* Reasons why NOT update hrtimer in prb_open_block:
829	* 1) It will increase complexity to distinguish the two caller scenario.
830	* 2) hrtimer_cancel and hrtimer_start need to be called if you want to update
831	* TMO of an already enqueued hrtimer, leading to complex shutdown logic.
832	*
833	* One side effect of NOT update hrtimer when called by tpacket_rcv is that
834	* a newly opened block triggered by tpacket_rcv may be retired earlier than
835	* expected. On the other hand, if timeout is updated in prb_open_block, the
836	* frequent reception of network packets that leads to prb_open_block being
837	* called may cause hrtimer to be removed and enqueued repeatedly.
838	*/
839	static void prb_open_block(struct tpacket_kbdq_core *pkc1,
840	struct tpacket_block_desc *pbd1)
841	{
842	struct timespec64 ts;
843	struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
844
845	smp_rmb();
846
847	/* We could have just memset this but we will lose the
848	* flexibility of making the priv area sticky
849	*/
850
851	BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
852	BLOCK_NUM_PKTS(pbd1) = 0;
853	BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
854
855	ktime_get_real_ts64(tv: &ts);
856
857	h1->ts_first_pkt.ts_sec = ts.tv_sec;
858	h1->ts_first_pkt.ts_nsec = ts.tv_nsec;
859
860	pkc1->pkblk_start = (char *)pbd1;
861	pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
862
863	BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
864	BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;
865
866	pbd1->version = pkc1->version;
867	pkc1->prev = pkc1->nxt_offset;
868	pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;
869
870	prb_thaw_queue(pkc: pkc1);
871
872	smp_wmb();
873	}
874
875	/*
876	* Queue freeze logic:
877	* 1) Assume tp_block_nr = 8 blocks.
878	* 2) At time 't0', user opens Rx ring.
879	* 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
880	* 4) user-space is either sleeping or processing block '0'.
881	* 5) tpacket_rcv is currently filling block '7', since there is no space left,
882	* it will close block-7,loop around and try to fill block '0'.
883	* call-flow:
884	* __packet_lookup_frame_in_block
885	* prb_retire_current_block()
886	* prb_dispatch_next_block()
887	* |->(BLOCK_STATUS == USER) evaluates to true
888	* 5.1) Since block-0 is currently in-use, we just freeze the queue.
889	* 6) Now there are two cases:
890	* 6.1) Link goes idle right after the queue is frozen.
891	* But remember, the last open_block() refreshed the timer.
892	* When this timer expires,it will refresh itself so that we can
893	* re-open block-0 in near future.
894	* 6.2) Link is busy and keeps on receiving packets. This is a simple
895	* case and __packet_lookup_frame_in_block will check if block-0
896	* is free and can now be re-used.
897	*/
898	static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
899	struct packet_sock *po)
900	{
901	pkc->reset_pending_on_curr_blk = 1;
902	po->stats.stats3.tp_freeze_q_cnt++;
903	}
904
905	#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))
906
907	/*
908	* If the next block is free then we will dispatch it
909	* and return a good offset.
910	* Else, we will freeze the queue.
911	* So, caller must check the return value.
912	*/
913	static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
914	struct packet_sock *po)
915	{
916	struct tpacket_block_desc *pbd;
917
918	smp_rmb();
919
920	/* 1. Get current block num */
921	pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
922
923	/* 2. If this block is currently in_use then freeze the queue */
924	if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
925	prb_freeze_queue(pkc, po);
926	return NULL;
927	}
928
929	/*
930	* 3.
931	* open this block and return the offset where the first packet
932	* needs to get stored.
933	*/
934	prb_open_block(pkc1: pkc, pbd1: pbd);
935	return (void *)pkc->nxt_offset;
936	}
937
938	static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
939	struct packet_sock *po, unsigned int status)
940	{
941	struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
942
943	/* retire/close the current block */
944	if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
945	/*
946	* Plug the case where copy_bits() is in progress on
947	* cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
948	* have space to copy the pkt in the current block and
949	* called prb_retire_current_block()
950	*
951	* We don't need to worry about the TMO case because
952	* the timer-handler already handled this case.
953	*/
954	if (!(status & TP_STATUS_BLK_TMO)) {
955	/* Waiting for skb_copy_bits to finish... */
956	write_lock(&pkc->blk_fill_in_prog_lock);
957	write_unlock(&pkc->blk_fill_in_prog_lock);
958	}
959	prb_close_block(pkc1: pkc, pbd1: pbd, po, stat: status);
960	return;
961	}
962	}
963
964	static int prb_curr_blk_in_use(struct tpacket_block_desc *pbd)
965	{
966	return TP_STATUS_USER & BLOCK_STATUS(pbd);
967	}
968
969	static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
970	{
971	return pkc->reset_pending_on_curr_blk;
972	}
973
974	static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
975	__releases(&pkc->blk_fill_in_prog_lock)
976	{
977	struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
978
979	read_unlock(&pkc->blk_fill_in_prog_lock);
980	}
981
982	static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
983	struct tpacket3_hdr *ppd)
984	{
985	ppd->hv1.tp_rxhash = skb_get_hash(skb: pkc->skb);
986	}
987
988	static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
989	struct tpacket3_hdr *ppd)
990	{
991	ppd->hv1.tp_rxhash = 0;
992	}
993
994	static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
995	struct tpacket3_hdr *ppd)
996	{
997	struct packet_sock *po = container_of(pkc, struct packet_sock, rx_ring.prb_bdqc);
998
999	if (skb_vlan_tag_present(pkc->skb)) {
1000	ppd->hv1.tp_vlan_tci = skb_vlan_tag_get(pkc->skb);
1001	ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
1002	ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
1003	} else if (unlikely(po->sk.sk_type == SOCK_DGRAM && eth_type_vlan(pkc->skb->protocol))) {
1004	ppd->hv1.tp_vlan_tci = vlan_get_tci(skb: pkc->skb, dev: pkc->skb->dev);
1005	ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->protocol);
1006	ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
1007	} else {
1008	ppd->hv1.tp_vlan_tci = 0;
1009	ppd->hv1.tp_vlan_tpid = 0;
1010	ppd->tp_status = TP_STATUS_AVAILABLE;
1011	}
1012	}
1013
1014	static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
1015	struct tpacket3_hdr *ppd)
1016	{
1017	ppd->hv1.tp_padding = 0;
1018	prb_fill_vlan_info(pkc, ppd);
1019
1020	if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
1021	prb_fill_rxhash(pkc, ppd);
1022	else
1023	prb_clear_rxhash(pkc, ppd);
1024	}
1025
1026	static void prb_fill_curr_block(char *curr,
1027	struct tpacket_kbdq_core *pkc,
1028	struct tpacket_block_desc *pbd,
1029	unsigned int len)
1030	__acquires(&pkc->blk_fill_in_prog_lock)
1031	{
1032	struct tpacket3_hdr *ppd;
1033
1034	ppd = (struct tpacket3_hdr *)curr;
1035	ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
1036	pkc->prev = curr;
1037	pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
1038	BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
1039	BLOCK_NUM_PKTS(pbd) += 1;
1040	read_lock(&pkc->blk_fill_in_prog_lock);
1041	prb_run_all_ft_ops(pkc, ppd);
1042	}
1043
1044	/* Assumes caller has the sk->rx_queue.lock */
1045	static void *__packet_lookup_frame_in_block(struct packet_sock *po,
1046	struct sk_buff *skb,
1047	unsigned int len
1048	)
1049	{
1050	struct tpacket_kbdq_core *pkc;
1051	struct tpacket_block_desc *pbd;
1052	char *curr, *end;
1053
1054	pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
1055	pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1056
1057	/* Queue is frozen when user space is lagging behind */
1058	if (prb_queue_frozen(pkc)) {
1059	/*
1060	* Check if that last block which caused the queue to freeze,
1061	* is still in_use by user-space.
1062	*/
1063	if (prb_curr_blk_in_use(pbd)) {
1064	/* Can't record this packet */
1065	return NULL;
1066	} else {
1067	/*
1068	* Ok, the block was released by user-space.
1069	* Now let's open that block.
1070	* opening a block also thaws the queue.
1071	* Thawing is a side effect.
1072	*/
1073	prb_open_block(pkc1: pkc, pbd1: pbd);
1074	}
1075	}
1076
1077	smp_mb();
1078	curr = pkc->nxt_offset;
1079	pkc->skb = skb;
1080	end = (char *)pbd + pkc->kblk_size;
1081
1082	/* first try the current block */
1083	if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
1084	prb_fill_curr_block(curr, pkc, pbd, len);
1085	return (void *)curr;
1086	}
1087
1088	/* Ok, close the current block */
1089	prb_retire_current_block(pkc, po, status: 0);
1090
1091	/* Now, try to dispatch the next block */
1092	curr = (char *)prb_dispatch_next_block(pkc, po);
1093	if (curr) {
1094	pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
1095	prb_fill_curr_block(curr, pkc, pbd, len);
1096	return (void *)curr;
1097	}
1098
1099	/*
1100	* No free blocks are available.user_space hasn't caught up yet.
1101	* Queue was just frozen and now this packet will get dropped.
1102	*/
1103	return NULL;
1104	}
1105
1106	static void *packet_current_rx_frame(struct packet_sock *po,
1107	struct sk_buff *skb,
1108	int status, unsigned int len)
1109	{
1110	char *curr = NULL;
1111	switch (po->tp_version) {
1112	case TPACKET_V1:
1113	case TPACKET_V2:
1114	curr = packet_lookup_frame(po, rb: &po->rx_ring,
1115	position: po->rx_ring.head, status);
1116	return curr;
1117	case TPACKET_V3:
1118	return __packet_lookup_frame_in_block(po, skb, len);
1119	default:
1120	WARN(1, "TPACKET version not supported\n");
1121	BUG();
1122	return NULL;
1123	}
1124	}
1125
1126	static void *prb_lookup_block(const struct packet_sock *po,
1127	const struct packet_ring_buffer *rb,
1128	unsigned int idx,
1129	int status)
1130	{
1131	struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(rb);
1132	struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);
1133
1134	if (status != BLOCK_STATUS(pbd))
1135	return NULL;
1136	return pbd;
1137	}
1138
1139	static int prb_previous_blk_num(struct packet_ring_buffer *rb)
1140	{
1141	unsigned int prev;
1142	if (rb->prb_bdqc.kactive_blk_num)
1143	prev = rb->prb_bdqc.kactive_blk_num-1;
1144	else
1145	prev = rb->prb_bdqc.knum_blocks-1;
1146	return prev;
1147	}
1148
1149	/* Assumes caller has held the rx_queue.lock */
1150	static void *__prb_previous_block(struct packet_sock *po,
1151	struct packet_ring_buffer *rb,
1152	int status)
1153	{
1154	unsigned int previous = prb_previous_blk_num(rb);
1155	return prb_lookup_block(po, rb, idx: previous, status);
1156	}
1157
1158	static void *packet_previous_rx_frame(struct packet_sock *po,
1159	struct packet_ring_buffer *rb,
1160	int status)
1161	{
1162	if (po->tp_version <= TPACKET_V2)
1163	return packet_previous_frame(po, rb, status);
1164
1165	return __prb_previous_block(po, rb, status);
1166	}
1167
1168	static void packet_increment_rx_head(struct packet_sock *po,
1169	struct packet_ring_buffer *rb)
1170	{
1171	switch (po->tp_version) {
1172	case TPACKET_V1:
1173	case TPACKET_V2:
1174	return packet_increment_head(buff: rb);
1175	case TPACKET_V3:
1176	default:
1177	WARN(1, "TPACKET version not supported.\n");
1178	BUG();
1179	return;
1180	}
1181	}
1182
1183	static void *packet_previous_frame(struct packet_sock *po,
1184	struct packet_ring_buffer *rb,
1185	int status)
1186	{
1187	unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
1188	return packet_lookup_frame(po, rb, position: previous, status);
1189	}
1190
1191	static void packet_increment_head(struct packet_ring_buffer *buff)
1192	{
1193	buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
1194	}
1195
1196	static void packet_inc_pending(struct packet_ring_buffer *rb)
1197	{
1198	this_cpu_inc(*rb->pending_refcnt);
1199	}
1200
1201	static void packet_dec_pending(struct packet_ring_buffer *rb)
1202	{
1203	this_cpu_dec(*rb->pending_refcnt);
1204	}
1205
1206	static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
1207	{
1208	unsigned int refcnt = 0;
1209	int cpu;
1210
1211	/* We don't use pending refcount in rx_ring. */
1212	if (rb->pending_refcnt == NULL)
1213	return 0;
1214
1215	for_each_possible_cpu(cpu)
1216	refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);
1217
1218	return refcnt;
1219	}
1220
1221	static int packet_alloc_pending(struct packet_sock *po)
1222	{
1223	po->rx_ring.pending_refcnt = NULL;
1224
1225	po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
1226	if (unlikely(po->tx_ring.pending_refcnt == NULL))
1227	return -ENOBUFS;
1228
1229	return 0;
1230	}
1231
1232	static void packet_free_pending(struct packet_sock *po)
1233	{
1234	free_percpu(pdata: po->tx_ring.pending_refcnt);
1235	}
1236
1237	#define ROOM_POW_OFF 2
1238	#define ROOM_NONE 0x0
1239	#define ROOM_LOW 0x1
1240	#define ROOM_NORMAL 0x2
1241
1242	static bool __tpacket_has_room(const struct packet_sock *po, int pow_off)
1243	{
1244	int idx, len;
1245
1246	len = READ_ONCE(po->rx_ring.frame_max) + 1;
1247	idx = READ_ONCE(po->rx_ring.head);
1248	if (pow_off)
1249	idx += len >> pow_off;
1250	if (idx >= len)
1251	idx -= len;
1252	return packet_lookup_frame(po, rb: &po->rx_ring, position: idx, TP_STATUS_KERNEL);
1253	}
1254
1255	static bool __tpacket_v3_has_room(const struct packet_sock *po, int pow_off)
1256	{
1257	int idx, len;
1258
1259	len = READ_ONCE(po->rx_ring.prb_bdqc.knum_blocks);
1260	idx = READ_ONCE(po->rx_ring.prb_bdqc.kactive_blk_num);
1261	if (pow_off)
1262	idx += len >> pow_off;
1263	if (idx >= len)
1264	idx -= len;
1265	return prb_lookup_block(po, rb: &po->rx_ring, idx, TP_STATUS_KERNEL);
1266	}
1267
1268	static int __packet_rcv_has_room(const struct packet_sock *po,
1269	const struct sk_buff *skb)
1270	{
1271	const struct sock *sk = &po->sk;
1272	int ret = ROOM_NONE;
1273
1274	if (po->prot_hook.func != tpacket_rcv) {
1275	int rcvbuf = READ_ONCE(sk->sk_rcvbuf);
1276	int avail = rcvbuf - atomic_read(v: &sk->sk_rmem_alloc)
1277	- (skb ? skb->truesize : 0);
1278
1279	if (avail > (rcvbuf >> ROOM_POW_OFF))
1280	return ROOM_NORMAL;
1281	else if (avail > 0)
1282	return ROOM_LOW;
1283	else
1284	return ROOM_NONE;
1285	}
1286
1287	if (po->tp_version == TPACKET_V3) {
1288	if (__tpacket_v3_has_room(po, ROOM_POW_OFF))
1289	ret = ROOM_NORMAL;
1290	else if (__tpacket_v3_has_room(po, pow_off: 0))
1291	ret = ROOM_LOW;
1292	} else {
1293	if (__tpacket_has_room(po, ROOM_POW_OFF))
1294	ret = ROOM_NORMAL;
1295	else if (__tpacket_has_room(po, pow_off: 0))
1296	ret = ROOM_LOW;
1297	}
1298
1299	return ret;
1300	}
1301
1302	static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
1303	{
1304	bool pressure;
1305	int ret;
1306
1307	ret = __packet_rcv_has_room(po, skb);
1308	pressure = ret != ROOM_NORMAL;
1309
1310	if (packet_sock_flag(po, flag: PACKET_SOCK_PRESSURE) != pressure)
1311	packet_sock_flag_set(po, flag: PACKET_SOCK_PRESSURE, val: pressure);
1312
1313	return ret;
1314	}
1315
1316	static void packet_rcv_try_clear_pressure(struct packet_sock *po)
1317	{
1318	if (packet_sock_flag(po, flag: PACKET_SOCK_PRESSURE) &&
1319	__packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
1320	packet_sock_flag_set(po, flag: PACKET_SOCK_PRESSURE, val: false);
1321	}
1322
1323	static void packet_sock_destruct(struct sock *sk)
1324	{
1325	skb_queue_purge(list: &sk->sk_error_queue);
1326
1327	WARN_ON(atomic_read(&sk->sk_rmem_alloc));
1328	WARN_ON(refcount_read(&sk->sk_wmem_alloc));
1329
1330	if (!sock_flag(sk, flag: SOCK_DEAD)) {
1331	pr_err("Attempt to release alive packet socket: %p\n", sk);
1332	return;
1333	}
1334	}
1335
1336	static bool fanout_flow_is_huge(struct packet_sock *po, struct sk_buff *skb)
1337	{
1338	u32 *history = po->rollover->history;
1339	u32 victim, rxhash;
1340	int i, count = 0;
1341
1342	rxhash = skb_get_hash(skb);
1343	for (i = 0; i < ROLLOVER_HLEN; i++)
1344	if (READ_ONCE(history[i]) == rxhash)
1345	count++;
1346
1347	victim = get_random_u32_below(ROLLOVER_HLEN);
1348
1349	/* Avoid dirtying the cache line if possible */
1350	if (READ_ONCE(history[victim]) != rxhash)
1351	WRITE_ONCE(history[victim], rxhash);
1352
1353	return count > (ROLLOVER_HLEN >> 1);
1354	}
1355
1356	static unsigned int fanout_demux_hash(struct packet_fanout *f,
1357	struct sk_buff *skb,
1358	unsigned int num)
1359	{
1360	return reciprocal_scale(val: __skb_get_hash_symmetric(skb), ep_ro: num);
1361	}
1362
1363	static unsigned int fanout_demux_lb(struct packet_fanout *f,
1364	struct sk_buff *skb,
1365	unsigned int num)
1366	{
1367	unsigned int val = atomic_inc_return(v: &f->rr_cur);
1368
1369	return val % num;
1370	}
1371
1372	static unsigned int fanout_demux_cpu(struct packet_fanout *f,
1373	struct sk_buff *skb,
1374	unsigned int num)
1375	{
1376	return smp_processor_id() % num;
1377	}
1378
1379	static unsigned int fanout_demux_rnd(struct packet_fanout *f,
1380	struct sk_buff *skb,
1381	unsigned int num)
1382	{
1383	return get_random_u32_below(ceil: num);
1384	}
1385
1386	static unsigned int fanout_demux_rollover(struct packet_fanout *f,
1387	struct sk_buff *skb,
1388	unsigned int idx, bool try_self,
1389	unsigned int num)
1390	{
1391	struct packet_sock *po, *po_next, *po_skip = NULL;
1392	unsigned int i, j, room = ROOM_NONE;
1393
1394	po = pkt_sk(rcu_dereference(f->arr[idx]));
1395
1396	if (try_self) {
1397	room = packet_rcv_has_room(po, skb);
1398	if (room == ROOM_NORMAL ||
1399	(room == ROOM_LOW && !fanout_flow_is_huge(po, skb)))
1400	return idx;
1401	po_skip = po;
1402	}
1403
1404	i = j = min_t(int, po->rollover->sock, num - 1);
1405	do {
1406	po_next = pkt_sk(rcu_dereference(f->arr[i]));
1407	if (po_next != po_skip &&
1408	!packet_sock_flag(po: po_next, flag: PACKET_SOCK_PRESSURE) &&
1409	packet_rcv_has_room(po: po_next, skb) == ROOM_NORMAL) {
1410	if (i != j)
1411	po->rollover->sock = i;
1412	atomic_long_inc(v: &po->rollover->num);
1413	if (room == ROOM_LOW)
1414	atomic_long_inc(v: &po->rollover->num_huge);
1415	return i;
1416	}
1417
1418	if (++i == num)
1419	i = 0;
1420	} while (i != j);
1421
1422	atomic_long_inc(v: &po->rollover->num_failed);
1423	return idx;
1424	}
1425
1426	static unsigned int fanout_demux_qm(struct packet_fanout *f,
1427	struct sk_buff *skb,
1428	unsigned int num)
1429	{
1430	return skb_get_queue_mapping(skb) % num;
1431	}
1432
1433	static unsigned int fanout_demux_bpf(struct packet_fanout *f,
1434	struct sk_buff *skb,
1435	unsigned int num)
1436	{
1437	struct bpf_prog *prog;
1438	unsigned int ret = 0;
1439
1440	rcu_read_lock();
1441	prog = rcu_dereference(f->bpf_prog);
1442	if (prog)
1443	ret = bpf_prog_run_clear_cb(prog, skb) % num;
1444	rcu_read_unlock();
1445
1446	return ret;
1447	}
1448
1449	static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
1450	{
1451	return f->flags & (flag >> 8);
1452	}
1453
1454	static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
1455	struct packet_type *pt, struct net_device *orig_dev)
1456	{
1457	struct packet_fanout *f = pt->af_packet_priv;
1458	unsigned int num = READ_ONCE(f->num_members);
1459	struct net *net = read_pnet(pnet: &f->net);
1460	struct packet_sock *po;
1461	unsigned int idx;
1462
1463	if (!net_eq(net1: dev_net(dev), net2: net) || !num) {
1464	kfree_skb(skb);
1465	return 0;
1466	}
1467
1468	if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
1469	skb = ip_check_defrag(net, skb, user: IP_DEFRAG_AF_PACKET);
1470	if (!skb)
1471	return 0;
1472	}
1473	switch (f->type) {
1474	case PACKET_FANOUT_HASH:
1475	default:
1476	idx = fanout_demux_hash(f, skb, num);
1477	break;
1478	case PACKET_FANOUT_LB:
1479	idx = fanout_demux_lb(f, skb, num);
1480	break;
1481	case PACKET_FANOUT_CPU:
1482	idx = fanout_demux_cpu(f, skb, num);
1483	break;
1484	case PACKET_FANOUT_RND:
1485	idx = fanout_demux_rnd(f, skb, num);
1486	break;
1487	case PACKET_FANOUT_QM:
1488	idx = fanout_demux_qm(f, skb, num);
1489	break;
1490	case PACKET_FANOUT_ROLLOVER:
1491	idx = fanout_demux_rollover(f, skb, idx: 0, try_self: false, num);
1492	break;
1493	case PACKET_FANOUT_CBPF:
1494	case PACKET_FANOUT_EBPF:
1495	idx = fanout_demux_bpf(f, skb, num);
1496	break;
1497	}
1498
1499	if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER))
1500	idx = fanout_demux_rollover(f, skb, idx, try_self: true, num);
1501
1502	po = pkt_sk(rcu_dereference(f->arr[idx]));
1503	return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
1504	}
1505
1506	DEFINE_MUTEX(fanout_mutex);
1507	EXPORT_SYMBOL_GPL(fanout_mutex);
1508	static LIST_HEAD(fanout_list);
1509	static u16 fanout_next_id;
1510
1511	static void __fanout_link(struct sock *sk, struct packet_sock *po)
1512	{
1513	struct packet_fanout *f = po->fanout;
1514
1515	spin_lock(lock: &f->lock);
1516	rcu_assign_pointer(f->arr[f->num_members], sk);
1517	smp_wmb();
1518	f->num_members++;
1519	if (f->num_members == 1)
1520	dev_add_pack(pt: &f->prot_hook);
1521	spin_unlock(lock: &f->lock);
1522	}
1523
1524	static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
1525	{
1526	struct packet_fanout *f = po->fanout;
1527	int i;
1528
1529	spin_lock(lock: &f->lock);
1530	for (i = 0; i < f->num_members; i++) {
1531	if (rcu_dereference_protected(f->arr[i],
1532	lockdep_is_held(&f->lock)) == sk)
1533	break;
1534	}
1535	BUG_ON(i >= f->num_members);
1536	rcu_assign_pointer(f->arr[i],
1537	rcu_dereference_protected(f->arr[f->num_members - 1],
1538	lockdep_is_held(&f->lock)));
1539	f->num_members--;
1540	if (f->num_members == 0)
1541	__dev_remove_pack(pt: &f->prot_hook);
1542	spin_unlock(lock: &f->lock);
1543	}
1544
1545	static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
1546	{
1547	if (sk->sk_family != PF_PACKET)
1548	return false;
1549
1550	return ptype->af_packet_priv == pkt_sk(sk)->fanout;
1551	}
1552
1553	static void fanout_init_data(struct packet_fanout *f)
1554	{
1555	switch (f->type) {
1556	case PACKET_FANOUT_LB:
1557	atomic_set(v: &f->rr_cur, i: 0);
1558	break;
1559	case PACKET_FANOUT_CBPF:
1560	case PACKET_FANOUT_EBPF:
1561	RCU_INIT_POINTER(f->bpf_prog, NULL);
1562	break;
1563	}
1564	}
1565
1566	static void __fanout_set_data_bpf(struct packet_fanout *f, struct bpf_prog *new)
1567	{
1568	struct bpf_prog *old;
1569
1570	spin_lock(lock: &f->lock);
1571	old = rcu_dereference_protected(f->bpf_prog, lockdep_is_held(&f->lock));
1572	rcu_assign_pointer(f->bpf_prog, new);
1573	spin_unlock(lock: &f->lock);
1574
1575	if (old) {
1576	synchronize_net();
1577	bpf_prog_destroy(fp: old);
1578	}
1579	}
1580
1581	static int fanout_set_data_cbpf(struct packet_sock *po, sockptr_t data,
1582	unsigned int len)
1583	{
1584	struct bpf_prog *new;
1585	struct sock_fprog fprog;
1586	int ret;
1587
1588	if (sock_flag(sk: &po->sk, flag: SOCK_FILTER_LOCKED))
1589	return -EPERM;
1590
1591	ret = copy_bpf_fprog_from_user(dst: &fprog, src: data, len);
1592	if (ret)
1593	return ret;
1594
1595	ret = bpf_prog_create_from_user(pfp: &new, fprog: &fprog, NULL, save_orig: false);
1596	if (ret)
1597	return ret;
1598
1599	__fanout_set_data_bpf(f: po->fanout, new);
1600	return 0;
1601	}
1602
1603	static int fanout_set_data_ebpf(struct packet_sock *po, sockptr_t data,
1604	unsigned int len)
1605	{
1606	struct bpf_prog *new;
1607	u32 fd;
1608
1609	if (sock_flag(sk: &po->sk, flag: SOCK_FILTER_LOCKED))
1610	return -EPERM;
1611	if (len != sizeof(fd))
1612	return -EINVAL;
1613	if (copy_from_sockptr(dst: &fd, src: data, size: len))
1614	return -EFAULT;
1615
1616	new = bpf_prog_get_type(ufd: fd, type: BPF_PROG_TYPE_SOCKET_FILTER);
1617	if (IS_ERR(ptr: new))
1618	return PTR_ERR(ptr: new);
1619
1620	__fanout_set_data_bpf(f: po->fanout, new);
1621	return 0;
1622	}
1623
1624	static int fanout_set_data(struct packet_sock *po, sockptr_t data,
1625	unsigned int len)
1626	{
1627	switch (po->fanout->type) {
1628	case PACKET_FANOUT_CBPF:
1629	return fanout_set_data_cbpf(po, data, len);
1630	case PACKET_FANOUT_EBPF:
1631	return fanout_set_data_ebpf(po, data, len);
1632	default:
1633	return -EINVAL;
1634	}
1635	}
1636
1637	static void fanout_release_data(struct packet_fanout *f)
1638	{
1639	switch (f->type) {
1640	case PACKET_FANOUT_CBPF:
1641	case PACKET_FANOUT_EBPF:
1642	__fanout_set_data_bpf(f, NULL);
1643	}
1644	}
1645
1646	static bool __fanout_id_is_free(struct sock *sk, u16 candidate_id)
1647	{
1648	struct packet_fanout *f;
1649
1650	list_for_each_entry(f, &fanout_list, list) {
1651	if (f->id == candidate_id &&
1652	read_pnet(pnet: &f->net) == sock_net(sk)) {
1653	return false;
1654	}
1655	}
1656	return true;
1657	}
1658
1659	static bool fanout_find_new_id(struct sock *sk, u16 *new_id)
1660	{
1661	u16 id = fanout_next_id;
1662
1663	do {
1664	if (__fanout_id_is_free(sk, candidate_id: id)) {
1665	*new_id = id;
1666	fanout_next_id = id + 1;
1667	return true;
1668	}
1669
1670	id++;
1671	} while (id != fanout_next_id);
1672
1673	return false;
1674	}
1675
1676	static int fanout_add(struct sock *sk, struct fanout_args *args)
1677	{
1678	struct packet_rollover *rollover = NULL;
1679	struct packet_sock *po = pkt_sk(sk);
1680	u16 type_flags = args->type_flags;
1681	struct packet_fanout *f, *match;
1682	u8 type = type_flags & 0xff;
1683	u8 flags = type_flags >> 8;
1684	u16 id = args->id;
1685	int err;
1686
1687	switch (type) {
1688	case PACKET_FANOUT_ROLLOVER:
1689	if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
1690	return -EINVAL;
1691	break;
1692	case PACKET_FANOUT_HASH:
1693	case PACKET_FANOUT_LB:
1694	case PACKET_FANOUT_CPU:
1695	case PACKET_FANOUT_RND:
1696	case PACKET_FANOUT_QM:
1697	case PACKET_FANOUT_CBPF:
1698	case PACKET_FANOUT_EBPF:
1699	break;
1700	default:
1701	return -EINVAL;
1702	}
1703
1704	mutex_lock(&fanout_mutex);
1705
1706	err = -EALREADY;
1707	if (po->fanout)
1708	goto out;
1709
1710	if (type == PACKET_FANOUT_ROLLOVER ||
1711	(type_flags & PACKET_FANOUT_FLAG_ROLLOVER)) {
1712	err = -ENOMEM;
1713	rollover = kzalloc(sizeof(*rollover), GFP_KERNEL);
1714	if (!rollover)
1715	goto out;
1716	atomic_long_set(v: &rollover->num, i: 0);
1717	atomic_long_set(v: &rollover->num_huge, i: 0);
1718	atomic_long_set(v: &rollover->num_failed, i: 0);
1719	}
1720
1721	if (type_flags & PACKET_FANOUT_FLAG_UNIQUEID) {
1722	if (id != 0) {
1723	err = -EINVAL;
1724	goto out;
1725	}
1726	if (!fanout_find_new_id(sk, new_id: &id)) {
1727	err = -ENOMEM;
1728	goto out;
1729	}
1730	/* ephemeral flag for the first socket in the group: drop it */
1731	flags &= ~(PACKET_FANOUT_FLAG_UNIQUEID >> 8);
1732	}
1733
1734	match = NULL;
1735	list_for_each_entry(f, &fanout_list, list) {
1736	if (f->id == id &&
1737	read_pnet(pnet: &f->net) == sock_net(sk)) {
1738	match = f;
1739	break;
1740	}
1741	}
1742	err = -EINVAL;
1743	if (match) {
1744	if (match->flags != flags)
1745	goto out;
1746	if (args->max_num_members &&
1747	args->max_num_members != match->max_num_members)
1748	goto out;
1749	} else {
1750	if (args->max_num_members > PACKET_FANOUT_MAX)
1751	goto out;
1752	if (!args->max_num_members)
1753	/* legacy PACKET_FANOUT_MAX */
1754	args->max_num_members = 256;
1755	err = -ENOMEM;
1756	match = kvzalloc(struct_size(match, arr, args->max_num_members),
1757	GFP_KERNEL);
1758	if (!match)
1759	goto out;
1760	write_pnet(pnet: &match->net, net: sock_net(sk));
1761	match->id = id;
1762	match->type = type;
1763	match->flags = flags;
1764	INIT_LIST_HEAD(list: &match->list);
1765	spin_lock_init(&match->lock);
1766	refcount_set(r: &match->sk_ref, n: 0);
1767	fanout_init_data(f: match);
1768	match->prot_hook.type = po->prot_hook.type;
1769	match->prot_hook.dev = po->prot_hook.dev;
1770	match->prot_hook.func = packet_rcv_fanout;
1771	match->prot_hook.af_packet_priv = match;
1772	match->prot_hook.af_packet_net = read_pnet(pnet: &match->net);
1773	match->prot_hook.id_match = match_fanout_group;
1774	match->max_num_members = args->max_num_members;
1775	match->prot_hook.ignore_outgoing = type_flags & PACKET_FANOUT_FLAG_IGNORE_OUTGOING;
1776	list_add(new: &match->list, head: &fanout_list);
1777	}
1778	err = -EINVAL;
1779
1780	spin_lock(lock: &po->bind_lock);
1781	if (po->num &&
1782	match->type == type &&
1783	match->prot_hook.type == po->prot_hook.type &&
1784	match->prot_hook.dev == po->prot_hook.dev) {
1785	err = -ENOSPC;
1786	if (refcount_read(r: &match->sk_ref) < match->max_num_members) {
1787	/* Paired with packet_setsockopt(PACKET_FANOUT_DATA) */
1788	WRITE_ONCE(po->fanout, match);
1789
1790	po->rollover = rollover;
1791	rollover = NULL;
1792	refcount_set(r: &match->sk_ref, n: refcount_read(r: &match->sk_ref) + 1);
1793	if (packet_sock_flag(po, flag: PACKET_SOCK_RUNNING)) {
1794	__dev_remove_pack(pt: &po->prot_hook);
1795	__fanout_link(sk, po);
1796	}
1797	err = 0;
1798	}
1799	}
1800	spin_unlock(lock: &po->bind_lock);
1801
1802	if (err && !refcount_read(r: &match->sk_ref)) {
1803	list_del(entry: &match->list);
1804	kvfree(addr: match);
1805	}
1806
1807	out:
1808	kfree(objp: rollover);
1809	mutex_unlock(lock: &fanout_mutex);
1810	return err;
1811	}
1812
1813	/* If pkt_sk(sk)->fanout->sk_ref is zero, this function removes
1814	* pkt_sk(sk)->fanout from fanout_list and returns pkt_sk(sk)->fanout.
1815	* It is the responsibility of the caller to call fanout_release_data() and
1816	* free the returned packet_fanout (after synchronize_net())
1817	*/
1818	static struct packet_fanout *fanout_release(struct sock *sk)
1819	{
1820	struct packet_sock *po = pkt_sk(sk);
1821	struct packet_fanout *f;
1822
1823	mutex_lock(&fanout_mutex);
1824	f = po->fanout;
1825	if (f) {
1826	po->fanout = NULL;
1827
1828	if (refcount_dec_and_test(r: &f->sk_ref))
1829	list_del(entry: &f->list);
1830	else
1831	f = NULL;
1832	}
1833	mutex_unlock(lock: &fanout_mutex);
1834
1835	return f;
1836	}
1837
1838	static bool packet_extra_vlan_len_allowed(const struct net_device *dev,
1839	struct sk_buff *skb)
1840	{
1841	/* Earlier code assumed this would be a VLAN pkt, double-check
1842	* this now that we have the actual packet in hand. We can only
1843	* do this check on Ethernet devices.
1844	*/
1845	if (unlikely(dev->type != ARPHRD_ETHER))
1846	return false;
1847
1848	skb_reset_mac_header(skb);
1849	return likely(eth_hdr(skb)->h_proto == htons(ETH_P_8021Q));
1850	}
1851
1852	static const struct proto_ops packet_ops;
1853
1854	static const struct proto_ops packet_ops_spkt;
1855
1856	static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
1857	struct packet_type *pt, struct net_device *orig_dev)
1858	{
1859	struct sock *sk;
1860	struct sockaddr_pkt *spkt;
1861
1862	/*
1863	* When we registered the protocol we saved the socket in the data
1864	* field for just this event.
1865	*/
1866
1867	sk = pt->af_packet_priv;
1868
1869	/*
1870	* Yank back the headers [hope the device set this
1871	* right or kerboom...]
1872	*
1873	* Incoming packets have ll header pulled,
1874	* push it back.
1875	*
1876	* For outgoing ones skb->data == skb_mac_header(skb)
1877	* so that this procedure is noop.
1878	*/
1879
1880	if (skb->pkt_type == PACKET_LOOPBACK)
1881	goto out;
1882
1883	if (!net_eq(net1: dev_net(dev), net2: sock_net(sk)))
1884	goto out;
1885
1886	skb = skb_share_check(skb, GFP_ATOMIC);
1887	if (skb == NULL)
1888	goto oom;
1889
1890	/* drop any routing info */
1891	skb_dst_drop(skb);
1892
1893	/* drop conntrack reference */
1894	nf_reset_ct(skb);
1895
1896	spkt = &PACKET_SKB_CB(skb)->sa.pkt;
1897
1898	skb_push(skb, len: skb->data - skb_mac_header(skb));
1899
1900	/*
1901	* The SOCK_PACKET socket receives _all_ frames.
1902	*/
1903
1904	spkt->spkt_family = dev->type;
1905	strscpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
1906	spkt->spkt_protocol = skb->protocol;
1907
1908	/*
1909	* Charge the memory to the socket. This is done specifically
1910	* to prevent sockets using all the memory up.
1911	*/
1912
1913	if (sock_queue_rcv_skb(sk, skb) == 0)
1914	return 0;
1915
1916	out:
1917	kfree_skb(skb);
1918	oom:
1919	return 0;
1920	}
1921
1922	static void packet_parse_headers(struct sk_buff *skb, struct socket *sock)
1923	{
1924	int depth;
1925
1926	if ((!skb->protocol || skb->protocol == htons(ETH_P_ALL)) &&
1927	sock->type == SOCK_RAW) {
1928	skb_reset_mac_header(skb);
1929	skb->protocol = dev_parse_header_protocol(skb);
1930	}
1931
1932	/* Move network header to the right position for VLAN tagged packets */
1933	if (likely(skb->dev->type == ARPHRD_ETHER) &&
1934	eth_type_vlan(ethertype: skb->protocol) &&
1935	vlan_get_protocol_and_depth(skb, type: skb->protocol, depth: &depth) != 0)
1936	skb_set_network_header(skb, offset: depth);
1937
1938	skb_probe_transport_header(skb);
1939	}
1940
1941	/*
1942	* Output a raw packet to a device layer. This bypasses all the other
1943	* protocol layers and you must therefore supply it with a complete frame
1944	*/
1945
1946	static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg,
1947	size_t len)
1948	{
1949	struct sock *sk = sock->sk;
1950	DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
1951	struct sk_buff *skb = NULL;
1952	struct net_device *dev;
1953	struct sockcm_cookie sockc;
1954	__be16 proto = 0;
1955	int err;
1956	int extra_len = 0;
1957
1958	/*
1959	* Get and verify the address.
1960	*/
1961
1962	if (saddr) {
1963	if (msg->msg_namelen < sizeof(struct sockaddr))
1964	return -EINVAL;
1965	if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
1966	proto = saddr->spkt_protocol;
1967	} else
1968	return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
1969
1970	/*
1971	* Find the device first to size check it
1972	*/
1973
1974	saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
1975	retry:
1976	rcu_read_lock();
1977	dev = dev_get_by_name_rcu(net: sock_net(sk), name: saddr->spkt_device);
1978	err = -ENODEV;
1979	if (dev == NULL)
1980	goto out_unlock;
1981
1982	err = -ENETDOWN;
1983	if (!(dev->flags & IFF_UP))
1984	goto out_unlock;
1985
1986	/*
1987	* You may not queue a frame bigger than the mtu. This is the lowest level
1988	* raw protocol and you must do your own fragmentation at this level.
1989	*/
1990
1991	if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
1992	if (!netif_supports_nofcs(dev)) {
1993	err = -EPROTONOSUPPORT;
1994	goto out_unlock;
1995	}
1996	extra_len = 4; /* We're doing our own CRC */
1997	}
1998
1999	err = -EMSGSIZE;
2000	if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
2001	goto out_unlock;
2002
2003	if (!skb) {
2004	size_t reserved = LL_RESERVED_SPACE(dev);
2005	int tlen = dev->needed_tailroom;
2006	unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
2007
2008	rcu_read_unlock();
2009	skb = sock_wmalloc(sk, size: len + reserved + tlen, force: 0, GFP_KERNEL);
2010	if (skb == NULL)
2011	return -ENOBUFS;
2012	/* FIXME: Save some space for broken drivers that write a hard
2013	* header at transmission time by themselves. PPP is the notable
2014	* one here. This should really be fixed at the driver level.
2015	*/
2016	skb_reserve(skb, len: reserved);
2017	skb_reset_network_header(skb);
2018
2019	/* Try to align data part correctly */
2020	if (hhlen) {
2021	skb->data -= hhlen;
2022	skb->tail -= hhlen;
2023	if (len < hhlen)
2024	skb_reset_network_header(skb);
2025	}
2026	err = memcpy_from_msg(data: skb_put(skb, len), msg, len);
2027	if (err)
2028	goto out_free;
2029	goto retry;
2030	}
2031
2032	if (!dev_validate_header(dev, ll_header: skb->data, len) || !skb->len) {
2033	err = -EINVAL;
2034	goto out_unlock;
2035	}
2036	if (len > (dev->mtu + dev->hard_header_len + extra_len) &&
2037	!packet_extra_vlan_len_allowed(dev, skb)) {
2038	err = -EMSGSIZE;
2039	goto out_unlock;
2040	}
2041
2042	sockcm_init(sockc: &sockc, sk);
2043	if (msg->msg_controllen) {
2044	err = sock_cmsg_send(sk, msg, sockc: &sockc);
2045	if (unlikely(err))
2046	goto out_unlock;
2047	}
2048
2049	skb->protocol = proto;
2050	skb->dev = dev;
2051	skb->priority = sockc.priority;
2052	skb->mark = sockc.mark;
2053	skb_set_delivery_type_by_clockid(skb, kt: sockc.transmit_time, clockid: sk->sk_clockid);
2054	skb_setup_tx_timestamp(skb, sockc: &sockc);
2055
2056	if (unlikely(extra_len == 4))
2057	skb->no_fcs = 1;
2058
2059	packet_parse_headers(skb, sock);
2060
2061	dev_queue_xmit(skb);
2062	rcu_read_unlock();
2063	return len;
2064
2065	out_unlock:
2066	rcu_read_unlock();
2067	out_free:
2068	kfree_skb(skb);
2069	return err;
2070	}
2071
2072	static unsigned int run_filter(struct sk_buff *skb,
2073	const struct sock *sk,
2074	unsigned int res)
2075	{
2076	struct sk_filter *filter;
2077
2078	rcu_read_lock();
2079	filter = rcu_dereference(sk->sk_filter);
2080	if (filter != NULL)
2081	res = bpf_prog_run_clear_cb(prog: filter->prog, skb);
2082	rcu_read_unlock();
2083
2084	return res;
2085	}
2086
2087	static int packet_rcv_vnet(struct msghdr *msg, const struct sk_buff *skb,
2088	size_t *len, int vnet_hdr_sz)
2089	{
2090	struct virtio_net_hdr_mrg_rxbuf vnet_hdr = { .num_buffers = 0 };
2091
2092	if (*len < vnet_hdr_sz)
2093	return -EINVAL;
2094	*len -= vnet_hdr_sz;
2095
2096	if (virtio_net_hdr_from_skb(skb, hdr: (struct virtio_net_hdr *)&vnet_hdr, vio_le(), has_data_valid: true, vlan_hlen: 0))
2097	return -EINVAL;
2098
2099	return memcpy_to_msg(msg, data: (void *)&vnet_hdr, len: vnet_hdr_sz);
2100	}
2101
2102	/*
2103	* This function makes lazy skb cloning in hope that most of packets
2104	* are discarded by BPF.
2105	*
2106	* Note tricky part: we DO mangle shared skb! skb->data, skb->len
2107	* and skb->cb are mangled. It works because (and until) packets
2108	* falling here are owned by current CPU. Output packets are cloned
2109	* by dev_queue_xmit_nit(), input packets are processed by net_bh
2110	* sequentially, so that if we return skb to original state on exit,
2111	* we will not harm anyone.
2112	*/
2113
2114	static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
2115	struct packet_type *pt, struct net_device *orig_dev)
2116	{
2117	enum skb_drop_reason drop_reason = SKB_CONSUMED;
2118	struct sock *sk = NULL;
2119	struct sockaddr_ll *sll;
2120	struct packet_sock *po;
2121	u8 *skb_head = skb->data;
2122	int skb_len = skb->len;
2123	unsigned int snaplen, res;
2124
2125	if (skb->pkt_type == PACKET_LOOPBACK)
2126	goto drop;
2127
2128	sk = pt->af_packet_priv;
2129	po = pkt_sk(sk);
2130
2131	if (!net_eq(net1: dev_net(dev), net2: sock_net(sk)))
2132	goto drop;
2133
2134	skb->dev = dev;
2135
2136	if (dev_has_header(dev)) {
2137	/* The device has an explicit notion of ll header,
2138	* exported to higher levels.
2139	*
2140	* Otherwise, the device hides details of its frame
2141	* structure, so that corresponding packet head is
2142	* never delivered to user.
2143	*/
2144	if (sk->sk_type != SOCK_DGRAM)
2145	skb_push(skb, len: skb->data - skb_mac_header(skb));
2146	else if (skb->pkt_type == PACKET_OUTGOING) {
2147	/* Special case: outgoing packets have ll header at head */
2148	skb_pull(skb, len: skb_network_offset(skb));
2149	}
2150	}
2151
2152	snaplen = skb_frags_readable(skb) ? skb->len : skb_headlen(skb);
2153
2154	res = run_filter(skb, sk, res: snaplen);
2155	if (!res)
2156	goto drop_n_restore;
2157	if (snaplen > res)
2158	snaplen = res;
2159
2160	if (atomic_read(v: &sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
2161	goto drop_n_acct;
2162
2163	if (skb_shared(skb)) {
2164	struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
2165	if (nskb == NULL)
2166	goto drop_n_acct;
2167
2168	if (skb_head != skb->data) {
2169	skb->data = skb_head;
2170	skb->len = skb_len;
2171	}
2172	consume_skb(skb);
2173	skb = nskb;
2174	}
2175
2176	sock_skb_cb_check_size(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8);
2177
2178	sll = &PACKET_SKB_CB(skb)->sa.ll;
2179	sll->sll_hatype = dev->type;
2180	sll->sll_pkttype = skb->pkt_type;
2181	if (unlikely(packet_sock_flag(po, PACKET_SOCK_ORIGDEV)))
2182	sll->sll_ifindex = orig_dev->ifindex;
2183	else
2184	sll->sll_ifindex = dev->ifindex;
2185
2186	sll->sll_halen = dev_parse_header(skb, haddr: sll->sll_addr);
2187
2188	/* sll->sll_family and sll->sll_protocol are set in packet_recvmsg().
2189	* Use their space for storing the original skb length.
2190	*/
2191	PACKET_SKB_CB(skb)->sa.origlen = skb->len;
2192
2193	if (pskb_trim(skb, len: snaplen))
2194	goto drop_n_acct;
2195
2196	skb_set_owner_r(skb, sk);
2197	skb->dev = NULL;
2198	skb_dst_drop(skb);
2199
2200	/* drop conntrack reference */
2201	nf_reset_ct(skb);
2202
2203	spin_lock(lock: &sk->sk_receive_queue.lock);
2204	po->stats.stats1.tp_packets++;
2205	sock_skb_set_dropcount(sk, skb);
2206	skb_clear_delivery_time(skb);
2207	__skb_queue_tail(list: &sk->sk_receive_queue, newsk: skb);
2208	spin_unlock(lock: &sk->sk_receive_queue.lock);
2209	sk->sk_data_ready(sk);
2210	return 0;
2211
2212	drop_n_acct:
2213	atomic_inc(v: &po->tp_drops);
2214	sk_drops_inc(sk);
2215	drop_reason = SKB_DROP_REASON_PACKET_SOCK_ERROR;
2216
2217	drop_n_restore:
2218	if (skb_head != skb->data && skb_shared(skb)) {
2219	skb->data = skb_head;
2220	skb->len = skb_len;
2221	}
2222	drop:
2223	sk_skb_reason_drop(sk, skb, reason: drop_reason);
2224	return 0;
2225	}
2226
2227	static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
2228	struct packet_type *pt, struct net_device *orig_dev)
2229	{
2230	enum skb_drop_reason drop_reason = SKB_CONSUMED;
2231	struct sock *sk = NULL;
2232	struct packet_sock *po;
2233	struct sockaddr_ll *sll;
2234	union tpacket_uhdr h;
2235	u8 *skb_head = skb->data;
2236	int skb_len = skb->len;
2237	unsigned int snaplen, res;
2238	unsigned long status = TP_STATUS_USER;
2239	unsigned short macoff, hdrlen;
2240	unsigned int netoff;
2241	struct sk_buff *copy_skb = NULL;
2242	struct timespec64 ts;
2243	__u32 ts_status;
2244	unsigned int slot_id = 0;
2245	int vnet_hdr_sz = 0;
2246
2247	/* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
2248	* We may add members to them until current aligned size without forcing
2249	* userspace to call getsockopt(..., PACKET_HDRLEN, ...).
2250	*/
2251	BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
2252	BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);
2253
2254	if (skb->pkt_type == PACKET_LOOPBACK)
2255	goto drop;
2256
2257	sk = pt->af_packet_priv;
2258	po = pkt_sk(sk);
2259
2260	if (!net_eq(net1: dev_net(dev), net2: sock_net(sk)))
2261	goto drop;
2262
2263	if (dev_has_header(dev)) {
2264	if (sk->sk_type != SOCK_DGRAM)
2265	skb_push(skb, len: skb->data - skb_mac_header(skb));
2266	else if (skb->pkt_type == PACKET_OUTGOING) {
2267	/* Special case: outgoing packets have ll header at head */
2268	skb_pull(skb, len: skb_network_offset(skb));
2269	}
2270	}
2271
2272	snaplen = skb_frags_readable(skb) ? skb->len : skb_headlen(skb);
2273
2274	res = run_filter(skb, sk, res: snaplen);
2275	if (!res)
2276	goto drop_n_restore;
2277
2278	/* If we are flooded, just give up */
2279	if (__packet_rcv_has_room(po, skb) == ROOM_NONE) {
2280	atomic_inc(v: &po->tp_drops);
2281	goto drop_n_restore;
2282	}
2283
2284	if (skb->ip_summed == CHECKSUM_PARTIAL)
2285	status |= TP_STATUS_CSUMNOTREADY;
2286	else if (skb->pkt_type != PACKET_OUTGOING &&
2287	skb_csum_unnecessary(skb))
2288	status |= TP_STATUS_CSUM_VALID;
2289	if (skb_is_gso(skb) && skb_is_gso_tcp(skb))
2290	status |= TP_STATUS_GSO_TCP;
2291
2292	if (snaplen > res)
2293	snaplen = res;
2294
2295	if (sk->sk_type == SOCK_DGRAM) {
2296	macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
2297	po->tp_reserve;
2298	} else {
2299	unsigned int maclen = skb_network_offset(skb);
2300	netoff = TPACKET_ALIGN(po->tp_hdrlen +
2301	(maclen < 16 ? 16 : maclen)) +
2302	po->tp_reserve;
2303	vnet_hdr_sz = READ_ONCE(po->vnet_hdr_sz);
2304	if (vnet_hdr_sz)
2305	netoff += vnet_hdr_sz;
2306	macoff = netoff - maclen;
2307	}
2308	if (netoff > USHRT_MAX) {
2309	atomic_inc(v: &po->tp_drops);
2310	goto drop_n_restore;
2311	}
2312	if (po->tp_version <= TPACKET_V2) {
2313	if (macoff + snaplen > po->rx_ring.frame_size) {
2314	if (READ_ONCE(po->copy_thresh) &&
2315	atomic_read(v: &sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
2316	if (skb_shared(skb)) {
2317	copy_skb = skb_clone(skb, GFP_ATOMIC);
2318	} else {
2319	copy_skb = skb_get(skb);
2320	skb_head = skb->data;
2321	}
2322	if (copy_skb) {
2323	memset(&PACKET_SKB_CB(copy_skb)->sa.ll, 0,
2324	sizeof(PACKET_SKB_CB(copy_skb)->sa.ll));
2325	skb_set_owner_r(skb: copy_skb, sk);
2326	}
2327	}
2328	snaplen = po->rx_ring.frame_size - macoff;
2329	if ((int)snaplen < 0) {
2330	snaplen = 0;
2331	vnet_hdr_sz = 0;
2332	}
2333	}
2334	} else if (unlikely(macoff + snaplen >
2335	GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
2336	u32 nval;
2337
2338	nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
2339	pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
2340	snaplen, nval, macoff);
2341	snaplen = nval;
2342	if (unlikely((int)snaplen < 0)) {
2343	snaplen = 0;
2344	macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
2345	vnet_hdr_sz = 0;
2346	}
2347	}
2348	spin_lock(lock: &sk->sk_receive_queue.lock);
2349	h.raw = packet_current_rx_frame(po, skb,
2350	TP_STATUS_KERNEL, len: (macoff+snaplen));
2351	if (!h.raw)
2352	goto drop_n_account;
2353
2354	if (po->tp_version <= TPACKET_V2) {
2355	slot_id = po->rx_ring.head;
2356	if (test_bit(slot_id, po->rx_ring.rx_owner_map))
2357	goto drop_n_account;
2358	__set_bit(slot_id, po->rx_ring.rx_owner_map);
2359	}
2360
2361	if (vnet_hdr_sz &&
2362	virtio_net_hdr_from_skb(skb, hdr: h.raw + macoff -
2363	sizeof(struct virtio_net_hdr),
2364	vio_le(), has_data_valid: true, vlan_hlen: 0)) {
2365	if (po->tp_version == TPACKET_V3)
2366	prb_clear_blk_fill_status(rb: &po->rx_ring);
2367	goto drop_n_account;
2368	}
2369
2370	if (po->tp_version <= TPACKET_V2) {
2371	packet_increment_rx_head(po, rb: &po->rx_ring);
2372	/*
2373	* LOSING will be reported till you read the stats,
2374	* because it's COR - Clear On Read.
2375	* Anyways, moving it for V1/V2 only as V3 doesn't need this
2376	* at packet level.
2377	*/
2378	if (atomic_read(v: &po->tp_drops))
2379	status |= TP_STATUS_LOSING;
2380	}
2381
2382	po->stats.stats1.tp_packets++;
2383	if (copy_skb) {
2384	status |= TP_STATUS_COPY;
2385	skb_clear_delivery_time(skb: copy_skb);
2386	__skb_queue_tail(list: &sk->sk_receive_queue, newsk: copy_skb);
2387	}
2388	spin_unlock(lock: &sk->sk_receive_queue.lock);
2389
2390	skb_copy_bits(skb, offset: 0, to: h.raw + macoff, len: snaplen);
2391
2392	/* Always timestamp; prefer an existing software timestamp taken
2393	* closer to the time of capture.
2394	*/
2395	ts_status = tpacket_get_timestamp(skb, ts: &ts,
2396	READ_ONCE(po->tp_tstamp) |
2397	SOF_TIMESTAMPING_SOFTWARE);
2398	if (!ts_status)
2399	ktime_get_real_ts64(tv: &ts);
2400
2401	status |= ts_status;
2402
2403	switch (po->tp_version) {
2404	case TPACKET_V1:
2405	h.h1->tp_len = skb->len;
2406	h.h1->tp_snaplen = snaplen;
2407	h.h1->tp_mac = macoff;
2408	h.h1->tp_net = netoff;
2409	h.h1->tp_sec = ts.tv_sec;
2410	h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
2411	hdrlen = sizeof(*h.h1);
2412	break;
2413	case TPACKET_V2:
2414	h.h2->tp_len = skb->len;
2415	h.h2->tp_snaplen = snaplen;
2416	h.h2->tp_mac = macoff;
2417	h.h2->tp_net = netoff;
2418	h.h2->tp_sec = ts.tv_sec;
2419	h.h2->tp_nsec = ts.tv_nsec;
2420	if (skb_vlan_tag_present(skb)) {
2421	h.h2->tp_vlan_tci = skb_vlan_tag_get(skb);
2422	h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
2423	status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
2424	} else if (unlikely(sk->sk_type == SOCK_DGRAM && eth_type_vlan(skb->protocol))) {
2425	h.h2->tp_vlan_tci = vlan_get_tci(skb, dev: skb->dev);
2426	h.h2->tp_vlan_tpid = ntohs(skb->protocol);
2427	status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
2428	} else {
2429	h.h2->tp_vlan_tci = 0;
2430	h.h2->tp_vlan_tpid = 0;
2431	}
2432	memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
2433	hdrlen = sizeof(*h.h2);
2434	break;
2435	case TPACKET_V3:
2436	/* tp_nxt_offset,vlan are already populated above.
2437	* So DONT clear those fields here
2438	*/
2439	h.h3->tp_status |= status;
2440	h.h3->tp_len = skb->len;
2441	h.h3->tp_snaplen = snaplen;
2442	h.h3->tp_mac = macoff;
2443	h.h3->tp_net = netoff;
2444	h.h3->tp_sec = ts.tv_sec;
2445	h.h3->tp_nsec = ts.tv_nsec;
2446	memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
2447	hdrlen = sizeof(*h.h3);
2448	break;
2449	default:
2450	BUG();
2451	}
2452
2453	sll = h.raw + TPACKET_ALIGN(hdrlen);
2454	sll->sll_halen = dev_parse_header(skb, haddr: sll->sll_addr);
2455	sll->sll_family = AF_PACKET;
2456	sll->sll_hatype = dev->type;
2457	sll->sll_protocol = (sk->sk_type == SOCK_DGRAM) ?
2458	vlan_get_protocol_dgram(skb) : skb->protocol;
2459	sll->sll_pkttype = skb->pkt_type;
2460	if (unlikely(packet_sock_flag(po, PACKET_SOCK_ORIGDEV)))
2461	sll->sll_ifindex = orig_dev->ifindex;
2462	else
2463	sll->sll_ifindex = dev->ifindex;
2464
2465	smp_mb();
2466
2467	#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
2468	if (po->tp_version <= TPACKET_V2) {
2469	u8 *start, *end;
2470
2471	end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
2472	macoff + snaplen);
2473
2474	for (start = h.raw; start < end; start += PAGE_SIZE)
2475	flush_dcache_page(pgv_to_page(start));
2476	}
2477	smp_wmb();
2478	#endif
2479
2480	if (po->tp_version <= TPACKET_V2) {
2481	spin_lock(lock: &sk->sk_receive_queue.lock);
2482	__packet_set_status(po, frame: h.raw, status);
2483	__clear_bit(slot_id, po->rx_ring.rx_owner_map);
2484	spin_unlock(lock: &sk->sk_receive_queue.lock);
2485	sk->sk_data_ready(sk);
2486	} else if (po->tp_version == TPACKET_V3) {
2487	prb_clear_blk_fill_status(rb: &po->rx_ring);
2488	}
2489
2490	drop_n_restore:
2491	if (skb_head != skb->data && skb_shared(skb)) {
2492	skb->data = skb_head;
2493	skb->len = skb_len;
2494	}
2495	drop:
2496	sk_skb_reason_drop(sk, skb, reason: drop_reason);
2497	return 0;
2498
2499	drop_n_account:
2500	spin_unlock(lock: &sk->sk_receive_queue.lock);
2501	atomic_inc(v: &po->tp_drops);
2502	drop_reason = SKB_DROP_REASON_PACKET_SOCK_ERROR;
2503
2504	sk->sk_data_ready(sk);
2505	sk_skb_reason_drop(sk, skb: copy_skb, reason: drop_reason);
2506	goto drop_n_restore;
2507	}
2508
2509	static void tpacket_destruct_skb(struct sk_buff *skb)
2510	{
2511	struct packet_sock *po = pkt_sk(skb->sk);
2512
2513	if (likely(po->tx_ring.pg_vec)) {
2514	void *ph;
2515	__u32 ts;
2516
2517	ph = skb_zcopy_get_nouarg(skb);
2518	packet_dec_pending(rb: &po->tx_ring);
2519
2520	ts = __packet_set_timestamp(po, frame: ph, skb);
2521	__packet_set_status(po, frame: ph, TP_STATUS_AVAILABLE | ts);
2522
2523	complete(&po->skb_completion);
2524	}
2525
2526	sock_wfree(skb);
2527	}
2528
2529	static int __packet_snd_vnet_parse(struct virtio_net_hdr *vnet_hdr, size_t len)
2530	{
2531	if ((vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
2532	(__virtio16_to_cpu(vio_le(), val: vnet_hdr->csum_start) +
2533	__virtio16_to_cpu(vio_le(), val: vnet_hdr->csum_offset) + 2 >
2534	__virtio16_to_cpu(vio_le(), val: vnet_hdr->hdr_len)))
2535	vnet_hdr->hdr_len = __cpu_to_virtio16(vio_le(),
2536	val: __virtio16_to_cpu(vio_le(), val: vnet_hdr->csum_start) +
2537	__virtio16_to_cpu(vio_le(), val: vnet_hdr->csum_offset) + 2);
2538
2539	if (__virtio16_to_cpu(vio_le(), val: vnet_hdr->hdr_len) > len)
2540	return -EINVAL;
2541
2542	return 0;
2543	}
2544
2545	static int packet_snd_vnet_parse(struct msghdr *msg, size_t *len,
2546	struct virtio_net_hdr *vnet_hdr, int vnet_hdr_sz)
2547	{
2548	int ret;
2549
2550	if (*len < vnet_hdr_sz)
2551	return -EINVAL;
2552	*len -= vnet_hdr_sz;
2553
2554	if (!copy_from_iter_full(addr: vnet_hdr, bytes: sizeof(*vnet_hdr), i: &msg->msg_iter))
2555	return -EFAULT;
2556
2557	ret = __packet_snd_vnet_parse(vnet_hdr, len: *len);
2558	if (ret)
2559	return ret;
2560
2561	/* move iter to point to the start of mac header */
2562	if (vnet_hdr_sz != sizeof(struct virtio_net_hdr))
2563	iov_iter_advance(i: &msg->msg_iter, bytes: vnet_hdr_sz - sizeof(struct virtio_net_hdr));
2564
2565	return 0;
2566	}
2567
2568	static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
2569	void *frame, struct net_device *dev, void *data, int tp_len,
2570	__be16 proto, unsigned char *addr, int hlen, int copylen,
2571	const struct sockcm_cookie *sockc)
2572	{
2573	union tpacket_uhdr ph;
2574	int to_write, offset, len, nr_frags, len_max;
2575	struct socket *sock = po->sk.sk_socket;
2576	struct page *page;
2577	int err;
2578
2579	ph.raw = frame;
2580
2581	skb->protocol = proto;
2582	skb->dev = dev;
2583	skb->priority = sockc->priority;
2584	skb->mark = sockc->mark;
2585	skb_set_delivery_type_by_clockid(skb, kt: sockc->transmit_time, clockid: po->sk.sk_clockid);
2586	skb_setup_tx_timestamp(skb, sockc);
2587	skb_zcopy_set_nouarg(skb, val: ph.raw);
2588
2589	skb_reserve(skb, len: hlen);
2590	skb_reset_network_header(skb);
2591
2592	to_write = tp_len;
2593
2594	if (sock->type == SOCK_DGRAM) {
2595	err = dev_hard_header(skb, dev, ntohs(proto), daddr: addr,
2596	NULL, len: tp_len);
2597	if (unlikely(err < 0))
2598	return -EINVAL;
2599	} else if (copylen) {
2600	int hdrlen = min_t(int, copylen, tp_len);
2601
2602	skb_push(skb, len: dev->hard_header_len);
2603	skb_put(skb, len: copylen - dev->hard_header_len);
2604	err = skb_store_bits(skb, offset: 0, from: data, len: hdrlen);
2605	if (unlikely(err))
2606	return err;
2607	if (!dev_validate_header(dev, ll_header: skb->data, len: hdrlen))
2608	return -EINVAL;
2609
2610	data += hdrlen;
2611	to_write -= hdrlen;
2612	}
2613
2614	offset = offset_in_page(data);
2615	len_max = PAGE_SIZE - offset;
2616	len = ((to_write > len_max) ? len_max : to_write);
2617
2618	skb->data_len = to_write;
2619	skb->len += to_write;
2620	skb->truesize += to_write;
2621	refcount_add(i: to_write, r: &po->sk.sk_wmem_alloc);
2622
2623	while (likely(to_write)) {
2624	nr_frags = skb_shinfo(skb)->nr_frags;
2625
2626	if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
2627	pr_err("Packet exceed the number of skb frags(%u)\n",
2628	(unsigned int)MAX_SKB_FRAGS);
2629	return -EFAULT;
2630	}
2631
2632	page = pgv_to_page(addr: data);
2633	data += len;
2634	flush_dcache_page(page);
2635	get_page(page);
2636	skb_fill_page_desc(skb, i: nr_frags, page, off: offset, size: len);
2637	to_write -= len;
2638	offset = 0;
2639	len_max = PAGE_SIZE;
2640	len = ((to_write > len_max) ? len_max : to_write);
2641	}
2642
2643	packet_parse_headers(skb, sock);
2644
2645	return tp_len;
2646	}
2647
2648	static int tpacket_parse_header(struct packet_sock *po, void *frame,
2649	int size_max, void **data)
2650	{
2651	union tpacket_uhdr ph;
2652	int tp_len, off;
2653
2654	ph.raw = frame;
2655
2656	switch (po->tp_version) {
2657	case TPACKET_V3:
2658	if (ph.h3->tp_next_offset != 0) {
2659	pr_warn_once("variable sized slot not supported");
2660	return -EINVAL;
2661	}
2662	tp_len = ph.h3->tp_len;
2663	break;
2664	case TPACKET_V2:
2665	tp_len = ph.h2->tp_len;
2666	break;
2667	default:
2668	tp_len = ph.h1->tp_len;
2669	break;
2670	}
2671	if (unlikely(tp_len > size_max)) {
2672	pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
2673	return -EMSGSIZE;
2674	}
2675
2676	if (unlikely(packet_sock_flag(po, PACKET_SOCK_TX_HAS_OFF))) {
2677	int off_min, off_max;
2678
2679	off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2680	off_max = po->tx_ring.frame_size - tp_len;
2681	if (po->sk.sk_type == SOCK_DGRAM) {
2682	switch (po->tp_version) {
2683	case TPACKET_V3:
2684	off = ph.h3->tp_net;
2685	break;
2686	case TPACKET_V2:
2687	off = ph.h2->tp_net;
2688	break;
2689	default:
2690	off = ph.h1->tp_net;
2691	break;
2692	}
2693	} else {
2694	switch (po->tp_version) {
2695	case TPACKET_V3:
2696	off = ph.h3->tp_mac;
2697	break;
2698	case TPACKET_V2:
2699	off = ph.h2->tp_mac;
2700	break;
2701	default:
2702	off = ph.h1->tp_mac;
2703	break;
2704	}
2705	}
2706	if (unlikely((off < off_min) || (off_max < off)))
2707	return -EINVAL;
2708	} else {
2709	off = po->tp_hdrlen - sizeof(struct sockaddr_ll);
2710	}
2711
2712	*data = frame + off;
2713	return tp_len;
2714	}
2715
2716	static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
2717	{
2718	struct sk_buff *skb = NULL;
2719	struct net_device *dev;
2720	struct virtio_net_hdr *vnet_hdr = NULL;
2721	struct sockcm_cookie sockc;
2722	__be16 proto;
2723	int err, reserve = 0;
2724	void *ph;
2725	DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
2726	bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
2727	int vnet_hdr_sz = READ_ONCE(po->vnet_hdr_sz);
2728	unsigned char *addr = NULL;
2729	int tp_len, size_max;
2730	void *data;
2731	int len_sum = 0;
2732	int status = TP_STATUS_AVAILABLE;
2733	int hlen, tlen, copylen = 0;
2734	long timeo;
2735
2736	mutex_lock(&po->pg_vec_lock);
2737
2738	/* packet_sendmsg() check on tx_ring.pg_vec was lockless,
2739	* we need to confirm it under protection of pg_vec_lock.
2740	*/
2741	if (unlikely(!po->tx_ring.pg_vec)) {
2742	err = -EBUSY;
2743	goto out;
2744	}
2745	if (likely(saddr == NULL)) {
2746	dev = packet_cached_dev_get(po);
2747	proto = READ_ONCE(po->num);
2748	} else {
2749	err = -EINVAL;
2750	if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2751	goto out;
2752	if (msg->msg_namelen < (saddr->sll_halen
2753	+ offsetof(struct sockaddr_ll,
2754	sll_addr)))
2755	goto out;
2756	proto = saddr->sll_protocol;
2757	dev = dev_get_by_index(net: sock_net(sk: &po->sk), ifindex: saddr->sll_ifindex);
2758	if (po->sk.sk_socket->type == SOCK_DGRAM) {
2759	if (dev && msg->msg_namelen < dev->addr_len +
2760	offsetof(struct sockaddr_ll, sll_addr))
2761	goto out_put;
2762	addr = saddr->sll_addr;
2763	}
2764	}
2765
2766	err = -ENXIO;
2767	if (unlikely(dev == NULL))
2768	goto out;
2769	err = -ENETDOWN;
2770	if (unlikely(!(dev->flags & IFF_UP)))
2771	goto out_put;
2772
2773	sockcm_init(sockc: &sockc, sk: &po->sk);
2774	if (msg->msg_controllen) {
2775	err = sock_cmsg_send(sk: &po->sk, msg, sockc: &sockc);
2776	if (unlikely(err))
2777	goto out_put;
2778	}
2779
2780	if (po->sk.sk_socket->type == SOCK_RAW)
2781	reserve = dev->hard_header_len;
2782	size_max = po->tx_ring.frame_size
2783	- (po->tp_hdrlen - sizeof(struct sockaddr_ll));
2784
2785	if ((size_max > dev->mtu + reserve + VLAN_HLEN) && !vnet_hdr_sz)
2786	size_max = dev->mtu + reserve + VLAN_HLEN;
2787
2788	timeo = sock_sndtimeo(sk: &po->sk, noblock: msg->msg_flags & MSG_DONTWAIT);
2789	reinit_completion(x: &po->skb_completion);
2790
2791	do {
2792	ph = packet_current_frame(po, rb: &po->tx_ring,
2793	TP_STATUS_SEND_REQUEST);
2794	if (unlikely(ph == NULL)) {
2795	/* Note: packet_read_pending() might be slow if we
2796	* have to call it as it's per_cpu variable, but in
2797	* fast-path we don't have to call it, only when ph
2798	* is NULL, we need to check the pending_refcnt.
2799	*/
2800	if (need_wait && packet_read_pending(rb: &po->tx_ring)) {
2801	timeo = wait_for_completion_interruptible_timeout(x: &po->skb_completion, timeout: timeo);
2802	if (timeo <= 0) {
2803	err = !timeo ? -ETIMEDOUT : -ERESTARTSYS;
2804	goto out_put;
2805	}
2806	/* check for additional frames */
2807	continue;
2808	} else
2809	break;
2810	}
2811
2812	skb = NULL;
2813	tp_len = tpacket_parse_header(po, frame: ph, size_max, data: &data);
2814	if (tp_len < 0)
2815	goto tpacket_error;
2816
2817	status = TP_STATUS_SEND_REQUEST;
2818	hlen = LL_RESERVED_SPACE(dev);
2819	tlen = dev->needed_tailroom;
2820	if (vnet_hdr_sz) {
2821	vnet_hdr = data;
2822	data += vnet_hdr_sz;
2823	tp_len -= vnet_hdr_sz;
2824	if (tp_len < 0 ||
2825	__packet_snd_vnet_parse(vnet_hdr, len: tp_len)) {
2826	tp_len = -EINVAL;
2827	goto tpacket_error;
2828	}
2829	copylen = __virtio16_to_cpu(vio_le(),
2830	val: vnet_hdr->hdr_len);
2831	}
2832	copylen = max_t(int, copylen, dev->hard_header_len);
2833	skb = sock_alloc_send_skb(sk: &po->sk,
2834	size: hlen + tlen + sizeof(struct sockaddr_ll) +
2835	(copylen - dev->hard_header_len),
2836	noblock: !need_wait, errcode: &err);
2837
2838	if (unlikely(skb == NULL)) {
2839	/* we assume the socket was initially writeable ... */
2840	if (likely(len_sum > 0))
2841	err = len_sum;
2842	goto out_status;
2843	}
2844	tp_len = tpacket_fill_skb(po, skb, frame: ph, dev, data, tp_len, proto,
2845	addr, hlen, copylen, sockc: &sockc);
2846	if (likely(tp_len >= 0) &&
2847	tp_len > dev->mtu + reserve &&
2848	!vnet_hdr_sz &&
2849	!packet_extra_vlan_len_allowed(dev, skb))
2850	tp_len = -EMSGSIZE;
2851
2852	if (unlikely(tp_len < 0)) {
2853	tpacket_error:
2854	if (packet_sock_flag(po, flag: PACKET_SOCK_TP_LOSS)) {
2855	__packet_set_status(po, frame: ph,
2856	TP_STATUS_AVAILABLE);
2857	packet_increment_head(buff: &po->tx_ring);
2858	kfree_skb(skb);
2859	continue;
2860	} else {
2861	status = TP_STATUS_WRONG_FORMAT;
2862	err = tp_len;
2863	goto out_status;
2864	}
2865	}
2866
2867	if (vnet_hdr_sz) {
2868	if (virtio_net_hdr_to_skb(skb, hdr: vnet_hdr, vio_le())) {
2869	tp_len = -EINVAL;
2870	goto tpacket_error;
2871	}
2872	virtio_net_hdr_set_proto(skb, hdr: vnet_hdr);
2873	}
2874
2875	skb->destructor = tpacket_destruct_skb;
2876	__packet_set_status(po, frame: ph, TP_STATUS_SENDING);
2877	packet_inc_pending(rb: &po->tx_ring);
2878
2879	status = TP_STATUS_SEND_REQUEST;
2880	err = packet_xmit(po, skb);
2881	if (unlikely(err != 0)) {
2882	if (err > 0)
2883	err = net_xmit_errno(err);
2884	if (err && __packet_get_status(po, frame: ph) ==
2885	TP_STATUS_AVAILABLE) {
2886	/* skb was destructed already */
2887	skb = NULL;
2888	goto out_status;
2889	}
2890	/*
2891	* skb was dropped but not destructed yet;
2892	* let's treat it like congestion or err < 0
2893	*/
2894	err = 0;
2895	}
2896	packet_increment_head(buff: &po->tx_ring);
2897	len_sum += tp_len;
2898	} while (1);
2899
2900	err = len_sum;
2901	goto out_put;
2902
2903	out_status:
2904	__packet_set_status(po, frame: ph, status);
2905	kfree_skb(skb);
2906	out_put:
2907	dev_put(dev);
2908	out:
2909	mutex_unlock(lock: &po->pg_vec_lock);
2910	return err;
2911	}
2912
2913	static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
2914	size_t reserve, size_t len,
2915	size_t linear, int noblock,
2916	int *err)
2917	{
2918	struct sk_buff *skb;
2919
2920	/* Under a page? Don't bother with paged skb. */
2921	if (prepad + len < PAGE_SIZE || !linear)
2922	linear = len;
2923
2924	if (len - linear > MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER))
2925	linear = len - MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER);
2926	skb = sock_alloc_send_pskb(sk, header_len: prepad + linear, data_len: len - linear, noblock,
2927	errcode: err, PAGE_ALLOC_COSTLY_ORDER);
2928	if (!skb)
2929	return NULL;
2930
2931	skb_reserve(skb, len: reserve);
2932	skb_put(skb, len: linear);
2933	skb->data_len = len - linear;
2934	skb->len += len - linear;
2935
2936	return skb;
2937	}
2938
2939	static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
2940	{
2941	struct sock *sk = sock->sk;
2942	DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
2943	struct sk_buff *skb;
2944	struct net_device *dev;
2945	__be16 proto;
2946	unsigned char *addr = NULL;
2947	int err, reserve = 0;
2948	struct sockcm_cookie sockc;
2949	struct virtio_net_hdr vnet_hdr = { 0 };
2950	int offset = 0;
2951	struct packet_sock *po = pkt_sk(sk);
2952	int vnet_hdr_sz = READ_ONCE(po->vnet_hdr_sz);
2953	int hlen, tlen, linear;
2954	int extra_len = 0;
2955
2956	/*
2957	* Get and verify the address.
2958	*/
2959
2960	if (likely(saddr == NULL)) {
2961	dev = packet_cached_dev_get(po);
2962	proto = READ_ONCE(po->num);
2963	} else {
2964	err = -EINVAL;
2965	if (msg->msg_namelen < sizeof(struct sockaddr_ll))
2966	goto out;
2967	if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
2968	goto out;
2969	proto = saddr->sll_protocol;
2970	dev = dev_get_by_index(net: sock_net(sk), ifindex: saddr->sll_ifindex);
2971	if (sock->type == SOCK_DGRAM) {
2972	if (dev && msg->msg_namelen < dev->addr_len +
2973	offsetof(struct sockaddr_ll, sll_addr))
2974	goto out_unlock;
2975	addr = saddr->sll_addr;
2976	}
2977	}
2978
2979	err = -ENXIO;
2980	if (unlikely(dev == NULL))
2981	goto out_unlock;
2982	err = -ENETDOWN;
2983	if (unlikely(!(dev->flags & IFF_UP)))
2984	goto out_unlock;
2985
2986	sockcm_init(sockc: &sockc, sk);
2987	if (msg->msg_controllen) {
2988	err = sock_cmsg_send(sk, msg, sockc: &sockc);
2989	if (unlikely(err))
2990	goto out_unlock;
2991	}
2992
2993	if (sock->type == SOCK_RAW)
2994	reserve = dev->hard_header_len;
2995	if (vnet_hdr_sz) {
2996	err = packet_snd_vnet_parse(msg, len: &len, vnet_hdr: &vnet_hdr, vnet_hdr_sz);
2997	if (err)
2998	goto out_unlock;
2999	}
3000
3001	if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
3002	if (!netif_supports_nofcs(dev)) {
3003	err = -EPROTONOSUPPORT;
3004	goto out_unlock;
3005	}
3006	extra_len = 4; /* We're doing our own CRC */
3007	}
3008
3009	err = -EMSGSIZE;
3010	if (!vnet_hdr.gso_type &&
3011	(len > dev->mtu + reserve + VLAN_HLEN + extra_len))
3012	goto out_unlock;
3013
3014	err = -ENOBUFS;
3015	hlen = LL_RESERVED_SPACE(dev);
3016	tlen = dev->needed_tailroom;
3017	linear = __virtio16_to_cpu(vio_le(), val: vnet_hdr.hdr_len);
3018	linear = max(linear, min_t(int, len, dev->hard_header_len));
3019	skb = packet_alloc_skb(sk, prepad: hlen + tlen, reserve: hlen, len, linear,
3020	noblock: msg->msg_flags & MSG_DONTWAIT, err: &err);
3021	if (skb == NULL)
3022	goto out_unlock;
3023
3024	skb_reset_network_header(skb);
3025
3026	err = -EINVAL;
3027	if (sock->type == SOCK_DGRAM) {
3028	offset = dev_hard_header(skb, dev, ntohs(proto), daddr: addr, NULL, len);
3029	if (unlikely(offset < 0))
3030	goto out_free;
3031	} else if (reserve) {
3032	skb_reserve(skb, len: -reserve);
3033	if (len < reserve + sizeof(struct ipv6hdr) &&
3034	dev->min_header_len != dev->hard_header_len)
3035	skb_reset_network_header(skb);
3036	}
3037
3038	/* Returns -EFAULT on error */
3039	err = skb_copy_datagram_from_iter(skb, offset, from: &msg->msg_iter, len);
3040	if (err)
3041	goto out_free;
3042
3043	if ((sock->type == SOCK_RAW &&
3044	!dev_validate_header(dev, ll_header: skb->data, len)) || !skb->len) {
3045	err = -EINVAL;
3046	goto out_free;
3047	}
3048
3049	skb_setup_tx_timestamp(skb, sockc: &sockc);
3050
3051	if (!vnet_hdr.gso_type && (len > dev->mtu + reserve + extra_len) &&
3052	!packet_extra_vlan_len_allowed(dev, skb)) {
3053	err = -EMSGSIZE;
3054	goto out_free;
3055	}
3056
3057	skb->protocol = proto;
3058	skb->dev = dev;
3059	skb->priority = sockc.priority;
3060	skb->mark = sockc.mark;
3061	skb_set_delivery_type_by_clockid(skb, kt: sockc.transmit_time, clockid: sk->sk_clockid);
3062
3063	if (unlikely(extra_len == 4))
3064	skb->no_fcs = 1;
3065
3066	packet_parse_headers(skb, sock);
3067
3068	if (vnet_hdr_sz) {
3069	err = virtio_net_hdr_to_skb(skb, hdr: &vnet_hdr, vio_le());
3070	if (err)
3071	goto out_free;
3072	len += vnet_hdr_sz;
3073	virtio_net_hdr_set_proto(skb, hdr: &vnet_hdr);
3074	}
3075
3076	err = packet_xmit(po, skb);
3077
3078	if (unlikely(err != 0)) {
3079	if (err > 0)
3080	err = net_xmit_errno(err);
3081	if (err)
3082	goto out_unlock;
3083	}
3084
3085	dev_put(dev);
3086
3087	return len;
3088
3089	out_free:
3090	kfree_skb(skb);
3091	out_unlock:
3092	dev_put(dev);
3093	out:
3094	return err;
3095	}
3096
3097	static int packet_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
3098	{
3099	struct sock *sk = sock->sk;
3100	struct packet_sock *po = pkt_sk(sk);
3101
3102	/* Reading tx_ring.pg_vec without holding pg_vec_lock is racy.
3103	* tpacket_snd() will redo the check safely.
3104	*/
3105	if (data_race(po->tx_ring.pg_vec))
3106	return tpacket_snd(po, msg);
3107
3108	return packet_snd(sock, msg, len);
3109	}
3110
3111	/*
3112	* Close a PACKET socket. This is fairly simple. We immediately go
3113	* to 'closed' state and remove our protocol entry in the device list.
3114	*/
3115
3116	static int packet_release(struct socket *sock)
3117	{
3118	struct sock *sk = sock->sk;
3119	struct packet_sock *po;
3120	struct packet_fanout *f;
3121	struct net *net;
3122	union tpacket_req_u req_u;
3123
3124	if (!sk)
3125	return 0;
3126
3127	net = sock_net(sk);
3128	po = pkt_sk(sk);
3129
3130	mutex_lock(&net->packet.sklist_lock);
3131	sk_del_node_init_rcu(sk);
3132	mutex_unlock(lock: &net->packet.sklist_lock);
3133
3134	sock_prot_inuse_add(net, prot: sk->sk_prot, val: -1);
3135
3136	spin_lock(lock: &po->bind_lock);
3137	unregister_prot_hook(sk, sync: false);
3138	packet_cached_dev_reset(po);
3139
3140	if (po->prot_hook.dev) {
3141	netdev_put(dev: po->prot_hook.dev, tracker: &po->prot_hook.dev_tracker);
3142	po->prot_hook.dev = NULL;
3143	}
3144	spin_unlock(lock: &po->bind_lock);
3145
3146	packet_flush_mclist(sk);
3147
3148	lock_sock(sk);
3149	if (po->rx_ring.pg_vec) {
3150	memset(&req_u, 0, sizeof(req_u));
3151	packet_set_ring(sk, req_u: &req_u, closing: 1, tx_ring: 0);
3152	}
3153
3154	if (po->tx_ring.pg_vec) {
3155	memset(&req_u, 0, sizeof(req_u));
3156	packet_set_ring(sk, req_u: &req_u, closing: 1, tx_ring: 1);
3157	}
3158	release_sock(sk);
3159
3160	f = fanout_release(sk);
3161
3162	synchronize_net();
3163
3164	kfree(objp: po->rollover);
3165	if (f) {
3166	fanout_release_data(f);
3167	kvfree(addr: f);
3168	}
3169	/*
3170	* Now the socket is dead. No more input will appear.
3171	*/
3172	sock_orphan(sk);
3173	sock->sk = NULL;
3174
3175	/* Purge queues */
3176
3177	skb_queue_purge(list: &sk->sk_receive_queue);
3178	packet_free_pending(po);
3179
3180	sock_put(sk);
3181	return 0;
3182	}
3183
3184	/*
3185	* Attach a packet hook.
3186	*/
3187
3188	static int packet_do_bind(struct sock *sk, const char *name, int ifindex,
3189	__be16 proto)
3190	{
3191	struct packet_sock *po = pkt_sk(sk);
3192	struct net_device *dev = NULL;
3193	bool unlisted = false;
3194	bool need_rehook;
3195	int ret = 0;
3196
3197	lock_sock(sk);
3198	spin_lock(lock: &po->bind_lock);
3199	if (!proto)
3200	proto = po->num;
3201
3202	rcu_read_lock();
3203
3204	if (po->fanout) {
3205	ret = -EINVAL;
3206	goto out_unlock;
3207	}
3208
3209	if (name) {
3210	dev = dev_get_by_name_rcu(net: sock_net(sk), name);
3211	if (!dev) {
3212	ret = -ENODEV;
3213	goto out_unlock;
3214	}
3215	} else if (ifindex) {
3216	dev = dev_get_by_index_rcu(net: sock_net(sk), ifindex);
3217	if (!dev) {
3218	ret = -ENODEV;
3219	goto out_unlock;
3220	}
3221	}
3222
3223	need_rehook = po->prot_hook.type != proto || po->prot_hook.dev != dev;
3224
3225	if (need_rehook) {
3226	dev_hold(dev);
3227	if (packet_sock_flag(po, flag: PACKET_SOCK_RUNNING)) {
3228	rcu_read_unlock();
3229	/* prevents packet_notifier() from calling
3230	* register_prot_hook()
3231	*/
3232	WRITE_ONCE(po->num, 0);
3233	__unregister_prot_hook(sk, sync: true);
3234	rcu_read_lock();
3235	if (dev)
3236	unlisted = !dev_get_by_index_rcu(net: sock_net(sk),
3237	ifindex: dev->ifindex);
3238	}
3239
3240	BUG_ON(packet_sock_flag(po, PACKET_SOCK_RUNNING));
3241	WRITE_ONCE(po->num, proto);
3242	po->prot_hook.type = proto;
3243
3244	netdev_put(dev: po->prot_hook.dev, tracker: &po->prot_hook.dev_tracker);
3245
3246	if (unlikely(unlisted)) {
3247	po->prot_hook.dev = NULL;
3248	WRITE_ONCE(po->ifindex, -1);
3249	packet_cached_dev_reset(po);
3250	} else {
3251	netdev_hold(dev, tracker: &po->prot_hook.dev_tracker,
3252	GFP_ATOMIC);
3253	po->prot_hook.dev = dev;
3254	WRITE_ONCE(po->ifindex, dev ? dev->ifindex : 0);
3255	packet_cached_dev_assign(po, dev);
3256	}
3257	dev_put(dev);
3258	}
3259
3260	if (proto == 0 || !need_rehook)
3261	goto out_unlock;
3262
3263	if (!unlisted && (!dev || (dev->flags & IFF_UP))) {
3264	register_prot_hook(sk);
3265	} else {
3266	sk->sk_err = ENETDOWN;
3267	if (!sock_flag(sk, flag: SOCK_DEAD))
3268	sk_error_report(sk);
3269	}
3270
3271	out_unlock:
3272	rcu_read_unlock();
3273	spin_unlock(lock: &po->bind_lock);
3274	release_sock(sk);
3275	return ret;
3276	}
3277
3278	/*
3279	* Bind a packet socket to a device
3280	*/
3281
3282	static int packet_bind_spkt(struct socket *sock, struct sockaddr_unsized *uaddr,
3283	int addr_len)
3284	{
3285	struct sock *sk = sock->sk;
3286	struct sockaddr *sa = (struct sockaddr *)uaddr;
3287	char name[sizeof(sa->sa_data) + 1];
3288
3289	/*
3290	* Check legality
3291	*/
3292
3293	if (addr_len != sizeof(struct sockaddr))
3294	return -EINVAL;
3295	/* uaddr->sa_data comes from the userspace, it's not guaranteed to be
3296	* zero-terminated.
3297	*/
3298	memcpy(name, sa->sa_data, sizeof(sa->sa_data));
3299	name[sizeof(sa->sa_data)] = 0;
3300
3301	return packet_do_bind(sk, name, ifindex: 0, proto: 0);
3302	}
3303
3304	static int packet_bind(struct socket *sock, struct sockaddr_unsized *uaddr, int addr_len)
3305	{
3306	struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
3307	struct sock *sk = sock->sk;
3308
3309	/*
3310	* Check legality
3311	*/
3312
3313	if (addr_len < sizeof(struct sockaddr_ll))
3314	return -EINVAL;
3315	if (sll->sll_family != AF_PACKET)
3316	return -EINVAL;
3317
3318	return packet_do_bind(sk, NULL, ifindex: sll->sll_ifindex, proto: sll->sll_protocol);
3319	}
3320
3321	static struct proto packet_proto = {
3322	.name = "PACKET",
3323	.owner = THIS_MODULE,
3324	.obj_size = sizeof(struct packet_sock),
3325	};
3326
3327	/*
3328	* Create a packet of type SOCK_PACKET.
3329	*/
3330
3331	static int packet_create(struct net *net, struct socket *sock, int protocol,
3332	int kern)
3333	{
3334	struct sock *sk;
3335	struct packet_sock *po;
3336	__be16 proto = (__force __be16)protocol; /* weird, but documented */
3337	int err;
3338
3339	if (!ns_capable(ns: net->user_ns, CAP_NET_RAW))
3340	return -EPERM;
3341	if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
3342	sock->type != SOCK_PACKET)
3343	return -ESOCKTNOSUPPORT;
3344
3345	sock->state = SS_UNCONNECTED;
3346
3347	err = -ENOBUFS;
3348	sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, prot: &packet_proto, kern);
3349	if (sk == NULL)
3350	goto out;
3351
3352	sock->ops = &packet_ops;
3353	if (sock->type == SOCK_PACKET)
3354	sock->ops = &packet_ops_spkt;
3355
3356	po = pkt_sk(sk);
3357	err = packet_alloc_pending(po);
3358	if (err)
3359	goto out_sk_free;
3360
3361	sock_init_data(sock, sk);
3362
3363	init_completion(x: &po->skb_completion);
3364	sk->sk_family = PF_PACKET;
3365	po->num = proto;
3366
3367	packet_cached_dev_reset(po);
3368
3369	sk->sk_destruct = packet_sock_destruct;
3370
3371	/*
3372	* Attach a protocol block
3373	*/
3374
3375	spin_lock_init(&po->bind_lock);
3376	mutex_init(&po->pg_vec_lock);
3377	po->rollover = NULL;
3378	po->prot_hook.func = packet_rcv;
3379
3380	if (sock->type == SOCK_PACKET)
3381	po->prot_hook.func = packet_rcv_spkt;
3382
3383	po->prot_hook.af_packet_priv = sk;
3384	po->prot_hook.af_packet_net = sock_net(sk);
3385
3386	if (proto) {
3387	po->prot_hook.type = proto;
3388	__register_prot_hook(sk);
3389	}
3390
3391	mutex_lock(&net->packet.sklist_lock);
3392	sk_add_node_tail_rcu(sk, list: &net->packet.sklist);
3393	mutex_unlock(lock: &net->packet.sklist_lock);
3394
3395	sock_prot_inuse_add(net, prot: &packet_proto, val: 1);
3396
3397	return 0;
3398	out_sk_free:
3399	sk_free(sk);
3400	out:
3401	return err;
3402	}
3403
3404	/*
3405	* Pull a packet from our receive queue and hand it to the user.
3406	* If necessary we block.
3407	*/
3408
3409	static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
3410	int flags)
3411	{
3412	struct sock *sk = sock->sk;
3413	struct sk_buff *skb;
3414	int copied, err;
3415	int vnet_hdr_len = READ_ONCE(pkt_sk(sk)->vnet_hdr_sz);
3416	unsigned int origlen = 0;
3417
3418	err = -EINVAL;
3419	if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
3420	goto out;
3421
3422	#if 0
3423	/* What error should we return now? EUNATTACH? */
3424	if (pkt_sk(sk)->ifindex < 0)
3425	return -ENODEV;
3426	#endif
3427
3428	if (flags & MSG_ERRQUEUE) {
3429	err = sock_recv_errqueue(sk, msg, len,
3430	SOL_PACKET, PACKET_TX_TIMESTAMP);
3431	goto out;
3432	}
3433
3434	/*
3435	* Call the generic datagram receiver. This handles all sorts
3436	* of horrible races and re-entrancy so we can forget about it
3437	* in the protocol layers.
3438	*
3439	* Now it will return ENETDOWN, if device have just gone down,
3440	* but then it will block.
3441	*/
3442
3443	skb = skb_recv_datagram(sk, flags, err: &err);
3444
3445	/*
3446	* An error occurred so return it. Because skb_recv_datagram()
3447	* handles the blocking we don't see and worry about blocking
3448	* retries.
3449	*/
3450
3451	if (skb == NULL)
3452	goto out;
3453
3454	packet_rcv_try_clear_pressure(pkt_sk(sk));
3455
3456	if (vnet_hdr_len) {
3457	err = packet_rcv_vnet(msg, skb, len: &len, vnet_hdr_sz: vnet_hdr_len);
3458	if (err)
3459	goto out_free;
3460	}
3461
3462	/* You lose any data beyond the buffer you gave. If it worries
3463	* a user program they can ask the device for its MTU
3464	* anyway.
3465	*/
3466	copied = skb->len;
3467	if (copied > len) {
3468	copied = len;
3469	msg->msg_flags |= MSG_TRUNC;
3470	}
3471
3472	err = skb_copy_datagram_msg(from: skb, offset: 0, msg, size: copied);
3473	if (err)
3474	goto out_free;
3475
3476	if (sock->type != SOCK_PACKET) {
3477	struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3478
3479	/* Original length was stored in sockaddr_ll fields */
3480	origlen = PACKET_SKB_CB(skb)->sa.origlen;
3481	sll->sll_family = AF_PACKET;
3482	sll->sll_protocol = (sock->type == SOCK_DGRAM) ?
3483	vlan_get_protocol_dgram(skb) : skb->protocol;
3484	}
3485
3486	sock_recv_cmsgs(msg, sk, skb);
3487
3488	if (msg->msg_name) {
3489	const size_t max_len = min(sizeof(skb->cb),
3490	sizeof(struct sockaddr_storage));
3491	int copy_len;
3492
3493	/* If the address length field is there to be filled
3494	* in, we fill it in now.
3495	*/
3496	if (sock->type == SOCK_PACKET) {
3497	__sockaddr_check_size(sizeof(struct sockaddr_pkt));
3498	msg->msg_namelen = sizeof(struct sockaddr_pkt);
3499	copy_len = msg->msg_namelen;
3500	} else {
3501	struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3502
3503	msg->msg_namelen = sll->sll_halen +
3504	offsetof(struct sockaddr_ll, sll_addr);
3505	copy_len = msg->msg_namelen;
3506	if (msg->msg_namelen < sizeof(struct sockaddr_ll)) {
3507	memset(msg->msg_name +
3508	offsetof(struct sockaddr_ll, sll_addr),
3509	0, sizeof(sll->sll_addr));
3510	msg->msg_namelen = sizeof(struct sockaddr_ll);
3511	}
3512	}
3513	if (WARN_ON_ONCE(copy_len > max_len)) {
3514	copy_len = max_len;
3515	msg->msg_namelen = copy_len;
3516	}
3517	memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa, copy_len);
3518	}
3519
3520	if (packet_sock_flag(pkt_sk(sk), flag: PACKET_SOCK_AUXDATA)) {
3521	struct tpacket_auxdata aux;
3522
3523	aux.tp_status = TP_STATUS_USER;
3524	if (skb->ip_summed == CHECKSUM_PARTIAL)
3525	aux.tp_status |= TP_STATUS_CSUMNOTREADY;
3526	else if (skb->pkt_type != PACKET_OUTGOING &&
3527	skb_csum_unnecessary(skb))
3528	aux.tp_status |= TP_STATUS_CSUM_VALID;
3529	if (skb_is_gso(skb) && skb_is_gso_tcp(skb))
3530	aux.tp_status |= TP_STATUS_GSO_TCP;
3531
3532	aux.tp_len = origlen;
3533	aux.tp_snaplen = skb->len;
3534	aux.tp_mac = 0;
3535	aux.tp_net = skb_network_offset(skb);
3536	if (skb_vlan_tag_present(skb)) {
3537	aux.tp_vlan_tci = skb_vlan_tag_get(skb);
3538	aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
3539	aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
3540	} else if (unlikely(sock->type == SOCK_DGRAM && eth_type_vlan(skb->protocol))) {
3541	struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
3542	struct net_device *dev;
3543
3544	rcu_read_lock();
3545	dev = dev_get_by_index_rcu(net: sock_net(sk), ifindex: sll->sll_ifindex);
3546	if (dev) {
3547	aux.tp_vlan_tci = vlan_get_tci(skb, dev);
3548	aux.tp_vlan_tpid = ntohs(skb->protocol);
3549	aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
3550	} else {
3551	aux.tp_vlan_tci = 0;
3552	aux.tp_vlan_tpid = 0;
3553	}
3554	rcu_read_unlock();
3555	} else {
3556	aux.tp_vlan_tci = 0;
3557	aux.tp_vlan_tpid = 0;
3558	}
3559	put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, len: sizeof(aux), data: &aux);
3560	}
3561
3562	/*
3563	* Free or return the buffer as appropriate. Again this
3564	* hides all the races and re-entrancy issues from us.
3565	*/
3566	err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
3567
3568	out_free:
3569	skb_free_datagram(sk, skb);
3570	out:
3571	return err;
3572	}
3573
3574	static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
3575	int peer)
3576	{
3577	struct net_device *dev;
3578	struct sock *sk = sock->sk;
3579
3580	if (peer)
3581	return -EOPNOTSUPP;
3582
3583	uaddr->sa_family = AF_PACKET;
3584	memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data));
3585	rcu_read_lock();
3586	dev = dev_get_by_index_rcu(net: sock_net(sk), READ_ONCE(pkt_sk(sk)->ifindex));
3587	if (dev)
3588	strscpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
3589	rcu_read_unlock();
3590
3591	return sizeof(*uaddr);
3592	}
3593
3594	static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
3595	int peer)
3596	{
3597	struct net_device *dev;
3598	struct sock *sk = sock->sk;
3599	struct packet_sock *po = pkt_sk(sk);
3600	DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
3601	int ifindex;
3602
3603	if (peer)
3604	return -EOPNOTSUPP;
3605
3606	ifindex = READ_ONCE(po->ifindex);
3607	sll->sll_family = AF_PACKET;
3608	sll->sll_ifindex = ifindex;
3609	sll->sll_protocol = READ_ONCE(po->num);
3610	sll->sll_pkttype = 0;
3611	rcu_read_lock();
3612	dev = dev_get_by_index_rcu(net: sock_net(sk), ifindex);
3613	if (dev) {
3614	sll->sll_hatype = dev->type;
3615	sll->sll_halen = dev->addr_len;
3616
3617	/* Let __fortify_memcpy_chk() know the actual buffer size. */
3618	memcpy(((struct sockaddr_storage *)sll)->__data +
3619	offsetof(struct sockaddr_ll, sll_addr) -
3620	offsetofend(struct sockaddr_ll, sll_family),
3621	dev->dev_addr, dev->addr_len);
3622	} else {
3623	sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
3624	sll->sll_halen = 0;
3625	}
3626	rcu_read_unlock();
3627
3628	return offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
3629	}
3630
3631	static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
3632	int what)
3633	{
3634	switch (i->type) {
3635	case PACKET_MR_MULTICAST:
3636	if (i->alen != dev->addr_len)
3637	return -EINVAL;
3638	if (what > 0)
3639	return dev_mc_add(dev, addr: i->addr);
3640	else
3641	return dev_mc_del(dev, addr: i->addr);
3642	break;
3643	case PACKET_MR_PROMISC:
3644	return dev_set_promiscuity(dev, inc: what);
3645	case PACKET_MR_ALLMULTI:
3646	return dev_set_allmulti(dev, inc: what);
3647	case PACKET_MR_UNICAST:
3648	if (i->alen != dev->addr_len)
3649	return -EINVAL;
3650	if (what > 0)
3651	return dev_uc_add(dev, addr: i->addr);
3652	else
3653	return dev_uc_del(dev, addr: i->addr);
3654	break;
3655	default:
3656	break;
3657	}
3658	return 0;
3659	}
3660
3661	static void packet_dev_mclist_delete(struct net_device *dev,
3662	struct packet_mclist **mlp,
3663	struct list_head *list)
3664	{
3665	struct packet_mclist *ml;
3666
3667	while ((ml = *mlp) != NULL) {
3668	if (ml->ifindex == dev->ifindex) {
3669	list_add(new: &ml->remove_list, head: list);
3670	*mlp = ml->next;
3671	} else
3672	mlp = &ml->next;
3673	}
3674	}
3675
3676	static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
3677	{
3678	struct packet_sock *po = pkt_sk(sk);
3679	struct packet_mclist *ml, *i;
3680	struct net_device *dev;
3681	int err;
3682
3683	rtnl_lock();
3684
3685	err = -ENODEV;
3686	dev = __dev_get_by_index(net: sock_net(sk), ifindex: mreq->mr_ifindex);
3687	if (!dev)
3688	goto done;
3689
3690	err = -EINVAL;
3691	if (mreq->mr_alen > dev->addr_len)
3692	goto done;
3693
3694	err = -ENOBUFS;
3695	i = kmalloc(sizeof(*i), GFP_KERNEL);
3696	if (i == NULL)
3697	goto done;
3698
3699	err = 0;
3700	for (ml = po->mclist; ml; ml = ml->next) {
3701	if (ml->ifindex == mreq->mr_ifindex &&
3702	ml->type == mreq->mr_type &&
3703	ml->alen == mreq->mr_alen &&
3704	memcmp(p: ml->addr, q: mreq->mr_address, size: ml->alen) == 0) {
3705	ml->count++;
3706	/* Free the new element ... */
3707	kfree(objp: i);
3708	goto done;
3709	}
3710	}
3711
3712	i->type = mreq->mr_type;
3713	i->ifindex = mreq->mr_ifindex;
3714	i->alen = mreq->mr_alen;
3715	memcpy(i->addr, mreq->mr_address, i->alen);
3716	memset(i->addr + i->alen, 0, sizeof(i->addr) - i->alen);
3717	i->count = 1;
3718	INIT_LIST_HEAD(list: &i->remove_list);
3719	i->next = po->mclist;
3720	po->mclist = i;
3721	err = packet_dev_mc(dev, i, what: 1);
3722	if (err) {
3723	po->mclist = i->next;
3724	kfree(objp: i);
3725	}
3726
3727	done:
3728	rtnl_unlock();
3729	return err;
3730	}
3731
3732	static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
3733	{
3734	struct packet_mclist *ml, **mlp;
3735
3736	rtnl_lock();
3737
3738	for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
3739	if (ml->ifindex == mreq->mr_ifindex &&
3740	ml->type == mreq->mr_type &&
3741	ml->alen == mreq->mr_alen &&
3742	memcmp(p: ml->addr, q: mreq->mr_address, size: ml->alen) == 0) {
3743	if (--ml->count == 0) {
3744	struct net_device *dev;
3745	*mlp = ml->next;
3746	dev = __dev_get_by_index(net: sock_net(sk), ifindex: ml->ifindex);
3747	if (dev)
3748	packet_dev_mc(dev, i: ml, what: -1);
3749	kfree(objp: ml);
3750	}
3751	break;
3752	}
3753	}
3754	rtnl_unlock();
3755	return 0;
3756	}
3757
3758	static void packet_flush_mclist(struct sock *sk)
3759	{
3760	struct packet_sock *po = pkt_sk(sk);
3761	struct packet_mclist *ml;
3762
3763	if (!po->mclist)
3764	return;
3765
3766	rtnl_lock();
3767	while ((ml = po->mclist) != NULL) {
3768	struct net_device *dev;
3769
3770	po->mclist = ml->next;
3771	dev = __dev_get_by_index(net: sock_net(sk), ifindex: ml->ifindex);
3772	if (dev != NULL)
3773	packet_dev_mc(dev, i: ml, what: -1);
3774	kfree(objp: ml);
3775	}
3776	rtnl_unlock();
3777	}
3778
3779	static int
3780	packet_setsockopt(struct socket *sock, int level, int optname, sockptr_t optval,
3781	unsigned int optlen)
3782	{
3783	struct sock *sk = sock->sk;
3784	struct packet_sock *po = pkt_sk(sk);
3785	int ret;
3786
3787	if (level != SOL_PACKET)
3788	return -ENOPROTOOPT;
3789
3790	switch (optname) {
3791	case PACKET_ADD_MEMBERSHIP:
3792	case PACKET_DROP_MEMBERSHIP:
3793	{
3794	struct packet_mreq_max mreq;
3795	int len = optlen;
3796	memset(&mreq, 0, sizeof(mreq));
3797	if (len < sizeof(struct packet_mreq))
3798	return -EINVAL;
3799	if (len > sizeof(mreq))
3800	len = sizeof(mreq);
3801	if (copy_from_sockptr(dst: &mreq, src: optval, size: len))
3802	return -EFAULT;
3803	if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
3804	return -EINVAL;
3805	if (optname == PACKET_ADD_MEMBERSHIP)
3806	ret = packet_mc_add(sk, mreq: &mreq);
3807	else
3808	ret = packet_mc_drop(sk, mreq: &mreq);
3809	return ret;
3810	}
3811
3812	case PACKET_RX_RING:
3813	case PACKET_TX_RING:
3814	{
3815	union tpacket_req_u req_u;
3816
3817	ret = -EINVAL;
3818	lock_sock(sk);
3819	switch (po->tp_version) {
3820	case TPACKET_V1:
3821	case TPACKET_V2:
3822	if (optlen < sizeof(req_u.req))
3823	break;
3824	ret = copy_from_sockptr(dst: &req_u.req, src: optval,
3825	size: sizeof(req_u.req)) ?
3826	-EINVAL : 0;
3827	break;
3828	case TPACKET_V3:
3829	default:
3830	if (optlen < sizeof(req_u.req3))
3831	break;
3832	ret = copy_from_sockptr(dst: &req_u.req3, src: optval,
3833	size: sizeof(req_u.req3)) ?
3834	-EINVAL : 0;
3835	break;
3836	}
3837	if (!ret)
3838	ret = packet_set_ring(sk, req_u: &req_u, closing: 0,
3839	tx_ring: optname == PACKET_TX_RING);
3840	release_sock(sk);
3841	return ret;
3842	}
3843	case PACKET_COPY_THRESH:
3844	{
3845	int val;
3846
3847	if (optlen != sizeof(val))
3848	return -EINVAL;
3849	if (copy_from_sockptr(dst: &val, src: optval, size: sizeof(val)))
3850	return -EFAULT;
3851
3852	WRITE_ONCE(pkt_sk(sk)->copy_thresh, val);
3853	return 0;
3854	}
3855	case PACKET_VERSION:
3856	{
3857	int val;
3858
3859	if (optlen != sizeof(val))
3860	return -EINVAL;
3861	if (copy_from_sockptr(dst: &val, src: optval, size: sizeof(val)))
3862	return -EFAULT;
3863	switch (val) {
3864	case TPACKET_V1:
3865	case TPACKET_V2:
3866	case TPACKET_V3:
3867	break;
3868	default:
3869	return -EINVAL;
3870	}
3871	lock_sock(sk);
3872	if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3873	ret = -EBUSY;
3874	} else {
3875	po->tp_version = val;
3876	ret = 0;
3877	}
3878	release_sock(sk);
3879	return ret;
3880	}
3881	case PACKET_RESERVE:
3882	{
3883	unsigned int val;
3884
3885	if (optlen != sizeof(val))
3886	return -EINVAL;
3887	if (copy_from_sockptr(dst: &val, src: optval, size: sizeof(val)))
3888	return -EFAULT;
3889	if (val > INT_MAX)
3890	return -EINVAL;
3891	lock_sock(sk);
3892	if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3893	ret = -EBUSY;
3894	} else {
3895	po->tp_reserve = val;
3896	ret = 0;
3897	}
3898	release_sock(sk);
3899	return ret;
3900	}
3901	case PACKET_LOSS:
3902	{
3903	unsigned int val;
3904
3905	if (optlen != sizeof(val))
3906	return -EINVAL;
3907	if (copy_from_sockptr(dst: &val, src: optval, size: sizeof(val)))
3908	return -EFAULT;
3909
3910	lock_sock(sk);
3911	if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3912	ret = -EBUSY;
3913	} else {
3914	packet_sock_flag_set(po, flag: PACKET_SOCK_TP_LOSS, val);
3915	ret = 0;
3916	}
3917	release_sock(sk);
3918	return ret;
3919	}
3920	case PACKET_AUXDATA:
3921	{
3922	int val;
3923
3924	if (optlen < sizeof(val))
3925	return -EINVAL;
3926	if (copy_from_sockptr(dst: &val, src: optval, size: sizeof(val)))
3927	return -EFAULT;
3928
3929	packet_sock_flag_set(po, flag: PACKET_SOCK_AUXDATA, val);
3930	return 0;
3931	}
3932	case PACKET_ORIGDEV:
3933	{
3934	int val;
3935
3936	if (optlen < sizeof(val))
3937	return -EINVAL;
3938	if (copy_from_sockptr(dst: &val, src: optval, size: sizeof(val)))
3939	return -EFAULT;
3940
3941	packet_sock_flag_set(po, flag: PACKET_SOCK_ORIGDEV, val);
3942	return 0;
3943	}
3944	case PACKET_VNET_HDR:
3945	case PACKET_VNET_HDR_SZ:
3946	{
3947	int val, hdr_len;
3948
3949	if (sock->type != SOCK_RAW)
3950	return -EINVAL;
3951	if (optlen < sizeof(val))
3952	return -EINVAL;
3953	if (copy_from_sockptr(dst: &val, src: optval, size: sizeof(val)))
3954	return -EFAULT;
3955
3956	if (optname == PACKET_VNET_HDR_SZ) {
3957	if (val && val != sizeof(struct virtio_net_hdr) &&
3958	val != sizeof(struct virtio_net_hdr_mrg_rxbuf))
3959	return -EINVAL;
3960	hdr_len = val;
3961	} else {
3962	hdr_len = val ? sizeof(struct virtio_net_hdr) : 0;
3963	}
3964	lock_sock(sk);
3965	if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
3966	ret = -EBUSY;
3967	} else {
3968	WRITE_ONCE(po->vnet_hdr_sz, hdr_len);
3969	ret = 0;
3970	}
3971	release_sock(sk);
3972	return ret;
3973	}
3974	case PACKET_TIMESTAMP:
3975	{
3976	int val;
3977
3978	if (optlen != sizeof(val))
3979	return -EINVAL;
3980	if (copy_from_sockptr(dst: &val, src: optval, size: sizeof(val)))
3981	return -EFAULT;
3982
3983	WRITE_ONCE(po->tp_tstamp, val);
3984	return 0;
3985	}
3986	case PACKET_FANOUT:
3987	{
3988	struct fanout_args args = { 0 };
3989
3990	if (optlen != sizeof(int) && optlen != sizeof(args))
3991	return -EINVAL;
3992	if (copy_from_sockptr(dst: &args, src: optval, size: optlen))
3993	return -EFAULT;
3994
3995	return fanout_add(sk, args: &args);
3996	}
3997	case PACKET_FANOUT_DATA:
3998	{
3999	/* Paired with the WRITE_ONCE() in fanout_add() */
4000	if (!READ_ONCE(po->fanout))
4001	return -EINVAL;
4002
4003	return fanout_set_data(po, data: optval, len: optlen);
4004	}
4005	case PACKET_IGNORE_OUTGOING:
4006	{
4007	int val;
4008
4009	if (optlen != sizeof(val))
4010	return -EINVAL;
4011	if (copy_from_sockptr(dst: &val, src: optval, size: sizeof(val)))
4012	return -EFAULT;
4013	if (val < 0 || val > 1)
4014	return -EINVAL;
4015
4016	WRITE_ONCE(po->prot_hook.ignore_outgoing, !!val);
4017	return 0;
4018	}
4019	case PACKET_TX_HAS_OFF:
4020	{
4021	unsigned int val;
4022
4023	if (optlen != sizeof(val))
4024	return -EINVAL;
4025	if (copy_from_sockptr(dst: &val, src: optval, size: sizeof(val)))
4026	return -EFAULT;
4027
4028	lock_sock(sk);
4029	if (!po->rx_ring.pg_vec && !po->tx_ring.pg_vec)
4030	packet_sock_flag_set(po, flag: PACKET_SOCK_TX_HAS_OFF, val);
4031
4032	release_sock(sk);
4033	return 0;
4034	}
4035	case PACKET_QDISC_BYPASS:
4036	{
4037	int val;
4038
4039	if (optlen != sizeof(val))
4040	return -EINVAL;
4041	if (copy_from_sockptr(dst: &val, src: optval, size: sizeof(val)))
4042	return -EFAULT;
4043
4044	packet_sock_flag_set(po, flag: PACKET_SOCK_QDISC_BYPASS, val);
4045	return 0;
4046	}
4047	default:
4048	return -ENOPROTOOPT;
4049	}
4050	}
4051
4052	static int packet_getsockopt(struct socket *sock, int level, int optname,
4053	char __user *optval, int __user *optlen)
4054	{
4055	int len;
4056	int val, lv = sizeof(val);
4057	struct sock *sk = sock->sk;
4058	struct packet_sock *po = pkt_sk(sk);
4059	void *data = &val;
4060	union tpacket_stats_u st;
4061	struct tpacket_rollover_stats rstats;
4062	int drops;
4063
4064	if (level != SOL_PACKET)
4065	return -ENOPROTOOPT;
4066
4067	if (get_user(len, optlen))
4068	return -EFAULT;
4069
4070	if (len < 0)
4071	return -EINVAL;
4072
4073	switch (optname) {
4074	case PACKET_STATISTICS:
4075	spin_lock_bh(lock: &sk->sk_receive_queue.lock);
4076	memcpy(&st, &po->stats, sizeof(st));
4077	memset(&po->stats, 0, sizeof(po->stats));
4078	spin_unlock_bh(lock: &sk->sk_receive_queue.lock);
4079	drops = atomic_xchg(v: &po->tp_drops, new: 0);
4080
4081	if (po->tp_version == TPACKET_V3) {
4082	lv = sizeof(struct tpacket_stats_v3);
4083	st.stats3.tp_drops = drops;
4084	st.stats3.tp_packets += drops;
4085	data = &st.stats3;
4086	} else {
4087	lv = sizeof(struct tpacket_stats);
4088	st.stats1.tp_drops = drops;
4089	st.stats1.tp_packets += drops;
4090	data = &st.stats1;
4091	}
4092
4093	break;
4094	case PACKET_AUXDATA:
4095	val = packet_sock_flag(po, flag: PACKET_SOCK_AUXDATA);
4096	break;
4097	case PACKET_ORIGDEV:
4098	val = packet_sock_flag(po, flag: PACKET_SOCK_ORIGDEV);
4099	break;
4100	case PACKET_VNET_HDR:
4101	val = !!READ_ONCE(po->vnet_hdr_sz);
4102	break;
4103	case PACKET_VNET_HDR_SZ:
4104	val = READ_ONCE(po->vnet_hdr_sz);
4105	break;
4106	case PACKET_COPY_THRESH:
4107	val = READ_ONCE(pkt_sk(sk)->copy_thresh);
4108	break;
4109	case PACKET_VERSION:
4110	val = po->tp_version;
4111	break;
4112	case PACKET_HDRLEN:
4113	if (len > sizeof(int))
4114	len = sizeof(int);
4115	if (len < sizeof(int))
4116	return -EINVAL;
4117	if (copy_from_user(to: &val, from: optval, n: len))
4118	return -EFAULT;
4119	switch (val) {
4120	case TPACKET_V1:
4121	val = sizeof(struct tpacket_hdr);
4122	break;
4123	case TPACKET_V2:
4124	val = sizeof(struct tpacket2_hdr);
4125	break;
4126	case TPACKET_V3:
4127	val = sizeof(struct tpacket3_hdr);
4128	break;
4129	default:
4130	return -EINVAL;
4131	}
4132	break;
4133	case PACKET_RESERVE:
4134	val = po->tp_reserve;
4135	break;
4136	case PACKET_LOSS:
4137	val = packet_sock_flag(po, flag: PACKET_SOCK_TP_LOSS);
4138	break;
4139	case PACKET_TIMESTAMP:
4140	val = READ_ONCE(po->tp_tstamp);
4141	break;
4142	case PACKET_FANOUT:
4143	val = (po->fanout ?
4144	((u32)po->fanout->id |
4145	((u32)po->fanout->type << 16) |
4146	((u32)po->fanout->flags << 24)) :
4147	0);
4148	break;
4149	case PACKET_IGNORE_OUTGOING:
4150	val = READ_ONCE(po->prot_hook.ignore_outgoing);
4151	break;
4152	case PACKET_ROLLOVER_STATS:
4153	if (!po->rollover)
4154	return -EINVAL;
4155	rstats.tp_all = atomic_long_read(v: &po->rollover->num);
4156	rstats.tp_huge = atomic_long_read(v: &po->rollover->num_huge);
4157	rstats.tp_failed = atomic_long_read(v: &po->rollover->num_failed);
4158	data = &rstats;
4159	lv = sizeof(rstats);
4160	break;
4161	case PACKET_TX_HAS_OFF:
4162	val = packet_sock_flag(po, flag: PACKET_SOCK_TX_HAS_OFF);
4163	break;
4164	case PACKET_QDISC_BYPASS:
4165	val = packet_sock_flag(po, flag: PACKET_SOCK_QDISC_BYPASS);
4166	break;
4167	default:
4168	return -ENOPROTOOPT;
4169	}
4170
4171	if (len > lv)
4172	len = lv;
4173	if (put_user(len, optlen))
4174	return -EFAULT;
4175	if (copy_to_user(to: optval, from: data, n: len))
4176	return -EFAULT;
4177	return 0;
4178	}
4179
4180	static int packet_notifier(struct notifier_block *this,
4181	unsigned long msg, void *ptr)
4182	{
4183	struct net_device *dev = netdev_notifier_info_to_dev(info: ptr);
4184	struct net *net = dev_net(dev);
4185	struct packet_mclist *ml, *tmp;
4186	LIST_HEAD(mclist);
4187	struct sock *sk;
4188
4189	rcu_read_lock();
4190	sk_for_each_rcu(sk, &net->packet.sklist) {
4191	struct packet_sock *po = pkt_sk(sk);
4192
4193	switch (msg) {
4194	case NETDEV_UNREGISTER:
4195	if (po->mclist)
4196	packet_dev_mclist_delete(dev, mlp: &po->mclist,
4197	list: &mclist);
4198	fallthrough;
4199
4200	case NETDEV_DOWN:
4201	if (dev->ifindex == po->ifindex) {
4202	spin_lock(lock: &po->bind_lock);
4203	if (packet_sock_flag(po, flag: PACKET_SOCK_RUNNING)) {
4204	__unregister_prot_hook(sk, sync: false);
4205	sk->sk_err = ENETDOWN;
4206	if (!sock_flag(sk, flag: SOCK_DEAD))
4207	sk_error_report(sk);
4208	}
4209	if (msg == NETDEV_UNREGISTER) {
4210	packet_cached_dev_reset(po);
4211	WRITE_ONCE(po->ifindex, -1);
4212	netdev_put(dev: po->prot_hook.dev,
4213	tracker: &po->prot_hook.dev_tracker);
4214	po->prot_hook.dev = NULL;
4215	}
4216	spin_unlock(lock: &po->bind_lock);
4217	}
4218	break;
4219	case NETDEV_UP:
4220	if (dev->ifindex == po->ifindex) {
4221	spin_lock(lock: &po->bind_lock);
4222	if (po->num)
4223	register_prot_hook(sk);
4224	spin_unlock(lock: &po->bind_lock);
4225	}
4226	break;
4227	}
4228	}
4229	rcu_read_unlock();
4230
4231	/* packet_dev_mc might grab instance locks so can't run under rcu */
4232	list_for_each_entry_safe(ml, tmp, &mclist, remove_list) {
4233	packet_dev_mc(dev, i: ml, what: -1);
4234	kfree(objp: ml);
4235	}
4236
4237	return NOTIFY_DONE;
4238	}
4239
4240
4241	static int packet_ioctl(struct socket *sock, unsigned int cmd,
4242	unsigned long arg)
4243	{
4244	struct sock *sk = sock->sk;
4245
4246	switch (cmd) {
4247	case SIOCOUTQ:
4248	{
4249	int amount = sk_wmem_alloc_get(sk);
4250
4251	return put_user(amount, (int __user *)arg);
4252	}
4253	case SIOCINQ:
4254	{
4255	struct sk_buff *skb;
4256	int amount = 0;
4257
4258	spin_lock_bh(lock: &sk->sk_receive_queue.lock);
4259	skb = skb_peek(list_: &sk->sk_receive_queue);
4260	if (skb)
4261	amount = skb->len;
4262	spin_unlock_bh(lock: &sk->sk_receive_queue.lock);
4263	return put_user(amount, (int __user *)arg);
4264	}
4265	#ifdef CONFIG_INET
4266	case SIOCADDRT:
4267	case SIOCDELRT:
4268	case SIOCDARP:
4269	case SIOCGARP:
4270	case SIOCSARP:
4271	case SIOCGIFADDR:
4272	case SIOCSIFADDR:
4273	case SIOCGIFBRDADDR:
4274	case SIOCSIFBRDADDR:
4275	case SIOCGIFNETMASK:
4276	case SIOCSIFNETMASK:
4277	case SIOCGIFDSTADDR:
4278	case SIOCSIFDSTADDR:
4279	case SIOCSIFFLAGS:
4280	return inet_dgram_ops.ioctl(sock, cmd, arg);
4281	#endif
4282
4283	default:
4284	return -ENOIOCTLCMD;
4285	}
4286	return 0;
4287	}
4288
4289	static __poll_t packet_poll(struct file *file, struct socket *sock,
4290	poll_table *wait)
4291	{
4292	struct sock *sk = sock->sk;
4293	struct packet_sock *po = pkt_sk(sk);
4294	__poll_t mask = datagram_poll(file, sock, wait);
4295
4296	spin_lock_bh(lock: &sk->sk_receive_queue.lock);
4297	if (po->rx_ring.pg_vec) {
4298	if (!packet_previous_rx_frame(po, rb: &po->rx_ring,
4299	TP_STATUS_KERNEL))
4300	mask |= EPOLLIN | EPOLLRDNORM;
4301	}
4302	packet_rcv_try_clear_pressure(po);
4303	spin_unlock_bh(lock: &sk->sk_receive_queue.lock);
4304	spin_lock_bh(lock: &sk->sk_write_queue.lock);
4305	if (po->tx_ring.pg_vec) {
4306	if (packet_current_frame(po, rb: &po->tx_ring, TP_STATUS_AVAILABLE))
4307	mask |= EPOLLOUT | EPOLLWRNORM;
4308	}
4309	spin_unlock_bh(lock: &sk->sk_write_queue.lock);
4310	return mask;
4311	}
4312
4313
4314	/* Dirty? Well, I still did not learn better way to account
4315	* for user mmaps.
4316	*/
4317
4318	static void packet_mm_open(struct vm_area_struct *vma)
4319	{
4320	struct file *file = vma->vm_file;
4321	struct socket *sock = file->private_data;
4322	struct sock *sk = sock->sk;
4323
4324	if (sk)
4325	atomic_long_inc(v: &pkt_sk(sk)->mapped);
4326	}
4327
4328	static void packet_mm_close(struct vm_area_struct *vma)
4329	{
4330	struct file *file = vma->vm_file;
4331	struct socket *sock = file->private_data;
4332	struct sock *sk = sock->sk;
4333
4334	if (sk)
4335	atomic_long_dec(v: &pkt_sk(sk)->mapped);
4336	}
4337
4338	static const struct vm_operations_struct packet_mmap_ops = {
4339	.open = packet_mm_open,
4340	.close = packet_mm_close,
4341	};
4342
4343	static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
4344	unsigned int len)
4345	{
4346	int i;
4347
4348	for (i = 0; i < len; i++) {
4349	if (likely(pg_vec[i].buffer)) {
4350	if (is_vmalloc_addr(x: pg_vec[i].buffer))
4351	vfree(addr: pg_vec[i].buffer);
4352	else
4353	free_pages(addr: (unsigned long)pg_vec[i].buffer,
4354	order);
4355	pg_vec[i].buffer = NULL;
4356	}
4357	}
4358	kfree(objp: pg_vec);
4359	}
4360
4361	static char *alloc_one_pg_vec_page(unsigned long order)
4362	{
4363	char *buffer;
4364	gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
4365	__GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;
4366
4367	buffer = (char *) __get_free_pages(gfp_flags, order);
4368	if (buffer)
4369	return buffer;
4370
4371	/* __get_free_pages failed, fall back to vmalloc */
4372	buffer = vzalloc(array_size((1 << order), PAGE_SIZE));
4373	if (buffer)
4374	return buffer;
4375
4376	/* vmalloc failed, lets dig into swap here */
4377	gfp_flags &= ~__GFP_NORETRY;
4378	buffer = (char *) __get_free_pages(gfp_flags, order);
4379	if (buffer)
4380	return buffer;
4381
4382	/* complete and utter failure */
4383	return NULL;
4384	}
4385
4386	static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
4387	{
4388	unsigned int block_nr = req->tp_block_nr;
4389	struct pgv *pg_vec;
4390	int i;
4391
4392	pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL | __GFP_NOWARN);
4393	if (unlikely(!pg_vec))
4394	goto out;
4395
4396	for (i = 0; i < block_nr; i++) {
4397	pg_vec[i].buffer = alloc_one_pg_vec_page(order);
4398	if (unlikely(!pg_vec[i].buffer))
4399	goto out_free_pgvec;
4400	}
4401
4402	out:
4403	return pg_vec;
4404
4405	out_free_pgvec:
4406	free_pg_vec(pg_vec, order, len: block_nr);
4407	pg_vec = NULL;
4408	goto out;
4409	}
4410
4411	static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
4412	int closing, int tx_ring)
4413	{
4414	struct pgv *pg_vec = NULL;
4415	struct packet_sock *po = pkt_sk(sk);
4416	unsigned long *rx_owner_map = NULL;
4417	int was_running, order = 0;
4418	struct packet_ring_buffer *rb;
4419	struct sk_buff_head *rb_queue;
4420	__be16 num;
4421	int err;
4422	/* Added to avoid minimal code churn */
4423	struct tpacket_req *req = &req_u->req;
4424
4425	rb = tx_ring ? &po->tx_ring : &po->rx_ring;
4426	rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
4427
4428	err = -EBUSY;
4429	if (!closing) {
4430	if (atomic_long_read(v: &po->mapped))
4431	goto out;
4432	if (packet_read_pending(rb))
4433	goto out;
4434	}
4435
4436	if (req->tp_block_nr) {
4437	unsigned int min_frame_size;
4438
4439	/* Sanity tests and some calculations */
4440	err = -EBUSY;
4441	if (unlikely(rb->pg_vec))
4442	goto out;
4443
4444	switch (po->tp_version) {
4445	case TPACKET_V1:
4446	po->tp_hdrlen = TPACKET_HDRLEN;
4447	break;
4448	case TPACKET_V2:
4449	po->tp_hdrlen = TPACKET2_HDRLEN;
4450	break;
4451	case TPACKET_V3:
4452	po->tp_hdrlen = TPACKET3_HDRLEN;
4453	break;
4454	}
4455
4456	err = -EINVAL;
4457	if (unlikely((int)req->tp_block_size <= 0))
4458	goto out;
4459	if (unlikely(!PAGE_ALIGNED(req->tp_block_size)))
4460	goto out;
4461	min_frame_size = po->tp_hdrlen + po->tp_reserve;
4462	if (po->tp_version >= TPACKET_V3 &&
4463	req->tp_block_size <
4464	BLK_PLUS_PRIV((u64)req_u->req3.tp_sizeof_priv) + min_frame_size)
4465	goto out;
4466	if (unlikely(req->tp_frame_size < min_frame_size))
4467	goto out;
4468	if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
4469	goto out;
4470
4471	rb->frames_per_block = req->tp_block_size / req->tp_frame_size;
4472	if (unlikely(rb->frames_per_block == 0))
4473	goto out;
4474	if (unlikely(rb->frames_per_block > UINT_MAX / req->tp_block_nr))
4475	goto out;
4476	if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
4477	req->tp_frame_nr))
4478	goto out;
4479
4480	err = -ENOMEM;
4481	order = get_order(size: req->tp_block_size);
4482	pg_vec = alloc_pg_vec(req, order);
4483	if (unlikely(!pg_vec))
4484	goto out;
4485	switch (po->tp_version) {
4486	case TPACKET_V3:
4487	/* Block transmit is not supported yet */
4488	if (!tx_ring) {
4489	init_prb_bdqc(po, rb, pg_vec, req_u);
4490	} else {
4491	struct tpacket_req3 *req3 = &req_u->req3;
4492
4493	if (req3->tp_retire_blk_tov ||
4494	req3->tp_sizeof_priv ||
4495	req3->tp_feature_req_word) {
4496	err = -EINVAL;
4497	goto out_free_pg_vec;
4498	}
4499	}
4500	break;
4501	default:
4502	if (!tx_ring) {
4503	rx_owner_map = bitmap_alloc(nbits: req->tp_frame_nr,
4504	GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
4505	if (!rx_owner_map)
4506	goto out_free_pg_vec;
4507	}
4508	break;
4509	}
4510	}
4511	/* Done */
4512	else {
4513	err = -EINVAL;
4514	if (unlikely(req->tp_frame_nr))
4515	goto out;
4516	}
4517
4518
4519	/* Detach socket from network */
4520	spin_lock(lock: &po->bind_lock);
4521	was_running = packet_sock_flag(po, flag: PACKET_SOCK_RUNNING);
4522	num = po->num;
4523	WRITE_ONCE(po->num, 0);
4524	if (was_running)
4525	__unregister_prot_hook(sk, sync: false);
4526
4527	spin_unlock(lock: &po->bind_lock);
4528
4529	synchronize_net();
4530
4531	err = -EBUSY;
4532	mutex_lock(&po->pg_vec_lock);
4533	if (closing || atomic_long_read(v: &po->mapped) == 0) {
4534	err = 0;
4535	spin_lock_bh(lock: &rb_queue->lock);
4536	swap(rb->pg_vec, pg_vec);
4537	if (po->tp_version <= TPACKET_V2)
4538	swap(rb->rx_owner_map, rx_owner_map);
4539	rb->frame_max = (req->tp_frame_nr - 1);
4540	rb->head = 0;
4541	rb->frame_size = req->tp_frame_size;
4542	spin_unlock_bh(lock: &rb_queue->lock);
4543
4544	swap(rb->pg_vec_order, order);
4545	swap(rb->pg_vec_len, req->tp_block_nr);
4546
4547	rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
4548	po->prot_hook.func = (po->rx_ring.pg_vec) ?
4549	tpacket_rcv : packet_rcv;
4550	skb_queue_purge(list: rb_queue);
4551	if (atomic_long_read(v: &po->mapped))
4552	pr_err("packet_mmap: vma is busy: %ld\n",
4553	atomic_long_read(&po->mapped));
4554	}
4555	mutex_unlock(lock: &po->pg_vec_lock);
4556
4557	spin_lock(lock: &po->bind_lock);
4558	WRITE_ONCE(po->num, num);
4559	if (was_running)
4560	register_prot_hook(sk);
4561
4562	spin_unlock(lock: &po->bind_lock);
4563	if (pg_vec && (po->tp_version > TPACKET_V2)) {
4564	/* Because we don't support block-based V3 on tx-ring */
4565	if (!tx_ring)
4566	prb_shutdown_retire_blk_timer(po, rb_queue);
4567	}
4568
4569	out_free_pg_vec:
4570	if (pg_vec) {
4571	bitmap_free(bitmap: rx_owner_map);
4572	free_pg_vec(pg_vec, order, len: req->tp_block_nr);
4573	}
4574	out:
4575	return err;
4576	}
4577
4578	static int packet_mmap(struct file *file, struct socket *sock,
4579	struct vm_area_struct *vma)
4580	{
4581	struct sock *sk = sock->sk;
4582	struct packet_sock *po = pkt_sk(sk);
4583	unsigned long size, expected_size;
4584	struct packet_ring_buffer *rb;
4585	unsigned long start;
4586	int err = -EINVAL;
4587	int i;
4588
4589	if (vma->vm_pgoff)
4590	return -EINVAL;
4591
4592	mutex_lock(&po->pg_vec_lock);
4593
4594	expected_size = 0;
4595	for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4596	if (rb->pg_vec) {
4597	expected_size += rb->pg_vec_len
4598	* rb->pg_vec_pages
4599	* PAGE_SIZE;
4600	}
4601	}
4602
4603	if (expected_size == 0)
4604	goto out;
4605
4606	size = vma->vm_end - vma->vm_start;
4607	if (size != expected_size)
4608	goto out;
4609
4610	start = vma->vm_start;
4611	for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
4612	if (rb->pg_vec == NULL)
4613	continue;
4614
4615	for (i = 0; i < rb->pg_vec_len; i++) {
4616	struct page *page;
4617	void *kaddr = rb->pg_vec[i].buffer;
4618	int pg_num;
4619
4620	for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
4621	page = pgv_to_page(addr: kaddr);
4622	err = vm_insert_page(vma, addr: start, page);
4623	if (unlikely(err))
4624	goto out;
4625	start += PAGE_SIZE;
4626	kaddr += PAGE_SIZE;
4627	}
4628	}
4629	}
4630
4631	atomic_long_inc(v: &po->mapped);
4632	vma->vm_ops = &packet_mmap_ops;
4633	err = 0;
4634
4635	out:
4636	mutex_unlock(lock: &po->pg_vec_lock);
4637	return err;
4638	}
4639
4640	static const struct proto_ops packet_ops_spkt = {
4641	.family = PF_PACKET,
4642	.owner = THIS_MODULE,
4643	.release = packet_release,
4644	.bind = packet_bind_spkt,
4645	.connect = sock_no_connect,
4646	.socketpair = sock_no_socketpair,
4647	.accept = sock_no_accept,
4648	.getname = packet_getname_spkt,
4649	.poll = datagram_poll,
4650	.ioctl = packet_ioctl,
4651	.gettstamp = sock_gettstamp,
4652	.listen = sock_no_listen,
4653	.shutdown = sock_no_shutdown,
4654	.sendmsg = packet_sendmsg_spkt,
4655	.recvmsg = packet_recvmsg,
4656	.mmap = sock_no_mmap,
4657	};
4658
4659	static const struct proto_ops packet_ops = {
4660	.family = PF_PACKET,
4661	.owner = THIS_MODULE,
4662	.release = packet_release,
4663	.bind = packet_bind,
4664	.connect = sock_no_connect,
4665	.socketpair = sock_no_socketpair,
4666	.accept = sock_no_accept,
4667	.getname = packet_getname,
4668	.poll = packet_poll,
4669	.ioctl = packet_ioctl,
4670	.gettstamp = sock_gettstamp,
4671	.listen = sock_no_listen,
4672	.shutdown = sock_no_shutdown,
4673	.setsockopt = packet_setsockopt,
4674	.getsockopt = packet_getsockopt,
4675	.sendmsg = packet_sendmsg,
4676	.recvmsg = packet_recvmsg,
4677	.mmap = packet_mmap,
4678	};
4679
4680	static const struct net_proto_family packet_family_ops = {
4681	.family = PF_PACKET,
4682	.create = packet_create,
4683	.owner = THIS_MODULE,
4684	};
4685
4686	static struct notifier_block packet_netdev_notifier = {
4687	.notifier_call = packet_notifier,
4688	};
4689
4690	#ifdef CONFIG_PROC_FS
4691
4692	static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
4693	__acquires(RCU)
4694	{
4695	struct net *net = seq_file_net(seq);
4696
4697	rcu_read_lock();
4698	return seq_hlist_start_head_rcu(head: &net->packet.sklist, pos: *pos);
4699	}
4700
4701	static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4702	{
4703	struct net *net = seq_file_net(seq);
4704	return seq_hlist_next_rcu(v, head: &net->packet.sklist, ppos: pos);
4705	}
4706
4707	static void packet_seq_stop(struct seq_file *seq, void *v)
4708	__releases(RCU)
4709	{
4710	rcu_read_unlock();
4711	}
4712
4713	static int packet_seq_show(struct seq_file *seq, void *v)
4714	{
4715	if (v == SEQ_START_TOKEN)
4716	seq_printf(m: seq,
4717	fmt: "%*sRefCnt Type Proto Iface R Rmem User Inode\n",
4718	IS_ENABLED(CONFIG_64BIT) ? -17 : -9, "sk");
4719	else {
4720	struct sock *s = sk_entry(node: v);
4721	const struct packet_sock *po = pkt_sk(s);
4722
4723	seq_printf(m: seq,
4724	fmt: "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
4725	s,
4726	refcount_read(r: &s->sk_refcnt),
4727	s->sk_type,
4728	ntohs(READ_ONCE(po->num)),
4729	READ_ONCE(po->ifindex),
4730	packet_sock_flag(po, flag: PACKET_SOCK_RUNNING),
4731	atomic_read(v: &s->sk_rmem_alloc),
4732	from_kuid_munged(to: seq_user_ns(seq), uid: sk_uid(sk: s)),
4733	sock_i_ino(sk: s));
4734	}
4735
4736	return 0;
4737	}
4738
4739	static const struct seq_operations packet_seq_ops = {
4740	.start = packet_seq_start,
4741	.next = packet_seq_next,
4742	.stop = packet_seq_stop,
4743	.show = packet_seq_show,
4744	};
4745	#endif
4746
4747	static int __net_init packet_net_init(struct net *net)
4748	{
4749	mutex_init(&net->packet.sklist_lock);
4750	INIT_HLIST_HEAD(&net->packet.sklist);
4751
4752	#ifdef CONFIG_PROC_FS
4753	if (!proc_create_net("packet", 0, net->proc_net, &packet_seq_ops,
4754	sizeof(struct seq_net_private)))
4755	return -ENOMEM;
4756	#endif /* CONFIG_PROC_FS */
4757
4758	return 0;
4759	}
4760
4761	static void __net_exit packet_net_exit(struct net *net)
4762	{
4763	remove_proc_entry("packet", net->proc_net);
4764	WARN_ON_ONCE(!hlist_empty(&net->packet.sklist));
4765	}
4766
4767	static struct pernet_operations packet_net_ops = {
4768	.init = packet_net_init,
4769	.exit = packet_net_exit,
4770	};
4771
4772
4773	static void __exit packet_exit(void)
4774	{
4775	sock_unregister(PF_PACKET);
4776	proto_unregister(prot: &packet_proto);
4777	unregister_netdevice_notifier(nb: &packet_netdev_notifier);
4778	unregister_pernet_subsys(&packet_net_ops);
4779	}
4780
4781	static int __init packet_init(void)
4782	{
4783	int rc;
4784
4785	rc = register_pernet_subsys(&packet_net_ops);
4786	if (rc)
4787	goto out;
4788	rc = register_netdevice_notifier(nb: &packet_netdev_notifier);
4789	if (rc)
4790	goto out_pernet;
4791	rc = proto_register(prot: &packet_proto, alloc_slab: 0);
4792	if (rc)
4793	goto out_notifier;
4794	rc = sock_register(fam: &packet_family_ops);
4795	if (rc)
4796	goto out_proto;
4797
4798	return 0;
4799
4800	out_proto:
4801	proto_unregister(prot: &packet_proto);
4802	out_notifier:
4803	unregister_netdevice_notifier(nb: &packet_netdev_notifier);
4804	out_pernet:
4805	unregister_pernet_subsys(&packet_net_ops);
4806	out:
4807	return rc;
4808	}
4809
4810	module_init(packet_init);
4811	module_exit(packet_exit);
4812	MODULE_DESCRIPTION("Packet socket support (AF_PACKET)");
4813	MODULE_LICENSE("GPL");
4814	MODULE_ALIAS_NETPROTO(PF_PACKET);
4815