1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	*
4	* Copyright (C) 2009-2016 John Crispin <blogic@openwrt.org>
5	* Copyright (C) 2009-2016 Felix Fietkau <nbd@openwrt.org>
6	* Copyright (C) 2013-2016 Michael Lee <igvtee@gmail.com>
7	*/
8
9	#include <linux/of.h>
10	#include <linux/of_mdio.h>
11	#include <linux/of_net.h>
12	#include <linux/of_address.h>
13	#include <linux/mfd/syscon.h>
14	#include <linux/platform_device.h>
15	#include <linux/regmap.h>
16	#include <linux/clk.h>
17	#include <linux/pm_runtime.h>
18	#include <linux/if_vlan.h>
19	#include <linux/reset.h>
20	#include <linux/tcp.h>
21	#include <linux/interrupt.h>
22	#include <linux/pinctrl/devinfo.h>
23	#include <linux/phylink.h>
24	#include <linux/pcs/pcs-mtk-lynxi.h>
25	#include <linux/jhash.h>
26	#include <linux/bitfield.h>
27	#include <net/dsa.h>
28	#include <net/dst_metadata.h>
29	#include <net/page_pool/helpers.h>
30	#include <linux/genalloc.h>
31
32	#include "mtk_eth_soc.h"
33	#include "mtk_wed.h"
34
35	static int mtk_msg_level = -1;
36	module_param_named(msg_level, mtk_msg_level, int, 0);
37	MODULE_PARM_DESC(msg_level, "Message level (-1=defaults,0=none,...,16=all)");
38
39	#define MTK_ETHTOOL_STAT(x) { #x, \
40	offsetof(struct mtk_hw_stats, x) / sizeof(u64) }
41
42	#define MTK_ETHTOOL_XDP_STAT(x) { #x, \
43	offsetof(struct mtk_hw_stats, xdp_stats.x) / \
44	sizeof(u64) }
45
46	static const struct mtk_reg_map mtk_reg_map = {
47	.tx_irq_mask = 0x1a1c,
48	.tx_irq_status = 0x1a18,
49	.pdma = {
50	.rx_ptr = 0x0900,
51	.rx_cnt_cfg = 0x0904,
52	.pcrx_ptr = 0x0908,
53	.glo_cfg = 0x0a04,
54	.rst_idx = 0x0a08,
55	.delay_irq = 0x0a0c,
56	.irq_status = 0x0a20,
57	.irq_mask = 0x0a28,
58	.adma_rx_dbg0 = 0x0a38,
59	.int_grp = 0x0a50,
60	},
61	.qdma = {
62	.qtx_cfg = 0x1800,
63	.qtx_sch = 0x1804,
64	.rx_ptr = 0x1900,
65	.rx_cnt_cfg = 0x1904,
66	.qcrx_ptr = 0x1908,
67	.glo_cfg = 0x1a04,
68	.rst_idx = 0x1a08,
69	.delay_irq = 0x1a0c,
70	.fc_th = 0x1a10,
71	.tx_sch_rate = 0x1a14,
72	.int_grp = 0x1a20,
73	.hred = 0x1a44,
74	.ctx_ptr = 0x1b00,
75	.dtx_ptr = 0x1b04,
76	.crx_ptr = 0x1b10,
77	.drx_ptr = 0x1b14,
78	.fq_head = 0x1b20,
79	.fq_tail = 0x1b24,
80	.fq_count = 0x1b28,
81	.fq_blen = 0x1b2c,
82	},
83	.gdm1_cnt = 0x2400,
84	.gdma_to_ppe = {
85	[0] = 0x4444,
86	},
87	.ppe_base = 0x0c00,
88	.wdma_base = {
89	[0] = 0x2800,
90	[1] = 0x2c00,
91	},
92	.pse_iq_sta = 0x0110,
93	.pse_oq_sta = 0x0118,
94	};
95
96	static const struct mtk_reg_map mt7628_reg_map = {
97	.tx_irq_mask = 0x0a28,
98	.tx_irq_status = 0x0a20,
99	.pdma = {
100	.rx_ptr = 0x0900,
101	.rx_cnt_cfg = 0x0904,
102	.pcrx_ptr = 0x0908,
103	.glo_cfg = 0x0a04,
104	.rst_idx = 0x0a08,
105	.delay_irq = 0x0a0c,
106	.irq_status = 0x0a20,
107	.irq_mask = 0x0a28,
108	.int_grp = 0x0a50,
109	},
110	};
111
112	static const struct mtk_reg_map mt7986_reg_map = {
113	.tx_irq_mask = 0x461c,
114	.tx_irq_status = 0x4618,
115	.pdma = {
116	.rx_ptr = 0x4100,
117	.rx_cnt_cfg = 0x4104,
118	.pcrx_ptr = 0x4108,
119	.glo_cfg = 0x4204,
120	.rst_idx = 0x4208,
121	.delay_irq = 0x420c,
122	.irq_status = 0x4220,
123	.irq_mask = 0x4228,
124	.adma_rx_dbg0 = 0x4238,
125	.int_grp = 0x4250,
126	},
127	.qdma = {
128	.qtx_cfg = 0x4400,
129	.qtx_sch = 0x4404,
130	.rx_ptr = 0x4500,
131	.rx_cnt_cfg = 0x4504,
132	.qcrx_ptr = 0x4508,
133	.glo_cfg = 0x4604,
134	.rst_idx = 0x4608,
135	.delay_irq = 0x460c,
136	.fc_th = 0x4610,
137	.int_grp = 0x4620,
138	.hred = 0x4644,
139	.ctx_ptr = 0x4700,
140	.dtx_ptr = 0x4704,
141	.crx_ptr = 0x4710,
142	.drx_ptr = 0x4714,
143	.fq_head = 0x4720,
144	.fq_tail = 0x4724,
145	.fq_count = 0x4728,
146	.fq_blen = 0x472c,
147	.tx_sch_rate = 0x4798,
148	},
149	.gdm1_cnt = 0x1c00,
150	.gdma_to_ppe = {
151	[0] = 0x3333,
152	[1] = 0x4444,
153	},
154	.ppe_base = 0x2000,
155	.wdma_base = {
156	[0] = 0x4800,
157	[1] = 0x4c00,
158	},
159	.pse_iq_sta = 0x0180,
160	.pse_oq_sta = 0x01a0,
161	};
162
163	static const struct mtk_reg_map mt7988_reg_map = {
164	.tx_irq_mask = 0x461c,
165	.tx_irq_status = 0x4618,
166	.pdma = {
167	.rx_ptr = 0x6900,
168	.rx_cnt_cfg = 0x6904,
169	.pcrx_ptr = 0x6908,
170	.glo_cfg = 0x6a04,
171	.rst_idx = 0x6a08,
172	.delay_irq = 0x6a0c,
173	.irq_status = 0x6a20,
174	.irq_mask = 0x6a28,
175	.adma_rx_dbg0 = 0x6a38,
176	.int_grp = 0x6a50,
177	},
178	.qdma = {
179	.qtx_cfg = 0x4400,
180	.qtx_sch = 0x4404,
181	.rx_ptr = 0x4500,
182	.rx_cnt_cfg = 0x4504,
183	.qcrx_ptr = 0x4508,
184	.glo_cfg = 0x4604,
185	.rst_idx = 0x4608,
186	.delay_irq = 0x460c,
187	.fc_th = 0x4610,
188	.int_grp = 0x4620,
189	.hred = 0x4644,
190	.ctx_ptr = 0x4700,
191	.dtx_ptr = 0x4704,
192	.crx_ptr = 0x4710,
193	.drx_ptr = 0x4714,
194	.fq_head = 0x4720,
195	.fq_tail = 0x4724,
196	.fq_count = 0x4728,
197	.fq_blen = 0x472c,
198	.tx_sch_rate = 0x4798,
199	},
200	.gdm1_cnt = 0x1c00,
201	.gdma_to_ppe = {
202	[0] = 0x3333,
203	[1] = 0x4444,
204	[2] = 0xcccc,
205	},
206	.ppe_base = 0x2000,
207	.wdma_base = {
208	[0] = 0x4800,
209	[1] = 0x4c00,
210	[2] = 0x5000,
211	},
212	.pse_iq_sta = 0x0180,
213	.pse_oq_sta = 0x01a0,
214	};
215
216	/* strings used by ethtool */
217	static const struct mtk_ethtool_stats {
218	char str[ETH_GSTRING_LEN];
219	u32 offset;
220	} mtk_ethtool_stats[] = {
221	MTK_ETHTOOL_STAT(tx_bytes),
222	MTK_ETHTOOL_STAT(tx_packets),
223	MTK_ETHTOOL_STAT(tx_skip),
224	MTK_ETHTOOL_STAT(tx_collisions),
225	MTK_ETHTOOL_STAT(rx_bytes),
226	MTK_ETHTOOL_STAT(rx_packets),
227	MTK_ETHTOOL_STAT(rx_overflow),
228	MTK_ETHTOOL_STAT(rx_fcs_errors),
229	MTK_ETHTOOL_STAT(rx_short_errors),
230	MTK_ETHTOOL_STAT(rx_long_errors),
231	MTK_ETHTOOL_STAT(rx_checksum_errors),
232	MTK_ETHTOOL_STAT(rx_flow_control_packets),
233	MTK_ETHTOOL_XDP_STAT(rx_xdp_redirect),
234	MTK_ETHTOOL_XDP_STAT(rx_xdp_pass),
235	MTK_ETHTOOL_XDP_STAT(rx_xdp_drop),
236	MTK_ETHTOOL_XDP_STAT(rx_xdp_tx),
237	MTK_ETHTOOL_XDP_STAT(rx_xdp_tx_errors),
238	MTK_ETHTOOL_XDP_STAT(tx_xdp_xmit),
239	MTK_ETHTOOL_XDP_STAT(tx_xdp_xmit_errors),
240	};
241
242	static const char * const mtk_clks_source_name[] = {
243	"ethif",
244	"sgmiitop",
245	"esw",
246	"gp0",
247	"gp1",
248	"gp2",
249	"gp3",
250	"xgp1",
251	"xgp2",
252	"xgp3",
253	"crypto",
254	"fe",
255	"trgpll",
256	"sgmii_tx250m",
257	"sgmii_rx250m",
258	"sgmii_cdr_ref",
259	"sgmii_cdr_fb",
260	"sgmii2_tx250m",
261	"sgmii2_rx250m",
262	"sgmii2_cdr_ref",
263	"sgmii2_cdr_fb",
264	"sgmii_ck",
265	"eth2pll",
266	"wocpu0",
267	"wocpu1",
268	"netsys0",
269	"netsys1",
270	"ethwarp_wocpu2",
271	"ethwarp_wocpu1",
272	"ethwarp_wocpu0",
273	"top_sgm0_sel",
274	"top_sgm1_sel",
275	"top_eth_gmii_sel",
276	"top_eth_refck_50m_sel",
277	"top_eth_sys_200m_sel",
278	"top_eth_sys_sel",
279	"top_eth_xgmii_sel",
280	"top_eth_mii_sel",
281	"top_netsys_sel",
282	"top_netsys_500m_sel",
283	"top_netsys_pao_2x_sel",
284	"top_netsys_sync_250m_sel",
285	"top_netsys_ppefb_250m_sel",
286	"top_netsys_warp_sel",
287	};
288
289	void mtk_w32(struct mtk_eth *eth, u32 val, unsigned reg)
290	{
291	__raw_writel(val, addr: eth->base + reg);
292	}
293
294	u32 mtk_r32(struct mtk_eth *eth, unsigned reg)
295	{
296	return __raw_readl(addr: eth->base + reg);
297	}
298
299	u32 mtk_m32(struct mtk_eth *eth, u32 mask, u32 set, unsigned int reg)
300	{
301	u32 val;
302
303	val = mtk_r32(eth, reg);
304	val &= ~mask;
305	val |= set;
306	mtk_w32(eth, val, reg);
307	return reg;
308	}
309
310	static int mtk_mdio_busy_wait(struct mtk_eth *eth)
311	{
312	unsigned long t_start = jiffies;
313
314	while (1) {
315	if (!(mtk_r32(eth, MTK_PHY_IAC) & PHY_IAC_ACCESS))
316	return 0;
317	if (time_after(jiffies, t_start + PHY_IAC_TIMEOUT))
318	break;
319	cond_resched();
320	}
321
322	dev_err(eth->dev, "mdio: MDIO timeout\n");
323	return -ETIMEDOUT;
324	}
325
326	static int _mtk_mdio_write_c22(struct mtk_eth *eth, u32 phy_addr, u32 phy_reg,
327	u32 write_data)
328	{
329	int ret;
330
331	ret = mtk_mdio_busy_wait(eth);
332	if (ret < 0)
333	return ret;
334
335	mtk_w32(eth, PHY_IAC_ACCESS |
336	PHY_IAC_START_C22 |
337	PHY_IAC_CMD_WRITE |
338	PHY_IAC_REG(phy_reg) |
339	PHY_IAC_ADDR(phy_addr) |
340	PHY_IAC_DATA(write_data),
341	MTK_PHY_IAC);
342
343	ret = mtk_mdio_busy_wait(eth);
344	if (ret < 0)
345	return ret;
346
347	return 0;
348	}
349
350	static int _mtk_mdio_write_c45(struct mtk_eth *eth, u32 phy_addr,
351	u32 devad, u32 phy_reg, u32 write_data)
352	{
353	int ret;
354
355	ret = mtk_mdio_busy_wait(eth);
356	if (ret < 0)
357	return ret;
358
359	mtk_w32(eth, PHY_IAC_ACCESS |
360	PHY_IAC_START_C45 |
361	PHY_IAC_CMD_C45_ADDR |
362	PHY_IAC_REG(devad) |
363	PHY_IAC_ADDR(phy_addr) |
364	PHY_IAC_DATA(phy_reg),
365	MTK_PHY_IAC);
366
367	ret = mtk_mdio_busy_wait(eth);
368	if (ret < 0)
369	return ret;
370
371	mtk_w32(eth, PHY_IAC_ACCESS |
372	PHY_IAC_START_C45 |
373	PHY_IAC_CMD_WRITE |
374	PHY_IAC_REG(devad) |
375	PHY_IAC_ADDR(phy_addr) |
376	PHY_IAC_DATA(write_data),
377	MTK_PHY_IAC);
378
379	ret = mtk_mdio_busy_wait(eth);
380	if (ret < 0)
381	return ret;
382
383	return 0;
384	}
385
386	static int _mtk_mdio_read_c22(struct mtk_eth *eth, u32 phy_addr, u32 phy_reg)
387	{
388	int ret;
389
390	ret = mtk_mdio_busy_wait(eth);
391	if (ret < 0)
392	return ret;
393
394	mtk_w32(eth, PHY_IAC_ACCESS |
395	PHY_IAC_START_C22 |
396	PHY_IAC_CMD_C22_READ |
397	PHY_IAC_REG(phy_reg) |
398	PHY_IAC_ADDR(phy_addr),
399	MTK_PHY_IAC);
400
401	ret = mtk_mdio_busy_wait(eth);
402	if (ret < 0)
403	return ret;
404
405	return mtk_r32(eth, MTK_PHY_IAC) & PHY_IAC_DATA_MASK;
406	}
407
408	static int _mtk_mdio_read_c45(struct mtk_eth *eth, u32 phy_addr,
409	u32 devad, u32 phy_reg)
410	{
411	int ret;
412
413	ret = mtk_mdio_busy_wait(eth);
414	if (ret < 0)
415	return ret;
416
417	mtk_w32(eth, PHY_IAC_ACCESS |
418	PHY_IAC_START_C45 |
419	PHY_IAC_CMD_C45_ADDR |
420	PHY_IAC_REG(devad) |
421	PHY_IAC_ADDR(phy_addr) |
422	PHY_IAC_DATA(phy_reg),
423	MTK_PHY_IAC);
424
425	ret = mtk_mdio_busy_wait(eth);
426	if (ret < 0)
427	return ret;
428
429	mtk_w32(eth, PHY_IAC_ACCESS |
430	PHY_IAC_START_C45 |
431	PHY_IAC_CMD_C45_READ |
432	PHY_IAC_REG(devad) |
433	PHY_IAC_ADDR(phy_addr),
434	MTK_PHY_IAC);
435
436	ret = mtk_mdio_busy_wait(eth);
437	if (ret < 0)
438	return ret;
439
440	return mtk_r32(eth, MTK_PHY_IAC) & PHY_IAC_DATA_MASK;
441	}
442
443	static int mtk_mdio_write_c22(struct mii_bus *bus, int phy_addr,
444	int phy_reg, u16 val)
445	{
446	struct mtk_eth *eth = bus->priv;
447
448	return _mtk_mdio_write_c22(eth, phy_addr, phy_reg, write_data: val);
449	}
450
451	static int mtk_mdio_write_c45(struct mii_bus *bus, int phy_addr,
452	int devad, int phy_reg, u16 val)
453	{
454	struct mtk_eth *eth = bus->priv;
455
456	return _mtk_mdio_write_c45(eth, phy_addr, devad, phy_reg, write_data: val);
457	}
458
459	static int mtk_mdio_read_c22(struct mii_bus *bus, int phy_addr, int phy_reg)
460	{
461	struct mtk_eth *eth = bus->priv;
462
463	return _mtk_mdio_read_c22(eth, phy_addr, phy_reg);
464	}
465
466	static int mtk_mdio_read_c45(struct mii_bus *bus, int phy_addr, int devad,
467	int phy_reg)
468	{
469	struct mtk_eth *eth = bus->priv;
470
471	return _mtk_mdio_read_c45(eth, phy_addr, devad, phy_reg);
472	}
473
474	static int mt7621_gmac0_rgmii_adjust(struct mtk_eth *eth,
475	phy_interface_t interface)
476	{
477	u32 val;
478
479	val = (interface == PHY_INTERFACE_MODE_TRGMII) ?
480	ETHSYS_TRGMII_MT7621_DDR_PLL : 0;
481
482	regmap_update_bits(map: eth->ethsys, ETHSYS_CLKCFG0,
483	ETHSYS_TRGMII_MT7621_MASK, val);
484
485	return 0;
486	}
487
488	static void mtk_gmac0_rgmii_adjust(struct mtk_eth *eth,
489	phy_interface_t interface)
490	{
491	int ret;
492
493	if (interface == PHY_INTERFACE_MODE_TRGMII) {
494	mtk_w32(eth, TRGMII_MODE, INTF_MODE);
495	ret = clk_set_rate(clk: eth->clks[MTK_CLK_TRGPLL], rate: 500000000);
496	if (ret)
497	dev_err(eth->dev, "Failed to set trgmii pll: %d\n", ret);
498	return;
499	}
500
501	dev_err(eth->dev, "Missing PLL configuration, ethernet may not work\n");
502	}
503
504	static void mtk_setup_bridge_switch(struct mtk_eth *eth)
505	{
506	/* Force Port1 XGMAC Link Up */
507	mtk_m32(eth, mask: 0, MTK_XGMAC_FORCE_MODE(MTK_GMAC1_ID),
508	MTK_XGMAC_STS(MTK_GMAC1_ID));
509
510	/* Adjust GSW bridge IPG to 11 */
511	mtk_m32(eth, GSWTX_IPG_MASK | GSWRX_IPG_MASK,
512	set: (GSW_IPG_11 << GSWTX_IPG_SHIFT) |
513	(GSW_IPG_11 << GSWRX_IPG_SHIFT),
514	MTK_GSW_CFG);
515	}
516
517	static struct phylink_pcs *mtk_mac_select_pcs(struct phylink_config *config,
518	phy_interface_t interface)
519	{
520	struct mtk_mac *mac = container_of(config, struct mtk_mac,
521	phylink_config);
522	struct mtk_eth *eth = mac->hw;
523	unsigned int sid;
524
525	if (interface == PHY_INTERFACE_MODE_SGMII ||
526	phy_interface_mode_is_8023z(mode: interface)) {
527	sid = (MTK_HAS_CAPS(eth->soc->caps, MTK_SHARED_SGMII)) ?
528	0 : mac->id;
529
530	return eth->sgmii_pcs[sid];
531	}
532
533	return NULL;
534	}
535
536	static int mtk_mac_prepare(struct phylink_config *config, unsigned int mode,
537	phy_interface_t iface)
538	{
539	struct mtk_mac *mac = container_of(config, struct mtk_mac,
540	phylink_config);
541	struct mtk_eth *eth = mac->hw;
542
543	if (mtk_interface_mode_is_xgmii(eth, interface: iface) &&
544	mac->id != MTK_GMAC1_ID) {
545	mtk_m32(eth: mac->hw, XMAC_MCR_TRX_DISABLE,
546	XMAC_MCR_TRX_DISABLE, MTK_XMAC_MCR(mac->id));
547
548	mtk_m32(eth: mac->hw, MTK_XGMAC_FORCE_MODE(mac->id) |
549	MTK_XGMAC_FORCE_LINK(mac->id),
550	MTK_XGMAC_FORCE_MODE(mac->id), MTK_XGMAC_STS(mac->id));
551	}
552
553	return 0;
554	}
555
556	static void mtk_mac_config(struct phylink_config *config, unsigned int mode,
557	const struct phylink_link_state *state)
558	{
559	struct mtk_mac *mac = container_of(config, struct mtk_mac,
560	phylink_config);
561	struct mtk_eth *eth = mac->hw;
562	int val, ge_mode, err = 0;
563	u32 i;
564
565	/* MT76x8 has no hardware settings between for the MAC */
566	if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628) &&
567	mac->interface != state->interface) {
568	/* Setup soc pin functions */
569	switch (state->interface) {
570	case PHY_INTERFACE_MODE_TRGMII:
571	case PHY_INTERFACE_MODE_RGMII_TXID:
572	case PHY_INTERFACE_MODE_RGMII_RXID:
573	case PHY_INTERFACE_MODE_RGMII_ID:
574	case PHY_INTERFACE_MODE_RGMII:
575	case PHY_INTERFACE_MODE_MII:
576	if (MTK_HAS_CAPS(eth->soc->caps, MTK_RGMII)) {
577	err = mtk_gmac_rgmii_path_setup(eth, mac_id: mac->id);
578	if (err)
579	goto init_err;
580	}
581	break;
582	case PHY_INTERFACE_MODE_1000BASEX:
583	case PHY_INTERFACE_MODE_2500BASEX:
584	case PHY_INTERFACE_MODE_SGMII:
585	err = mtk_gmac_sgmii_path_setup(eth, mac_id: mac->id);
586	if (err)
587	goto init_err;
588	break;
589	case PHY_INTERFACE_MODE_GMII:
590	if (MTK_HAS_CAPS(eth->soc->caps, MTK_GEPHY)) {
591	err = mtk_gmac_gephy_path_setup(eth, mac_id: mac->id);
592	if (err)
593	goto init_err;
594	}
595	break;
596	case PHY_INTERFACE_MODE_INTERNAL:
597	if (mac->id == MTK_GMAC2_ID &&
598	MTK_HAS_CAPS(eth->soc->caps, MTK_2P5GPHY)) {
599	err = mtk_gmac_2p5gphy_path_setup(eth, mac_id: mac->id);
600	if (err)
601	goto init_err;
602	}
603	break;
604	default:
605	goto err_phy;
606	}
607
608	/* Setup clock for 1st gmac */
609	if (!mac->id && state->interface != PHY_INTERFACE_MODE_SGMII &&
610	!phy_interface_mode_is_8023z(mode: state->interface) &&
611	MTK_HAS_CAPS(mac->hw->soc->caps, MTK_GMAC1_TRGMII)) {
612	if (MTK_HAS_CAPS(mac->hw->soc->caps,
613	MTK_TRGMII_MT7621_CLK)) {
614	if (mt7621_gmac0_rgmii_adjust(eth: mac->hw,
615	interface: state->interface))
616	goto err_phy;
617	} else {
618	mtk_gmac0_rgmii_adjust(eth: mac->hw,
619	interface: state->interface);
620
621	/* mt7623_pad_clk_setup */
622	for (i = 0 ; i < NUM_TRGMII_CTRL; i++)
623	mtk_w32(eth: mac->hw,
624	TD_DM_DRVP(8) | TD_DM_DRVN(8),
625	TRGMII_TD_ODT(i));
626
627	/* Assert/release MT7623 RXC reset */
628	mtk_m32(eth: mac->hw, mask: 0, RXC_RST | RXC_DQSISEL,
629	TRGMII_RCK_CTRL);
630	mtk_m32(eth: mac->hw, RXC_RST, set: 0, TRGMII_RCK_CTRL);
631	}
632	}
633
634	switch (state->interface) {
635	case PHY_INTERFACE_MODE_MII:
636	case PHY_INTERFACE_MODE_GMII:
637	ge_mode = 1;
638	break;
639	default:
640	ge_mode = 0;
641	break;
642	}
643
644	/* put the gmac into the right mode */
645	regmap_read(map: eth->ethsys, ETHSYS_SYSCFG0, val: &val);
646	val &= ~SYSCFG0_GE_MODE(SYSCFG0_GE_MASK, mac->id);
647	val |= SYSCFG0_GE_MODE(ge_mode, mac->id);
648	regmap_write(map: eth->ethsys, ETHSYS_SYSCFG0, val);
649
650	mac->interface = state->interface;
651	}
652
653	/* SGMII */
654	if (state->interface == PHY_INTERFACE_MODE_SGMII ||
655	phy_interface_mode_is_8023z(mode: state->interface)) {
656	/* The path GMAC to SGMII will be enabled once the SGMIISYS is
657	* being setup done.
658	*/
659	regmap_read(map: eth->ethsys, ETHSYS_SYSCFG0, val: &val);
660
661	regmap_update_bits(map: eth->ethsys, ETHSYS_SYSCFG0,
662	SYSCFG0_SGMII_MASK,
663	val: ~(u32)SYSCFG0_SGMII_MASK);
664
665	/* Save the syscfg0 value for mac_finish */
666	mac->syscfg0 = val;
667	} else if (phylink_autoneg_inband(mode)) {
668	dev_err(eth->dev,
669	"In-band mode not supported in non SGMII mode!\n");
670	return;
671	}
672
673	/* Setup gmac */
674	if (mtk_interface_mode_is_xgmii(eth, interface: state->interface)) {
675	mtk_w32(eth: mac->hw, MTK_GDMA_XGDM_SEL, MTK_GDMA_EG_CTRL(mac->id));
676	mtk_w32(eth: mac->hw, MAC_MCR_FORCE_LINK_DOWN, MTK_MAC_MCR(mac->id));
677
678	if (mac->id == MTK_GMAC1_ID)
679	mtk_setup_bridge_switch(eth);
680	}
681
682	return;
683
684	err_phy:
685	dev_err(eth->dev, "%s: GMAC%d mode %s not supported!\n", __func__,
686	mac->id, phy_modes(state->interface));
687	return;
688
689	init_err:
690	dev_err(eth->dev, "%s: GMAC%d mode %s err: %d!\n", __func__,
691	mac->id, phy_modes(state->interface), err);
692	}
693
694	static int mtk_mac_finish(struct phylink_config *config, unsigned int mode,
695	phy_interface_t interface)
696	{
697	struct mtk_mac *mac = container_of(config, struct mtk_mac,
698	phylink_config);
699	struct mtk_eth *eth = mac->hw;
700	u32 mcr_cur, mcr_new;
701
702	/* Enable SGMII */
703	if (interface == PHY_INTERFACE_MODE_SGMII ||
704	phy_interface_mode_is_8023z(mode: interface))
705	regmap_update_bits(map: eth->ethsys, ETHSYS_SYSCFG0,
706	SYSCFG0_SGMII_MASK, val: mac->syscfg0);
707
708	/* Setup gmac */
709	mcr_cur = mtk_r32(eth: mac->hw, MTK_MAC_MCR(mac->id));
710	mcr_new = mcr_cur;
711	mcr_new |= MAC_MCR_IPG_CFG | MAC_MCR_FORCE_MODE |
712	MAC_MCR_BACKOFF_EN | MAC_MCR_BACKPR_EN | MAC_MCR_RX_FIFO_CLR_DIS;
713
714	/* Only update control register when needed! */
715	if (mcr_new != mcr_cur)
716	mtk_w32(eth: mac->hw, val: mcr_new, MTK_MAC_MCR(mac->id));
717
718	return 0;
719	}
720
721	static void mtk_mac_link_down(struct phylink_config *config, unsigned int mode,
722	phy_interface_t interface)
723	{
724	struct mtk_mac *mac = container_of(config, struct mtk_mac,
725	phylink_config);
726
727	if (!mtk_interface_mode_is_xgmii(eth: mac->hw, interface)) {
728	/* GMAC modes */
729	mtk_m32(eth: mac->hw,
730	MAC_MCR_TX_EN | MAC_MCR_RX_EN | MAC_MCR_FORCE_LINK, set: 0,
731	MTK_MAC_MCR(mac->id));
732	} else if (mac->id != MTK_GMAC1_ID) {
733	/* XGMAC except for built-in switch */
734	mtk_m32(eth: mac->hw, XMAC_MCR_TRX_DISABLE, XMAC_MCR_TRX_DISABLE,
735	MTK_XMAC_MCR(mac->id));
736	mtk_m32(eth: mac->hw, MTK_XGMAC_FORCE_LINK(mac->id), set: 0,
737	MTK_XGMAC_STS(mac->id));
738	}
739	}
740
741	static void mtk_set_queue_speed(struct mtk_eth *eth, unsigned int idx,
742	int speed)
743	{
744	const struct mtk_soc_data *soc = eth->soc;
745	u32 ofs, val;
746
747	if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA))
748	return;
749
750	val = MTK_QTX_SCH_MIN_RATE_EN |
751	/* minimum: 10 Mbps */
752	FIELD_PREP(MTK_QTX_SCH_MIN_RATE_MAN, 1) |
753	FIELD_PREP(MTK_QTX_SCH_MIN_RATE_EXP, 4) |
754	MTK_QTX_SCH_LEAKY_BUCKET_SIZE;
755	if (mtk_is_netsys_v1(eth))
756	val |= MTK_QTX_SCH_LEAKY_BUCKET_EN;
757
758	if (IS_ENABLED(CONFIG_SOC_MT7621)) {
759	switch (speed) {
760	case SPEED_10:
761	val |= MTK_QTX_SCH_MAX_RATE_EN |
762	FIELD_PREP(MTK_QTX_SCH_MAX_RATE_MAN, 103) |
763	FIELD_PREP(MTK_QTX_SCH_MAX_RATE_EXP, 2) |
764	FIELD_PREP(MTK_QTX_SCH_MAX_RATE_WEIGHT, 1);
765	break;
766	case SPEED_100:
767	val |= MTK_QTX_SCH_MAX_RATE_EN |
768	FIELD_PREP(MTK_QTX_SCH_MAX_RATE_MAN, 103) |
769	FIELD_PREP(MTK_QTX_SCH_MAX_RATE_EXP, 3) |
770	FIELD_PREP(MTK_QTX_SCH_MAX_RATE_WEIGHT, 1);
771	break;
772	case SPEED_1000:
773	val |= MTK_QTX_SCH_MAX_RATE_EN |
774	FIELD_PREP(MTK_QTX_SCH_MAX_RATE_MAN, 105) |
775	FIELD_PREP(MTK_QTX_SCH_MAX_RATE_EXP, 4) |
776	FIELD_PREP(MTK_QTX_SCH_MAX_RATE_WEIGHT, 10);
777	break;
778	default:
779	break;
780	}
781	} else {
782	switch (speed) {
783	case SPEED_10:
784	val |= MTK_QTX_SCH_MAX_RATE_EN |
785	FIELD_PREP(MTK_QTX_SCH_MAX_RATE_MAN, 1) |
786	FIELD_PREP(MTK_QTX_SCH_MAX_RATE_EXP, 4) |
787	FIELD_PREP(MTK_QTX_SCH_MAX_RATE_WEIGHT, 1);
788	break;
789	case SPEED_100:
790	val |= MTK_QTX_SCH_MAX_RATE_EN |
791	FIELD_PREP(MTK_QTX_SCH_MAX_RATE_MAN, 1) |
792	FIELD_PREP(MTK_QTX_SCH_MAX_RATE_EXP, 5) |
793	FIELD_PREP(MTK_QTX_SCH_MAX_RATE_WEIGHT, 1);
794	break;
795	case SPEED_1000:
796	val |= MTK_QTX_SCH_MAX_RATE_EN |
797	FIELD_PREP(MTK_QTX_SCH_MAX_RATE_MAN, 1) |
798	FIELD_PREP(MTK_QTX_SCH_MAX_RATE_EXP, 6) |
799	FIELD_PREP(MTK_QTX_SCH_MAX_RATE_WEIGHT, 10);
800	break;
801	default:
802	break;
803	}
804	}
805
806	ofs = MTK_QTX_OFFSET * idx;
807	mtk_w32(eth, val, reg: soc->reg_map->qdma.qtx_sch + ofs);
808	}
809
810	static void mtk_gdm_mac_link_up(struct mtk_mac *mac,
811	struct phy_device *phy,
812	unsigned int mode, phy_interface_t interface,
813	int speed, int duplex, bool tx_pause,
814	bool rx_pause)
815	{
816	u32 mcr;
817
818	mcr = mtk_r32(eth: mac->hw, MTK_MAC_MCR(mac->id));
819	mcr &= ~(MAC_MCR_SPEED_100 | MAC_MCR_SPEED_1000 |
820	MAC_MCR_FORCE_DPX | MAC_MCR_FORCE_TX_FC |
821	MAC_MCR_FORCE_RX_FC);
822
823	/* Configure speed */
824	mac->speed = speed;
825	switch (speed) {
826	case SPEED_2500:
827	case SPEED_1000:
828	mcr |= MAC_MCR_SPEED_1000;
829	break;
830	case SPEED_100:
831	mcr |= MAC_MCR_SPEED_100;
832	break;
833	}
834
835	/* Configure duplex */
836	if (duplex == DUPLEX_FULL)
837	mcr |= MAC_MCR_FORCE_DPX;
838
839	/* Configure pause modes - phylink will avoid these for half duplex */
840	if (tx_pause)
841	mcr |= MAC_MCR_FORCE_TX_FC;
842	if (rx_pause)
843	mcr |= MAC_MCR_FORCE_RX_FC;
844
845	mcr |= MAC_MCR_TX_EN | MAC_MCR_RX_EN | MAC_MCR_FORCE_LINK;
846	mtk_w32(eth: mac->hw, val: mcr, MTK_MAC_MCR(mac->id));
847	}
848
849	static void mtk_xgdm_mac_link_up(struct mtk_mac *mac,
850	struct phy_device *phy,
851	unsigned int mode, phy_interface_t interface,
852	int speed, int duplex, bool tx_pause,
853	bool rx_pause)
854	{
855	u32 mcr;
856
857	if (mac->id == MTK_GMAC1_ID)
858	return;
859
860	/* Eliminate the interference(before link-up) caused by PHY noise */
861	mtk_m32(eth: mac->hw, XMAC_LOGIC_RST, set: 0, MTK_XMAC_LOGIC_RST(mac->id));
862	mdelay(20);
863	mtk_m32(eth: mac->hw, XMAC_GLB_CNTCLR, XMAC_GLB_CNTCLR,
864	MTK_XMAC_CNT_CTRL(mac->id));
865
866	mtk_m32(eth: mac->hw, MTK_XGMAC_FORCE_LINK(mac->id),
867	MTK_XGMAC_FORCE_LINK(mac->id), MTK_XGMAC_STS(mac->id));
868
869	mcr = mtk_r32(eth: mac->hw, MTK_XMAC_MCR(mac->id));
870	mcr &= ~(XMAC_MCR_FORCE_TX_FC | XMAC_MCR_FORCE_RX_FC |
871	XMAC_MCR_TRX_DISABLE);
872	/* Configure pause modes -
873	* phylink will avoid these for half duplex
874	*/
875	if (tx_pause)
876	mcr |= XMAC_MCR_FORCE_TX_FC;
877	if (rx_pause)
878	mcr |= XMAC_MCR_FORCE_RX_FC;
879
880	mtk_w32(eth: mac->hw, val: mcr, MTK_XMAC_MCR(mac->id));
881	}
882
883	static void mtk_mac_link_up(struct phylink_config *config,
884	struct phy_device *phy,
885	unsigned int mode, phy_interface_t interface,
886	int speed, int duplex, bool tx_pause, bool rx_pause)
887	{
888	struct mtk_mac *mac = container_of(config, struct mtk_mac,
889	phylink_config);
890
891	if (mtk_interface_mode_is_xgmii(eth: mac->hw, interface))
892	mtk_xgdm_mac_link_up(mac, phy, mode, interface, speed, duplex,
893	tx_pause, rx_pause);
894	else
895	mtk_gdm_mac_link_up(mac, phy, mode, interface, speed, duplex,
896	tx_pause, rx_pause);
897	}
898
899	static void mtk_mac_disable_tx_lpi(struct phylink_config *config)
900	{
901	struct mtk_mac *mac = container_of(config, struct mtk_mac,
902	phylink_config);
903	struct mtk_eth *eth = mac->hw;
904
905	mtk_m32(eth, MAC_MCR_EEE100M | MAC_MCR_EEE1G, set: 0, MTK_MAC_MCR(mac->id));
906	}
907
908	static int mtk_mac_enable_tx_lpi(struct phylink_config *config, u32 timer,
909	bool tx_clk_stop)
910	{
911	struct mtk_mac *mac = container_of(config, struct mtk_mac,
912	phylink_config);
913	struct mtk_eth *eth = mac->hw;
914	u32 val;
915
916	if (mtk_interface_mode_is_xgmii(eth, interface: mac->interface))
917	return -EOPNOTSUPP;
918
919	/* Tx idle timer in ms */
920	timer = DIV_ROUND_UP(timer, 1000);
921
922	/* If the timer is zero, then set LPI_MODE, which allows the
923	* system to enter LPI mode immediately rather than waiting for
924	* the LPI threshold.
925	*/
926	if (!timer)
927	val = MAC_EEE_LPI_MODE;
928	else if (FIELD_FIT(MAC_EEE_LPI_TXIDLE_THD, timer))
929	val = FIELD_PREP(MAC_EEE_LPI_TXIDLE_THD, timer);
930	else
931	val = MAC_EEE_LPI_TXIDLE_THD;
932
933	if (tx_clk_stop)
934	val |= MAC_EEE_CKG_TXIDLE;
935
936	/* PHY Wake-up time, this field does not have a reset value, so use the
937	* reset value from MT7531 (36us for 100M and 17us for 1000M).
938	*/
939	val |= FIELD_PREP(MAC_EEE_WAKEUP_TIME_1000, 17) |
940	FIELD_PREP(MAC_EEE_WAKEUP_TIME_100, 36);
941
942	mtk_w32(eth, val, MTK_MAC_EEECR(mac->id));
943	mtk_m32(eth, mask: 0, MAC_MCR_EEE100M | MAC_MCR_EEE1G, MTK_MAC_MCR(mac->id));
944
945	return 0;
946	}
947
948	static const struct phylink_mac_ops mtk_phylink_ops = {
949	.mac_prepare = mtk_mac_prepare,
950	.mac_select_pcs = mtk_mac_select_pcs,
951	.mac_config = mtk_mac_config,
952	.mac_finish = mtk_mac_finish,
953	.mac_link_down = mtk_mac_link_down,
954	.mac_link_up = mtk_mac_link_up,
955	.mac_disable_tx_lpi = mtk_mac_disable_tx_lpi,
956	.mac_enable_tx_lpi = mtk_mac_enable_tx_lpi,
957	};
958
959	static void mtk_mdio_config(struct mtk_eth *eth)
960	{
961	u32 val;
962
963	/* Configure MDC Divider */
964	val = FIELD_PREP(PPSC_MDC_CFG, eth->mdc_divider);
965
966	/* Configure MDC Turbo Mode */
967	if (mtk_is_netsys_v3_or_greater(eth))
968	mtk_m32(eth, mask: 0, MISC_MDC_TURBO, MTK_MAC_MISC_V3);
969	else
970	val |= PPSC_MDC_TURBO;
971
972	mtk_m32(eth, PPSC_MDC_CFG, set: val, MTK_PPSC);
973	}
974
975	static int mtk_mdio_init(struct mtk_eth *eth)
976	{
977	unsigned int max_clk = 2500000;
978	struct device_node *mii_np;
979	int ret;
980	u32 val;
981
982	mii_np = of_get_available_child_by_name(node: eth->dev->of_node, name: "mdio-bus");
983	if (!mii_np) {
984	dev_err(eth->dev, "no %s child node found", "mdio-bus");
985	return -ENODEV;
986	}
987
988	eth->mii_bus = devm_mdiobus_alloc(dev: eth->dev);
989	if (!eth->mii_bus) {
990	ret = -ENOMEM;
991	goto err_put_node;
992	}
993
994	eth->mii_bus->name = "mdio";
995	eth->mii_bus->read = mtk_mdio_read_c22;
996	eth->mii_bus->write = mtk_mdio_write_c22;
997	eth->mii_bus->read_c45 = mtk_mdio_read_c45;
998	eth->mii_bus->write_c45 = mtk_mdio_write_c45;
999	eth->mii_bus->priv = eth;
1000	eth->mii_bus->parent = eth->dev;
1001
1002	snprintf(buf: eth->mii_bus->id, MII_BUS_ID_SIZE, fmt: "%pOFn", mii_np);
1003
1004	if (!of_property_read_u32(np: mii_np, propname: "clock-frequency", out_value: &val)) {
1005	if (val > MDC_MAX_FREQ || val < MDC_MAX_FREQ / MDC_MAX_DIVIDER) {
1006	dev_err(eth->dev, "MDIO clock frequency out of range");
1007	ret = -EINVAL;
1008	goto err_put_node;
1009	}
1010	max_clk = val;
1011	}
1012	eth->mdc_divider = min_t(unsigned int, DIV_ROUND_UP(MDC_MAX_FREQ, max_clk), 63);
1013	mtk_mdio_config(eth);
1014	dev_dbg(eth->dev, "MDC is running on %d Hz\n", MDC_MAX_FREQ / eth->mdc_divider);
1015	ret = of_mdiobus_register(mdio: eth->mii_bus, np: mii_np);
1016
1017	err_put_node:
1018	of_node_put(node: mii_np);
1019	return ret;
1020	}
1021
1022	static void mtk_mdio_cleanup(struct mtk_eth *eth)
1023	{
1024	if (!eth->mii_bus)
1025	return;
1026
1027	mdiobus_unregister(bus: eth->mii_bus);
1028	}
1029
1030	static inline void mtk_tx_irq_disable(struct mtk_eth *eth, u32 mask)
1031	{
1032	unsigned long flags;
1033	u32 val;
1034
1035	spin_lock_irqsave(&eth->tx_irq_lock, flags);
1036	val = mtk_r32(eth, reg: eth->soc->reg_map->tx_irq_mask);
1037	mtk_w32(eth, val: val & ~mask, reg: eth->soc->reg_map->tx_irq_mask);
1038	spin_unlock_irqrestore(lock: &eth->tx_irq_lock, flags);
1039	}
1040
1041	static inline void mtk_tx_irq_enable(struct mtk_eth *eth, u32 mask)
1042	{
1043	unsigned long flags;
1044	u32 val;
1045
1046	spin_lock_irqsave(&eth->tx_irq_lock, flags);
1047	val = mtk_r32(eth, reg: eth->soc->reg_map->tx_irq_mask);
1048	mtk_w32(eth, val: val | mask, reg: eth->soc->reg_map->tx_irq_mask);
1049	spin_unlock_irqrestore(lock: &eth->tx_irq_lock, flags);
1050	}
1051
1052	static inline void mtk_rx_irq_disable(struct mtk_eth *eth, u32 mask)
1053	{
1054	unsigned long flags;
1055	u32 val;
1056
1057	spin_lock_irqsave(&eth->rx_irq_lock, flags);
1058	val = mtk_r32(eth, reg: eth->soc->reg_map->pdma.irq_mask);
1059	mtk_w32(eth, val: val & ~mask, reg: eth->soc->reg_map->pdma.irq_mask);
1060	spin_unlock_irqrestore(lock: &eth->rx_irq_lock, flags);
1061	}
1062
1063	static inline void mtk_rx_irq_enable(struct mtk_eth *eth, u32 mask)
1064	{
1065	unsigned long flags;
1066	u32 val;
1067
1068	spin_lock_irqsave(&eth->rx_irq_lock, flags);
1069	val = mtk_r32(eth, reg: eth->soc->reg_map->pdma.irq_mask);
1070	mtk_w32(eth, val: val | mask, reg: eth->soc->reg_map->pdma.irq_mask);
1071	spin_unlock_irqrestore(lock: &eth->rx_irq_lock, flags);
1072	}
1073
1074	static int mtk_set_mac_address(struct net_device *dev, void *p)
1075	{
1076	int ret = eth_mac_addr(dev, p);
1077	struct mtk_mac *mac = netdev_priv(dev);
1078	struct mtk_eth *eth = mac->hw;
1079	const char *macaddr = dev->dev_addr;
1080
1081	if (ret)
1082	return ret;
1083
1084	if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
1085	return -EBUSY;
1086
1087	spin_lock_bh(lock: &mac->hw->page_lock);
1088	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
1089	mtk_w32(eth: mac->hw, val: (macaddr[0] << 8) | macaddr[1],
1090	MT7628_SDM_MAC_ADRH);
1091	mtk_w32(eth: mac->hw, val: (macaddr[2] << 24) | (macaddr[3] << 16) |
1092	(macaddr[4] << 8) | macaddr[5],
1093	MT7628_SDM_MAC_ADRL);
1094	} else {
1095	mtk_w32(eth: mac->hw, val: (macaddr[0] << 8) | macaddr[1],
1096	MTK_GDMA_MAC_ADRH(mac->id));
1097	mtk_w32(eth: mac->hw, val: (macaddr[2] << 24) | (macaddr[3] << 16) |
1098	(macaddr[4] << 8) | macaddr[5],
1099	MTK_GDMA_MAC_ADRL(mac->id));
1100	}
1101	spin_unlock_bh(lock: &mac->hw->page_lock);
1102
1103	return 0;
1104	}
1105
1106	void mtk_stats_update_mac(struct mtk_mac *mac)
1107	{
1108	struct mtk_hw_stats *hw_stats = mac->hw_stats;
1109	struct mtk_eth *eth = mac->hw;
1110
1111	u64_stats_update_begin(syncp: &hw_stats->syncp);
1112
1113	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
1114	hw_stats->tx_packets += mtk_r32(eth: mac->hw, MT7628_SDM_TPCNT);
1115	hw_stats->tx_bytes += mtk_r32(eth: mac->hw, MT7628_SDM_TBCNT);
1116	hw_stats->rx_packets += mtk_r32(eth: mac->hw, MT7628_SDM_RPCNT);
1117	hw_stats->rx_bytes += mtk_r32(eth: mac->hw, MT7628_SDM_RBCNT);
1118	hw_stats->rx_checksum_errors +=
1119	mtk_r32(eth: mac->hw, MT7628_SDM_CS_ERR);
1120	} else {
1121	const struct mtk_reg_map *reg_map = eth->soc->reg_map;
1122	unsigned int offs = hw_stats->reg_offset;
1123	u64 stats;
1124
1125	hw_stats->rx_bytes += mtk_r32(eth: mac->hw, reg: reg_map->gdm1_cnt + offs);
1126	stats = mtk_r32(eth: mac->hw, reg: reg_map->gdm1_cnt + 0x4 + offs);
1127	if (stats)
1128	hw_stats->rx_bytes += (stats << 32);
1129	hw_stats->rx_packets +=
1130	mtk_r32(eth: mac->hw, reg: reg_map->gdm1_cnt + 0x8 + offs);
1131	hw_stats->rx_overflow +=
1132	mtk_r32(eth: mac->hw, reg: reg_map->gdm1_cnt + 0x10 + offs);
1133	hw_stats->rx_fcs_errors +=
1134	mtk_r32(eth: mac->hw, reg: reg_map->gdm1_cnt + 0x14 + offs);
1135	hw_stats->rx_short_errors +=
1136	mtk_r32(eth: mac->hw, reg: reg_map->gdm1_cnt + 0x18 + offs);
1137	hw_stats->rx_long_errors +=
1138	mtk_r32(eth: mac->hw, reg: reg_map->gdm1_cnt + 0x1c + offs);
1139	hw_stats->rx_checksum_errors +=
1140	mtk_r32(eth: mac->hw, reg: reg_map->gdm1_cnt + 0x20 + offs);
1141	hw_stats->rx_flow_control_packets +=
1142	mtk_r32(eth: mac->hw, reg: reg_map->gdm1_cnt + 0x24 + offs);
1143
1144	if (mtk_is_netsys_v3_or_greater(eth)) {
1145	hw_stats->tx_skip +=
1146	mtk_r32(eth: mac->hw, reg: reg_map->gdm1_cnt + 0x50 + offs);
1147	hw_stats->tx_collisions +=
1148	mtk_r32(eth: mac->hw, reg: reg_map->gdm1_cnt + 0x54 + offs);
1149	hw_stats->tx_bytes +=
1150	mtk_r32(eth: mac->hw, reg: reg_map->gdm1_cnt + 0x40 + offs);
1151	stats = mtk_r32(eth: mac->hw, reg: reg_map->gdm1_cnt + 0x44 + offs);
1152	if (stats)
1153	hw_stats->tx_bytes += (stats << 32);
1154	hw_stats->tx_packets +=
1155	mtk_r32(eth: mac->hw, reg: reg_map->gdm1_cnt + 0x48 + offs);
1156	} else {
1157	hw_stats->tx_skip +=
1158	mtk_r32(eth: mac->hw, reg: reg_map->gdm1_cnt + 0x28 + offs);
1159	hw_stats->tx_collisions +=
1160	mtk_r32(eth: mac->hw, reg: reg_map->gdm1_cnt + 0x2c + offs);
1161	hw_stats->tx_bytes +=
1162	mtk_r32(eth: mac->hw, reg: reg_map->gdm1_cnt + 0x30 + offs);
1163	stats = mtk_r32(eth: mac->hw, reg: reg_map->gdm1_cnt + 0x34 + offs);
1164	if (stats)
1165	hw_stats->tx_bytes += (stats << 32);
1166	hw_stats->tx_packets +=
1167	mtk_r32(eth: mac->hw, reg: reg_map->gdm1_cnt + 0x38 + offs);
1168	}
1169	}
1170
1171	u64_stats_update_end(syncp: &hw_stats->syncp);
1172	}
1173
1174	static void mtk_stats_update(struct mtk_eth *eth)
1175	{
1176	int i;
1177
1178	for (i = 0; i < MTK_MAX_DEVS; i++) {
1179	if (!eth->mac[i] || !eth->mac[i]->hw_stats)
1180	continue;
1181	if (spin_trylock(lock: &eth->mac[i]->hw_stats->stats_lock)) {
1182	mtk_stats_update_mac(mac: eth->mac[i]);
1183	spin_unlock(lock: &eth->mac[i]->hw_stats->stats_lock);
1184	}
1185	}
1186	}
1187
1188	static void mtk_get_stats64(struct net_device *dev,
1189	struct rtnl_link_stats64 *storage)
1190	{
1191	struct mtk_mac *mac = netdev_priv(dev);
1192	struct mtk_hw_stats *hw_stats = mac->hw_stats;
1193	unsigned int start;
1194
1195	if (netif_running(dev) && netif_device_present(dev)) {
1196	if (spin_trylock_bh(lock: &hw_stats->stats_lock)) {
1197	mtk_stats_update_mac(mac);
1198	spin_unlock_bh(lock: &hw_stats->stats_lock);
1199	}
1200	}
1201
1202	do {
1203	start = u64_stats_fetch_begin(syncp: &hw_stats->syncp);
1204	storage->rx_packets = hw_stats->rx_packets;
1205	storage->tx_packets = hw_stats->tx_packets;
1206	storage->rx_bytes = hw_stats->rx_bytes;
1207	storage->tx_bytes = hw_stats->tx_bytes;
1208	storage->collisions = hw_stats->tx_collisions;
1209	storage->rx_length_errors = hw_stats->rx_short_errors +
1210	hw_stats->rx_long_errors;
1211	storage->rx_over_errors = hw_stats->rx_overflow;
1212	storage->rx_crc_errors = hw_stats->rx_fcs_errors;
1213	storage->rx_errors = hw_stats->rx_checksum_errors;
1214	storage->tx_aborted_errors = hw_stats->tx_skip;
1215	} while (u64_stats_fetch_retry(syncp: &hw_stats->syncp, start));
1216
1217	storage->tx_errors = dev->stats.tx_errors;
1218	storage->rx_dropped = dev->stats.rx_dropped;
1219	storage->tx_dropped = dev->stats.tx_dropped;
1220	}
1221
1222	static inline int mtk_max_frag_size(int mtu)
1223	{
1224	/* make sure buf_size will be at least MTK_MAX_RX_LENGTH */
1225	if (mtu + MTK_RX_ETH_HLEN < MTK_MAX_RX_LENGTH_2K)
1226	mtu = MTK_MAX_RX_LENGTH_2K - MTK_RX_ETH_HLEN;
1227
1228	return SKB_DATA_ALIGN(MTK_RX_HLEN + mtu) +
1229	SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1230	}
1231
1232	static inline int mtk_max_buf_size(int frag_size)
1233	{
1234	int buf_size = frag_size - NET_SKB_PAD - NET_IP_ALIGN -
1235	SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1236
1237	WARN_ON(buf_size < MTK_MAX_RX_LENGTH_2K);
1238
1239	return buf_size;
1240	}
1241
1242	static bool mtk_rx_get_desc(struct mtk_eth *eth, struct mtk_rx_dma_v2 *rxd,
1243	struct mtk_rx_dma_v2 *dma_rxd)
1244	{
1245	rxd->rxd2 = READ_ONCE(dma_rxd->rxd2);
1246	if (!(rxd->rxd2 & RX_DMA_DONE))
1247	return false;
1248
1249	rxd->rxd1 = READ_ONCE(dma_rxd->rxd1);
1250	rxd->rxd3 = READ_ONCE(dma_rxd->rxd3);
1251	rxd->rxd4 = READ_ONCE(dma_rxd->rxd4);
1252	if (mtk_is_netsys_v3_or_greater(eth)) {
1253	rxd->rxd5 = READ_ONCE(dma_rxd->rxd5);
1254	rxd->rxd6 = READ_ONCE(dma_rxd->rxd6);
1255	}
1256
1257	return true;
1258	}
1259
1260	static void *mtk_max_lro_buf_alloc(gfp_t gfp_mask)
1261	{
1262	unsigned int size = mtk_max_frag_size(MTK_MAX_LRO_RX_LENGTH);
1263	unsigned long data;
1264
1265	data = __get_free_pages(gfp_mask | __GFP_COMP | __GFP_NOWARN,
1266	get_order(size));
1267
1268	return (void *)data;
1269	}
1270
1271	static void *mtk_dma_ring_alloc(struct mtk_eth *eth, size_t size,
1272	dma_addr_t *dma_handle, bool use_sram)
1273	{
1274	void *dma_ring;
1275
1276	if (use_sram && eth->sram_pool) {
1277	dma_ring = (void *)gen_pool_alloc(pool: eth->sram_pool, size);
1278	if (!dma_ring)
1279	return dma_ring;
1280	*dma_handle = gen_pool_virt_to_phys(pool: eth->sram_pool,
1281	(unsigned long)dma_ring);
1282	} else {
1283	dma_ring = dma_alloc_coherent(dev: eth->dma_dev, size, dma_handle,
1284	GFP_KERNEL);
1285	}
1286
1287	return dma_ring;
1288	}
1289
1290	static void mtk_dma_ring_free(struct mtk_eth *eth, size_t size, void *dma_ring,
1291	dma_addr_t dma_handle, bool in_sram)
1292	{
1293	if (in_sram && eth->sram_pool)
1294	gen_pool_free(pool: eth->sram_pool, addr: (unsigned long)dma_ring, size);
1295	else
1296	dma_free_coherent(dev: eth->dma_dev, size, cpu_addr: dma_ring, dma_handle);
1297	}
1298
1299	/* the qdma core needs scratch memory to be setup */
1300	static int mtk_init_fq_dma(struct mtk_eth *eth)
1301	{
1302	const struct mtk_soc_data *soc = eth->soc;
1303	dma_addr_t phy_ring_tail;
1304	int cnt = soc->tx.fq_dma_size;
1305	dma_addr_t dma_addr;
1306	int i, j, len;
1307
1308	eth->scratch_ring = mtk_dma_ring_alloc(eth, size: cnt * soc->tx.desc_size,
1309	dma_handle: &eth->phy_scratch_ring, use_sram: true);
1310
1311	if (unlikely(!eth->scratch_ring))
1312	return -ENOMEM;
1313
1314	phy_ring_tail = eth->phy_scratch_ring + soc->tx.desc_size * (cnt - 1);
1315
1316	for (j = 0; j < DIV_ROUND_UP(soc->tx.fq_dma_size, MTK_FQ_DMA_LENGTH); j++) {
1317	len = min_t(int, cnt - j * MTK_FQ_DMA_LENGTH, MTK_FQ_DMA_LENGTH);
1318	eth->scratch_head[j] = kcalloc(len, MTK_QDMA_PAGE_SIZE, GFP_KERNEL);
1319
1320	if (unlikely(!eth->scratch_head[j]))
1321	return -ENOMEM;
1322
1323	dma_addr = dma_map_single(eth->dma_dev,
1324	eth->scratch_head[j], len * MTK_QDMA_PAGE_SIZE,
1325	DMA_FROM_DEVICE);
1326
1327	if (unlikely(dma_mapping_error(eth->dma_dev, dma_addr)))
1328	return -ENOMEM;
1329
1330	for (i = 0; i < len; i++) {
1331	struct mtk_tx_dma_v2 *txd;
1332
1333	txd = eth->scratch_ring + (j * MTK_FQ_DMA_LENGTH + i) * soc->tx.desc_size;
1334	txd->txd1 = dma_addr + i * MTK_QDMA_PAGE_SIZE;
1335	if (j * MTK_FQ_DMA_LENGTH + i < cnt)
1336	txd->txd2 = eth->phy_scratch_ring +
1337	(j * MTK_FQ_DMA_LENGTH + i + 1) * soc->tx.desc_size;
1338
1339	txd->txd3 = TX_DMA_PLEN0(MTK_QDMA_PAGE_SIZE);
1340	if (MTK_HAS_CAPS(soc->caps, MTK_36BIT_DMA))
1341	txd->txd3 |= TX_DMA_PREP_ADDR64(dma_addr + i * MTK_QDMA_PAGE_SIZE);
1342
1343	txd->txd4 = 0;
1344	if (mtk_is_netsys_v2_or_greater(eth)) {
1345	txd->txd5 = 0;
1346	txd->txd6 = 0;
1347	txd->txd7 = 0;
1348	txd->txd8 = 0;
1349	}
1350	}
1351	}
1352
1353	mtk_w32(eth, val: eth->phy_scratch_ring, reg: soc->reg_map->qdma.fq_head);
1354	mtk_w32(eth, val: phy_ring_tail, reg: soc->reg_map->qdma.fq_tail);
1355	mtk_w32(eth, val: (cnt << 16) | cnt, reg: soc->reg_map->qdma.fq_count);
1356	mtk_w32(eth, MTK_QDMA_PAGE_SIZE << 16, reg: soc->reg_map->qdma.fq_blen);
1357
1358	return 0;
1359	}
1360
1361	static void *mtk_qdma_phys_to_virt(struct mtk_tx_ring *ring, u32 desc)
1362	{
1363	return ring->dma + (desc - ring->phys);
1364	}
1365
1366	static struct mtk_tx_buf *mtk_desc_to_tx_buf(struct mtk_tx_ring *ring,
1367	void *txd, u32 txd_size)
1368	{
1369	int idx = (txd - ring->dma) / txd_size;
1370
1371	return &ring->buf[idx];
1372	}
1373
1374	static struct mtk_tx_dma *qdma_to_pdma(struct mtk_tx_ring *ring,
1375	struct mtk_tx_dma *dma)
1376	{
1377	return ring->dma_pdma - (struct mtk_tx_dma *)ring->dma + dma;
1378	}
1379
1380	static int txd_to_idx(struct mtk_tx_ring *ring, void *dma, u32 txd_size)
1381	{
1382	return (dma - ring->dma) / txd_size;
1383	}
1384
1385	static void mtk_tx_unmap(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf,
1386	struct xdp_frame_bulk *bq, bool napi)
1387	{
1388	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
1389	if (tx_buf->flags & MTK_TX_FLAGS_SINGLE0) {
1390	dma_unmap_single(eth->dma_dev,
1391	dma_unmap_addr(tx_buf, dma_addr0),
1392	dma_unmap_len(tx_buf, dma_len0),
1393	DMA_TO_DEVICE);
1394	} else if (tx_buf->flags & MTK_TX_FLAGS_PAGE0) {
1395	dma_unmap_page(eth->dma_dev,
1396	dma_unmap_addr(tx_buf, dma_addr0),
1397	dma_unmap_len(tx_buf, dma_len0),
1398	DMA_TO_DEVICE);
1399	}
1400	} else {
1401	if (dma_unmap_len(tx_buf, dma_len0)) {
1402	dma_unmap_page(eth->dma_dev,
1403	dma_unmap_addr(tx_buf, dma_addr0),
1404	dma_unmap_len(tx_buf, dma_len0),
1405	DMA_TO_DEVICE);
1406	}
1407
1408	if (dma_unmap_len(tx_buf, dma_len1)) {
1409	dma_unmap_page(eth->dma_dev,
1410	dma_unmap_addr(tx_buf, dma_addr1),
1411	dma_unmap_len(tx_buf, dma_len1),
1412	DMA_TO_DEVICE);
1413	}
1414	}
1415
1416	if (tx_buf->data && tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) {
1417	if (tx_buf->type == MTK_TYPE_SKB) {
1418	struct sk_buff *skb = tx_buf->data;
1419
1420	if (napi)
1421	napi_consume_skb(skb, budget: napi);
1422	else
1423	dev_kfree_skb_any(skb);
1424	} else {
1425	struct xdp_frame *xdpf = tx_buf->data;
1426
1427	if (napi && tx_buf->type == MTK_TYPE_XDP_TX)
1428	xdp_return_frame_rx_napi(xdpf);
1429	else if (bq)
1430	xdp_return_frame_bulk(xdpf, bq);
1431	else
1432	xdp_return_frame(xdpf);
1433	}
1434	}
1435	tx_buf->flags = 0;
1436	tx_buf->data = NULL;
1437	}
1438
1439	static void setup_tx_buf(struct mtk_eth *eth, struct mtk_tx_buf *tx_buf,
1440	struct mtk_tx_dma *txd, dma_addr_t mapped_addr,
1441	size_t size, int idx)
1442	{
1443	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
1444	dma_unmap_addr_set(tx_buf, dma_addr0, mapped_addr);
1445	dma_unmap_len_set(tx_buf, dma_len0, size);
1446	} else {
1447	if (idx & 1) {
1448	txd->txd3 = mapped_addr;
1449	txd->txd2 |= TX_DMA_PLEN1(size);
1450	dma_unmap_addr_set(tx_buf, dma_addr1, mapped_addr);
1451	dma_unmap_len_set(tx_buf, dma_len1, size);
1452	} else {
1453	tx_buf->data = (void *)MTK_DMA_DUMMY_DESC;
1454	txd->txd1 = mapped_addr;
1455	txd->txd2 = TX_DMA_PLEN0(size);
1456	dma_unmap_addr_set(tx_buf, dma_addr0, mapped_addr);
1457	dma_unmap_len_set(tx_buf, dma_len0, size);
1458	}
1459	}
1460	}
1461
1462	static void mtk_tx_set_dma_desc_v1(struct net_device *dev, void *txd,
1463	struct mtk_tx_dma_desc_info *info)
1464	{
1465	struct mtk_mac *mac = netdev_priv(dev);
1466	struct mtk_eth *eth = mac->hw;
1467	struct mtk_tx_dma *desc = txd;
1468	u32 data;
1469
1470	WRITE_ONCE(desc->txd1, info->addr);
1471
1472	data = TX_DMA_SWC | TX_DMA_PLEN0(info->size) |
1473	FIELD_PREP(TX_DMA_PQID, info->qid);
1474	if (info->last)
1475	data |= TX_DMA_LS0;
1476	WRITE_ONCE(desc->txd3, data);
1477
1478	data = (mac->id + 1) << TX_DMA_FPORT_SHIFT; /* forward port */
1479	if (info->first) {
1480	if (info->gso)
1481	data |= TX_DMA_TSO;
1482	/* tx checksum offload */
1483	if (info->csum)
1484	data |= TX_DMA_CHKSUM;
1485	/* vlan header offload */
1486	if (info->vlan)
1487	data |= TX_DMA_INS_VLAN | info->vlan_tci;
1488	}
1489	WRITE_ONCE(desc->txd4, data);
1490	}
1491
1492	static void mtk_tx_set_dma_desc_v2(struct net_device *dev, void *txd,
1493	struct mtk_tx_dma_desc_info *info)
1494	{
1495	struct mtk_mac *mac = netdev_priv(dev);
1496	struct mtk_tx_dma_v2 *desc = txd;
1497	struct mtk_eth *eth = mac->hw;
1498	u32 data;
1499
1500	WRITE_ONCE(desc->txd1, info->addr);
1501
1502	data = TX_DMA_PLEN0(info->size);
1503	if (info->last)
1504	data |= TX_DMA_LS0;
1505
1506	if (MTK_HAS_CAPS(eth->soc->caps, MTK_36BIT_DMA))
1507	data |= TX_DMA_PREP_ADDR64(info->addr);
1508
1509	WRITE_ONCE(desc->txd3, data);
1510
1511	/* set forward port */
1512	switch (mac->id) {
1513	case MTK_GMAC1_ID:
1514	data = PSE_GDM1_PORT << TX_DMA_FPORT_SHIFT_V2;
1515	break;
1516	case MTK_GMAC2_ID:
1517	data = PSE_GDM2_PORT << TX_DMA_FPORT_SHIFT_V2;
1518	break;
1519	case MTK_GMAC3_ID:
1520	data = PSE_GDM3_PORT << TX_DMA_FPORT_SHIFT_V2;
1521	break;
1522	}
1523
1524	data |= TX_DMA_SWC_V2 | QID_BITS_V2(info->qid);
1525	WRITE_ONCE(desc->txd4, data);
1526
1527	data = 0;
1528	if (info->first) {
1529	if (info->gso)
1530	data |= TX_DMA_TSO_V2;
1531	/* tx checksum offload */
1532	if (info->csum)
1533	data |= TX_DMA_CHKSUM_V2;
1534	if (mtk_is_netsys_v3_or_greater(eth) && netdev_uses_dsa(dev))
1535	data |= TX_DMA_SPTAG_V3;
1536	}
1537	WRITE_ONCE(desc->txd5, data);
1538
1539	data = 0;
1540	if (info->first && info->vlan)
1541	data |= TX_DMA_INS_VLAN_V2 | info->vlan_tci;
1542	WRITE_ONCE(desc->txd6, data);
1543
1544	WRITE_ONCE(desc->txd7, 0);
1545	WRITE_ONCE(desc->txd8, 0);
1546	}
1547
1548	static void mtk_tx_set_dma_desc(struct net_device *dev, void *txd,
1549	struct mtk_tx_dma_desc_info *info)
1550	{
1551	struct mtk_mac *mac = netdev_priv(dev);
1552	struct mtk_eth *eth = mac->hw;
1553
1554	if (mtk_is_netsys_v2_or_greater(eth))
1555	mtk_tx_set_dma_desc_v2(dev, txd, info);
1556	else
1557	mtk_tx_set_dma_desc_v1(dev, txd, info);
1558	}
1559
1560	static int mtk_tx_map(struct sk_buff *skb, struct net_device *dev,
1561	int tx_num, struct mtk_tx_ring *ring, bool gso)
1562	{
1563	struct mtk_tx_dma_desc_info txd_info = {
1564	.size = skb_headlen(skb),
1565	.gso = gso,
1566	.csum = skb->ip_summed == CHECKSUM_PARTIAL,
1567	.vlan = skb_vlan_tag_present(skb),
1568	.qid = skb_get_queue_mapping(skb),
1569	.vlan_tci = skb_vlan_tag_get(skb),
1570	.first = true,
1571	.last = !skb_is_nonlinear(skb),
1572	};
1573	struct netdev_queue *txq;
1574	struct mtk_mac *mac = netdev_priv(dev);
1575	struct mtk_eth *eth = mac->hw;
1576	const struct mtk_soc_data *soc = eth->soc;
1577	struct mtk_tx_dma *itxd, *txd;
1578	struct mtk_tx_dma *itxd_pdma, *txd_pdma;
1579	struct mtk_tx_buf *itx_buf, *tx_buf;
1580	int i, n_desc = 1;
1581	int queue = skb_get_queue_mapping(skb);
1582	int k = 0;
1583
1584	txq = netdev_get_tx_queue(dev, index: queue);
1585	itxd = ring->next_free;
1586	itxd_pdma = qdma_to_pdma(ring, dma: itxd);
1587	if (itxd == ring->last_free)
1588	return -ENOMEM;
1589
1590	itx_buf = mtk_desc_to_tx_buf(ring, txd: itxd, txd_size: soc->tx.desc_size);
1591	memset(itx_buf, 0, sizeof(*itx_buf));
1592
1593	txd_info.addr = dma_map_single(eth->dma_dev, skb->data, txd_info.size,
1594	DMA_TO_DEVICE);
1595	if (unlikely(dma_mapping_error(eth->dma_dev, txd_info.addr)))
1596	return -ENOMEM;
1597
1598	mtk_tx_set_dma_desc(dev, txd: itxd, info: &txd_info);
1599
1600	itx_buf->flags |= MTK_TX_FLAGS_SINGLE0;
1601	itx_buf->mac_id = mac->id;
1602	setup_tx_buf(eth, tx_buf: itx_buf, txd: itxd_pdma, mapped_addr: txd_info.addr, size: txd_info.size,
1603	idx: k++);
1604
1605	/* TX SG offload */
1606	txd = itxd;
1607	txd_pdma = qdma_to_pdma(ring, dma: txd);
1608
1609	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1610	skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1611	unsigned int offset = 0;
1612	int frag_size = skb_frag_size(frag);
1613
1614	while (frag_size) {
1615	bool new_desc = true;
1616
1617	if (MTK_HAS_CAPS(soc->caps, MTK_QDMA) ||
1618	(i & 0x1)) {
1619	txd = mtk_qdma_phys_to_virt(ring, desc: txd->txd2);
1620	txd_pdma = qdma_to_pdma(ring, dma: txd);
1621	if (txd == ring->last_free)
1622	goto err_dma;
1623
1624	n_desc++;
1625	} else {
1626	new_desc = false;
1627	}
1628
1629	memset(&txd_info, 0, sizeof(struct mtk_tx_dma_desc_info));
1630	txd_info.size = min_t(unsigned int, frag_size,
1631	soc->tx.dma_max_len);
1632	txd_info.qid = queue;
1633	txd_info.last = i == skb_shinfo(skb)->nr_frags - 1 &&
1634	!(frag_size - txd_info.size);
1635	txd_info.addr = skb_frag_dma_map(eth->dma_dev, frag,
1636	offset, txd_info.size,
1637	DMA_TO_DEVICE);
1638	if (unlikely(dma_mapping_error(eth->dma_dev, txd_info.addr)))
1639	goto err_dma;
1640
1641	mtk_tx_set_dma_desc(dev, txd, info: &txd_info);
1642
1643	tx_buf = mtk_desc_to_tx_buf(ring, txd,
1644	txd_size: soc->tx.desc_size);
1645	if (new_desc)
1646	memset(tx_buf, 0, sizeof(*tx_buf));
1647	tx_buf->data = (void *)MTK_DMA_DUMMY_DESC;
1648	tx_buf->flags |= MTK_TX_FLAGS_PAGE0;
1649	tx_buf->mac_id = mac->id;
1650
1651	setup_tx_buf(eth, tx_buf, txd: txd_pdma, mapped_addr: txd_info.addr,
1652	size: txd_info.size, idx: k++);
1653
1654	frag_size -= txd_info.size;
1655	offset += txd_info.size;
1656	}
1657	}
1658
1659	/* store skb to cleanup */
1660	itx_buf->type = MTK_TYPE_SKB;
1661	itx_buf->data = skb;
1662
1663	if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA)) {
1664	if (k & 0x1)
1665	txd_pdma->txd2 |= TX_DMA_LS0;
1666	else
1667	txd_pdma->txd2 |= TX_DMA_LS1;
1668	}
1669
1670	netdev_tx_sent_queue(dev_queue: txq, bytes: skb->len);
1671	skb_tx_timestamp(skb);
1672
1673	ring->next_free = mtk_qdma_phys_to_virt(ring, desc: txd->txd2);
1674	atomic_sub(i: n_desc, v: &ring->free_count);
1675
1676	/* make sure that all changes to the dma ring are flushed before we
1677	* continue
1678	*/
1679	wmb();
1680
1681	if (MTK_HAS_CAPS(soc->caps, MTK_QDMA)) {
1682	if (netif_xmit_stopped(dev_queue: txq) || !netdev_xmit_more())
1683	mtk_w32(eth, val: txd->txd2, reg: soc->reg_map->qdma.ctx_ptr);
1684	} else {
1685	int next_idx;
1686
1687	next_idx = NEXT_DESP_IDX(txd_to_idx(ring, txd, soc->tx.desc_size),
1688	ring->dma_size);
1689	mtk_w32(eth, val: next_idx, MT7628_TX_CTX_IDX0);
1690	}
1691
1692	return 0;
1693
1694	err_dma:
1695	do {
1696	tx_buf = mtk_desc_to_tx_buf(ring, txd: itxd, txd_size: soc->tx.desc_size);
1697
1698	/* unmap dma */
1699	mtk_tx_unmap(eth, tx_buf, NULL, napi: false);
1700
1701	itxd->txd3 = TX_DMA_LS0 | TX_DMA_OWNER_CPU;
1702	if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA))
1703	itxd_pdma->txd2 = TX_DMA_DESP2_DEF;
1704
1705	itxd = mtk_qdma_phys_to_virt(ring, desc: itxd->txd2);
1706	itxd_pdma = qdma_to_pdma(ring, dma: itxd);
1707	} while (itxd != txd);
1708
1709	return -ENOMEM;
1710	}
1711
1712	static int mtk_cal_txd_req(struct mtk_eth *eth, struct sk_buff *skb)
1713	{
1714	int i, nfrags = 1;
1715	skb_frag_t *frag;
1716
1717	if (skb_is_gso(skb)) {
1718	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1719	frag = &skb_shinfo(skb)->frags[i];
1720	nfrags += DIV_ROUND_UP(skb_frag_size(frag),
1721	eth->soc->tx.dma_max_len);
1722	}
1723	} else {
1724	nfrags += skb_shinfo(skb)->nr_frags;
1725	}
1726
1727	return nfrags;
1728	}
1729
1730	static int mtk_queue_stopped(struct mtk_eth *eth)
1731	{
1732	int i;
1733
1734	for (i = 0; i < MTK_MAX_DEVS; i++) {
1735	if (!eth->netdev[i])
1736	continue;
1737	if (netif_queue_stopped(dev: eth->netdev[i]))
1738	return 1;
1739	}
1740
1741	return 0;
1742	}
1743
1744	static void mtk_wake_queue(struct mtk_eth *eth)
1745	{
1746	int i;
1747
1748	for (i = 0; i < MTK_MAX_DEVS; i++) {
1749	if (!eth->netdev[i])
1750	continue;
1751	netif_tx_wake_all_queues(dev: eth->netdev[i]);
1752	}
1753	}
1754
1755	static netdev_tx_t mtk_start_xmit(struct sk_buff *skb, struct net_device *dev)
1756	{
1757	struct mtk_mac *mac = netdev_priv(dev);
1758	struct mtk_eth *eth = mac->hw;
1759	struct mtk_tx_ring *ring = &eth->tx_ring;
1760	struct net_device_stats *stats = &dev->stats;
1761	bool gso = false;
1762	int tx_num;
1763
1764	if (skb_vlan_tag_present(skb) &&
1765	!eth_proto_is_802_3(proto: eth_hdr(skb)->h_proto)) {
1766	skb = __vlan_hwaccel_push_inside(skb);
1767	if (!skb)
1768	goto dropped;
1769	}
1770
1771	/* normally we can rely on the stack not calling this more than once,
1772	* however we have 2 queues running on the same ring so we need to lock
1773	* the ring access
1774	*/
1775	spin_lock(lock: &eth->page_lock);
1776
1777	if (unlikely(test_bit(MTK_RESETTING, &eth->state)))
1778	goto drop;
1779
1780	tx_num = mtk_cal_txd_req(eth, skb);
1781	if (unlikely(atomic_read(&ring->free_count) <= tx_num)) {
1782	netif_tx_stop_all_queues(dev);
1783	netif_err(eth, tx_queued, dev,
1784	"Tx Ring full when queue awake!\n");
1785	spin_unlock(lock: &eth->page_lock);
1786	return NETDEV_TX_BUSY;
1787	}
1788
1789	/* TSO: fill MSS info in tcp checksum field */
1790	if (skb_is_gso(skb)) {
1791	if (skb_cow_head(skb, headroom: 0)) {
1792	netif_warn(eth, tx_err, dev,
1793	"GSO expand head fail.\n");
1794	goto drop;
1795	}
1796
1797	if (skb_shinfo(skb)->gso_type &
1798	(SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) {
1799	gso = true;
1800	tcp_hdr(skb)->check = htons(skb_shinfo(skb)->gso_size);
1801	}
1802	}
1803
1804	if (mtk_tx_map(skb, dev, tx_num, ring, gso) < 0)
1805	goto drop;
1806
1807	if (unlikely(atomic_read(&ring->free_count) <= ring->thresh))
1808	netif_tx_stop_all_queues(dev);
1809
1810	spin_unlock(lock: &eth->page_lock);
1811
1812	return NETDEV_TX_OK;
1813
1814	drop:
1815	spin_unlock(lock: &eth->page_lock);
1816	dev_kfree_skb_any(skb);
1817	dropped:
1818	stats->tx_dropped++;
1819	return NETDEV_TX_OK;
1820	}
1821
1822	static struct mtk_rx_ring *mtk_get_rx_ring(struct mtk_eth *eth)
1823	{
1824	int i;
1825	struct mtk_rx_ring *ring;
1826	int idx;
1827
1828	if (!eth->hwlro)
1829	return &eth->rx_ring[0];
1830
1831	for (i = 0; i < MTK_MAX_RX_RING_NUM; i++) {
1832	struct mtk_rx_dma *rxd;
1833
1834	ring = &eth->rx_ring[i];
1835	idx = NEXT_DESP_IDX(ring->calc_idx, ring->dma_size);
1836	rxd = ring->dma + idx * eth->soc->rx.desc_size;
1837	if (rxd->rxd2 & RX_DMA_DONE) {
1838	ring->calc_idx_update = true;
1839	return ring;
1840	}
1841	}
1842
1843	return NULL;
1844	}
1845
1846	static void mtk_update_rx_cpu_idx(struct mtk_eth *eth)
1847	{
1848	struct mtk_rx_ring *ring;
1849	int i;
1850
1851	if (!eth->hwlro) {
1852	ring = &eth->rx_ring[0];
1853	mtk_w32(eth, val: ring->calc_idx, reg: ring->crx_idx_reg);
1854	} else {
1855	for (i = 0; i < MTK_MAX_RX_RING_NUM; i++) {
1856	ring = &eth->rx_ring[i];
1857	if (ring->calc_idx_update) {
1858	ring->calc_idx_update = false;
1859	mtk_w32(eth, val: ring->calc_idx, reg: ring->crx_idx_reg);
1860	}
1861	}
1862	}
1863	}
1864
1865	static bool mtk_page_pool_enabled(struct mtk_eth *eth)
1866	{
1867	return mtk_is_netsys_v2_or_greater(eth);
1868	}
1869
1870	static struct page_pool *mtk_create_page_pool(struct mtk_eth *eth,
1871	struct xdp_rxq_info *xdp_q,
1872	int id, int size)
1873	{
1874	struct page_pool_params pp_params = {
1875	.order = 0,
1876	.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
1877	.pool_size = size,
1878	.nid = NUMA_NO_NODE,
1879	.dev = eth->dma_dev,
1880	.offset = MTK_PP_HEADROOM,
1881	.max_len = MTK_PP_MAX_BUF_SIZE,
1882	};
1883	struct page_pool *pp;
1884	int err;
1885
1886	pp_params.dma_dir = rcu_access_pointer(eth->prog) ? DMA_BIDIRECTIONAL
1887	: DMA_FROM_DEVICE;
1888	pp = page_pool_create(params: &pp_params);
1889	if (IS_ERR(ptr: pp))
1890	return pp;
1891
1892	err = __xdp_rxq_info_reg(xdp_rxq: xdp_q, dev: eth->dummy_dev, queue_index: id,
1893	napi_id: eth->rx_napi.napi_id, PAGE_SIZE);
1894	if (err < 0)
1895	goto err_free_pp;
1896
1897	err = xdp_rxq_info_reg_mem_model(xdp_rxq: xdp_q, type: MEM_TYPE_PAGE_POOL, allocator: pp);
1898	if (err)
1899	goto err_unregister_rxq;
1900
1901	return pp;
1902
1903	err_unregister_rxq:
1904	xdp_rxq_info_unreg(xdp_rxq: xdp_q);
1905	err_free_pp:
1906	page_pool_destroy(pool: pp);
1907
1908	return ERR_PTR(error: err);
1909	}
1910
1911	static void *mtk_page_pool_get_buff(struct page_pool *pp, dma_addr_t *dma_addr,
1912	gfp_t gfp_mask)
1913	{
1914	struct page *page;
1915
1916	page = page_pool_alloc_pages(pool: pp, gfp: gfp_mask | __GFP_NOWARN);
1917	if (!page)
1918	return NULL;
1919
1920	*dma_addr = page_pool_get_dma_addr(page) + MTK_PP_HEADROOM;
1921	return page_address(page);
1922	}
1923
1924	static void mtk_rx_put_buff(struct mtk_rx_ring *ring, void *data, bool napi)
1925	{
1926	if (ring->page_pool)
1927	page_pool_put_full_page(pool: ring->page_pool,
1928	page: virt_to_head_page(x: data), allow_direct: napi);
1929	else
1930	skb_free_frag(addr: data);
1931	}
1932
1933	static int mtk_xdp_frame_map(struct mtk_eth *eth, struct net_device *dev,
1934	struct mtk_tx_dma_desc_info *txd_info,
1935	struct mtk_tx_dma *txd, struct mtk_tx_buf *tx_buf,
1936	void *data, u16 headroom, int index, bool dma_map)
1937	{
1938	struct mtk_tx_ring *ring = &eth->tx_ring;
1939	struct mtk_mac *mac = netdev_priv(dev);
1940	struct mtk_tx_dma *txd_pdma;
1941
1942	if (dma_map) { /* ndo_xdp_xmit */
1943	txd_info->addr = dma_map_single(eth->dma_dev, data,
1944	txd_info->size, DMA_TO_DEVICE);
1945	if (unlikely(dma_mapping_error(eth->dma_dev, txd_info->addr)))
1946	return -ENOMEM;
1947
1948	tx_buf->flags |= MTK_TX_FLAGS_SINGLE0;
1949	} else {
1950	struct page *page = virt_to_head_page(x: data);
1951
1952	txd_info->addr = page_pool_get_dma_addr(page) +
1953	sizeof(struct xdp_frame) + headroom;
1954	dma_sync_single_for_device(dev: eth->dma_dev, addr: txd_info->addr,
1955	size: txd_info->size, dir: DMA_BIDIRECTIONAL);
1956	}
1957	mtk_tx_set_dma_desc(dev, txd, info: txd_info);
1958
1959	tx_buf->mac_id = mac->id;
1960	tx_buf->type = dma_map ? MTK_TYPE_XDP_NDO : MTK_TYPE_XDP_TX;
1961	tx_buf->data = (void *)MTK_DMA_DUMMY_DESC;
1962
1963	txd_pdma = qdma_to_pdma(ring, dma: txd);
1964	setup_tx_buf(eth, tx_buf, txd: txd_pdma, mapped_addr: txd_info->addr, size: txd_info->size,
1965	idx: index);
1966
1967	return 0;
1968	}
1969
1970	static int mtk_xdp_submit_frame(struct mtk_eth *eth, struct xdp_frame *xdpf,
1971	struct net_device *dev, bool dma_map)
1972	{
1973	struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(frame: xdpf);
1974	const struct mtk_soc_data *soc = eth->soc;
1975	struct mtk_tx_ring *ring = &eth->tx_ring;
1976	struct mtk_mac *mac = netdev_priv(dev);
1977	struct mtk_tx_dma_desc_info txd_info = {
1978	.size = xdpf->len,
1979	.first = true,
1980	.last = !xdp_frame_has_frags(frame: xdpf),
1981	.qid = mac->id,
1982	};
1983	int err, index = 0, n_desc = 1, nr_frags;
1984	struct mtk_tx_buf *htx_buf, *tx_buf;
1985	struct mtk_tx_dma *htxd, *txd;
1986	void *data = xdpf->data;
1987
1988	if (unlikely(test_bit(MTK_RESETTING, &eth->state)))
1989	return -EBUSY;
1990
1991	nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0;
1992	if (unlikely(atomic_read(&ring->free_count) <= 1 + nr_frags))
1993	return -EBUSY;
1994
1995	spin_lock(lock: &eth->page_lock);
1996
1997	txd = ring->next_free;
1998	if (txd == ring->last_free) {
1999	spin_unlock(lock: &eth->page_lock);
2000	return -ENOMEM;
2001	}
2002	htxd = txd;
2003
2004	tx_buf = mtk_desc_to_tx_buf(ring, txd, txd_size: soc->tx.desc_size);
2005	memset(tx_buf, 0, sizeof(*tx_buf));
2006	htx_buf = tx_buf;
2007
2008	for (;;) {
2009	err = mtk_xdp_frame_map(eth, dev, txd_info: &txd_info, txd, tx_buf,
2010	data, headroom: xdpf->headroom, index, dma_map);
2011	if (err < 0)
2012	goto unmap;
2013
2014	if (txd_info.last)
2015	break;
2016
2017	if (MTK_HAS_CAPS(soc->caps, MTK_QDMA) || (index & 0x1)) {
2018	txd = mtk_qdma_phys_to_virt(ring, desc: txd->txd2);
2019	if (txd == ring->last_free)
2020	goto unmap;
2021
2022	tx_buf = mtk_desc_to_tx_buf(ring, txd,
2023	txd_size: soc->tx.desc_size);
2024	memset(tx_buf, 0, sizeof(*tx_buf));
2025	n_desc++;
2026	}
2027
2028	memset(&txd_info, 0, sizeof(struct mtk_tx_dma_desc_info));
2029	txd_info.size = skb_frag_size(frag: &sinfo->frags[index]);
2030	txd_info.last = index + 1 == nr_frags;
2031	txd_info.qid = mac->id;
2032	data = skb_frag_address(frag: &sinfo->frags[index]);
2033
2034	index++;
2035	}
2036	/* store xdpf for cleanup */
2037	htx_buf->data = xdpf;
2038
2039	if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA)) {
2040	struct mtk_tx_dma *txd_pdma = qdma_to_pdma(ring, dma: txd);
2041
2042	if (index & 1)
2043	txd_pdma->txd2 |= TX_DMA_LS0;
2044	else
2045	txd_pdma->txd2 |= TX_DMA_LS1;
2046	}
2047
2048	ring->next_free = mtk_qdma_phys_to_virt(ring, desc: txd->txd2);
2049	atomic_sub(i: n_desc, v: &ring->free_count);
2050
2051	/* make sure that all changes to the dma ring are flushed before we
2052	* continue
2053	*/
2054	wmb();
2055
2056	if (MTK_HAS_CAPS(soc->caps, MTK_QDMA)) {
2057	mtk_w32(eth, val: txd->txd2, reg: soc->reg_map->qdma.ctx_ptr);
2058	} else {
2059	int idx;
2060
2061	idx = txd_to_idx(ring, dma: txd, txd_size: soc->tx.desc_size);
2062	mtk_w32(eth, NEXT_DESP_IDX(idx, ring->dma_size),
2063	MT7628_TX_CTX_IDX0);
2064	}
2065
2066	spin_unlock(lock: &eth->page_lock);
2067
2068	return 0;
2069
2070	unmap:
2071	while (htxd != txd) {
2072	tx_buf = mtk_desc_to_tx_buf(ring, txd: htxd, txd_size: soc->tx.desc_size);
2073	mtk_tx_unmap(eth, tx_buf, NULL, napi: false);
2074
2075	htxd->txd3 = TX_DMA_LS0 | TX_DMA_OWNER_CPU;
2076	if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA)) {
2077	struct mtk_tx_dma *txd_pdma = qdma_to_pdma(ring, dma: htxd);
2078
2079	txd_pdma->txd2 = TX_DMA_DESP2_DEF;
2080	}
2081
2082	htxd = mtk_qdma_phys_to_virt(ring, desc: htxd->txd2);
2083	}
2084
2085	spin_unlock(lock: &eth->page_lock);
2086
2087	return err;
2088	}
2089
2090	static int mtk_xdp_xmit(struct net_device *dev, int num_frame,
2091	struct xdp_frame **frames, u32 flags)
2092	{
2093	struct mtk_mac *mac = netdev_priv(dev);
2094	struct mtk_hw_stats *hw_stats = mac->hw_stats;
2095	struct mtk_eth *eth = mac->hw;
2096	int i, nxmit = 0;
2097
2098	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
2099	return -EINVAL;
2100
2101	for (i = 0; i < num_frame; i++) {
2102	if (mtk_xdp_submit_frame(eth, xdpf: frames[i], dev, dma_map: true))
2103	break;
2104	nxmit++;
2105	}
2106
2107	u64_stats_update_begin(syncp: &hw_stats->syncp);
2108	hw_stats->xdp_stats.tx_xdp_xmit += nxmit;
2109	hw_stats->xdp_stats.tx_xdp_xmit_errors += num_frame - nxmit;
2110	u64_stats_update_end(syncp: &hw_stats->syncp);
2111
2112	return nxmit;
2113	}
2114
2115	static u32 mtk_xdp_run(struct mtk_eth *eth, struct mtk_rx_ring *ring,
2116	struct xdp_buff *xdp, struct net_device *dev)
2117	{
2118	struct mtk_mac *mac = netdev_priv(dev);
2119	struct mtk_hw_stats *hw_stats = mac->hw_stats;
2120	u64 *count = &hw_stats->xdp_stats.rx_xdp_drop;
2121	struct bpf_prog *prog;
2122	u32 act = XDP_PASS;
2123
2124	rcu_read_lock();
2125
2126	prog = rcu_dereference(eth->prog);
2127	if (!prog)
2128	goto out;
2129
2130	act = bpf_prog_run_xdp(prog, xdp);
2131	switch (act) {
2132	case XDP_PASS:
2133	count = &hw_stats->xdp_stats.rx_xdp_pass;
2134	goto update_stats;
2135	case XDP_REDIRECT:
2136	if (unlikely(xdp_do_redirect(dev, xdp, prog))) {
2137	act = XDP_DROP;
2138	break;
2139	}
2140
2141	count = &hw_stats->xdp_stats.rx_xdp_redirect;
2142	goto update_stats;
2143	case XDP_TX: {
2144	struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
2145
2146	if (!xdpf || mtk_xdp_submit_frame(eth, xdpf, dev, dma_map: false)) {
2147	count = &hw_stats->xdp_stats.rx_xdp_tx_errors;
2148	act = XDP_DROP;
2149	break;
2150	}
2151
2152	count = &hw_stats->xdp_stats.rx_xdp_tx;
2153	goto update_stats;
2154	}
2155	default:
2156	bpf_warn_invalid_xdp_action(dev, prog, act);
2157	fallthrough;
2158	case XDP_ABORTED:
2159	trace_xdp_exception(dev, xdp: prog, act);
2160	fallthrough;
2161	case XDP_DROP:
2162	break;
2163	}
2164
2165	page_pool_put_full_page(pool: ring->page_pool,
2166	page: virt_to_head_page(x: xdp->data), allow_direct: true);
2167
2168	update_stats:
2169	u64_stats_update_begin(syncp: &hw_stats->syncp);
2170	*count = *count + 1;
2171	u64_stats_update_end(syncp: &hw_stats->syncp);
2172	out:
2173	rcu_read_unlock();
2174
2175	return act;
2176	}
2177
2178	static int mtk_poll_rx(struct napi_struct *napi, int budget,
2179	struct mtk_eth *eth)
2180	{
2181	struct dim_sample dim_sample = {};
2182	struct mtk_rx_ring *ring;
2183	bool xdp_flush = false;
2184	int idx;
2185	struct sk_buff *skb;
2186	u64 addr64 = 0;
2187	u8 *data, *new_data;
2188	struct mtk_rx_dma_v2 *rxd, trxd;
2189	int done = 0, bytes = 0;
2190	dma_addr_t dma_addr = DMA_MAPPING_ERROR;
2191	int ppe_idx = 0;
2192
2193	while (done < budget) {
2194	unsigned int pktlen, *rxdcsum;
2195	struct net_device *netdev;
2196	u32 hash, reason;
2197	int mac = 0;
2198
2199	ring = mtk_get_rx_ring(eth);
2200	if (unlikely(!ring))
2201	goto rx_done;
2202
2203	idx = NEXT_DESP_IDX(ring->calc_idx, ring->dma_size);
2204	rxd = ring->dma + idx * eth->soc->rx.desc_size;
2205	data = ring->data[idx];
2206
2207	if (!mtk_rx_get_desc(eth, rxd: &trxd, dma_rxd: rxd))
2208	break;
2209
2210	/* find out which mac the packet come from. values start at 1 */
2211	if (mtk_is_netsys_v3_or_greater(eth)) {
2212	u32 val = RX_DMA_GET_SPORT_V2(trxd.rxd5);
2213
2214	switch (val) {
2215	case PSE_GDM1_PORT:
2216	case PSE_GDM2_PORT:
2217	mac = val - 1;
2218	break;
2219	case PSE_GDM3_PORT:
2220	mac = MTK_GMAC3_ID;
2221	break;
2222	default:
2223	break;
2224	}
2225	} else if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628) &&
2226	!(trxd.rxd4 & RX_DMA_SPECIAL_TAG)) {
2227	mac = RX_DMA_GET_SPORT(trxd.rxd4) - 1;
2228	}
2229
2230	if (unlikely(mac < 0 || mac >= MTK_MAX_DEVS ||
2231	!eth->netdev[mac]))
2232	goto release_desc;
2233
2234	netdev = eth->netdev[mac];
2235	ppe_idx = eth->mac[mac]->ppe_idx;
2236
2237	if (unlikely(test_bit(MTK_RESETTING, &eth->state)))
2238	goto release_desc;
2239
2240	pktlen = RX_DMA_GET_PLEN0(trxd.rxd2);
2241
2242	/* alloc new buffer */
2243	if (ring->page_pool) {
2244	struct page *page = virt_to_head_page(x: data);
2245	struct xdp_buff xdp;
2246	u32 ret, metasize;
2247
2248	new_data = mtk_page_pool_get_buff(pp: ring->page_pool,
2249	dma_addr: &dma_addr,
2250	GFP_ATOMIC);
2251	if (unlikely(!new_data)) {
2252	netdev->stats.rx_dropped++;
2253	goto release_desc;
2254	}
2255
2256	dma_sync_single_for_cpu(dev: eth->dma_dev,
2257	addr: page_pool_get_dma_addr(page) + MTK_PP_HEADROOM,
2258	size: pktlen, dir: page_pool_get_dma_dir(pool: ring->page_pool));
2259
2260	xdp_init_buff(xdp: &xdp, PAGE_SIZE, rxq: &ring->xdp_q);
2261	xdp_prepare_buff(xdp: &xdp, hard_start: data, MTK_PP_HEADROOM, data_len: pktlen,
2262	meta_valid: true);
2263	xdp_buff_clear_frags_flag(xdp: &xdp);
2264
2265	ret = mtk_xdp_run(eth, ring, xdp: &xdp, dev: netdev);
2266	if (ret == XDP_REDIRECT)
2267	xdp_flush = true;
2268
2269	if (ret != XDP_PASS)
2270	goto skip_rx;
2271
2272	skb = build_skb(data, PAGE_SIZE);
2273	if (unlikely(!skb)) {
2274	page_pool_put_full_page(pool: ring->page_pool,
2275	page, allow_direct: true);
2276	netdev->stats.rx_dropped++;
2277	goto skip_rx;
2278	}
2279
2280	skb_reserve(skb, len: xdp.data - xdp.data_hard_start);
2281	skb_put(skb, len: xdp.data_end - xdp.data);
2282	metasize = xdp.data - xdp.data_meta;
2283	if (metasize)
2284	skb_metadata_set(skb, meta_len: metasize);
2285	skb_mark_for_recycle(skb);
2286	} else {
2287	if (ring->frag_size <= PAGE_SIZE)
2288	new_data = napi_alloc_frag(fragsz: ring->frag_size);
2289	else
2290	new_data = mtk_max_lro_buf_alloc(GFP_ATOMIC);
2291
2292	if (unlikely(!new_data)) {
2293	netdev->stats.rx_dropped++;
2294	goto release_desc;
2295	}
2296
2297	dma_addr = dma_map_single(eth->dma_dev,
2298	new_data + NET_SKB_PAD + eth->ip_align,
2299	ring->buf_size, DMA_FROM_DEVICE);
2300	if (unlikely(dma_mapping_error(eth->dma_dev,
2301	dma_addr))) {
2302	skb_free_frag(addr: new_data);
2303	netdev->stats.rx_dropped++;
2304	goto release_desc;
2305	}
2306
2307	if (MTK_HAS_CAPS(eth->soc->caps, MTK_36BIT_DMA))
2308	addr64 = RX_DMA_GET_ADDR64(trxd.rxd2);
2309
2310	dma_unmap_single(eth->dma_dev, ((u64)trxd.rxd1 | addr64),
2311	ring->buf_size, DMA_FROM_DEVICE);
2312
2313	skb = build_skb(data, frag_size: ring->frag_size);
2314	if (unlikely(!skb)) {
2315	netdev->stats.rx_dropped++;
2316	skb_free_frag(addr: data);
2317	goto skip_rx;
2318	}
2319
2320	skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
2321	skb_put(skb, len: pktlen);
2322	}
2323
2324	skb->dev = netdev;
2325	bytes += skb->len;
2326
2327	if (mtk_is_netsys_v3_or_greater(eth)) {
2328	reason = FIELD_GET(MTK_RXD5_PPE_CPU_REASON, trxd.rxd5);
2329	hash = trxd.rxd5 & MTK_RXD5_FOE_ENTRY;
2330	if (hash != MTK_RXD5_FOE_ENTRY)
2331	skb_set_hash(skb, hash: jhash_1word(a: hash, initval: 0),
2332	type: PKT_HASH_TYPE_L4);
2333	rxdcsum = &trxd.rxd3;
2334	} else {
2335	reason = FIELD_GET(MTK_RXD4_PPE_CPU_REASON, trxd.rxd4);
2336	hash = trxd.rxd4 & MTK_RXD4_FOE_ENTRY;
2337	if (hash != MTK_RXD4_FOE_ENTRY)
2338	skb_set_hash(skb, hash: jhash_1word(a: hash, initval: 0),
2339	type: PKT_HASH_TYPE_L4);
2340	rxdcsum = &trxd.rxd4;
2341	}
2342
2343	if (*rxdcsum & eth->soc->rx.dma_l4_valid)
2344	skb->ip_summed = CHECKSUM_UNNECESSARY;
2345	else
2346	skb_checksum_none_assert(skb);
2347	skb->protocol = eth_type_trans(skb, dev: netdev);
2348
2349	/* When using VLAN untagging in combination with DSA, the
2350	* hardware treats the MTK special tag as a VLAN and untags it.
2351	*/
2352	if (mtk_is_netsys_v1(eth) && (trxd.rxd2 & RX_DMA_VTAG) &&
2353	netdev_uses_dsa(dev: netdev)) {
2354	unsigned int port = RX_DMA_VPID(trxd.rxd3) & GENMASK(2, 0);
2355
2356	if (port < ARRAY_SIZE(eth->dsa_meta) &&
2357	eth->dsa_meta[port])
2358	skb_dst_set_noref(skb, dst: &eth->dsa_meta[port]->dst);
2359	}
2360
2361	if (reason == MTK_PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED)
2362	mtk_ppe_check_skb(ppe: eth->ppe[ppe_idx], skb, hash);
2363
2364	skb_record_rx_queue(skb, rx_queue: 0);
2365	napi_gro_receive(napi, skb);
2366
2367	skip_rx:
2368	ring->data[idx] = new_data;
2369	rxd->rxd1 = (unsigned int)dma_addr;
2370	release_desc:
2371	if (MTK_HAS_CAPS(eth->soc->caps, MTK_36BIT_DMA)) {
2372	if (unlikely(dma_addr == DMA_MAPPING_ERROR))
2373	addr64 = FIELD_GET(RX_DMA_ADDR64_MASK,
2374	rxd->rxd2);
2375	else
2376	addr64 = RX_DMA_PREP_ADDR64(dma_addr);
2377	}
2378
2379	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
2380	rxd->rxd2 = RX_DMA_LSO;
2381	else
2382	rxd->rxd2 = RX_DMA_PREP_PLEN0(ring->buf_size) | addr64;
2383
2384	ring->calc_idx = idx;
2385	done++;
2386	}
2387
2388	rx_done:
2389	if (done) {
2390	/* make sure that all changes to the dma ring are flushed before
2391	* we continue
2392	*/
2393	wmb();
2394	mtk_update_rx_cpu_idx(eth);
2395	}
2396
2397	eth->rx_packets += done;
2398	eth->rx_bytes += bytes;
2399	dim_update_sample(event_ctr: eth->rx_events, packets: eth->rx_packets, bytes: eth->rx_bytes,
2400	s: &dim_sample);
2401	net_dim(dim: &eth->rx_dim, end_sample: &dim_sample);
2402
2403	if (xdp_flush)
2404	xdp_do_flush();
2405
2406	return done;
2407	}
2408
2409	struct mtk_poll_state {
2410	struct netdev_queue *txq;
2411	unsigned int total;
2412	unsigned int done;
2413	unsigned int bytes;
2414	};
2415
2416	static void
2417	mtk_poll_tx_done(struct mtk_eth *eth, struct mtk_poll_state *state, u8 mac,
2418	struct sk_buff *skb)
2419	{
2420	struct netdev_queue *txq;
2421	struct net_device *dev;
2422	unsigned int bytes = skb->len;
2423
2424	state->total++;
2425	eth->tx_packets++;
2426	eth->tx_bytes += bytes;
2427
2428	dev = eth->netdev[mac];
2429	if (!dev)
2430	return;
2431
2432	txq = netdev_get_tx_queue(dev, index: skb_get_queue_mapping(skb));
2433	if (state->txq == txq) {
2434	state->done++;
2435	state->bytes += bytes;
2436	return;
2437	}
2438
2439	if (state->txq)
2440	netdev_tx_completed_queue(dev_queue: state->txq, pkts: state->done, bytes: state->bytes);
2441
2442	state->txq = txq;
2443	state->done = 1;
2444	state->bytes = bytes;
2445	}
2446
2447	static int mtk_poll_tx_qdma(struct mtk_eth *eth, int budget,
2448	struct mtk_poll_state *state)
2449	{
2450	const struct mtk_reg_map *reg_map = eth->soc->reg_map;
2451	struct mtk_tx_ring *ring = &eth->tx_ring;
2452	struct mtk_tx_buf *tx_buf;
2453	struct xdp_frame_bulk bq;
2454	struct mtk_tx_dma *desc;
2455	u32 cpu, dma;
2456
2457	cpu = ring->last_free_ptr;
2458	dma = mtk_r32(eth, reg: reg_map->qdma.drx_ptr);
2459
2460	desc = mtk_qdma_phys_to_virt(ring, desc: cpu);
2461	xdp_frame_bulk_init(bq: &bq);
2462
2463	while ((cpu != dma) && budget) {
2464	u32 next_cpu = desc->txd2;
2465
2466	desc = mtk_qdma_phys_to_virt(ring, desc: desc->txd2);
2467	if ((desc->txd3 & TX_DMA_OWNER_CPU) == 0)
2468	break;
2469
2470	tx_buf = mtk_desc_to_tx_buf(ring, txd: desc,
2471	txd_size: eth->soc->tx.desc_size);
2472	if (!tx_buf->data)
2473	break;
2474
2475	if (tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) {
2476	if (tx_buf->type == MTK_TYPE_SKB)
2477	mtk_poll_tx_done(eth, state, mac: tx_buf->mac_id,
2478	skb: tx_buf->data);
2479
2480	budget--;
2481	}
2482	mtk_tx_unmap(eth, tx_buf, bq: &bq, napi: true);
2483
2484	ring->last_free = desc;
2485	atomic_inc(v: &ring->free_count);
2486
2487	cpu = next_cpu;
2488	}
2489	xdp_flush_frame_bulk(bq: &bq);
2490
2491	ring->last_free_ptr = cpu;
2492	mtk_w32(eth, val: cpu, reg: reg_map->qdma.crx_ptr);
2493
2494	return budget;
2495	}
2496
2497	static int mtk_poll_tx_pdma(struct mtk_eth *eth, int budget,
2498	struct mtk_poll_state *state)
2499	{
2500	struct mtk_tx_ring *ring = &eth->tx_ring;
2501	struct mtk_tx_buf *tx_buf;
2502	struct xdp_frame_bulk bq;
2503	struct mtk_tx_dma *desc;
2504	u32 cpu, dma;
2505
2506	cpu = ring->cpu_idx;
2507	dma = mtk_r32(eth, MT7628_TX_DTX_IDX0);
2508	xdp_frame_bulk_init(bq: &bq);
2509
2510	while ((cpu != dma) && budget) {
2511	tx_buf = &ring->buf[cpu];
2512	if (!tx_buf->data)
2513	break;
2514
2515	if (tx_buf->data != (void *)MTK_DMA_DUMMY_DESC) {
2516	if (tx_buf->type == MTK_TYPE_SKB)
2517	mtk_poll_tx_done(eth, state, mac: 0, skb: tx_buf->data);
2518	budget--;
2519	}
2520	mtk_tx_unmap(eth, tx_buf, bq: &bq, napi: true);
2521
2522	desc = ring->dma + cpu * eth->soc->tx.desc_size;
2523	ring->last_free = desc;
2524	atomic_inc(v: &ring->free_count);
2525
2526	cpu = NEXT_DESP_IDX(cpu, ring->dma_size);
2527	}
2528	xdp_flush_frame_bulk(bq: &bq);
2529
2530	ring->cpu_idx = cpu;
2531
2532	return budget;
2533	}
2534
2535	static int mtk_poll_tx(struct mtk_eth *eth, int budget)
2536	{
2537	struct mtk_tx_ring *ring = &eth->tx_ring;
2538	struct dim_sample dim_sample = {};
2539	struct mtk_poll_state state = {};
2540
2541	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
2542	budget = mtk_poll_tx_qdma(eth, budget, state: &state);
2543	else
2544	budget = mtk_poll_tx_pdma(eth, budget, state: &state);
2545
2546	if (state.txq)
2547	netdev_tx_completed_queue(dev_queue: state.txq, pkts: state.done, bytes: state.bytes);
2548
2549	dim_update_sample(event_ctr: eth->tx_events, packets: eth->tx_packets, bytes: eth->tx_bytes,
2550	s: &dim_sample);
2551	net_dim(dim: &eth->tx_dim, end_sample: &dim_sample);
2552
2553	if (mtk_queue_stopped(eth) &&
2554	(atomic_read(v: &ring->free_count) > ring->thresh))
2555	mtk_wake_queue(eth);
2556
2557	return state.total;
2558	}
2559
2560	static void mtk_handle_status_irq(struct mtk_eth *eth)
2561	{
2562	u32 status2 = mtk_r32(eth, MTK_INT_STATUS2);
2563
2564	if (unlikely(status2 & (MTK_GDM1_AF | MTK_GDM2_AF))) {
2565	mtk_stats_update(eth);
2566	mtk_w32(eth, val: (MTK_GDM1_AF | MTK_GDM2_AF),
2567	MTK_INT_STATUS2);
2568	}
2569	}
2570
2571	static int mtk_napi_tx(struct napi_struct *napi, int budget)
2572	{
2573	struct mtk_eth *eth = container_of(napi, struct mtk_eth, tx_napi);
2574	const struct mtk_reg_map *reg_map = eth->soc->reg_map;
2575	int tx_done = 0;
2576
2577	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
2578	mtk_handle_status_irq(eth);
2579	mtk_w32(eth, MTK_TX_DONE_INT, reg: reg_map->tx_irq_status);
2580	tx_done = mtk_poll_tx(eth, budget);
2581
2582	if (unlikely(netif_msg_intr(eth))) {
2583	dev_info(eth->dev,
2584	"done tx %d, intr 0x%08x/0x%x\n", tx_done,
2585	mtk_r32(eth, reg_map->tx_irq_status),
2586	mtk_r32(eth, reg_map->tx_irq_mask));
2587	}
2588
2589	if (tx_done == budget)
2590	return budget;
2591
2592	if (mtk_r32(eth, reg: reg_map->tx_irq_status) & MTK_TX_DONE_INT)
2593	return budget;
2594
2595	if (napi_complete_done(n: napi, work_done: tx_done))
2596	mtk_tx_irq_enable(eth, MTK_TX_DONE_INT);
2597
2598	return tx_done;
2599	}
2600
2601	static int mtk_napi_rx(struct napi_struct *napi, int budget)
2602	{
2603	struct mtk_eth *eth = container_of(napi, struct mtk_eth, rx_napi);
2604	const struct mtk_reg_map *reg_map = eth->soc->reg_map;
2605	int rx_done_total = 0;
2606
2607	mtk_handle_status_irq(eth);
2608
2609	do {
2610	int rx_done;
2611
2612	mtk_w32(eth, val: eth->soc->rx.irq_done_mask,
2613	reg: reg_map->pdma.irq_status);
2614	rx_done = mtk_poll_rx(napi, budget: budget - rx_done_total, eth);
2615	rx_done_total += rx_done;
2616
2617	if (unlikely(netif_msg_intr(eth))) {
2618	dev_info(eth->dev,
2619	"done rx %d, intr 0x%08x/0x%x\n", rx_done,
2620	mtk_r32(eth, reg_map->pdma.irq_status),
2621	mtk_r32(eth, reg_map->pdma.irq_mask));
2622	}
2623
2624	if (rx_done_total == budget)
2625	return budget;
2626
2627	} while (mtk_r32(eth, reg: reg_map->pdma.irq_status) &
2628	eth->soc->rx.irq_done_mask);
2629
2630	if (napi_complete_done(n: napi, work_done: rx_done_total))
2631	mtk_rx_irq_enable(eth, mask: eth->soc->rx.irq_done_mask);
2632
2633	return rx_done_total;
2634	}
2635
2636	static int mtk_tx_alloc(struct mtk_eth *eth)
2637	{
2638	const struct mtk_soc_data *soc = eth->soc;
2639	struct mtk_tx_ring *ring = &eth->tx_ring;
2640	int i, sz = soc->tx.desc_size;
2641	struct mtk_tx_dma_v2 *txd;
2642	int ring_size;
2643	u32 ofs, val;
2644
2645	if (MTK_HAS_CAPS(soc->caps, MTK_QDMA))
2646	ring_size = MTK_QDMA_RING_SIZE;
2647	else
2648	ring_size = soc->tx.dma_size;
2649
2650	ring->buf = kcalloc(ring_size, sizeof(*ring->buf),
2651	GFP_KERNEL);
2652	if (!ring->buf)
2653	goto no_tx_mem;
2654
2655	ring->dma = mtk_dma_ring_alloc(eth, size: ring_size * sz, dma_handle: &ring->phys, use_sram: true);
2656	if (!ring->dma)
2657	goto no_tx_mem;
2658
2659	for (i = 0; i < ring_size; i++) {
2660	int next = (i + 1) % ring_size;
2661	u32 next_ptr = ring->phys + next * sz;
2662
2663	txd = ring->dma + i * sz;
2664	txd->txd2 = next_ptr;
2665	txd->txd3 = TX_DMA_LS0 | TX_DMA_OWNER_CPU;
2666	txd->txd4 = 0;
2667	if (mtk_is_netsys_v2_or_greater(eth)) {
2668	txd->txd5 = 0;
2669	txd->txd6 = 0;
2670	txd->txd7 = 0;
2671	txd->txd8 = 0;
2672	}
2673	}
2674
2675	/* On MT7688 (PDMA only) this driver uses the ring->dma structs
2676	* only as the framework. The real HW descriptors are the PDMA
2677	* descriptors in ring->dma_pdma.
2678	*/
2679	if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA)) {
2680	ring->dma_pdma = dma_alloc_coherent(dev: eth->dma_dev, size: ring_size * sz,
2681	dma_handle: &ring->phys_pdma, GFP_KERNEL);
2682	if (!ring->dma_pdma)
2683	goto no_tx_mem;
2684
2685	for (i = 0; i < ring_size; i++) {
2686	ring->dma_pdma[i].txd2 = TX_DMA_DESP2_DEF;
2687	ring->dma_pdma[i].txd4 = 0;
2688	}
2689	}
2690
2691	ring->dma_size = ring_size;
2692	atomic_set(v: &ring->free_count, i: ring_size - 2);
2693	ring->next_free = ring->dma;
2694	ring->last_free = (void *)txd;
2695	ring->last_free_ptr = (u32)(ring->phys + ((ring_size - 1) * sz));
2696	ring->thresh = MAX_SKB_FRAGS;
2697
2698	/* make sure that all changes to the dma ring are flushed before we
2699	* continue
2700	*/
2701	wmb();
2702
2703	if (MTK_HAS_CAPS(soc->caps, MTK_QDMA)) {
2704	mtk_w32(eth, val: ring->phys, reg: soc->reg_map->qdma.ctx_ptr);
2705	mtk_w32(eth, val: ring->phys, reg: soc->reg_map->qdma.dtx_ptr);
2706	mtk_w32(eth,
2707	val: ring->phys + ((ring_size - 1) * sz),
2708	reg: soc->reg_map->qdma.crx_ptr);
2709	mtk_w32(eth, val: ring->last_free_ptr, reg: soc->reg_map->qdma.drx_ptr);
2710
2711	for (i = 0, ofs = 0; i < MTK_QDMA_NUM_QUEUES; i++) {
2712	val = (QDMA_RES_THRES << 8) | QDMA_RES_THRES;
2713	mtk_w32(eth, val, reg: soc->reg_map->qdma.qtx_cfg + ofs);
2714
2715	val = MTK_QTX_SCH_MIN_RATE_EN |
2716	/* minimum: 10 Mbps */
2717	FIELD_PREP(MTK_QTX_SCH_MIN_RATE_MAN, 1) |
2718	FIELD_PREP(MTK_QTX_SCH_MIN_RATE_EXP, 4) |
2719	MTK_QTX_SCH_LEAKY_BUCKET_SIZE;
2720	if (mtk_is_netsys_v1(eth))
2721	val |= MTK_QTX_SCH_LEAKY_BUCKET_EN;
2722	mtk_w32(eth, val, reg: soc->reg_map->qdma.qtx_sch + ofs);
2723	ofs += MTK_QTX_OFFSET;
2724	}
2725	val = MTK_QDMA_TX_SCH_MAX_WFQ | (MTK_QDMA_TX_SCH_MAX_WFQ << 16);
2726	mtk_w32(eth, val, reg: soc->reg_map->qdma.tx_sch_rate);
2727	if (mtk_is_netsys_v2_or_greater(eth))
2728	mtk_w32(eth, val, reg: soc->reg_map->qdma.tx_sch_rate + 4);
2729	} else {
2730	mtk_w32(eth, val: ring->phys_pdma, MT7628_TX_BASE_PTR0);
2731	mtk_w32(eth, val: ring_size, MT7628_TX_MAX_CNT0);
2732	mtk_w32(eth, val: 0, MT7628_TX_CTX_IDX0);
2733	mtk_w32(eth, MT7628_PST_DTX_IDX0, reg: soc->reg_map->pdma.rst_idx);
2734	}
2735
2736	return 0;
2737
2738	no_tx_mem:
2739	return -ENOMEM;
2740	}
2741
2742	static void mtk_tx_clean(struct mtk_eth *eth)
2743	{
2744	const struct mtk_soc_data *soc = eth->soc;
2745	struct mtk_tx_ring *ring = &eth->tx_ring;
2746	int i;
2747
2748	if (ring->buf) {
2749	for (i = 0; i < ring->dma_size; i++)
2750	mtk_tx_unmap(eth, tx_buf: &ring->buf[i], NULL, napi: false);
2751	kfree(objp: ring->buf);
2752	ring->buf = NULL;
2753	}
2754
2755	if (ring->dma) {
2756	mtk_dma_ring_free(eth, size: ring->dma_size * soc->tx.desc_size,
2757	dma_ring: ring->dma, dma_handle: ring->phys, in_sram: true);
2758	ring->dma = NULL;
2759	}
2760
2761	if (ring->dma_pdma) {
2762	dma_free_coherent(dev: eth->dma_dev,
2763	size: ring->dma_size * soc->tx.desc_size,
2764	cpu_addr: ring->dma_pdma, dma_handle: ring->phys_pdma);
2765	ring->dma_pdma = NULL;
2766	}
2767	}
2768
2769	static int mtk_rx_alloc(struct mtk_eth *eth, int ring_no, int rx_flag)
2770	{
2771	const struct mtk_reg_map *reg_map = eth->soc->reg_map;
2772	const struct mtk_soc_data *soc = eth->soc;
2773	struct mtk_rx_ring *ring;
2774	int rx_data_len, rx_dma_size;
2775	int i;
2776
2777	if (rx_flag == MTK_RX_FLAGS_QDMA) {
2778	if (ring_no)
2779	return -EINVAL;
2780	ring = &eth->rx_ring_qdma;
2781	} else {
2782	ring = &eth->rx_ring[ring_no];
2783	}
2784
2785	if (rx_flag == MTK_RX_FLAGS_HWLRO) {
2786	rx_data_len = MTK_MAX_LRO_RX_LENGTH;
2787	rx_dma_size = MTK_HW_LRO_DMA_SIZE;
2788	} else {
2789	rx_data_len = ETH_DATA_LEN;
2790	rx_dma_size = soc->rx.dma_size;
2791	}
2792
2793	ring->frag_size = mtk_max_frag_size(mtu: rx_data_len);
2794	ring->buf_size = mtk_max_buf_size(frag_size: ring->frag_size);
2795	ring->data = kcalloc(rx_dma_size, sizeof(*ring->data),
2796	GFP_KERNEL);
2797	if (!ring->data)
2798	return -ENOMEM;
2799
2800	if (mtk_page_pool_enabled(eth)) {
2801	struct page_pool *pp;
2802
2803	pp = mtk_create_page_pool(eth, xdp_q: &ring->xdp_q, id: ring_no,
2804	size: rx_dma_size);
2805	if (IS_ERR(ptr: pp))
2806	return PTR_ERR(ptr: pp);
2807
2808	ring->page_pool = pp;
2809	}
2810
2811	ring->dma = mtk_dma_ring_alloc(eth,
2812	size: rx_dma_size * eth->soc->rx.desc_size,
2813	dma_handle: &ring->phys,
2814	use_sram: rx_flag == MTK_RX_FLAGS_NORMAL);
2815	if (!ring->dma)
2816	return -ENOMEM;
2817
2818	for (i = 0; i < rx_dma_size; i++) {
2819	struct mtk_rx_dma_v2 *rxd;
2820	dma_addr_t dma_addr;
2821	void *data;
2822
2823	rxd = ring->dma + i * eth->soc->rx.desc_size;
2824	if (ring->page_pool) {
2825	data = mtk_page_pool_get_buff(pp: ring->page_pool,
2826	dma_addr: &dma_addr, GFP_KERNEL);
2827	if (!data)
2828	return -ENOMEM;
2829	} else {
2830	if (ring->frag_size <= PAGE_SIZE)
2831	data = netdev_alloc_frag(fragsz: ring->frag_size);
2832	else
2833	data = mtk_max_lro_buf_alloc(GFP_KERNEL);
2834
2835	if (!data)
2836	return -ENOMEM;
2837
2838	dma_addr = dma_map_single(eth->dma_dev,
2839	data + NET_SKB_PAD + eth->ip_align,
2840	ring->buf_size, DMA_FROM_DEVICE);
2841	if (unlikely(dma_mapping_error(eth->dma_dev,
2842	dma_addr))) {
2843	skb_free_frag(addr: data);
2844	return -ENOMEM;
2845	}
2846	}
2847	rxd->rxd1 = (unsigned int)dma_addr;
2848	ring->data[i] = data;
2849
2850	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
2851	rxd->rxd2 = RX_DMA_LSO;
2852	else
2853	rxd->rxd2 = RX_DMA_PREP_PLEN0(ring->buf_size);
2854
2855	if (MTK_HAS_CAPS(eth->soc->caps, MTK_36BIT_DMA))
2856	rxd->rxd2 |= RX_DMA_PREP_ADDR64(dma_addr);
2857
2858	rxd->rxd3 = 0;
2859	rxd->rxd4 = 0;
2860	if (mtk_is_netsys_v3_or_greater(eth)) {
2861	rxd->rxd5 = 0;
2862	rxd->rxd6 = 0;
2863	rxd->rxd7 = 0;
2864	rxd->rxd8 = 0;
2865	}
2866	}
2867
2868	ring->dma_size = rx_dma_size;
2869	ring->calc_idx_update = false;
2870	ring->calc_idx = rx_dma_size - 1;
2871	if (rx_flag == MTK_RX_FLAGS_QDMA)
2872	ring->crx_idx_reg = reg_map->qdma.qcrx_ptr +
2873	ring_no * MTK_QRX_OFFSET;
2874	else
2875	ring->crx_idx_reg = reg_map->pdma.pcrx_ptr +
2876	ring_no * MTK_QRX_OFFSET;
2877	/* make sure that all changes to the dma ring are flushed before we
2878	* continue
2879	*/
2880	wmb();
2881
2882	if (rx_flag == MTK_RX_FLAGS_QDMA) {
2883	mtk_w32(eth, val: ring->phys,
2884	reg: reg_map->qdma.rx_ptr + ring_no * MTK_QRX_OFFSET);
2885	mtk_w32(eth, val: rx_dma_size,
2886	reg: reg_map->qdma.rx_cnt_cfg + ring_no * MTK_QRX_OFFSET);
2887	mtk_w32(eth, MTK_PST_DRX_IDX_CFG(ring_no),
2888	reg: reg_map->qdma.rst_idx);
2889	} else {
2890	mtk_w32(eth, val: ring->phys,
2891	reg: reg_map->pdma.rx_ptr + ring_no * MTK_QRX_OFFSET);
2892	mtk_w32(eth, val: rx_dma_size,
2893	reg: reg_map->pdma.rx_cnt_cfg + ring_no * MTK_QRX_OFFSET);
2894	mtk_w32(eth, MTK_PST_DRX_IDX_CFG(ring_no),
2895	reg: reg_map->pdma.rst_idx);
2896	}
2897	mtk_w32(eth, val: ring->calc_idx, reg: ring->crx_idx_reg);
2898
2899	return 0;
2900	}
2901
2902	static void mtk_rx_clean(struct mtk_eth *eth, struct mtk_rx_ring *ring, bool in_sram)
2903	{
2904	u64 addr64 = 0;
2905	int i;
2906
2907	if (ring->data && ring->dma) {
2908	for (i = 0; i < ring->dma_size; i++) {
2909	struct mtk_rx_dma *rxd;
2910
2911	if (!ring->data[i])
2912	continue;
2913
2914	rxd = ring->dma + i * eth->soc->rx.desc_size;
2915	if (!rxd->rxd1)
2916	continue;
2917
2918	if (MTK_HAS_CAPS(eth->soc->caps, MTK_36BIT_DMA))
2919	addr64 = RX_DMA_GET_ADDR64(rxd->rxd2);
2920
2921	dma_unmap_single(eth->dma_dev, ((u64)rxd->rxd1 | addr64),
2922	ring->buf_size, DMA_FROM_DEVICE);
2923	mtk_rx_put_buff(ring, data: ring->data[i], napi: false);
2924	}
2925	kfree(objp: ring->data);
2926	ring->data = NULL;
2927	}
2928
2929	if (ring->dma) {
2930	mtk_dma_ring_free(eth, size: ring->dma_size * eth->soc->rx.desc_size,
2931	dma_ring: ring->dma, dma_handle: ring->phys, in_sram);
2932	ring->dma = NULL;
2933	}
2934
2935	if (ring->page_pool) {
2936	if (xdp_rxq_info_is_reg(xdp_rxq: &ring->xdp_q))
2937	xdp_rxq_info_unreg(xdp_rxq: &ring->xdp_q);
2938	page_pool_destroy(pool: ring->page_pool);
2939	ring->page_pool = NULL;
2940	}
2941	}
2942
2943	static int mtk_hwlro_rx_init(struct mtk_eth *eth)
2944	{
2945	int i;
2946	u32 ring_ctrl_dw1 = 0, ring_ctrl_dw2 = 0, ring_ctrl_dw3 = 0;
2947	u32 lro_ctrl_dw0 = 0, lro_ctrl_dw3 = 0;
2948
2949	/* set LRO rings to auto-learn modes */
2950	ring_ctrl_dw2 |= MTK_RING_AUTO_LERAN_MODE;
2951
2952	/* validate LRO ring */
2953	ring_ctrl_dw2 |= MTK_RING_VLD;
2954
2955	/* set AGE timer (unit: 20us) */
2956	ring_ctrl_dw2 |= MTK_RING_AGE_TIME_H;
2957	ring_ctrl_dw1 |= MTK_RING_AGE_TIME_L;
2958
2959	/* set max AGG timer (unit: 20us) */
2960	ring_ctrl_dw2 |= MTK_RING_MAX_AGG_TIME;
2961
2962	/* set max LRO AGG count */
2963	ring_ctrl_dw2 |= MTK_RING_MAX_AGG_CNT_L;
2964	ring_ctrl_dw3 |= MTK_RING_MAX_AGG_CNT_H;
2965
2966	for (i = 1; i < MTK_MAX_RX_RING_NUM; i++) {
2967	mtk_w32(eth, val: ring_ctrl_dw1, MTK_LRO_CTRL_DW1_CFG(i));
2968	mtk_w32(eth, val: ring_ctrl_dw2, MTK_LRO_CTRL_DW2_CFG(i));
2969	mtk_w32(eth, val: ring_ctrl_dw3, MTK_LRO_CTRL_DW3_CFG(i));
2970	}
2971
2972	/* IPv4 checksum update enable */
2973	lro_ctrl_dw0 |= MTK_L3_CKS_UPD_EN;
2974
2975	/* switch priority comparison to packet count mode */
2976	lro_ctrl_dw0 |= MTK_LRO_ALT_PKT_CNT_MODE;
2977
2978	/* bandwidth threshold setting */
2979	mtk_w32(eth, MTK_HW_LRO_BW_THRE, MTK_PDMA_LRO_CTRL_DW2);
2980
2981	/* auto-learn score delta setting */
2982	mtk_w32(eth, MTK_HW_LRO_REPLACE_DELTA, MTK_PDMA_LRO_ALT_SCORE_DELTA);
2983
2984	/* set refresh timer for altering flows to 1 sec. (unit: 20us) */
2985	mtk_w32(eth, val: (MTK_HW_LRO_TIMER_UNIT << 16) | MTK_HW_LRO_REFRESH_TIME,
2986	MTK_PDMA_LRO_ALT_REFRESH_TIMER);
2987
2988	/* set HW LRO mode & the max aggregation count for rx packets */
2989	lro_ctrl_dw3 |= MTK_ADMA_MODE | (MTK_HW_LRO_MAX_AGG_CNT & 0xff);
2990
2991	/* the minimal remaining room of SDL0 in RXD for lro aggregation */
2992	lro_ctrl_dw3 |= MTK_LRO_MIN_RXD_SDL;
2993
2994	/* enable HW LRO */
2995	lro_ctrl_dw0 |= MTK_LRO_EN;
2996
2997	mtk_w32(eth, val: lro_ctrl_dw3, MTK_PDMA_LRO_CTRL_DW3);
2998	mtk_w32(eth, val: lro_ctrl_dw0, MTK_PDMA_LRO_CTRL_DW0);
2999
3000	return 0;
3001	}
3002
3003	static void mtk_hwlro_rx_uninit(struct mtk_eth *eth)
3004	{
3005	int i;
3006	u32 val;
3007
3008	/* relinquish lro rings, flush aggregated packets */
3009	mtk_w32(eth, MTK_LRO_RING_RELINQUISH_REQ, MTK_PDMA_LRO_CTRL_DW0);
3010
3011	/* wait for relinquishments done */
3012	for (i = 0; i < 10; i++) {
3013	val = mtk_r32(eth, MTK_PDMA_LRO_CTRL_DW0);
3014	if (val & MTK_LRO_RING_RELINQUISH_DONE) {
3015	msleep(msecs: 20);
3016	continue;
3017	}
3018	break;
3019	}
3020
3021	/* invalidate lro rings */
3022	for (i = 1; i < MTK_MAX_RX_RING_NUM; i++)
3023	mtk_w32(eth, val: 0, MTK_LRO_CTRL_DW2_CFG(i));
3024
3025	/* disable HW LRO */
3026	mtk_w32(eth, val: 0, MTK_PDMA_LRO_CTRL_DW0);
3027	}
3028
3029	static void mtk_hwlro_val_ipaddr(struct mtk_eth *eth, int idx, __be32 ip)
3030	{
3031	u32 reg_val;
3032
3033	reg_val = mtk_r32(eth, MTK_LRO_CTRL_DW2_CFG(idx));
3034
3035	/* invalidate the IP setting */
3036	mtk_w32(eth, val: (reg_val & ~MTK_RING_MYIP_VLD), MTK_LRO_CTRL_DW2_CFG(idx));
3037
3038	mtk_w32(eth, val: ip, MTK_LRO_DIP_DW0_CFG(idx));
3039
3040	/* validate the IP setting */
3041	mtk_w32(eth, val: (reg_val | MTK_RING_MYIP_VLD), MTK_LRO_CTRL_DW2_CFG(idx));
3042	}
3043
3044	static void mtk_hwlro_inval_ipaddr(struct mtk_eth *eth, int idx)
3045	{
3046	u32 reg_val;
3047
3048	reg_val = mtk_r32(eth, MTK_LRO_CTRL_DW2_CFG(idx));
3049
3050	/* invalidate the IP setting */
3051	mtk_w32(eth, val: (reg_val & ~MTK_RING_MYIP_VLD), MTK_LRO_CTRL_DW2_CFG(idx));
3052
3053	mtk_w32(eth, val: 0, MTK_LRO_DIP_DW0_CFG(idx));
3054	}
3055
3056	static int mtk_hwlro_get_ip_cnt(struct mtk_mac *mac)
3057	{
3058	int cnt = 0;
3059	int i;
3060
3061	for (i = 0; i < MTK_MAX_LRO_IP_CNT; i++) {
3062	if (mac->hwlro_ip[i])
3063	cnt++;
3064	}
3065
3066	return cnt;
3067	}
3068
3069	static int mtk_hwlro_add_ipaddr(struct net_device *dev,
3070	struct ethtool_rxnfc *cmd)
3071	{
3072	struct ethtool_rx_flow_spec *fsp =
3073	(struct ethtool_rx_flow_spec *)&cmd->fs;
3074	struct mtk_mac *mac = netdev_priv(dev);
3075	struct mtk_eth *eth = mac->hw;
3076	int hwlro_idx;
3077
3078	if ((fsp->flow_type != TCP_V4_FLOW) ||
3079	(!fsp->h_u.tcp_ip4_spec.ip4dst) ||
3080	(fsp->location > 1))
3081	return -EINVAL;
3082
3083	mac->hwlro_ip[fsp->location] = htonl(fsp->h_u.tcp_ip4_spec.ip4dst);
3084	hwlro_idx = (mac->id * MTK_MAX_LRO_IP_CNT) + fsp->location;
3085
3086	mac->hwlro_ip_cnt = mtk_hwlro_get_ip_cnt(mac);
3087
3088	mtk_hwlro_val_ipaddr(eth, idx: hwlro_idx, ip: mac->hwlro_ip[fsp->location]);
3089
3090	return 0;
3091	}
3092
3093	static int mtk_hwlro_del_ipaddr(struct net_device *dev,
3094	struct ethtool_rxnfc *cmd)
3095	{
3096	struct ethtool_rx_flow_spec *fsp =
3097	(struct ethtool_rx_flow_spec *)&cmd->fs;
3098	struct mtk_mac *mac = netdev_priv(dev);
3099	struct mtk_eth *eth = mac->hw;
3100	int hwlro_idx;
3101
3102	if (fsp->location > 1)
3103	return -EINVAL;
3104
3105	mac->hwlro_ip[fsp->location] = 0;
3106	hwlro_idx = (mac->id * MTK_MAX_LRO_IP_CNT) + fsp->location;
3107
3108	mac->hwlro_ip_cnt = mtk_hwlro_get_ip_cnt(mac);
3109
3110	mtk_hwlro_inval_ipaddr(eth, idx: hwlro_idx);
3111
3112	return 0;
3113	}
3114
3115	static void mtk_hwlro_netdev_disable(struct net_device *dev)
3116	{
3117	struct mtk_mac *mac = netdev_priv(dev);
3118	struct mtk_eth *eth = mac->hw;
3119	int i, hwlro_idx;
3120
3121	for (i = 0; i < MTK_MAX_LRO_IP_CNT; i++) {
3122	mac->hwlro_ip[i] = 0;
3123	hwlro_idx = (mac->id * MTK_MAX_LRO_IP_CNT) + i;
3124
3125	mtk_hwlro_inval_ipaddr(eth, idx: hwlro_idx);
3126	}
3127
3128	mac->hwlro_ip_cnt = 0;
3129	}
3130
3131	static int mtk_hwlro_get_fdir_entry(struct net_device *dev,
3132	struct ethtool_rxnfc *cmd)
3133	{
3134	struct mtk_mac *mac = netdev_priv(dev);
3135	struct ethtool_rx_flow_spec *fsp =
3136	(struct ethtool_rx_flow_spec *)&cmd->fs;
3137
3138	if (fsp->location >= ARRAY_SIZE(mac->hwlro_ip))
3139	return -EINVAL;
3140
3141	/* only tcp dst ipv4 is meaningful, others are meaningless */
3142	fsp->flow_type = TCP_V4_FLOW;
3143	fsp->h_u.tcp_ip4_spec.ip4dst = ntohl(mac->hwlro_ip[fsp->location]);
3144	fsp->m_u.tcp_ip4_spec.ip4dst = 0;
3145
3146	fsp->h_u.tcp_ip4_spec.ip4src = 0;
3147	fsp->m_u.tcp_ip4_spec.ip4src = 0xffffffff;
3148	fsp->h_u.tcp_ip4_spec.psrc = 0;
3149	fsp->m_u.tcp_ip4_spec.psrc = 0xffff;
3150	fsp->h_u.tcp_ip4_spec.pdst = 0;
3151	fsp->m_u.tcp_ip4_spec.pdst = 0xffff;
3152	fsp->h_u.tcp_ip4_spec.tos = 0;
3153	fsp->m_u.tcp_ip4_spec.tos = 0xff;
3154
3155	return 0;
3156	}
3157
3158	static int mtk_hwlro_get_fdir_all(struct net_device *dev,
3159	struct ethtool_rxnfc *cmd,
3160	u32 *rule_locs)
3161	{
3162	struct mtk_mac *mac = netdev_priv(dev);
3163	int cnt = 0;
3164	int i;
3165
3166	for (i = 0; i < MTK_MAX_LRO_IP_CNT; i++) {
3167	if (cnt == cmd->rule_cnt)
3168	return -EMSGSIZE;
3169
3170	if (mac->hwlro_ip[i]) {
3171	rule_locs[cnt] = i;
3172	cnt++;
3173	}
3174	}
3175
3176	cmd->rule_cnt = cnt;
3177
3178	return 0;
3179	}
3180
3181	static netdev_features_t mtk_fix_features(struct net_device *dev,
3182	netdev_features_t features)
3183	{
3184	if (!(features & NETIF_F_LRO)) {
3185	struct mtk_mac *mac = netdev_priv(dev);
3186	int ip_cnt = mtk_hwlro_get_ip_cnt(mac);
3187
3188	if (ip_cnt) {
3189	netdev_info(dev, format: "RX flow is programmed, LRO should keep on\n");
3190
3191	features |= NETIF_F_LRO;
3192	}
3193	}
3194
3195	return features;
3196	}
3197
3198	static int mtk_set_features(struct net_device *dev, netdev_features_t features)
3199	{
3200	netdev_features_t diff = dev->features ^ features;
3201
3202	if ((diff & NETIF_F_LRO) && !(features & NETIF_F_LRO))
3203	mtk_hwlro_netdev_disable(dev);
3204
3205	return 0;
3206	}
3207
3208	/* wait for DMA to finish whatever it is doing before we start using it again */
3209	static int mtk_dma_busy_wait(struct mtk_eth *eth)
3210	{
3211	unsigned int reg;
3212	int ret;
3213	u32 val;
3214
3215	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
3216	reg = eth->soc->reg_map->qdma.glo_cfg;
3217	else
3218	reg = eth->soc->reg_map->pdma.glo_cfg;
3219
3220	ret = readx_poll_timeout_atomic(__raw_readl, eth->base + reg, val,
3221	!(val & (MTK_RX_DMA_BUSY | MTK_TX_DMA_BUSY)),
3222	5, MTK_DMA_BUSY_TIMEOUT_US);
3223	if (ret)
3224	dev_err(eth->dev, "DMA init timeout\n");
3225
3226	return ret;
3227	}
3228
3229	static int mtk_dma_init(struct mtk_eth *eth)
3230	{
3231	int err;
3232	u32 i;
3233
3234	if (mtk_dma_busy_wait(eth))
3235	return -EBUSY;
3236
3237	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
3238	/* QDMA needs scratch memory for internal reordering of the
3239	* descriptors
3240	*/
3241	err = mtk_init_fq_dma(eth);
3242	if (err)
3243	return err;
3244	}
3245
3246	err = mtk_tx_alloc(eth);
3247	if (err)
3248	return err;
3249
3250	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
3251	err = mtk_rx_alloc(eth, ring_no: 0, rx_flag: MTK_RX_FLAGS_QDMA);
3252	if (err)
3253	return err;
3254	}
3255
3256	err = mtk_rx_alloc(eth, ring_no: 0, rx_flag: MTK_RX_FLAGS_NORMAL);
3257	if (err)
3258	return err;
3259
3260	if (eth->hwlro) {
3261	for (i = 1; i < MTK_MAX_RX_RING_NUM; i++) {
3262	err = mtk_rx_alloc(eth, ring_no: i, rx_flag: MTK_RX_FLAGS_HWLRO);
3263	if (err)
3264	return err;
3265	}
3266	err = mtk_hwlro_rx_init(eth);
3267	if (err)
3268	return err;
3269	}
3270
3271	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
3272	/* Enable random early drop and set drop threshold
3273	* automatically
3274	*/
3275	mtk_w32(eth, FC_THRES_DROP_MODE | FC_THRES_DROP_EN |
3276	FC_THRES_MIN, reg: eth->soc->reg_map->qdma.fc_th);
3277	mtk_w32(eth, val: 0x0, reg: eth->soc->reg_map->qdma.hred);
3278	}
3279
3280	return 0;
3281	}
3282
3283	static void mtk_dma_free(struct mtk_eth *eth)
3284	{
3285	const struct mtk_soc_data *soc = eth->soc;
3286	int i, j, txqs = 1;
3287
3288	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
3289	txqs = MTK_QDMA_NUM_QUEUES;
3290
3291	for (i = 0; i < MTK_MAX_DEVS; i++) {
3292	if (!eth->netdev[i])
3293	continue;
3294
3295	for (j = 0; j < txqs; j++)
3296	netdev_tx_reset_subqueue(dev: eth->netdev[i], qid: j);
3297	}
3298
3299	if (eth->scratch_ring) {
3300	mtk_dma_ring_free(eth, size: soc->tx.fq_dma_size * soc->tx.desc_size,
3301	dma_ring: eth->scratch_ring, dma_handle: eth->phy_scratch_ring,
3302	in_sram: true);
3303	eth->scratch_ring = NULL;
3304	eth->phy_scratch_ring = 0;
3305	}
3306
3307	mtk_tx_clean(eth);
3308	mtk_rx_clean(eth, ring: &eth->rx_ring[0], in_sram: true);
3309	mtk_rx_clean(eth, ring: &eth->rx_ring_qdma, in_sram: false);
3310
3311	if (eth->hwlro) {
3312	mtk_hwlro_rx_uninit(eth);
3313	for (i = 1; i < MTK_MAX_RX_RING_NUM; i++)
3314	mtk_rx_clean(eth, ring: &eth->rx_ring[i], in_sram: false);
3315	}
3316
3317	for (i = 0; i < DIV_ROUND_UP(soc->tx.fq_dma_size, MTK_FQ_DMA_LENGTH); i++) {
3318	kfree(objp: eth->scratch_head[i]);
3319	eth->scratch_head[i] = NULL;
3320	}
3321	}
3322
3323	static bool mtk_hw_reset_check(struct mtk_eth *eth)
3324	{
3325	u32 val = mtk_r32(eth, MTK_INT_STATUS2);
3326
3327	return (val & MTK_FE_INT_FQ_EMPTY) || (val & MTK_FE_INT_RFIFO_UF) ||
3328	(val & MTK_FE_INT_RFIFO_OV) || (val & MTK_FE_INT_TSO_FAIL) ||
3329	(val & MTK_FE_INT_TSO_ALIGN) || (val & MTK_FE_INT_TSO_ILLEGAL);
3330	}
3331
3332	static void mtk_tx_timeout(struct net_device *dev, unsigned int txqueue)
3333	{
3334	struct mtk_mac *mac = netdev_priv(dev);
3335	struct mtk_eth *eth = mac->hw;
3336
3337	if (test_bit(MTK_RESETTING, &eth->state))
3338	return;
3339
3340	if (!mtk_hw_reset_check(eth))
3341	return;
3342
3343	eth->netdev[mac->id]->stats.tx_errors++;
3344	netif_err(eth, tx_err, dev, "transmit timed out\n");
3345
3346	schedule_work(work: &eth->pending_work);
3347	}
3348
3349	static int mtk_get_irqs(struct platform_device *pdev, struct mtk_eth *eth)
3350	{
3351	int i;
3352
3353	/* future SoCs beginning with MT7988 should use named IRQs in dts */
3354	eth->irq[MTK_FE_IRQ_TX] = platform_get_irq_byname_optional(dev: pdev, name: "fe1");
3355	eth->irq[MTK_FE_IRQ_RX] = platform_get_irq_byname_optional(dev: pdev, name: "fe2");
3356	if (eth->irq[MTK_FE_IRQ_TX] >= 0 && eth->irq[MTK_FE_IRQ_RX] >= 0)
3357	return 0;
3358
3359	/* only use legacy mode if platform_get_irq_byname_optional returned -ENXIO */
3360	if (eth->irq[MTK_FE_IRQ_TX] != -ENXIO)
3361	return dev_err_probe(dev: &pdev->dev, err: eth->irq[MTK_FE_IRQ_TX],
3362	fmt: "Error requesting FE TX IRQ\n");
3363
3364	if (eth->irq[MTK_FE_IRQ_RX] != -ENXIO)
3365	return dev_err_probe(dev: &pdev->dev, err: eth->irq[MTK_FE_IRQ_RX],
3366	fmt: "Error requesting FE RX IRQ\n");
3367
3368	if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SHARED_INT))
3369	dev_warn(&pdev->dev, "legacy DT: missing interrupt-names.");
3370
3371	/* legacy way:
3372	* On MTK_SHARED_INT SoCs (MT7621 + MT7628) the first IRQ is taken
3373	* from devicetree and used for both RX and TX - it is shared.
3374	* On SoCs with non-shared IRQs the first entry is not used,
3375	* the second is for TX, and the third is for RX.
3376	*/
3377	for (i = 0; i < MTK_FE_IRQ_NUM; i++) {
3378	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SHARED_INT)) {
3379	if (i == MTK_FE_IRQ_SHARED)
3380	eth->irq[MTK_FE_IRQ_SHARED] = platform_get_irq(pdev, i);
3381	else
3382	eth->irq[i] = eth->irq[MTK_FE_IRQ_SHARED];
3383	} else {
3384	eth->irq[i] = platform_get_irq(pdev, i + 1);
3385	}
3386
3387	if (eth->irq[i] < 0) {
3388	dev_err(&pdev->dev, "no IRQ%d resource found\n", i);
3389	return -ENXIO;
3390	}
3391	}
3392
3393	return 0;
3394	}
3395
3396	static irqreturn_t mtk_handle_irq_rx(int irq, void *_eth)
3397	{
3398	struct mtk_eth *eth = _eth;
3399
3400	eth->rx_events++;
3401	if (likely(napi_schedule_prep(&eth->rx_napi))) {
3402	mtk_rx_irq_disable(eth, mask: eth->soc->rx.irq_done_mask);
3403	__napi_schedule(n: &eth->rx_napi);
3404	}
3405
3406	return IRQ_HANDLED;
3407	}
3408
3409	static irqreturn_t mtk_handle_irq_tx(int irq, void *_eth)
3410	{
3411	struct mtk_eth *eth = _eth;
3412
3413	eth->tx_events++;
3414	if (likely(napi_schedule_prep(&eth->tx_napi))) {
3415	mtk_tx_irq_disable(eth, MTK_TX_DONE_INT);
3416	__napi_schedule(n: &eth->tx_napi);
3417	}
3418
3419	return IRQ_HANDLED;
3420	}
3421
3422	static irqreturn_t mtk_handle_irq(int irq, void *_eth)
3423	{
3424	struct mtk_eth *eth = _eth;
3425	const struct mtk_reg_map *reg_map = eth->soc->reg_map;
3426
3427	if (mtk_r32(eth, reg: reg_map->pdma.irq_mask) &
3428	eth->soc->rx.irq_done_mask) {
3429	if (mtk_r32(eth, reg: reg_map->pdma.irq_status) &
3430	eth->soc->rx.irq_done_mask)
3431	mtk_handle_irq_rx(irq, _eth);
3432	}
3433	if (mtk_r32(eth, reg: reg_map->tx_irq_mask) & MTK_TX_DONE_INT) {
3434	if (mtk_r32(eth, reg: reg_map->tx_irq_status) & MTK_TX_DONE_INT)
3435	mtk_handle_irq_tx(irq, _eth);
3436	}
3437
3438	return IRQ_HANDLED;
3439	}
3440
3441	#ifdef CONFIG_NET_POLL_CONTROLLER
3442	static void mtk_poll_controller(struct net_device *dev)
3443	{
3444	struct mtk_mac *mac = netdev_priv(dev);
3445	struct mtk_eth *eth = mac->hw;
3446
3447	mtk_tx_irq_disable(eth, MTK_TX_DONE_INT);
3448	mtk_rx_irq_disable(eth, mask: eth->soc->rx.irq_done_mask);
3449	mtk_handle_irq_rx(irq: eth->irq[MTK_FE_IRQ_RX], eth: dev);
3450	mtk_tx_irq_enable(eth, MTK_TX_DONE_INT);
3451	mtk_rx_irq_enable(eth, mask: eth->soc->rx.irq_done_mask);
3452	}
3453	#endif
3454
3455	static int mtk_start_dma(struct mtk_eth *eth)
3456	{
3457	u32 val, rx_2b_offset = (NET_IP_ALIGN == 2) ? MTK_RX_2B_OFFSET : 0;
3458	const struct mtk_reg_map *reg_map = eth->soc->reg_map;
3459	int err;
3460
3461	err = mtk_dma_init(eth);
3462	if (err) {
3463	mtk_dma_free(eth);
3464	return err;
3465	}
3466
3467	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
3468	val = mtk_r32(eth, reg: reg_map->qdma.glo_cfg);
3469	val |= MTK_TX_DMA_EN | MTK_RX_DMA_EN |
3470	MTK_TX_BT_32DWORDS | MTK_NDP_CO_PRO |
3471	MTK_RX_2B_OFFSET | MTK_TX_WB_DDONE;
3472
3473	if (mtk_is_netsys_v2_or_greater(eth))
3474	val |= MTK_MUTLI_CNT | MTK_RESV_BUF |
3475	MTK_WCOMP_EN | MTK_DMAD_WR_WDONE |
3476	MTK_CHK_DDONE_EN;
3477	else
3478	val |= MTK_RX_BT_32DWORDS;
3479	mtk_w32(eth, val, reg: reg_map->qdma.glo_cfg);
3480
3481	mtk_w32(eth,
3482	MTK_RX_DMA_EN | rx_2b_offset |
3483	MTK_RX_BT_32DWORDS | MTK_MULTI_EN,
3484	reg: reg_map->pdma.glo_cfg);
3485	} else {
3486	mtk_w32(eth, MTK_TX_WB_DDONE | MTK_TX_DMA_EN | MTK_RX_DMA_EN |
3487	MTK_MULTI_EN | MTK_PDMA_SIZE_8DWORDS,
3488	reg: reg_map->pdma.glo_cfg);
3489	}
3490
3491	return 0;
3492	}
3493
3494	static void mtk_gdm_config(struct mtk_eth *eth, u32 id, u32 config)
3495	{
3496	u32 val;
3497
3498	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
3499	return;
3500
3501	val = mtk_r32(eth, MTK_GDMA_FWD_CFG(id));
3502
3503	/* default setup the forward port to send frame to PDMA */
3504	val &= ~0xffff;
3505
3506	/* Enable RX checksum */
3507	val |= MTK_GDMA_ICS_EN | MTK_GDMA_TCS_EN | MTK_GDMA_UCS_EN;
3508
3509	val |= config;
3510
3511	if (eth->netdev[id] && netdev_uses_dsa(dev: eth->netdev[id]))
3512	val |= MTK_GDMA_SPECIAL_TAG;
3513
3514	mtk_w32(eth, val, MTK_GDMA_FWD_CFG(id));
3515	}
3516
3517
3518	static bool mtk_uses_dsa(struct net_device *dev)
3519	{
3520	#if IS_ENABLED(CONFIG_NET_DSA)
3521	return netdev_uses_dsa(dev) &&
3522	dev->dsa_ptr->tag_ops->proto == DSA_TAG_PROTO_MTK;
3523	#else
3524	return false;
3525	#endif
3526	}
3527
3528	static int mtk_device_event(struct notifier_block *n, unsigned long event, void *ptr)
3529	{
3530	struct mtk_mac *mac = container_of(n, struct mtk_mac, device_notifier);
3531	struct mtk_eth *eth = mac->hw;
3532	struct net_device *dev = netdev_notifier_info_to_dev(info: ptr);
3533	struct ethtool_link_ksettings s;
3534	struct net_device *ldev;
3535	struct list_head *iter;
3536	struct dsa_port *dp;
3537
3538	if (event != NETDEV_CHANGE)
3539	return NOTIFY_DONE;
3540
3541	netdev_for_each_lower_dev(dev, ldev, iter) {
3542	if (netdev_priv(dev: ldev) == mac)
3543	goto found;
3544	}
3545
3546	return NOTIFY_DONE;
3547
3548	found:
3549	if (!dsa_user_dev_check(dev))
3550	return NOTIFY_DONE;
3551
3552	if (__ethtool_get_link_ksettings(dev, link_ksettings: &s))
3553	return NOTIFY_DONE;
3554
3555	if (s.base.speed == 0 || s.base.speed == ((__u32)-1))
3556	return NOTIFY_DONE;
3557
3558	dp = dsa_port_from_netdev(netdev: dev);
3559	if (dp->index >= MTK_QDMA_NUM_QUEUES)
3560	return NOTIFY_DONE;
3561
3562	if (mac->speed > 0 && mac->speed <= s.base.speed)
3563	s.base.speed = 0;
3564
3565	mtk_set_queue_speed(eth, idx: dp->index + 3, speed: s.base.speed);
3566
3567	return NOTIFY_DONE;
3568	}
3569
3570	static int mtk_open(struct net_device *dev)
3571	{
3572	struct mtk_mac *mac = netdev_priv(dev);
3573	struct mtk_eth *eth = mac->hw;
3574	struct mtk_mac *target_mac;
3575	int i, err, ppe_num;
3576
3577	ppe_num = eth->soc->ppe_num;
3578
3579	err = phylink_of_phy_connect(mac->phylink, mac->of_node, flags: 0);
3580	if (err) {
3581	netdev_err(dev, format: "%s: could not attach PHY: %d\n", __func__,
3582	err);
3583	return err;
3584	}
3585
3586	/* we run 2 netdevs on the same dma ring so we only bring it up once */
3587	if (!refcount_read(r: &eth->dma_refcnt)) {
3588	const struct mtk_soc_data *soc = eth->soc;
3589	u32 gdm_config;
3590	int i;
3591
3592	err = mtk_start_dma(eth);
3593	if (err) {
3594	phylink_disconnect_phy(mac->phylink);
3595	return err;
3596	}
3597
3598	for (i = 0; i < ARRAY_SIZE(eth->ppe); i++)
3599	mtk_ppe_start(ppe: eth->ppe[i]);
3600
3601	for (i = 0; i < MTK_MAX_DEVS; i++) {
3602	if (!eth->netdev[i])
3603	continue;
3604
3605	target_mac = netdev_priv(dev: eth->netdev[i]);
3606	if (!soc->offload_version) {
3607	target_mac->ppe_idx = 0;
3608	gdm_config = MTK_GDMA_TO_PDMA;
3609	} else if (ppe_num >= 3 && target_mac->id == 2) {
3610	target_mac->ppe_idx = 2;
3611	gdm_config = soc->reg_map->gdma_to_ppe[2];
3612	} else if (ppe_num >= 2 && target_mac->id == 1) {
3613	target_mac->ppe_idx = 1;
3614	gdm_config = soc->reg_map->gdma_to_ppe[1];
3615	} else {
3616	target_mac->ppe_idx = 0;
3617	gdm_config = soc->reg_map->gdma_to_ppe[0];
3618	}
3619	mtk_gdm_config(eth, id: target_mac->id, config: gdm_config);
3620	}
3621
3622	napi_enable(n: &eth->tx_napi);
3623	napi_enable(n: &eth->rx_napi);
3624	mtk_tx_irq_enable(eth, MTK_TX_DONE_INT);
3625	mtk_rx_irq_enable(eth, mask: soc->rx.irq_done_mask);
3626	refcount_set(r: &eth->dma_refcnt, n: 1);
3627	} else {
3628	refcount_inc(r: &eth->dma_refcnt);
3629	}
3630
3631	phylink_start(mac->phylink);
3632	netif_tx_start_all_queues(dev);
3633
3634	if (mtk_is_netsys_v2_or_greater(eth))
3635	return 0;
3636
3637	if (mtk_uses_dsa(dev) && !eth->prog) {
3638	for (i = 0; i < ARRAY_SIZE(eth->dsa_meta); i++) {
3639	struct metadata_dst *md_dst = eth->dsa_meta[i];
3640
3641	if (md_dst)
3642	continue;
3643
3644	md_dst = metadata_dst_alloc(optslen: 0, type: METADATA_HW_PORT_MUX,
3645	GFP_KERNEL);
3646	if (!md_dst)
3647	return -ENOMEM;
3648
3649	md_dst->u.port_info.port_id = i;
3650	eth->dsa_meta[i] = md_dst;
3651	}
3652	} else {
3653	/* Hardware DSA untagging and VLAN RX offloading need to be
3654	* disabled if at least one MAC does not use DSA.
3655	*/
3656	u32 val = mtk_r32(eth, MTK_CDMP_IG_CTRL);
3657
3658	val &= ~MTK_CDMP_STAG_EN;
3659	mtk_w32(eth, val, MTK_CDMP_IG_CTRL);
3660
3661	mtk_w32(eth, val: 0, MTK_CDMP_EG_CTRL);
3662	}
3663
3664	return 0;
3665	}
3666
3667	static void mtk_stop_dma(struct mtk_eth *eth, u32 glo_cfg)
3668	{
3669	u32 val;
3670	int i;
3671
3672	/* stop the dma engine */
3673	spin_lock_bh(lock: &eth->page_lock);
3674	val = mtk_r32(eth, reg: glo_cfg);
3675	mtk_w32(eth, val: val & ~(MTK_TX_WB_DDONE | MTK_RX_DMA_EN | MTK_TX_DMA_EN),
3676	reg: glo_cfg);
3677	spin_unlock_bh(lock: &eth->page_lock);
3678
3679	/* wait for dma stop */
3680	for (i = 0; i < 10; i++) {
3681	val = mtk_r32(eth, reg: glo_cfg);
3682	if (val & (MTK_TX_DMA_BUSY | MTK_RX_DMA_BUSY)) {
3683	msleep(msecs: 20);
3684	continue;
3685	}
3686	break;
3687	}
3688	}
3689
3690	static int mtk_stop(struct net_device *dev)
3691	{
3692	struct mtk_mac *mac = netdev_priv(dev);
3693	struct mtk_eth *eth = mac->hw;
3694	int i;
3695
3696	phylink_stop(mac->phylink);
3697
3698	netif_tx_disable(dev);
3699
3700	phylink_disconnect_phy(mac->phylink);
3701
3702	/* only shutdown DMA if this is the last user */
3703	if (!refcount_dec_and_test(r: &eth->dma_refcnt))
3704	return 0;
3705
3706	for (i = 0; i < MTK_MAX_DEVS; i++)
3707	mtk_gdm_config(eth, id: i, MTK_GDMA_DROP_ALL);
3708
3709	mtk_tx_irq_disable(eth, MTK_TX_DONE_INT);
3710	mtk_rx_irq_disable(eth, mask: eth->soc->rx.irq_done_mask);
3711	napi_disable(n: &eth->tx_napi);
3712	napi_disable(n: &eth->rx_napi);
3713
3714	cancel_work_sync(work: &eth->rx_dim.work);
3715	cancel_work_sync(work: &eth->tx_dim.work);
3716
3717	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
3718	mtk_stop_dma(eth, glo_cfg: eth->soc->reg_map->qdma.glo_cfg);
3719	mtk_stop_dma(eth, glo_cfg: eth->soc->reg_map->pdma.glo_cfg);
3720
3721	mtk_dma_free(eth);
3722
3723	for (i = 0; i < ARRAY_SIZE(eth->ppe); i++)
3724	mtk_ppe_stop(ppe: eth->ppe[i]);
3725
3726	return 0;
3727	}
3728
3729	static int mtk_xdp_setup(struct net_device *dev, struct bpf_prog *prog,
3730	struct netlink_ext_ack *extack)
3731	{
3732	struct mtk_mac *mac = netdev_priv(dev);
3733	struct mtk_eth *eth = mac->hw;
3734	struct bpf_prog *old_prog;
3735	bool need_update;
3736
3737	if (eth->hwlro) {
3738	NL_SET_ERR_MSG_MOD(extack, "XDP not supported with HWLRO");
3739	return -EOPNOTSUPP;
3740	}
3741
3742	if (dev->mtu > MTK_PP_MAX_BUF_SIZE) {
3743	NL_SET_ERR_MSG_MOD(extack, "MTU too large for XDP");
3744	return -EOPNOTSUPP;
3745	}
3746
3747	need_update = !!eth->prog != !!prog;
3748	if (netif_running(dev) && need_update)
3749	mtk_stop(dev);
3750
3751	old_prog = rcu_replace_pointer(eth->prog, prog, lockdep_rtnl_is_held());
3752	if (old_prog)
3753	bpf_prog_put(prog: old_prog);
3754
3755	if (netif_running(dev) && need_update)
3756	return mtk_open(dev);
3757
3758	return 0;
3759	}
3760
3761	static int mtk_xdp(struct net_device *dev, struct netdev_bpf *xdp)
3762	{
3763	switch (xdp->command) {
3764	case XDP_SETUP_PROG:
3765	return mtk_xdp_setup(dev, prog: xdp->prog, extack: xdp->extack);
3766	default:
3767	return -EINVAL;
3768	}
3769	}
3770
3771	static void ethsys_reset(struct mtk_eth *eth, u32 reset_bits)
3772	{
3773	regmap_update_bits(map: eth->ethsys, ETHSYS_RSTCTRL,
3774	mask: reset_bits,
3775	val: reset_bits);
3776
3777	usleep_range(min: 1000, max: 1100);
3778	regmap_update_bits(map: eth->ethsys, ETHSYS_RSTCTRL,
3779	mask: reset_bits,
3780	val: ~reset_bits);
3781	mdelay(10);
3782	}
3783
3784	static void mtk_clk_disable(struct mtk_eth *eth)
3785	{
3786	int clk;
3787
3788	for (clk = MTK_CLK_MAX - 1; clk >= 0; clk--)
3789	clk_disable_unprepare(clk: eth->clks[clk]);
3790	}
3791
3792	static int mtk_clk_enable(struct mtk_eth *eth)
3793	{
3794	int clk, ret;
3795
3796	for (clk = 0; clk < MTK_CLK_MAX ; clk++) {
3797	ret = clk_prepare_enable(clk: eth->clks[clk]);
3798	if (ret)
3799	goto err_disable_clks;
3800	}
3801
3802	return 0;
3803
3804	err_disable_clks:
3805	while (--clk >= 0)
3806	clk_disable_unprepare(clk: eth->clks[clk]);
3807
3808	return ret;
3809	}
3810
3811	static void mtk_dim_rx(struct work_struct *work)
3812	{
3813	struct dim *dim = container_of(work, struct dim, work);
3814	struct mtk_eth *eth = container_of(dim, struct mtk_eth, rx_dim);
3815	const struct mtk_reg_map *reg_map = eth->soc->reg_map;
3816	struct dim_cq_moder cur_profile;
3817	u32 val, cur;
3818
3819	cur_profile = net_dim_get_rx_moderation(cq_period_mode: eth->rx_dim.mode,
3820	ix: dim->profile_ix);
3821	spin_lock_bh(lock: &eth->dim_lock);
3822
3823	val = mtk_r32(eth, reg: reg_map->pdma.delay_irq);
3824	val &= MTK_PDMA_DELAY_TX_MASK;
3825	val |= MTK_PDMA_DELAY_RX_EN;
3826
3827	cur = min_t(u32, DIV_ROUND_UP(cur_profile.usec, 20), MTK_PDMA_DELAY_PTIME_MASK);
3828	val |= cur << MTK_PDMA_DELAY_RX_PTIME_SHIFT;
3829
3830	cur = min_t(u32, cur_profile.pkts, MTK_PDMA_DELAY_PINT_MASK);
3831	val |= cur << MTK_PDMA_DELAY_RX_PINT_SHIFT;
3832
3833	mtk_w32(eth, val, reg: reg_map->pdma.delay_irq);
3834	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
3835	mtk_w32(eth, val, reg: reg_map->qdma.delay_irq);
3836
3837	spin_unlock_bh(lock: &eth->dim_lock);
3838
3839	dim->state = DIM_START_MEASURE;
3840	}
3841
3842	static void mtk_dim_tx(struct work_struct *work)
3843	{
3844	struct dim *dim = container_of(work, struct dim, work);
3845	struct mtk_eth *eth = container_of(dim, struct mtk_eth, tx_dim);
3846	const struct mtk_reg_map *reg_map = eth->soc->reg_map;
3847	struct dim_cq_moder cur_profile;
3848	u32 val, cur;
3849
3850	cur_profile = net_dim_get_tx_moderation(cq_period_mode: eth->tx_dim.mode,
3851	ix: dim->profile_ix);
3852	spin_lock_bh(lock: &eth->dim_lock);
3853
3854	val = mtk_r32(eth, reg: reg_map->pdma.delay_irq);
3855	val &= MTK_PDMA_DELAY_RX_MASK;
3856	val |= MTK_PDMA_DELAY_TX_EN;
3857
3858	cur = min_t(u32, DIV_ROUND_UP(cur_profile.usec, 20), MTK_PDMA_DELAY_PTIME_MASK);
3859	val |= cur << MTK_PDMA_DELAY_TX_PTIME_SHIFT;
3860
3861	cur = min_t(u32, cur_profile.pkts, MTK_PDMA_DELAY_PINT_MASK);
3862	val |= cur << MTK_PDMA_DELAY_TX_PINT_SHIFT;
3863
3864	mtk_w32(eth, val, reg: reg_map->pdma.delay_irq);
3865	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
3866	mtk_w32(eth, val, reg: reg_map->qdma.delay_irq);
3867
3868	spin_unlock_bh(lock: &eth->dim_lock);
3869
3870	dim->state = DIM_START_MEASURE;
3871	}
3872
3873	static void mtk_set_mcr_max_rx(struct mtk_mac *mac, u32 val)
3874	{
3875	struct mtk_eth *eth = mac->hw;
3876	u32 mcr_cur, mcr_new;
3877
3878	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
3879	return;
3880
3881	mcr_cur = mtk_r32(eth: mac->hw, MTK_MAC_MCR(mac->id));
3882	mcr_new = mcr_cur & ~MAC_MCR_MAX_RX_MASK;
3883
3884	if (val <= 1518)
3885	mcr_new |= MAC_MCR_MAX_RX(MAC_MCR_MAX_RX_1518);
3886	else if (val <= 1536)
3887	mcr_new |= MAC_MCR_MAX_RX(MAC_MCR_MAX_RX_1536);
3888	else if (val <= 1552)
3889	mcr_new |= MAC_MCR_MAX_RX(MAC_MCR_MAX_RX_1552);
3890	else
3891	mcr_new |= MAC_MCR_MAX_RX(MAC_MCR_MAX_RX_2048);
3892
3893	if (mcr_new != mcr_cur)
3894	mtk_w32(eth: mac->hw, val: mcr_new, MTK_MAC_MCR(mac->id));
3895	}
3896
3897	static void mtk_hw_reset(struct mtk_eth *eth)
3898	{
3899	u32 val;
3900
3901	if (mtk_is_netsys_v2_or_greater(eth))
3902	regmap_write(map: eth->ethsys, ETHSYS_FE_RST_CHK_IDLE_EN, val: 0);
3903
3904	if (mtk_is_netsys_v3_or_greater(eth)) {
3905	val = RSTCTRL_PPE0_V3;
3906
3907	if (MTK_HAS_CAPS(eth->soc->caps, MTK_RSTCTRL_PPE1))
3908	val |= RSTCTRL_PPE1_V3;
3909
3910	if (MTK_HAS_CAPS(eth->soc->caps, MTK_RSTCTRL_PPE2))
3911	val |= RSTCTRL_PPE2;
3912
3913	val |= RSTCTRL_WDMA0 | RSTCTRL_WDMA1 | RSTCTRL_WDMA2;
3914	} else if (mtk_is_netsys_v2_or_greater(eth)) {
3915	val = RSTCTRL_PPE0_V2;
3916
3917	if (MTK_HAS_CAPS(eth->soc->caps, MTK_RSTCTRL_PPE1))
3918	val |= RSTCTRL_PPE1;
3919	} else {
3920	val = RSTCTRL_PPE0;
3921	}
3922
3923	ethsys_reset(eth, RSTCTRL_ETH | RSTCTRL_FE | val);
3924
3925	if (mtk_is_netsys_v3_or_greater(eth))
3926	regmap_write(map: eth->ethsys, ETHSYS_FE_RST_CHK_IDLE_EN,
3927	val: 0x6f8ff);
3928	else if (mtk_is_netsys_v2_or_greater(eth))
3929	regmap_write(map: eth->ethsys, ETHSYS_FE_RST_CHK_IDLE_EN,
3930	val: 0x3ffffff);
3931	}
3932
3933	static u32 mtk_hw_reset_read(struct mtk_eth *eth)
3934	{
3935	u32 val;
3936
3937	regmap_read(map: eth->ethsys, ETHSYS_RSTCTRL, val: &val);
3938	return val;
3939	}
3940
3941	static void mtk_hw_warm_reset(struct mtk_eth *eth)
3942	{
3943	u32 rst_mask, val;
3944
3945	regmap_update_bits(map: eth->ethsys, ETHSYS_RSTCTRL, RSTCTRL_FE,
3946	RSTCTRL_FE);
3947	if (readx_poll_timeout_atomic(mtk_hw_reset_read, eth, val,
3948	val & RSTCTRL_FE, 1, 1000)) {
3949	dev_err(eth->dev, "warm reset failed\n");
3950	mtk_hw_reset(eth);
3951	return;
3952	}
3953
3954	if (mtk_is_netsys_v3_or_greater(eth)) {
3955	rst_mask = RSTCTRL_ETH | RSTCTRL_PPE0_V3;
3956	if (MTK_HAS_CAPS(eth->soc->caps, MTK_RSTCTRL_PPE1))
3957	rst_mask |= RSTCTRL_PPE1_V3;
3958	if (MTK_HAS_CAPS(eth->soc->caps, MTK_RSTCTRL_PPE2))
3959	rst_mask |= RSTCTRL_PPE2;
3960
3961	rst_mask |= RSTCTRL_WDMA0 | RSTCTRL_WDMA1 | RSTCTRL_WDMA2;
3962	} else if (mtk_is_netsys_v2_or_greater(eth)) {
3963	rst_mask = RSTCTRL_ETH | RSTCTRL_PPE0_V2;
3964	if (MTK_HAS_CAPS(eth->soc->caps, MTK_RSTCTRL_PPE1))
3965	rst_mask |= RSTCTRL_PPE1;
3966	} else {
3967	rst_mask = RSTCTRL_ETH | RSTCTRL_PPE0;
3968	}
3969
3970	regmap_update_bits(map: eth->ethsys, ETHSYS_RSTCTRL, mask: rst_mask, val: rst_mask);
3971
3972	udelay(usec: 1);
3973	val = mtk_hw_reset_read(eth);
3974	if (!(val & rst_mask))
3975	dev_err(eth->dev, "warm reset stage0 failed %08x (%08x)\n",
3976	val, rst_mask);
3977
3978	rst_mask |= RSTCTRL_FE;
3979	regmap_update_bits(map: eth->ethsys, ETHSYS_RSTCTRL, mask: rst_mask, val: ~rst_mask);
3980
3981	udelay(usec: 1);
3982	val = mtk_hw_reset_read(eth);
3983	if (val & rst_mask)
3984	dev_err(eth->dev, "warm reset stage1 failed %08x (%08x)\n",
3985	val, rst_mask);
3986	}
3987
3988	static bool mtk_hw_check_dma_hang(struct mtk_eth *eth)
3989	{
3990	const struct mtk_reg_map *reg_map = eth->soc->reg_map;
3991	bool gmac1_tx, gmac2_tx, gdm1_tx, gdm2_tx;
3992	bool oq_hang, cdm1_busy, adma_busy;
3993	bool wtx_busy, cdm_full, oq_free;
3994	u32 wdidx, val, gdm1_fc, gdm2_fc;
3995	bool qfsm_hang, qfwd_hang;
3996	bool ret = false;
3997
3998	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
3999	return false;
4000
4001	/* WDMA sanity checks */
4002	wdidx = mtk_r32(eth, reg: reg_map->wdma_base[0] + 0xc);
4003
4004	val = mtk_r32(eth, reg: reg_map->wdma_base[0] + 0x204);
4005	wtx_busy = FIELD_GET(MTK_TX_DMA_BUSY, val);
4006
4007	val = mtk_r32(eth, reg: reg_map->wdma_base[0] + 0x230);
4008	cdm_full = !FIELD_GET(MTK_CDM_TXFIFO_RDY, val);
4009
4010	oq_free = (!(mtk_r32(eth, reg: reg_map->pse_oq_sta) & GENMASK(24, 16)) &&
4011	!(mtk_r32(eth, reg: reg_map->pse_oq_sta + 0x4) & GENMASK(8, 0)) &&
4012	!(mtk_r32(eth, reg: reg_map->pse_oq_sta + 0x10) & GENMASK(24, 16)));
4013
4014	if (wdidx == eth->reset.wdidx && wtx_busy && cdm_full && oq_free) {
4015	if (++eth->reset.wdma_hang_count > 2) {
4016	eth->reset.wdma_hang_count = 0;
4017	ret = true;
4018	}
4019	goto out;
4020	}
4021
4022	/* QDMA sanity checks */
4023	qfsm_hang = !!mtk_r32(eth, reg: reg_map->qdma.qtx_cfg + 0x234);
4024	qfwd_hang = !mtk_r32(eth, reg: reg_map->qdma.qtx_cfg + 0x308);
4025
4026	gdm1_tx = FIELD_GET(GENMASK(31, 16), mtk_r32(eth, MTK_FE_GDM1_FSM)) > 0;
4027	gdm2_tx = FIELD_GET(GENMASK(31, 16), mtk_r32(eth, MTK_FE_GDM2_FSM)) > 0;
4028	gmac1_tx = FIELD_GET(GENMASK(31, 24), mtk_r32(eth, MTK_MAC_FSM(0))) != 1;
4029	gmac2_tx = FIELD_GET(GENMASK(31, 24), mtk_r32(eth, MTK_MAC_FSM(1))) != 1;
4030	gdm1_fc = mtk_r32(eth, reg: reg_map->gdm1_cnt + 0x24);
4031	gdm2_fc = mtk_r32(eth, reg: reg_map->gdm1_cnt + 0x64);
4032
4033	if (qfsm_hang && qfwd_hang &&
4034	((gdm1_tx && gmac1_tx && gdm1_fc < 1) ||
4035	(gdm2_tx && gmac2_tx && gdm2_fc < 1))) {
4036	if (++eth->reset.qdma_hang_count > 2) {
4037	eth->reset.qdma_hang_count = 0;
4038	ret = true;
4039	}
4040	goto out;
4041	}
4042
4043	/* ADMA sanity checks */
4044	oq_hang = !!(mtk_r32(eth, reg: reg_map->pse_oq_sta) & GENMASK(8, 0));
4045	cdm1_busy = !!(mtk_r32(eth, MTK_FE_CDM1_FSM) & GENMASK(31, 16));
4046	adma_busy = !(mtk_r32(eth, reg: reg_map->pdma.adma_rx_dbg0) & GENMASK(4, 0)) &&
4047	!(mtk_r32(eth, reg: reg_map->pdma.adma_rx_dbg0) & BIT(6));
4048
4049	if (oq_hang && cdm1_busy && adma_busy) {
4050	if (++eth->reset.adma_hang_count > 2) {
4051	eth->reset.adma_hang_count = 0;
4052	ret = true;
4053	}
4054	goto out;
4055	}
4056
4057	eth->reset.wdma_hang_count = 0;
4058	eth->reset.qdma_hang_count = 0;
4059	eth->reset.adma_hang_count = 0;
4060	out:
4061	eth->reset.wdidx = wdidx;
4062
4063	return ret;
4064	}
4065
4066	static void mtk_hw_reset_monitor_work(struct work_struct *work)
4067	{
4068	struct delayed_work *del_work = to_delayed_work(work);
4069	struct mtk_eth *eth = container_of(del_work, struct mtk_eth,
4070	reset.monitor_work);
4071
4072	if (test_bit(MTK_RESETTING, &eth->state))
4073	goto out;
4074
4075	/* DMA stuck checks */
4076	if (mtk_hw_check_dma_hang(eth))
4077	schedule_work(work: &eth->pending_work);
4078
4079	out:
4080	schedule_delayed_work(dwork: &eth->reset.monitor_work,
4081	MTK_DMA_MONITOR_TIMEOUT);
4082	}
4083
4084	static int mtk_hw_init(struct mtk_eth *eth, bool reset)
4085	{
4086	u32 dma_mask = ETHSYS_DMA_AG_MAP_PDMA | ETHSYS_DMA_AG_MAP_QDMA |
4087	ETHSYS_DMA_AG_MAP_PPE;
4088	const struct mtk_reg_map *reg_map = eth->soc->reg_map;
4089	int i, val, ret;
4090
4091	if (!reset && test_and_set_bit(nr: MTK_HW_INIT, addr: &eth->state))
4092	return 0;
4093
4094	if (!reset) {
4095	pm_runtime_enable(dev: eth->dev);
4096	pm_runtime_get_sync(dev: eth->dev);
4097
4098	ret = mtk_clk_enable(eth);
4099	if (ret)
4100	goto err_disable_pm;
4101	}
4102
4103	if (eth->ethsys)
4104	regmap_update_bits(map: eth->ethsys, ETHSYS_DMA_AG_MAP, mask: dma_mask,
4105	val: of_dma_is_coherent(np: eth->dma_dev->of_node) * dma_mask);
4106
4107	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
4108	ret = device_reset(dev: eth->dev);
4109	if (ret) {
4110	dev_err(eth->dev, "MAC reset failed!\n");
4111	goto err_disable_pm;
4112	}
4113
4114	/* set interrupt delays based on current Net DIM sample */
4115	mtk_dim_rx(work: &eth->rx_dim.work);
4116	mtk_dim_tx(work: &eth->tx_dim.work);
4117
4118	/* disable delay and normal interrupt */
4119	mtk_tx_irq_disable(eth, mask: ~0);
4120	mtk_rx_irq_disable(eth, mask: ~0);
4121
4122	return 0;
4123	}
4124
4125	msleep(msecs: 100);
4126
4127	if (reset)
4128	mtk_hw_warm_reset(eth);
4129	else
4130	mtk_hw_reset(eth);
4131
4132	/* No MT7628/88 support yet */
4133	if (reset && !MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
4134	mtk_mdio_config(eth);
4135
4136	if (mtk_is_netsys_v3_or_greater(eth)) {
4137	/* Set FE to PDMAv2 if necessary */
4138	val = mtk_r32(eth, MTK_FE_GLO_MISC);
4139	mtk_w32(eth, val: val | BIT(4), MTK_FE_GLO_MISC);
4140	}
4141
4142	if (eth->pctl) {
4143	/* Set GE2 driving and slew rate */
4144	regmap_write(map: eth->pctl, GPIO_DRV_SEL10, val: 0xa00);
4145
4146	/* set GE2 TDSEL */
4147	regmap_write(map: eth->pctl, GPIO_OD33_CTRL8, val: 0x5);
4148
4149	/* set GE2 TUNE */
4150	regmap_write(map: eth->pctl, GPIO_BIAS_CTRL, val: 0x0);
4151	}
4152
4153	/* Set linkdown as the default for each GMAC. Its own MCR would be set
4154	* up with the more appropriate value when mtk_mac_config call is being
4155	* invoked.
4156	*/
4157	for (i = 0; i < MTK_MAX_DEVS; i++) {
4158	struct net_device *dev = eth->netdev[i];
4159
4160	if (!dev)
4161	continue;
4162
4163	mtk_w32(eth, MAC_MCR_FORCE_LINK_DOWN, MTK_MAC_MCR(i));
4164	mtk_set_mcr_max_rx(mac: netdev_priv(dev),
4165	val: dev->mtu + MTK_RX_ETH_HLEN);
4166	}
4167
4168	/* Indicates CDM to parse the MTK special tag from CPU
4169	* which also is working out for untag packets.
4170	*/
4171	val = mtk_r32(eth, MTK_CDMQ_IG_CTRL);
4172	mtk_w32(eth, val: val | MTK_CDMQ_STAG_EN, MTK_CDMQ_IG_CTRL);
4173	if (mtk_is_netsys_v1(eth)) {
4174	val = mtk_r32(eth, MTK_CDMP_IG_CTRL);
4175	mtk_w32(eth, val: val | MTK_CDMP_STAG_EN, MTK_CDMP_IG_CTRL);
4176
4177	mtk_w32(eth, val: 1, MTK_CDMP_EG_CTRL);
4178	}
4179
4180	/* set interrupt delays based on current Net DIM sample */
4181	mtk_dim_rx(work: &eth->rx_dim.work);
4182	mtk_dim_tx(work: &eth->tx_dim.work);
4183
4184	/* disable delay and normal interrupt */
4185	mtk_tx_irq_disable(eth, mask: ~0);
4186	mtk_rx_irq_disable(eth, mask: ~0);
4187
4188	/* FE int grouping */
4189	mtk_w32(eth, MTK_TX_DONE_INT, reg: reg_map->pdma.int_grp);
4190	mtk_w32(eth, val: eth->soc->rx.irq_done_mask, reg: reg_map->pdma.int_grp + 4);
4191	mtk_w32(eth, MTK_TX_DONE_INT, reg: reg_map->qdma.int_grp);
4192	mtk_w32(eth, val: eth->soc->rx.irq_done_mask, reg: reg_map->qdma.int_grp + 4);
4193	mtk_w32(eth, val: 0x21021000, MTK_FE_INT_GRP);
4194
4195	if (mtk_is_netsys_v3_or_greater(eth)) {
4196	/* PSE dummy page mechanism */
4197	mtk_w32(eth, PSE_DUMMY_WORK_GDM(1) | PSE_DUMMY_WORK_GDM(2) |
4198	PSE_DUMMY_WORK_GDM(3) | DUMMY_PAGE_THR, PSE_DUMY_REQ);
4199
4200	/* PSE free buffer drop threshold */
4201	mtk_w32(eth, val: 0x00600009, PSE_IQ_REV(8));
4202
4203	/* PSE should not drop port8, port9 and port13 packets from
4204	* WDMA Tx
4205	*/
4206	mtk_w32(eth, val: 0x00002300, PSE_DROP_CFG);
4207
4208	/* PSE should drop packets to port8, port9 and port13 on WDMA Rx
4209	* ring full
4210	*/
4211	mtk_w32(eth, val: 0x00002300, PSE_PPE_DROP(0));
4212	mtk_w32(eth, val: 0x00002300, PSE_PPE_DROP(1));
4213	mtk_w32(eth, val: 0x00002300, PSE_PPE_DROP(2));
4214
4215	/* GDM and CDM Threshold */
4216	mtk_w32(eth, val: 0x08000707, MTK_CDMW0_THRES);
4217	mtk_w32(eth, val: 0x00000077, MTK_CDMW1_THRES);
4218
4219	/* Disable GDM1 RX CRC stripping */
4220	mtk_m32(eth, MTK_GDMA_STRP_CRC, set: 0, MTK_GDMA_FWD_CFG(0));
4221
4222	/* PSE GDM3 MIB counter has incorrect hw default values,
4223	* so the driver ought to read clear the values beforehand
4224	* in case ethtool retrieve wrong mib values.
4225	*/
4226	for (i = 0; i < 0x80; i += 0x4)
4227	mtk_r32(eth, reg: reg_map->gdm1_cnt + 0x100 + i);
4228	} else if (!mtk_is_netsys_v1(eth)) {
4229	/* PSE should not drop port8 and port9 packets from WDMA Tx */
4230	mtk_w32(eth, val: 0x00000300, PSE_DROP_CFG);
4231
4232	/* PSE should drop packets to port 8/9 on WDMA Rx ring full */
4233	mtk_w32(eth, val: 0x00000300, PSE_PPE_DROP(0));
4234
4235	/* PSE Free Queue Flow Control */
4236	mtk_w32(eth, val: 0x01fa01f4, PSE_FQFC_CFG2);
4237
4238	/* PSE config input queue threshold */
4239	mtk_w32(eth, val: 0x001a000e, PSE_IQ_REV(1));
4240	mtk_w32(eth, val: 0x01ff001a, PSE_IQ_REV(2));
4241	mtk_w32(eth, val: 0x000e01ff, PSE_IQ_REV(3));
4242	mtk_w32(eth, val: 0x000e000e, PSE_IQ_REV(4));
4243	mtk_w32(eth, val: 0x000e000e, PSE_IQ_REV(5));
4244	mtk_w32(eth, val: 0x000e000e, PSE_IQ_REV(6));
4245	mtk_w32(eth, val: 0x000e000e, PSE_IQ_REV(7));
4246	mtk_w32(eth, val: 0x000e000e, PSE_IQ_REV(8));
4247
4248	/* PSE config output queue threshold */
4249	mtk_w32(eth, val: 0x000f000a, PSE_OQ_TH(1));
4250	mtk_w32(eth, val: 0x001a000f, PSE_OQ_TH(2));
4251	mtk_w32(eth, val: 0x000f001a, PSE_OQ_TH(3));
4252	mtk_w32(eth, val: 0x01ff000f, PSE_OQ_TH(4));
4253	mtk_w32(eth, val: 0x000f000f, PSE_OQ_TH(5));
4254	mtk_w32(eth, val: 0x0006000f, PSE_OQ_TH(6));
4255	mtk_w32(eth, val: 0x00060006, PSE_OQ_TH(7));
4256	mtk_w32(eth, val: 0x00060006, PSE_OQ_TH(8));
4257
4258	/* GDM and CDM Threshold */
4259	mtk_w32(eth, val: 0x00000004, MTK_GDM2_THRES);
4260	mtk_w32(eth, val: 0x00000004, MTK_CDMW0_THRES);
4261	mtk_w32(eth, val: 0x00000004, MTK_CDMW1_THRES);
4262	mtk_w32(eth, val: 0x00000004, MTK_CDME0_THRES);
4263	mtk_w32(eth, val: 0x00000004, MTK_CDME1_THRES);
4264	mtk_w32(eth, val: 0x00000004, MTK_CDMM_THRES);
4265	}
4266
4267	return 0;
4268
4269	err_disable_pm:
4270	if (!reset) {
4271	pm_runtime_put_sync(dev: eth->dev);
4272	pm_runtime_disable(dev: eth->dev);
4273	}
4274
4275	return ret;
4276	}
4277
4278	static int mtk_hw_deinit(struct mtk_eth *eth)
4279	{
4280	if (!test_and_clear_bit(nr: MTK_HW_INIT, addr: &eth->state))
4281	return 0;
4282
4283	mtk_clk_disable(eth);
4284
4285	pm_runtime_put_sync(dev: eth->dev);
4286	pm_runtime_disable(dev: eth->dev);
4287
4288	return 0;
4289	}
4290
4291	static void mtk_uninit(struct net_device *dev)
4292	{
4293	struct mtk_mac *mac = netdev_priv(dev);
4294	struct mtk_eth *eth = mac->hw;
4295
4296	phylink_disconnect_phy(mac->phylink);
4297	mtk_tx_irq_disable(eth, mask: ~0);
4298	mtk_rx_irq_disable(eth, mask: ~0);
4299	}
4300
4301	static int mtk_change_mtu(struct net_device *dev, int new_mtu)
4302	{
4303	int length = new_mtu + MTK_RX_ETH_HLEN;
4304	struct mtk_mac *mac = netdev_priv(dev);
4305	struct mtk_eth *eth = mac->hw;
4306
4307	if (rcu_access_pointer(eth->prog) &&
4308	length > MTK_PP_MAX_BUF_SIZE) {
4309	netdev_err(dev, format: "Invalid MTU for XDP mode\n");
4310	return -EINVAL;
4311	}
4312
4313	mtk_set_mcr_max_rx(mac, val: length);
4314	WRITE_ONCE(dev->mtu, new_mtu);
4315
4316	return 0;
4317	}
4318
4319	static int mtk_do_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
4320	{
4321	struct mtk_mac *mac = netdev_priv(dev);
4322
4323	switch (cmd) {
4324	case SIOCGMIIPHY:
4325	case SIOCGMIIREG:
4326	case SIOCSMIIREG:
4327	return phylink_mii_ioctl(mac->phylink, ifr, cmd);
4328	default:
4329	break;
4330	}
4331
4332	return -EOPNOTSUPP;
4333	}
4334
4335	static void mtk_prepare_for_reset(struct mtk_eth *eth)
4336	{
4337	u32 val;
4338	int i;
4339
4340	/* set FE PPE ports link down */
4341	for (i = MTK_GMAC1_ID;
4342	i <= (mtk_is_netsys_v3_or_greater(eth) ? MTK_GMAC3_ID : MTK_GMAC2_ID);
4343	i += 2) {
4344	val = mtk_r32(eth, MTK_FE_GLO_CFG(i)) | MTK_FE_LINK_DOWN_P(PSE_PPE0_PORT);
4345	if (MTK_HAS_CAPS(eth->soc->caps, MTK_RSTCTRL_PPE1))
4346	val |= MTK_FE_LINK_DOWN_P(PSE_PPE1_PORT);
4347	if (MTK_HAS_CAPS(eth->soc->caps, MTK_RSTCTRL_PPE2))
4348	val |= MTK_FE_LINK_DOWN_P(PSE_PPE2_PORT);
4349	mtk_w32(eth, val, MTK_FE_GLO_CFG(i));
4350	}
4351
4352	/* adjust PPE configurations to prepare for reset */
4353	for (i = 0; i < ARRAY_SIZE(eth->ppe); i++)
4354	mtk_ppe_prepare_reset(ppe: eth->ppe[i]);
4355
4356	/* disable NETSYS interrupts */
4357	mtk_w32(eth, val: 0, MTK_FE_INT_ENABLE);
4358
4359	/* force link down GMAC */
4360	for (i = 0; i < 2; i++) {
4361	val = mtk_r32(eth, MTK_MAC_MCR(i)) & ~MAC_MCR_FORCE_LINK;
4362	mtk_w32(eth, val, MTK_MAC_MCR(i));
4363	}
4364	}
4365
4366	static void mtk_pending_work(struct work_struct *work)
4367	{
4368	struct mtk_eth *eth = container_of(work, struct mtk_eth, pending_work);
4369	unsigned long restart = 0;
4370	u32 val;
4371	int i;
4372
4373	rtnl_lock();
4374	set_bit(nr: MTK_RESETTING, addr: &eth->state);
4375
4376	mtk_prepare_for_reset(eth);
4377	mtk_wed_fe_reset();
4378	/* Run again reset preliminary configuration in order to avoid any
4379	* possible race during FE reset since it can run releasing RTNL lock.
4380	*/
4381	mtk_prepare_for_reset(eth);
4382
4383	/* stop all devices to make sure that dma is properly shut down */
4384	for (i = 0; i < MTK_MAX_DEVS; i++) {
4385	if (!eth->netdev[i] || !netif_running(dev: eth->netdev[i]))
4386	continue;
4387
4388	mtk_stop(dev: eth->netdev[i]);
4389	__set_bit(i, &restart);
4390	}
4391
4392	usleep_range(min: 15000, max: 16000);
4393
4394	if (eth->dev->pins)
4395	pinctrl_select_state(p: eth->dev->pins->p,
4396	s: eth->dev->pins->default_state);
4397	mtk_hw_init(eth, reset: true);
4398
4399	/* restart DMA and enable IRQs */
4400	for (i = 0; i < MTK_MAX_DEVS; i++) {
4401	if (!eth->netdev[i] || !test_bit(i, &restart))
4402	continue;
4403
4404	if (mtk_open(dev: eth->netdev[i])) {
4405	netif_alert(eth, ifup, eth->netdev[i],
4406	"Driver up/down cycle failed\n");
4407	dev_close(dev: eth->netdev[i]);
4408	}
4409	}
4410
4411	/* set FE PPE ports link up */
4412	for (i = MTK_GMAC1_ID;
4413	i <= (mtk_is_netsys_v3_or_greater(eth) ? MTK_GMAC3_ID : MTK_GMAC2_ID);
4414	i += 2) {
4415	val = mtk_r32(eth, MTK_FE_GLO_CFG(i)) & ~MTK_FE_LINK_DOWN_P(PSE_PPE0_PORT);
4416	if (MTK_HAS_CAPS(eth->soc->caps, MTK_RSTCTRL_PPE1))
4417	val &= ~MTK_FE_LINK_DOWN_P(PSE_PPE1_PORT);
4418	if (MTK_HAS_CAPS(eth->soc->caps, MTK_RSTCTRL_PPE2))
4419	val &= ~MTK_FE_LINK_DOWN_P(PSE_PPE2_PORT);
4420
4421	mtk_w32(eth, val, MTK_FE_GLO_CFG(i));
4422	}
4423
4424	clear_bit(nr: MTK_RESETTING, addr: &eth->state);
4425
4426	mtk_wed_fe_reset_complete();
4427
4428	rtnl_unlock();
4429	}
4430
4431	static int mtk_free_dev(struct mtk_eth *eth)
4432	{
4433	int i;
4434
4435	for (i = 0; i < MTK_MAX_DEVS; i++) {
4436	if (!eth->netdev[i])
4437	continue;
4438	free_netdev(dev: eth->netdev[i]);
4439	}
4440
4441	for (i = 0; i < ARRAY_SIZE(eth->dsa_meta); i++) {
4442	if (!eth->dsa_meta[i])
4443	break;
4444	metadata_dst_free(eth->dsa_meta[i]);
4445	}
4446
4447	return 0;
4448	}
4449
4450	static int mtk_unreg_dev(struct mtk_eth *eth)
4451	{
4452	int i;
4453
4454	for (i = 0; i < MTK_MAX_DEVS; i++) {
4455	struct mtk_mac *mac;
4456	if (!eth->netdev[i])
4457	continue;
4458	mac = netdev_priv(dev: eth->netdev[i]);
4459	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
4460	unregister_netdevice_notifier(nb: &mac->device_notifier);
4461	unregister_netdev(dev: eth->netdev[i]);
4462	}
4463
4464	return 0;
4465	}
4466
4467	static void mtk_sgmii_destroy(struct mtk_eth *eth)
4468	{
4469	int i;
4470
4471	for (i = 0; i < MTK_MAX_DEVS; i++)
4472	mtk_pcs_lynxi_destroy(pcs: eth->sgmii_pcs[i]);
4473	}
4474
4475	static int mtk_cleanup(struct mtk_eth *eth)
4476	{
4477	mtk_sgmii_destroy(eth);
4478	mtk_unreg_dev(eth);
4479	mtk_free_dev(eth);
4480	cancel_work_sync(work: &eth->pending_work);
4481	cancel_delayed_work_sync(dwork: &eth->reset.monitor_work);
4482
4483	return 0;
4484	}
4485
4486	static int mtk_get_link_ksettings(struct net_device *ndev,
4487	struct ethtool_link_ksettings *cmd)
4488	{
4489	struct mtk_mac *mac = netdev_priv(dev: ndev);
4490
4491	if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
4492	return -EBUSY;
4493
4494	return phylink_ethtool_ksettings_get(mac->phylink, cmd);
4495	}
4496
4497	static int mtk_set_link_ksettings(struct net_device *ndev,
4498	const struct ethtool_link_ksettings *cmd)
4499	{
4500	struct mtk_mac *mac = netdev_priv(dev: ndev);
4501
4502	if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
4503	return -EBUSY;
4504
4505	return phylink_ethtool_ksettings_set(mac->phylink, cmd);
4506	}
4507
4508	static void mtk_get_drvinfo(struct net_device *dev,
4509	struct ethtool_drvinfo *info)
4510	{
4511	struct mtk_mac *mac = netdev_priv(dev);
4512
4513	strscpy(info->driver, mac->hw->dev->driver->name, sizeof(info->driver));
4514	strscpy(info->bus_info, dev_name(mac->hw->dev), sizeof(info->bus_info));
4515	info->n_stats = ARRAY_SIZE(mtk_ethtool_stats);
4516	}
4517
4518	static u32 mtk_get_msglevel(struct net_device *dev)
4519	{
4520	struct mtk_mac *mac = netdev_priv(dev);
4521
4522	return mac->hw->msg_enable;
4523	}
4524
4525	static void mtk_set_msglevel(struct net_device *dev, u32 value)
4526	{
4527	struct mtk_mac *mac = netdev_priv(dev);
4528
4529	mac->hw->msg_enable = value;
4530	}
4531
4532	static int mtk_nway_reset(struct net_device *dev)
4533	{
4534	struct mtk_mac *mac = netdev_priv(dev);
4535
4536	if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
4537	return -EBUSY;
4538
4539	if (!mac->phylink)
4540	return -ENOTSUPP;
4541
4542	return phylink_ethtool_nway_reset(mac->phylink);
4543	}
4544
4545	static void mtk_get_strings(struct net_device *dev, u32 stringset, u8 *data)
4546	{
4547	int i;
4548
4549	switch (stringset) {
4550	case ETH_SS_STATS: {
4551	struct mtk_mac *mac = netdev_priv(dev);
4552
4553	for (i = 0; i < ARRAY_SIZE(mtk_ethtool_stats); i++)
4554	ethtool_puts(data: &data, str: mtk_ethtool_stats[i].str);
4555	if (mtk_page_pool_enabled(eth: mac->hw))
4556	page_pool_ethtool_stats_get_strings(data);
4557	break;
4558	}
4559	default:
4560	break;
4561	}
4562	}
4563
4564	static int mtk_get_sset_count(struct net_device *dev, int sset)
4565	{
4566	switch (sset) {
4567	case ETH_SS_STATS: {
4568	int count = ARRAY_SIZE(mtk_ethtool_stats);
4569	struct mtk_mac *mac = netdev_priv(dev);
4570
4571	if (mtk_page_pool_enabled(eth: mac->hw))
4572	count += page_pool_ethtool_stats_get_count();
4573	return count;
4574	}
4575	default:
4576	return -EOPNOTSUPP;
4577	}
4578	}
4579
4580	static void mtk_ethtool_pp_stats(struct mtk_eth *eth, u64 *data)
4581	{
4582	struct page_pool_stats stats = {};
4583	int i;
4584
4585	for (i = 0; i < ARRAY_SIZE(eth->rx_ring); i++) {
4586	struct mtk_rx_ring *ring = &eth->rx_ring[i];
4587
4588	if (!ring->page_pool)
4589	continue;
4590
4591	page_pool_get_stats(pool: ring->page_pool, stats: &stats);
4592	}
4593	page_pool_ethtool_stats_get(data, stats: &stats);
4594	}
4595
4596	static void mtk_get_ethtool_stats(struct net_device *dev,
4597	struct ethtool_stats *stats, u64 *data)
4598	{
4599	struct mtk_mac *mac = netdev_priv(dev);
4600	struct mtk_hw_stats *hwstats = mac->hw_stats;
4601	u64 *data_src, *data_dst;
4602	unsigned int start;
4603	int i;
4604
4605	if (unlikely(test_bit(MTK_RESETTING, &mac->hw->state)))
4606	return;
4607
4608	if (netif_running(dev) && netif_device_present(dev)) {
4609	if (spin_trylock_bh(lock: &hwstats->stats_lock)) {
4610	mtk_stats_update_mac(mac);
4611	spin_unlock_bh(lock: &hwstats->stats_lock);
4612	}
4613	}
4614
4615	data_src = (u64 *)hwstats;
4616
4617	do {
4618	data_dst = data;
4619	start = u64_stats_fetch_begin(syncp: &hwstats->syncp);
4620
4621	for (i = 0; i < ARRAY_SIZE(mtk_ethtool_stats); i++)
4622	*data_dst++ = *(data_src + mtk_ethtool_stats[i].offset);
4623	if (mtk_page_pool_enabled(eth: mac->hw))
4624	mtk_ethtool_pp_stats(eth: mac->hw, data: data_dst);
4625	} while (u64_stats_fetch_retry(syncp: &hwstats->syncp, start));
4626	}
4627
4628	static int mtk_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd,
4629	u32 *rule_locs)
4630	{
4631	int ret = -EOPNOTSUPP;
4632
4633	switch (cmd->cmd) {
4634	case ETHTOOL_GRXRINGS:
4635	if (dev->hw_features & NETIF_F_LRO) {
4636	cmd->data = MTK_MAX_RX_RING_NUM;
4637	ret = 0;
4638	}
4639	break;
4640	case ETHTOOL_GRXCLSRLCNT:
4641	if (dev->hw_features & NETIF_F_LRO) {
4642	struct mtk_mac *mac = netdev_priv(dev);
4643
4644	cmd->rule_cnt = mac->hwlro_ip_cnt;
4645	ret = 0;
4646	}
4647	break;
4648	case ETHTOOL_GRXCLSRULE:
4649	if (dev->hw_features & NETIF_F_LRO)
4650	ret = mtk_hwlro_get_fdir_entry(dev, cmd);
4651	break;
4652	case ETHTOOL_GRXCLSRLALL:
4653	if (dev->hw_features & NETIF_F_LRO)
4654	ret = mtk_hwlro_get_fdir_all(dev, cmd,
4655	rule_locs);
4656	break;
4657	default:
4658	break;
4659	}
4660
4661	return ret;
4662	}
4663
4664	static int mtk_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd)
4665	{
4666	int ret = -EOPNOTSUPP;
4667
4668	switch (cmd->cmd) {
4669	case ETHTOOL_SRXCLSRLINS:
4670	if (dev->hw_features & NETIF_F_LRO)
4671	ret = mtk_hwlro_add_ipaddr(dev, cmd);
4672	break;
4673	case ETHTOOL_SRXCLSRLDEL:
4674	if (dev->hw_features & NETIF_F_LRO)
4675	ret = mtk_hwlro_del_ipaddr(dev, cmd);
4676	break;
4677	default:
4678	break;
4679	}
4680
4681	return ret;
4682	}
4683
4684	static void mtk_get_pauseparam(struct net_device *dev, struct ethtool_pauseparam *pause)
4685	{
4686	struct mtk_mac *mac = netdev_priv(dev);
4687
4688	phylink_ethtool_get_pauseparam(mac->phylink, pause);
4689	}
4690
4691	static int mtk_set_pauseparam(struct net_device *dev, struct ethtool_pauseparam *pause)
4692	{
4693	struct mtk_mac *mac = netdev_priv(dev);
4694
4695	return phylink_ethtool_set_pauseparam(mac->phylink, pause);
4696	}
4697
4698	static int mtk_get_eee(struct net_device *dev, struct ethtool_keee *eee)
4699	{
4700	struct mtk_mac *mac = netdev_priv(dev);
4701
4702	return phylink_ethtool_get_eee(link: mac->phylink, eee);
4703	}
4704
4705	static int mtk_set_eee(struct net_device *dev, struct ethtool_keee *eee)
4706	{
4707	struct mtk_mac *mac = netdev_priv(dev);
4708
4709	return phylink_ethtool_set_eee(link: mac->phylink, eee);
4710	}
4711
4712	static u16 mtk_select_queue(struct net_device *dev, struct sk_buff *skb,
4713	struct net_device *sb_dev)
4714	{
4715	struct mtk_mac *mac = netdev_priv(dev);
4716	unsigned int queue = 0;
4717
4718	if (netdev_uses_dsa(dev))
4719	queue = skb_get_queue_mapping(skb) + 3;
4720	else
4721	queue = mac->id;
4722
4723	if (queue >= dev->num_tx_queues)
4724	queue = 0;
4725
4726	return queue;
4727	}
4728
4729	static const struct ethtool_ops mtk_ethtool_ops = {
4730	.get_link_ksettings = mtk_get_link_ksettings,
4731	.set_link_ksettings = mtk_set_link_ksettings,
4732	.get_drvinfo = mtk_get_drvinfo,
4733	.get_msglevel = mtk_get_msglevel,
4734	.set_msglevel = mtk_set_msglevel,
4735	.nway_reset = mtk_nway_reset,
4736	.get_link = ethtool_op_get_link,
4737	.get_strings = mtk_get_strings,
4738	.get_sset_count = mtk_get_sset_count,
4739	.get_ethtool_stats = mtk_get_ethtool_stats,
4740	.get_pauseparam = mtk_get_pauseparam,
4741	.set_pauseparam = mtk_set_pauseparam,
4742	.get_rxnfc = mtk_get_rxnfc,
4743	.set_rxnfc = mtk_set_rxnfc,
4744	.get_eee = mtk_get_eee,
4745	.set_eee = mtk_set_eee,
4746	};
4747
4748	static const struct net_device_ops mtk_netdev_ops = {
4749	.ndo_uninit = mtk_uninit,
4750	.ndo_open = mtk_open,
4751	.ndo_stop = mtk_stop,
4752	.ndo_start_xmit = mtk_start_xmit,
4753	.ndo_set_mac_address = mtk_set_mac_address,
4754	.ndo_validate_addr = eth_validate_addr,
4755	.ndo_eth_ioctl = mtk_do_ioctl,
4756	.ndo_change_mtu = mtk_change_mtu,
4757	.ndo_tx_timeout = mtk_tx_timeout,
4758	.ndo_get_stats64 = mtk_get_stats64,
4759	.ndo_fix_features = mtk_fix_features,
4760	.ndo_set_features = mtk_set_features,
4761	#ifdef CONFIG_NET_POLL_CONTROLLER
4762	.ndo_poll_controller = mtk_poll_controller,
4763	#endif
4764	.ndo_setup_tc = mtk_eth_setup_tc,
4765	.ndo_bpf = mtk_xdp,
4766	.ndo_xdp_xmit = mtk_xdp_xmit,
4767	.ndo_select_queue = mtk_select_queue,
4768	};
4769
4770	static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np)
4771	{
4772	const __be32 *_id = of_get_property(node: np, name: "reg", NULL);
4773	phy_interface_t phy_mode;
4774	struct phylink *phylink;
4775	struct mtk_mac *mac;
4776	int id, err;
4777	int txqs = 1;
4778	u32 val;
4779
4780	if (!_id) {
4781	dev_err(eth->dev, "missing mac id\n");
4782	return -EINVAL;
4783	}
4784
4785	id = be32_to_cpup(p: _id);
4786	if (id >= MTK_MAX_DEVS) {
4787	dev_err(eth->dev, "%d is not a valid mac id\n", id);
4788	return -EINVAL;
4789	}
4790
4791	if (eth->netdev[id]) {
4792	dev_err(eth->dev, "duplicate mac id found: %d\n", id);
4793	return -EINVAL;
4794	}
4795
4796	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
4797	txqs = MTK_QDMA_NUM_QUEUES;
4798
4799	eth->netdev[id] = alloc_etherdev_mqs(sizeof_priv: sizeof(*mac), txqs, rxqs: 1);
4800	if (!eth->netdev[id]) {
4801	dev_err(eth->dev, "alloc_etherdev failed\n");
4802	return -ENOMEM;
4803	}
4804	mac = netdev_priv(dev: eth->netdev[id]);
4805	eth->mac[id] = mac;
4806	mac->id = id;
4807	mac->hw = eth;
4808	mac->of_node = np;
4809
4810	err = of_get_ethdev_address(np: mac->of_node, dev: eth->netdev[id]);
4811	if (err == -EPROBE_DEFER)
4812	return err;
4813
4814	if (err) {
4815	/* If the mac address is invalid, use random mac address */
4816	eth_hw_addr_random(dev: eth->netdev[id]);
4817	dev_err(eth->dev, "generated random MAC address %pM\n",
4818	eth->netdev[id]->dev_addr);
4819	}
4820
4821	memset(mac->hwlro_ip, 0, sizeof(mac->hwlro_ip));
4822	mac->hwlro_ip_cnt = 0;
4823
4824	mac->hw_stats = devm_kzalloc(dev: eth->dev,
4825	size: sizeof(*mac->hw_stats),
4826	GFP_KERNEL);
4827	if (!mac->hw_stats) {
4828	dev_err(eth->dev, "failed to allocate counter memory\n");
4829	err = -ENOMEM;
4830	goto free_netdev;
4831	}
4832	spin_lock_init(&mac->hw_stats->stats_lock);
4833	u64_stats_init(syncp: &mac->hw_stats->syncp);
4834
4835	if (mtk_is_netsys_v3_or_greater(eth))
4836	mac->hw_stats->reg_offset = id * 0x80;
4837	else
4838	mac->hw_stats->reg_offset = id * 0x40;
4839
4840	/* phylink create */
4841	err = of_get_phy_mode(np, interface: &phy_mode);
4842	if (err) {
4843	dev_err(eth->dev, "incorrect phy-mode\n");
4844	goto free_netdev;
4845	}
4846
4847	/* mac config is not set */
4848	mac->interface = PHY_INTERFACE_MODE_NA;
4849	mac->speed = SPEED_UNKNOWN;
4850
4851	mac->phylink_config.dev = &eth->netdev[id]->dev;
4852	mac->phylink_config.type = PHYLINK_NETDEV;
4853	mac->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE |
4854	MAC_10 | MAC_100 | MAC_1000 | MAC_2500FD;
4855	mac->phylink_config.lpi_capabilities = MAC_100FD | MAC_1000FD |
4856	MAC_2500FD;
4857	mac->phylink_config.lpi_timer_default = 1000;
4858
4859	/* MT7623 gmac0 is now missing its speed-specific PLL configuration
4860	* in its .mac_config method (since state->speed is not valid there.
4861	* Disable support for MII, GMII and RGMII.
4862	*/
4863	if (!mac->hw->soc->disable_pll_modes || mac->id != 0) {
4864	__set_bit(PHY_INTERFACE_MODE_MII,
4865	mac->phylink_config.supported_interfaces);
4866	__set_bit(PHY_INTERFACE_MODE_GMII,
4867	mac->phylink_config.supported_interfaces);
4868
4869	if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_RGMII))
4870	phy_interface_set_rgmii(intf: mac->phylink_config.supported_interfaces);
4871	}
4872
4873	if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_TRGMII) && !mac->id)
4874	__set_bit(PHY_INTERFACE_MODE_TRGMII,
4875	mac->phylink_config.supported_interfaces);
4876
4877	/* TRGMII is not permitted on MT7621 if using DDR2 */
4878	if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_GMAC1_TRGMII) &&
4879	MTK_HAS_CAPS(mac->hw->soc->caps, MTK_TRGMII_MT7621_CLK)) {
4880	regmap_read(map: eth->ethsys, ETHSYS_SYSCFG, val: &val);
4881	if (val & SYSCFG_DRAM_TYPE_DDR2)
4882	__clear_bit(PHY_INTERFACE_MODE_TRGMII,
4883	mac->phylink_config.supported_interfaces);
4884	}
4885
4886	if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_SGMII)) {
4887	__set_bit(PHY_INTERFACE_MODE_SGMII,
4888	mac->phylink_config.supported_interfaces);
4889	__set_bit(PHY_INTERFACE_MODE_1000BASEX,
4890	mac->phylink_config.supported_interfaces);
4891	__set_bit(PHY_INTERFACE_MODE_2500BASEX,
4892	mac->phylink_config.supported_interfaces);
4893	}
4894
4895	if (mtk_is_netsys_v3_or_greater(eth: mac->hw) &&
4896	MTK_HAS_CAPS(mac->hw->soc->caps, MTK_ESW) &&
4897	id == MTK_GMAC1_ID) {
4898	mac->phylink_config.mac_capabilities = MAC_ASYM_PAUSE |
4899	MAC_SYM_PAUSE |
4900	MAC_10000FD;
4901	phy_interface_zero(intf: mac->phylink_config.supported_interfaces);
4902	__set_bit(PHY_INTERFACE_MODE_INTERNAL,
4903	mac->phylink_config.supported_interfaces);
4904	}
4905
4906	phylink = phylink_create(&mac->phylink_config,
4907	of_fwnode_handle(mac->of_node),
4908	phy_mode, &mtk_phylink_ops);
4909	if (IS_ERR(ptr: phylink)) {
4910	err = PTR_ERR(ptr: phylink);
4911	goto free_netdev;
4912	}
4913
4914	mac->phylink = phylink;
4915
4916	if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_2P5GPHY) &&
4917	id == MTK_GMAC2_ID)
4918	__set_bit(PHY_INTERFACE_MODE_INTERNAL,
4919	mac->phylink_config.supported_interfaces);
4920
4921	SET_NETDEV_DEV(eth->netdev[id], eth->dev);
4922	eth->netdev[id]->watchdog_timeo = 5 * HZ;
4923	eth->netdev[id]->netdev_ops = &mtk_netdev_ops;
4924	eth->netdev[id]->base_addr = (unsigned long)eth->base;
4925
4926	eth->netdev[id]->hw_features = eth->soc->hw_features;
4927	if (eth->hwlro)
4928	eth->netdev[id]->hw_features |= NETIF_F_LRO;
4929
4930	eth->netdev[id]->vlan_features = eth->soc->hw_features &
4931	~NETIF_F_HW_VLAN_CTAG_TX;
4932	eth->netdev[id]->features |= eth->soc->hw_features;
4933	eth->netdev[id]->ethtool_ops = &mtk_ethtool_ops;
4934
4935	eth->netdev[id]->irq = eth->irq[MTK_FE_IRQ_SHARED];
4936	eth->netdev[id]->dev.of_node = np;
4937
4938	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
4939	eth->netdev[id]->max_mtu = MTK_MAX_RX_LENGTH - MTK_RX_ETH_HLEN;
4940	else
4941	eth->netdev[id]->max_mtu = MTK_MAX_RX_LENGTH_2K - MTK_RX_ETH_HLEN;
4942
4943	if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) {
4944	mac->device_notifier.notifier_call = mtk_device_event;
4945	register_netdevice_notifier(nb: &mac->device_notifier);
4946	}
4947
4948	if (mtk_page_pool_enabled(eth))
4949	eth->netdev[id]->xdp_features = NETDEV_XDP_ACT_BASIC |
4950	NETDEV_XDP_ACT_REDIRECT |
4951	NETDEV_XDP_ACT_NDO_XMIT |
4952	NETDEV_XDP_ACT_NDO_XMIT_SG;
4953
4954	return 0;
4955
4956	free_netdev:
4957	free_netdev(dev: eth->netdev[id]);
4958	return err;
4959	}
4960
4961	void mtk_eth_set_dma_device(struct mtk_eth *eth, struct device *dma_dev)
4962	{
4963	struct net_device *dev, *tmp;
4964	LIST_HEAD(dev_list);
4965	int i;
4966
4967	rtnl_lock();
4968
4969	for (i = 0; i < MTK_MAX_DEVS; i++) {
4970	dev = eth->netdev[i];
4971
4972	if (!dev || !(dev->flags & IFF_UP))
4973	continue;
4974
4975	list_add_tail(new: &dev->close_list, head: &dev_list);
4976	}
4977
4978	netif_close_many(head: &dev_list, unlink: false);
4979
4980	eth->dma_dev = dma_dev;
4981
4982	list_for_each_entry_safe(dev, tmp, &dev_list, close_list) {
4983	list_del_init(entry: &dev->close_list);
4984	dev_open(dev, NULL);
4985	}
4986
4987	rtnl_unlock();
4988	}
4989
4990	static int mtk_sgmii_init(struct mtk_eth *eth)
4991	{
4992	struct device_node *np;
4993	struct regmap *regmap;
4994	u32 flags;
4995	int i;
4996
4997	for (i = 0; i < MTK_MAX_DEVS; i++) {
4998	np = of_parse_phandle(np: eth->dev->of_node, phandle_name: "mediatek,sgmiisys", index: i);
4999	if (!np)
5000	break;
5001
5002	regmap = syscon_node_to_regmap(np);
5003	flags = 0;
5004	if (of_property_read_bool(np, propname: "mediatek,pnswap"))
5005	flags |= MTK_SGMII_FLAG_PN_SWAP;
5006
5007	of_node_put(node: np);
5008
5009	if (IS_ERR(ptr: regmap))
5010	return PTR_ERR(ptr: regmap);
5011
5012	eth->sgmii_pcs[i] = mtk_pcs_lynxi_create(dev: eth->dev, regmap,
5013	ana_rgc3: eth->soc->ana_rgc3,
5014	flags);
5015	}
5016
5017	return 0;
5018	}
5019
5020	static int mtk_setup_legacy_sram(struct mtk_eth *eth, struct resource *res)
5021	{
5022	dev_warn(eth->dev, "legacy DT: using hard-coded SRAM offset.\n");
5023
5024	if (res->start + MTK_ETH_SRAM_OFFSET + MTK_ETH_NETSYS_V2_SRAM_SIZE - 1 >
5025	res->end)
5026	return -EINVAL;
5027
5028	eth->sram_pool = devm_gen_pool_create(dev: eth->dev,
5029	const_ilog2(MTK_ETH_SRAM_GRANULARITY),
5030	NUMA_NO_NODE, name: dev_name(dev: eth->dev));
5031
5032	if (IS_ERR(ptr: eth->sram_pool))
5033	return PTR_ERR(ptr: eth->sram_pool);
5034
5035	return gen_pool_add_virt(pool: eth->sram_pool,
5036	addr: (unsigned long)eth->base + MTK_ETH_SRAM_OFFSET,
5037	phys: res->start + MTK_ETH_SRAM_OFFSET,
5038	MTK_ETH_NETSYS_V2_SRAM_SIZE, NUMA_NO_NODE);
5039	}
5040
5041	static int mtk_probe(struct platform_device *pdev)
5042	{
5043	struct resource *res = NULL;
5044	struct device_node *mac_np;
5045	struct mtk_eth *eth;
5046	int err, i;
5047
5048	eth = devm_kzalloc(dev: &pdev->dev, size: sizeof(*eth), GFP_KERNEL);
5049	if (!eth)
5050	return -ENOMEM;
5051
5052	eth->soc = of_device_get_match_data(dev: &pdev->dev);
5053
5054	eth->dev = &pdev->dev;
5055	eth->dma_dev = &pdev->dev;
5056	eth->base = devm_platform_ioremap_resource(pdev, index: 0);
5057	if (IS_ERR(ptr: eth->base))
5058	return PTR_ERR(ptr: eth->base);
5059
5060	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628))
5061	eth->ip_align = NET_IP_ALIGN;
5062
5063	if (MTK_HAS_CAPS(eth->soc->caps, MTK_36BIT_DMA)) {
5064	err = dma_set_mask(dev: &pdev->dev, DMA_BIT_MASK(36));
5065	if (!err)
5066	err = dma_set_coherent_mask(dev: &pdev->dev, DMA_BIT_MASK(32));
5067
5068	if (err) {
5069	dev_err(&pdev->dev, "Wrong DMA config\n");
5070	return -EINVAL;
5071	}
5072	}
5073
5074	spin_lock_init(&eth->page_lock);
5075	spin_lock_init(&eth->tx_irq_lock);
5076	spin_lock_init(&eth->rx_irq_lock);
5077	spin_lock_init(&eth->dim_lock);
5078
5079	eth->rx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
5080	INIT_WORK(&eth->rx_dim.work, mtk_dim_rx);
5081	INIT_DELAYED_WORK(&eth->reset.monitor_work, mtk_hw_reset_monitor_work);
5082
5083	eth->tx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
5084	INIT_WORK(&eth->tx_dim.work, mtk_dim_tx);
5085
5086	if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
5087	eth->ethsys = syscon_regmap_lookup_by_phandle(np: pdev->dev.of_node,
5088	property: "mediatek,ethsys");
5089	if (IS_ERR(ptr: eth->ethsys)) {
5090	dev_err(&pdev->dev, "no ethsys regmap found\n");
5091	return PTR_ERR(ptr: eth->ethsys);
5092	}
5093	}
5094
5095	if (MTK_HAS_CAPS(eth->soc->caps, MTK_INFRA)) {
5096	eth->infra = syscon_regmap_lookup_by_phandle(np: pdev->dev.of_node,
5097	property: "mediatek,infracfg");
5098	if (IS_ERR(ptr: eth->infra)) {
5099	dev_err(&pdev->dev, "no infracfg regmap found\n");
5100	return PTR_ERR(ptr: eth->infra);
5101	}
5102	}
5103
5104	if (of_dma_is_coherent(np: pdev->dev.of_node)) {
5105	struct regmap *cci;
5106
5107	cci = syscon_regmap_lookup_by_phandle(np: pdev->dev.of_node,
5108	property: "cci-control-port");
5109	/* enable CPU/bus coherency */
5110	if (!IS_ERR(ptr: cci))
5111	regmap_write(map: cci, reg: 0, val: 3);
5112	}
5113
5114	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SGMII)) {
5115	err = mtk_sgmii_init(eth);
5116
5117	if (err)
5118	return err;
5119	}
5120
5121	if (eth->soc->required_pctl) {
5122	eth->pctl = syscon_regmap_lookup_by_phandle(np: pdev->dev.of_node,
5123	property: "mediatek,pctl");
5124	if (IS_ERR(ptr: eth->pctl)) {
5125	dev_err(&pdev->dev, "no pctl regmap found\n");
5126	err = PTR_ERR(ptr: eth->pctl);
5127	goto err_destroy_sgmii;
5128	}
5129	}
5130
5131	if (mtk_is_netsys_v2_or_greater(eth)) {
5132	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
5133	if (!res) {
5134	err = -EINVAL;
5135	goto err_destroy_sgmii;
5136	}
5137
5138	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SRAM)) {
5139	eth->sram_pool = of_gen_pool_get(np: pdev->dev.of_node,
5140	propname: "sram", index: 0);
5141	if (!eth->sram_pool) {
5142	if (!mtk_is_netsys_v3_or_greater(eth)) {
5143	err = mtk_setup_legacy_sram(eth, res);
5144	if (err)
5145	goto err_destroy_sgmii;
5146	} else {
5147	dev_err(&pdev->dev,
5148	"Could not get SRAM pool\n");
5149	err = -EINVAL;
5150	goto err_destroy_sgmii;
5151	}
5152	}
5153	}
5154	}
5155
5156	if (eth->soc->offload_version) {
5157	for (i = 0;; i++) {
5158	struct device_node *np;
5159	phys_addr_t wdma_phy;
5160	u32 wdma_base;
5161
5162	if (i >= ARRAY_SIZE(eth->soc->reg_map->wdma_base))
5163	break;
5164
5165	np = of_parse_phandle(np: pdev->dev.of_node,
5166	phandle_name: "mediatek,wed", index: i);
5167	if (!np)
5168	break;
5169
5170	wdma_base = eth->soc->reg_map->wdma_base[i];
5171	wdma_phy = res ? res->start + wdma_base : 0;
5172	mtk_wed_add_hw(np, eth, wdma: eth->base + wdma_base,
5173	wdma_phy, index: i);
5174	}
5175	}
5176
5177	err = mtk_get_irqs(pdev, eth);
5178	if (err)
5179	goto err_wed_exit;
5180
5181	for (i = 0; i < ARRAY_SIZE(eth->clks); i++) {
5182	eth->clks[i] = devm_clk_get(dev: eth->dev,
5183	id: mtk_clks_source_name[i]);
5184	if (IS_ERR(ptr: eth->clks[i])) {
5185	if (PTR_ERR(ptr: eth->clks[i]) == -EPROBE_DEFER) {
5186	err = -EPROBE_DEFER;
5187	goto err_wed_exit;
5188	}
5189	if (eth->soc->required_clks & BIT(i)) {
5190	dev_err(&pdev->dev, "clock %s not found\n",
5191	mtk_clks_source_name[i]);
5192	err = -EINVAL;
5193	goto err_wed_exit;
5194	}
5195	eth->clks[i] = NULL;
5196	}
5197	}
5198
5199	eth->msg_enable = netif_msg_init(debug_value: mtk_msg_level, MTK_DEFAULT_MSG_ENABLE);
5200	INIT_WORK(&eth->pending_work, mtk_pending_work);
5201
5202	err = mtk_hw_init(eth, reset: false);
5203	if (err)
5204	goto err_wed_exit;
5205
5206	eth->hwlro = MTK_HAS_CAPS(eth->soc->caps, MTK_HWLRO);
5207
5208	for_each_child_of_node(pdev->dev.of_node, mac_np) {
5209	if (!of_device_is_compatible(device: mac_np,
5210	"mediatek,eth-mac"))
5211	continue;
5212
5213	if (!of_device_is_available(device: mac_np))
5214	continue;
5215
5216	err = mtk_add_mac(eth, np: mac_np);
5217	if (err) {
5218	of_node_put(node: mac_np);
5219	goto err_deinit_hw;
5220	}
5221	}
5222
5223	if (MTK_HAS_CAPS(eth->soc->caps, MTK_SHARED_INT)) {
5224	err = devm_request_irq(dev: eth->dev, irq: eth->irq[MTK_FE_IRQ_SHARED],
5225	handler: mtk_handle_irq, irqflags: 0,
5226	devname: dev_name(dev: eth->dev), dev_id: eth);
5227	} else {
5228	err = devm_request_irq(dev: eth->dev, irq: eth->irq[MTK_FE_IRQ_TX],
5229	handler: mtk_handle_irq_tx, irqflags: 0,
5230	devname: dev_name(dev: eth->dev), dev_id: eth);
5231	if (err)
5232	goto err_free_dev;
5233
5234	err = devm_request_irq(dev: eth->dev, irq: eth->irq[MTK_FE_IRQ_RX],
5235	handler: mtk_handle_irq_rx, irqflags: 0,
5236	devname: dev_name(dev: eth->dev), dev_id: eth);
5237	}
5238	if (err)
5239	goto err_free_dev;
5240
5241	/* No MT7628/88 support yet */
5242	if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) {
5243	err = mtk_mdio_init(eth);
5244	if (err)
5245	goto err_free_dev;
5246	}
5247
5248	if (eth->soc->offload_version) {
5249	u8 ppe_num = eth->soc->ppe_num;
5250
5251	ppe_num = min_t(u8, ARRAY_SIZE(eth->ppe), ppe_num);
5252	for (i = 0; i < ppe_num; i++) {
5253	u32 ppe_addr = eth->soc->reg_map->ppe_base;
5254
5255	ppe_addr += (i == 2 ? 0xc00 : i * 0x400);
5256	eth->ppe[i] = mtk_ppe_init(eth, base: eth->base + ppe_addr, index: i);
5257
5258	if (!eth->ppe[i]) {
5259	err = -ENOMEM;
5260	goto err_deinit_ppe;
5261	}
5262	err = mtk_eth_offload_init(eth, id: i);
5263
5264	if (err)
5265	goto err_deinit_ppe;
5266	}
5267	}
5268
5269	for (i = 0; i < MTK_MAX_DEVS; i++) {
5270	if (!eth->netdev[i])
5271	continue;
5272
5273	err = register_netdev(dev: eth->netdev[i]);
5274	if (err) {
5275	dev_err(eth->dev, "error bringing up device\n");
5276	goto err_deinit_ppe;
5277	} else
5278	netif_info(eth, probe, eth->netdev[i],
5279	"mediatek frame engine at 0x%08lx, irq %d\n",
5280	eth->netdev[i]->base_addr, eth->irq[MTK_FE_IRQ_SHARED]);
5281	}
5282
5283	/* we run 2 devices on the same DMA ring so we need a dummy device
5284	* for NAPI to work
5285	*/
5286	eth->dummy_dev = alloc_netdev_dummy(sizeof_priv: 0);
5287	if (!eth->dummy_dev) {
5288	err = -ENOMEM;
5289	dev_err(eth->dev, "failed to allocated dummy device\n");
5290	goto err_unreg_netdev;
5291	}
5292	netif_napi_add(dev: eth->dummy_dev, napi: &eth->tx_napi, poll: mtk_napi_tx);
5293	netif_napi_add(dev: eth->dummy_dev, napi: &eth->rx_napi, poll: mtk_napi_rx);
5294
5295	platform_set_drvdata(pdev, data: eth);
5296	schedule_delayed_work(dwork: &eth->reset.monitor_work,
5297	MTK_DMA_MONITOR_TIMEOUT);
5298
5299	return 0;
5300
5301	err_unreg_netdev:
5302	mtk_unreg_dev(eth);
5303	err_deinit_ppe:
5304	mtk_ppe_deinit(eth);
5305	mtk_mdio_cleanup(eth);
5306	err_free_dev:
5307	mtk_free_dev(eth);
5308	err_deinit_hw:
5309	mtk_hw_deinit(eth);
5310	err_wed_exit:
5311	mtk_wed_exit();
5312	err_destroy_sgmii:
5313	mtk_sgmii_destroy(eth);
5314
5315	return err;
5316	}
5317
5318	static void mtk_remove(struct platform_device *pdev)
5319	{
5320	struct mtk_eth *eth = platform_get_drvdata(pdev);
5321	struct mtk_mac *mac;
5322	int i;
5323
5324	/* stop all devices to make sure that dma is properly shut down */
5325	for (i = 0; i < MTK_MAX_DEVS; i++) {
5326	if (!eth->netdev[i])
5327	continue;
5328	mtk_stop(dev: eth->netdev[i]);
5329	mac = netdev_priv(dev: eth->netdev[i]);
5330	phylink_disconnect_phy(mac->phylink);
5331	}
5332
5333	mtk_wed_exit();
5334	mtk_hw_deinit(eth);
5335
5336	netif_napi_del(napi: &eth->tx_napi);
5337	netif_napi_del(napi: &eth->rx_napi);
5338	mtk_cleanup(eth);
5339	free_netdev(dev: eth->dummy_dev);
5340	mtk_mdio_cleanup(eth);
5341	}
5342
5343	static const struct mtk_soc_data mt2701_data = {
5344	.reg_map = &mtk_reg_map,
5345	.caps = MT7623_CAPS | MTK_HWLRO,
5346	.hw_features = MTK_HW_FEATURES,
5347	.required_clks = MT7623_CLKS_BITMAP,
5348	.required_pctl = true,
5349	.version = 1,
5350	.tx = {
5351	.desc_size = sizeof(struct mtk_tx_dma),
5352	.dma_max_len = MTK_TX_DMA_BUF_LEN,
5353	.dma_len_offset = 16,
5354	.dma_size = MTK_DMA_SIZE(2K),
5355	.fq_dma_size = MTK_DMA_SIZE(2K),
5356	},
5357	.rx = {
5358	.desc_size = sizeof(struct mtk_rx_dma),
5359	.irq_done_mask = MTK_RX_DONE_INT,
5360	.dma_l4_valid = RX_DMA_L4_VALID,
5361	.dma_size = MTK_DMA_SIZE(2K),
5362	.dma_max_len = MTK_TX_DMA_BUF_LEN,
5363	.dma_len_offset = 16,
5364	},
5365	};
5366
5367	static const struct mtk_soc_data mt7621_data = {
5368	.reg_map = &mtk_reg_map,
5369	.caps = MT7621_CAPS,
5370	.hw_features = MTK_HW_FEATURES,
5371	.required_clks = MT7621_CLKS_BITMAP,
5372	.required_pctl = false,
5373	.version = 1,
5374	.offload_version = 1,
5375	.ppe_num = 1,
5376	.hash_offset = 2,
5377	.foe_entry_size = MTK_FOE_ENTRY_V1_SIZE,
5378	.tx = {
5379	.desc_size = sizeof(struct mtk_tx_dma),
5380	.dma_max_len = MTK_TX_DMA_BUF_LEN,
5381	.dma_len_offset = 16,
5382	.dma_size = MTK_DMA_SIZE(2K),
5383	.fq_dma_size = MTK_DMA_SIZE(2K),
5384	},
5385	.rx = {
5386	.desc_size = sizeof(struct mtk_rx_dma),
5387	.irq_done_mask = MTK_RX_DONE_INT,
5388	.dma_l4_valid = RX_DMA_L4_VALID,
5389	.dma_size = MTK_DMA_SIZE(2K),
5390	.dma_max_len = MTK_TX_DMA_BUF_LEN,
5391	.dma_len_offset = 16,
5392	},
5393	};
5394
5395	static const struct mtk_soc_data mt7622_data = {
5396	.reg_map = &mtk_reg_map,
5397	.ana_rgc3 = 0x2028,
5398	.caps = MT7622_CAPS | MTK_HWLRO,
5399	.hw_features = MTK_HW_FEATURES,
5400	.required_clks = MT7622_CLKS_BITMAP,
5401	.required_pctl = false,
5402	.version = 1,
5403	.offload_version = 2,
5404	.ppe_num = 1,
5405	.hash_offset = 2,
5406	.has_accounting = true,
5407	.foe_entry_size = MTK_FOE_ENTRY_V1_SIZE,
5408	.tx = {
5409	.desc_size = sizeof(struct mtk_tx_dma),
5410	.dma_max_len = MTK_TX_DMA_BUF_LEN,
5411	.dma_len_offset = 16,
5412	.dma_size = MTK_DMA_SIZE(2K),
5413	.fq_dma_size = MTK_DMA_SIZE(2K),
5414	},
5415	.rx = {
5416	.desc_size = sizeof(struct mtk_rx_dma),
5417	.irq_done_mask = MTK_RX_DONE_INT,
5418	.dma_l4_valid = RX_DMA_L4_VALID,
5419	.dma_size = MTK_DMA_SIZE(2K),
5420	.dma_max_len = MTK_TX_DMA_BUF_LEN,
5421	.dma_len_offset = 16,
5422	},
5423	};
5424
5425	static const struct mtk_soc_data mt7623_data = {
5426	.reg_map = &mtk_reg_map,
5427	.caps = MT7623_CAPS | MTK_HWLRO,
5428	.hw_features = MTK_HW_FEATURES,
5429	.required_clks = MT7623_CLKS_BITMAP,
5430	.required_pctl = true,
5431	.version = 1,
5432	.offload_version = 1,
5433	.ppe_num = 1,
5434	.hash_offset = 2,
5435	.foe_entry_size = MTK_FOE_ENTRY_V1_SIZE,
5436	.disable_pll_modes = true,
5437	.tx = {
5438	.desc_size = sizeof(struct mtk_tx_dma),
5439	.dma_max_len = MTK_TX_DMA_BUF_LEN,
5440	.dma_len_offset = 16,
5441	.dma_size = MTK_DMA_SIZE(2K),
5442	.fq_dma_size = MTK_DMA_SIZE(2K),
5443	},
5444	.rx = {
5445	.desc_size = sizeof(struct mtk_rx_dma),
5446	.irq_done_mask = MTK_RX_DONE_INT,
5447	.dma_l4_valid = RX_DMA_L4_VALID,
5448	.dma_size = MTK_DMA_SIZE(2K),
5449	.dma_max_len = MTK_TX_DMA_BUF_LEN,
5450	.dma_len_offset = 16,
5451	},
5452	};
5453
5454	static const struct mtk_soc_data mt7629_data = {
5455	.reg_map = &mtk_reg_map,
5456	.ana_rgc3 = 0x128,
5457	.caps = MT7629_CAPS | MTK_HWLRO,
5458	.hw_features = MTK_HW_FEATURES,
5459	.required_clks = MT7629_CLKS_BITMAP,
5460	.required_pctl = false,
5461	.has_accounting = true,
5462	.version = 1,
5463	.tx = {
5464	.desc_size = sizeof(struct mtk_tx_dma),
5465	.dma_max_len = MTK_TX_DMA_BUF_LEN,
5466	.dma_len_offset = 16,
5467	.dma_size = MTK_DMA_SIZE(2K),
5468	.fq_dma_size = MTK_DMA_SIZE(2K),
5469	},
5470	.rx = {
5471	.desc_size = sizeof(struct mtk_rx_dma),
5472	.irq_done_mask = MTK_RX_DONE_INT,
5473	.dma_l4_valid = RX_DMA_L4_VALID,
5474	.dma_size = MTK_DMA_SIZE(2K),
5475	.dma_max_len = MTK_TX_DMA_BUF_LEN,
5476	.dma_len_offset = 16,
5477	},
5478	};
5479
5480	static const struct mtk_soc_data mt7981_data = {
5481	.reg_map = &mt7986_reg_map,
5482	.ana_rgc3 = 0x128,
5483	.caps = MT7981_CAPS,
5484	.hw_features = MTK_HW_FEATURES,
5485	.required_clks = MT7981_CLKS_BITMAP,
5486	.required_pctl = false,
5487	.version = 2,
5488	.offload_version = 2,
5489	.ppe_num = 2,
5490	.hash_offset = 4,
5491	.has_accounting = true,
5492	.foe_entry_size = MTK_FOE_ENTRY_V2_SIZE,
5493	.tx = {
5494	.desc_size = sizeof(struct mtk_tx_dma_v2),
5495	.dma_max_len = MTK_TX_DMA_BUF_LEN_V2,
5496	.dma_len_offset = 8,
5497	.dma_size = MTK_DMA_SIZE(2K),
5498	.fq_dma_size = MTK_DMA_SIZE(2K),
5499	},
5500	.rx = {
5501	.desc_size = sizeof(struct mtk_rx_dma),
5502	.irq_done_mask = MTK_RX_DONE_INT,
5503	.dma_l4_valid = RX_DMA_L4_VALID_V2,
5504	.dma_max_len = MTK_TX_DMA_BUF_LEN,
5505	.dma_len_offset = 16,
5506	.dma_size = MTK_DMA_SIZE(2K),
5507	},
5508	};
5509
5510	static const struct mtk_soc_data mt7986_data = {
5511	.reg_map = &mt7986_reg_map,
5512	.ana_rgc3 = 0x128,
5513	.caps = MT7986_CAPS,
5514	.hw_features = MTK_HW_FEATURES,
5515	.required_clks = MT7986_CLKS_BITMAP,
5516	.required_pctl = false,
5517	.version = 2,
5518	.offload_version = 2,
5519	.ppe_num = 2,
5520	.hash_offset = 4,
5521	.has_accounting = true,
5522	.foe_entry_size = MTK_FOE_ENTRY_V2_SIZE,
5523	.tx = {
5524	.desc_size = sizeof(struct mtk_tx_dma_v2),
5525	.dma_max_len = MTK_TX_DMA_BUF_LEN_V2,
5526	.dma_len_offset = 8,
5527	.dma_size = MTK_DMA_SIZE(2K),
5528	.fq_dma_size = MTK_DMA_SIZE(2K),
5529	},
5530	.rx = {
5531	.desc_size = sizeof(struct mtk_rx_dma),
5532	.irq_done_mask = MTK_RX_DONE_INT,
5533	.dma_l4_valid = RX_DMA_L4_VALID_V2,
5534	.dma_max_len = MTK_TX_DMA_BUF_LEN,
5535	.dma_len_offset = 16,
5536	.dma_size = MTK_DMA_SIZE(2K),
5537	},
5538	};
5539
5540	static const struct mtk_soc_data mt7988_data = {
5541	.reg_map = &mt7988_reg_map,
5542	.ana_rgc3 = 0x128,
5543	.caps = MT7988_CAPS,
5544	.hw_features = MTK_HW_FEATURES,
5545	.required_clks = MT7988_CLKS_BITMAP,
5546	.required_pctl = false,
5547	.version = 3,
5548	.offload_version = 2,
5549	.ppe_num = 3,
5550	.hash_offset = 4,
5551	.has_accounting = true,
5552	.foe_entry_size = MTK_FOE_ENTRY_V3_SIZE,
5553	.tx = {
5554	.desc_size = sizeof(struct mtk_tx_dma_v2),
5555	.dma_max_len = MTK_TX_DMA_BUF_LEN_V2,
5556	.dma_len_offset = 8,
5557	.dma_size = MTK_DMA_SIZE(2K),
5558	.fq_dma_size = MTK_DMA_SIZE(4K),
5559	},
5560	.rx = {
5561	.desc_size = sizeof(struct mtk_rx_dma_v2),
5562	.irq_done_mask = MTK_RX_DONE_INT_V2,
5563	.dma_l4_valid = RX_DMA_L4_VALID_V2,
5564	.dma_max_len = MTK_TX_DMA_BUF_LEN_V2,
5565	.dma_len_offset = 8,
5566	.dma_size = MTK_DMA_SIZE(2K),
5567	},
5568	};
5569
5570	static const struct mtk_soc_data rt5350_data = {
5571	.reg_map = &mt7628_reg_map,
5572	.caps = MT7628_CAPS,
5573	.hw_features = MTK_HW_FEATURES_MT7628,
5574	.required_clks = MT7628_CLKS_BITMAP,
5575	.required_pctl = false,
5576	.version = 1,
5577	.tx = {
5578	.desc_size = sizeof(struct mtk_tx_dma),
5579	.dma_max_len = MTK_TX_DMA_BUF_LEN,
5580	.dma_len_offset = 16,
5581	.dma_size = MTK_DMA_SIZE(2K),
5582	},
5583	.rx = {
5584	.desc_size = sizeof(struct mtk_rx_dma),
5585	.irq_done_mask = MTK_RX_DONE_INT,
5586	.dma_l4_valid = RX_DMA_L4_VALID_PDMA,
5587	.dma_max_len = MTK_TX_DMA_BUF_LEN,
5588	.dma_len_offset = 16,
5589	.dma_size = MTK_DMA_SIZE(2K),
5590	},
5591	};
5592
5593	const struct of_device_id of_mtk_match[] = {
5594	{ .compatible = "mediatek,mt2701-eth", .data = &mt2701_data },
5595	{ .compatible = "mediatek,mt7621-eth", .data = &mt7621_data },
5596	{ .compatible = "mediatek,mt7622-eth", .data = &mt7622_data },
5597	{ .compatible = "mediatek,mt7623-eth", .data = &mt7623_data },
5598	{ .compatible = "mediatek,mt7629-eth", .data = &mt7629_data },
5599	{ .compatible = "mediatek,mt7981-eth", .data = &mt7981_data },
5600	{ .compatible = "mediatek,mt7986-eth", .data = &mt7986_data },
5601	{ .compatible = "mediatek,mt7988-eth", .data = &mt7988_data },
5602	{ .compatible = "ralink,rt5350-eth", .data = &rt5350_data },
5603	{},
5604	};
5605	MODULE_DEVICE_TABLE(of, of_mtk_match);
5606
5607	static struct platform_driver mtk_driver = {
5608	.probe = mtk_probe,
5609	.remove = mtk_remove,
5610	.driver = {
5611	.name = "mtk_soc_eth",
5612	.of_match_table = of_mtk_match,
5613	},
5614	};
5615
5616	module_platform_driver(mtk_driver);
5617
5618	MODULE_LICENSE("GPL");
5619	MODULE_AUTHOR("John Crispin <blogic@openwrt.org>");
5620	MODULE_DESCRIPTION("Ethernet driver for MediaTek SoC");
5621	MODULE_IMPORT_NS("NETDEV_INTERNAL");
5622