1	/*
2	* Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3	*
4	* This software is available to you under a choice of one of two
5	* licenses. You may choose to be licensed under the terms of the GNU
6	* General Public License (GPL) Version 2, available from the file
7	* COPYING in the main directory of this source tree, or the
8	* OpenIB.org BSD license below:
9	*
10	* Redistribution and use in source and binary forms, with or
11	* without modification, are permitted provided that the following
12	* conditions are met:
13	*
14	* - Redistributions of source code must retain the above
15	* copyright notice, this list of conditions and the following
16	* disclaimer.
17	*
18	* - Redistributions in binary form must reproduce the above
19	* copyright notice, this list of conditions and the following
20	* disclaimer in the documentation and/or other materials
21	* provided with the distribution.
22	*
23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30	* SOFTWARE.
31	*/
32
33	#include <linux/etherdevice.h>
34	#include <linux/idr.h>
35	#include <linux/mlx5/driver.h>
36	#include <linux/mlx5/mlx5_ifc.h>
37	#include <linux/mlx5/vport.h>
38	#include <linux/mlx5/fs.h>
39	#include "mlx5_core.h"
40	#include "eswitch.h"
41	#include "esw/indir_table.h"
42	#include "esw/acl/ofld.h"
43	#include "rdma.h"
44	#include "en.h"
45	#include "fs_core.h"
46	#include "lib/mlx5.h"
47	#include "lib/devcom.h"
48	#include "lib/eq.h"
49	#include "lib/fs_chains.h"
50	#include "en_tc.h"
51	#include "en/mapping.h"
52	#include "devlink.h"
53	#include "lag/lag.h"
54	#include "en/tc/post_meter.h"
55	#include "fw_reset.h"
56
57	/* There are two match-all miss flows, one for unicast dst mac and
58	* one for multicast.
59	*/
60	#define MLX5_ESW_MISS_FLOWS (2)
61	#define UPLINK_REP_INDEX 0
62
63	#define MLX5_ESW_VPORT_TBL_SIZE 128
64	#define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4
65
66	#define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1)
67
68	#define MLX5_ESW_MAX_CTRL_EQS 4
69	#define MLX5_ESW_DEFAULT_SF_COMP_EQS 8
70
71	static struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = {
72	.max_fte = MLX5_ESW_VPORT_TBL_SIZE,
73	.max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS,
74	.flags = 0,
75	};
76
77	static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw,
78	u16 vport_num)
79	{
80	return xa_load(&esw->offloads.vport_reps, index: vport_num);
81	}
82
83	static void
84	mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw,
85	struct mlx5_flow_spec *spec,
86	struct mlx5_esw_flow_attr *attr)
87	{
88	if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) || !attr || !attr->in_rep)
89	return;
90
91	if (attr->int_port) {
92	spec->flow_context.flow_source = mlx5e_tc_int_port_get_flow_source(int_port: attr->int_port);
93
94	return;
95	}
96
97	spec->flow_context.flow_source = (attr->in_rep->vport == MLX5_VPORT_UPLINK) ?
98	MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK :
99	MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
100	}
101
102	/* Actually only the upper 16 bits of reg c0 need to be cleared, but the lower 16 bits
103	* are not needed as well in the following process. So clear them all for simplicity.
104	*/
105	void
106	mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec)
107	{
108	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
109	void *misc2;
110
111	misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
112	MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
113
114	misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
115	MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
116
117	if (!memchr_inv(p: misc2, c: 0, MLX5_ST_SZ_BYTES(fte_match_set_misc2)))
118	spec->match_criteria_enable &= ~MLX5_MATCH_MISC_PARAMETERS_2;
119	}
120	}
121
122	static void
123	mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw,
124	struct mlx5_flow_spec *spec,
125	struct mlx5_flow_attr *attr,
126	struct mlx5_eswitch *src_esw,
127	u16 vport)
128	{
129	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
130	u32 metadata;
131	void *misc2;
132	void *misc;
133
134	/* Use metadata matching because vport is not represented by single
135	* VHCA in dual-port RoCE mode, and matching on source vport may fail.
136	*/
137	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
138	if (mlx5_esw_indir_table_decap_vport(attr))
139	vport = mlx5_esw_indir_table_decap_vport(attr);
140
141	if (!attr->chain && esw_attr && esw_attr->int_port)
142	metadata =
143	mlx5e_tc_int_port_get_metadata_for_match(int_port: esw_attr->int_port);
144	else
145	metadata =
146	mlx5_eswitch_get_vport_metadata_for_match(esw: src_esw, vport_num: vport);
147
148	misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
149	MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, metadata);
150
151	misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
152	MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0,
153	mlx5_eswitch_get_vport_metadata_mask());
154
155	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
156	} else {
157	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
158	MLX5_SET(fte_match_set_misc, misc, source_port, vport);
159
160	if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
161	MLX5_SET(fte_match_set_misc, misc,
162	source_eswitch_owner_vhca_id,
163	MLX5_CAP_GEN(src_esw->dev, vhca_id));
164
165	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
166	MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
167	if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
168	MLX5_SET_TO_ONES(fte_match_set_misc, misc,
169	source_eswitch_owner_vhca_id);
170
171	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
172	}
173	}
174
175	static int
176	esw_setup_decap_indir(struct mlx5_eswitch *esw,
177	struct mlx5_flow_attr *attr)
178	{
179	struct mlx5_flow_table *ft;
180
181	if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
182	return -EOPNOTSUPP;
183
184	ft = mlx5_esw_indir_table_get(esw, attr,
185	vport: mlx5_esw_indir_table_decap_vport(attr), decap: true);
186	return PTR_ERR_OR_ZERO(ptr: ft);
187	}
188
189	static void
190	esw_cleanup_decap_indir(struct mlx5_eswitch *esw,
191	struct mlx5_flow_attr *attr)
192	{
193	if (mlx5_esw_indir_table_decap_vport(attr))
194	mlx5_esw_indir_table_put(esw,
195	vport: mlx5_esw_indir_table_decap_vport(attr),
196	decap: true);
197	}
198
199	static int
200	esw_setup_mtu_dest(struct mlx5_flow_destination *dest,
201	struct mlx5e_meter_attr *meter,
202	int i)
203	{
204	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_RANGE;
205	dest[i].range.field = MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN;
206	dest[i].range.min = 0;
207	dest[i].range.max = meter->params.mtu;
208	dest[i].range.hit_ft = mlx5e_post_meter_get_mtu_true_ft(post_meter: meter->post_meter);
209	dest[i].range.miss_ft = mlx5e_post_meter_get_mtu_false_ft(post_meter: meter->post_meter);
210
211	return 0;
212	}
213
214	static int
215	esw_setup_sampler_dest(struct mlx5_flow_destination *dest,
216	struct mlx5_flow_act *flow_act,
217	u32 sampler_id,
218	int i)
219	{
220	flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
221	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER;
222	dest[i].sampler_id = sampler_id;
223
224	return 0;
225	}
226
227	static int
228	esw_setup_ft_dest(struct mlx5_flow_destination *dest,
229	struct mlx5_flow_act *flow_act,
230	struct mlx5_eswitch *esw,
231	struct mlx5_flow_attr *attr,
232	int i)
233	{
234	flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
235	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
236	dest[i].ft = attr->dest_ft;
237
238	if (mlx5_esw_indir_table_decap_vport(attr))
239	return esw_setup_decap_indir(esw, attr);
240	return 0;
241	}
242
243	static void
244	esw_setup_accept_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
245	struct mlx5_fs_chains *chains, int i)
246	{
247	if (mlx5_chains_ignore_flow_level_supported(chains))
248	flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
249	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
250	dest[i].ft = mlx5_chains_get_tc_end_ft(chains);
251	}
252
253	static void
254	esw_setup_slow_path_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
255	struct mlx5_eswitch *esw, int i)
256	{
257	if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level))
258	flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
259	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
260	dest[i].ft = mlx5_eswitch_get_slow_fdb(esw);
261	}
262
263	static int
264	esw_setup_chain_dest(struct mlx5_flow_destination *dest,
265	struct mlx5_flow_act *flow_act,
266	struct mlx5_fs_chains *chains,
267	u32 chain, u32 prio, u32 level,
268	int i)
269	{
270	struct mlx5_flow_table *ft;
271
272	flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
273	ft = mlx5_chains_get_table(chains, chain, prio, level);
274	if (IS_ERR(ptr: ft))
275	return PTR_ERR(ptr: ft);
276
277	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
278	dest[i].ft = ft;
279	return 0;
280	}
281
282	static void esw_put_dest_tables_loop(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr,
283	int from, int to)
284	{
285	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
286	struct mlx5_fs_chains *chains = esw_chains(esw);
287	int i;
288
289	for (i = from; i < to; i++)
290	if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
291	mlx5_chains_put_table(chains, chain: 0, prio: 1, level: 0);
292	else if (mlx5_esw_indir_table_needed(esw, attr, vport_num: esw_attr->dests[i].vport,
293	dest_mdev: esw_attr->dests[i].mdev))
294	mlx5_esw_indir_table_put(esw, vport: esw_attr->dests[i].vport, decap: false);
295	}
296
297	static bool
298	esw_is_chain_src_port_rewrite(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr)
299	{
300	int i;
301
302	for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
303	if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
304	return true;
305	return false;
306	}
307
308	static int
309	esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination *dest,
310	struct mlx5_flow_act *flow_act,
311	struct mlx5_eswitch *esw,
312	struct mlx5_fs_chains *chains,
313	struct mlx5_flow_attr *attr,
314	int *i)
315	{
316	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
317	int err;
318
319	if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
320	return -EOPNOTSUPP;
321
322	/* flow steering cannot handle more than one dest with the same ft
323	* in a single flow
324	*/
325	if (esw_attr->out_count - esw_attr->split_count > 1)
326	return -EOPNOTSUPP;
327
328	err = esw_setup_chain_dest(dest, flow_act, chains, chain: attr->dest_chain, prio: 1, level: 0, i: *i);
329	if (err)
330	return err;
331
332	if (esw_attr->dests[esw_attr->split_count].pkt_reformat) {
333	flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
334	flow_act->pkt_reformat = esw_attr->dests[esw_attr->split_count].pkt_reformat;
335	}
336	(*i)++;
337
338	return 0;
339	}
340
341	static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch *esw,
342	struct mlx5_flow_attr *attr)
343	{
344	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
345
346	esw_put_dest_tables_loop(esw, attr, from: esw_attr->split_count, to: esw_attr->out_count);
347	}
348
349	static bool
350	esw_is_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
351	{
352	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
353	bool result = false;
354	int i;
355
356	/* Indirect table is supported only for flows with in_port uplink
357	* and the destination is vport on the same eswitch as the uplink,
358	* return false in case at least one of destinations doesn't meet
359	* this criteria.
360	*/
361	for (i = esw_attr->split_count; i < esw_attr->out_count; i++) {
362	if (esw_attr->dests[i].vport_valid &&
363	mlx5_esw_indir_table_needed(esw, attr, vport_num: esw_attr->dests[i].vport,
364	dest_mdev: esw_attr->dests[i].mdev)) {
365	result = true;
366	} else {
367	result = false;
368	break;
369	}
370	}
371	return result;
372	}
373
374	static int
375	esw_setup_indir_table(struct mlx5_flow_destination *dest,
376	struct mlx5_flow_act *flow_act,
377	struct mlx5_eswitch *esw,
378	struct mlx5_flow_attr *attr,
379	int *i)
380	{
381	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
382	int j, err;
383
384	if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
385	return -EOPNOTSUPP;
386
387	for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) {
388	flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
389	dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
390
391	dest[*i].ft = mlx5_esw_indir_table_get(esw, attr,
392	vport: esw_attr->dests[j].vport, decap: false);
393	if (IS_ERR(ptr: dest[*i].ft)) {
394	err = PTR_ERR(ptr: dest[*i].ft);
395	goto err_indir_tbl_get;
396	}
397	}
398
399	if (mlx5_esw_indir_table_decap_vport(attr)) {
400	err = esw_setup_decap_indir(esw, attr);
401	if (err)
402	goto err_indir_tbl_get;
403	}
404
405	return 0;
406
407	err_indir_tbl_get:
408	esw_put_dest_tables_loop(esw, attr, from: esw_attr->split_count, to: j);
409	return err;
410	}
411
412	static void esw_cleanup_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
413	{
414	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
415
416	esw_put_dest_tables_loop(esw, attr, from: esw_attr->split_count, to: esw_attr->out_count);
417	esw_cleanup_decap_indir(esw, attr);
418	}
419
420	static void
421	esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level)
422	{
423	mlx5_chains_put_table(chains, chain, prio, level);
424	}
425
426	static bool esw_same_vhca_id(struct mlx5_core_dev *mdev1, struct mlx5_core_dev *mdev2)
427	{
428	return MLX5_CAP_GEN(mdev1, vhca_id) == MLX5_CAP_GEN(mdev2, vhca_id);
429	}
430
431	static bool esw_setup_uplink_fwd_ipsec_needed(struct mlx5_eswitch *esw,
432	struct mlx5_esw_flow_attr *esw_attr,
433	int attr_idx)
434	{
435	if (esw->offloads.ft_ipsec_tx_pol &&
436	esw_attr->dests[attr_idx].vport_valid &&
437	esw_attr->dests[attr_idx].vport == MLX5_VPORT_UPLINK &&
438	/* To be aligned with software, encryption is needed only for tunnel device */
439	(esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) &&
440	esw_attr->dests[attr_idx].vport != esw_attr->in_rep->vport &&
441	esw_same_vhca_id(mdev1: esw_attr->dests[attr_idx].mdev, mdev2: esw->dev))
442	return true;
443
444	return false;
445	}
446
447	static bool esw_flow_dests_fwd_ipsec_check(struct mlx5_eswitch *esw,
448	struct mlx5_esw_flow_attr *esw_attr)
449	{
450	int i;
451
452	if (!esw->offloads.ft_ipsec_tx_pol)
453	return true;
454
455	for (i = 0; i < esw_attr->split_count; i++)
456	if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx: i))
457	return false;
458
459	for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
460	if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx: i) &&
461	(esw_attr->out_count - esw_attr->split_count > 1))
462	return false;
463
464	return true;
465	}
466
467	static void
468	esw_setup_dest_fwd_vport(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
469	struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
470	int attr_idx, int dest_idx, bool pkt_reformat)
471	{
472	dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
473	dest[dest_idx].vport.num = esw_attr->dests[attr_idx].vport;
474	if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
475	dest[dest_idx].vport.vhca_id =
476	MLX5_CAP_GEN(esw_attr->dests[attr_idx].mdev, vhca_id);
477	dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
478	if (dest[dest_idx].vport.num == MLX5_VPORT_UPLINK &&
479	mlx5_lag_is_mpesw(dev: esw->dev))
480	dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_UPLINK;
481	}
482	if (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) {
483	if (pkt_reformat) {
484	flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
485	flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
486	}
487	dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID;
488	dest[dest_idx].vport.pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
489	}
490	}
491
492	static void
493	esw_setup_dest_fwd_ipsec(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
494	struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
495	int attr_idx, int dest_idx, bool pkt_reformat)
496	{
497	dest[dest_idx].ft = esw->offloads.ft_ipsec_tx_pol;
498	dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
499	if (pkt_reformat &&
500	esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) {
501	flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
502	flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
503	}
504	}
505
506	static void
507	esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
508	struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
509	int attr_idx, int dest_idx, bool pkt_reformat)
510	{
511	if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx))
512	esw_setup_dest_fwd_ipsec(dest, flow_act, esw, esw_attr,
513	attr_idx, dest_idx, pkt_reformat);
514	else
515	esw_setup_dest_fwd_vport(dest, flow_act, esw, esw_attr,
516	attr_idx, dest_idx, pkt_reformat);
517	}
518
519	static int
520	esw_setup_vport_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
521	struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
522	int i)
523	{
524	int j;
525
526	for (j = esw_attr->split_count; j < esw_attr->out_count; j++, i++)
527	esw_setup_vport_dest(dest, flow_act, esw, esw_attr, attr_idx: j, dest_idx: i, pkt_reformat: true);
528	return i;
529	}
530
531	static bool
532	esw_src_port_rewrite_supported(struct mlx5_eswitch *esw)
533	{
534	return MLX5_CAP_GEN(esw->dev, reg_c_preserve) &&
535	mlx5_eswitch_vport_match_metadata_enabled(esw) &&
536	MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level);
537	}
538
539	static bool
540	esw_dests_to_int_external(struct mlx5_flow_destination *dests, int max_dest)
541	{
542	bool internal_dest = false, external_dest = false;
543	int i;
544
545	for (i = 0; i < max_dest; i++) {
546	if (dests[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT &&
547	dests[i].type != MLX5_FLOW_DESTINATION_TYPE_UPLINK)
548	continue;
549
550	/* Uplink dest is external, but considered as internal
551	* if there is reformat because firmware uses LB+hairpin to support it.
552	*/
553	if (dests[i].vport.num == MLX5_VPORT_UPLINK &&
554	!(dests[i].vport.flags & MLX5_FLOW_DEST_VPORT_REFORMAT_ID))
555	external_dest = true;
556	else
557	internal_dest = true;
558
559	if (internal_dest && external_dest)
560	return true;
561	}
562
563	return false;
564	}
565
566	static int
567	esw_setup_dests(struct mlx5_flow_destination *dest,
568	struct mlx5_flow_act *flow_act,
569	struct mlx5_eswitch *esw,
570	struct mlx5_flow_attr *attr,
571	struct mlx5_flow_spec *spec,
572	int *i)
573	{
574	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
575	struct mlx5_fs_chains *chains = esw_chains(esw);
576	int err = 0;
577
578	if (!mlx5_eswitch_termtbl_required(esw, attr, flow_act, spec) &&
579	esw_src_port_rewrite_supported(esw))
580	attr->flags |= MLX5_ATTR_FLAG_SRC_REWRITE;
581
582	if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH) {
583	esw_setup_slow_path_dest(dest, flow_act, esw, i: *i);
584	(*i)++;
585	goto out;
586	}
587
588	if (attr->flags & MLX5_ATTR_FLAG_SAMPLE) {
589	esw_setup_sampler_dest(dest, flow_act, sampler_id: attr->sample_attr.sampler_id, i: *i);
590	(*i)++;
591	} else if (attr->flags & MLX5_ATTR_FLAG_ACCEPT) {
592	esw_setup_accept_dest(dest, flow_act, chains, i: *i);
593	(*i)++;
594	} else if (attr->flags & MLX5_ATTR_FLAG_MTU) {
595	err = esw_setup_mtu_dest(dest, meter: &attr->meter_attr, i: *i);
596	(*i)++;
597	} else if (esw_is_indir_table(esw, attr)) {
598	err = esw_setup_indir_table(dest, flow_act, esw, attr, i);
599	} else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) {
600	err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i);
601	} else {
602	*i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, i: *i);
603
604	if (attr->dest_ft) {
605	err = esw_setup_ft_dest(dest, flow_act, esw, attr, i: *i);
606	(*i)++;
607	} else if (attr->dest_chain) {
608	err = esw_setup_chain_dest(dest, flow_act, chains, chain: attr->dest_chain,
609	prio: 1, level: 0, i: *i);
610	(*i)++;
611	}
612	}
613
614	if (attr->extra_split_ft) {
615	flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
616	dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
617	dest[*i].ft = attr->extra_split_ft;
618	(*i)++;
619	}
620
621	out:
622	return err;
623	}
624
625	static void
626	esw_cleanup_dests(struct mlx5_eswitch *esw,
627	struct mlx5_flow_attr *attr)
628	{
629	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
630	struct mlx5_fs_chains *chains = esw_chains(esw);
631
632	if (attr->dest_ft) {
633	esw_cleanup_decap_indir(esw, attr);
634	} else if (!mlx5e_tc_attr_flags_skip(attr_flags: attr->flags)) {
635	if (attr->dest_chain)
636	esw_cleanup_chain_dest(chains, chain: attr->dest_chain, prio: 1, level: 0);
637	else if (esw_is_indir_table(esw, attr))
638	esw_cleanup_indir_table(esw, attr);
639	else if (esw_is_chain_src_port_rewrite(esw, esw_attr))
640	esw_cleanup_chain_src_port_rewrite(esw, attr);
641	}
642	}
643
644	static void
645	esw_setup_meter(struct mlx5_flow_attr *attr, struct mlx5_flow_act *flow_act)
646	{
647	struct mlx5e_flow_meter_handle *meter;
648
649	meter = attr->meter_attr.meter;
650	flow_act->exe_aso.type = attr->exe_aso_type;
651	flow_act->exe_aso.object_id = meter->obj_id;
652	flow_act->exe_aso.base_id = mlx5e_flow_meter_get_base_id(meter);
653	flow_act->exe_aso.flow_meter.meter_idx = meter->idx;
654	flow_act->exe_aso.flow_meter.init_color = MLX5_FLOW_METER_COLOR_GREEN;
655	/* use metadata reg 5 for packet color */
656	flow_act->exe_aso.return_reg_id = 5;
657	}
658
659	struct mlx5_flow_handle *
660	mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
661	struct mlx5_flow_spec *spec,
662	struct mlx5_flow_attr *attr)
663	{
664	struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
665	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
666	struct mlx5_fs_chains *chains = esw_chains(esw);
667	bool split = !!(esw_attr->split_count);
668	struct mlx5_vport_tbl_attr fwd_attr;
669	struct mlx5_flow_destination *dest;
670	struct mlx5_flow_handle *rule;
671	struct mlx5_flow_table *fdb;
672	int i = 0;
673
674	if (esw->mode != MLX5_ESWITCH_OFFLOADS)
675	return ERR_PTR(error: -EOPNOTSUPP);
676
677	if (!mlx5_eswitch_vlan_actions_supported(dev: esw->dev, vlan_depth: 1))
678	return ERR_PTR(error: -EOPNOTSUPP);
679
680	if (!esw_flow_dests_fwd_ipsec_check(esw, esw_attr))
681	return ERR_PTR(error: -EOPNOTSUPP);
682
683	dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL);
684	if (!dest)
685	return ERR_PTR(error: -ENOMEM);
686
687	flow_act.action = attr->action;
688
689	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) {
690	flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]);
691	flow_act.vlan[0].vid = esw_attr->vlan_vid[0];
692	flow_act.vlan[0].prio = esw_attr->vlan_prio[0];
693	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) {
694	flow_act.vlan[1].ethtype = ntohs(esw_attr->vlan_proto[1]);
695	flow_act.vlan[1].vid = esw_attr->vlan_vid[1];
696	flow_act.vlan[1].prio = esw_attr->vlan_prio[1];
697	}
698	}
699
700	mlx5_eswitch_set_rule_flow_source(esw, spec, attr: esw_attr);
701
702	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
703	int err;
704
705	err = esw_setup_dests(dest, flow_act: &flow_act, esw, attr, spec, i: &i);
706	if (err) {
707	rule = ERR_PTR(error: err);
708	goto err_create_goto_table;
709	}
710
711	/* Header rewrite with combined wire+loopback in FDB is not allowed */
712	if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) &&
713	esw_dests_to_int_external(dests: dest, max_dest: i)) {
714	esw_warn(esw->dev,
715	"FDB: Header rewrite with forwarding to both internal and external dests is not allowed\n");
716	rule = ERR_PTR(error: -EINVAL);
717	goto err_esw_get;
718	}
719	}
720
721	if (esw_attr->decap_pkt_reformat)
722	flow_act.pkt_reformat = esw_attr->decap_pkt_reformat;
723
724	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
725	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
726	dest[i].counter = attr->counter;
727	i++;
728	}
729
730	if (attr->outer_match_level != MLX5_MATCH_NONE)
731	spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
732	if (attr->inner_match_level != MLX5_MATCH_NONE)
733	spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS;
734
735	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
736	flow_act.modify_hdr = attr->modify_hdr;
737
738	if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) &&
739	attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER)
740	esw_setup_meter(attr, flow_act: &flow_act);
741
742	if (split) {
743	fwd_attr.chain = attr->chain;
744	fwd_attr.prio = attr->prio;
745	fwd_attr.vport = esw_attr->in_rep->vport;
746	fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
747
748	fdb = mlx5_esw_vporttbl_get(esw, attr: &fwd_attr);
749	} else {
750	if (attr->chain || attr->prio)
751	fdb = mlx5_chains_get_table(chains, chain: attr->chain,
752	prio: attr->prio, level: 0);
753	else
754	fdb = attr->ft;
755
756	if (!(attr->flags & MLX5_ATTR_FLAG_NO_IN_PORT))
757	mlx5_eswitch_set_rule_source_port(esw, spec, attr,
758	src_esw: esw_attr->in_mdev->priv.eswitch,
759	vport: esw_attr->in_rep->vport);
760	}
761	if (IS_ERR(ptr: fdb)) {
762	rule = ERR_CAST(ptr: fdb);
763	goto err_esw_get;
764	}
765
766	if (!i) {
767	kfree(objp: dest);
768	dest = NULL;
769	}
770
771	if (mlx5_eswitch_termtbl_required(esw, attr, flow_act: &flow_act, spec))
772	rule = mlx5_eswitch_add_termtbl_rule(esw, ft: fdb, spec, attr: esw_attr,
773	flow_act: &flow_act, dest, num_dest: i);
774	else
775	rule = mlx5_add_flow_rules(ft: fdb, spec, flow_act: &flow_act, dest, num_dest: i);
776	if (IS_ERR(ptr: rule))
777	goto err_add_rule;
778	else
779	atomic64_inc(v: &esw->offloads.num_flows);
780
781	kfree(objp: dest);
782	return rule;
783
784	err_add_rule:
785	if (split)
786	mlx5_esw_vporttbl_put(esw, attr: &fwd_attr);
787	else if (attr->chain || attr->prio)
788	mlx5_chains_put_table(chains, chain: attr->chain, prio: attr->prio, level: 0);
789	err_esw_get:
790	esw_cleanup_dests(esw, attr);
791	err_create_goto_table:
792	kfree(objp: dest);
793	return rule;
794	}
795
796	struct mlx5_flow_handle *
797	mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw,
798	struct mlx5_flow_spec *spec,
799	struct mlx5_flow_attr *attr)
800	{
801	struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
802	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
803	struct mlx5_fs_chains *chains = esw_chains(esw);
804	struct mlx5_vport_tbl_attr fwd_attr;
805	struct mlx5_flow_destination *dest;
806	struct mlx5_flow_table *fast_fdb;
807	struct mlx5_flow_table *fwd_fdb;
808	struct mlx5_flow_handle *rule;
809	int i, err = 0;
810
811	dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL);
812	if (!dest)
813	return ERR_PTR(error: -ENOMEM);
814
815	fast_fdb = mlx5_chains_get_table(chains, chain: attr->chain, prio: attr->prio, level: 0);
816	if (IS_ERR(ptr: fast_fdb)) {
817	rule = ERR_CAST(ptr: fast_fdb);
818	goto err_get_fast;
819	}
820
821	fwd_attr.chain = attr->chain;
822	fwd_attr.prio = attr->prio;
823	fwd_attr.vport = esw_attr->in_rep->vport;
824	fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
825	fwd_fdb = mlx5_esw_vporttbl_get(esw, attr: &fwd_attr);
826	if (IS_ERR(ptr: fwd_fdb)) {
827	rule = ERR_CAST(ptr: fwd_fdb);
828	goto err_get_fwd;
829	}
830
831	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
832	for (i = 0; i < esw_attr->split_count; i++) {
833	if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
834	/* Source port rewrite (forward to ovs internal port or statck device) isn't
835	* supported in the rule of split action.
836	*/
837	err = -EOPNOTSUPP;
838	else
839	esw_setup_vport_dest(dest, flow_act: &flow_act, esw, esw_attr, attr_idx: i, dest_idx: i, pkt_reformat: false);
840
841	if (err) {
842	rule = ERR_PTR(error: err);
843	goto err_chain_src_rewrite;
844	}
845	}
846	dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
847	dest[i].ft = fwd_fdb;
848	i++;
849
850	mlx5_eswitch_set_rule_source_port(esw, spec, attr,
851	src_esw: esw_attr->in_mdev->priv.eswitch,
852	vport: esw_attr->in_rep->vport);
853
854	if (attr->outer_match_level != MLX5_MATCH_NONE)
855	spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
856
857	flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
858	rule = mlx5_add_flow_rules(ft: fast_fdb, spec, flow_act: &flow_act, dest, num_dest: i);
859
860	if (IS_ERR(ptr: rule)) {
861	i = esw_attr->split_count;
862	goto err_chain_src_rewrite;
863	}
864
865	atomic64_inc(v: &esw->offloads.num_flows);
866
867	kfree(objp: dest);
868	return rule;
869	err_chain_src_rewrite:
870	mlx5_esw_vporttbl_put(esw, attr: &fwd_attr);
871	err_get_fwd:
872	mlx5_chains_put_table(chains, chain: attr->chain, prio: attr->prio, level: 0);
873	err_get_fast:
874	kfree(objp: dest);
875	return rule;
876	}
877
878	static void
879	__mlx5_eswitch_del_rule(struct mlx5_eswitch *esw,
880	struct mlx5_flow_handle *rule,
881	struct mlx5_flow_attr *attr,
882	bool fwd_rule)
883	{
884	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
885	struct mlx5_fs_chains *chains = esw_chains(esw);
886	bool split = (esw_attr->split_count > 0);
887	struct mlx5_vport_tbl_attr fwd_attr;
888	int i;
889
890	mlx5_del_flow_rules(fr: rule);
891
892	if (!mlx5e_tc_attr_flags_skip(attr_flags: attr->flags)) {
893	/* unref the term table */
894	for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) {
895	if (esw_attr->dests[i].termtbl)
896	mlx5_eswitch_termtbl_put(esw, tt: esw_attr->dests[i].termtbl);
897	}
898	}
899
900	atomic64_dec(v: &esw->offloads.num_flows);
901
902	if (fwd_rule || split) {
903	fwd_attr.chain = attr->chain;
904	fwd_attr.prio = attr->prio;
905	fwd_attr.vport = esw_attr->in_rep->vport;
906	fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
907	}
908
909	if (fwd_rule) {
910	mlx5_esw_vporttbl_put(esw, attr: &fwd_attr);
911	mlx5_chains_put_table(chains, chain: attr->chain, prio: attr->prio, level: 0);
912	} else {
913	if (split)
914	mlx5_esw_vporttbl_put(esw, attr: &fwd_attr);
915	else if (attr->chain || attr->prio)
916	mlx5_chains_put_table(chains, chain: attr->chain, prio: attr->prio, level: 0);
917	esw_cleanup_dests(esw, attr);
918	}
919	}
920
921	void
922	mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw,
923	struct mlx5_flow_handle *rule,
924	struct mlx5_flow_attr *attr)
925	{
926	__mlx5_eswitch_del_rule(esw, rule, attr, fwd_rule: false);
927	}
928
929	void
930	mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw,
931	struct mlx5_flow_handle *rule,
932	struct mlx5_flow_attr *attr)
933	{
934	__mlx5_eswitch_del_rule(esw, rule, attr, fwd_rule: true);
935	}
936
937	struct mlx5_flow_handle *
938	mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw,
939	struct mlx5_eswitch *from_esw,
940	struct mlx5_eswitch_rep *rep,
941	u32 sqn)
942	{
943	struct mlx5_flow_act flow_act = {0};
944	struct mlx5_flow_destination dest = {};
945	struct mlx5_flow_handle *flow_rule;
946	struct mlx5_flow_spec *spec;
947	void *misc;
948	u16 vport;
949
950	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
951	if (!spec) {
952	flow_rule = ERR_PTR(error: -ENOMEM);
953	goto out;
954	}
955
956	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
957	MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn);
958
959	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
960	MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn);
961
962	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
963
964	/* source vport is the esw manager */
965	vport = from_esw->manager_vport;
966
967	if (mlx5_eswitch_vport_match_metadata_enabled(esw: on_esw)) {
968	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
969	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
970	mlx5_eswitch_get_vport_metadata_for_match(from_esw, vport));
971
972	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
973	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
974	mlx5_eswitch_get_vport_metadata_mask());
975
976	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
977	} else {
978	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
979	MLX5_SET(fte_match_set_misc, misc, source_port, vport);
980
981	if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch))
982	MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
983	MLX5_CAP_GEN(from_esw->dev, vhca_id));
984
985	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
986	MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
987
988	if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch))
989	MLX5_SET_TO_ONES(fte_match_set_misc, misc,
990	source_eswitch_owner_vhca_id);
991
992	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
993	}
994
995	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
996	dest.vport.num = rep->vport;
997	dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id);
998	dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
999	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1000
1001	if (rep->vport == MLX5_VPORT_UPLINK &&
1002	on_esw == from_esw && on_esw->offloads.ft_ipsec_tx_pol) {
1003	dest.ft = on_esw->offloads.ft_ipsec_tx_pol;
1004	flow_act.flags = FLOW_ACT_IGNORE_FLOW_LEVEL;
1005	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1006	} else {
1007	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1008	dest.vport.num = rep->vport;
1009	dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id);
1010	dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
1011	}
1012
1013	if (MLX5_CAP_ESW_FLOWTABLE(on_esw->dev, flow_source) &&
1014	rep->vport == MLX5_VPORT_UPLINK)
1015	spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
1016
1017	flow_rule = mlx5_add_flow_rules(ft: mlx5_eswitch_get_slow_fdb(esw: on_esw),
1018	spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
1019	if (IS_ERR(ptr: flow_rule))
1020	esw_warn(on_esw->dev, "FDB: Failed to add send to vport rule err %pe\n",
1021	flow_rule);
1022	out:
1023	kvfree(addr: spec);
1024	return flow_rule;
1025	}
1026	EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule);
1027
1028	void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule)
1029	{
1030	mlx5_del_flow_rules(fr: rule);
1031	}
1032
1033	void mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle *rule)
1034	{
1035	if (rule)
1036	mlx5_del_flow_rules(fr: rule);
1037	}
1038
1039	struct mlx5_flow_handle *
1040	mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num)
1041	{
1042	struct mlx5_flow_destination dest = {};
1043	struct mlx5_flow_act flow_act = {0};
1044	struct mlx5_flow_handle *flow_rule;
1045	struct mlx5_flow_spec *spec;
1046
1047	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1048	if (!spec)
1049	return ERR_PTR(error: -ENOMEM);
1050
1051	MLX5_SET(fte_match_param, spec->match_criteria,
1052	misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask());
1053	MLX5_SET(fte_match_param, spec->match_criteria,
1054	misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
1055	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_1,
1056	ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK);
1057
1058	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1059	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1060	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1061
1062	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0,
1063	mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num));
1064	dest.vport.num = vport_num;
1065
1066	flow_rule = mlx5_add_flow_rules(ft: mlx5_eswitch_get_slow_fdb(esw),
1067	spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
1068	if (IS_ERR(ptr: flow_rule))
1069	esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule vport %d, err %pe\n",
1070	vport_num, flow_rule);
1071
1072	kvfree(addr: spec);
1073	return flow_rule;
1074	}
1075
1076	static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw)
1077	{
1078	return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
1079	MLX5_FDB_TO_VPORT_REG_C_1;
1080	}
1081
1082	static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable)
1083	{
1084	u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {};
1085	u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {};
1086	u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {};
1087	u8 curr, wanted;
1088	int err;
1089
1090	if (!mlx5_eswitch_reg_c1_loopback_supported(esw) &&
1091	!mlx5_eswitch_vport_match_metadata_enabled(esw))
1092	return 0;
1093
1094	MLX5_SET(query_esw_vport_context_in, in, opcode,
1095	MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT);
1096	err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out);
1097	if (err)
1098	return err;
1099
1100	curr = MLX5_GET(query_esw_vport_context_out, out,
1101	esw_vport_context.fdb_to_vport_reg_c_id);
1102	wanted = MLX5_FDB_TO_VPORT_REG_C_0;
1103	if (mlx5_eswitch_reg_c1_loopback_supported(esw))
1104	wanted |= MLX5_FDB_TO_VPORT_REG_C_1;
1105
1106	if (enable)
1107	curr |= wanted;
1108	else
1109	curr &= ~wanted;
1110
1111	MLX5_SET(modify_esw_vport_context_in, min,
1112	esw_vport_context.fdb_to_vport_reg_c_id, curr);
1113	MLX5_SET(modify_esw_vport_context_in, min,
1114	field_select.fdb_to_vport_reg_c_id, 1);
1115
1116	err = mlx5_eswitch_modify_esw_vport_context(dev: esw->dev, vport: 0, other_vport: false, in: min);
1117	if (!err) {
1118	if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1))
1119	esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1120	else
1121	esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1122	}
1123
1124	return err;
1125	}
1126
1127	static void peer_miss_rules_setup(struct mlx5_eswitch *esw,
1128	struct mlx5_core_dev *peer_dev,
1129	struct mlx5_flow_spec *spec,
1130	struct mlx5_flow_destination *dest)
1131	{
1132	void *misc;
1133
1134	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1135	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1136	misc_parameters_2);
1137	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1138	mlx5_eswitch_get_vport_metadata_mask());
1139
1140	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1141	} else {
1142	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1143	misc_parameters);
1144
1145	MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
1146	MLX5_CAP_GEN(peer_dev, vhca_id));
1147
1148	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
1149
1150	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1151	misc_parameters);
1152	MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1153	MLX5_SET_TO_ONES(fte_match_set_misc, misc,
1154	source_eswitch_owner_vhca_id);
1155	}
1156
1157	dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1158	dest->vport.num = peer_dev->priv.eswitch->manager_vport;
1159	dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id);
1160	dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
1161	}
1162
1163	static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw,
1164	struct mlx5_eswitch *peer_esw,
1165	struct mlx5_flow_spec *spec,
1166	u16 vport)
1167	{
1168	void *misc;
1169
1170	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1171	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1172	misc_parameters_2);
1173	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1174	mlx5_eswitch_get_vport_metadata_for_match(peer_esw,
1175	vport));
1176	} else {
1177	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1178	misc_parameters);
1179	MLX5_SET(fte_match_set_misc, misc, source_port, vport);
1180	}
1181	}
1182
1183	static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
1184	struct mlx5_core_dev *peer_dev)
1185	{
1186	struct mlx5_eswitch *peer_esw = peer_dev->priv.eswitch;
1187	struct mlx5_flow_destination dest = {};
1188	struct mlx5_flow_act flow_act = {0};
1189	struct mlx5_flow_handle **flows;
1190	struct mlx5_flow_handle *flow;
1191	struct mlx5_vport *peer_vport;
1192	struct mlx5_flow_spec *spec;
1193	int err, pfindex;
1194	unsigned long i;
1195	void *misc;
1196
1197	if (!MLX5_VPORT_MANAGER(peer_dev) &&
1198	!mlx5_core_is_ecpf_esw_manager(dev: peer_dev))
1199	return 0;
1200
1201	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1202	if (!spec)
1203	return -ENOMEM;
1204
1205	peer_miss_rules_setup(esw, peer_dev, spec, dest: &dest);
1206
1207	flows = kvcalloc(peer_esw->total_vports, sizeof(*flows), GFP_KERNEL);
1208	if (!flows) {
1209	err = -ENOMEM;
1210	goto alloc_flows_err;
1211	}
1212
1213	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1214	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1215	misc_parameters);
1216
1217	if (mlx5_core_is_ecpf_esw_manager(dev: peer_dev) &&
1218	mlx5_esw_host_functions_enabled(dev: peer_dev)) {
1219	peer_vport = mlx5_eswitch_get_vport(esw: peer_esw, vport_num: MLX5_VPORT_PF);
1220	esw_set_peer_miss_rule_source_port(esw, peer_esw, spec,
1221	vport: MLX5_VPORT_PF);
1222
1223	flow = mlx5_add_flow_rules(ft: mlx5_eswitch_get_slow_fdb(esw),
1224	spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
1225	if (IS_ERR(ptr: flow)) {
1226	err = PTR_ERR(ptr: flow);
1227	goto add_pf_flow_err;
1228	}
1229	flows[peer_vport->index] = flow;
1230	}
1231
1232	if (mlx5_ecpf_vport_exists(dev: peer_dev)) {
1233	peer_vport = mlx5_eswitch_get_vport(esw: peer_esw, vport_num: MLX5_VPORT_ECPF);
1234	MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF);
1235	flow = mlx5_add_flow_rules(ft: mlx5_eswitch_get_slow_fdb(esw),
1236	spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
1237	if (IS_ERR(ptr: flow)) {
1238	err = PTR_ERR(ptr: flow);
1239	goto add_ecpf_flow_err;
1240	}
1241	flows[peer_vport->index] = flow;
1242	}
1243
1244	if (mlx5_esw_host_functions_enabled(dev: esw->dev)) {
1245	mlx5_esw_for_each_vf_vport(peer_esw, i, peer_vport,
1246	mlx5_core_max_vfs(peer_dev)) {
1247	esw_set_peer_miss_rule_source_port(esw, peer_esw,
1248	spec,
1249	vport: peer_vport->vport);
1250
1251	flow = mlx5_add_flow_rules(ft: mlx5_eswitch_get_slow_fdb(esw),
1252	spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
1253	if (IS_ERR(ptr: flow)) {
1254	err = PTR_ERR(ptr: flow);
1255	goto add_vf_flow_err;
1256	}
1257	flows[peer_vport->index] = flow;
1258	}
1259	}
1260
1261	if (mlx5_core_ec_sriov_enabled(dev: peer_dev)) {
1262	mlx5_esw_for_each_ec_vf_vport(peer_esw, i, peer_vport,
1263	mlx5_core_max_ec_vfs(peer_dev)) {
1264	esw_set_peer_miss_rule_source_port(esw, peer_esw,
1265	spec,
1266	vport: peer_vport->vport);
1267	flow = mlx5_add_flow_rules(ft: esw->fdb_table.offloads.slow_fdb,
1268	spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
1269	if (IS_ERR(ptr: flow)) {
1270	err = PTR_ERR(ptr: flow);
1271	goto add_ec_vf_flow_err;
1272	}
1273	flows[peer_vport->index] = flow;
1274	}
1275	}
1276
1277	pfindex = mlx5_get_dev_index(dev: peer_dev);
1278	if (pfindex >= MLX5_MAX_PORTS) {
1279	esw_warn(esw->dev, "Peer dev index(%d) is over the max num defined(%d)\n",
1280	pfindex, MLX5_MAX_PORTS);
1281	err = -EINVAL;
1282	goto add_ec_vf_flow_err;
1283	}
1284	esw->fdb_table.offloads.peer_miss_rules[pfindex] = flows;
1285
1286	kvfree(addr: spec);
1287	return 0;
1288
1289	add_ec_vf_flow_err:
1290	mlx5_esw_for_each_ec_vf_vport(peer_esw, i, peer_vport,
1291	mlx5_core_max_ec_vfs(peer_dev)) {
1292	if (!flows[peer_vport->index])
1293	continue;
1294	mlx5_del_flow_rules(fr: flows[peer_vport->index]);
1295	}
1296	add_vf_flow_err:
1297	mlx5_esw_for_each_vf_vport(peer_esw, i, peer_vport,
1298	mlx5_core_max_vfs(peer_dev)) {
1299	if (!flows[peer_vport->index])
1300	continue;
1301	mlx5_del_flow_rules(fr: flows[peer_vport->index]);
1302	}
1303	if (mlx5_ecpf_vport_exists(dev: peer_dev)) {
1304	peer_vport = mlx5_eswitch_get_vport(esw: peer_esw, vport_num: MLX5_VPORT_ECPF);
1305	mlx5_del_flow_rules(fr: flows[peer_vport->index]);
1306	}
1307	add_ecpf_flow_err:
1308
1309	if (mlx5_core_is_ecpf_esw_manager(dev: peer_dev) &&
1310	mlx5_esw_host_functions_enabled(dev: peer_dev)) {
1311	peer_vport = mlx5_eswitch_get_vport(esw: peer_esw, vport_num: MLX5_VPORT_PF);
1312	mlx5_del_flow_rules(fr: flows[peer_vport->index]);
1313	}
1314	add_pf_flow_err:
1315	esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err);
1316	kvfree(addr: flows);
1317	alloc_flows_err:
1318	kvfree(addr: spec);
1319	return err;
1320	}
1321
1322	static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
1323	struct mlx5_core_dev *peer_dev)
1324	{
1325	struct mlx5_eswitch *peer_esw = peer_dev->priv.eswitch;
1326	u16 peer_index = mlx5_get_dev_index(dev: peer_dev);
1327	struct mlx5_flow_handle **flows;
1328	struct mlx5_vport *peer_vport;
1329	unsigned long i;
1330
1331	flows = esw->fdb_table.offloads.peer_miss_rules[peer_index];
1332	if (!flows)
1333	return;
1334
1335	if (mlx5_core_ec_sriov_enabled(dev: peer_dev)) {
1336	mlx5_esw_for_each_ec_vf_vport(peer_esw, i, peer_vport,
1337	mlx5_core_max_ec_vfs(peer_dev))
1338	mlx5_del_flow_rules(fr: flows[peer_vport->index]);
1339	}
1340
1341	mlx5_esw_for_each_vf_vport(peer_esw, i, peer_vport,
1342	mlx5_core_max_vfs(peer_dev))
1343	mlx5_del_flow_rules(fr: flows[peer_vport->index]);
1344
1345	if (mlx5_ecpf_vport_exists(dev: peer_dev)) {
1346	peer_vport = mlx5_eswitch_get_vport(esw: peer_esw, vport_num: MLX5_VPORT_ECPF);
1347	mlx5_del_flow_rules(fr: flows[peer_vport->index]);
1348	}
1349
1350	if (mlx5_core_is_ecpf_esw_manager(dev: peer_dev)) {
1351	peer_vport = mlx5_eswitch_get_vport(esw: peer_esw, vport_num: MLX5_VPORT_PF);
1352	mlx5_del_flow_rules(fr: flows[peer_vport->index]);
1353	}
1354
1355	kvfree(addr: flows);
1356	esw->fdb_table.offloads.peer_miss_rules[peer_index] = NULL;
1357	}
1358
1359	static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw)
1360	{
1361	struct mlx5_flow_act flow_act = {0};
1362	struct mlx5_flow_destination dest = {};
1363	struct mlx5_flow_handle *flow_rule = NULL;
1364	struct mlx5_flow_spec *spec;
1365	void *headers_c;
1366	void *headers_v;
1367	int err = 0;
1368	u8 *dmac_c;
1369	u8 *dmac_v;
1370
1371	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1372	if (!spec) {
1373	err = -ENOMEM;
1374	goto out;
1375	}
1376
1377	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
1378	headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1379	outer_headers);
1380	dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c,
1381	outer_headers.dmac_47_16);
1382	dmac_c[0] = 0x01;
1383
1384	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1385	dest.vport.num = esw->manager_vport;
1386	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1387
1388	flow_rule = mlx5_add_flow_rules(ft: mlx5_eswitch_get_slow_fdb(esw),
1389	spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
1390	if (IS_ERR(ptr: flow_rule)) {
1391	err = PTR_ERR(ptr: flow_rule);
1392	esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err);
1393	goto out;
1394	}
1395
1396	esw->fdb_table.offloads.miss_rule_uni = flow_rule;
1397
1398	headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1399	outer_headers);
1400	dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v,
1401	outer_headers.dmac_47_16);
1402	dmac_v[0] = 0x01;
1403	flow_rule = mlx5_add_flow_rules(ft: mlx5_eswitch_get_slow_fdb(esw),
1404	spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
1405	if (IS_ERR(ptr: flow_rule)) {
1406	err = PTR_ERR(ptr: flow_rule);
1407	esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err);
1408	mlx5_del_flow_rules(fr: esw->fdb_table.offloads.miss_rule_uni);
1409	goto out;
1410	}
1411
1412	esw->fdb_table.offloads.miss_rule_multi = flow_rule;
1413
1414	out:
1415	kvfree(addr: spec);
1416	return err;
1417	}
1418
1419	struct mlx5_flow_handle *
1420	esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag)
1421	{
1422	struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
1423	struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore;
1424	struct mlx5_flow_context *flow_context;
1425	struct mlx5_flow_handle *flow_rule;
1426	struct mlx5_flow_destination dest;
1427	struct mlx5_flow_spec *spec;
1428	void *misc;
1429
1430	if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
1431	return ERR_PTR(error: -EOPNOTSUPP);
1432
1433	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1434	if (!spec)
1435	return ERR_PTR(error: -ENOMEM);
1436
1437	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1438	misc_parameters_2);
1439	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1440	ESW_REG_C0_USER_DATA_METADATA_MASK);
1441	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1442	misc_parameters_2);
1443	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag);
1444	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1445	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1446	MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1447	flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id;
1448
1449	flow_context = &spec->flow_context;
1450	flow_context->flags |= FLOW_CONTEXT_HAS_TAG;
1451	flow_context->flow_tag = tag;
1452	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1453	dest.ft = esw->offloads.ft_offloads;
1454
1455	flow_rule = mlx5_add_flow_rules(ft, spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
1456	kvfree(addr: spec);
1457
1458	if (IS_ERR(ptr: flow_rule))
1459	esw_warn(esw->dev,
1460	"Failed to create restore rule for tag: %d, err(%d)\n",
1461	tag, (int)PTR_ERR(flow_rule));
1462
1463	return flow_rule;
1464	}
1465
1466	#define MAX_PF_SQ 256
1467	#define MAX_SQ_NVPORTS 32
1468
1469	void
1470	mlx5_esw_set_flow_group_source_port(struct mlx5_eswitch *esw,
1471	u32 *flow_group_in,
1472	int match_params)
1473	{
1474	void *match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1475	flow_group_in,
1476	match_criteria);
1477
1478	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1479	MLX5_SET(create_flow_group_in, flow_group_in,
1480	match_criteria_enable,
1481	MLX5_MATCH_MISC_PARAMETERS_2 | match_params);
1482
1483	MLX5_SET(fte_match_param, match_criteria,
1484	misc_parameters_2.metadata_reg_c_0,
1485	mlx5_eswitch_get_vport_metadata_mask());
1486	} else {
1487	MLX5_SET(create_flow_group_in, flow_group_in,
1488	match_criteria_enable,
1489	MLX5_MATCH_MISC_PARAMETERS | match_params);
1490
1491	MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1492	misc_parameters.source_port);
1493	}
1494	}
1495
1496	#if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
1497	static void esw_vport_tbl_put(struct mlx5_eswitch *esw)
1498	{
1499	struct mlx5_vport_tbl_attr attr;
1500	struct mlx5_vport *vport;
1501	unsigned long i;
1502
1503	attr.chain = 0;
1504	attr.prio = 1;
1505	mlx5_esw_for_each_vport(esw, i, vport) {
1506	attr.vport = vport->vport;
1507	attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
1508	mlx5_esw_vporttbl_put(esw, attr: &attr);
1509	}
1510	}
1511
1512	static int esw_vport_tbl_get(struct mlx5_eswitch *esw)
1513	{
1514	struct mlx5_vport_tbl_attr attr;
1515	struct mlx5_flow_table *fdb;
1516	struct mlx5_vport *vport;
1517	unsigned long i;
1518
1519	attr.chain = 0;
1520	attr.prio = 1;
1521	mlx5_esw_for_each_vport(esw, i, vport) {
1522	attr.vport = vport->vport;
1523	attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
1524	fdb = mlx5_esw_vporttbl_get(esw, attr: &attr);
1525	if (IS_ERR(ptr: fdb))
1526	goto out;
1527	}
1528	return 0;
1529
1530	out:
1531	esw_vport_tbl_put(esw);
1532	return PTR_ERR(ptr: fdb);
1533	}
1534
1535	#define fdb_modify_header_fwd_to_table_supported(esw) \
1536	(MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table))
1537	static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags)
1538	{
1539	struct mlx5_core_dev *dev = esw->dev;
1540
1541	if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ignore_flow_level))
1542	*flags |= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
1543
1544	if (!MLX5_CAP_ESW_FLOWTABLE(dev, multi_fdb_encap) &&
1545	esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) {
1546	*flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1547	esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n");
1548	} else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) {
1549	*flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1550	esw_warn(dev, "Tc chains and priorities offload aren't supported\n");
1551	} else if (!fdb_modify_header_fwd_to_table_supported(esw)) {
1552	/* Disabled when ttl workaround is needed, e.g
1553	* when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig
1554	*/
1555	esw_warn(dev,
1556	"Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n");
1557	*flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1558	} else {
1559	*flags |= MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1560	esw_info(dev, "Supported tc chains and prios offload\n");
1561	}
1562
1563	if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1564	*flags |= MLX5_CHAINS_FT_TUNNEL_SUPPORTED;
1565	}
1566
1567	static int
1568	esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1569	{
1570	struct mlx5_core_dev *dev = esw->dev;
1571	struct mlx5_flow_table *nf_ft, *ft;
1572	struct mlx5_chains_attr attr = {};
1573	struct mlx5_fs_chains *chains;
1574	int err;
1575
1576	esw_init_chains_offload_flags(esw, flags: &attr.flags);
1577	attr.ns = MLX5_FLOW_NAMESPACE_FDB;
1578	attr.max_grp_num = esw->params.large_group_num;
1579	attr.default_ft = miss_fdb;
1580	attr.mapping = esw->offloads.reg_c0_obj_pool;
1581	attr.fs_base_prio = FDB_BYPASS_PATH;
1582
1583	chains = mlx5_chains_create(dev, attr: &attr);
1584	if (IS_ERR(ptr: chains)) {
1585	err = PTR_ERR(ptr: chains);
1586	esw_warn(dev, "Failed to create fdb chains err(%d)\n", err);
1587	return err;
1588	}
1589	mlx5_chains_print_info(chains);
1590
1591	esw->fdb_table.offloads.esw_chains_priv = chains;
1592
1593	/* Create tc_end_ft which is the always created ft chain */
1594	nf_ft = mlx5_chains_get_table(chains, chain: mlx5_chains_get_nf_ft_chain(chains),
1595	prio: 1, level: 0);
1596	if (IS_ERR(ptr: nf_ft)) {
1597	err = PTR_ERR(ptr: nf_ft);
1598	goto nf_ft_err;
1599	}
1600
1601	/* Always open the root for fast path */
1602	ft = mlx5_chains_get_table(chains, chain: 0, prio: 1, level: 0);
1603	if (IS_ERR(ptr: ft)) {
1604	err = PTR_ERR(ptr: ft);
1605	goto level_0_err;
1606	}
1607
1608	/* Open level 1 for split fdb rules now if prios isn't supported */
1609	if (!mlx5_chains_prios_supported(chains)) {
1610	err = esw_vport_tbl_get(esw);
1611	if (err)
1612	goto level_1_err;
1613	}
1614
1615	mlx5_chains_set_end_ft(chains, ft: nf_ft);
1616
1617	return 0;
1618
1619	level_1_err:
1620	mlx5_chains_put_table(chains, chain: 0, prio: 1, level: 0);
1621	level_0_err:
1622	mlx5_chains_put_table(chains, chain: mlx5_chains_get_nf_ft_chain(chains), prio: 1, level: 0);
1623	nf_ft_err:
1624	mlx5_chains_destroy(chains);
1625	esw->fdb_table.offloads.esw_chains_priv = NULL;
1626
1627	return err;
1628	}
1629
1630	static void
1631	esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1632	{
1633	if (!mlx5_chains_prios_supported(chains))
1634	esw_vport_tbl_put(esw);
1635	mlx5_chains_put_table(chains, chain: 0, prio: 1, level: 0);
1636	mlx5_chains_put_table(chains, chain: mlx5_chains_get_nf_ft_chain(chains), prio: 1, level: 0);
1637	mlx5_chains_destroy(chains);
1638	}
1639
1640	#else /* CONFIG_MLX5_CLS_ACT */
1641
1642	static int
1643	esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1644	{ return 0; }
1645
1646	static void
1647	esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1648	{}
1649
1650	#endif
1651
1652	static int
1653	esw_create_send_to_vport_group(struct mlx5_eswitch *esw,
1654	struct mlx5_flow_table *fdb,
1655	u32 *flow_group_in,
1656	int *ix)
1657	{
1658	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1659	struct mlx5_flow_group *g;
1660	void *match_criteria;
1661	int count, err = 0;
1662
1663	memset(flow_group_in, 0, inlen);
1664
1665	mlx5_esw_set_flow_group_source_port(esw, flow_group_in, match_params: MLX5_MATCH_MISC_PARAMETERS);
1666
1667	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
1668	MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn);
1669
1670	if (!mlx5_eswitch_vport_match_metadata_enabled(esw) &&
1671	MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
1672	MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1673	misc_parameters.source_eswitch_owner_vhca_id);
1674	MLX5_SET(create_flow_group_in, flow_group_in,
1675	source_eswitch_owner_vhca_id_valid, 1);
1676	}
1677
1678	/* See comment at table_size calculation */
1679	count = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ);
1680	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
1681	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, *ix + count - 1);
1682	*ix += count;
1683
1684	g = mlx5_create_flow_group(ft: fdb, in: flow_group_in);
1685	if (IS_ERR(ptr: g)) {
1686	err = PTR_ERR(ptr: g);
1687	esw_warn(esw->dev, "Failed to create send-to-vport flow group err(%d)\n", err);
1688	goto out;
1689	}
1690	esw->fdb_table.offloads.send_to_vport_grp = g;
1691
1692	out:
1693	return err;
1694	}
1695
1696	static int
1697	esw_create_meta_send_to_vport_group(struct mlx5_eswitch *esw,
1698	struct mlx5_flow_table *fdb,
1699	u32 *flow_group_in,
1700	int *ix)
1701	{
1702	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1703	struct mlx5_flow_group *g;
1704	void *match_criteria;
1705	int err = 0;
1706
1707	if (!esw_src_port_rewrite_supported(esw))
1708	return 0;
1709
1710	memset(flow_group_in, 0, inlen);
1711
1712	MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1713	MLX5_MATCH_MISC_PARAMETERS_2);
1714
1715	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
1716
1717	MLX5_SET(fte_match_param, match_criteria,
1718	misc_parameters_2.metadata_reg_c_0,
1719	mlx5_eswitch_get_vport_metadata_mask());
1720	MLX5_SET(fte_match_param, match_criteria,
1721	misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
1722
1723	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1724	MLX5_SET(create_flow_group_in, flow_group_in,
1725	end_flow_index, *ix + esw->total_vports - 1);
1726	*ix += esw->total_vports;
1727
1728	g = mlx5_create_flow_group(ft: fdb, in: flow_group_in);
1729	if (IS_ERR(ptr: g)) {
1730	err = PTR_ERR(ptr: g);
1731	esw_warn(esw->dev,
1732	"Failed to create send-to-vport meta flow group err(%d)\n", err);
1733	goto send_vport_meta_err;
1734	}
1735	esw->fdb_table.offloads.send_to_vport_meta_grp = g;
1736
1737	return 0;
1738
1739	send_vport_meta_err:
1740	return err;
1741	}
1742
1743	static int
1744	esw_create_peer_esw_miss_group(struct mlx5_eswitch *esw,
1745	struct mlx5_flow_table *fdb,
1746	u32 *flow_group_in,
1747	int *ix)
1748	{
1749	int max_peer_ports = (esw->total_vports - 1) * (MLX5_MAX_PORTS - 1);
1750	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1751	struct mlx5_flow_group *g;
1752	void *match_criteria;
1753	int err = 0;
1754
1755	if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
1756	return 0;
1757
1758	memset(flow_group_in, 0, inlen);
1759
1760	mlx5_esw_set_flow_group_source_port(esw, flow_group_in, match_params: 0);
1761
1762	if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1763	match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1764	flow_group_in,
1765	match_criteria);
1766
1767	MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1768	misc_parameters.source_eswitch_owner_vhca_id);
1769
1770	MLX5_SET(create_flow_group_in, flow_group_in,
1771	source_eswitch_owner_vhca_id_valid, 1);
1772	}
1773
1774	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1775	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1776	*ix + max_peer_ports);
1777	*ix += max_peer_ports + 1;
1778
1779	g = mlx5_create_flow_group(ft: fdb, in: flow_group_in);
1780	if (IS_ERR(ptr: g)) {
1781	err = PTR_ERR(ptr: g);
1782	esw_warn(esw->dev, "Failed to create peer miss flow group err(%d)\n", err);
1783	goto out;
1784	}
1785	esw->fdb_table.offloads.peer_miss_grp = g;
1786
1787	out:
1788	return err;
1789	}
1790
1791	static int
1792	esw_create_miss_group(struct mlx5_eswitch *esw,
1793	struct mlx5_flow_table *fdb,
1794	u32 *flow_group_in,
1795	int *ix)
1796	{
1797	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1798	struct mlx5_flow_group *g;
1799	void *match_criteria;
1800	int err = 0;
1801	u8 *dmac;
1802
1803	memset(flow_group_in, 0, inlen);
1804
1805	MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1806	MLX5_MATCH_OUTER_HEADERS);
1807	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
1808	match_criteria);
1809	dmac = MLX5_ADDR_OF(fte_match_param, match_criteria,
1810	outer_headers.dmac_47_16);
1811	dmac[0] = 0x01;
1812
1813	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1814	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1815	*ix + MLX5_ESW_MISS_FLOWS);
1816
1817	g = mlx5_create_flow_group(ft: fdb, in: flow_group_in);
1818	if (IS_ERR(ptr: g)) {
1819	err = PTR_ERR(ptr: g);
1820	esw_warn(esw->dev, "Failed to create miss flow group err(%d)\n", err);
1821	goto miss_err;
1822	}
1823	esw->fdb_table.offloads.miss_grp = g;
1824
1825	err = esw_add_fdb_miss_rule(esw);
1826	if (err)
1827	goto miss_rule_err;
1828
1829	return 0;
1830
1831	miss_rule_err:
1832	mlx5_destroy_flow_group(fg: esw->fdb_table.offloads.miss_grp);
1833	miss_err:
1834	return err;
1835	}
1836
1837	static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw)
1838	{
1839	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1840	struct mlx5_flow_table_attr ft_attr = {};
1841	struct mlx5_core_dev *dev = esw->dev;
1842	struct mlx5_flow_namespace *root_ns;
1843	struct mlx5_flow_table *fdb = NULL;
1844	int table_size, ix = 0, err = 0;
1845	u32 flags = 0, *flow_group_in;
1846
1847	esw_debug(esw->dev, "Create offloads FDB Tables\n");
1848
1849	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
1850	if (!flow_group_in)
1851	return -ENOMEM;
1852
1853	root_ns = mlx5_get_flow_namespace(dev, type: MLX5_FLOW_NAMESPACE_FDB);
1854	if (!root_ns) {
1855	esw_warn(dev, "Failed to get FDB flow namespace\n");
1856	err = -EOPNOTSUPP;
1857	goto ns_err;
1858	}
1859	esw->fdb_table.offloads.ns = root_ns;
1860	err = mlx5_flow_namespace_set_mode(ns: root_ns,
1861	mode: esw->dev->priv.steering->mode);
1862	if (err) {
1863	esw_warn(dev, "Failed to set FDB namespace steering mode\n");
1864	goto ns_err;
1865	}
1866
1867	/* To be strictly correct:
1868	* MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ)
1869	* should be:
1870	* esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ +
1871	* peer_esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ
1872	* but as the peer device might not be in switchdev mode it's not
1873	* possible. We use the fact that by default FW sets max vfs and max sfs
1874	* to the same value on both devices. If it needs to be changed in the future note
1875	* the peer miss group should also be created based on the number of
1876	* total vports of the peer (currently is also uses esw->total_vports).
1877	*/
1878	table_size = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) +
1879	esw->total_vports * MLX5_MAX_PORTS + MLX5_ESW_MISS_FLOWS;
1880
1881	/* create the slow path fdb with encap set, so further table instances
1882	* can be created at run time while VFs are probed if the FW allows that.
1883	*/
1884	if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1885	flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT |
1886	MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
1887
1888	ft_attr.flags = flags;
1889	ft_attr.max_fte = table_size;
1890	ft_attr.prio = FDB_SLOW_PATH;
1891
1892	fdb = mlx5_create_flow_table(ns: root_ns, ft_attr: &ft_attr);
1893	if (IS_ERR(ptr: fdb)) {
1894	err = PTR_ERR(ptr: fdb);
1895	esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err);
1896	goto slow_fdb_err;
1897	}
1898	esw->fdb_table.offloads.slow_fdb = fdb;
1899
1900	/* Create empty TC-miss managed table. This allows plugging in following
1901	* priorities without directly exposing their level 0 table to
1902	* eswitch_offloads and passing it as miss_fdb to following call to
1903	* esw_chains_create().
1904	*/
1905	memset(&ft_attr, 0, sizeof(ft_attr));
1906	ft_attr.prio = FDB_TC_MISS;
1907	esw->fdb_table.offloads.tc_miss_table = mlx5_create_flow_table(ns: root_ns, ft_attr: &ft_attr);
1908	if (IS_ERR(ptr: esw->fdb_table.offloads.tc_miss_table)) {
1909	err = PTR_ERR(ptr: esw->fdb_table.offloads.tc_miss_table);
1910	esw_warn(dev, "Failed to create TC miss FDB Table err %d\n", err);
1911	goto tc_miss_table_err;
1912	}
1913
1914	err = esw_chains_create(esw, miss_fdb: esw->fdb_table.offloads.tc_miss_table);
1915	if (err) {
1916	esw_warn(dev, "Failed to open fdb chains err(%d)\n", err);
1917	goto fdb_chains_err;
1918	}
1919
1920	err = esw_create_send_to_vport_group(esw, fdb, flow_group_in, ix: &ix);
1921	if (err)
1922	goto send_vport_err;
1923
1924	err = esw_create_meta_send_to_vport_group(esw, fdb, flow_group_in, ix: &ix);
1925	if (err)
1926	goto send_vport_meta_err;
1927
1928	err = esw_create_peer_esw_miss_group(esw, fdb, flow_group_in, ix: &ix);
1929	if (err)
1930	goto peer_miss_err;
1931
1932	err = esw_create_miss_group(esw, fdb, flow_group_in, ix: &ix);
1933	if (err)
1934	goto miss_err;
1935
1936	kvfree(addr: flow_group_in);
1937	return 0;
1938
1939	miss_err:
1940	if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
1941	mlx5_destroy_flow_group(fg: esw->fdb_table.offloads.peer_miss_grp);
1942	peer_miss_err:
1943	if (esw->fdb_table.offloads.send_to_vport_meta_grp)
1944	mlx5_destroy_flow_group(fg: esw->fdb_table.offloads.send_to_vport_meta_grp);
1945	send_vport_meta_err:
1946	mlx5_destroy_flow_group(fg: esw->fdb_table.offloads.send_to_vport_grp);
1947	send_vport_err:
1948	esw_chains_destroy(esw, esw_chains(esw));
1949	fdb_chains_err:
1950	mlx5_destroy_flow_table(ft: esw->fdb_table.offloads.tc_miss_table);
1951	tc_miss_table_err:
1952	mlx5_destroy_flow_table(ft: mlx5_eswitch_get_slow_fdb(esw));
1953	slow_fdb_err:
1954	/* Holds true only as long as DMFS is the default */
1955	mlx5_flow_namespace_set_mode(ns: root_ns, mode: MLX5_FLOW_STEERING_MODE_DMFS);
1956	ns_err:
1957	kvfree(addr: flow_group_in);
1958	return err;
1959	}
1960
1961	static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw)
1962	{
1963	if (!mlx5_eswitch_get_slow_fdb(esw))
1964	return;
1965
1966	esw_debug(esw->dev, "Destroy offloads FDB Tables\n");
1967	mlx5_del_flow_rules(fr: esw->fdb_table.offloads.miss_rule_multi);
1968	mlx5_del_flow_rules(fr: esw->fdb_table.offloads.miss_rule_uni);
1969	mlx5_destroy_flow_group(fg: esw->fdb_table.offloads.send_to_vport_grp);
1970	if (esw->fdb_table.offloads.send_to_vport_meta_grp)
1971	mlx5_destroy_flow_group(fg: esw->fdb_table.offloads.send_to_vport_meta_grp);
1972	if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
1973	mlx5_destroy_flow_group(fg: esw->fdb_table.offloads.peer_miss_grp);
1974	mlx5_destroy_flow_group(fg: esw->fdb_table.offloads.miss_grp);
1975
1976	esw_chains_destroy(esw, esw_chains(esw));
1977
1978	mlx5_destroy_flow_table(ft: esw->fdb_table.offloads.tc_miss_table);
1979	mlx5_destroy_flow_table(ft: mlx5_eswitch_get_slow_fdb(esw));
1980	/* Holds true only as long as DMFS is the default */
1981	mlx5_flow_namespace_set_mode(ns: esw->fdb_table.offloads.ns,
1982	mode: MLX5_FLOW_STEERING_MODE_DMFS);
1983	}
1984
1985	static int esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch *esw)
1986	{
1987	int nvports;
1988
1989	nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS;
1990	if (mlx5e_tc_int_port_supported(esw))
1991	nvports += MLX5E_TC_MAX_INT_PORT_NUM;
1992
1993	return nvports;
1994	}
1995
1996	static int esw_create_offloads_table(struct mlx5_eswitch *esw)
1997	{
1998	struct mlx5_flow_table_attr ft_attr = {};
1999	struct mlx5_core_dev *dev = esw->dev;
2000	struct mlx5_flow_table *ft_offloads;
2001	struct mlx5_flow_namespace *ns;
2002	int err = 0;
2003
2004	ns = mlx5_get_flow_namespace(dev, type: MLX5_FLOW_NAMESPACE_OFFLOADS);
2005	if (!ns) {
2006	esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
2007	return -EOPNOTSUPP;
2008	}
2009
2010	ft_attr.max_fte = esw_get_nr_ft_offloads_steering_src_ports(esw) +
2011	MLX5_ESW_FT_OFFLOADS_DROP_RULE;
2012	ft_attr.prio = 1;
2013
2014	ft_offloads = mlx5_create_flow_table(ns, ft_attr: &ft_attr);
2015	if (IS_ERR(ptr: ft_offloads)) {
2016	err = PTR_ERR(ptr: ft_offloads);
2017	esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err);
2018	return err;
2019	}
2020
2021	esw->offloads.ft_offloads = ft_offloads;
2022	return 0;
2023	}
2024
2025	static void esw_destroy_offloads_table(struct mlx5_eswitch *esw)
2026	{
2027	struct mlx5_esw_offload *offloads = &esw->offloads;
2028
2029	mlx5_destroy_flow_table(ft: offloads->ft_offloads);
2030	}
2031
2032	static int esw_create_vport_rx_group(struct mlx5_eswitch *esw)
2033	{
2034	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2035	struct mlx5_flow_group *g;
2036	u32 *flow_group_in;
2037	int nvports;
2038	int err = 0;
2039
2040	nvports = esw_get_nr_ft_offloads_steering_src_ports(esw);
2041	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2042	if (!flow_group_in)
2043	return -ENOMEM;
2044
2045	mlx5_esw_set_flow_group_source_port(esw, flow_group_in, match_params: 0);
2046
2047	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2048	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1);
2049
2050	g = mlx5_create_flow_group(ft: esw->offloads.ft_offloads, in: flow_group_in);
2051
2052	if (IS_ERR(ptr: g)) {
2053	err = PTR_ERR(ptr: g);
2054	mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err);
2055	goto out;
2056	}
2057
2058	esw->offloads.vport_rx_group = g;
2059	out:
2060	kvfree(addr: flow_group_in);
2061	return err;
2062	}
2063
2064	static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw)
2065	{
2066	mlx5_destroy_flow_group(fg: esw->offloads.vport_rx_group);
2067	}
2068
2069	static int esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch *esw)
2070	{
2071	/* ft_offloads table is enlarged by MLX5_ESW_FT_OFFLOADS_DROP_RULE (1)
2072	* for the drop rule, which is placed at the end of the table.
2073	* So return the total of vport and int_port as rule index.
2074	*/
2075	return esw_get_nr_ft_offloads_steering_src_ports(esw);
2076	}
2077
2078	static int esw_create_vport_rx_drop_group(struct mlx5_eswitch *esw)
2079	{
2080	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2081	struct mlx5_flow_group *g;
2082	u32 *flow_group_in;
2083	int flow_index;
2084	int err = 0;
2085
2086	flow_index = esw_create_vport_rx_drop_rule_index(esw);
2087
2088	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2089	if (!flow_group_in)
2090	return -ENOMEM;
2091
2092	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_index);
2093	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_index);
2094
2095	g = mlx5_create_flow_group(ft: esw->offloads.ft_offloads, in: flow_group_in);
2096
2097	if (IS_ERR(ptr: g)) {
2098	err = PTR_ERR(ptr: g);
2099	mlx5_core_warn(esw->dev, "Failed to create vport rx drop group err %d\n", err);
2100	goto out;
2101	}
2102
2103	esw->offloads.vport_rx_drop_group = g;
2104	out:
2105	kvfree(addr: flow_group_in);
2106	return err;
2107	}
2108
2109	static void esw_destroy_vport_rx_drop_group(struct mlx5_eswitch *esw)
2110	{
2111	if (esw->offloads.vport_rx_drop_group)
2112	mlx5_destroy_flow_group(fg: esw->offloads.vport_rx_drop_group);
2113	}
2114
2115	void
2116	mlx5_esw_set_spec_source_port(struct mlx5_eswitch *esw,
2117	u16 vport,
2118	struct mlx5_flow_spec *spec)
2119	{
2120	void *misc;
2121
2122	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
2123	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
2124	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2125	mlx5_eswitch_get_vport_metadata_for_match(esw, vport));
2126
2127	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
2128	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2129	mlx5_eswitch_get_vport_metadata_mask());
2130
2131	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
2132	} else {
2133	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
2134	MLX5_SET(fte_match_set_misc, misc, source_port, vport);
2135
2136	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
2137	MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
2138
2139	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
2140	}
2141	}
2142
2143	struct mlx5_flow_handle *
2144	mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport,
2145	struct mlx5_flow_destination *dest)
2146	{
2147	struct mlx5_flow_act flow_act = {0};
2148	struct mlx5_flow_handle *flow_rule;
2149	struct mlx5_flow_spec *spec;
2150
2151	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
2152	if (!spec) {
2153	flow_rule = ERR_PTR(error: -ENOMEM);
2154	goto out;
2155	}
2156
2157	mlx5_esw_set_spec_source_port(esw, vport, spec);
2158
2159	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2160	flow_rule = mlx5_add_flow_rules(ft: esw->offloads.ft_offloads, spec,
2161	flow_act: &flow_act, dest, num_dest: 1);
2162	if (IS_ERR(ptr: flow_rule)) {
2163	esw_warn(esw->dev,
2164	"fs offloads: Failed to add vport rx rule err %pe\n",
2165	flow_rule);
2166	goto out;
2167	}
2168
2169	out:
2170	kvfree(addr: spec);
2171	return flow_rule;
2172	}
2173
2174	static int esw_create_vport_rx_drop_rule(struct mlx5_eswitch *esw)
2175	{
2176	struct mlx5_flow_act flow_act = {};
2177	struct mlx5_flow_handle *flow_rule;
2178
2179	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
2180	flow_rule = mlx5_add_flow_rules(ft: esw->offloads.ft_offloads, NULL,
2181	flow_act: &flow_act, NULL, num_dest: 0);
2182	if (IS_ERR(ptr: flow_rule)) {
2183	esw_warn(esw->dev,
2184	"fs offloads: Failed to add vport rx drop rule err %pe\n",
2185	flow_rule);
2186	return PTR_ERR(ptr: flow_rule);
2187	}
2188
2189	esw->offloads.vport_rx_drop_rule = flow_rule;
2190
2191	return 0;
2192	}
2193
2194	static void esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch *esw)
2195	{
2196	if (esw->offloads.vport_rx_drop_rule)
2197	mlx5_del_flow_rules(fr: esw->offloads.vport_rx_drop_rule);
2198	}
2199
2200	static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode)
2201	{
2202	u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2;
2203	struct mlx5_core_dev *dev = esw->dev;
2204	struct mlx5_vport *vport;
2205	unsigned long i;
2206
2207	if (!MLX5_CAP_GEN(dev, vport_group_manager))
2208	return -EOPNOTSUPP;
2209
2210	if (!mlx5_esw_is_fdb_created(esw))
2211	return -EOPNOTSUPP;
2212
2213	switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
2214	case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
2215	mlx5_mode = MLX5_INLINE_MODE_NONE;
2216	goto out;
2217	case MLX5_CAP_INLINE_MODE_L2:
2218	mlx5_mode = MLX5_INLINE_MODE_L2;
2219	goto out;
2220	case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
2221	goto query_vports;
2222	}
2223
2224	query_vports:
2225	mlx5_query_nic_vport_min_inline(mdev: dev, vport: esw->first_host_vport, min_inline: &prev_mlx5_mode);
2226	mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
2227	mlx5_query_nic_vport_min_inline(mdev: dev, vport: vport->vport, min_inline: &mlx5_mode);
2228	if (prev_mlx5_mode != mlx5_mode)
2229	return -EINVAL;
2230	prev_mlx5_mode = mlx5_mode;
2231	}
2232
2233	out:
2234	*mode = mlx5_mode;
2235	return 0;
2236	}
2237
2238	static void esw_destroy_restore_table(struct mlx5_eswitch *esw)
2239	{
2240	struct mlx5_esw_offload *offloads = &esw->offloads;
2241
2242	if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
2243	return;
2244
2245	mlx5_modify_header_dealloc(dev: esw->dev, modify_hdr: offloads->restore_copy_hdr_id);
2246	mlx5_destroy_flow_group(fg: offloads->restore_group);
2247	mlx5_destroy_flow_table(ft: offloads->ft_offloads_restore);
2248	}
2249
2250	static int esw_create_restore_table(struct mlx5_eswitch *esw)
2251	{
2252	u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
2253	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2254	struct mlx5_flow_table_attr ft_attr = {};
2255	struct mlx5_core_dev *dev = esw->dev;
2256	struct mlx5_flow_namespace *ns;
2257	struct mlx5_modify_hdr *mod_hdr;
2258	void *match_criteria, *misc;
2259	struct mlx5_flow_table *ft;
2260	struct mlx5_flow_group *g;
2261	u32 *flow_group_in;
2262	int err = 0;
2263
2264	if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
2265	return 0;
2266
2267	ns = mlx5_get_flow_namespace(dev, type: MLX5_FLOW_NAMESPACE_OFFLOADS);
2268	if (!ns) {
2269	esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
2270	return -EOPNOTSUPP;
2271	}
2272
2273	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2274	if (!flow_group_in) {
2275	err = -ENOMEM;
2276	goto out_free;
2277	}
2278
2279	ft_attr.max_fte = 1 << ESW_REG_C0_USER_DATA_METADATA_BITS;
2280	ft = mlx5_create_flow_table(ns, ft_attr: &ft_attr);
2281	if (IS_ERR(ptr: ft)) {
2282	err = PTR_ERR(ptr: ft);
2283	esw_warn(esw->dev, "Failed to create restore table, err %d\n",
2284	err);
2285	goto out_free;
2286	}
2287
2288	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
2289	match_criteria);
2290	misc = MLX5_ADDR_OF(fte_match_param, match_criteria,
2291	misc_parameters_2);
2292
2293	MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2294	ESW_REG_C0_USER_DATA_METADATA_MASK);
2295	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2296	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
2297	ft_attr.max_fte - 1);
2298	MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
2299	MLX5_MATCH_MISC_PARAMETERS_2);
2300	g = mlx5_create_flow_group(ft, in: flow_group_in);
2301	if (IS_ERR(ptr: g)) {
2302	err = PTR_ERR(ptr: g);
2303	esw_warn(dev, "Failed to create restore flow group, err: %d\n",
2304	err);
2305	goto err_group;
2306	}
2307
2308	MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY);
2309	MLX5_SET(copy_action_in, modact, src_field,
2310	MLX5_ACTION_IN_FIELD_METADATA_REG_C_1);
2311	MLX5_SET(copy_action_in, modact, dst_field,
2312	MLX5_ACTION_IN_FIELD_METADATA_REG_B);
2313	mod_hdr = mlx5_modify_header_alloc(dev: esw->dev,
2314	ns_type: MLX5_FLOW_NAMESPACE_KERNEL, num_actions: 1,
2315	modify_actions: modact);
2316	if (IS_ERR(ptr: mod_hdr)) {
2317	err = PTR_ERR(ptr: mod_hdr);
2318	esw_warn(dev, "Failed to create restore mod header, err: %d\n",
2319	err);
2320	goto err_mod_hdr;
2321	}
2322
2323	esw->offloads.ft_offloads_restore = ft;
2324	esw->offloads.restore_group = g;
2325	esw->offloads.restore_copy_hdr_id = mod_hdr;
2326
2327	kvfree(addr: flow_group_in);
2328
2329	return 0;
2330
2331	err_mod_hdr:
2332	mlx5_destroy_flow_group(fg: g);
2333	err_group:
2334	mlx5_destroy_flow_table(ft);
2335	out_free:
2336	kvfree(addr: flow_group_in);
2337
2338	return err;
2339	}
2340
2341	static void esw_mode_change(struct mlx5_eswitch *esw, u16 mode)
2342	{
2343	mlx5_devcom_comp_lock(devcom: esw->dev->priv.hca_devcom_comp);
2344	if (esw->dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_IB_ADEV ||
2345	mlx5_core_mp_enabled(dev: esw->dev)) {
2346	esw->mode = mode;
2347	mlx5_rescan_drivers_locked(dev: esw->dev);
2348	mlx5_devcom_comp_unlock(devcom: esw->dev->priv.hca_devcom_comp);
2349	return;
2350	}
2351
2352	esw->dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
2353	mlx5_rescan_drivers_locked(dev: esw->dev);
2354	esw->mode = mode;
2355	esw->dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
2356	mlx5_rescan_drivers_locked(dev: esw->dev);
2357	mlx5_devcom_comp_unlock(devcom: esw->dev->priv.hca_devcom_comp);
2358	}
2359
2360	static void mlx5_esw_fdb_drop_destroy(struct mlx5_eswitch *esw)
2361	{
2362	if (!esw->fdb_table.offloads.drop_root)
2363	return;
2364
2365	esw_debug(esw->dev, "Destroying FDB drop root table %#x fc %#x\n",
2366	esw->fdb_table.offloads.drop_root->id,
2367	esw->fdb_table.offloads.drop_root_fc->id);
2368	mlx5_del_flow_rules(fr: esw->fdb_table.offloads.drop_root_rule);
2369	/* Don't free flow counter here, can be reused on a later activation */
2370	mlx5_destroy_flow_table(ft: esw->fdb_table.offloads.drop_root);
2371	esw->fdb_table.offloads.drop_root_rule = NULL;
2372	esw->fdb_table.offloads.drop_root = NULL;
2373	}
2374
2375	static int mlx5_esw_fdb_drop_create(struct mlx5_eswitch *esw)
2376	{
2377	struct mlx5_flow_destination drop_fc_dst = {};
2378	struct mlx5_flow_table_attr ft_attr = {};
2379	struct mlx5_flow_destination *dst = NULL;
2380	struct mlx5_core_dev *dev = esw->dev;
2381	struct mlx5_flow_namespace *root_ns;
2382	struct mlx5_flow_act flow_act = {};
2383	struct mlx5_flow_handle *flow_rule;
2384	struct mlx5_flow_table *table;
2385	int err = 0, dst_num = 0;
2386
2387	if (esw->fdb_table.offloads.drop_root)
2388	return 0;
2389
2390	root_ns = esw->fdb_table.offloads.ns;
2391
2392	ft_attr.prio = FDB_DROP_ROOT;
2393	ft_attr.max_fte = 1;
2394	ft_attr.autogroup.max_num_groups = 1;
2395	table = mlx5_create_auto_grouped_flow_table(ns: root_ns, ft_attr: &ft_attr);
2396	if (IS_ERR(ptr: table)) {
2397	esw_warn(dev, "Failed to create fdb drop root table, err %pe\n",
2398	table);
2399	return PTR_ERR(ptr: table);
2400	}
2401
2402	/* Drop FC reusable, create once on first deactivation of FDB */
2403	if (!esw->fdb_table.offloads.drop_root_fc) {
2404	struct mlx5_fc *counter = mlx5_fc_create(dev, aging: 0);
2405
2406	err = PTR_ERR_OR_ZERO(ptr: counter);
2407	if (err)
2408	esw_warn(esw->dev, "create fdb drop fc err %d\n", err);
2409	else
2410	esw->fdb_table.offloads.drop_root_fc = counter;
2411	}
2412
2413	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
2414
2415	if (esw->fdb_table.offloads.drop_root_fc) {
2416	flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
2417	drop_fc_dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
2418	drop_fc_dst.counter = esw->fdb_table.offloads.drop_root_fc;
2419	dst = &drop_fc_dst;
2420	dst_num++;
2421	}
2422
2423	flow_rule = mlx5_add_flow_rules(ft: table, NULL, flow_act: &flow_act, dest: dst, num_dest: dst_num);
2424	err = PTR_ERR_OR_ZERO(ptr: flow_rule);
2425	if (err) {
2426	esw_warn(esw->dev,
2427	"fs offloads: Failed to add vport rx drop rule err %d\n",
2428	err);
2429	goto err_flow_rule;
2430	}
2431
2432	esw->fdb_table.offloads.drop_root = table;
2433	esw->fdb_table.offloads.drop_root_rule = flow_rule;
2434	esw_debug(esw->dev, "Created FDB drop root table %#x fc %#x\n",
2435	table->id, dst ? dst->counter->id : 0);
2436	return 0;
2437
2438	err_flow_rule:
2439	/* no need to free drop fc, esw_offloads_steering_cleanup will do it */
2440	mlx5_destroy_flow_table(ft: table);
2441	return err;
2442	}
2443
2444	static void mlx5_esw_fdb_active(struct mlx5_eswitch *esw)
2445	{
2446	struct mlx5_vport *vport;
2447	unsigned long i;
2448
2449	mlx5_esw_fdb_drop_destroy(esw);
2450	mlx5_mpfs_enable(dev: esw->dev);
2451
2452	mlx5_esw_for_each_vf_vport(esw, i, vport, U16_MAX) {
2453	if (!vport->adjacent)
2454	continue;
2455	esw_debug(esw->dev, "Connecting vport %d to eswitch\n",
2456	vport->vport);
2457	mlx5_esw_adj_vport_modify(dev: esw->dev, vport: vport->vport, connect: true);
2458	}
2459
2460	esw->offloads_inactive = false;
2461	esw_warn(esw->dev, "MPFS/FDB active\n");
2462	}
2463
2464	static void mlx5_esw_fdb_inactive(struct mlx5_eswitch *esw)
2465	{
2466	struct mlx5_vport *vport;
2467	unsigned long i;
2468
2469	mlx5_mpfs_disable(dev: esw->dev);
2470	mlx5_esw_fdb_drop_create(esw);
2471
2472	mlx5_esw_for_each_vf_vport(esw, i, vport, U16_MAX) {
2473	if (!vport->adjacent)
2474	continue;
2475	esw_debug(esw->dev, "Disconnecting vport %u from eswitch\n",
2476	vport->vport);
2477
2478	mlx5_esw_adj_vport_modify(dev: esw->dev, vport: vport->vport, connect: false);
2479	}
2480
2481	esw->offloads_inactive = true;
2482	esw_warn(esw->dev, "MPFS/FDB inactive\n");
2483	}
2484
2485	static int esw_offloads_start(struct mlx5_eswitch *esw,
2486	struct netlink_ext_ack *extack)
2487	{
2488	int err;
2489
2490	esw_mode_change(esw, mode: MLX5_ESWITCH_OFFLOADS);
2491	err = mlx5_eswitch_enable_locked(esw, num_vfs: esw->dev->priv.sriov.num_vfs);
2492	if (err) {
2493	NL_SET_ERR_MSG_MOD(extack,
2494	"Failed setting eswitch to offloads");
2495	esw_mode_change(esw, mode: MLX5_ESWITCH_LEGACY);
2496	return err;
2497	}
2498	if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) {
2499	if (mlx5_eswitch_inline_mode_get(esw,
2500	mode: &esw->offloads.inline_mode)) {
2501	esw->offloads.inline_mode = MLX5_INLINE_MODE_L2;
2502	NL_SET_ERR_MSG_MOD(extack,
2503	"Inline mode is different between vports");
2504	}
2505	}
2506	return 0;
2507	}
2508
2509	void mlx5_esw_offloads_rep_remove(struct mlx5_eswitch *esw,
2510	const struct mlx5_vport *vport)
2511	{
2512	struct mlx5_eswitch_rep *rep = xa_load(&esw->offloads.vport_reps,
2513	index: vport->vport);
2514
2515	if (!rep)
2516	return;
2517	xa_erase(&esw->offloads.vport_reps, index: vport->vport);
2518	kfree(objp: rep);
2519	}
2520
2521	int mlx5_esw_offloads_rep_add(struct mlx5_eswitch *esw,
2522	const struct mlx5_vport *vport)
2523	{
2524	struct mlx5_eswitch_rep *rep;
2525	int rep_type;
2526	int err;
2527
2528	rep = kzalloc(sizeof(*rep), GFP_KERNEL);
2529	if (!rep)
2530	return -ENOMEM;
2531
2532	rep->vport = vport->vport;
2533	rep->vport_index = vport->index;
2534	for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) {
2535	if (!esw->offloads.rep_ops[rep_type]) {
2536	atomic_set(v: &rep->rep_data[rep_type].state,
2537	i: REP_UNREGISTERED);
2538	continue;
2539	}
2540	/* Dynamic/delegated vports add their representors after
2541	* mlx5_eswitch_register_vport_reps, so mark them as registered
2542	* for them to be loaded later with the others.
2543	*/
2544	rep->esw = esw;
2545	atomic_set(v: &rep->rep_data[rep_type].state, i: REP_REGISTERED);
2546	}
2547	err = xa_insert(xa: &esw->offloads.vport_reps, index: rep->vport, entry: rep, GFP_KERNEL);
2548	if (err)
2549	goto insert_err;
2550
2551	return 0;
2552
2553	insert_err:
2554	kfree(objp: rep);
2555	return err;
2556	}
2557
2558	static void mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch *esw,
2559	struct mlx5_eswitch_rep *rep)
2560	{
2561	xa_erase(&esw->offloads.vport_reps, index: rep->vport);
2562	kfree(objp: rep);
2563	}
2564
2565	static void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw)
2566	{
2567	struct mlx5_eswitch_rep *rep;
2568	unsigned long i;
2569
2570	mlx5_esw_for_each_rep(esw, i, rep)
2571	mlx5_esw_offloads_rep_cleanup(esw, rep);
2572	xa_destroy(&esw->offloads.vport_reps);
2573	}
2574
2575	static int esw_offloads_init_reps(struct mlx5_eswitch *esw)
2576	{
2577	struct mlx5_vport *vport;
2578	unsigned long i;
2579	int err;
2580
2581	xa_init(xa: &esw->offloads.vport_reps);
2582
2583	mlx5_esw_for_each_vport(esw, i, vport) {
2584	err = mlx5_esw_offloads_rep_add(esw, vport);
2585	if (err)
2586	goto err;
2587	}
2588	return 0;
2589
2590	err:
2591	esw_offloads_cleanup_reps(esw);
2592	return err;
2593	}
2594
2595	static int esw_port_metadata_set(struct devlink *devlink, u32 id,
2596	struct devlink_param_gset_ctx *ctx,
2597	struct netlink_ext_ack *extack)
2598	{
2599	struct mlx5_core_dev *dev = devlink_priv(devlink);
2600	struct mlx5_eswitch *esw = dev->priv.eswitch;
2601	int err = 0;
2602
2603	down_write(sem: &esw->mode_lock);
2604	if (mlx5_esw_is_fdb_created(esw)) {
2605	err = -EBUSY;
2606	goto done;
2607	}
2608	if (!mlx5_esw_vport_match_metadata_supported(esw)) {
2609	err = -EOPNOTSUPP;
2610	goto done;
2611	}
2612	if (ctx->val.vbool)
2613	esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA;
2614	else
2615	esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA;
2616	done:
2617	up_write(sem: &esw->mode_lock);
2618	return err;
2619	}
2620
2621	static int esw_port_metadata_get(struct devlink *devlink, u32 id,
2622	struct devlink_param_gset_ctx *ctx,
2623	struct netlink_ext_ack *extack)
2624	{
2625	struct mlx5_core_dev *dev = devlink_priv(devlink);
2626
2627	ctx->val.vbool = mlx5_eswitch_vport_match_metadata_enabled(esw: dev->priv.eswitch);
2628	return 0;
2629	}
2630
2631	static int esw_port_metadata_validate(struct devlink *devlink, u32 id,
2632	union devlink_param_value val,
2633	struct netlink_ext_ack *extack)
2634	{
2635	struct mlx5_core_dev *dev = devlink_priv(devlink);
2636	u8 esw_mode;
2637
2638	esw_mode = mlx5_eswitch_mode(dev);
2639	if (esw_mode == MLX5_ESWITCH_OFFLOADS) {
2640	NL_SET_ERR_MSG_MOD(extack,
2641	"E-Switch must either disabled or non switchdev mode");
2642	return -EBUSY;
2643	}
2644	return 0;
2645	}
2646
2647	static const struct devlink_param esw_devlink_params[] = {
2648	DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_ESW_PORT_METADATA,
2649	"esw_port_metadata", DEVLINK_PARAM_TYPE_BOOL,
2650	BIT(DEVLINK_PARAM_CMODE_RUNTIME),
2651	esw_port_metadata_get,
2652	esw_port_metadata_set,
2653	esw_port_metadata_validate),
2654	};
2655
2656	int esw_offloads_init(struct mlx5_eswitch *esw)
2657	{
2658	int err;
2659
2660	err = esw_offloads_init_reps(esw);
2661	if (err)
2662	return err;
2663
2664	if (MLX5_ESWITCH_MANAGER(esw->dev) &&
2665	mlx5_esw_vport_match_metadata_supported(esw))
2666	esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA;
2667
2668	err = devl_params_register(devlink: priv_to_devlink(priv: esw->dev),
2669	params: esw_devlink_params,
2670	ARRAY_SIZE(esw_devlink_params));
2671	if (err)
2672	goto err_params;
2673
2674	return 0;
2675
2676	err_params:
2677	esw_offloads_cleanup_reps(esw);
2678	return err;
2679	}
2680
2681	void esw_offloads_cleanup(struct mlx5_eswitch *esw)
2682	{
2683	devl_params_unregister(devlink: priv_to_devlink(priv: esw->dev),
2684	params: esw_devlink_params,
2685	ARRAY_SIZE(esw_devlink_params));
2686	esw_offloads_cleanup_reps(esw);
2687	}
2688
2689	static int __esw_offloads_load_rep(struct mlx5_eswitch *esw,
2690	struct mlx5_eswitch_rep *rep, u8 rep_type)
2691	{
2692	if (atomic_cmpxchg(v: &rep->rep_data[rep_type].state,
2693	old: REP_REGISTERED, new: REP_LOADED) == REP_REGISTERED)
2694	return esw->offloads.rep_ops[rep_type]->load(esw->dev, rep);
2695
2696	return 0;
2697	}
2698
2699	static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw,
2700	struct mlx5_eswitch_rep *rep, u8 rep_type)
2701	{
2702	if (atomic_cmpxchg(v: &rep->rep_data[rep_type].state,
2703	old: REP_LOADED, new: REP_REGISTERED) == REP_LOADED) {
2704	if (rep_type == REP_ETH)
2705	__esw_offloads_unload_rep(esw, rep, rep_type: REP_IB);
2706	esw->offloads.rep_ops[rep_type]->unload(rep);
2707	}
2708	}
2709
2710	static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type)
2711	{
2712	struct mlx5_eswitch_rep *rep;
2713	unsigned long i;
2714
2715	mlx5_esw_for_each_rep(esw, i, rep)
2716	__esw_offloads_unload_rep(esw, rep, rep_type);
2717	}
2718
2719	static int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num)
2720	{
2721	struct mlx5_eswitch_rep *rep;
2722	int rep_type;
2723	int err;
2724
2725	rep = mlx5_eswitch_get_rep(esw, vport_num);
2726	for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) {
2727	err = __esw_offloads_load_rep(esw, rep, rep_type);
2728	if (err)
2729	goto err_reps;
2730	}
2731
2732	return 0;
2733
2734	err_reps:
2735	atomic_set(v: &rep->rep_data[rep_type].state, i: REP_REGISTERED);
2736	for (--rep_type; rep_type >= 0; rep_type--)
2737	__esw_offloads_unload_rep(esw, rep, rep_type);
2738	return err;
2739	}
2740
2741	static void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num)
2742	{
2743	struct mlx5_eswitch_rep *rep;
2744	int rep_type;
2745
2746	rep = mlx5_eswitch_get_rep(esw, vport_num);
2747	for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--)
2748	__esw_offloads_unload_rep(esw, rep, rep_type);
2749	}
2750
2751	int mlx5_esw_offloads_init_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2752	{
2753	if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2754	return 0;
2755
2756	return mlx5_esw_offloads_pf_vf_devlink_port_init(esw, vport);
2757	}
2758
2759	void mlx5_esw_offloads_cleanup_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2760	{
2761	if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2762	return;
2763
2764	mlx5_esw_offloads_pf_vf_devlink_port_cleanup(esw, vport);
2765	}
2766
2767	int mlx5_esw_offloads_init_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport,
2768	struct mlx5_devlink_port *dl_port,
2769	u32 controller, u32 sfnum)
2770	{
2771	return mlx5_esw_offloads_sf_devlink_port_init(esw, vport, dl_port, controller, sfnum);
2772	}
2773
2774	void mlx5_esw_offloads_cleanup_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2775	{
2776	mlx5_esw_offloads_sf_devlink_port_cleanup(esw, vport);
2777	}
2778
2779	int mlx5_esw_offloads_load_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2780	{
2781	int err;
2782
2783	if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2784	return 0;
2785
2786	err = mlx5_esw_offloads_devlink_port_register(esw, vport);
2787	if (err)
2788	return err;
2789
2790	err = mlx5_esw_offloads_rep_load(esw, vport_num: vport->vport);
2791	if (err)
2792	goto load_err;
2793	return err;
2794
2795	load_err:
2796	mlx5_esw_offloads_devlink_port_unregister(vport);
2797	return err;
2798	}
2799
2800	void mlx5_esw_offloads_unload_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2801	{
2802	if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2803	return;
2804
2805	mlx5_esw_offloads_rep_unload(esw, vport_num: vport->vport);
2806
2807	mlx5_esw_offloads_devlink_port_unregister(vport);
2808	}
2809
2810	static int esw_set_slave_root_fdb(struct mlx5_core_dev *master,
2811	struct mlx5_core_dev *slave)
2812	{
2813	u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {};
2814	u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {};
2815	struct mlx5_flow_root_namespace *root;
2816	struct mlx5_flow_namespace *ns;
2817	int err;
2818
2819	MLX5_SET(set_flow_table_root_in, in, opcode,
2820	MLX5_CMD_OP_SET_FLOW_TABLE_ROOT);
2821	MLX5_SET(set_flow_table_root_in, in, table_type,
2822	FS_FT_FDB);
2823
2824	if (master) {
2825	ns = mlx5_get_flow_namespace(dev: master,
2826	type: MLX5_FLOW_NAMESPACE_FDB);
2827	root = find_root(node: &ns->node);
2828	mutex_lock(&root->chain_lock);
2829	MLX5_SET(set_flow_table_root_in, in,
2830	table_eswitch_owner_vhca_id_valid, 1);
2831	MLX5_SET(set_flow_table_root_in, in,
2832	table_eswitch_owner_vhca_id,
2833	MLX5_CAP_GEN(master, vhca_id));
2834	MLX5_SET(set_flow_table_root_in, in, table_id,
2835	root->root_ft->id);
2836	} else {
2837	ns = mlx5_get_flow_namespace(dev: slave,
2838	type: MLX5_FLOW_NAMESPACE_FDB);
2839	root = find_root(node: &ns->node);
2840	mutex_lock(&root->chain_lock);
2841	MLX5_SET(set_flow_table_root_in, in, table_id,
2842	root->root_ft->id);
2843	}
2844
2845	err = mlx5_cmd_exec(dev: slave, in, in_size: sizeof(in), out, out_size: sizeof(out));
2846	mutex_unlock(lock: &root->chain_lock);
2847
2848	return err;
2849	}
2850
2851	static int __esw_set_master_egress_rule(struct mlx5_core_dev *master,
2852	struct mlx5_core_dev *slave,
2853	struct mlx5_vport *vport,
2854	struct mlx5_flow_table *acl)
2855	{
2856	u16 slave_index = MLX5_CAP_GEN(slave, vhca_id);
2857	struct mlx5_flow_handle *flow_rule = NULL;
2858	struct mlx5_flow_destination dest = {};
2859	struct mlx5_flow_act flow_act = {};
2860	struct mlx5_flow_spec *spec;
2861	int err = 0;
2862	void *misc;
2863
2864	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
2865	if (!spec)
2866	return -ENOMEM;
2867
2868	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
2869	misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
2870	misc_parameters);
2871	MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_UPLINK);
2872	MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, slave_index);
2873
2874	misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
2875	MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
2876	MLX5_SET_TO_ONES(fte_match_set_misc, misc,
2877	source_eswitch_owner_vhca_id);
2878
2879	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2880	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
2881	dest.vport.num = slave->priv.eswitch->manager_vport;
2882	dest.vport.vhca_id = MLX5_CAP_GEN(slave, vhca_id);
2883	dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
2884
2885	flow_rule = mlx5_add_flow_rules(ft: acl, spec, flow_act: &flow_act,
2886	dest: &dest, num_dest: 1);
2887	if (IS_ERR(ptr: flow_rule)) {
2888	err = PTR_ERR(ptr: flow_rule);
2889	} else {
2890	err = xa_insert(xa: &vport->egress.offloads.bounce_rules,
2891	index: slave_index, entry: flow_rule, GFP_KERNEL);
2892	if (err)
2893	mlx5_del_flow_rules(fr: flow_rule);
2894	}
2895
2896	kvfree(addr: spec);
2897	return err;
2898	}
2899
2900	static int esw_master_egress_create_resources(struct mlx5_eswitch *esw,
2901	struct mlx5_flow_namespace *egress_ns,
2902	struct mlx5_vport *vport, size_t count)
2903	{
2904	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2905	struct mlx5_flow_table_attr ft_attr = {
2906	.max_fte = count, .prio = 0, .level = 0,
2907	};
2908	struct mlx5_flow_table *acl;
2909	struct mlx5_flow_group *g;
2910	void *match_criteria;
2911	u32 *flow_group_in;
2912	int err;
2913
2914	if (vport->egress.acl)
2915	return 0;
2916
2917	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2918	if (!flow_group_in)
2919	return -ENOMEM;
2920
2921	if (vport->vport || mlx5_core_is_ecpf(dev: esw->dev))
2922	ft_attr.flags = MLX5_FLOW_TABLE_OTHER_VPORT;
2923
2924	acl = mlx5_create_vport_flow_table(ns: egress_ns, ft_attr: &ft_attr, vport: vport->vport);
2925	if (IS_ERR(ptr: acl)) {
2926	err = PTR_ERR(ptr: acl);
2927	goto out;
2928	}
2929
2930	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
2931	match_criteria);
2932	MLX5_SET_TO_ONES(fte_match_param, match_criteria,
2933	misc_parameters.source_port);
2934	MLX5_SET_TO_ONES(fte_match_param, match_criteria,
2935	misc_parameters.source_eswitch_owner_vhca_id);
2936	MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
2937	MLX5_MATCH_MISC_PARAMETERS);
2938
2939	MLX5_SET(create_flow_group_in, flow_group_in,
2940	source_eswitch_owner_vhca_id_valid, 1);
2941	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2942	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, count);
2943
2944	g = mlx5_create_flow_group(ft: acl, in: flow_group_in);
2945	if (IS_ERR(ptr: g)) {
2946	err = PTR_ERR(ptr: g);
2947	goto err_group;
2948	}
2949
2950	vport->egress.acl = acl;
2951	vport->egress.offloads.bounce_grp = g;
2952	vport->egress.type = VPORT_EGRESS_ACL_TYPE_SHARED_FDB;
2953	xa_init_flags(xa: &vport->egress.offloads.bounce_rules, XA_FLAGS_ALLOC);
2954
2955	kvfree(addr: flow_group_in);
2956
2957	return 0;
2958
2959	err_group:
2960	mlx5_destroy_flow_table(ft: acl);
2961	out:
2962	kvfree(addr: flow_group_in);
2963	return err;
2964	}
2965
2966	static void esw_master_egress_destroy_resources(struct mlx5_vport *vport)
2967	{
2968	if (!xa_empty(xa: &vport->egress.offloads.bounce_rules))
2969	return;
2970	mlx5_destroy_flow_group(fg: vport->egress.offloads.bounce_grp);
2971	vport->egress.offloads.bounce_grp = NULL;
2972	mlx5_destroy_flow_table(ft: vport->egress.acl);
2973	vport->egress.acl = NULL;
2974	}
2975
2976	static int esw_set_master_egress_rule(struct mlx5_core_dev *master,
2977	struct mlx5_core_dev *slave, size_t count)
2978	{
2979	struct mlx5_eswitch *esw = master->priv.eswitch;
2980	u16 slave_index = MLX5_CAP_GEN(slave, vhca_id);
2981	struct mlx5_flow_namespace *egress_ns;
2982	struct mlx5_vport *vport;
2983	int err;
2984
2985	vport = mlx5_eswitch_get_vport(esw, vport_num: esw->manager_vport);
2986	if (IS_ERR(ptr: vport))
2987	return PTR_ERR(ptr: vport);
2988
2989	egress_ns = mlx5_get_flow_vport_namespace(dev: master,
2990	type: MLX5_FLOW_NAMESPACE_ESW_EGRESS,
2991	vport_idx: vport->index);
2992	if (!egress_ns)
2993	return -EINVAL;
2994
2995	if (vport->egress.acl && vport->egress.type != VPORT_EGRESS_ACL_TYPE_SHARED_FDB)
2996	return 0;
2997
2998	err = esw_master_egress_create_resources(esw, egress_ns, vport, count);
2999	if (err)
3000	return err;
3001
3002	if (xa_load(&vport->egress.offloads.bounce_rules, index: slave_index))
3003	return -EINVAL;
3004
3005	err = __esw_set_master_egress_rule(master, slave, vport, acl: vport->egress.acl);
3006	if (err)
3007	goto err_rule;
3008
3009	return 0;
3010
3011	err_rule:
3012	esw_master_egress_destroy_resources(vport);
3013	return err;
3014	}
3015
3016	static void esw_unset_master_egress_rule(struct mlx5_core_dev *dev,
3017	struct mlx5_core_dev *slave_dev)
3018	{
3019	struct mlx5_vport *vport;
3020
3021	vport = mlx5_eswitch_get_vport(esw: dev->priv.eswitch,
3022	vport_num: dev->priv.eswitch->manager_vport);
3023
3024	esw_acl_egress_ofld_bounce_rule_destroy(vport, MLX5_CAP_GEN(slave_dev, vhca_id));
3025
3026	if (xa_empty(xa: &vport->egress.offloads.bounce_rules)) {
3027	esw_acl_egress_ofld_cleanup(vport);
3028	xa_destroy(&vport->egress.offloads.bounce_rules);
3029	}
3030	}
3031
3032	int mlx5_eswitch_offloads_single_fdb_add_one(struct mlx5_eswitch *master_esw,
3033	struct mlx5_eswitch *slave_esw, int max_slaves)
3034	{
3035	int err;
3036
3037	err = esw_set_slave_root_fdb(master: master_esw->dev,
3038	slave: slave_esw->dev);
3039	if (err)
3040	return err;
3041
3042	err = esw_set_master_egress_rule(master: master_esw->dev,
3043	slave: slave_esw->dev, count: max_slaves);
3044	if (err)
3045	goto err_acl;
3046
3047	return err;
3048
3049	err_acl:
3050	esw_set_slave_root_fdb(NULL, slave: slave_esw->dev);
3051	return err;
3052	}
3053
3054	void mlx5_eswitch_offloads_single_fdb_del_one(struct mlx5_eswitch *master_esw,
3055	struct mlx5_eswitch *slave_esw)
3056	{
3057	esw_set_slave_root_fdb(NULL, slave: slave_esw->dev);
3058	esw_unset_master_egress_rule(dev: master_esw->dev, slave_dev: slave_esw->dev);
3059	}
3060
3061	#define ESW_OFFLOADS_DEVCOM_PAIR (0)
3062	#define ESW_OFFLOADS_DEVCOM_UNPAIR (1)
3063
3064	static void mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch *esw,
3065	struct mlx5_eswitch *peer_esw)
3066	{
3067	const struct mlx5_eswitch_rep_ops *ops;
3068	struct mlx5_eswitch_rep *rep;
3069	unsigned long i;
3070	u8 rep_type;
3071
3072	mlx5_esw_for_each_rep(esw, i, rep) {
3073	rep_type = NUM_REP_TYPES;
3074	while (rep_type--) {
3075	ops = esw->offloads.rep_ops[rep_type];
3076	if (atomic_read(v: &rep->rep_data[rep_type].state) == REP_LOADED &&
3077	ops->event)
3078	ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_UNPAIR, peer_esw);
3079	}
3080	}
3081	}
3082
3083	static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw,
3084	struct mlx5_eswitch *peer_esw)
3085	{
3086	#if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
3087	mlx5e_tc_clean_fdb_peer_flows(esw);
3088	#endif
3089	mlx5_esw_offloads_rep_event_unpair(esw, peer_esw);
3090	esw_del_fdb_peer_miss_rules(esw, peer_dev: peer_esw->dev);
3091	}
3092
3093	static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw,
3094	struct mlx5_eswitch *peer_esw)
3095	{
3096	const struct mlx5_eswitch_rep_ops *ops;
3097	struct mlx5_eswitch_rep *rep;
3098	unsigned long i;
3099	u8 rep_type;
3100	int err;
3101
3102	err = esw_add_fdb_peer_miss_rules(esw, peer_dev: peer_esw->dev);
3103	if (err)
3104	return err;
3105
3106	mlx5_esw_for_each_rep(esw, i, rep) {
3107	for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) {
3108	ops = esw->offloads.rep_ops[rep_type];
3109	if (atomic_read(v: &rep->rep_data[rep_type].state) == REP_LOADED &&
3110	ops->event) {
3111	err = ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_PAIR, peer_esw);
3112	if (err)
3113	goto err_out;
3114	}
3115	}
3116	}
3117
3118	return 0;
3119
3120	err_out:
3121	mlx5_esw_offloads_unpair(esw, peer_esw);
3122	return err;
3123	}
3124
3125	static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw,
3126	struct mlx5_eswitch *peer_esw,
3127	bool pair)
3128	{
3129	u16 peer_vhca_id = MLX5_CAP_GEN(peer_esw->dev, vhca_id);
3130	u16 vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id);
3131	struct mlx5_flow_root_namespace *peer_ns;
3132	struct mlx5_flow_root_namespace *ns;
3133	int err;
3134
3135	peer_ns = peer_esw->dev->priv.steering->fdb_root_ns;
3136	ns = esw->dev->priv.steering->fdb_root_ns;
3137
3138	if (pair) {
3139	err = mlx5_flow_namespace_set_peer(ns, peer_ns, peer_vhca_id);
3140	if (err)
3141	return err;
3142
3143	err = mlx5_flow_namespace_set_peer(ns: peer_ns, peer_ns: ns, peer_vhca_id: vhca_id);
3144	if (err) {
3145	mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id);
3146	return err;
3147	}
3148	} else {
3149	mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id);
3150	mlx5_flow_namespace_set_peer(ns: peer_ns, NULL, peer_vhca_id: vhca_id);
3151	}
3152
3153	return 0;
3154	}
3155
3156	static int mlx5_esw_offloads_devcom_event(int event,
3157	void *my_data,
3158	void *event_data)
3159	{
3160	struct mlx5_eswitch *esw = my_data;
3161	struct mlx5_eswitch *peer_esw = event_data;
3162	u16 esw_i, peer_esw_i;
3163	bool esw_paired;
3164	int err;
3165
3166	peer_esw_i = MLX5_CAP_GEN(peer_esw->dev, vhca_id);
3167	esw_i = MLX5_CAP_GEN(esw->dev, vhca_id);
3168	esw_paired = !!xa_load(&esw->paired, index: peer_esw_i);
3169
3170	switch (event) {
3171	case ESW_OFFLOADS_DEVCOM_PAIR:
3172	if (mlx5_eswitch_vport_match_metadata_enabled(esw) !=
3173	mlx5_eswitch_vport_match_metadata_enabled(esw: peer_esw))
3174	break;
3175
3176	if (esw_paired)
3177	break;
3178
3179	err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, pair: true);
3180	if (err)
3181	goto err_out;
3182
3183	err = mlx5_esw_offloads_pair(esw, peer_esw);
3184	if (err)
3185	goto err_peer;
3186
3187	err = mlx5_esw_offloads_pair(esw: peer_esw, peer_esw: esw);
3188	if (err)
3189	goto err_pair;
3190
3191	err = xa_insert(xa: &esw->paired, index: peer_esw_i, entry: peer_esw, GFP_KERNEL);
3192	if (err)
3193	goto err_xa;
3194
3195	err = xa_insert(xa: &peer_esw->paired, index: esw_i, entry: esw, GFP_KERNEL);
3196	if (err)
3197	goto err_peer_xa;
3198
3199	esw->num_peers++;
3200	peer_esw->num_peers++;
3201	mlx5_devcom_comp_set_ready(devcom: esw->devcom, ready: true);
3202	break;
3203
3204	case ESW_OFFLOADS_DEVCOM_UNPAIR:
3205	if (!esw_paired)
3206	break;
3207
3208	peer_esw->num_peers--;
3209	esw->num_peers--;
3210	if (!esw->num_peers && !peer_esw->num_peers)
3211	mlx5_devcom_comp_set_ready(devcom: esw->devcom, ready: false);
3212	xa_erase(&peer_esw->paired, index: esw_i);
3213	xa_erase(&esw->paired, index: peer_esw_i);
3214	mlx5_esw_offloads_unpair(esw: peer_esw, peer_esw: esw);
3215	mlx5_esw_offloads_unpair(esw, peer_esw);
3216	mlx5_esw_offloads_set_ns_peer(esw, peer_esw, pair: false);
3217	break;
3218	}
3219
3220	return 0;
3221
3222	err_peer_xa:
3223	xa_erase(&esw->paired, index: peer_esw_i);
3224	err_xa:
3225	mlx5_esw_offloads_unpair(esw: peer_esw, peer_esw: esw);
3226	err_pair:
3227	mlx5_esw_offloads_unpair(esw, peer_esw);
3228	err_peer:
3229	mlx5_esw_offloads_set_ns_peer(esw, peer_esw, pair: false);
3230	err_out:
3231	mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d",
3232	event, err);
3233	return err;
3234	}
3235
3236	void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw,
3237	const struct mlx5_devcom_match_attr *attr)
3238	{
3239	int i;
3240
3241	for (i = 0; i < MLX5_MAX_PORTS; i++)
3242	INIT_LIST_HEAD(list: &esw->offloads.peer_flows[i]);
3243	mutex_init(&esw->offloads.peer_mutex);
3244
3245	if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
3246	return;
3247
3248	if ((MLX5_VPORT_MANAGER(esw->dev) || mlx5_core_is_ecpf_esw_manager(dev: esw->dev)) &&
3249	!mlx5_lag_is_supported(dev: esw->dev))
3250	return;
3251
3252	xa_init(xa: &esw->paired);
3253	esw->num_peers = 0;
3254	esw->devcom = mlx5_devcom_register_component(devc: esw->dev->priv.devc,
3255	id: MLX5_DEVCOM_ESW_OFFLOADS,
3256	attr,
3257	handler: mlx5_esw_offloads_devcom_event,
3258	data: esw);
3259	if (!esw->devcom)
3260	return;
3261
3262	mlx5_devcom_send_event(devcom: esw->devcom,
3263	ESW_OFFLOADS_DEVCOM_PAIR,
3264	ESW_OFFLOADS_DEVCOM_UNPAIR,
3265	event_data: esw);
3266	}
3267
3268	void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw)
3269	{
3270	if (!esw->devcom)
3271	return;
3272
3273	mlx5_devcom_send_event(devcom: esw->devcom,
3274	ESW_OFFLOADS_DEVCOM_UNPAIR,
3275	ESW_OFFLOADS_DEVCOM_UNPAIR,
3276	event_data: esw);
3277
3278	mlx5_devcom_unregister_component(devcom: esw->devcom);
3279	xa_destroy(&esw->paired);
3280	esw->devcom = NULL;
3281	}
3282
3283	bool mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch *esw)
3284	{
3285	return mlx5_devcom_comp_is_ready(devcom: esw->devcom);
3286	}
3287
3288	bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw)
3289	{
3290	if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl))
3291	return false;
3292
3293	if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
3294	MLX5_FDB_TO_VPORT_REG_C_0))
3295	return false;
3296
3297	return true;
3298	}
3299
3300	#define MLX5_ESW_METADATA_RSVD_UPLINK 1
3301
3302	/* Share the same metadata for uplink's. This is fine because:
3303	* (a) In shared FDB mode (LAG) both uplink's are treated the
3304	* same and tagged with the same metadata.
3305	* (b) In non shared FDB mode, packets from physical port0
3306	* cannot hit eswitch of PF1 and vice versa.
3307	*/
3308	static u32 mlx5_esw_match_metadata_reserved(struct mlx5_eswitch *esw)
3309	{
3310	return MLX5_ESW_METADATA_RSVD_UPLINK;
3311	}
3312
3313	u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw)
3314	{
3315	u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1;
3316	/* Reserve 0xf for internal port offload */
3317	u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 2;
3318	u32 pf_num;
3319	int id;
3320
3321	/* Only 4 bits of pf_num */
3322	pf_num = mlx5_get_dev_index(dev: esw->dev);
3323	if (pf_num > max_pf_num)
3324	return 0;
3325
3326	/* Metadata is 4 bits of PFNUM and 12 bits of unique id */
3327	/* Use only non-zero vport_id (2-4095) for all PF's */
3328	id = ida_alloc_range(&esw->offloads.vport_metadata_ida,
3329	MLX5_ESW_METADATA_RSVD_UPLINK + 1,
3330	max: vport_end_ida, GFP_KERNEL);
3331	if (id < 0)
3332	return 0;
3333	id = (pf_num << ESW_VPORT_BITS) | id;
3334	return id;
3335	}
3336
3337	void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata)
3338	{
3339	u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1;
3340
3341	/* Metadata contains only 12 bits of actual ida id */
3342	ida_free(&esw->offloads.vport_metadata_ida, id: metadata & vport_bit_mask);
3343	}
3344
3345	static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw,
3346	struct mlx5_vport *vport)
3347	{
3348	if (vport->vport == MLX5_VPORT_UPLINK)
3349	vport->default_metadata = mlx5_esw_match_metadata_reserved(esw);
3350	else
3351	vport->default_metadata = mlx5_esw_match_metadata_alloc(esw);
3352
3353	vport->metadata = vport->default_metadata;
3354	return vport->metadata ? 0 : -ENOSPC;
3355	}
3356
3357	static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw,
3358	struct mlx5_vport *vport)
3359	{
3360	if (!vport->default_metadata)
3361	return;
3362
3363	if (vport->vport == MLX5_VPORT_UPLINK)
3364	return;
3365
3366	WARN_ON(vport->metadata != vport->default_metadata);
3367	mlx5_esw_match_metadata_free(esw, metadata: vport->default_metadata);
3368	}
3369
3370	static void esw_offloads_metadata_uninit(struct mlx5_eswitch *esw)
3371	{
3372	struct mlx5_vport *vport;
3373	unsigned long i;
3374
3375	if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
3376	return;
3377
3378	mlx5_esw_for_each_vport(esw, i, vport)
3379	esw_offloads_vport_metadata_cleanup(esw, vport);
3380	}
3381
3382	static int esw_offloads_metadata_init(struct mlx5_eswitch *esw)
3383	{
3384	struct mlx5_vport *vport;
3385	unsigned long i;
3386	int err;
3387
3388	if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
3389	return 0;
3390
3391	mlx5_esw_for_each_vport(esw, i, vport) {
3392	err = esw_offloads_vport_metadata_setup(esw, vport);
3393	if (err)
3394	goto metadata_err;
3395	}
3396
3397	return 0;
3398
3399	metadata_err:
3400	esw_offloads_metadata_uninit(esw);
3401	return err;
3402	}
3403
3404	int
3405	esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw,
3406	struct mlx5_vport *vport)
3407	{
3408	int err;
3409
3410	err = esw_acl_ingress_ofld_setup(esw, vport);
3411	if (err)
3412	return err;
3413
3414	err = esw_acl_egress_ofld_setup(esw, vport);
3415	if (err)
3416	goto egress_err;
3417
3418	return 0;
3419
3420	egress_err:
3421	esw_acl_ingress_ofld_cleanup(esw, vport);
3422	return err;
3423	}
3424
3425	void
3426	esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw,
3427	struct mlx5_vport *vport)
3428	{
3429	esw_acl_egress_ofld_cleanup(vport);
3430	esw_acl_ingress_ofld_cleanup(esw, vport);
3431	}
3432
3433	static int esw_create_offloads_acl_tables(struct mlx5_eswitch *esw)
3434	{
3435	struct mlx5_vport *uplink, *manager;
3436	int ret;
3437
3438	uplink = mlx5_eswitch_get_vport(esw, vport_num: MLX5_VPORT_UPLINK);
3439	if (IS_ERR(ptr: uplink))
3440	return PTR_ERR(ptr: uplink);
3441
3442	ret = esw_vport_create_offloads_acl_tables(esw, vport: uplink);
3443	if (ret)
3444	return ret;
3445
3446	manager = mlx5_eswitch_get_vport(esw, vport_num: esw->manager_vport);
3447	if (IS_ERR(ptr: manager)) {
3448	ret = PTR_ERR(ptr: manager);
3449	goto err_manager;
3450	}
3451
3452	ret = esw_vport_create_offloads_acl_tables(esw, vport: manager);
3453	if (ret)
3454	goto err_manager;
3455
3456	return 0;
3457
3458	err_manager:
3459	esw_vport_destroy_offloads_acl_tables(esw, vport: uplink);
3460	return ret;
3461	}
3462
3463	static void esw_destroy_offloads_acl_tables(struct mlx5_eswitch *esw)
3464	{
3465	struct mlx5_vport *vport;
3466
3467	vport = mlx5_eswitch_get_vport(esw, vport_num: esw->manager_vport);
3468	if (!IS_ERR(ptr: vport))
3469	esw_vport_destroy_offloads_acl_tables(esw, vport);
3470
3471	vport = mlx5_eswitch_get_vport(esw, vport_num: MLX5_VPORT_UPLINK);
3472	if (!IS_ERR(ptr: vport))
3473	esw_vport_destroy_offloads_acl_tables(esw, vport);
3474	}
3475
3476	int mlx5_eswitch_reload_ib_reps(struct mlx5_eswitch *esw)
3477	{
3478	struct mlx5_eswitch_rep *rep;
3479	unsigned long i;
3480	int ret;
3481
3482	if (!esw || esw->mode != MLX5_ESWITCH_OFFLOADS)
3483	return 0;
3484
3485	rep = mlx5_eswitch_get_rep(esw, vport_num: MLX5_VPORT_UPLINK);
3486	if (atomic_read(v: &rep->rep_data[REP_ETH].state) != REP_LOADED)
3487	return 0;
3488
3489	ret = __esw_offloads_load_rep(esw, rep, rep_type: REP_IB);
3490	if (ret)
3491	return ret;
3492
3493	mlx5_esw_for_each_rep(esw, i, rep) {
3494	if (atomic_read(v: &rep->rep_data[REP_ETH].state) == REP_LOADED)
3495	__esw_offloads_load_rep(esw, rep, rep_type: REP_IB);
3496	}
3497
3498	return 0;
3499	}
3500
3501	static int esw_offloads_steering_init(struct mlx5_eswitch *esw)
3502	{
3503	struct mlx5_esw_indir_table *indir;
3504	int err;
3505
3506	memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb));
3507	mutex_init(&esw->fdb_table.offloads.vports.lock);
3508	hash_init(esw->fdb_table.offloads.vports.table);
3509	atomic64_set(v: &esw->user_count, i: 0);
3510
3511	indir = mlx5_esw_indir_table_init();
3512	if (IS_ERR(ptr: indir)) {
3513	err = PTR_ERR(ptr: indir);
3514	goto create_indir_err;
3515	}
3516	esw->fdb_table.offloads.indir = indir;
3517
3518	err = esw_create_offloads_acl_tables(esw);
3519	if (err)
3520	goto create_acl_err;
3521
3522	err = esw_create_offloads_table(esw);
3523	if (err)
3524	goto create_offloads_err;
3525
3526	err = esw_create_restore_table(esw);
3527	if (err)
3528	goto create_restore_err;
3529
3530	err = esw_create_offloads_fdb_tables(esw);
3531	if (err)
3532	goto create_fdb_err;
3533
3534	err = esw_create_vport_rx_group(esw);
3535	if (err)
3536	goto create_fg_err;
3537
3538	err = esw_create_vport_rx_drop_group(esw);
3539	if (err)
3540	goto create_rx_drop_fg_err;
3541
3542	err = esw_create_vport_rx_drop_rule(esw);
3543	if (err)
3544	goto create_rx_drop_rule_err;
3545
3546	return 0;
3547
3548	create_rx_drop_rule_err:
3549	esw_destroy_vport_rx_drop_group(esw);
3550	create_rx_drop_fg_err:
3551	esw_destroy_vport_rx_group(esw);
3552	create_fg_err:
3553	esw_destroy_offloads_fdb_tables(esw);
3554	create_fdb_err:
3555	esw_destroy_restore_table(esw);
3556	create_restore_err:
3557	esw_destroy_offloads_table(esw);
3558	create_offloads_err:
3559	esw_destroy_offloads_acl_tables(esw);
3560	create_acl_err:
3561	mlx5_esw_indir_table_destroy(indir: esw->fdb_table.offloads.indir);
3562	create_indir_err:
3563	mutex_destroy(lock: &esw->fdb_table.offloads.vports.lock);
3564	return err;
3565	}
3566
3567	static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw)
3568	{
3569	mlx5_esw_fdb_drop_destroy(esw);
3570	if (esw->fdb_table.offloads.drop_root_fc)
3571	mlx5_fc_destroy(dev: esw->dev, counter: esw->fdb_table.offloads.drop_root_fc);
3572	esw->fdb_table.offloads.drop_root_fc = NULL;
3573	esw_destroy_vport_rx_drop_rule(esw);
3574	esw_destroy_vport_rx_drop_group(esw);
3575	esw_destroy_vport_rx_group(esw);
3576	esw_destroy_offloads_fdb_tables(esw);
3577	esw_destroy_restore_table(esw);
3578	esw_destroy_offloads_table(esw);
3579	esw_destroy_offloads_acl_tables(esw);
3580	mlx5_esw_indir_table_destroy(indir: esw->fdb_table.offloads.indir);
3581	mutex_destroy(lock: &esw->fdb_table.offloads.vports.lock);
3582	}
3583
3584	static void
3585	esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out)
3586	{
3587	struct devlink *devlink;
3588	bool host_pf_disabled;
3589	u16 new_num_vfs;
3590
3591	new_num_vfs = MLX5_GET(query_esw_functions_out, out,
3592	host_params_context.host_num_of_vfs);
3593	host_pf_disabled = MLX5_GET(query_esw_functions_out, out,
3594	host_params_context.host_pf_disabled);
3595
3596	if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled)
3597	return;
3598
3599	devlink = priv_to_devlink(priv: esw->dev);
3600	devl_lock(devlink);
3601	/* Number of VFs can only change from "0 to x" or "x to 0". */
3602	if (esw->esw_funcs.num_vfs > 0) {
3603	mlx5_eswitch_unload_vf_vports(esw, num_vfs: esw->esw_funcs.num_vfs);
3604	} else {
3605	int err;
3606
3607	err = mlx5_eswitch_load_vf_vports(esw, num_vfs: new_num_vfs,
3608	enabled_events: MLX5_VPORT_UC_ADDR_CHANGE);
3609	if (err) {
3610	devl_unlock(devlink);
3611	return;
3612	}
3613	}
3614	esw->esw_funcs.num_vfs = new_num_vfs;
3615	devl_unlock(devlink);
3616	}
3617
3618	static void esw_functions_changed_event_handler(struct work_struct *work)
3619	{
3620	struct mlx5_host_work *host_work;
3621	struct mlx5_eswitch *esw;
3622	const u32 *out;
3623
3624	host_work = container_of(work, struct mlx5_host_work, work);
3625	esw = host_work->esw;
3626
3627	out = mlx5_esw_query_functions(dev: esw->dev);
3628	if (IS_ERR(ptr: out))
3629	goto out;
3630
3631	esw_vfs_changed_event_handler(esw, out);
3632	kvfree(addr: out);
3633	out:
3634	kfree(objp: host_work);
3635	}
3636
3637	int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data)
3638	{
3639	struct mlx5_esw_functions *esw_funcs;
3640	struct mlx5_host_work *host_work;
3641	struct mlx5_eswitch *esw;
3642
3643	host_work = kzalloc(sizeof(*host_work), GFP_ATOMIC);
3644	if (!host_work)
3645	return NOTIFY_DONE;
3646
3647	esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb);
3648	esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs);
3649
3650	host_work->esw = esw;
3651
3652	INIT_WORK(&host_work->work, esw_functions_changed_event_handler);
3653	queue_work(wq: esw->work_queue, work: &host_work->work);
3654
3655	return NOTIFY_OK;
3656	}
3657
3658	static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw)
3659	{
3660	const u32 *query_host_out;
3661
3662	if (!mlx5_core_is_ecpf_esw_manager(dev: esw->dev))
3663	return 0;
3664
3665	query_host_out = mlx5_esw_query_functions(dev: esw->dev);
3666	if (IS_ERR(ptr: query_host_out))
3667	return PTR_ERR(ptr: query_host_out);
3668
3669	/* Mark non local controller with non zero controller number. */
3670	esw->offloads.host_number = MLX5_GET(query_esw_functions_out, query_host_out,
3671	host_params_context.host_number);
3672	kvfree(addr: query_host_out);
3673	return 0;
3674	}
3675
3676	bool mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch *esw, u32 controller)
3677	{
3678	/* Local controller is always valid */
3679	if (controller == 0)
3680	return true;
3681
3682	if (!mlx5_core_is_ecpf_esw_manager(dev: esw->dev))
3683	return false;
3684
3685	/* External host number starts with zero in device */
3686	return (controller == esw->offloads.host_number + 1);
3687	}
3688
3689	int esw_offloads_enable(struct mlx5_eswitch *esw)
3690	{
3691	u8 mapping_id[MLX5_SW_IMAGE_GUID_MAX_BYTES];
3692	struct mapping_ctx *reg_c0_obj_pool;
3693	struct mlx5_vport *vport;
3694	unsigned long i;
3695	u8 id_len;
3696	int err;
3697
3698	mutex_init(&esw->offloads.termtbl_mutex);
3699	mlx5_esw_adjacent_vhcas_setup(esw);
3700
3701	err = mlx5_rdma_enable_roce(dev: esw->dev);
3702	if (err)
3703	goto err_roce;
3704
3705	err = mlx5_esw_host_number_init(esw);
3706	if (err)
3707	goto err_metadata;
3708
3709	err = esw_offloads_metadata_init(esw);
3710	if (err)
3711	goto err_metadata;
3712
3713	err = esw_set_passing_vport_metadata(esw, enable: true);
3714	if (err)
3715	goto err_vport_metadata;
3716
3717	mlx5_query_nic_sw_system_image_guid(mdev: esw->dev, buf: mapping_id, len: &id_len);
3718
3719	reg_c0_obj_pool = mapping_create_for_id(id: mapping_id, id_len,
3720	type: MAPPING_TYPE_CHAIN,
3721	data_size: sizeof(struct mlx5_mapped_obj),
3722	ESW_REG_C0_USER_DATA_METADATA_MASK,
3723	delayed_removal: true);
3724
3725	if (IS_ERR(ptr: reg_c0_obj_pool)) {
3726	err = PTR_ERR(ptr: reg_c0_obj_pool);
3727	goto err_pool;
3728	}
3729	esw->offloads.reg_c0_obj_pool = reg_c0_obj_pool;
3730
3731	err = esw_offloads_steering_init(esw);
3732	if (err)
3733	goto err_steering_init;
3734
3735	if (esw->offloads_inactive)
3736	mlx5_esw_fdb_inactive(esw);
3737	else
3738	mlx5_esw_fdb_active(esw);
3739
3740	/* Representor will control the vport link state */
3741	mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs)
3742	vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN;
3743	if (mlx5_core_ec_sriov_enabled(dev: esw->dev))
3744	mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs)
3745	vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN;
3746
3747	/* Uplink vport rep must load first. */
3748	err = mlx5_esw_offloads_rep_load(esw, vport_num: MLX5_VPORT_UPLINK);
3749	if (err)
3750	goto err_uplink;
3751
3752	err = mlx5_eswitch_enable_pf_vf_vports(esw, enabled_events: MLX5_VPORT_UC_ADDR_CHANGE);
3753	if (err)
3754	goto err_vports;
3755
3756	return 0;
3757
3758	err_vports:
3759	mlx5_esw_offloads_rep_unload(esw, vport_num: MLX5_VPORT_UPLINK);
3760	err_uplink:
3761	esw_offloads_steering_cleanup(esw);
3762	err_steering_init:
3763	mapping_destroy(ctx: reg_c0_obj_pool);
3764	err_pool:
3765	esw_set_passing_vport_metadata(esw, enable: false);
3766	err_vport_metadata:
3767	esw_offloads_metadata_uninit(esw);
3768	err_metadata:
3769	mlx5_rdma_disable_roce(dev: esw->dev);
3770	err_roce:
3771	mlx5_esw_adjacent_vhcas_cleanup(esw);
3772	mutex_destroy(lock: &esw->offloads.termtbl_mutex);
3773	return err;
3774	}
3775
3776	static int esw_offloads_stop(struct mlx5_eswitch *esw,
3777	struct netlink_ext_ack *extack)
3778	{
3779	int err;
3780
3781	esw_mode_change(esw, mode: MLX5_ESWITCH_LEGACY);
3782
3783	/* If changing from switchdev to legacy mode without sriov enabled,
3784	* no need to create legacy fdb.
3785	*/
3786	if (!mlx5_core_is_pf(dev: esw->dev) || !mlx5_sriov_is_enabled(dev: esw->dev))
3787	return 0;
3788
3789	err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS);
3790	if (err)
3791	NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy");
3792
3793	return err;
3794	}
3795
3796	void esw_offloads_disable(struct mlx5_eswitch *esw)
3797	{
3798	mlx5_eswitch_disable_pf_vf_vports(esw);
3799	mlx5_esw_offloads_rep_unload(esw, vport_num: MLX5_VPORT_UPLINK);
3800	esw_set_passing_vport_metadata(esw, enable: false);
3801	esw_offloads_steering_cleanup(esw);
3802	mapping_destroy(ctx: esw->offloads.reg_c0_obj_pool);
3803	esw_offloads_metadata_uninit(esw);
3804	mlx5_rdma_disable_roce(dev: esw->dev);
3805	mlx5_esw_adjacent_vhcas_cleanup(esw);
3806	/* must be done after vhcas cleanup to avoid adjacent vports connect */
3807	if (esw->offloads_inactive)
3808	mlx5_esw_fdb_active(esw); /* legacy mode always active */
3809	mutex_destroy(lock: &esw->offloads.termtbl_mutex);
3810	}
3811
3812	static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode)
3813	{
3814	switch (mode) {
3815	case DEVLINK_ESWITCH_MODE_LEGACY:
3816	*mlx5_mode = MLX5_ESWITCH_LEGACY;
3817	break;
3818	case DEVLINK_ESWITCH_MODE_SWITCHDEV:
3819	case DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE:
3820	*mlx5_mode = MLX5_ESWITCH_OFFLOADS;
3821	break;
3822	default:
3823	return -EINVAL;
3824	}
3825
3826	return 0;
3827	}
3828
3829	static int esw_mode_to_devlink(struct mlx5_eswitch *esw, u16 *mode)
3830	{
3831	switch (esw->mode) {
3832	case MLX5_ESWITCH_LEGACY:
3833	*mode = DEVLINK_ESWITCH_MODE_LEGACY;
3834	break;
3835	case MLX5_ESWITCH_OFFLOADS:
3836	if (esw->offloads_inactive)
3837	*mode = DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE;
3838	else
3839	*mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;
3840	break;
3841	default:
3842	return -EINVAL;
3843	}
3844
3845	return 0;
3846	}
3847
3848	static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode)
3849	{
3850	switch (mode) {
3851	case DEVLINK_ESWITCH_INLINE_MODE_NONE:
3852	*mlx5_mode = MLX5_INLINE_MODE_NONE;
3853	break;
3854	case DEVLINK_ESWITCH_INLINE_MODE_LINK:
3855	*mlx5_mode = MLX5_INLINE_MODE_L2;
3856	break;
3857	case DEVLINK_ESWITCH_INLINE_MODE_NETWORK:
3858	*mlx5_mode = MLX5_INLINE_MODE_IP;
3859	break;
3860	case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT:
3861	*mlx5_mode = MLX5_INLINE_MODE_TCP_UDP;
3862	break;
3863	default:
3864	return -EINVAL;
3865	}
3866
3867	return 0;
3868	}
3869
3870	static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode)
3871	{
3872	switch (mlx5_mode) {
3873	case MLX5_INLINE_MODE_NONE:
3874	*mode = DEVLINK_ESWITCH_INLINE_MODE_NONE;
3875	break;
3876	case MLX5_INLINE_MODE_L2:
3877	*mode = DEVLINK_ESWITCH_INLINE_MODE_LINK;
3878	break;
3879	case MLX5_INLINE_MODE_IP:
3880	*mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK;
3881	break;
3882	case MLX5_INLINE_MODE_TCP_UDP:
3883	*mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT;
3884	break;
3885	default:
3886	return -EINVAL;
3887	}
3888
3889	return 0;
3890	}
3891
3892	int mlx5_eswitch_block_mode(struct mlx5_core_dev *dev)
3893	{
3894	struct mlx5_eswitch *esw = dev->priv.eswitch;
3895	int err;
3896
3897	if (!mlx5_esw_allowed(esw))
3898	return 0;
3899
3900	/* Take TC into account */
3901	err = mlx5_esw_try_lock(esw);
3902	if (err < 0)
3903	return err;
3904
3905	esw->offloads.num_block_mode++;
3906	mlx5_esw_unlock(esw);
3907	return 0;
3908	}
3909
3910	void mlx5_eswitch_unblock_mode(struct mlx5_core_dev *dev)
3911	{
3912	struct mlx5_eswitch *esw = dev->priv.eswitch;
3913
3914	if (!mlx5_esw_allowed(esw))
3915	return;
3916
3917	down_write(sem: &esw->mode_lock);
3918	esw->offloads.num_block_mode--;
3919	up_write(sem: &esw->mode_lock);
3920	}
3921
3922	/* Returns false only when uplink netdev exists and its netns is different from
3923	* devlink's netns. True for all others so entering switchdev mode is allowed.
3924	*/
3925	static bool mlx5_devlink_netdev_netns_immutable_set(struct devlink *devlink,
3926	bool immutable)
3927	{
3928	struct mlx5_core_dev *mdev = devlink_priv(devlink);
3929	struct net_device *netdev;
3930	bool ret;
3931
3932	netdev = mlx5_uplink_netdev_get(mdev);
3933	if (!netdev)
3934	return true;
3935
3936	rtnl_lock();
3937	netdev->netns_immutable = immutable;
3938	ret = net_eq(net1: dev_net(dev: netdev), net2: devlink_net(devlink));
3939	rtnl_unlock();
3940
3941	mlx5_uplink_netdev_put(mdev, netdev);
3942	return ret;
3943	}
3944
3945	/* Returns true when only changing between active and inactive switchdev mode */
3946	static bool mlx5_devlink_switchdev_active_mode_change(struct mlx5_eswitch *esw,
3947	u16 devlink_mode)
3948	{
3949	/* current mode is not switchdev */
3950	if (esw->mode != MLX5_ESWITCH_OFFLOADS)
3951	return false;
3952
3953	/* new mode is not switchdev */
3954	if (devlink_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV &&
3955	devlink_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE)
3956	return false;
3957
3958	/* already inactive: no change in current state */
3959	if (devlink_mode == DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE &&
3960	esw->offloads_inactive)
3961	return false;
3962
3963	/* already active: no change in current state */
3964	if (devlink_mode == DEVLINK_ESWITCH_MODE_SWITCHDEV &&
3965	!esw->offloads_inactive)
3966	return false;
3967
3968	down_write(sem: &esw->mode_lock);
3969	esw->offloads_inactive = !esw->offloads_inactive;
3970	esw->eswitch_operation_in_progress = true;
3971	up_write(sem: &esw->mode_lock);
3972
3973	if (esw->offloads_inactive)
3974	mlx5_esw_fdb_inactive(esw);
3975	else
3976	mlx5_esw_fdb_active(esw);
3977
3978	down_write(sem: &esw->mode_lock);
3979	esw->eswitch_operation_in_progress = false;
3980	up_write(sem: &esw->mode_lock);
3981	return true;
3982	}
3983
3984	#define MLX5_ESW_HOLD_TIMEOUT_MS 7000
3985	#define MLX5_ESW_HOLD_RETRY_DELAY_MS 500
3986
3987	void mlx5_eswitch_safe_aux_devs_remove(struct mlx5_core_dev *dev)
3988	{
3989	unsigned long timeout;
3990	bool hold_esw = true;
3991
3992	/* Wait for any concurrent eswitch mode transition to complete. */
3993	if (!mlx5_esw_hold(dev)) {
3994	timeout = jiffies + msecs_to_jiffies(MLX5_ESW_HOLD_TIMEOUT_MS);
3995	while (!mlx5_esw_hold(dev)) {
3996	if (!time_before(jiffies, timeout)) {
3997	hold_esw = false;
3998	break;
3999	}
4000	msleep(MLX5_ESW_HOLD_RETRY_DELAY_MS);
4001	}
4002	}
4003	if (hold_esw) {
4004	if (mlx5_eswitch_mode(dev) == MLX5_ESWITCH_OFFLOADS)
4005	mlx5_core_reps_aux_devs_remove(dev);
4006	mlx5_esw_release(dev);
4007	}
4008	}
4009
4010	int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode,
4011	struct netlink_ext_ack *extack)
4012	{
4013	u16 cur_mlx5_mode, mlx5_mode = 0;
4014	struct mlx5_eswitch *esw;
4015	int err = 0;
4016
4017	esw = mlx5_devlink_eswitch_get(devlink);
4018	if (IS_ERR(ptr: esw))
4019	return PTR_ERR(ptr: esw);
4020
4021	if (mlx5_fw_reset_in_progress(dev: esw->dev)) {
4022	NL_SET_ERR_MSG_MOD(extack, "Can't change eswitch mode during firmware reset");
4023	return -EBUSY;
4024	}
4025
4026	if (esw_mode_from_devlink(mode, mlx5_mode: &mlx5_mode))
4027	return -EINVAL;
4028
4029	if (mlx5_mode == MLX5_ESWITCH_OFFLOADS && mlx5_get_sd(dev: esw->dev)) {
4030	NL_SET_ERR_MSG_MOD(extack,
4031	"Can't change E-Switch mode to switchdev when multi-PF netdev (Socket Direct) is configured.");
4032	return -EPERM;
4033	}
4034
4035	/* Avoid try_lock, active/inactive mode change is not restricted */
4036	if (mlx5_devlink_switchdev_active_mode_change(esw, devlink_mode: mode))
4037	return 0;
4038
4039	mlx5_lag_disable_change(dev: esw->dev);
4040	err = mlx5_esw_try_lock(esw);
4041	if (err < 0) {
4042	NL_SET_ERR_MSG_MOD(extack, "Can't change mode, E-Switch is busy");
4043	goto enable_lag;
4044	}
4045	cur_mlx5_mode = err;
4046	err = 0;
4047
4048	if (cur_mlx5_mode == mlx5_mode)
4049	goto unlock;
4050
4051	if (esw->offloads.num_block_mode) {
4052	NL_SET_ERR_MSG_MOD(extack,
4053	"Can't change eswitch mode when IPsec SA and/or policies are configured");
4054	err = -EOPNOTSUPP;
4055	goto unlock;
4056	}
4057
4058	esw->eswitch_operation_in_progress = true;
4059	up_write(sem: &esw->mode_lock);
4060
4061	if (mlx5_mode == MLX5_ESWITCH_OFFLOADS &&
4062	!mlx5_devlink_netdev_netns_immutable_set(devlink, immutable: true)) {
4063	NL_SET_ERR_MSG_MOD(extack,
4064	"Can't change E-Switch mode to switchdev when netdev net namespace has diverged from the devlink's.");
4065	err = -EINVAL;
4066	goto skip;
4067	}
4068
4069	if (mlx5_mode == MLX5_ESWITCH_LEGACY)
4070	esw->dev->priv.flags |= MLX5_PRIV_FLAGS_SWITCH_LEGACY;
4071	mlx5_eswitch_disable_locked(esw);
4072	if (mlx5_mode == MLX5_ESWITCH_OFFLOADS) {
4073	if (mlx5_devlink_trap_get_num_active(dev: esw->dev)) {
4074	NL_SET_ERR_MSG_MOD(extack,
4075	"Can't change mode while devlink traps are active");
4076	err = -EOPNOTSUPP;
4077	goto skip;
4078	}
4079	esw->offloads_inactive =
4080	(mode == DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE);
4081	err = esw_offloads_start(esw, extack);
4082	} else if (mlx5_mode == MLX5_ESWITCH_LEGACY) {
4083	err = esw_offloads_stop(esw, extack);
4084	} else {
4085	err = -EINVAL;
4086	}
4087
4088	skip:
4089	if (mlx5_mode == MLX5_ESWITCH_OFFLOADS && err)
4090	mlx5_devlink_netdev_netns_immutable_set(devlink, immutable: false);
4091	down_write(sem: &esw->mode_lock);
4092	esw->eswitch_operation_in_progress = false;
4093	unlock:
4094	mlx5_esw_unlock(esw);
4095	enable_lag:
4096	mlx5_lag_enable_change(dev: esw->dev);
4097	return err;
4098	}
4099
4100	int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode)
4101	{
4102	struct mlx5_eswitch *esw;
4103
4104	esw = mlx5_devlink_eswitch_get(devlink);
4105	if (IS_ERR(ptr: esw))
4106	return PTR_ERR(ptr: esw);
4107
4108	return esw_mode_to_devlink(esw, mode);
4109	}
4110
4111	static int mlx5_esw_vports_inline_set(struct mlx5_eswitch *esw, u8 mlx5_mode,
4112	struct netlink_ext_ack *extack)
4113	{
4114	struct mlx5_core_dev *dev = esw->dev;
4115	struct mlx5_vport *vport;
4116	u16 err_vport_num = 0;
4117	unsigned long i;
4118	int err = 0;
4119
4120	mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
4121	err = mlx5_modify_nic_vport_min_inline(mdev: dev, vport: vport->vport, min_inline: mlx5_mode);
4122	if (err) {
4123	err_vport_num = vport->vport;
4124	NL_SET_ERR_MSG_MOD(extack,
4125	"Failed to set min inline on vport");
4126	goto revert_inline_mode;
4127	}
4128	}
4129	if (mlx5_core_ec_sriov_enabled(dev: esw->dev)) {
4130	mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) {
4131	err = mlx5_modify_nic_vport_min_inline(mdev: dev, vport: vport->vport, min_inline: mlx5_mode);
4132	if (err) {
4133	err_vport_num = vport->vport;
4134	NL_SET_ERR_MSG_MOD(extack,
4135	"Failed to set min inline on vport");
4136	goto revert_ec_vf_inline_mode;
4137	}
4138	}
4139	}
4140	return 0;
4141
4142	revert_ec_vf_inline_mode:
4143	mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) {
4144	if (vport->vport == err_vport_num)
4145	break;
4146	mlx5_modify_nic_vport_min_inline(mdev: dev,
4147	vport: vport->vport,
4148	min_inline: esw->offloads.inline_mode);
4149	}
4150	revert_inline_mode:
4151	mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
4152	if (vport->vport == err_vport_num)
4153	break;
4154	mlx5_modify_nic_vport_min_inline(mdev: dev,
4155	vport: vport->vport,
4156	min_inline: esw->offloads.inline_mode);
4157	}
4158	return err;
4159	}
4160
4161	int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode,
4162	struct netlink_ext_ack *extack)
4163	{
4164	struct mlx5_core_dev *dev = devlink_priv(devlink);
4165	struct mlx5_eswitch *esw;
4166	u8 mlx5_mode;
4167	int err;
4168
4169	esw = mlx5_devlink_eswitch_get(devlink);
4170	if (IS_ERR(ptr: esw))
4171	return PTR_ERR(ptr: esw);
4172
4173	down_write(sem: &esw->mode_lock);
4174
4175	switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
4176	case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
4177	if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) {
4178	err = 0;
4179	goto out;
4180	}
4181
4182	fallthrough;
4183	case MLX5_CAP_INLINE_MODE_L2:
4184	NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set");
4185	err = -EOPNOTSUPP;
4186	goto out;
4187	case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
4188	break;
4189	}
4190
4191	if (atomic64_read(v: &esw->offloads.num_flows) > 0) {
4192	NL_SET_ERR_MSG_MOD(extack,
4193	"Can't set inline mode when flows are configured");
4194	err = -EOPNOTSUPP;
4195	goto out;
4196	}
4197
4198	err = esw_inline_mode_from_devlink(mode, mlx5_mode: &mlx5_mode);
4199	if (err)
4200	goto out;
4201
4202	esw->eswitch_operation_in_progress = true;
4203	up_write(sem: &esw->mode_lock);
4204
4205	err = mlx5_esw_vports_inline_set(esw, mlx5_mode, extack);
4206	if (!err)
4207	esw->offloads.inline_mode = mlx5_mode;
4208
4209	down_write(sem: &esw->mode_lock);
4210	esw->eswitch_operation_in_progress = false;
4211	up_write(sem: &esw->mode_lock);
4212	return 0;
4213
4214	out:
4215	up_write(sem: &esw->mode_lock);
4216	return err;
4217	}
4218
4219	int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode)
4220	{
4221	struct mlx5_eswitch *esw;
4222
4223	esw = mlx5_devlink_eswitch_get(devlink);
4224	if (IS_ERR(ptr: esw))
4225	return PTR_ERR(ptr: esw);
4226
4227	return esw_inline_mode_to_devlink(mlx5_mode: esw->offloads.inline_mode, mode);
4228	}
4229
4230	bool mlx5_eswitch_block_encap(struct mlx5_core_dev *dev, bool from_fdb)
4231	{
4232	struct mlx5_eswitch *esw = dev->priv.eswitch;
4233	enum devlink_eswitch_encap_mode encap;
4234	bool allow_tunnel = false;
4235
4236	if (!mlx5_esw_allowed(esw))
4237	return true;
4238
4239	down_write(sem: &esw->mode_lock);
4240	encap = esw->offloads.encap;
4241	if (esw->mode == MLX5_ESWITCH_LEGACY ||
4242	(encap == DEVLINK_ESWITCH_ENCAP_MODE_NONE && !from_fdb)) {
4243	allow_tunnel = true;
4244	esw->offloads.num_block_encap++;
4245	}
4246	up_write(sem: &esw->mode_lock);
4247
4248	return allow_tunnel;
4249	}
4250
4251	void mlx5_eswitch_unblock_encap(struct mlx5_core_dev *dev)
4252	{
4253	struct mlx5_eswitch *esw = dev->priv.eswitch;
4254
4255	if (!mlx5_esw_allowed(esw))
4256	return;
4257
4258	down_write(sem: &esw->mode_lock);
4259	esw->offloads.num_block_encap--;
4260	up_write(sem: &esw->mode_lock);
4261	}
4262
4263	int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink,
4264	enum devlink_eswitch_encap_mode encap,
4265	struct netlink_ext_ack *extack)
4266	{
4267	struct mlx5_core_dev *dev = devlink_priv(devlink);
4268	struct mlx5_eswitch *esw;
4269	int err = 0;
4270
4271	esw = mlx5_devlink_eswitch_get(devlink);
4272	if (IS_ERR(ptr: esw))
4273	return PTR_ERR(ptr: esw);
4274
4275	down_write(sem: &esw->mode_lock);
4276
4277	if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE &&
4278	(!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) ||
4279	!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) {
4280	err = -EOPNOTSUPP;
4281	goto unlock;
4282	}
4283
4284	if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) {
4285	err = -EOPNOTSUPP;
4286	goto unlock;
4287	}
4288
4289	if (esw->mode == MLX5_ESWITCH_LEGACY) {
4290	esw->offloads.encap = encap;
4291	goto unlock;
4292	}
4293
4294	if (esw->offloads.encap == encap)
4295	goto unlock;
4296
4297	if (atomic64_read(v: &esw->offloads.num_flows) > 0) {
4298	NL_SET_ERR_MSG_MOD(extack,
4299	"Can't set encapsulation when flows are configured");
4300	err = -EOPNOTSUPP;
4301	goto unlock;
4302	}
4303
4304	if (esw->offloads.num_block_encap) {
4305	NL_SET_ERR_MSG_MOD(extack,
4306	"Can't set encapsulation when IPsec SA and/or policies are configured");
4307	err = -EOPNOTSUPP;
4308	goto unlock;
4309	}
4310
4311	esw->eswitch_operation_in_progress = true;
4312	up_write(sem: &esw->mode_lock);
4313
4314	esw_destroy_offloads_fdb_tables(esw);
4315
4316	esw->offloads.encap = encap;
4317
4318	err = esw_create_offloads_fdb_tables(esw);
4319
4320	if (err) {
4321	NL_SET_ERR_MSG_MOD(extack,
4322	"Failed re-creating fast FDB table");
4323	esw->offloads.encap = !encap;
4324	(void)esw_create_offloads_fdb_tables(esw);
4325	}
4326
4327	down_write(sem: &esw->mode_lock);
4328	esw->eswitch_operation_in_progress = false;
4329
4330	unlock:
4331	up_write(sem: &esw->mode_lock);
4332	return err;
4333	}
4334
4335	int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink,
4336	enum devlink_eswitch_encap_mode *encap)
4337	{
4338	struct mlx5_eswitch *esw;
4339
4340	esw = mlx5_devlink_eswitch_get(devlink);
4341	if (IS_ERR(ptr: esw))
4342	return PTR_ERR(ptr: esw);
4343
4344	*encap = esw->offloads.encap;
4345	return 0;
4346	}
4347
4348	static bool
4349	mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num)
4350	{
4351	/* Currently, only ECPF based device has representor for host PF. */
4352	if (vport_num == MLX5_VPORT_PF &&
4353	(!mlx5_core_is_ecpf_esw_manager(dev: esw->dev) ||
4354	!mlx5_esw_host_functions_enabled(dev: esw->dev)))
4355	return false;
4356
4357	if (vport_num == MLX5_VPORT_ECPF &&
4358	!mlx5_ecpf_vport_exists(dev: esw->dev))
4359	return false;
4360
4361	return true;
4362	}
4363
4364	void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw,
4365	const struct mlx5_eswitch_rep_ops *ops,
4366	u8 rep_type)
4367	{
4368	struct mlx5_eswitch_rep_data *rep_data;
4369	struct mlx5_eswitch_rep *rep;
4370	unsigned long i;
4371
4372	esw->offloads.rep_ops[rep_type] = ops;
4373	mlx5_esw_for_each_rep(esw, i, rep) {
4374	if (likely(mlx5_eswitch_vport_has_rep(esw, rep->vport))) {
4375	rep->esw = esw;
4376	rep_data = &rep->rep_data[rep_type];
4377	atomic_set(v: &rep_data->state, i: REP_REGISTERED);
4378	}
4379	}
4380	}
4381	EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps);
4382
4383	void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type)
4384	{
4385	struct mlx5_eswitch_rep *rep;
4386	unsigned long i;
4387
4388	if (esw->mode == MLX5_ESWITCH_OFFLOADS)
4389	__unload_reps_all_vport(esw, rep_type);
4390
4391	mlx5_esw_for_each_rep(esw, i, rep)
4392	atomic_set(v: &rep->rep_data[rep_type].state, i: REP_UNREGISTERED);
4393	}
4394	EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps);
4395
4396	void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type)
4397	{
4398	struct mlx5_eswitch_rep *rep;
4399
4400	rep = mlx5_eswitch_get_rep(esw, vport_num: MLX5_VPORT_UPLINK);
4401	return rep->rep_data[rep_type].priv;
4402	}
4403
4404	void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw,
4405	u16 vport,
4406	u8 rep_type)
4407	{
4408	struct mlx5_eswitch_rep *rep;
4409
4410	rep = mlx5_eswitch_get_rep(esw, vport_num: vport);
4411
4412	if (atomic_read(v: &rep->rep_data[rep_type].state) == REP_LOADED &&
4413	esw->offloads.rep_ops[rep_type]->get_proto_dev)
4414	return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep);
4415	return NULL;
4416	}
4417	EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev);
4418
4419	void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type)
4420	{
4421	return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type);
4422	}
4423	EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev);
4424
4425	struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw,
4426	u16 vport)
4427	{
4428	return mlx5_eswitch_get_rep(esw, vport_num: vport);
4429	}
4430	EXPORT_SYMBOL(mlx5_eswitch_vport_rep);
4431
4432	bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw)
4433	{
4434	return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED);
4435	}
4436	EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled);
4437
4438	bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw)
4439	{
4440	return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA);
4441	}
4442	EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled);
4443
4444	u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw,
4445	u16 vport_num)
4446	{
4447	struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
4448
4449	if (WARN_ON_ONCE(IS_ERR(vport)))
4450	return 0;
4451
4452	return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS);
4453	}
4454	EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match);
4455
4456	int mlx5_esw_vport_vhca_id_map(struct mlx5_eswitch *esw,
4457	struct mlx5_vport *vport)
4458	{
4459	u16 *old_entry, *vhca_map_entry, vhca_id;
4460
4461	if (WARN_ONCE(MLX5_VPORT_INVAL_VHCA_ID(vport),
4462	"vport %d vhca_id is not set", vport->vport)) {
4463	int err;
4464
4465	err = mlx5_vport_get_vhca_id(dev: vport->dev, vport: vport->vport,
4466	vhca_id: &vhca_id);
4467	if (err)
4468	return err;
4469	vport->vhca_id = vhca_id;
4470	}
4471
4472	vhca_id = vport->vhca_id;
4473	vhca_map_entry = kmalloc(sizeof(*vhca_map_entry), GFP_KERNEL);
4474	if (!vhca_map_entry)
4475	return -ENOMEM;
4476
4477	*vhca_map_entry = vport->vport;
4478	old_entry = xa_store(&esw->offloads.vhca_map, index: vhca_id, entry: vhca_map_entry, GFP_KERNEL);
4479	if (xa_is_err(entry: old_entry)) {
4480	kfree(objp: vhca_map_entry);
4481	return xa_err(entry: old_entry);
4482	}
4483	kfree(objp: old_entry);
4484	return 0;
4485	}
4486
4487	void mlx5_esw_vport_vhca_id_unmap(struct mlx5_eswitch *esw,
4488	struct mlx5_vport *vport)
4489	{
4490	u16 *vhca_map_entry;
4491
4492	vhca_map_entry = xa_erase(&esw->offloads.vhca_map, index: vport->vhca_id);
4493	kfree(objp: vhca_map_entry);
4494	}
4495
4496	int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num)
4497	{
4498	u16 *res = xa_load(&esw->offloads.vhca_map, index: vhca_id);
4499
4500	if (!res)
4501	return -ENOENT;
4502
4503	*vport_num = *res;
4504	return 0;
4505	}
4506
4507	u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw,
4508	u16 vport_num)
4509	{
4510	struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
4511
4512	if (WARN_ON_ONCE(IS_ERR(vport)))
4513	return 0;
4514
4515	return vport->metadata;
4516	}
4517	EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set);
4518
4519	int mlx5_devlink_port_fn_hw_addr_get(struct devlink_port *port,
4520	u8 *hw_addr, int *hw_addr_len,
4521	struct netlink_ext_ack *extack)
4522	{
4523	struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(devlink: port->devlink);
4524	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(dl_port: port);
4525
4526	mutex_lock(&esw->state_lock);
4527
4528	mlx5_query_nic_vport_mac_address(mdev: esw->dev, vport: vport->vport, other: true,
4529	addr: vport->info.mac);
4530	ether_addr_copy(dst: hw_addr, src: vport->info.mac);
4531	*hw_addr_len = ETH_ALEN;
4532	mutex_unlock(lock: &esw->state_lock);
4533	return 0;
4534	}
4535
4536	int mlx5_devlink_port_fn_hw_addr_set(struct devlink_port *port,
4537	const u8 *hw_addr, int hw_addr_len,
4538	struct netlink_ext_ack *extack)
4539	{
4540	struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(devlink: port->devlink);
4541	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(dl_port: port);
4542
4543	return mlx5_eswitch_set_vport_mac(esw, vport: vport->vport, mac: hw_addr);
4544	}
4545
4546	int mlx5_devlink_port_fn_migratable_get(struct devlink_port *port, bool *is_enabled,
4547	struct netlink_ext_ack *extack)
4548	{
4549	struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(devlink: port->devlink);
4550	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(dl_port: port);
4551
4552	if (!MLX5_CAP_GEN(esw->dev, migration)) {
4553	NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration");
4554	return -EOPNOTSUPP;
4555	}
4556
4557	if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4558	NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4559	return -EOPNOTSUPP;
4560	}
4561
4562	mutex_lock(&esw->state_lock);
4563	*is_enabled = vport->info.mig_enabled;
4564	mutex_unlock(lock: &esw->state_lock);
4565	return 0;
4566	}
4567
4568	int mlx5_devlink_port_fn_migratable_set(struct devlink_port *port, bool enable,
4569	struct netlink_ext_ack *extack)
4570	{
4571	struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(devlink: port->devlink);
4572	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(dl_port: port);
4573	int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
4574	void *query_ctx;
4575	void *hca_caps;
4576	int err;
4577
4578	if (!MLX5_CAP_GEN(esw->dev, migration)) {
4579	NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration");
4580	return -EOPNOTSUPP;
4581	}
4582
4583	if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4584	NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4585	return -EOPNOTSUPP;
4586	}
4587
4588	mutex_lock(&esw->state_lock);
4589
4590	if (vport->info.mig_enabled == enable) {
4591	err = 0;
4592	goto out;
4593	}
4594
4595	query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
4596	if (!query_ctx) {
4597	err = -ENOMEM;
4598	goto out;
4599	}
4600
4601	err = mlx5_vport_get_other_func_cap(dev: esw->dev, vport: vport->vport, out: query_ctx,
4602	opmod: MLX5_CAP_GENERAL_2);
4603	if (err) {
4604	NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps");
4605	goto out_free;
4606	}
4607
4608	hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
4609	MLX5_SET(cmd_hca_cap_2, hca_caps, migratable, enable);
4610
4611	err = mlx5_vport_set_other_func_cap(dev: esw->dev, hca_cap: hca_caps, vport: vport->vport,
4612	opmod: MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE2);
4613	if (err) {
4614	NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA migratable cap");
4615	goto out_free;
4616	}
4617
4618	vport->info.mig_enabled = enable;
4619
4620	out_free:
4621	kfree(objp: query_ctx);
4622	out:
4623	mutex_unlock(lock: &esw->state_lock);
4624	return err;
4625	}
4626
4627	int mlx5_devlink_port_fn_roce_get(struct devlink_port *port, bool *is_enabled,
4628	struct netlink_ext_ack *extack)
4629	{
4630	struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(devlink: port->devlink);
4631	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(dl_port: port);
4632
4633	if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4634	NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4635	return -EOPNOTSUPP;
4636	}
4637
4638	mutex_lock(&esw->state_lock);
4639	*is_enabled = vport->info.roce_enabled;
4640	mutex_unlock(lock: &esw->state_lock);
4641	return 0;
4642	}
4643
4644	int mlx5_devlink_port_fn_roce_set(struct devlink_port *port, bool enable,
4645	struct netlink_ext_ack *extack)
4646	{
4647	struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(devlink: port->devlink);
4648	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(dl_port: port);
4649	int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
4650	u16 vport_num = vport->vport;
4651	void *query_ctx;
4652	void *hca_caps;
4653	int err;
4654
4655	if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4656	NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4657	return -EOPNOTSUPP;
4658	}
4659
4660	mutex_lock(&esw->state_lock);
4661
4662	if (vport->info.roce_enabled == enable) {
4663	err = 0;
4664	goto out;
4665	}
4666
4667	query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
4668	if (!query_ctx) {
4669	err = -ENOMEM;
4670	goto out;
4671	}
4672
4673	err = mlx5_vport_get_other_func_cap(dev: esw->dev, vport: vport_num, out: query_ctx,
4674	opmod: MLX5_CAP_GENERAL);
4675	if (err) {
4676	NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps");
4677	goto out_free;
4678	}
4679
4680	hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
4681	MLX5_SET(cmd_hca_cap, hca_caps, roce, enable);
4682
4683	err = mlx5_vport_set_other_func_cap(dev: esw->dev, hca_cap: hca_caps, vport: vport_num,
4684	opmod: MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE);
4685	if (err) {
4686	NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA roce cap");
4687	goto out_free;
4688	}
4689
4690	vport->info.roce_enabled = enable;
4691
4692	out_free:
4693	kfree(objp: query_ctx);
4694	out:
4695	mutex_unlock(lock: &esw->state_lock);
4696	return err;
4697	}
4698
4699	int
4700	mlx5_eswitch_restore_ipsec_rule(struct mlx5_eswitch *esw, struct mlx5_flow_handle *rule,
4701	struct mlx5_esw_flow_attr *esw_attr, int attr_idx)
4702	{
4703	struct mlx5_flow_destination new_dest = {};
4704	struct mlx5_flow_destination old_dest = {};
4705
4706	if (!esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx))
4707	return 0;
4708
4709	esw_setup_dest_fwd_ipsec(dest: &old_dest, NULL, esw, esw_attr, attr_idx, dest_idx: 0, pkt_reformat: false);
4710	esw_setup_dest_fwd_vport(dest: &new_dest, NULL, esw, esw_attr, attr_idx, dest_idx: 0, pkt_reformat: false);
4711
4712	return mlx5_modify_rule_destination(handler: rule, new_dest: &new_dest, old_dest: &old_dest);
4713	}
4714
4715	#ifdef CONFIG_XFRM_OFFLOAD
4716	int mlx5_devlink_port_fn_ipsec_crypto_get(struct devlink_port *port, bool *is_enabled,
4717	struct netlink_ext_ack *extack)
4718	{
4719	struct mlx5_eswitch *esw;
4720	struct mlx5_vport *vport;
4721	int err = 0;
4722
4723	esw = mlx5_devlink_eswitch_get(devlink: port->devlink);
4724	if (IS_ERR(ptr: esw))
4725	return PTR_ERR(ptr: esw);
4726
4727	if (!mlx5_esw_ipsec_vf_offload_supported(dev: esw->dev)) {
4728	NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPSec crypto");
4729	return -EOPNOTSUPP;
4730	}
4731
4732	vport = mlx5_devlink_port_vport_get(dl_port: port);
4733
4734	mutex_lock(&esw->state_lock);
4735	if (!vport->enabled) {
4736	err = -EOPNOTSUPP;
4737	goto unlock;
4738	}
4739
4740	*is_enabled = vport->info.ipsec_crypto_enabled;
4741	unlock:
4742	mutex_unlock(lock: &esw->state_lock);
4743	return err;
4744	}
4745
4746	int mlx5_devlink_port_fn_ipsec_crypto_set(struct devlink_port *port, bool enable,
4747	struct netlink_ext_ack *extack)
4748	{
4749	struct mlx5_eswitch *esw;
4750	struct mlx5_vport *vport;
4751	u16 vport_num;
4752	int err;
4753
4754	esw = mlx5_devlink_eswitch_get(devlink: port->devlink);
4755	if (IS_ERR(ptr: esw))
4756	return PTR_ERR(ptr: esw);
4757
4758	vport_num = mlx5_esw_devlink_port_index_to_vport_num(dl_port_index: port->index);
4759	err = mlx5_esw_ipsec_vf_crypto_offload_supported(dev: esw->dev, vport_num);
4760	if (err) {
4761	NL_SET_ERR_MSG_MOD(extack,
4762	"Device doesn't support IPsec crypto");
4763	return err;
4764	}
4765
4766	vport = mlx5_devlink_port_vport_get(dl_port: port);
4767
4768	mutex_lock(&esw->state_lock);
4769	if (!vport->enabled) {
4770	err = -EOPNOTSUPP;
4771	NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled");
4772	goto unlock;
4773	}
4774
4775	if (vport->info.ipsec_crypto_enabled == enable)
4776	goto unlock;
4777
4778	if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) {
4779	err = -EBUSY;
4780	goto unlock;
4781	}
4782
4783	err = mlx5_esw_ipsec_vf_crypto_offload_set(esw, vport, enable);
4784	if (err) {
4785	NL_SET_ERR_MSG_MOD(extack, "Failed to set IPsec crypto");
4786	goto unlock;
4787	}
4788
4789	vport->info.ipsec_crypto_enabled = enable;
4790	if (enable)
4791	esw->enabled_ipsec_vf_count++;
4792	else
4793	esw->enabled_ipsec_vf_count--;
4794	unlock:
4795	mutex_unlock(lock: &esw->state_lock);
4796	return err;
4797	}
4798
4799	int mlx5_devlink_port_fn_ipsec_packet_get(struct devlink_port *port, bool *is_enabled,
4800	struct netlink_ext_ack *extack)
4801	{
4802	struct mlx5_eswitch *esw;
4803	struct mlx5_vport *vport;
4804	int err = 0;
4805
4806	esw = mlx5_devlink_eswitch_get(devlink: port->devlink);
4807	if (IS_ERR(ptr: esw))
4808	return PTR_ERR(ptr: esw);
4809
4810	if (!mlx5_esw_ipsec_vf_offload_supported(dev: esw->dev)) {
4811	NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPsec packet");
4812	return -EOPNOTSUPP;
4813	}
4814
4815	vport = mlx5_devlink_port_vport_get(dl_port: port);
4816
4817	mutex_lock(&esw->state_lock);
4818	if (!vport->enabled) {
4819	err = -EOPNOTSUPP;
4820	goto unlock;
4821	}
4822
4823	*is_enabled = vport->info.ipsec_packet_enabled;
4824	unlock:
4825	mutex_unlock(lock: &esw->state_lock);
4826	return err;
4827	}
4828
4829	int mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port *port,
4830	bool enable,
4831	struct netlink_ext_ack *extack)
4832	{
4833	struct mlx5_eswitch *esw;
4834	struct mlx5_vport *vport;
4835	u16 vport_num;
4836	int err;
4837
4838	esw = mlx5_devlink_eswitch_get(devlink: port->devlink);
4839	if (IS_ERR(ptr: esw))
4840	return PTR_ERR(ptr: esw);
4841
4842	vport_num = mlx5_esw_devlink_port_index_to_vport_num(dl_port_index: port->index);
4843	err = mlx5_esw_ipsec_vf_packet_offload_supported(dev: esw->dev, vport_num);
4844	if (err) {
4845	NL_SET_ERR_MSG_MOD(extack,
4846	"Device doesn't support IPsec packet mode");
4847	return err;
4848	}
4849
4850	vport = mlx5_devlink_port_vport_get(dl_port: port);
4851	mutex_lock(&esw->state_lock);
4852	if (!vport->enabled) {
4853	err = -EOPNOTSUPP;
4854	NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled");
4855	goto unlock;
4856	}
4857
4858	if (vport->info.ipsec_packet_enabled == enable)
4859	goto unlock;
4860
4861	if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) {
4862	err = -EBUSY;
4863	goto unlock;
4864	}
4865
4866	err = mlx5_esw_ipsec_vf_packet_offload_set(esw, vport, enable);
4867	if (err) {
4868	NL_SET_ERR_MSG_MOD(extack,
4869	"Failed to set IPsec packet mode");
4870	goto unlock;
4871	}
4872
4873	vport->info.ipsec_packet_enabled = enable;
4874	if (enable)
4875	esw->enabled_ipsec_vf_count++;
4876	else
4877	esw->enabled_ipsec_vf_count--;
4878	unlock:
4879	mutex_unlock(lock: &esw->state_lock);
4880	return err;
4881	}
4882	#endif /* CONFIG_XFRM_OFFLOAD */
4883
4884	int
4885	mlx5_devlink_port_fn_max_io_eqs_get(struct devlink_port *port, u32 *max_io_eqs,
4886	struct netlink_ext_ack *extack)
4887	{
4888	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(dl_port: port);
4889	int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
4890	u16 vport_num = vport->vport;
4891	struct mlx5_eswitch *esw;
4892	void *query_ctx;
4893	void *hca_caps;
4894	u32 max_eqs;
4895	int err;
4896
4897	esw = mlx5_devlink_eswitch_nocheck_get(devlink: port->devlink);
4898	if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4899	NL_SET_ERR_MSG_MOD(extack,
4900	"Device doesn't support VHCA management");
4901	return -EOPNOTSUPP;
4902	}
4903
4904	if (!MLX5_CAP_GEN_2(esw->dev, max_num_eqs_24b)) {
4905	NL_SET_ERR_MSG_MOD(extack,
4906	"Device doesn't support getting the max number of EQs");
4907	return -EOPNOTSUPP;
4908	}
4909
4910	query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
4911	if (!query_ctx)
4912	return -ENOMEM;
4913
4914	mutex_lock(&esw->state_lock);
4915	err = mlx5_vport_get_other_func_cap(dev: esw->dev, vport: vport_num, out: query_ctx,
4916	opmod: MLX5_CAP_GENERAL_2);
4917	if (err) {
4918	NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps");
4919	goto out;
4920	}
4921
4922	hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
4923	max_eqs = MLX5_GET(cmd_hca_cap_2, hca_caps, max_num_eqs_24b);
4924	if (max_eqs < MLX5_ESW_MAX_CTRL_EQS)
4925	*max_io_eqs = 0;
4926	else
4927	*max_io_eqs = max_eqs - MLX5_ESW_MAX_CTRL_EQS;
4928	out:
4929	mutex_unlock(lock: &esw->state_lock);
4930	kfree(objp: query_ctx);
4931	return err;
4932	}
4933
4934	int
4935	mlx5_devlink_port_fn_max_io_eqs_set(struct devlink_port *port, u32 max_io_eqs,
4936	struct netlink_ext_ack *extack)
4937	{
4938	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(dl_port: port);
4939	int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
4940	u16 vport_num = vport->vport;
4941	struct mlx5_eswitch *esw;
4942	void *query_ctx;
4943	void *hca_caps;
4944	u16 max_eqs;
4945	int err;
4946
4947	esw = mlx5_devlink_eswitch_nocheck_get(devlink: port->devlink);
4948	if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4949	NL_SET_ERR_MSG_MOD(extack,
4950	"Device doesn't support VHCA management");
4951	return -EOPNOTSUPP;
4952	}
4953
4954	if (!MLX5_CAP_GEN_2(esw->dev, max_num_eqs_24b)) {
4955	NL_SET_ERR_MSG_MOD(extack,
4956	"Device doesn't support changing the max number of EQs");
4957	return -EOPNOTSUPP;
4958	}
4959
4960	if (check_add_overflow(max_io_eqs, MLX5_ESW_MAX_CTRL_EQS, &max_eqs)) {
4961	NL_SET_ERR_MSG_MOD(extack, "Supplied value out of range");
4962	return -EINVAL;
4963	}
4964
4965	query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
4966	if (!query_ctx)
4967	return -ENOMEM;
4968
4969	mutex_lock(&esw->state_lock);
4970	err = mlx5_vport_get_other_func_cap(dev: esw->dev, vport: vport_num, out: query_ctx,
4971	opmod: MLX5_CAP_GENERAL_2);
4972	if (err) {
4973	NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps");
4974	goto out;
4975	}
4976
4977	hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
4978	MLX5_SET(cmd_hca_cap_2, hca_caps, max_num_eqs_24b, max_eqs);
4979
4980	if (mlx5_esw_is_sf_vport(esw, vport_num))
4981	MLX5_SET(cmd_hca_cap_2, hca_caps, sf_eq_usage, 1);
4982
4983	err = mlx5_vport_set_other_func_cap(dev: esw->dev, hca_cap: hca_caps, vport: vport_num,
4984	opmod: MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE2);
4985	if (err)
4986	NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA caps");
4987	vport->max_eqs_set = true;
4988	out:
4989	mutex_unlock(lock: &esw->state_lock);
4990	kfree(objp: query_ctx);
4991	return err;
4992	}
4993
4994	int
4995	mlx5_devlink_port_fn_max_io_eqs_set_sf_default(struct devlink_port *port,
4996	struct netlink_ext_ack *extack)
4997	{
4998	return mlx5_devlink_port_fn_max_io_eqs_set(port,
4999	MLX5_ESW_DEFAULT_SF_COMP_EQS,
5000	extack);
5001	}
5002