// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2024 NVIDIA Corporation & Affiliates */

#include "internal.h"

/* We chain submatchers by applying three rules on a subrule: modify header (to
 * set register C6), jump to table (to the next submatcher) and the mandatory
 * last rule.
 */
#define HWS_NUM_CHAIN_ACTIONS 3

static const struct rhashtable_params hws_rules_hash_params = {
	.key_len = sizeof_field(struct mlx5hws_bwc_complex_subrule_data,
				match_tag),
	.key_offset =
		offsetof(struct mlx5hws_bwc_complex_subrule_data, match_tag),
	.head_offset =
		offsetof(struct mlx5hws_bwc_complex_subrule_data, hash_node),
	.automatic_shrinking = true, .min_size = 1,
};

static bool
hws_match_params_exceeds_definer(struct mlx5hws_context *ctx,
				 u8 match_criteria_enable,
				 struct mlx5hws_match_parameters *mask,
				 bool allow_jumbo)
{
	struct mlx5hws_definer match_layout = {0};
	struct mlx5hws_match_template *mt;
	bool is_complex = false;
	int ret;

	if (!match_criteria_enable)
		return false; /* empty matcher */

	mt = mlx5hws_match_template_create(ctx,
					   mask->match_buf,
					   mask->match_sz,
					   match_criteria_enable);
	if (!mt) {
		mlx5hws_err(ctx, "Complex matcher: failed creating match template\n");
		return false;
	}

	ret = mlx5hws_definer_calc_layout(ctx, mt, &match_layout, allow_jumbo);
	if (ret) {
		/* The only case that we're interested in is E2BIG,
		 * which means that the match parameters need to be
		 * split into complex martcher.
		 * For all other cases (good or bad) - just return true
		 * and let the usual match creation path handle it,
		 * both for good and bad flows.
		 */
		if (ret == -E2BIG) {
			is_complex = true;
			mlx5hws_dbg(ctx, "Matcher definer layout: need complex matcher\n");
		} else {
			mlx5hws_err(ctx, "Failed to calculate matcher definer layout\n");
		}
	} else {
		kfree(mt->fc);
	}

	mlx5hws_match_template_destroy(mt);

	return is_complex;
}

bool mlx5hws_bwc_match_params_is_complex(struct mlx5hws_context *ctx,
					 u8 match_criteria_enable,
					 struct mlx5hws_match_parameters *mask)
{
	return hws_match_params_exceeds_definer(ctx, match_criteria_enable,
						mask, true);
}

static int
hws_get_last_set_dword_idx(const struct mlx5hws_match_parameters *mask)
{
	int i;

	for (i = mask->match_sz / 4 - 1; i >= 0; i--)
		if (mask->match_buf[i])
			return i;

	return -1;
}

static bool hws_match_mask_is_empty(const struct mlx5hws_match_parameters *mask)
{
	return hws_get_last_set_dword_idx(mask) == -1;
}

static bool hws_dword_is_inner_ipaddr_off(int dword_off)
{
	/* IPv4 and IPv6 addresses share the same entry via a union, and the
	 * source and dest addresses are contiguous in the fte_match_param. So
	 * we need to check 8 words.
	 */
	static const int inner_ip_dword_off =
		__mlx5_dw_off(fte_match_param, inner_headers.src_ipv4_src_ipv6);

	return dword_off >= inner_ip_dword_off &&
	       dword_off < inner_ip_dword_off + 8;
}

static bool hws_dword_is_outer_ipaddr_off(int dword_off)
{
	static const int outer_ip_dword_off =
		__mlx5_dw_off(fte_match_param, outer_headers.src_ipv4_src_ipv6);

	return dword_off >= outer_ip_dword_off &&
	       dword_off < outer_ip_dword_off + 8;
}

static void hws_add_dword_to_mask(struct mlx5hws_match_parameters *mask,
				  const struct mlx5hws_match_parameters *orig,
				  int dword_idx, bool *added_inner_ipv,
				  bool *added_outer_ipv)
{
	mask->match_buf[dword_idx] |= orig->match_buf[dword_idx];

	*added_inner_ipv = false;
	*added_outer_ipv = false;

	/* Any IP address fragment must be accompanied by a match on IP version.
	 * Use the `added_ipv` variables to keep track if we added IP versions
	 * specifically for this dword, so that we can roll them back if the
	 * match params become too large to fit into a definer.
	 */
	if (hws_dword_is_inner_ipaddr_off(dword_idx) &&
	    !MLX5_GET(fte_match_param, mask->match_buf,
		      inner_headers.ip_version)) {
		MLX5_SET(fte_match_param, mask->match_buf,
			 inner_headers.ip_version, 0xf);
		*added_inner_ipv = true;
	}
	if (hws_dword_is_outer_ipaddr_off(dword_idx) &&
	    !MLX5_GET(fte_match_param, mask->match_buf,
		      outer_headers.ip_version)) {
		MLX5_SET(fte_match_param, mask->match_buf,
			 outer_headers.ip_version, 0xf);
		*added_outer_ipv = true;
	}
}

static void hws_remove_dword_from_mask(struct mlx5hws_match_parameters *mask,
				       int dword_idx, bool added_inner_ipv,
				       bool added_outer_ipv)
{
	mask->match_buf[dword_idx] = 0;
	if (added_inner_ipv)
		MLX5_SET(fte_match_param, mask->match_buf,
			 inner_headers.ip_version, 0);
	if (added_outer_ipv)
		MLX5_SET(fte_match_param, mask->match_buf,
			 outer_headers.ip_version, 0);
}

/* Avoid leaving a single lower dword in `mask` if there are others present in
 * `orig`. Splitting IPv6 addresses like this causes them to be interpreted as
 * IPv4.
 */
static void hws_avoid_ipv6_split_of(struct mlx5hws_match_parameters *orig,
				    struct mlx5hws_match_parameters *mask,
				    int off)
{
	/* Masks are allocated to a full fte_match_param, but it can't hurt to
	 * double check.
	 */
	if (orig->match_sz <= off + 3 || mask->match_sz <= off + 3)
		return;

	/* Lower dword is not set, nothing to do. */
	if (!mask->match_buf[off + 3])
		return;

	/* Higher dwords also present in `mask`, no ambiguity. */
	if (mask->match_buf[off] || mask->match_buf[off + 1] ||
	    mask->match_buf[off + 2])
		return;

	/* There are no higher dwords in `orig`, i.e. we match on IPv4. */
	if (!orig->match_buf[off] && !orig->match_buf[off + 1] &&
	    !orig->match_buf[off + 2])
		return;

	/* Put the lower dword back in `orig`. It is always safe to do this, the
	 * dword will just be picked up in the next submask.
	 */
	orig->match_buf[off + 3] = mask->match_buf[off + 3];
	mask->match_buf[off + 3] = 0;
}

static void hws_avoid_ipv6_split(struct mlx5hws_match_parameters *orig,
				 struct mlx5hws_match_parameters *mask)
{
	hws_avoid_ipv6_split_of(orig, mask,
				__mlx5_dw_off(fte_match_param,
					      outer_headers.src_ipv4_src_ipv6));
	hws_avoid_ipv6_split_of(orig, mask,
				__mlx5_dw_off(fte_match_param,
					      outer_headers.dst_ipv4_dst_ipv6));
	hws_avoid_ipv6_split_of(orig, mask,
				__mlx5_dw_off(fte_match_param,
					      inner_headers.src_ipv4_src_ipv6));
	hws_avoid_ipv6_split_of(orig, mask,
				__mlx5_dw_off(fte_match_param,
					      inner_headers.dst_ipv4_dst_ipv6));
}

/* Build a subset of the `orig` match parameters into `mask`. This subset is
 * guaranteed to fit in a single definer an as such is a candidate for being a
 * part of a complex matcher. Upon successful execution, the match params that
 * go into `mask` are cleared from `orig`.
 */
static int hws_get_simple_params(struct mlx5hws_context *ctx, u8 match_criteria,
				 struct mlx5hws_match_parameters *orig,
				 struct mlx5hws_match_parameters *mask)
{
	bool added_inner_ipv, added_outer_ipv;
	int dword_idx;
	u32 *backup;
	int ret;

	dword_idx = hws_get_last_set_dword_idx(orig);
	/* Nothing to do, we consumed all of the match params before. */
	if (dword_idx == -1)
		return 0;

	backup = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
	if (!backup)
		return -ENOMEM;

	while (1) {
		dword_idx = hws_get_last_set_dword_idx(orig);
		/* Nothing to do, we consumed all of the original match params
		 * into this subset, which still fits into a single matcher.
		 */
		if (dword_idx == -1) {
			ret = 0;
			goto free_backup;
		}

		memcpy(backup, mask->match_buf, mask->match_sz);

		/* Try to add this dword to the current subset. */
		hws_add_dword_to_mask(mask, orig, dword_idx, &added_inner_ipv,
				      &added_outer_ipv);

		if (hws_match_params_exceeds_definer(ctx, match_criteria, mask,
						     false)) {
			/* We just added a match param that makes the definer
			 * too large. Revert and return what we had before.
			 * Note that we can't just zero out the affected fields,
			 * because it's possible that the dword we're looking at
			 * wasn't zero before (e.g. it included auto-added
			 * matches in IP version. This is why we employ the
			 * rather cumbersome memcpy for backing up.
			 */
			memcpy(mask->match_buf, backup, mask->match_sz);
			/* Possible future improvement: We can't add any more
			 * dwords, but it may be possible to squeeze in
			 * individual bytes, as definers have special slots for
			 * those.
			 *
			 * For now, keep the code simple. This results in an
			 * extra submatcher in some cases, but it's good enough.
			 */
			ret = 0;
			break;
		}

		/* The current subset of match params still fits in a single
		 * definer. Remove the dword from the original mask.
		 *
		 * Also remove any explicit match on IP version if we just
		 * included one here. We will still automatically add it to
		 * accompany any IP address fragment, but do not need to
		 * consider it by itself.
		 */
		hws_remove_dword_from_mask(orig, dword_idx, added_inner_ipv,
					   added_outer_ipv);
	}

	/* Make sure we have not picked up a single lower dword of an IPv6
	 * address, as the firmware will erroneously treat it as an IPv4
	 * address.
	 */
	hws_avoid_ipv6_split(orig, mask);

free_backup:
	kfree(backup);

	return ret;
}

static int
hws_bwc_matcher_split_mask(struct mlx5hws_context *ctx, u8 match_criteria,
			   const struct mlx5hws_match_parameters *mask,
			   struct mlx5hws_match_parameters *submasks,
			   int *num_submasks)
{
	struct mlx5hws_match_parameters mask_copy;
	int ret, i = 0;

	mask_copy.match_sz = MLX5_ST_SZ_BYTES(fte_match_param);
	mask_copy.match_buf = kzalloc(mask_copy.match_sz, GFP_KERNEL);
	if (!mask_copy.match_buf)
		return -ENOMEM;

	memcpy(mask_copy.match_buf, mask->match_buf, mask->match_sz);

	while (!hws_match_mask_is_empty(&mask_copy)) {
		if (i >= MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS) {
			mlx5hws_err(ctx,
				    "Complex matcher: mask too large for %d matchers\n",
				    MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS);
			ret = -E2BIG;
			goto free_copy;
		}
		/* All but the first matcher need to match on register C6 to
		 * connect pieces of the complex rule together.
		 */
		if (i > 0) {
			MLX5_SET(fte_match_param, submasks[i].match_buf,
				 misc_parameters_2.metadata_reg_c_6, -1);
			match_criteria |= MLX5HWS_DEFINER_MATCH_CRITERIA_MISC2;
		}
		ret = hws_get_simple_params(ctx, match_criteria, &mask_copy,
					    &submasks[i]);
		if (ret < 0)
			goto free_copy;
		i++;
	}

	*num_submasks = i;
	ret = 0;

free_copy:
	kfree(mask_copy.match_buf);

	return ret;
}

static struct mlx5hws_table *
hws_isolated_table_create(const struct mlx5hws_bwc_matcher *cmatcher)
{
	struct mlx5hws_bwc_complex_submatcher *first_subm;
	struct mlx5hws_cmd_ft_modify_attr ft_attr = {0};
	struct mlx5hws_table_attr tbl_attr = {0};
	struct mlx5hws_table *orig_tbl;
	struct mlx5hws_context *ctx;
	struct mlx5hws_table *tbl;
	int ret;

	first_subm = &cmatcher->complex->submatchers[0];
	orig_tbl = first_subm->tbl;
	ctx = orig_tbl->ctx;

	tbl_attr.type = orig_tbl->type;
	tbl_attr.level = orig_tbl->level;
	tbl = mlx5hws_table_create(ctx, &tbl_attr);
	if (!tbl)
		return ERR_PTR(-EINVAL);

	/* Set the default miss of the isolated table to point
	 * to the end anchor of the original matcher.
	 */
	mlx5hws_cmd_set_attr_connect_miss_tbl(ctx, tbl->fw_ft_type,
					      tbl->type, &ft_attr);
	ft_attr.table_miss_id = first_subm->bwc_matcher->matcher->end_ft_id;

	ret = mlx5hws_cmd_flow_table_modify(ctx->mdev, &ft_attr, tbl->ft_id);
	if (ret) {
		mlx5hws_err(ctx, "Complex matcher: failed to set isolated tbl default miss\n");
		goto destroy_tbl;
	}

	return tbl;

destroy_tbl:
	mlx5hws_table_destroy(tbl);

	return ERR_PTR(ret);
}

static int hws_submatcher_init_first(struct mlx5hws_bwc_matcher *cmatcher,
				     struct mlx5hws_table *table, u32 priority,
				     u8 match_criteria,
				     struct mlx5hws_match_parameters *mask)
{
	enum mlx5hws_action_type action_types[HWS_NUM_CHAIN_ACTIONS];
	struct mlx5hws_bwc_complex_submatcher *subm;
	int ret;

	subm = &cmatcher->complex->submatchers[0];

	/* The first submatcher lives in the original table and does not have an
	 * associated jump to table action. It also points to the outer complex
	 * matcher.
	 */
	subm->tbl = table;
	subm->action_tbl = NULL;
	subm->bwc_matcher = cmatcher;

	action_types[0] = MLX5HWS_ACTION_TYP_MODIFY_HDR;
	action_types[1] = MLX5HWS_ACTION_TYP_TBL;
	action_types[2] = MLX5HWS_ACTION_TYP_LAST;

	ret = mlx5hws_bwc_matcher_create_simple(subm->bwc_matcher, subm->tbl,
						priority, match_criteria, mask,
						action_types);
	if (ret)
		return ret;

	subm->bwc_matcher->matcher_type = MLX5HWS_BWC_MATCHER_COMPLEX_FIRST;

	ret = rhashtable_init(&subm->rules_hash, &hws_rules_hash_params);
	if (ret)
		goto destroy_matcher;
	mutex_init(&subm->hash_lock);
	ida_init(&subm->chain_ida);

	return 0;

destroy_matcher:
	mlx5hws_bwc_matcher_destroy_simple(subm->bwc_matcher);

	return ret;
}

static int hws_submatcher_init(struct mlx5hws_bwc_matcher *cmatcher, int idx,
			       struct mlx5hws_table *table, u32 priority,
			       u8 match_criteria,
			       struct mlx5hws_match_parameters *mask)
{
	enum mlx5hws_action_type action_types[HWS_NUM_CHAIN_ACTIONS];
	struct mlx5hws_bwc_complex_submatcher *subm;
	bool is_last;
	int ret;

	if (!idx)
		return hws_submatcher_init_first(cmatcher, table, priority,
						 match_criteria, mask);

	subm = &cmatcher->complex->submatchers[idx];
	is_last = idx == cmatcher->complex->num_submatchers - 1;

	subm->tbl = hws_isolated_table_create(cmatcher);
	if (IS_ERR(subm->tbl))
		return PTR_ERR(subm->tbl);

	subm->action_tbl =
		mlx5hws_action_create_dest_table(subm->tbl->ctx, subm->tbl,
						 MLX5HWS_ACTION_FLAG_HWS_FDB);
	if (!subm->action_tbl) {
		ret = -EINVAL;
		goto destroy_tbl;
	}

	subm->bwc_matcher = kzalloc(sizeof(*subm->bwc_matcher), GFP_KERNEL);
	if (!subm->bwc_matcher) {
		ret = -ENOMEM;
		goto destroy_action;
	}

	/* Every matcher other than the first also matches of register C6 to
	 * bind subrules together in the complex rule using the chain ids.
	 */
	match_criteria |= MLX5HWS_DEFINER_MATCH_CRITERIA_MISC2;

	action_types[0] = MLX5HWS_ACTION_TYP_MODIFY_HDR;
	action_types[1] = MLX5HWS_ACTION_TYP_TBL;
	action_types[2] = MLX5HWS_ACTION_TYP_LAST;

	/* Every matcher other than the last sets register C6 and jumps to the
	 * next submatcher's table. The final submatcher will use the
	 * user-supplied actions and will attach an action template at rule
	 * insertion time.
	 */
	ret = mlx5hws_bwc_matcher_create_simple(subm->bwc_matcher, subm->tbl,
						priority, match_criteria, mask,
						is_last ? NULL : action_types);
	if (ret)
		goto free_matcher;

	subm->bwc_matcher->matcher_type =
		MLX5HWS_BWC_MATCHER_COMPLEX_SUBMATCHER;

	ret = rhashtable_init(&subm->rules_hash, &hws_rules_hash_params);
	if (ret)
		goto destroy_matcher;
	mutex_init(&subm->hash_lock);
	ida_init(&subm->chain_ida);

	return 0;

destroy_matcher:
	mlx5hws_bwc_matcher_destroy_simple(subm->bwc_matcher);
free_matcher:
	kfree(subm->bwc_matcher);
destroy_action:
	mlx5hws_action_destroy(subm->action_tbl);
destroy_tbl:
	mlx5hws_table_destroy(subm->tbl);

	return ret;
}

static void hws_submatcher_destroy(struct mlx5hws_bwc_matcher *cmatcher,
				   int idx)
{
	struct mlx5hws_bwc_complex_submatcher *subm;

	subm = &cmatcher->complex->submatchers[idx];

	ida_destroy(&subm->chain_ida);
	mutex_destroy(&subm->hash_lock);
	rhashtable_destroy(&subm->rules_hash);

	if (subm->bwc_matcher) {
		mlx5hws_bwc_matcher_destroy_simple(subm->bwc_matcher);
		if (idx)
			kfree(subm->bwc_matcher);
	}

	/* We own all of the isolated tables, but not the original one. */
	if (idx) {
		mlx5hws_action_destroy(subm->action_tbl);
		mlx5hws_table_destroy(subm->tbl);
	}
}

static int
hws_complex_data_actions_init(struct mlx5hws_bwc_matcher_complex_data *cdata)
{
	struct mlx5hws_context *ctx = cdata->submatchers[0].tbl->ctx;
	u8 modify_hdr_action[MLX5_ST_SZ_BYTES(set_action_in)] = {0};
	struct mlx5hws_action_mh_pattern ptrn;
	int ret = 0;

	/* Create modify header action to set REG_C_6 */
	MLX5_SET(set_action_in, modify_hdr_action,
		 action_type, MLX5_MODIFICATION_TYPE_SET);
	MLX5_SET(set_action_in, modify_hdr_action,
		 field, MLX5_MODI_META_REG_C_6);
	MLX5_SET(set_action_in, modify_hdr_action,
		 length, 0); /* zero means length of 32 */
	MLX5_SET(set_action_in, modify_hdr_action, offset, 0);
	MLX5_SET(set_action_in, modify_hdr_action, data, 0);

	ptrn.data = (void *)modify_hdr_action;
	ptrn.sz = MLX5HWS_ACTION_DOUBLE_SIZE;

	cdata->action_metadata =
		mlx5hws_action_create_modify_header(ctx, 1, &ptrn, 0,
						    MLX5HWS_ACTION_FLAG_HWS_FDB);
	if (!cdata->action_metadata) {
		mlx5hws_err(ctx, "Complex matcher: failed to create set reg C6 action\n");
		return -EINVAL;
	}

	/* Create last action */
	cdata->action_last =
		mlx5hws_action_create_last(ctx, MLX5HWS_ACTION_FLAG_HWS_FDB);
	if (!cdata->action_last) {
		mlx5hws_err(ctx, "Complex matcher: failed to create last action\n");
		ret = -EINVAL;
		goto destroy_action_metadata;
	}

	return 0;

destroy_action_metadata:
	mlx5hws_action_destroy(cdata->action_metadata);

	return ret;
}

static void
hws_complex_data_actions_destroy(struct mlx5hws_bwc_matcher_complex_data *cdata)
{
	mlx5hws_action_destroy(cdata->action_last);
	mlx5hws_action_destroy(cdata->action_metadata);
}

int mlx5hws_bwc_matcher_create_complex(struct mlx5hws_bwc_matcher *bwc_matcher,
				       struct mlx5hws_table *table,
				       u32 priority, u8 match_criteria_enable,
				       struct mlx5hws_match_parameters *mask)
{
	struct mlx5hws_match_parameters
		submasks[MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS] = {0};
	struct mlx5hws_bwc_matcher_complex_data *cdata;
	struct mlx5hws_context *ctx = table->ctx;
	int num_submatchers;
	int i, ret;

	for (i = 0; i < ARRAY_SIZE(submasks); i++) {
		submasks[i].match_sz = MLX5_ST_SZ_BYTES(fte_match_param);
		submasks[i].match_buf = kzalloc(submasks[i].match_sz,
						GFP_KERNEL);
		if (!submasks[i].match_buf) {
			ret = -ENOMEM;
			goto free_submasks;
		}
	}

	ret = hws_bwc_matcher_split_mask(ctx, match_criteria_enable, mask,
					 submasks, &num_submatchers);
	if (ret)
		goto free_submasks;

	cdata = kzalloc(sizeof(*cdata), GFP_KERNEL);
	if (!cdata) {
		ret = -ENOMEM;
		goto free_submasks;
	}

	bwc_matcher->complex = cdata;
	cdata->num_submatchers = num_submatchers;

	for (i = 0; i < num_submatchers; i++) {
		ret = hws_submatcher_init(bwc_matcher, i, table, priority,
					  match_criteria_enable, &submasks[i]);
		if (ret)
			goto destroy_submatchers;
	}

	ret = hws_complex_data_actions_init(cdata);
	if (ret)
		goto destroy_submatchers;

	ret = 0;
	goto free_submasks;

destroy_submatchers:
	while (i--)
		hws_submatcher_destroy(bwc_matcher, i);
	kfree(cdata);
	bwc_matcher->complex = NULL;

free_submasks:
	for (i = 0; i < ARRAY_SIZE(submasks); i++)
		kfree(submasks[i].match_buf);

	return ret;
}

void
mlx5hws_bwc_matcher_destroy_complex(struct mlx5hws_bwc_matcher *bwc_matcher)
{
	int i;

	hws_complex_data_actions_destroy(bwc_matcher->complex);
	for (i = 0; i < bwc_matcher->complex->num_submatchers; i++)
		hws_submatcher_destroy(bwc_matcher, i);
	kfree(bwc_matcher->complex);
	bwc_matcher->complex = NULL;
}

static int
hws_complex_get_subrule_data(struct mlx5hws_bwc_rule *bwc_rule,
			     struct mlx5hws_bwc_complex_submatcher *subm,
			     u32 *match_params)
__must_hold(&subm->hash_lock)
{
	struct mlx5hws_bwc_matcher *bwc_matcher = subm->bwc_matcher;
	struct mlx5hws_bwc_complex_subrule_data *sr_data, *old_data;
	struct mlx5hws_match_template *mt;
	int ret;

	sr_data = kzalloc(sizeof(*sr_data), GFP_KERNEL);
	if (!sr_data)
		return -ENOMEM;

	ret = ida_alloc(&subm->chain_ida, GFP_KERNEL);
	if (ret < 0)
		goto free_sr_data;
	sr_data->chain_id = ret;

	refcount_set(&sr_data->refcount, 1);

	mt  = bwc_matcher->matcher->mt;
	mlx5hws_definer_create_tag(match_params, mt->fc, mt->fc_sz,
				   (u8 *)&sr_data->match_tag);

	old_data = rhashtable_lookup_get_insert_fast(&subm->rules_hash,
						     &sr_data->hash_node,
						     hws_rules_hash_params);
	if (IS_ERR(old_data)) {
		ret = PTR_ERR(old_data);
		goto free_ida;
	}

	if (old_data) {
		/* Rule with the same tag already exists - update refcount */
		refcount_inc(&old_data->refcount);
		/* Let the new rule use the same tag as the existing rule.
		 * Note that we don't have any indication for the rule creation
		 * process that a rule with similar matching params already
		 * exists - no harm done when this rule is be overwritten by
		 * the same STE.
		 * There's some performance advantage in skipping such cases,
		 * so this is left for future optimizations.
		 */
		bwc_rule->subrule_data = old_data;
		ret = 0;
		goto free_ida;
	}

	bwc_rule->subrule_data = sr_data;
	return 0;

free_ida:
	ida_free(&subm->chain_ida, sr_data->chain_id);
free_sr_data:
	kfree(sr_data);

	return ret;
}

static void
hws_complex_put_subrule_data(struct mlx5hws_bwc_rule *bwc_rule,
			     struct mlx5hws_bwc_complex_submatcher *subm,
			     bool *is_last_rule)
__must_hold(&subm->hash_lock)
{
	struct mlx5hws_bwc_complex_subrule_data *sr_data;

	if (is_last_rule)
		*is_last_rule = false;

	sr_data = bwc_rule->subrule_data;
	if (refcount_dec_and_test(&sr_data->refcount)) {
		rhashtable_remove_fast(&subm->rules_hash,
				       &sr_data->hash_node,
				       hws_rules_hash_params);
		ida_free(&subm->chain_ida, sr_data->chain_id);
		kfree(sr_data);
		if (is_last_rule)
			*is_last_rule = true;
	}

	bwc_rule->subrule_data = NULL;
}

static int hws_complex_subrule_create(struct mlx5hws_bwc_matcher *cmatcher,
				      struct mlx5hws_bwc_rule *subrule,
				      u32 *match_params, u32 flow_source,
				      int bwc_queue_idx, int subm_idx,
				      struct mlx5hws_rule_action *actions,
				      u32 *chain_id)
{
	struct mlx5hws_rule_action chain_actions[HWS_NUM_CHAIN_ACTIONS] = {0};
	u8 modify_hdr_action[MLX5_ST_SZ_BYTES(set_action_in)] = {0};
	struct mlx5hws_bwc_matcher_complex_data *cdata;
	struct mlx5hws_bwc_complex_submatcher *subm;
	int ret;

	cdata = cmatcher->complex;
	subm = &cdata->submatchers[subm_idx];

	mutex_lock(&subm->hash_lock);

	ret = hws_complex_get_subrule_data(subrule, subm, match_params);
	if (ret)
		goto unlock;

	*chain_id = subrule->subrule_data->chain_id;

	if (!actions) {
		MLX5_SET(set_action_in, modify_hdr_action, data, *chain_id);
		chain_actions[0].action = cdata->action_metadata;
		chain_actions[0].modify_header.data = modify_hdr_action;
		chain_actions[1].action =
			cdata->submatchers[subm_idx + 1].action_tbl;
		chain_actions[2].action = cdata->action_last;
		actions = chain_actions;
	}

	ret = mlx5hws_bwc_rule_create_simple(subrule, match_params, actions,
					     flow_source, bwc_queue_idx);
	if (ret)
		goto put_subrule_data;

	ret = 0;
	goto unlock;

put_subrule_data:
	hws_complex_put_subrule_data(subrule, subm, NULL);
unlock:
	mutex_unlock(&subm->hash_lock);

	return ret;
}

static int hws_complex_subrule_destroy(struct mlx5hws_bwc_rule *bwc_rule,
				       struct mlx5hws_bwc_matcher *cmatcher,
				       int subm_idx)
{
	struct mlx5hws_bwc_matcher_complex_data *cdata;
	struct mlx5hws_bwc_complex_submatcher *subm;
	struct mlx5hws_context *ctx;
	bool is_last_rule;
	int ret = 0;

	cdata = cmatcher->complex;
	subm = &cdata->submatchers[subm_idx];
	ctx = subm->tbl->ctx;

	mutex_lock(&subm->hash_lock);

	hws_complex_put_subrule_data(bwc_rule, subm, &is_last_rule);
	bwc_rule->rule->skip_delete = !is_last_rule;
	ret = mlx5hws_bwc_rule_destroy_simple(bwc_rule);
	if (unlikely(ret))
		mlx5hws_err(ctx,
			    "Complex rule: failed to delete subrule %d (%d)\n",
			    subm_idx, ret);

	if (subm_idx)
		mlx5hws_bwc_rule_free(bwc_rule);

	mutex_unlock(&subm->hash_lock);

	return ret;
}

int mlx5hws_bwc_rule_create_complex(struct mlx5hws_bwc_rule *bwc_rule,
				    struct mlx5hws_match_parameters *params,
				    u32 flow_source,
				    struct mlx5hws_rule_action rule_actions[],
				    u16 bwc_queue_idx)
{
	struct mlx5hws_bwc_rule
		*subrules[MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS] = {0};
	struct mlx5hws_bwc_matcher *cmatcher = bwc_rule->bwc_matcher;
	struct mlx5hws_bwc_matcher_complex_data *cdata;
	struct mlx5hws_rule_action *subrule_actions;
	struct mlx5hws_bwc_complex_submatcher *subm;
	struct mlx5hws_bwc_rule *subrule;
	u32 *match_params;
	u32 chain_id;
	int i, ret;

	cdata = cmatcher->complex;
	if (!cdata)
		return -EINVAL;

	/* Duplicate user data because we will modify it to set register C6
	 * values. For the same reason, make sure that we allocate a full
	 * match_param even if the user gave us fewer bytes. We need to ensure
	 * there is space for the match on C6.
	 */
	match_params = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
	if (!match_params)
		return -ENOMEM;

	memcpy(match_params, params->match_buf, params->match_sz);

	ret = hws_complex_subrule_create(cmatcher, bwc_rule, match_params,
					 flow_source, bwc_queue_idx, 0,
					 NULL, &chain_id);
	if (ret)
		goto free_match_params;
	subrules[0] = bwc_rule;

	for (i = 1; i < cdata->num_submatchers; i++) {
		subm = &cdata->submatchers[i];
		subrule = mlx5hws_bwc_rule_alloc(subm->bwc_matcher);
		if (!subrule) {
			ret = -ENOMEM;
			goto destroy_subrules;
		}

		/* Match on the previous subrule's chain_id. This is how
		 * subrules are connected in steering.
		 */
		MLX5_SET(fte_match_param, match_params,
			 misc_parameters_2.metadata_reg_c_6, chain_id);

		/* The last subrule uses the complex rule's user-specified
		 * actions. Everything else uses the chaining rules based on the
		 * next table and chain_id.
		 */
		subrule_actions =
			i == cdata->num_submatchers - 1 ? rule_actions : NULL;

		ret = hws_complex_subrule_create(cmatcher, subrule,
						 match_params, flow_source,
						 bwc_queue_idx, i,
						 subrule_actions, &chain_id);
		if (ret) {
			mlx5hws_bwc_rule_free(subrule);
			goto destroy_subrules;
		}

		subrules[i] = subrule;
	}

	for (i = 0; i < cdata->num_submatchers - 1; i++)
		subrules[i]->next_subrule = subrules[i + 1];

	kfree(match_params);

	return 0;

destroy_subrules:
	while (i--)
		hws_complex_subrule_destroy(subrules[i], cmatcher, i);
free_match_params:
	kfree(match_params);

	return ret;
}

int mlx5hws_bwc_rule_destroy_complex(struct mlx5hws_bwc_rule *bwc_rule)
{
	struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
	struct mlx5hws_bwc_rule
		*subrules[MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS] = {0};
	struct mlx5hws_bwc_matcher_complex_data *cdata;
	int i, err, ret_val;

	cdata = bwc_matcher->complex;

	/* Construct a list of all the subrules we need to destroy. */
	subrules[0] = bwc_rule;
	for (i = 1; i < cdata->num_submatchers; i++)
		subrules[i] = subrules[i - 1]->next_subrule;

	ret_val = 0;
	for (i = 0; i < cdata->num_submatchers; i++) {
		err = hws_complex_subrule_destroy(subrules[i], bwc_matcher, i);
		/* If something goes wrong, plow along to destroy all of the
		 * subrules but return an error upstack.
		 */
		if (unlikely(err))
			ret_val = err;
	}

	return ret_val;
}

static void
hws_bwc_matcher_init_move(struct mlx5hws_bwc_matcher *bwc_matcher)
{
	struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
	u16 bwc_queues = mlx5hws_bwc_queues(ctx);
	struct mlx5hws_bwc_rule *bwc_rule;
	struct list_head *rules_list;
	int i;

	for (i = 0; i < bwc_queues; i++) {
		rules_list = &bwc_matcher->rules[i];
		if (list_empty(rules_list))
			continue;

		list_for_each_entry(bwc_rule, rules_list, list_node) {
			if (!bwc_rule->subrule_data)
				continue;
			bwc_rule->subrule_data->was_moved = false;
		}
	}
}

int mlx5hws_bwc_matcher_complex_move(struct mlx5hws_bwc_matcher *bwc_matcher)
{
	struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
	struct mlx5hws_matcher *matcher = bwc_matcher->matcher;
	u16 bwc_queues = mlx5hws_bwc_queues(ctx);
	struct mlx5hws_bwc_rule *tmp_bwc_rule;
	struct mlx5hws_rule_attr rule_attr;
	int move_error = 0, poll_error = 0;
	struct mlx5hws_rule *tmp_rule;
	struct list_head *rules_list;
	u32 expected_completions = 1;
	int i, ret = 0;

	hws_bwc_matcher_init_move(bwc_matcher);

	mlx5hws_bwc_rule_fill_attr(bwc_matcher, 0, 0, &rule_attr);

	for (i = 0; i < bwc_queues; i++) {
		rules_list = &bwc_matcher->rules[i];
		if (list_empty(rules_list))
			continue;

		rule_attr.queue_id = mlx5hws_bwc_get_queue_id(ctx, i);

		list_for_each_entry(tmp_bwc_rule, rules_list, list_node) {
			/* Check if a rule with similar tag has already
			 * been moved.
			 */
			if (tmp_bwc_rule->subrule_data->was_moved) {
				/* This rule is a duplicate of rule with
				 * identical tag that has already been moved
				 * earlier. Just update this rule's RTCs.
				 */
				tmp_bwc_rule->rule->rtc_0 =
					tmp_bwc_rule->subrule_data->rtc_0;
				tmp_bwc_rule->rule->rtc_1 =
					tmp_bwc_rule->subrule_data->rtc_1;
				tmp_bwc_rule->rule->matcher =
					tmp_bwc_rule->rule->matcher->resize_dst;
				continue;
			}

			/* First time we're moving rule with this tag.
			 * Move it for real.
			 */
			tmp_rule = tmp_bwc_rule->rule;
			tmp_rule->skip_delete = false;
			ret = mlx5hws_matcher_resize_rule_move(matcher,
							       tmp_rule,
							       &rule_attr);
			if (unlikely(ret)) {
				if (!move_error) {
					mlx5hws_err(ctx,
						    "Moving complex BWC rule: move failed (%d), attempting to move rest of the rules\n",
						    ret);
					move_error = ret;
				}
				/* Rule wasn't queued, no need to poll */
				continue;
			}

			expected_completions = 1;
			ret = mlx5hws_bwc_queue_poll(ctx,
						     rule_attr.queue_id,
						     &expected_completions,
						     true);
			if (unlikely(ret)) {
				if (ret == -ETIMEDOUT) {
					mlx5hws_err(ctx,
						    "Moving complex BWC rule: timeout polling for completions (%d), aborting rehash\n",
						    ret);
					return ret;
				}
				if (!poll_error) {
					mlx5hws_err(ctx,
						    "Moving complex BWC rule: polling for completions failed (%d), attempting to move rest of the rules\n",
						    ret);
					poll_error = ret;
				}
			}

			/* Done moving the rule to the new matcher,
			 * now update RTCs for all the duplicated rules.
			 */
			tmp_bwc_rule->subrule_data->rtc_0 =
				tmp_bwc_rule->rule->rtc_0;
			tmp_bwc_rule->subrule_data->rtc_1 =
				tmp_bwc_rule->rule->rtc_1;

			tmp_bwc_rule->subrule_data->was_moved = true;
		}
	}

	/* Return the first error that happened */
	if (unlikely(move_error))
		return move_error;
	if (unlikely(poll_error))
		return poll_error;

	return ret;
}

int
mlx5hws_bwc_matcher_complex_move_first(struct mlx5hws_bwc_matcher *bwc_matcher)
{
	struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
	struct mlx5hws_bwc_matcher_complex_data *cdata;
	struct mlx5hws_table *isolated_tbl;
	u32 end_ft_id;
	int i, ret;

	cdata = bwc_matcher->complex;

	/* We are rehashing the first submatcher. We need to update the
	 * subsequent submatchers to point to the end_ft of this new matcher.
	 * This needs to be done before moving any rules to prevent possible
	 * steering loops.
	 */
	end_ft_id = bwc_matcher->matcher->resize_dst->end_ft_id;
	for (i = 1; i < cdata->num_submatchers; i++) {
		isolated_tbl = cdata->submatchers[i].tbl;
		ret = mlx5hws_matcher_update_end_ft_isolated(isolated_tbl,
							     end_ft_id);
		if (ret) {
			mlx5hws_err(ctx,
				    "Complex matcher: failed updating end_ft of isolated matcher (%d)\n",
				    ret);
			return ret;
		}
	}

	return mlx5hws_bwc_matcher_complex_move(bwc_matcher);
}