RDMA/mlx5: Separate flow steering logic from main.c

+1 -1

drivers/infiniband/hw/mlx5/Makefile

··· 23 23 mlx5_ib-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += odp.o 24 24 mlx5_ib-$(CONFIG_MLX5_ESWITCH) += ib_rep.o 25 25 mlx5_ib-$(CONFIG_INFINIBAND_USER_ACCESS) += devx.o \ 26 - flow.o \ 26 + fs.o \ 27 27 qos.o \ 28 28 std_types.o

+1 -58

drivers/infiniband/hw/mlx5/devx.c

··· 14 14 #include <linux/mlx5/driver.h> 15 15 #include <linux/mlx5/fs.h> 16 16 #include "mlx5_ib.h" 17 + #include "devx.h" 17 18 #include "qp.h" 18 19 #include <linux/xarray.h> 19 20 ··· 90 89 u8 is_destroyed:1; 91 90 }; 92 91 93 - #define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in) 94 - struct devx_obj { 95 - struct mlx5_ib_dev *ib_dev; 96 - u64 obj_id; 97 - u32 dinlen; /* destroy inbox length */ 98 - u32 dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW]; 99 - u32 flags; 100 - union { 101 - struct mlx5_ib_devx_mr devx_mr; 102 - struct mlx5_core_dct core_dct; 103 - struct mlx5_core_cq core_cq; 104 - u32 flow_counter_bulk_size; 105 - }; 106 - struct list_head event_sub; /* holds devx_event_subscription entries */ 107 - }; 108 - 109 92 struct devx_umem { 110 93 struct mlx5_core_dev *mdev; 111 94 struct ib_umem *umem; ··· 154 169 MLX5_SET(destroy_uctx_in, in, uid, uid); 155 170 156 171 mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out)); 157 - } 158 - 159 - bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type) 160 - { 161 - struct devx_obj *devx_obj = obj; 162 - u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 163 - 164 - switch (opcode) { 165 - case MLX5_CMD_OP_DESTROY_TIR: 166 - *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; 167 - *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, 168 - obj_id); 169 - return true; 170 - 171 - case MLX5_CMD_OP_DESTROY_FLOW_TABLE: 172 - *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 173 - *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox, 174 - table_id); 175 - return true; 176 - default: 177 - return false; 178 - } 179 - } 180 - 181 - bool mlx5_ib_devx_is_flow_counter(void *obj, u32 offset, u32 *counter_id) 182 - { 183 - struct devx_obj *devx_obj = obj; 184 - u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 185 - 186 - if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) { 187 - 188 - if (offset && offset >= devx_obj->flow_counter_bulk_size) 189 - return false; 190 - 191 - *counter_id = MLX5_GET(dealloc_flow_counter_in, 192 - devx_obj->dinbox, 193 - flow_counter_id); 194 - *counter_id += offset; 195 - return true; 196 - } 197 - 198 - return false; 199 172 } 200 173 201 174 static bool is_legacy_unaffiliated_event_num(u16 event_num)

+26

drivers/infiniband/hw/mlx5/devx.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 + /* 3 + * Copyright (c) 2019-2020, Mellanox Technologies inc. All rights reserved. 4 + */ 5 + 6 + #ifndef _MLX5_IB_DEVX_H 7 + #define _MLX5_IB_DEVX_H 8 + 9 + #include "mlx5_ib.h" 10 + 11 + #define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in) 12 + struct devx_obj { 13 + struct mlx5_ib_dev *ib_dev; 14 + u64 obj_id; 15 + u32 dinlen; /* destroy inbox length */ 16 + u32 dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW]; 17 + u32 flags; 18 + union { 19 + struct mlx5_ib_devx_mr devx_mr; 20 + struct mlx5_core_dct core_dct; 21 + struct mlx5_core_cq core_cq; 22 + u32 flow_counter_bulk_size; 23 + }; 24 + struct list_head event_sub; /* holds devx_event_subscription entries */ 25 + }; 26 + #endif /* _MLX5_IB_DEVX_H */

-765

drivers/infiniband/hw/mlx5/flow.c

··· 1 - // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 - /* 3 - * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved. 4 - */ 5 - 6 - #include <rdma/ib_user_verbs.h> 7 - #include <rdma/ib_verbs.h> 8 - #include <rdma/uverbs_types.h> 9 - #include <rdma/uverbs_ioctl.h> 10 - #include <rdma/uverbs_std_types.h> 11 - #include <rdma/mlx5_user_ioctl_cmds.h> 12 - #include <rdma/mlx5_user_ioctl_verbs.h> 13 - #include <rdma/ib_umem.h> 14 - #include <linux/mlx5/driver.h> 15 - #include <linux/mlx5/fs.h> 16 - #include "mlx5_ib.h" 17 - 18 - #define UVERBS_MODULE_NAME mlx5_ib 19 - #include <rdma/uverbs_named_ioctl.h> 20 - 21 - static int 22 - mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type, 23 - enum mlx5_flow_namespace_type *namespace) 24 - { 25 - switch (table_type) { 26 - case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX: 27 - *namespace = MLX5_FLOW_NAMESPACE_BYPASS; 28 - break; 29 - case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX: 30 - *namespace = MLX5_FLOW_NAMESPACE_EGRESS; 31 - break; 32 - case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB: 33 - *namespace = MLX5_FLOW_NAMESPACE_FDB; 34 - break; 35 - case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX: 36 - *namespace = MLX5_FLOW_NAMESPACE_RDMA_RX; 37 - break; 38 - case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX: 39 - *namespace = MLX5_FLOW_NAMESPACE_RDMA_TX; 40 - break; 41 - default: 42 - return -EINVAL; 43 - } 44 - 45 - return 0; 46 - } 47 - 48 - static const struct uverbs_attr_spec mlx5_ib_flow_type[] = { 49 - [MLX5_IB_FLOW_TYPE_NORMAL] = { 50 - .type = UVERBS_ATTR_TYPE_PTR_IN, 51 - .u.ptr = { 52 - .len = sizeof(u16), /* data is priority */ 53 - .min_len = sizeof(u16), 54 - } 55 - }, 56 - [MLX5_IB_FLOW_TYPE_SNIFFER] = { 57 - .type = UVERBS_ATTR_TYPE_PTR_IN, 58 - UVERBS_ATTR_NO_DATA(), 59 - }, 60 - [MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = { 61 - .type = UVERBS_ATTR_TYPE_PTR_IN, 62 - UVERBS_ATTR_NO_DATA(), 63 - }, 64 - [MLX5_IB_FLOW_TYPE_MC_DEFAULT] = { 65 - .type = UVERBS_ATTR_TYPE_PTR_IN, 66 - UVERBS_ATTR_NO_DATA(), 67 - }, 68 - }; 69 - 70 - static int get_dests(struct uverbs_attr_bundle *attrs, 71 - struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id, 72 - int *dest_type, struct ib_qp **qp, u32 *flags) 73 - { 74 - bool dest_devx, dest_qp; 75 - void *devx_obj; 76 - int err; 77 - 78 - dest_devx = uverbs_attr_is_valid(attrs, 79 - MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); 80 - dest_qp = uverbs_attr_is_valid(attrs, 81 - MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); 82 - 83 - *flags = 0; 84 - err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS, 85 - MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS | 86 - MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP); 87 - if (err) 88 - return err; 89 - 90 - /* Both flags are not allowed */ 91 - if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS && 92 - *flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) 93 - return -EINVAL; 94 - 95 - if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) { 96 - if (dest_devx && (dest_qp || *flags)) 97 - return -EINVAL; 98 - else if (dest_qp && *flags) 99 - return -EINVAL; 100 - } 101 - 102 - /* Allow only DEVX object, drop as dest for FDB */ 103 - if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB && !(dest_devx || 104 - (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP))) 105 - return -EINVAL; 106 - 107 - /* Allow only DEVX object or QP as dest when inserting to RDMA_RX */ 108 - if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) && 109 - ((!dest_devx && !dest_qp) || (dest_devx && dest_qp))) 110 - return -EINVAL; 111 - 112 - *qp = NULL; 113 - if (dest_devx) { 114 - devx_obj = 115 - uverbs_attr_get_obj(attrs, 116 - MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); 117 - 118 - /* Verify that the given DEVX object is a flow 119 - * steering destination. 120 - */ 121 - if (!mlx5_ib_devx_is_flow_dest(devx_obj, dest_id, dest_type)) 122 - return -EINVAL; 123 - /* Allow only flow table as dest when inserting to FDB or RDMA_RX */ 124 - if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB || 125 - fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) && 126 - *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) 127 - return -EINVAL; 128 - } else if (dest_qp) { 129 - struct mlx5_ib_qp *mqp; 130 - 131 - *qp = uverbs_attr_get_obj(attrs, 132 - MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); 133 - if (IS_ERR(*qp)) 134 - return PTR_ERR(*qp); 135 - 136 - if ((*qp)->qp_type != IB_QPT_RAW_PACKET) 137 - return -EINVAL; 138 - 139 - mqp = to_mqp(*qp); 140 - if (mqp->is_rss) 141 - *dest_id = mqp->rss_qp.tirn; 142 - else 143 - *dest_id = mqp->raw_packet_qp.rq.tirn; 144 - *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; 145 - } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) { 146 - *dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT; 147 - } 148 - 149 - if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR && 150 - fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) 151 - return -EINVAL; 152 - 153 - return 0; 154 - } 155 - 156 - #define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2 157 - static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)( 158 - struct uverbs_attr_bundle *attrs) 159 - { 160 - struct mlx5_flow_context flow_context = {.flow_tag = 161 - MLX5_FS_DEFAULT_FLOW_TAG}; 162 - u32 *offset_attr, offset = 0, counter_id = 0; 163 - int dest_id, dest_type, inlen, len, ret, i; 164 - struct mlx5_ib_flow_handler *flow_handler; 165 - struct mlx5_ib_flow_matcher *fs_matcher; 166 - struct ib_uobject **arr_flow_actions; 167 - struct ib_uflow_resources *uflow_res; 168 - struct mlx5_flow_act flow_act = {}; 169 - struct ib_qp *qp = NULL; 170 - void *devx_obj, *cmd_in; 171 - struct ib_uobject *uobj; 172 - struct mlx5_ib_dev *dev; 173 - u32 flags; 174 - 175 - if (!capable(CAP_NET_RAW)) 176 - return -EPERM; 177 - 178 - fs_matcher = uverbs_attr_get_obj(attrs, 179 - MLX5_IB_ATTR_CREATE_FLOW_MATCHER); 180 - uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE); 181 - dev = mlx5_udata_to_mdev(&attrs->driver_udata); 182 - 183 - if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags)) 184 - return -EINVAL; 185 - 186 - if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS) 187 - flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS; 188 - 189 - if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) 190 - flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 191 - 192 - len = uverbs_attr_get_uobjs_arr(attrs, 193 - MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions); 194 - if (len) { 195 - devx_obj = arr_flow_actions[0]->object; 196 - 197 - if (uverbs_attr_is_valid(attrs, 198 - MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) { 199 - 200 - int num_offsets = uverbs_attr_ptr_get_array_size( 201 - attrs, 202 - MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, 203 - sizeof(u32)); 204 - 205 - if (num_offsets != 1) 206 - return -EINVAL; 207 - 208 - offset_attr = uverbs_attr_get_alloced_ptr( 209 - attrs, 210 - MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET); 211 - offset = *offset_attr; 212 - } 213 - 214 - if (!mlx5_ib_devx_is_flow_counter(devx_obj, offset, 215 - &counter_id)) 216 - return -EINVAL; 217 - 218 - flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 219 - } 220 - 221 - cmd_in = uverbs_attr_get_alloced_ptr( 222 - attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); 223 - inlen = uverbs_attr_get_len(attrs, 224 - MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); 225 - 226 - uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS); 227 - if (!uflow_res) 228 - return -ENOMEM; 229 - 230 - len = uverbs_attr_get_uobjs_arr(attrs, 231 - MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &arr_flow_actions); 232 - for (i = 0; i < len; i++) { 233 - struct mlx5_ib_flow_action *maction = 234 - to_mflow_act(arr_flow_actions[i]->object); 235 - 236 - ret = parse_flow_flow_action(maction, false, &flow_act); 237 - if (ret) 238 - goto err_out; 239 - flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE, 240 - arr_flow_actions[i]->object); 241 - } 242 - 243 - ret = uverbs_copy_from(&flow_context.flow_tag, attrs, 244 - MLX5_IB_ATTR_CREATE_FLOW_TAG); 245 - if (!ret) { 246 - if (flow_context.flow_tag >= BIT(24)) { 247 - ret = -EINVAL; 248 - goto err_out; 249 - } 250 - flow_context.flags |= FLOW_CONTEXT_HAS_TAG; 251 - } 252 - 253 - flow_handler = mlx5_ib_raw_fs_rule_add(dev, fs_matcher, 254 - &flow_context, 255 - &flow_act, 256 - counter_id, 257 - cmd_in, inlen, 258 - dest_id, dest_type); 259 - if (IS_ERR(flow_handler)) { 260 - ret = PTR_ERR(flow_handler); 261 - goto err_out; 262 - } 263 - 264 - ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res); 265 - 266 - return 0; 267 - err_out: 268 - ib_uverbs_flow_resources_free(uflow_res); 269 - return ret; 270 - } 271 - 272 - static int flow_matcher_cleanup(struct ib_uobject *uobject, 273 - enum rdma_remove_reason why, 274 - struct uverbs_attr_bundle *attrs) 275 - { 276 - struct mlx5_ib_flow_matcher *obj = uobject->object; 277 - int ret; 278 - 279 - ret = ib_destroy_usecnt(&obj->usecnt, why, uobject); 280 - if (ret) 281 - return ret; 282 - 283 - kfree(obj); 284 - return 0; 285 - } 286 - 287 - static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs, 288 - struct mlx5_ib_flow_matcher *obj) 289 - { 290 - enum mlx5_ib_uapi_flow_table_type ft_type = 291 - MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX; 292 - u32 flags; 293 - int err; 294 - 295 - /* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older 296 - * users should switch to it. We leave this to not break userspace 297 - */ 298 - if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) && 299 - uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) 300 - return -EINVAL; 301 - 302 - if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) { 303 - err = uverbs_get_const(&ft_type, attrs, 304 - MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE); 305 - if (err) 306 - return err; 307 - 308 - err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type); 309 - if (err) 310 - return err; 311 - 312 - return 0; 313 - } 314 - 315 - if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) { 316 - err = uverbs_get_flags32(&flags, attrs, 317 - MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, 318 - IB_FLOW_ATTR_FLAGS_EGRESS); 319 - if (err) 320 - return err; 321 - 322 - if (flags) { 323 - mlx5_ib_ft_type_to_namespace( 324 - MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX, 325 - &obj->ns_type); 326 - return 0; 327 - } 328 - } 329 - 330 - obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS; 331 - 332 - return 0; 333 - } 334 - 335 - static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)( 336 - struct uverbs_attr_bundle *attrs) 337 - { 338 - struct ib_uobject *uobj = uverbs_attr_get_uobject( 339 - attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE); 340 - struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata); 341 - struct mlx5_ib_flow_matcher *obj; 342 - int err; 343 - 344 - obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL); 345 - if (!obj) 346 - return -ENOMEM; 347 - 348 - obj->mask_len = uverbs_attr_get_len( 349 - attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); 350 - err = uverbs_copy_from(&obj->matcher_mask, 351 - attrs, 352 - MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); 353 - if (err) 354 - goto end; 355 - 356 - obj->flow_type = uverbs_attr_get_enum_id( 357 - attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); 358 - 359 - if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) { 360 - err = uverbs_copy_from(&obj->priority, 361 - attrs, 362 - MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); 363 - if (err) 364 - goto end; 365 - } 366 - 367 - err = uverbs_copy_from(&obj->match_criteria_enable, 368 - attrs, 369 - MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA); 370 - if (err) 371 - goto end; 372 - 373 - err = mlx5_ib_matcher_ns(attrs, obj); 374 - if (err) 375 - goto end; 376 - 377 - uobj->object = obj; 378 - obj->mdev = dev->mdev; 379 - atomic_set(&obj->usecnt, 0); 380 - return 0; 381 - 382 - end: 383 - kfree(obj); 384 - return err; 385 - } 386 - 387 - void mlx5_ib_destroy_flow_action_raw(struct mlx5_ib_flow_action *maction) 388 - { 389 - switch (maction->flow_action_raw.sub_type) { 390 - case MLX5_IB_FLOW_ACTION_MODIFY_HEADER: 391 - mlx5_modify_header_dealloc(maction->flow_action_raw.dev->mdev, 392 - maction->flow_action_raw.modify_hdr); 393 - break; 394 - case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT: 395 - mlx5_packet_reformat_dealloc(maction->flow_action_raw.dev->mdev, 396 - maction->flow_action_raw.pkt_reformat); 397 - break; 398 - case MLX5_IB_FLOW_ACTION_DECAP: 399 - break; 400 - default: 401 - break; 402 - } 403 - } 404 - 405 - static struct ib_flow_action * 406 - mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev, 407 - enum mlx5_ib_uapi_flow_table_type ft_type, 408 - u8 num_actions, void *in) 409 - { 410 - enum mlx5_flow_namespace_type namespace; 411 - struct mlx5_ib_flow_action *maction; 412 - int ret; 413 - 414 - ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); 415 - if (ret) 416 - return ERR_PTR(-EINVAL); 417 - 418 - maction = kzalloc(sizeof(*maction), GFP_KERNEL); 419 - if (!maction) 420 - return ERR_PTR(-ENOMEM); 421 - 422 - maction->flow_action_raw.modify_hdr = 423 - mlx5_modify_header_alloc(dev->mdev, namespace, num_actions, in); 424 - 425 - if (IS_ERR(maction->flow_action_raw.modify_hdr)) { 426 - ret = PTR_ERR(maction->flow_action_raw.modify_hdr); 427 - kfree(maction); 428 - return ERR_PTR(ret); 429 - } 430 - maction->flow_action_raw.sub_type = 431 - MLX5_IB_FLOW_ACTION_MODIFY_HEADER; 432 - maction->flow_action_raw.dev = dev; 433 - 434 - return &maction->ib_action; 435 - } 436 - 437 - static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev) 438 - { 439 - return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 440 - max_modify_header_actions) || 441 - MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, 442 - max_modify_header_actions) || 443 - MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, 444 - max_modify_header_actions); 445 - } 446 - 447 - static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)( 448 - struct uverbs_attr_bundle *attrs) 449 - { 450 - struct ib_uobject *uobj = uverbs_attr_get_uobject( 451 - attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE); 452 - struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); 453 - enum mlx5_ib_uapi_flow_table_type ft_type; 454 - struct ib_flow_action *action; 455 - int num_actions; 456 - void *in; 457 - int ret; 458 - 459 - if (!mlx5_ib_modify_header_supported(mdev)) 460 - return -EOPNOTSUPP; 461 - 462 - in = uverbs_attr_get_alloced_ptr(attrs, 463 - MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM); 464 - 465 - num_actions = uverbs_attr_ptr_get_array_size( 466 - attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, 467 - MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)); 468 - if (num_actions < 0) 469 - return num_actions; 470 - 471 - ret = uverbs_get_const(&ft_type, attrs, 472 - MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE); 473 - if (ret) 474 - return ret; 475 - action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in); 476 - if (IS_ERR(action)) 477 - return PTR_ERR(action); 478 - 479 - uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev, 480 - IB_FLOW_ACTION_UNSPECIFIED); 481 - 482 - return 0; 483 - } 484 - 485 - static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev, 486 - u8 packet_reformat_type, 487 - u8 ft_type) 488 - { 489 - switch (packet_reformat_type) { 490 - case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: 491 - if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) 492 - return MLX5_CAP_FLOWTABLE(ibdev->mdev, 493 - encap_general_header); 494 - break; 495 - case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: 496 - if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) 497 - return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev, 498 - reformat_l2_to_l3_tunnel); 499 - break; 500 - case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: 501 - if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) 502 - return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, 503 - reformat_l3_tunnel_to_l2); 504 - break; 505 - case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2: 506 - if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) 507 - return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap); 508 - break; 509 - default: 510 - break; 511 - } 512 - 513 - return false; 514 - } 515 - 516 - static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt) 517 - { 518 - switch (dv_prt) { 519 - case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: 520 - *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL; 521 - break; 522 - case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: 523 - *prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2; 524 - break; 525 - case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: 526 - *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL; 527 - break; 528 - default: 529 - return -EINVAL; 530 - } 531 - 532 - return 0; 533 - } 534 - 535 - static int mlx5_ib_flow_action_create_packet_reformat_ctx( 536 - struct mlx5_ib_dev *dev, 537 - struct mlx5_ib_flow_action *maction, 538 - u8 ft_type, u8 dv_prt, 539 - void *in, size_t len) 540 - { 541 - enum mlx5_flow_namespace_type namespace; 542 - u8 prm_prt; 543 - int ret; 544 - 545 - ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); 546 - if (ret) 547 - return ret; 548 - 549 - ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &prm_prt); 550 - if (ret) 551 - return ret; 552 - 553 - maction->flow_action_raw.pkt_reformat = 554 - mlx5_packet_reformat_alloc(dev->mdev, prm_prt, len, 555 - in, namespace); 556 - if (IS_ERR(maction->flow_action_raw.pkt_reformat)) { 557 - ret = PTR_ERR(maction->flow_action_raw.pkt_reformat); 558 - return ret; 559 - } 560 - 561 - maction->flow_action_raw.sub_type = 562 - MLX5_IB_FLOW_ACTION_PACKET_REFORMAT; 563 - maction->flow_action_raw.dev = dev; 564 - 565 - return 0; 566 - } 567 - 568 - static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)( 569 - struct uverbs_attr_bundle *attrs) 570 - { 571 - struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, 572 - MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE); 573 - struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); 574 - enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt; 575 - enum mlx5_ib_uapi_flow_table_type ft_type; 576 - struct mlx5_ib_flow_action *maction; 577 - int ret; 578 - 579 - ret = uverbs_get_const(&ft_type, attrs, 580 - MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE); 581 - if (ret) 582 - return ret; 583 - 584 - ret = uverbs_get_const(&dv_prt, attrs, 585 - MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE); 586 - if (ret) 587 - return ret; 588 - 589 - if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type)) 590 - return -EOPNOTSUPP; 591 - 592 - maction = kzalloc(sizeof(*maction), GFP_KERNEL); 593 - if (!maction) 594 - return -ENOMEM; 595 - 596 - if (dv_prt == 597 - MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) { 598 - maction->flow_action_raw.sub_type = 599 - MLX5_IB_FLOW_ACTION_DECAP; 600 - maction->flow_action_raw.dev = mdev; 601 - } else { 602 - void *in; 603 - int len; 604 - 605 - in = uverbs_attr_get_alloced_ptr(attrs, 606 - MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); 607 - if (IS_ERR(in)) { 608 - ret = PTR_ERR(in); 609 - goto free_maction; 610 - } 611 - 612 - len = uverbs_attr_get_len(attrs, 613 - MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); 614 - 615 - ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev, 616 - maction, ft_type, dv_prt, in, len); 617 - if (ret) 618 - goto free_maction; 619 - } 620 - 621 - uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev, 622 - IB_FLOW_ACTION_UNSPECIFIED); 623 - return 0; 624 - 625 - free_maction: 626 - kfree(maction); 627 - return ret; 628 - } 629 - 630 - DECLARE_UVERBS_NAMED_METHOD( 631 - MLX5_IB_METHOD_CREATE_FLOW, 632 - UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, 633 - UVERBS_OBJECT_FLOW, 634 - UVERBS_ACCESS_NEW, 635 - UA_MANDATORY), 636 - UVERBS_ATTR_PTR_IN( 637 - MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE, 638 - UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), 639 - UA_MANDATORY, 640 - UA_ALLOC_AND_COPY), 641 - UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER, 642 - MLX5_IB_OBJECT_FLOW_MATCHER, 643 - UVERBS_ACCESS_READ, 644 - UA_MANDATORY), 645 - UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP, 646 - UVERBS_OBJECT_QP, 647 - UVERBS_ACCESS_READ), 648 - UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX, 649 - MLX5_IB_OBJECT_DEVX_OBJ, 650 - UVERBS_ACCESS_READ), 651 - UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, 652 - UVERBS_OBJECT_FLOW_ACTION, 653 - UVERBS_ACCESS_READ, 1, 654 - MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS, 655 - UA_OPTIONAL), 656 - UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG, 657 - UVERBS_ATTR_TYPE(u32), 658 - UA_OPTIONAL), 659 - UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, 660 - MLX5_IB_OBJECT_DEVX_OBJ, 661 - UVERBS_ACCESS_READ, 1, 1, 662 - UA_OPTIONAL), 663 - UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, 664 - UVERBS_ATTR_MIN_SIZE(sizeof(u32)), 665 - UA_OPTIONAL, 666 - UA_ALLOC_AND_COPY), 667 - UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS, 668 - enum mlx5_ib_create_flow_flags, 669 - UA_OPTIONAL)); 670 - 671 - DECLARE_UVERBS_NAMED_METHOD_DESTROY( 672 - MLX5_IB_METHOD_DESTROY_FLOW, 673 - UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, 674 - UVERBS_OBJECT_FLOW, 675 - UVERBS_ACCESS_DESTROY, 676 - UA_MANDATORY)); 677 - 678 - ADD_UVERBS_METHODS(mlx5_ib_fs, 679 - UVERBS_OBJECT_FLOW, 680 - &UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW), 681 - &UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW)); 682 - 683 - DECLARE_UVERBS_NAMED_METHOD( 684 - MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER, 685 - UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE, 686 - UVERBS_OBJECT_FLOW_ACTION, 687 - UVERBS_ACCESS_NEW, 688 - UA_MANDATORY), 689 - UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, 690 - UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES( 691 - set_add_copy_action_in_auto)), 692 - UA_MANDATORY, 693 - UA_ALLOC_AND_COPY), 694 - UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE, 695 - enum mlx5_ib_uapi_flow_table_type, 696 - UA_MANDATORY)); 697 - 698 - DECLARE_UVERBS_NAMED_METHOD( 699 - MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT, 700 - UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE, 701 - UVERBS_OBJECT_FLOW_ACTION, 702 - UVERBS_ACCESS_NEW, 703 - UA_MANDATORY), 704 - UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF, 705 - UVERBS_ATTR_MIN_SIZE(1), 706 - UA_ALLOC_AND_COPY, 707 - UA_OPTIONAL), 708 - UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE, 709 - enum mlx5_ib_uapi_flow_action_packet_reformat_type, 710 - UA_MANDATORY), 711 - UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE, 712 - enum mlx5_ib_uapi_flow_table_type, 713 - UA_MANDATORY)); 714 - 715 - ADD_UVERBS_METHODS( 716 - mlx5_ib_flow_actions, 717 - UVERBS_OBJECT_FLOW_ACTION, 718 - &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER), 719 - &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)); 720 - 721 - DECLARE_UVERBS_NAMED_METHOD( 722 - MLX5_IB_METHOD_FLOW_MATCHER_CREATE, 723 - UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE, 724 - MLX5_IB_OBJECT_FLOW_MATCHER, 725 - UVERBS_ACCESS_NEW, 726 - UA_MANDATORY), 727 - UVERBS_ATTR_PTR_IN( 728 - MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK, 729 - UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), 730 - UA_MANDATORY), 731 - UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE, 732 - mlx5_ib_flow_type, 733 - UA_MANDATORY), 734 - UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA, 735 - UVERBS_ATTR_TYPE(u8), 736 - UA_MANDATORY), 737 - UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, 738 - enum ib_flow_flags, 739 - UA_OPTIONAL), 740 - UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE, 741 - enum mlx5_ib_uapi_flow_table_type, 742 - UA_OPTIONAL)); 743 - 744 - DECLARE_UVERBS_NAMED_METHOD_DESTROY( 745 - MLX5_IB_METHOD_FLOW_MATCHER_DESTROY, 746 - UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE, 747 - MLX5_IB_OBJECT_FLOW_MATCHER, 748 - UVERBS_ACCESS_DESTROY, 749 - UA_MANDATORY)); 750 - 751 - DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER, 752 - UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup), 753 - &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE), 754 - &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY)); 755 - 756 - const struct uapi_definition mlx5_ib_flow_defs[] = { 757 - UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 758 - MLX5_IB_OBJECT_FLOW_MATCHER), 759 - UAPI_DEF_CHAIN_OBJ_TREE( 760 - UVERBS_OBJECT_FLOW, 761 - &mlx5_ib_fs), 762 - UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION, 763 - &mlx5_ib_flow_actions), 764 - {}, 765 - };

+2514

drivers/infiniband/hw/mlx5/fs.c

··· 1 + // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 + /* 3 + * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved. 4 + */ 5 + 6 + #include <rdma/ib_user_verbs.h> 7 + #include <rdma/ib_verbs.h> 8 + #include <rdma/uverbs_types.h> 9 + #include <rdma/uverbs_ioctl.h> 10 + #include <rdma/uverbs_std_types.h> 11 + #include <rdma/mlx5_user_ioctl_cmds.h> 12 + #include <rdma/mlx5_user_ioctl_verbs.h> 13 + #include <rdma/ib_umem.h> 14 + #include <linux/mlx5/driver.h> 15 + #include <linux/mlx5/fs.h> 16 + #include <linux/mlx5/fs_helpers.h> 17 + #include <linux/mlx5/accel.h> 18 + #include <linux/mlx5/eswitch.h> 19 + #include "mlx5_ib.h" 20 + #include "counters.h" 21 + #include "devx.h" 22 + #include "fs.h" 23 + 24 + #define UVERBS_MODULE_NAME mlx5_ib 25 + #include <rdma/uverbs_named_ioctl.h> 26 + 27 + enum { 28 + MATCH_CRITERIA_ENABLE_OUTER_BIT, 29 + MATCH_CRITERIA_ENABLE_MISC_BIT, 30 + MATCH_CRITERIA_ENABLE_INNER_BIT, 31 + MATCH_CRITERIA_ENABLE_MISC2_BIT 32 + }; 33 + 34 + #define HEADER_IS_ZERO(match_criteria, headers) \ 35 + !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \ 36 + 0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \ 37 + 38 + static u8 get_match_criteria_enable(u32 *match_criteria) 39 + { 40 + u8 match_criteria_enable; 41 + 42 + match_criteria_enable = 43 + (!HEADER_IS_ZERO(match_criteria, outer_headers)) << 44 + MATCH_CRITERIA_ENABLE_OUTER_BIT; 45 + match_criteria_enable |= 46 + (!HEADER_IS_ZERO(match_criteria, misc_parameters)) << 47 + MATCH_CRITERIA_ENABLE_MISC_BIT; 48 + match_criteria_enable |= 49 + (!HEADER_IS_ZERO(match_criteria, inner_headers)) << 50 + MATCH_CRITERIA_ENABLE_INNER_BIT; 51 + match_criteria_enable |= 52 + (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) << 53 + MATCH_CRITERIA_ENABLE_MISC2_BIT; 54 + 55 + return match_criteria_enable; 56 + } 57 + 58 + static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val) 59 + { 60 + u8 entry_mask; 61 + u8 entry_val; 62 + int err = 0; 63 + 64 + if (!mask) 65 + goto out; 66 + 67 + entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c, 68 + ip_protocol); 69 + entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v, 70 + ip_protocol); 71 + if (!entry_mask) { 72 + MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask); 73 + MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val); 74 + goto out; 75 + } 76 + /* Don't override existing ip protocol */ 77 + if (mask != entry_mask || val != entry_val) 78 + err = -EINVAL; 79 + out: 80 + return err; 81 + } 82 + 83 + static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val, 84 + bool inner) 85 + { 86 + if (inner) { 87 + MLX5_SET(fte_match_set_misc, 88 + misc_c, inner_ipv6_flow_label, mask); 89 + MLX5_SET(fte_match_set_misc, 90 + misc_v, inner_ipv6_flow_label, val); 91 + } else { 92 + MLX5_SET(fte_match_set_misc, 93 + misc_c, outer_ipv6_flow_label, mask); 94 + MLX5_SET(fte_match_set_misc, 95 + misc_v, outer_ipv6_flow_label, val); 96 + } 97 + } 98 + 99 + static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val) 100 + { 101 + MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask); 102 + MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val); 103 + MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2); 104 + MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2); 105 + } 106 + 107 + static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask) 108 + { 109 + if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) && 110 + !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL)) 111 + return -EOPNOTSUPP; 112 + 113 + if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) && 114 + !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP)) 115 + return -EOPNOTSUPP; 116 + 117 + if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) && 118 + !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS)) 119 + return -EOPNOTSUPP; 120 + 121 + if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) && 122 + !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL)) 123 + return -EOPNOTSUPP; 124 + 125 + return 0; 126 + } 127 + 128 + #define LAST_ETH_FIELD vlan_tag 129 + #define LAST_IB_FIELD sl 130 + #define LAST_IPV4_FIELD tos 131 + #define LAST_IPV6_FIELD traffic_class 132 + #define LAST_TCP_UDP_FIELD src_port 133 + #define LAST_TUNNEL_FIELD tunnel_id 134 + #define LAST_FLOW_TAG_FIELD tag_id 135 + #define LAST_DROP_FIELD size 136 + #define LAST_COUNTERS_FIELD counters 137 + 138 + /* Field is the last supported field */ 139 + #define FIELDS_NOT_SUPPORTED(filter, field)\ 140 + memchr_inv((void *)&filter.field +\ 141 + sizeof(filter.field), 0,\ 142 + sizeof(filter) -\ 143 + offsetof(typeof(filter), field) -\ 144 + sizeof(filter.field)) 145 + 146 + int parse_flow_flow_action(struct mlx5_ib_flow_action *maction, 147 + bool is_egress, 148 + struct mlx5_flow_act *action) 149 + { 150 + 151 + switch (maction->ib_action.type) { 152 + case IB_FLOW_ACTION_ESP: 153 + if (action->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT | 154 + MLX5_FLOW_CONTEXT_ACTION_DECRYPT)) 155 + return -EINVAL; 156 + /* Currently only AES_GCM keymat is supported by the driver */ 157 + action->esp_id = (uintptr_t)maction->esp_aes_gcm.ctx; 158 + action->action |= is_egress ? 159 + MLX5_FLOW_CONTEXT_ACTION_ENCRYPT : 160 + MLX5_FLOW_CONTEXT_ACTION_DECRYPT; 161 + return 0; 162 + case IB_FLOW_ACTION_UNSPECIFIED: 163 + if (maction->flow_action_raw.sub_type == 164 + MLX5_IB_FLOW_ACTION_MODIFY_HEADER) { 165 + if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 166 + return -EINVAL; 167 + action->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 168 + action->modify_hdr = 169 + maction->flow_action_raw.modify_hdr; 170 + return 0; 171 + } 172 + if (maction->flow_action_raw.sub_type == 173 + MLX5_IB_FLOW_ACTION_DECAP) { 174 + if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP) 175 + return -EINVAL; 176 + action->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP; 177 + return 0; 178 + } 179 + if (maction->flow_action_raw.sub_type == 180 + MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) { 181 + if (action->action & 182 + MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT) 183 + return -EINVAL; 184 + action->action |= 185 + MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 186 + action->pkt_reformat = 187 + maction->flow_action_raw.pkt_reformat; 188 + return 0; 189 + } 190 + fallthrough; 191 + default: 192 + return -EOPNOTSUPP; 193 + } 194 + } 195 + 196 + static int parse_flow_attr(struct mlx5_core_dev *mdev, 197 + struct mlx5_flow_spec *spec, 198 + const union ib_flow_spec *ib_spec, 199 + const struct ib_flow_attr *flow_attr, 200 + struct mlx5_flow_act *action, u32 prev_type) 201 + { 202 + struct mlx5_flow_context *flow_context = &spec->flow_context; 203 + u32 *match_c = spec->match_criteria; 204 + u32 *match_v = spec->match_value; 205 + void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c, 206 + misc_parameters); 207 + void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v, 208 + misc_parameters); 209 + void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c, 210 + misc_parameters_2); 211 + void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v, 212 + misc_parameters_2); 213 + void *headers_c; 214 + void *headers_v; 215 + int match_ipv; 216 + int ret; 217 + 218 + if (ib_spec->type & IB_FLOW_SPEC_INNER) { 219 + headers_c = MLX5_ADDR_OF(fte_match_param, match_c, 220 + inner_headers); 221 + headers_v = MLX5_ADDR_OF(fte_match_param, match_v, 222 + inner_headers); 223 + match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 224 + ft_field_support.inner_ip_version); 225 + } else { 226 + headers_c = MLX5_ADDR_OF(fte_match_param, match_c, 227 + outer_headers); 228 + headers_v = MLX5_ADDR_OF(fte_match_param, match_v, 229 + outer_headers); 230 + match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 231 + ft_field_support.outer_ip_version); 232 + } 233 + 234 + switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) { 235 + case IB_FLOW_SPEC_ETH: 236 + if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD)) 237 + return -EOPNOTSUPP; 238 + 239 + ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 240 + dmac_47_16), 241 + ib_spec->eth.mask.dst_mac); 242 + ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 243 + dmac_47_16), 244 + ib_spec->eth.val.dst_mac); 245 + 246 + ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 247 + smac_47_16), 248 + ib_spec->eth.mask.src_mac); 249 + ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 250 + smac_47_16), 251 + ib_spec->eth.val.src_mac); 252 + 253 + if (ib_spec->eth.mask.vlan_tag) { 254 + MLX5_SET(fte_match_set_lyr_2_4, headers_c, 255 + cvlan_tag, 1); 256 + MLX5_SET(fte_match_set_lyr_2_4, headers_v, 257 + cvlan_tag, 1); 258 + 259 + MLX5_SET(fte_match_set_lyr_2_4, headers_c, 260 + first_vid, ntohs(ib_spec->eth.mask.vlan_tag)); 261 + MLX5_SET(fte_match_set_lyr_2_4, headers_v, 262 + first_vid, ntohs(ib_spec->eth.val.vlan_tag)); 263 + 264 + MLX5_SET(fte_match_set_lyr_2_4, headers_c, 265 + first_cfi, 266 + ntohs(ib_spec->eth.mask.vlan_tag) >> 12); 267 + MLX5_SET(fte_match_set_lyr_2_4, headers_v, 268 + first_cfi, 269 + ntohs(ib_spec->eth.val.vlan_tag) >> 12); 270 + 271 + MLX5_SET(fte_match_set_lyr_2_4, headers_c, 272 + first_prio, 273 + ntohs(ib_spec->eth.mask.vlan_tag) >> 13); 274 + MLX5_SET(fte_match_set_lyr_2_4, headers_v, 275 + first_prio, 276 + ntohs(ib_spec->eth.val.vlan_tag) >> 13); 277 + } 278 + MLX5_SET(fte_match_set_lyr_2_4, headers_c, 279 + ethertype, ntohs(ib_spec->eth.mask.ether_type)); 280 + MLX5_SET(fte_match_set_lyr_2_4, headers_v, 281 + ethertype, ntohs(ib_spec->eth.val.ether_type)); 282 + break; 283 + case IB_FLOW_SPEC_IPV4: 284 + if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD)) 285 + return -EOPNOTSUPP; 286 + 287 + if (match_ipv) { 288 + MLX5_SET(fte_match_set_lyr_2_4, headers_c, 289 + ip_version, 0xf); 290 + MLX5_SET(fte_match_set_lyr_2_4, headers_v, 291 + ip_version, MLX5_FS_IPV4_VERSION); 292 + } else { 293 + MLX5_SET(fte_match_set_lyr_2_4, headers_c, 294 + ethertype, 0xffff); 295 + MLX5_SET(fte_match_set_lyr_2_4, headers_v, 296 + ethertype, ETH_P_IP); 297 + } 298 + 299 + memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 300 + src_ipv4_src_ipv6.ipv4_layout.ipv4), 301 + &ib_spec->ipv4.mask.src_ip, 302 + sizeof(ib_spec->ipv4.mask.src_ip)); 303 + memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 304 + src_ipv4_src_ipv6.ipv4_layout.ipv4), 305 + &ib_spec->ipv4.val.src_ip, 306 + sizeof(ib_spec->ipv4.val.src_ip)); 307 + memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 308 + dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 309 + &ib_spec->ipv4.mask.dst_ip, 310 + sizeof(ib_spec->ipv4.mask.dst_ip)); 311 + memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 312 + dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 313 + &ib_spec->ipv4.val.dst_ip, 314 + sizeof(ib_spec->ipv4.val.dst_ip)); 315 + 316 + set_tos(headers_c, headers_v, 317 + ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos); 318 + 319 + if (set_proto(headers_c, headers_v, 320 + ib_spec->ipv4.mask.proto, 321 + ib_spec->ipv4.val.proto)) 322 + return -EINVAL; 323 + break; 324 + case IB_FLOW_SPEC_IPV6: 325 + if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD)) 326 + return -EOPNOTSUPP; 327 + 328 + if (match_ipv) { 329 + MLX5_SET(fte_match_set_lyr_2_4, headers_c, 330 + ip_version, 0xf); 331 + MLX5_SET(fte_match_set_lyr_2_4, headers_v, 332 + ip_version, MLX5_FS_IPV6_VERSION); 333 + } else { 334 + MLX5_SET(fte_match_set_lyr_2_4, headers_c, 335 + ethertype, 0xffff); 336 + MLX5_SET(fte_match_set_lyr_2_4, headers_v, 337 + ethertype, ETH_P_IPV6); 338 + } 339 + 340 + memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 341 + src_ipv4_src_ipv6.ipv6_layout.ipv6), 342 + &ib_spec->ipv6.mask.src_ip, 343 + sizeof(ib_spec->ipv6.mask.src_ip)); 344 + memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 345 + src_ipv4_src_ipv6.ipv6_layout.ipv6), 346 + &ib_spec->ipv6.val.src_ip, 347 + sizeof(ib_spec->ipv6.val.src_ip)); 348 + memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 349 + dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 350 + &ib_spec->ipv6.mask.dst_ip, 351 + sizeof(ib_spec->ipv6.mask.dst_ip)); 352 + memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 353 + dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 354 + &ib_spec->ipv6.val.dst_ip, 355 + sizeof(ib_spec->ipv6.val.dst_ip)); 356 + 357 + set_tos(headers_c, headers_v, 358 + ib_spec->ipv6.mask.traffic_class, 359 + ib_spec->ipv6.val.traffic_class); 360 + 361 + if (set_proto(headers_c, headers_v, 362 + ib_spec->ipv6.mask.next_hdr, 363 + ib_spec->ipv6.val.next_hdr)) 364 + return -EINVAL; 365 + 366 + set_flow_label(misc_params_c, misc_params_v, 367 + ntohl(ib_spec->ipv6.mask.flow_label), 368 + ntohl(ib_spec->ipv6.val.flow_label), 369 + ib_spec->type & IB_FLOW_SPEC_INNER); 370 + break; 371 + case IB_FLOW_SPEC_ESP: 372 + if (ib_spec->esp.mask.seq) 373 + return -EOPNOTSUPP; 374 + 375 + MLX5_SET(fte_match_set_misc, misc_params_c, outer_esp_spi, 376 + ntohl(ib_spec->esp.mask.spi)); 377 + MLX5_SET(fte_match_set_misc, misc_params_v, outer_esp_spi, 378 + ntohl(ib_spec->esp.val.spi)); 379 + break; 380 + case IB_FLOW_SPEC_TCP: 381 + if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, 382 + LAST_TCP_UDP_FIELD)) 383 + return -EOPNOTSUPP; 384 + 385 + if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP)) 386 + return -EINVAL; 387 + 388 + MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport, 389 + ntohs(ib_spec->tcp_udp.mask.src_port)); 390 + MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport, 391 + ntohs(ib_spec->tcp_udp.val.src_port)); 392 + 393 + MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport, 394 + ntohs(ib_spec->tcp_udp.mask.dst_port)); 395 + MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport, 396 + ntohs(ib_spec->tcp_udp.val.dst_port)); 397 + break; 398 + case IB_FLOW_SPEC_UDP: 399 + if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, 400 + LAST_TCP_UDP_FIELD)) 401 + return -EOPNOTSUPP; 402 + 403 + if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP)) 404 + return -EINVAL; 405 + 406 + MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport, 407 + ntohs(ib_spec->tcp_udp.mask.src_port)); 408 + MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, 409 + ntohs(ib_spec->tcp_udp.val.src_port)); 410 + 411 + MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport, 412 + ntohs(ib_spec->tcp_udp.mask.dst_port)); 413 + MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 414 + ntohs(ib_spec->tcp_udp.val.dst_port)); 415 + break; 416 + case IB_FLOW_SPEC_GRE: 417 + if (ib_spec->gre.mask.c_ks_res0_ver) 418 + return -EOPNOTSUPP; 419 + 420 + if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE)) 421 + return -EINVAL; 422 + 423 + MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol, 424 + 0xff); 425 + MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 426 + IPPROTO_GRE); 427 + 428 + MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol, 429 + ntohs(ib_spec->gre.mask.protocol)); 430 + MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol, 431 + ntohs(ib_spec->gre.val.protocol)); 432 + 433 + memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c, 434 + gre_key.nvgre.hi), 435 + &ib_spec->gre.mask.key, 436 + sizeof(ib_spec->gre.mask.key)); 437 + memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v, 438 + gre_key.nvgre.hi), 439 + &ib_spec->gre.val.key, 440 + sizeof(ib_spec->gre.val.key)); 441 + break; 442 + case IB_FLOW_SPEC_MPLS: 443 + switch (prev_type) { 444 + case IB_FLOW_SPEC_UDP: 445 + if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 446 + ft_field_support.outer_first_mpls_over_udp), 447 + &ib_spec->mpls.mask.tag)) 448 + return -EOPNOTSUPP; 449 + 450 + memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 451 + outer_first_mpls_over_udp), 452 + &ib_spec->mpls.val.tag, 453 + sizeof(ib_spec->mpls.val.tag)); 454 + memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 455 + outer_first_mpls_over_udp), 456 + &ib_spec->mpls.mask.tag, 457 + sizeof(ib_spec->mpls.mask.tag)); 458 + break; 459 + case IB_FLOW_SPEC_GRE: 460 + if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 461 + ft_field_support.outer_first_mpls_over_gre), 462 + &ib_spec->mpls.mask.tag)) 463 + return -EOPNOTSUPP; 464 + 465 + memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 466 + outer_first_mpls_over_gre), 467 + &ib_spec->mpls.val.tag, 468 + sizeof(ib_spec->mpls.val.tag)); 469 + memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 470 + outer_first_mpls_over_gre), 471 + &ib_spec->mpls.mask.tag, 472 + sizeof(ib_spec->mpls.mask.tag)); 473 + break; 474 + default: 475 + if (ib_spec->type & IB_FLOW_SPEC_INNER) { 476 + if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 477 + ft_field_support.inner_first_mpls), 478 + &ib_spec->mpls.mask.tag)) 479 + return -EOPNOTSUPP; 480 + 481 + memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 482 + inner_first_mpls), 483 + &ib_spec->mpls.val.tag, 484 + sizeof(ib_spec->mpls.val.tag)); 485 + memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 486 + inner_first_mpls), 487 + &ib_spec->mpls.mask.tag, 488 + sizeof(ib_spec->mpls.mask.tag)); 489 + } else { 490 + if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 491 + ft_field_support.outer_first_mpls), 492 + &ib_spec->mpls.mask.tag)) 493 + return -EOPNOTSUPP; 494 + 495 + memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 496 + outer_first_mpls), 497 + &ib_spec->mpls.val.tag, 498 + sizeof(ib_spec->mpls.val.tag)); 499 + memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 500 + outer_first_mpls), 501 + &ib_spec->mpls.mask.tag, 502 + sizeof(ib_spec->mpls.mask.tag)); 503 + } 504 + } 505 + break; 506 + case IB_FLOW_SPEC_VXLAN_TUNNEL: 507 + if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask, 508 + LAST_TUNNEL_FIELD)) 509 + return -EOPNOTSUPP; 510 + 511 + MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni, 512 + ntohl(ib_spec->tunnel.mask.tunnel_id)); 513 + MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni, 514 + ntohl(ib_spec->tunnel.val.tunnel_id)); 515 + break; 516 + case IB_FLOW_SPEC_ACTION_TAG: 517 + if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag, 518 + LAST_FLOW_TAG_FIELD)) 519 + return -EOPNOTSUPP; 520 + if (ib_spec->flow_tag.tag_id >= BIT(24)) 521 + return -EINVAL; 522 + 523 + flow_context->flow_tag = ib_spec->flow_tag.tag_id; 524 + flow_context->flags |= FLOW_CONTEXT_HAS_TAG; 525 + break; 526 + case IB_FLOW_SPEC_ACTION_DROP: 527 + if (FIELDS_NOT_SUPPORTED(ib_spec->drop, 528 + LAST_DROP_FIELD)) 529 + return -EOPNOTSUPP; 530 + action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 531 + break; 532 + case IB_FLOW_SPEC_ACTION_HANDLE: 533 + ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act), 534 + flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action); 535 + if (ret) 536 + return ret; 537 + break; 538 + case IB_FLOW_SPEC_ACTION_COUNT: 539 + if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count, 540 + LAST_COUNTERS_FIELD)) 541 + return -EOPNOTSUPP; 542 + 543 + /* for now support only one counters spec per flow */ 544 + if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) 545 + return -EINVAL; 546 + 547 + action->counters = ib_spec->flow_count.counters; 548 + action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 549 + break; 550 + default: 551 + return -EINVAL; 552 + } 553 + 554 + return 0; 555 + } 556 + 557 + /* If a flow could catch both multicast and unicast packets, 558 + * it won't fall into the multicast flow steering table and this rule 559 + * could steal other multicast packets. 560 + */ 561 + static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr) 562 + { 563 + union ib_flow_spec *flow_spec; 564 + 565 + if (ib_attr->type != IB_FLOW_ATTR_NORMAL || 566 + ib_attr->num_of_specs < 1) 567 + return false; 568 + 569 + flow_spec = (union ib_flow_spec *)(ib_attr + 1); 570 + if (flow_spec->type == IB_FLOW_SPEC_IPV4) { 571 + struct ib_flow_spec_ipv4 *ipv4_spec; 572 + 573 + ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec; 574 + if (ipv4_is_multicast(ipv4_spec->val.dst_ip)) 575 + return true; 576 + 577 + return false; 578 + } 579 + 580 + if (flow_spec->type == IB_FLOW_SPEC_ETH) { 581 + struct ib_flow_spec_eth *eth_spec; 582 + 583 + eth_spec = (struct ib_flow_spec_eth *)flow_spec; 584 + return is_multicast_ether_addr(eth_spec->mask.dst_mac) && 585 + is_multicast_ether_addr(eth_spec->val.dst_mac); 586 + } 587 + 588 + return false; 589 + } 590 + 591 + enum valid_spec { 592 + VALID_SPEC_INVALID, 593 + VALID_SPEC_VALID, 594 + VALID_SPEC_NA, 595 + }; 596 + 597 + static enum valid_spec 598 + is_valid_esp_aes_gcm(struct mlx5_core_dev *mdev, 599 + const struct mlx5_flow_spec *spec, 600 + const struct mlx5_flow_act *flow_act, 601 + bool egress) 602 + { 603 + const u32 *match_c = spec->match_criteria; 604 + bool is_crypto = 605 + (flow_act->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT | 606 + MLX5_FLOW_CONTEXT_ACTION_DECRYPT)); 607 + bool is_ipsec = mlx5_fs_is_ipsec_flow(match_c); 608 + bool is_drop = flow_act->action & MLX5_FLOW_CONTEXT_ACTION_DROP; 609 + 610 + /* 611 + * Currently only crypto is supported in egress, when regular egress 612 + * rules would be supported, always return VALID_SPEC_NA. 613 + */ 614 + if (!is_crypto) 615 + return VALID_SPEC_NA; 616 + 617 + return is_crypto && is_ipsec && 618 + (!egress || (!is_drop && 619 + !(spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG))) ? 620 + VALID_SPEC_VALID : VALID_SPEC_INVALID; 621 + } 622 + 623 + static bool is_valid_spec(struct mlx5_core_dev *mdev, 624 + const struct mlx5_flow_spec *spec, 625 + const struct mlx5_flow_act *flow_act, 626 + bool egress) 627 + { 628 + /* We curretly only support ipsec egress flow */ 629 + return is_valid_esp_aes_gcm(mdev, spec, flow_act, egress) != VALID_SPEC_INVALID; 630 + } 631 + 632 + static bool is_valid_ethertype(struct mlx5_core_dev *mdev, 633 + const struct ib_flow_attr *flow_attr, 634 + bool check_inner) 635 + { 636 + union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1); 637 + int match_ipv = check_inner ? 638 + MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 639 + ft_field_support.inner_ip_version) : 640 + MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 641 + ft_field_support.outer_ip_version); 642 + int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0; 643 + bool ipv4_spec_valid, ipv6_spec_valid; 644 + unsigned int ip_spec_type = 0; 645 + bool has_ethertype = false; 646 + unsigned int spec_index; 647 + bool mask_valid = true; 648 + u16 eth_type = 0; 649 + bool type_valid; 650 + 651 + /* Validate that ethertype is correct */ 652 + for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { 653 + if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) && 654 + ib_spec->eth.mask.ether_type) { 655 + mask_valid = (ib_spec->eth.mask.ether_type == 656 + htons(0xffff)); 657 + has_ethertype = true; 658 + eth_type = ntohs(ib_spec->eth.val.ether_type); 659 + } else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) || 660 + (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) { 661 + ip_spec_type = ib_spec->type; 662 + } 663 + ib_spec = (void *)ib_spec + ib_spec->size; 664 + } 665 + 666 + type_valid = (!has_ethertype) || (!ip_spec_type); 667 + if (!type_valid && mask_valid) { 668 + ipv4_spec_valid = (eth_type == ETH_P_IP) && 669 + (ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit)); 670 + ipv6_spec_valid = (eth_type == ETH_P_IPV6) && 671 + (ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit)); 672 + 673 + type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) || 674 + (((eth_type == ETH_P_MPLS_UC) || 675 + (eth_type == ETH_P_MPLS_MC)) && match_ipv); 676 + } 677 + 678 + return type_valid; 679 + } 680 + 681 + static bool is_valid_attr(struct mlx5_core_dev *mdev, 682 + const struct ib_flow_attr *flow_attr) 683 + { 684 + return is_valid_ethertype(mdev, flow_attr, false) && 685 + is_valid_ethertype(mdev, flow_attr, true); 686 + } 687 + 688 + static void put_flow_table(struct mlx5_ib_dev *dev, 689 + struct mlx5_ib_flow_prio *prio, bool ft_added) 690 + { 691 + prio->refcount -= !!ft_added; 692 + if (!prio->refcount) { 693 + mlx5_destroy_flow_table(prio->flow_table); 694 + prio->flow_table = NULL; 695 + } 696 + } 697 + 698 + static int mlx5_ib_destroy_flow(struct ib_flow *flow_id) 699 + { 700 + struct mlx5_ib_flow_handler *handler = container_of(flow_id, 701 + struct mlx5_ib_flow_handler, 702 + ibflow); 703 + struct mlx5_ib_flow_handler *iter, *tmp; 704 + struct mlx5_ib_dev *dev = handler->dev; 705 + 706 + mutex_lock(&dev->flow_db->lock); 707 + 708 + list_for_each_entry_safe(iter, tmp, &handler->list, list) { 709 + mlx5_del_flow_rules(iter->rule); 710 + put_flow_table(dev, iter->prio, true); 711 + list_del(&iter->list); 712 + kfree(iter); 713 + } 714 + 715 + mlx5_del_flow_rules(handler->rule); 716 + put_flow_table(dev, handler->prio, true); 717 + mlx5_ib_counters_clear_description(handler->ibcounters); 718 + mutex_unlock(&dev->flow_db->lock); 719 + if (handler->flow_matcher) 720 + atomic_dec(&handler->flow_matcher->usecnt); 721 + kfree(handler); 722 + 723 + return 0; 724 + } 725 + 726 + static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap) 727 + { 728 + priority *= 2; 729 + if (!dont_trap) 730 + priority++; 731 + return priority; 732 + } 733 + 734 + enum flow_table_type { 735 + MLX5_IB_FT_RX, 736 + MLX5_IB_FT_TX 737 + }; 738 + 739 + #define MLX5_FS_MAX_TYPES 6 740 + #define MLX5_FS_MAX_ENTRIES BIT(16) 741 + 742 + static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_flow_namespace *ns, 743 + struct mlx5_ib_flow_prio *prio, 744 + int priority, 745 + int num_entries, int num_groups, 746 + u32 flags) 747 + { 748 + struct mlx5_flow_table_attr ft_attr = {}; 749 + struct mlx5_flow_table *ft; 750 + 751 + ft_attr.prio = priority; 752 + ft_attr.max_fte = num_entries; 753 + ft_attr.flags = flags; 754 + ft_attr.autogroup.max_num_groups = num_groups; 755 + ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 756 + if (IS_ERR(ft)) 757 + return ERR_CAST(ft); 758 + 759 + prio->flow_table = ft; 760 + prio->refcount = 0; 761 + return prio; 762 + } 763 + 764 + static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, 765 + struct ib_flow_attr *flow_attr, 766 + enum flow_table_type ft_type) 767 + { 768 + bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP; 769 + struct mlx5_flow_namespace *ns = NULL; 770 + struct mlx5_ib_flow_prio *prio; 771 + struct mlx5_flow_table *ft; 772 + int max_table_size; 773 + int num_entries; 774 + int num_groups; 775 + bool esw_encap; 776 + u32 flags = 0; 777 + int priority; 778 + 779 + max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 780 + log_max_ft_size)); 781 + esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != 782 + DEVLINK_ESWITCH_ENCAP_MODE_NONE; 783 + if (flow_attr->type == IB_FLOW_ATTR_NORMAL) { 784 + enum mlx5_flow_namespace_type fn_type; 785 + 786 + if (flow_is_multicast_only(flow_attr) && 787 + !dont_trap) 788 + priority = MLX5_IB_FLOW_MCAST_PRIO; 789 + else 790 + priority = ib_prio_to_core_prio(flow_attr->priority, 791 + dont_trap); 792 + if (ft_type == MLX5_IB_FT_RX) { 793 + fn_type = MLX5_FLOW_NAMESPACE_BYPASS; 794 + prio = &dev->flow_db->prios[priority]; 795 + if (!dev->is_rep && !esw_encap && 796 + MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap)) 797 + flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 798 + if (!dev->is_rep && !esw_encap && 799 + MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 800 + reformat_l3_tunnel_to_l2)) 801 + flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 802 + } else { 803 + max_table_size = 804 + BIT(MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, 805 + log_max_ft_size)); 806 + fn_type = MLX5_FLOW_NAMESPACE_EGRESS; 807 + prio = &dev->flow_db->egress_prios[priority]; 808 + if (!dev->is_rep && !esw_encap && 809 + MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat)) 810 + flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 811 + } 812 + ns = mlx5_get_flow_namespace(dev->mdev, fn_type); 813 + num_entries = MLX5_FS_MAX_ENTRIES; 814 + num_groups = MLX5_FS_MAX_TYPES; 815 + } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 816 + flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) { 817 + ns = mlx5_get_flow_namespace(dev->mdev, 818 + MLX5_FLOW_NAMESPACE_LEFTOVERS); 819 + build_leftovers_ft_param(&priority, 820 + &num_entries, 821 + &num_groups); 822 + prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO]; 823 + } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { 824 + if (!MLX5_CAP_FLOWTABLE(dev->mdev, 825 + allow_sniffer_and_nic_rx_shared_tir)) 826 + return ERR_PTR(-EOPNOTSUPP); 827 + 828 + ns = mlx5_get_flow_namespace(dev->mdev, ft_type == MLX5_IB_FT_RX ? 829 + MLX5_FLOW_NAMESPACE_SNIFFER_RX : 830 + MLX5_FLOW_NAMESPACE_SNIFFER_TX); 831 + 832 + prio = &dev->flow_db->sniffer[ft_type]; 833 + priority = 0; 834 + num_entries = 1; 835 + num_groups = 1; 836 + } 837 + 838 + if (!ns) 839 + return ERR_PTR(-EOPNOTSUPP); 840 + 841 + max_table_size = min_t(int, num_entries, max_table_size); 842 + 843 + ft = prio->flow_table; 844 + if (!ft) 845 + return _get_prio(ns, prio, priority, max_table_size, num_groups, 846 + flags); 847 + 848 + return prio; 849 + } 850 + 851 + static void set_underlay_qp(struct mlx5_ib_dev *dev, 852 + struct mlx5_flow_spec *spec, 853 + u32 underlay_qpn) 854 + { 855 + void *misc_params_c = MLX5_ADDR_OF(fte_match_param, 856 + spec->match_criteria, 857 + misc_parameters); 858 + void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 859 + misc_parameters); 860 + 861 + if (underlay_qpn && 862 + MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 863 + ft_field_support.bth_dst_qp)) { 864 + MLX5_SET(fte_match_set_misc, 865 + misc_params_v, bth_dst_qp, underlay_qpn); 866 + MLX5_SET(fte_match_set_misc, 867 + misc_params_c, bth_dst_qp, 0xffffff); 868 + } 869 + } 870 + 871 + static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev, 872 + struct mlx5_flow_spec *spec, 873 + struct mlx5_eswitch_rep *rep) 874 + { 875 + struct mlx5_eswitch *esw = dev->mdev->priv.eswitch; 876 + void *misc; 877 + 878 + if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 879 + misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 880 + misc_parameters_2); 881 + 882 + MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 883 + mlx5_eswitch_get_vport_metadata_for_match(esw, 884 + rep->vport)); 885 + misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 886 + misc_parameters_2); 887 + 888 + MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 889 + mlx5_eswitch_get_vport_metadata_mask()); 890 + } else { 891 + misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 892 + misc_parameters); 893 + 894 + MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport); 895 + 896 + misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 897 + misc_parameters); 898 + 899 + MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 900 + } 901 + } 902 + 903 + static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev, 904 + struct mlx5_ib_flow_prio *ft_prio, 905 + const struct ib_flow_attr *flow_attr, 906 + struct mlx5_flow_destination *dst, 907 + u32 underlay_qpn, 908 + struct mlx5_ib_create_flow *ucmd) 909 + { 910 + struct mlx5_flow_table *ft = ft_prio->flow_table; 911 + struct mlx5_ib_flow_handler *handler; 912 + struct mlx5_flow_act flow_act = {}; 913 + struct mlx5_flow_spec *spec; 914 + struct mlx5_flow_destination dest_arr[2] = {}; 915 + struct mlx5_flow_destination *rule_dst = dest_arr; 916 + const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr); 917 + unsigned int spec_index; 918 + u32 prev_type = 0; 919 + int err = 0; 920 + int dest_num = 0; 921 + bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; 922 + 923 + if (!is_valid_attr(dev->mdev, flow_attr)) 924 + return ERR_PTR(-EINVAL); 925 + 926 + if (dev->is_rep && is_egress) 927 + return ERR_PTR(-EINVAL); 928 + 929 + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 930 + handler = kzalloc(sizeof(*handler), GFP_KERNEL); 931 + if (!handler || !spec) { 932 + err = -ENOMEM; 933 + goto free; 934 + } 935 + 936 + INIT_LIST_HEAD(&handler->list); 937 + 938 + for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { 939 + err = parse_flow_attr(dev->mdev, spec, 940 + ib_flow, flow_attr, &flow_act, 941 + prev_type); 942 + if (err < 0) 943 + goto free; 944 + 945 + prev_type = ((union ib_flow_spec *)ib_flow)->type; 946 + ib_flow += ((union ib_flow_spec *)ib_flow)->size; 947 + } 948 + 949 + if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) { 950 + memcpy(&dest_arr[0], dst, sizeof(*dst)); 951 + dest_num++; 952 + } 953 + 954 + if (!flow_is_multicast_only(flow_attr)) 955 + set_underlay_qp(dev, spec, underlay_qpn); 956 + 957 + if (dev->is_rep) { 958 + struct mlx5_eswitch_rep *rep; 959 + 960 + rep = dev->port[flow_attr->port - 1].rep; 961 + if (!rep) { 962 + err = -EINVAL; 963 + goto free; 964 + } 965 + 966 + mlx5_ib_set_rule_source_port(dev, spec, rep); 967 + } 968 + 969 + spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria); 970 + 971 + if (is_egress && 972 + !is_valid_spec(dev->mdev, spec, &flow_act, is_egress)) { 973 + err = -EINVAL; 974 + goto free; 975 + } 976 + 977 + if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 978 + struct mlx5_ib_mcounters *mcounters; 979 + 980 + err = mlx5_ib_flow_counters_set_data(flow_act.counters, ucmd); 981 + if (err) 982 + goto free; 983 + 984 + mcounters = to_mcounters(flow_act.counters); 985 + handler->ibcounters = flow_act.counters; 986 + dest_arr[dest_num].type = 987 + MLX5_FLOW_DESTINATION_TYPE_COUNTER; 988 + dest_arr[dest_num].counter_id = 989 + mlx5_fc_id(mcounters->hw_cntrs_hndl); 990 + dest_num++; 991 + } 992 + 993 + if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) { 994 + if (!dest_num) 995 + rule_dst = NULL; 996 + } else { 997 + if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) 998 + flow_act.action |= 999 + MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO; 1000 + if (is_egress) 1001 + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; 1002 + else if (dest_num) 1003 + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1004 + } 1005 + 1006 + if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG) && 1007 + (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 1008 + flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { 1009 + mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n", 1010 + spec->flow_context.flow_tag, flow_attr->type); 1011 + err = -EINVAL; 1012 + goto free; 1013 + } 1014 + handler->rule = mlx5_add_flow_rules(ft, spec, 1015 + &flow_act, 1016 + rule_dst, dest_num); 1017 + 1018 + if (IS_ERR(handler->rule)) { 1019 + err = PTR_ERR(handler->rule); 1020 + goto free; 1021 + } 1022 + 1023 + ft_prio->refcount++; 1024 + handler->prio = ft_prio; 1025 + handler->dev = dev; 1026 + 1027 + ft_prio->flow_table = ft; 1028 + free: 1029 + if (err && handler) { 1030 + mlx5_ib_counters_clear_description(handler->ibcounters); 1031 + kfree(handler); 1032 + } 1033 + kvfree(spec); 1034 + return err ? ERR_PTR(err) : handler; 1035 + } 1036 + 1037 + static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev, 1038 + struct mlx5_ib_flow_prio *ft_prio, 1039 + const struct ib_flow_attr *flow_attr, 1040 + struct mlx5_flow_destination *dst) 1041 + { 1042 + return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL); 1043 + } 1044 + 1045 + enum { 1046 + LEFTOVERS_MC, 1047 + LEFTOVERS_UC, 1048 + }; 1049 + 1050 + static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev, 1051 + struct mlx5_ib_flow_prio *ft_prio, 1052 + struct ib_flow_attr *flow_attr, 1053 + struct mlx5_flow_destination *dst) 1054 + { 1055 + struct mlx5_ib_flow_handler *handler_ucast = NULL; 1056 + struct mlx5_ib_flow_handler *handler = NULL; 1057 + 1058 + static struct { 1059 + struct ib_flow_attr flow_attr; 1060 + struct ib_flow_spec_eth eth_flow; 1061 + } leftovers_specs[] = { 1062 + [LEFTOVERS_MC] = { 1063 + .flow_attr = { 1064 + .num_of_specs = 1, 1065 + .size = sizeof(leftovers_specs[0]) 1066 + }, 1067 + .eth_flow = { 1068 + .type = IB_FLOW_SPEC_ETH, 1069 + .size = sizeof(struct ib_flow_spec_eth), 1070 + .mask = {.dst_mac = {0x1} }, 1071 + .val = {.dst_mac = {0x1} } 1072 + } 1073 + }, 1074 + [LEFTOVERS_UC] = { 1075 + .flow_attr = { 1076 + .num_of_specs = 1, 1077 + .size = sizeof(leftovers_specs[0]) 1078 + }, 1079 + .eth_flow = { 1080 + .type = IB_FLOW_SPEC_ETH, 1081 + .size = sizeof(struct ib_flow_spec_eth), 1082 + .mask = {.dst_mac = {0x1} }, 1083 + .val = {.dst_mac = {} } 1084 + } 1085 + } 1086 + }; 1087 + 1088 + handler = create_flow_rule(dev, ft_prio, 1089 + &leftovers_specs[LEFTOVERS_MC].flow_attr, 1090 + dst); 1091 + if (!IS_ERR(handler) && 1092 + flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) { 1093 + handler_ucast = create_flow_rule(dev, ft_prio, 1094 + &leftovers_specs[LEFTOVERS_UC].flow_attr, 1095 + dst); 1096 + if (IS_ERR(handler_ucast)) { 1097 + mlx5_del_flow_rules(handler->rule); 1098 + ft_prio->refcount--; 1099 + kfree(handler); 1100 + handler = handler_ucast; 1101 + } else { 1102 + list_add(&handler_ucast->list, &handler->list); 1103 + } 1104 + } 1105 + 1106 + return handler; 1107 + } 1108 + 1109 + static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev, 1110 + struct mlx5_ib_flow_prio *ft_rx, 1111 + struct mlx5_ib_flow_prio *ft_tx, 1112 + struct mlx5_flow_destination *dst) 1113 + { 1114 + struct mlx5_ib_flow_handler *handler_rx; 1115 + struct mlx5_ib_flow_handler *handler_tx; 1116 + int err; 1117 + static const struct ib_flow_attr flow_attr = { 1118 + .num_of_specs = 0, 1119 + .size = sizeof(flow_attr) 1120 + }; 1121 + 1122 + handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst); 1123 + if (IS_ERR(handler_rx)) { 1124 + err = PTR_ERR(handler_rx); 1125 + goto err; 1126 + } 1127 + 1128 + handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst); 1129 + if (IS_ERR(handler_tx)) { 1130 + err = PTR_ERR(handler_tx); 1131 + goto err_tx; 1132 + } 1133 + 1134 + list_add(&handler_tx->list, &handler_rx->list); 1135 + 1136 + return handler_rx; 1137 + 1138 + err_tx: 1139 + mlx5_del_flow_rules(handler_rx->rule); 1140 + ft_rx->refcount--; 1141 + kfree(handler_rx); 1142 + err: 1143 + return ERR_PTR(err); 1144 + } 1145 + 1146 + 1147 + static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp, 1148 + struct ib_flow_attr *flow_attr, 1149 + int domain, 1150 + struct ib_udata *udata) 1151 + { 1152 + struct mlx5_ib_dev *dev = to_mdev(qp->device); 1153 + struct mlx5_ib_qp *mqp = to_mqp(qp); 1154 + struct mlx5_ib_flow_handler *handler = NULL; 1155 + struct mlx5_flow_destination *dst = NULL; 1156 + struct mlx5_ib_flow_prio *ft_prio_tx = NULL; 1157 + struct mlx5_ib_flow_prio *ft_prio; 1158 + bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; 1159 + struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr; 1160 + size_t min_ucmd_sz, required_ucmd_sz; 1161 + int err; 1162 + int underlay_qpn; 1163 + 1164 + if (udata && udata->inlen) { 1165 + min_ucmd_sz = offsetof(typeof(ucmd_hdr), reserved) + 1166 + sizeof(ucmd_hdr.reserved); 1167 + if (udata->inlen < min_ucmd_sz) 1168 + return ERR_PTR(-EOPNOTSUPP); 1169 + 1170 + err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz); 1171 + if (err) 1172 + return ERR_PTR(err); 1173 + 1174 + /* currently supports only one counters data */ 1175 + if (ucmd_hdr.ncounters_data > 1) 1176 + return ERR_PTR(-EINVAL); 1177 + 1178 + required_ucmd_sz = min_ucmd_sz + 1179 + sizeof(struct mlx5_ib_flow_counters_data) * 1180 + ucmd_hdr.ncounters_data; 1181 + if (udata->inlen > required_ucmd_sz && 1182 + !ib_is_udata_cleared(udata, required_ucmd_sz, 1183 + udata->inlen - required_ucmd_sz)) 1184 + return ERR_PTR(-EOPNOTSUPP); 1185 + 1186 + ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL); 1187 + if (!ucmd) 1188 + return ERR_PTR(-ENOMEM); 1189 + 1190 + err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz); 1191 + if (err) 1192 + goto free_ucmd; 1193 + } 1194 + 1195 + if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) { 1196 + err = -ENOMEM; 1197 + goto free_ucmd; 1198 + } 1199 + 1200 + if (domain != IB_FLOW_DOMAIN_USER || 1201 + flow_attr->port > dev->num_ports || 1202 + (flow_attr->flags & ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP | 1203 + IB_FLOW_ATTR_FLAGS_EGRESS))) { 1204 + err = -EINVAL; 1205 + goto free_ucmd; 1206 + } 1207 + 1208 + if (is_egress && 1209 + (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 1210 + flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { 1211 + err = -EINVAL; 1212 + goto free_ucmd; 1213 + } 1214 + 1215 + dst = kzalloc(sizeof(*dst), GFP_KERNEL); 1216 + if (!dst) { 1217 + err = -ENOMEM; 1218 + goto free_ucmd; 1219 + } 1220 + 1221 + mutex_lock(&dev->flow_db->lock); 1222 + 1223 + ft_prio = get_flow_table(dev, flow_attr, 1224 + is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX); 1225 + if (IS_ERR(ft_prio)) { 1226 + err = PTR_ERR(ft_prio); 1227 + goto unlock; 1228 + } 1229 + if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { 1230 + ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX); 1231 + if (IS_ERR(ft_prio_tx)) { 1232 + err = PTR_ERR(ft_prio_tx); 1233 + ft_prio_tx = NULL; 1234 + goto destroy_ft; 1235 + } 1236 + } 1237 + 1238 + if (is_egress) { 1239 + dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT; 1240 + } else { 1241 + dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR; 1242 + if (mqp->is_rss) 1243 + dst->tir_num = mqp->rss_qp.tirn; 1244 + else 1245 + dst->tir_num = mqp->raw_packet_qp.rq.tirn; 1246 + } 1247 + 1248 + if (flow_attr->type == IB_FLOW_ATTR_NORMAL) { 1249 + underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ? 1250 + mqp->underlay_qpn : 1251 + 0; 1252 + handler = _create_flow_rule(dev, ft_prio, flow_attr, dst, 1253 + underlay_qpn, ucmd); 1254 + } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 1255 + flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) { 1256 + handler = create_leftovers_rule(dev, ft_prio, flow_attr, 1257 + dst); 1258 + } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { 1259 + handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst); 1260 + } else { 1261 + err = -EINVAL; 1262 + goto destroy_ft; 1263 + } 1264 + 1265 + if (IS_ERR(handler)) { 1266 + err = PTR_ERR(handler); 1267 + handler = NULL; 1268 + goto destroy_ft; 1269 + } 1270 + 1271 + mutex_unlock(&dev->flow_db->lock); 1272 + kfree(dst); 1273 + kfree(ucmd); 1274 + 1275 + return &handler->ibflow; 1276 + 1277 + destroy_ft: 1278 + put_flow_table(dev, ft_prio, false); 1279 + if (ft_prio_tx) 1280 + put_flow_table(dev, ft_prio_tx, false); 1281 + unlock: 1282 + mutex_unlock(&dev->flow_db->lock); 1283 + kfree(dst); 1284 + free_ucmd: 1285 + kfree(ucmd); 1286 + return ERR_PTR(err); 1287 + } 1288 + 1289 + static struct mlx5_ib_flow_prio * 1290 + _get_flow_table(struct mlx5_ib_dev *dev, 1291 + struct mlx5_ib_flow_matcher *fs_matcher, 1292 + bool mcast) 1293 + { 1294 + struct mlx5_flow_namespace *ns = NULL; 1295 + struct mlx5_ib_flow_prio *prio = NULL; 1296 + int max_table_size = 0; 1297 + bool esw_encap; 1298 + u32 flags = 0; 1299 + int priority; 1300 + 1301 + if (mcast) 1302 + priority = MLX5_IB_FLOW_MCAST_PRIO; 1303 + else 1304 + priority = ib_prio_to_core_prio(fs_matcher->priority, false); 1305 + 1306 + esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != 1307 + DEVLINK_ESWITCH_ENCAP_MODE_NONE; 1308 + if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) { 1309 + max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 1310 + log_max_ft_size)); 1311 + if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap) 1312 + flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 1313 + if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 1314 + reformat_l3_tunnel_to_l2) && 1315 + !esw_encap) 1316 + flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1317 + } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) { 1318 + max_table_size = BIT( 1319 + MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size)); 1320 + if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && !esw_encap) 1321 + flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1322 + } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) { 1323 + max_table_size = BIT( 1324 + MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size)); 1325 + if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap) 1326 + flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 1327 + if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, reformat_l3_tunnel_to_l2) && 1328 + esw_encap) 1329 + flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 1330 + priority = FDB_BYPASS_PATH; 1331 + } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) { 1332 + max_table_size = 1333 + BIT(MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, 1334 + log_max_ft_size)); 1335 + priority = fs_matcher->priority; 1336 + } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) { 1337 + max_table_size = 1338 + BIT(MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, 1339 + log_max_ft_size)); 1340 + priority = fs_matcher->priority; 1341 + } 1342 + 1343 + max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES); 1344 + 1345 + ns = mlx5_get_flow_namespace(dev->mdev, fs_matcher->ns_type); 1346 + if (!ns) 1347 + return ERR_PTR(-EOPNOTSUPP); 1348 + 1349 + if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) 1350 + prio = &dev->flow_db->prios[priority]; 1351 + else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) 1352 + prio = &dev->flow_db->egress_prios[priority]; 1353 + else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) 1354 + prio = &dev->flow_db->fdb; 1355 + else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) 1356 + prio = &dev->flow_db->rdma_rx[priority]; 1357 + else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) 1358 + prio = &dev->flow_db->rdma_tx[priority]; 1359 + 1360 + if (!prio) 1361 + return ERR_PTR(-EINVAL); 1362 + 1363 + if (prio->flow_table) 1364 + return prio; 1365 + 1366 + return _get_prio(ns, prio, priority, max_table_size, 1367 + MLX5_FS_MAX_TYPES, flags); 1368 + } 1369 + 1370 + static struct mlx5_ib_flow_handler * 1371 + _create_raw_flow_rule(struct mlx5_ib_dev *dev, 1372 + struct mlx5_ib_flow_prio *ft_prio, 1373 + struct mlx5_flow_destination *dst, 1374 + struct mlx5_ib_flow_matcher *fs_matcher, 1375 + struct mlx5_flow_context *flow_context, 1376 + struct mlx5_flow_act *flow_act, 1377 + void *cmd_in, int inlen, 1378 + int dst_num) 1379 + { 1380 + struct mlx5_ib_flow_handler *handler; 1381 + struct mlx5_flow_spec *spec; 1382 + struct mlx5_flow_table *ft = ft_prio->flow_table; 1383 + int err = 0; 1384 + 1385 + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1386 + handler = kzalloc(sizeof(*handler), GFP_KERNEL); 1387 + if (!handler || !spec) { 1388 + err = -ENOMEM; 1389 + goto free; 1390 + } 1391 + 1392 + INIT_LIST_HEAD(&handler->list); 1393 + 1394 + memcpy(spec->match_value, cmd_in, inlen); 1395 + memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params, 1396 + fs_matcher->mask_len); 1397 + spec->match_criteria_enable = fs_matcher->match_criteria_enable; 1398 + spec->flow_context = *flow_context; 1399 + 1400 + handler->rule = mlx5_add_flow_rules(ft, spec, 1401 + flow_act, dst, dst_num); 1402 + 1403 + if (IS_ERR(handler->rule)) { 1404 + err = PTR_ERR(handler->rule); 1405 + goto free; 1406 + } 1407 + 1408 + ft_prio->refcount++; 1409 + handler->prio = ft_prio; 1410 + handler->dev = dev; 1411 + ft_prio->flow_table = ft; 1412 + 1413 + free: 1414 + if (err) 1415 + kfree(handler); 1416 + kvfree(spec); 1417 + return err ? ERR_PTR(err) : handler; 1418 + } 1419 + 1420 + static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher, 1421 + void *match_v) 1422 + { 1423 + void *match_c; 1424 + void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4; 1425 + void *dmac, *dmac_mask; 1426 + void *ipv4, *ipv4_mask; 1427 + 1428 + if (!(fs_matcher->match_criteria_enable & 1429 + (1 << MATCH_CRITERIA_ENABLE_OUTER_BIT))) 1430 + return false; 1431 + 1432 + match_c = fs_matcher->matcher_mask.match_params; 1433 + match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v, 1434 + outer_headers); 1435 + match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c, 1436 + outer_headers); 1437 + 1438 + dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, 1439 + dmac_47_16); 1440 + dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, 1441 + dmac_47_16); 1442 + 1443 + if (is_multicast_ether_addr(dmac) && 1444 + is_multicast_ether_addr(dmac_mask)) 1445 + return true; 1446 + 1447 + ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, 1448 + dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 1449 + 1450 + ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, 1451 + dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 1452 + 1453 + if (ipv4_is_multicast(*(__be32 *)(ipv4)) && 1454 + ipv4_is_multicast(*(__be32 *)(ipv4_mask))) 1455 + return true; 1456 + 1457 + return false; 1458 + } 1459 + 1460 + static struct mlx5_ib_flow_handler *raw_fs_rule_add( 1461 + struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher, 1462 + struct mlx5_flow_context *flow_context, struct mlx5_flow_act *flow_act, 1463 + u32 counter_id, void *cmd_in, int inlen, int dest_id, int dest_type) 1464 + { 1465 + struct mlx5_flow_destination *dst; 1466 + struct mlx5_ib_flow_prio *ft_prio; 1467 + struct mlx5_ib_flow_handler *handler; 1468 + int dst_num = 0; 1469 + bool mcast; 1470 + int err; 1471 + 1472 + if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL) 1473 + return ERR_PTR(-EOPNOTSUPP); 1474 + 1475 + if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO) 1476 + return ERR_PTR(-ENOMEM); 1477 + 1478 + dst = kcalloc(2, sizeof(*dst), GFP_KERNEL); 1479 + if (!dst) 1480 + return ERR_PTR(-ENOMEM); 1481 + 1482 + mcast = raw_fs_is_multicast(fs_matcher, cmd_in); 1483 + mutex_lock(&dev->flow_db->lock); 1484 + 1485 + ft_prio = _get_flow_table(dev, fs_matcher, mcast); 1486 + if (IS_ERR(ft_prio)) { 1487 + err = PTR_ERR(ft_prio); 1488 + goto unlock; 1489 + } 1490 + 1491 + if (dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR) { 1492 + dst[dst_num].type = dest_type; 1493 + dst[dst_num++].tir_num = dest_id; 1494 + flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1495 + } else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) { 1496 + dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM; 1497 + dst[dst_num++].ft_num = dest_id; 1498 + flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1499 + } else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_PORT) { 1500 + dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT; 1501 + flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; 1502 + } 1503 + 1504 + 1505 + if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 1506 + dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1507 + dst[dst_num].counter_id = counter_id; 1508 + dst_num++; 1509 + } 1510 + 1511 + handler = _create_raw_flow_rule(dev, ft_prio, dst, fs_matcher, 1512 + flow_context, flow_act, 1513 + cmd_in, inlen, dst_num); 1514 + 1515 + if (IS_ERR(handler)) { 1516 + err = PTR_ERR(handler); 1517 + goto destroy_ft; 1518 + } 1519 + 1520 + mutex_unlock(&dev->flow_db->lock); 1521 + atomic_inc(&fs_matcher->usecnt); 1522 + handler->flow_matcher = fs_matcher; 1523 + 1524 + kfree(dst); 1525 + 1526 + return handler; 1527 + 1528 + destroy_ft: 1529 + put_flow_table(dev, ft_prio, false); 1530 + unlock: 1531 + mutex_unlock(&dev->flow_db->lock); 1532 + kfree(dst); 1533 + 1534 + return ERR_PTR(err); 1535 + } 1536 + 1537 + static u32 mlx5_ib_flow_action_flags_to_accel_xfrm_flags(u32 mlx5_flags) 1538 + { 1539 + u32 flags = 0; 1540 + 1541 + if (mlx5_flags & MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA) 1542 + flags |= MLX5_ACCEL_XFRM_FLAG_REQUIRE_METADATA; 1543 + 1544 + return flags; 1545 + } 1546 + 1547 + #define MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED \ 1548 + MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA 1549 + static struct ib_flow_action * 1550 + mlx5_ib_create_flow_action_esp(struct ib_device *device, 1551 + const struct ib_flow_action_attrs_esp *attr, 1552 + struct uverbs_attr_bundle *attrs) 1553 + { 1554 + struct mlx5_ib_dev *mdev = to_mdev(device); 1555 + struct ib_uverbs_flow_action_esp_keymat_aes_gcm *aes_gcm; 1556 + struct mlx5_accel_esp_xfrm_attrs accel_attrs = {}; 1557 + struct mlx5_ib_flow_action *action; 1558 + u64 action_flags; 1559 + u64 flags; 1560 + int err = 0; 1561 + 1562 + err = uverbs_get_flags64( 1563 + &action_flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS, 1564 + ((MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED << 1) - 1)); 1565 + if (err) 1566 + return ERR_PTR(err); 1567 + 1568 + flags = mlx5_ib_flow_action_flags_to_accel_xfrm_flags(action_flags); 1569 + 1570 + /* We current only support a subset of the standard features. Only a 1571 + * keymat of type AES_GCM, with icv_len == 16, iv_algo == SEQ and esn 1572 + * (with overlap). Full offload mode isn't supported. 1573 + */ 1574 + if (!attr->keymat || attr->replay || attr->encap || 1575 + attr->spi || attr->seq || attr->tfc_pad || 1576 + attr->hard_limit_pkts || 1577 + (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED | 1578 + IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT))) 1579 + return ERR_PTR(-EOPNOTSUPP); 1580 + 1581 + if (attr->keymat->protocol != 1582 + IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM) 1583 + return ERR_PTR(-EOPNOTSUPP); 1584 + 1585 + aes_gcm = &attr->keymat->keymat.aes_gcm; 1586 + 1587 + if (aes_gcm->icv_len != 16 || 1588 + aes_gcm->iv_algo != IB_UVERBS_FLOW_ACTION_IV_ALGO_SEQ) 1589 + return ERR_PTR(-EOPNOTSUPP); 1590 + 1591 + action = kmalloc(sizeof(*action), GFP_KERNEL); 1592 + if (!action) 1593 + return ERR_PTR(-ENOMEM); 1594 + 1595 + action->esp_aes_gcm.ib_flags = attr->flags; 1596 + memcpy(&accel_attrs.keymat.aes_gcm.aes_key, &aes_gcm->aes_key, 1597 + sizeof(accel_attrs.keymat.aes_gcm.aes_key)); 1598 + accel_attrs.keymat.aes_gcm.key_len = aes_gcm->key_len * 8; 1599 + memcpy(&accel_attrs.keymat.aes_gcm.salt, &aes_gcm->salt, 1600 + sizeof(accel_attrs.keymat.aes_gcm.salt)); 1601 + memcpy(&accel_attrs.keymat.aes_gcm.seq_iv, &aes_gcm->iv, 1602 + sizeof(accel_attrs.keymat.aes_gcm.seq_iv)); 1603 + accel_attrs.keymat.aes_gcm.icv_len = aes_gcm->icv_len * 8; 1604 + accel_attrs.keymat.aes_gcm.iv_algo = MLX5_ACCEL_ESP_AES_GCM_IV_ALGO_SEQ; 1605 + accel_attrs.keymat_type = MLX5_ACCEL_ESP_KEYMAT_AES_GCM; 1606 + 1607 + accel_attrs.esn = attr->esn; 1608 + if (attr->flags & IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED) 1609 + accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED; 1610 + if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW) 1611 + accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; 1612 + 1613 + if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT) 1614 + accel_attrs.action |= MLX5_ACCEL_ESP_ACTION_ENCRYPT; 1615 + 1616 + action->esp_aes_gcm.ctx = 1617 + mlx5_accel_esp_create_xfrm(mdev->mdev, &accel_attrs, flags); 1618 + if (IS_ERR(action->esp_aes_gcm.ctx)) { 1619 + err = PTR_ERR(action->esp_aes_gcm.ctx); 1620 + goto err_parse; 1621 + } 1622 + 1623 + action->esp_aes_gcm.ib_flags = attr->flags; 1624 + 1625 + return &action->ib_action; 1626 + 1627 + err_parse: 1628 + kfree(action); 1629 + return ERR_PTR(err); 1630 + } 1631 + 1632 + static int 1633 + mlx5_ib_modify_flow_action_esp(struct ib_flow_action *action, 1634 + const struct ib_flow_action_attrs_esp *attr, 1635 + struct uverbs_attr_bundle *attrs) 1636 + { 1637 + struct mlx5_ib_flow_action *maction = to_mflow_act(action); 1638 + struct mlx5_accel_esp_xfrm_attrs accel_attrs; 1639 + int err = 0; 1640 + 1641 + if (attr->keymat || attr->replay || attr->encap || 1642 + attr->spi || attr->seq || attr->tfc_pad || 1643 + attr->hard_limit_pkts || 1644 + (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED | 1645 + IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS | 1646 + IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW))) 1647 + return -EOPNOTSUPP; 1648 + 1649 + /* Only the ESN value or the MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP can 1650 + * be modified. 1651 + */ 1652 + if (!(maction->esp_aes_gcm.ib_flags & 1653 + IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED) && 1654 + attr->flags & (IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED | 1655 + IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)) 1656 + return -EINVAL; 1657 + 1658 + memcpy(&accel_attrs, &maction->esp_aes_gcm.ctx->attrs, 1659 + sizeof(accel_attrs)); 1660 + 1661 + accel_attrs.esn = attr->esn; 1662 + if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW) 1663 + accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; 1664 + else 1665 + accel_attrs.flags &= ~MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; 1666 + 1667 + err = mlx5_accel_esp_modify_xfrm(maction->esp_aes_gcm.ctx, 1668 + &accel_attrs); 1669 + if (err) 1670 + return err; 1671 + 1672 + maction->esp_aes_gcm.ib_flags &= 1673 + ~IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW; 1674 + maction->esp_aes_gcm.ib_flags |= 1675 + attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW; 1676 + 1677 + return 0; 1678 + } 1679 + 1680 + static void destroy_flow_action_raw(struct mlx5_ib_flow_action *maction) 1681 + { 1682 + switch (maction->flow_action_raw.sub_type) { 1683 + case MLX5_IB_FLOW_ACTION_MODIFY_HEADER: 1684 + mlx5_modify_header_dealloc(maction->flow_action_raw.dev->mdev, 1685 + maction->flow_action_raw.modify_hdr); 1686 + break; 1687 + case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT: 1688 + mlx5_packet_reformat_dealloc(maction->flow_action_raw.dev->mdev, 1689 + maction->flow_action_raw.pkt_reformat); 1690 + break; 1691 + case MLX5_IB_FLOW_ACTION_DECAP: 1692 + break; 1693 + default: 1694 + break; 1695 + } 1696 + } 1697 + 1698 + static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action) 1699 + { 1700 + struct mlx5_ib_flow_action *maction = to_mflow_act(action); 1701 + 1702 + switch (action->type) { 1703 + case IB_FLOW_ACTION_ESP: 1704 + /* 1705 + * We only support aes_gcm by now, so we implicitly know this is 1706 + * the underline crypto. 1707 + */ 1708 + mlx5_accel_esp_destroy_xfrm(maction->esp_aes_gcm.ctx); 1709 + break; 1710 + case IB_FLOW_ACTION_UNSPECIFIED: 1711 + destroy_flow_action_raw(maction); 1712 + break; 1713 + default: 1714 + WARN_ON(true); 1715 + break; 1716 + } 1717 + 1718 + kfree(maction); 1719 + return 0; 1720 + } 1721 + 1722 + static int 1723 + mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type, 1724 + enum mlx5_flow_namespace_type *namespace) 1725 + { 1726 + switch (table_type) { 1727 + case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX: 1728 + *namespace = MLX5_FLOW_NAMESPACE_BYPASS; 1729 + break; 1730 + case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX: 1731 + *namespace = MLX5_FLOW_NAMESPACE_EGRESS; 1732 + break; 1733 + case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB: 1734 + *namespace = MLX5_FLOW_NAMESPACE_FDB; 1735 + break; 1736 + case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX: 1737 + *namespace = MLX5_FLOW_NAMESPACE_RDMA_RX; 1738 + break; 1739 + case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX: 1740 + *namespace = MLX5_FLOW_NAMESPACE_RDMA_TX; 1741 + break; 1742 + default: 1743 + return -EINVAL; 1744 + } 1745 + 1746 + return 0; 1747 + } 1748 + 1749 + static const struct uverbs_attr_spec mlx5_ib_flow_type[] = { 1750 + [MLX5_IB_FLOW_TYPE_NORMAL] = { 1751 + .type = UVERBS_ATTR_TYPE_PTR_IN, 1752 + .u.ptr = { 1753 + .len = sizeof(u16), /* data is priority */ 1754 + .min_len = sizeof(u16), 1755 + } 1756 + }, 1757 + [MLX5_IB_FLOW_TYPE_SNIFFER] = { 1758 + .type = UVERBS_ATTR_TYPE_PTR_IN, 1759 + UVERBS_ATTR_NO_DATA(), 1760 + }, 1761 + [MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = { 1762 + .type = UVERBS_ATTR_TYPE_PTR_IN, 1763 + UVERBS_ATTR_NO_DATA(), 1764 + }, 1765 + [MLX5_IB_FLOW_TYPE_MC_DEFAULT] = { 1766 + .type = UVERBS_ATTR_TYPE_PTR_IN, 1767 + UVERBS_ATTR_NO_DATA(), 1768 + }, 1769 + }; 1770 + 1771 + static bool is_flow_dest(void *obj, int *dest_id, int *dest_type) 1772 + { 1773 + struct devx_obj *devx_obj = obj; 1774 + u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 1775 + 1776 + switch (opcode) { 1777 + case MLX5_CMD_OP_DESTROY_TIR: 1778 + *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; 1779 + *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, 1780 + obj_id); 1781 + return true; 1782 + 1783 + case MLX5_CMD_OP_DESTROY_FLOW_TABLE: 1784 + *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1785 + *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox, 1786 + table_id); 1787 + return true; 1788 + default: 1789 + return false; 1790 + } 1791 + } 1792 + 1793 + static int get_dests(struct uverbs_attr_bundle *attrs, 1794 + struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id, 1795 + int *dest_type, struct ib_qp **qp, u32 *flags) 1796 + { 1797 + bool dest_devx, dest_qp; 1798 + void *devx_obj; 1799 + int err; 1800 + 1801 + dest_devx = uverbs_attr_is_valid(attrs, 1802 + MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); 1803 + dest_qp = uverbs_attr_is_valid(attrs, 1804 + MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); 1805 + 1806 + *flags = 0; 1807 + err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS, 1808 + MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS | 1809 + MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP); 1810 + if (err) 1811 + return err; 1812 + 1813 + /* Both flags are not allowed */ 1814 + if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS && 1815 + *flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) 1816 + return -EINVAL; 1817 + 1818 + if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) { 1819 + if (dest_devx && (dest_qp || *flags)) 1820 + return -EINVAL; 1821 + else if (dest_qp && *flags) 1822 + return -EINVAL; 1823 + } 1824 + 1825 + /* Allow only DEVX object, drop as dest for FDB */ 1826 + if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB && !(dest_devx || 1827 + (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP))) 1828 + return -EINVAL; 1829 + 1830 + /* Allow only DEVX object or QP as dest when inserting to RDMA_RX */ 1831 + if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) && 1832 + ((!dest_devx && !dest_qp) || (dest_devx && dest_qp))) 1833 + return -EINVAL; 1834 + 1835 + *qp = NULL; 1836 + if (dest_devx) { 1837 + devx_obj = 1838 + uverbs_attr_get_obj(attrs, 1839 + MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX); 1840 + 1841 + /* Verify that the given DEVX object is a flow 1842 + * steering destination. 1843 + */ 1844 + if (!is_flow_dest(devx_obj, dest_id, dest_type)) 1845 + return -EINVAL; 1846 + /* Allow only flow table as dest when inserting to FDB or RDMA_RX */ 1847 + if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB || 1848 + fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) && 1849 + *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) 1850 + return -EINVAL; 1851 + } else if (dest_qp) { 1852 + struct mlx5_ib_qp *mqp; 1853 + 1854 + *qp = uverbs_attr_get_obj(attrs, 1855 + MLX5_IB_ATTR_CREATE_FLOW_DEST_QP); 1856 + if (IS_ERR(*qp)) 1857 + return PTR_ERR(*qp); 1858 + 1859 + if ((*qp)->qp_type != IB_QPT_RAW_PACKET) 1860 + return -EINVAL; 1861 + 1862 + mqp = to_mqp(*qp); 1863 + if (mqp->is_rss) 1864 + *dest_id = mqp->rss_qp.tirn; 1865 + else 1866 + *dest_id = mqp->raw_packet_qp.rq.tirn; 1867 + *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR; 1868 + } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) { 1869 + *dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT; 1870 + } 1871 + 1872 + if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR && 1873 + fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) 1874 + return -EINVAL; 1875 + 1876 + return 0; 1877 + } 1878 + 1879 + static bool is_flow_counter(void *obj, u32 offset, u32 *counter_id) 1880 + { 1881 + struct devx_obj *devx_obj = obj; 1882 + u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode); 1883 + 1884 + if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) { 1885 + 1886 + if (offset && offset >= devx_obj->flow_counter_bulk_size) 1887 + return false; 1888 + 1889 + *counter_id = MLX5_GET(dealloc_flow_counter_in, 1890 + devx_obj->dinbox, 1891 + flow_counter_id); 1892 + *counter_id += offset; 1893 + return true; 1894 + } 1895 + 1896 + return false; 1897 + } 1898 + 1899 + #define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2 1900 + static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)( 1901 + struct uverbs_attr_bundle *attrs) 1902 + { 1903 + struct mlx5_flow_context flow_context = {.flow_tag = 1904 + MLX5_FS_DEFAULT_FLOW_TAG}; 1905 + u32 *offset_attr, offset = 0, counter_id = 0; 1906 + int dest_id, dest_type, inlen, len, ret, i; 1907 + struct mlx5_ib_flow_handler *flow_handler; 1908 + struct mlx5_ib_flow_matcher *fs_matcher; 1909 + struct ib_uobject **arr_flow_actions; 1910 + struct ib_uflow_resources *uflow_res; 1911 + struct mlx5_flow_act flow_act = {}; 1912 + struct ib_qp *qp = NULL; 1913 + void *devx_obj, *cmd_in; 1914 + struct ib_uobject *uobj; 1915 + struct mlx5_ib_dev *dev; 1916 + u32 flags; 1917 + 1918 + if (!capable(CAP_NET_RAW)) 1919 + return -EPERM; 1920 + 1921 + fs_matcher = uverbs_attr_get_obj(attrs, 1922 + MLX5_IB_ATTR_CREATE_FLOW_MATCHER); 1923 + uobj = uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE); 1924 + dev = mlx5_udata_to_mdev(&attrs->driver_udata); 1925 + 1926 + if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags)) 1927 + return -EINVAL; 1928 + 1929 + if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS) 1930 + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS; 1931 + 1932 + if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP) 1933 + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 1934 + 1935 + len = uverbs_attr_get_uobjs_arr(attrs, 1936 + MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions); 1937 + if (len) { 1938 + devx_obj = arr_flow_actions[0]->object; 1939 + 1940 + if (uverbs_attr_is_valid(attrs, 1941 + MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) { 1942 + 1943 + int num_offsets = uverbs_attr_ptr_get_array_size( 1944 + attrs, 1945 + MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, 1946 + sizeof(u32)); 1947 + 1948 + if (num_offsets != 1) 1949 + return -EINVAL; 1950 + 1951 + offset_attr = uverbs_attr_get_alloced_ptr( 1952 + attrs, 1953 + MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET); 1954 + offset = *offset_attr; 1955 + } 1956 + 1957 + if (!is_flow_counter(devx_obj, offset, &counter_id)) 1958 + return -EINVAL; 1959 + 1960 + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 1961 + } 1962 + 1963 + cmd_in = uverbs_attr_get_alloced_ptr( 1964 + attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); 1965 + inlen = uverbs_attr_get_len(attrs, 1966 + MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE); 1967 + 1968 + uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS); 1969 + if (!uflow_res) 1970 + return -ENOMEM; 1971 + 1972 + len = uverbs_attr_get_uobjs_arr(attrs, 1973 + MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &arr_flow_actions); 1974 + for (i = 0; i < len; i++) { 1975 + struct mlx5_ib_flow_action *maction = 1976 + to_mflow_act(arr_flow_actions[i]->object); 1977 + 1978 + ret = parse_flow_flow_action(maction, false, &flow_act); 1979 + if (ret) 1980 + goto err_out; 1981 + flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE, 1982 + arr_flow_actions[i]->object); 1983 + } 1984 + 1985 + ret = uverbs_copy_from(&flow_context.flow_tag, attrs, 1986 + MLX5_IB_ATTR_CREATE_FLOW_TAG); 1987 + if (!ret) { 1988 + if (flow_context.flow_tag >= BIT(24)) { 1989 + ret = -EINVAL; 1990 + goto err_out; 1991 + } 1992 + flow_context.flags |= FLOW_CONTEXT_HAS_TAG; 1993 + } 1994 + 1995 + flow_handler = 1996 + raw_fs_rule_add(dev, fs_matcher, &flow_context, &flow_act, 1997 + counter_id, cmd_in, inlen, dest_id, dest_type); 1998 + if (IS_ERR(flow_handler)) { 1999 + ret = PTR_ERR(flow_handler); 2000 + goto err_out; 2001 + } 2002 + 2003 + ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res); 2004 + 2005 + return 0; 2006 + err_out: 2007 + ib_uverbs_flow_resources_free(uflow_res); 2008 + return ret; 2009 + } 2010 + 2011 + static int flow_matcher_cleanup(struct ib_uobject *uobject, 2012 + enum rdma_remove_reason why, 2013 + struct uverbs_attr_bundle *attrs) 2014 + { 2015 + struct mlx5_ib_flow_matcher *obj = uobject->object; 2016 + int ret; 2017 + 2018 + ret = ib_destroy_usecnt(&obj->usecnt, why, uobject); 2019 + if (ret) 2020 + return ret; 2021 + 2022 + kfree(obj); 2023 + return 0; 2024 + } 2025 + 2026 + static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs, 2027 + struct mlx5_ib_flow_matcher *obj) 2028 + { 2029 + enum mlx5_ib_uapi_flow_table_type ft_type = 2030 + MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX; 2031 + u32 flags; 2032 + int err; 2033 + 2034 + /* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older 2035 + * users should switch to it. We leave this to not break userspace 2036 + */ 2037 + if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) && 2038 + uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) 2039 + return -EINVAL; 2040 + 2041 + if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) { 2042 + err = uverbs_get_const(&ft_type, attrs, 2043 + MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE); 2044 + if (err) 2045 + return err; 2046 + 2047 + err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type); 2048 + if (err) 2049 + return err; 2050 + 2051 + return 0; 2052 + } 2053 + 2054 + if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) { 2055 + err = uverbs_get_flags32(&flags, attrs, 2056 + MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, 2057 + IB_FLOW_ATTR_FLAGS_EGRESS); 2058 + if (err) 2059 + return err; 2060 + 2061 + if (flags) { 2062 + mlx5_ib_ft_type_to_namespace( 2063 + MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX, 2064 + &obj->ns_type); 2065 + return 0; 2066 + } 2067 + } 2068 + 2069 + obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS; 2070 + 2071 + return 0; 2072 + } 2073 + 2074 + static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)( 2075 + struct uverbs_attr_bundle *attrs) 2076 + { 2077 + struct ib_uobject *uobj = uverbs_attr_get_uobject( 2078 + attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE); 2079 + struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata); 2080 + struct mlx5_ib_flow_matcher *obj; 2081 + int err; 2082 + 2083 + obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL); 2084 + if (!obj) 2085 + return -ENOMEM; 2086 + 2087 + obj->mask_len = uverbs_attr_get_len( 2088 + attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); 2089 + err = uverbs_copy_from(&obj->matcher_mask, 2090 + attrs, 2091 + MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK); 2092 + if (err) 2093 + goto end; 2094 + 2095 + obj->flow_type = uverbs_attr_get_enum_id( 2096 + attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); 2097 + 2098 + if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) { 2099 + err = uverbs_copy_from(&obj->priority, 2100 + attrs, 2101 + MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE); 2102 + if (err) 2103 + goto end; 2104 + } 2105 + 2106 + err = uverbs_copy_from(&obj->match_criteria_enable, 2107 + attrs, 2108 + MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA); 2109 + if (err) 2110 + goto end; 2111 + 2112 + err = mlx5_ib_matcher_ns(attrs, obj); 2113 + if (err) 2114 + goto end; 2115 + 2116 + uobj->object = obj; 2117 + obj->mdev = dev->mdev; 2118 + atomic_set(&obj->usecnt, 0); 2119 + return 0; 2120 + 2121 + end: 2122 + kfree(obj); 2123 + return err; 2124 + } 2125 + 2126 + static struct ib_flow_action * 2127 + mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev, 2128 + enum mlx5_ib_uapi_flow_table_type ft_type, 2129 + u8 num_actions, void *in) 2130 + { 2131 + enum mlx5_flow_namespace_type namespace; 2132 + struct mlx5_ib_flow_action *maction; 2133 + int ret; 2134 + 2135 + ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); 2136 + if (ret) 2137 + return ERR_PTR(-EINVAL); 2138 + 2139 + maction = kzalloc(sizeof(*maction), GFP_KERNEL); 2140 + if (!maction) 2141 + return ERR_PTR(-ENOMEM); 2142 + 2143 + maction->flow_action_raw.modify_hdr = 2144 + mlx5_modify_header_alloc(dev->mdev, namespace, num_actions, in); 2145 + 2146 + if (IS_ERR(maction->flow_action_raw.modify_hdr)) { 2147 + ret = PTR_ERR(maction->flow_action_raw.modify_hdr); 2148 + kfree(maction); 2149 + return ERR_PTR(ret); 2150 + } 2151 + maction->flow_action_raw.sub_type = 2152 + MLX5_IB_FLOW_ACTION_MODIFY_HEADER; 2153 + maction->flow_action_raw.dev = dev; 2154 + 2155 + return &maction->ib_action; 2156 + } 2157 + 2158 + static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev) 2159 + { 2160 + return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 2161 + max_modify_header_actions) || 2162 + MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, 2163 + max_modify_header_actions) || 2164 + MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, 2165 + max_modify_header_actions); 2166 + } 2167 + 2168 + static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)( 2169 + struct uverbs_attr_bundle *attrs) 2170 + { 2171 + struct ib_uobject *uobj = uverbs_attr_get_uobject( 2172 + attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE); 2173 + struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); 2174 + enum mlx5_ib_uapi_flow_table_type ft_type; 2175 + struct ib_flow_action *action; 2176 + int num_actions; 2177 + void *in; 2178 + int ret; 2179 + 2180 + if (!mlx5_ib_modify_header_supported(mdev)) 2181 + return -EOPNOTSUPP; 2182 + 2183 + in = uverbs_attr_get_alloced_ptr(attrs, 2184 + MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM); 2185 + 2186 + num_actions = uverbs_attr_ptr_get_array_size( 2187 + attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, 2188 + MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)); 2189 + if (num_actions < 0) 2190 + return num_actions; 2191 + 2192 + ret = uverbs_get_const(&ft_type, attrs, 2193 + MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE); 2194 + if (ret) 2195 + return ret; 2196 + action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in); 2197 + if (IS_ERR(action)) 2198 + return PTR_ERR(action); 2199 + 2200 + uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev, 2201 + IB_FLOW_ACTION_UNSPECIFIED); 2202 + 2203 + return 0; 2204 + } 2205 + 2206 + static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev, 2207 + u8 packet_reformat_type, 2208 + u8 ft_type) 2209 + { 2210 + switch (packet_reformat_type) { 2211 + case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: 2212 + if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) 2213 + return MLX5_CAP_FLOWTABLE(ibdev->mdev, 2214 + encap_general_header); 2215 + break; 2216 + case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: 2217 + if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX) 2218 + return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev, 2219 + reformat_l2_to_l3_tunnel); 2220 + break; 2221 + case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: 2222 + if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) 2223 + return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, 2224 + reformat_l3_tunnel_to_l2); 2225 + break; 2226 + case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2: 2227 + if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX) 2228 + return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap); 2229 + break; 2230 + default: 2231 + break; 2232 + } 2233 + 2234 + return false; 2235 + } 2236 + 2237 + static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt) 2238 + { 2239 + switch (dv_prt) { 2240 + case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL: 2241 + *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL; 2242 + break; 2243 + case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2: 2244 + *prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2; 2245 + break; 2246 + case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL: 2247 + *prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL; 2248 + break; 2249 + default: 2250 + return -EINVAL; 2251 + } 2252 + 2253 + return 0; 2254 + } 2255 + 2256 + static int mlx5_ib_flow_action_create_packet_reformat_ctx( 2257 + struct mlx5_ib_dev *dev, 2258 + struct mlx5_ib_flow_action *maction, 2259 + u8 ft_type, u8 dv_prt, 2260 + void *in, size_t len) 2261 + { 2262 + enum mlx5_flow_namespace_type namespace; 2263 + u8 prm_prt; 2264 + int ret; 2265 + 2266 + ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace); 2267 + if (ret) 2268 + return ret; 2269 + 2270 + ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &prm_prt); 2271 + if (ret) 2272 + return ret; 2273 + 2274 + maction->flow_action_raw.pkt_reformat = 2275 + mlx5_packet_reformat_alloc(dev->mdev, prm_prt, len, 2276 + in, namespace); 2277 + if (IS_ERR(maction->flow_action_raw.pkt_reformat)) { 2278 + ret = PTR_ERR(maction->flow_action_raw.pkt_reformat); 2279 + return ret; 2280 + } 2281 + 2282 + maction->flow_action_raw.sub_type = 2283 + MLX5_IB_FLOW_ACTION_PACKET_REFORMAT; 2284 + maction->flow_action_raw.dev = dev; 2285 + 2286 + return 0; 2287 + } 2288 + 2289 + static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)( 2290 + struct uverbs_attr_bundle *attrs) 2291 + { 2292 + struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, 2293 + MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE); 2294 + struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata); 2295 + enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt; 2296 + enum mlx5_ib_uapi_flow_table_type ft_type; 2297 + struct mlx5_ib_flow_action *maction; 2298 + int ret; 2299 + 2300 + ret = uverbs_get_const(&ft_type, attrs, 2301 + MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE); 2302 + if (ret) 2303 + return ret; 2304 + 2305 + ret = uverbs_get_const(&dv_prt, attrs, 2306 + MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE); 2307 + if (ret) 2308 + return ret; 2309 + 2310 + if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type)) 2311 + return -EOPNOTSUPP; 2312 + 2313 + maction = kzalloc(sizeof(*maction), GFP_KERNEL); 2314 + if (!maction) 2315 + return -ENOMEM; 2316 + 2317 + if (dv_prt == 2318 + MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) { 2319 + maction->flow_action_raw.sub_type = 2320 + MLX5_IB_FLOW_ACTION_DECAP; 2321 + maction->flow_action_raw.dev = mdev; 2322 + } else { 2323 + void *in; 2324 + int len; 2325 + 2326 + in = uverbs_attr_get_alloced_ptr(attrs, 2327 + MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); 2328 + if (IS_ERR(in)) { 2329 + ret = PTR_ERR(in); 2330 + goto free_maction; 2331 + } 2332 + 2333 + len = uverbs_attr_get_len(attrs, 2334 + MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF); 2335 + 2336 + ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev, 2337 + maction, ft_type, dv_prt, in, len); 2338 + if (ret) 2339 + goto free_maction; 2340 + } 2341 + 2342 + uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev, 2343 + IB_FLOW_ACTION_UNSPECIFIED); 2344 + return 0; 2345 + 2346 + free_maction: 2347 + kfree(maction); 2348 + return ret; 2349 + } 2350 + 2351 + DECLARE_UVERBS_NAMED_METHOD( 2352 + MLX5_IB_METHOD_CREATE_FLOW, 2353 + UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, 2354 + UVERBS_OBJECT_FLOW, 2355 + UVERBS_ACCESS_NEW, 2356 + UA_MANDATORY), 2357 + UVERBS_ATTR_PTR_IN( 2358 + MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE, 2359 + UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), 2360 + UA_MANDATORY, 2361 + UA_ALLOC_AND_COPY), 2362 + UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER, 2363 + MLX5_IB_OBJECT_FLOW_MATCHER, 2364 + UVERBS_ACCESS_READ, 2365 + UA_MANDATORY), 2366 + UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP, 2367 + UVERBS_OBJECT_QP, 2368 + UVERBS_ACCESS_READ), 2369 + UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX, 2370 + MLX5_IB_OBJECT_DEVX_OBJ, 2371 + UVERBS_ACCESS_READ), 2372 + UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, 2373 + UVERBS_OBJECT_FLOW_ACTION, 2374 + UVERBS_ACCESS_READ, 1, 2375 + MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS, 2376 + UA_OPTIONAL), 2377 + UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG, 2378 + UVERBS_ATTR_TYPE(u32), 2379 + UA_OPTIONAL), 2380 + UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, 2381 + MLX5_IB_OBJECT_DEVX_OBJ, 2382 + UVERBS_ACCESS_READ, 1, 1, 2383 + UA_OPTIONAL), 2384 + UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET, 2385 + UVERBS_ATTR_MIN_SIZE(sizeof(u32)), 2386 + UA_OPTIONAL, 2387 + UA_ALLOC_AND_COPY), 2388 + UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS, 2389 + enum mlx5_ib_create_flow_flags, 2390 + UA_OPTIONAL)); 2391 + 2392 + DECLARE_UVERBS_NAMED_METHOD_DESTROY( 2393 + MLX5_IB_METHOD_DESTROY_FLOW, 2394 + UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE, 2395 + UVERBS_OBJECT_FLOW, 2396 + UVERBS_ACCESS_DESTROY, 2397 + UA_MANDATORY)); 2398 + 2399 + ADD_UVERBS_METHODS(mlx5_ib_fs, 2400 + UVERBS_OBJECT_FLOW, 2401 + &UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW), 2402 + &UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW)); 2403 + 2404 + DECLARE_UVERBS_NAMED_METHOD( 2405 + MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER, 2406 + UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE, 2407 + UVERBS_OBJECT_FLOW_ACTION, 2408 + UVERBS_ACCESS_NEW, 2409 + UA_MANDATORY), 2410 + UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM, 2411 + UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES( 2412 + set_add_copy_action_in_auto)), 2413 + UA_MANDATORY, 2414 + UA_ALLOC_AND_COPY), 2415 + UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE, 2416 + enum mlx5_ib_uapi_flow_table_type, 2417 + UA_MANDATORY)); 2418 + 2419 + DECLARE_UVERBS_NAMED_METHOD( 2420 + MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT, 2421 + UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE, 2422 + UVERBS_OBJECT_FLOW_ACTION, 2423 + UVERBS_ACCESS_NEW, 2424 + UA_MANDATORY), 2425 + UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF, 2426 + UVERBS_ATTR_MIN_SIZE(1), 2427 + UA_ALLOC_AND_COPY, 2428 + UA_OPTIONAL), 2429 + UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE, 2430 + enum mlx5_ib_uapi_flow_action_packet_reformat_type, 2431 + UA_MANDATORY), 2432 + UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE, 2433 + enum mlx5_ib_uapi_flow_table_type, 2434 + UA_MANDATORY)); 2435 + 2436 + ADD_UVERBS_METHODS( 2437 + mlx5_ib_flow_actions, 2438 + UVERBS_OBJECT_FLOW_ACTION, 2439 + &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER), 2440 + &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)); 2441 + 2442 + DECLARE_UVERBS_NAMED_METHOD( 2443 + MLX5_IB_METHOD_FLOW_MATCHER_CREATE, 2444 + UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE, 2445 + MLX5_IB_OBJECT_FLOW_MATCHER, 2446 + UVERBS_ACCESS_NEW, 2447 + UA_MANDATORY), 2448 + UVERBS_ATTR_PTR_IN( 2449 + MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK, 2450 + UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)), 2451 + UA_MANDATORY), 2452 + UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE, 2453 + mlx5_ib_flow_type, 2454 + UA_MANDATORY), 2455 + UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA, 2456 + UVERBS_ATTR_TYPE(u8), 2457 + UA_MANDATORY), 2458 + UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS, 2459 + enum ib_flow_flags, 2460 + UA_OPTIONAL), 2461 + UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE, 2462 + enum mlx5_ib_uapi_flow_table_type, 2463 + UA_OPTIONAL)); 2464 + 2465 + DECLARE_UVERBS_NAMED_METHOD_DESTROY( 2466 + MLX5_IB_METHOD_FLOW_MATCHER_DESTROY, 2467 + UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE, 2468 + MLX5_IB_OBJECT_FLOW_MATCHER, 2469 + UVERBS_ACCESS_DESTROY, 2470 + UA_MANDATORY)); 2471 + 2472 + DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER, 2473 + UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup), 2474 + &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE), 2475 + &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY)); 2476 + 2477 + const struct uapi_definition mlx5_ib_flow_defs[] = { 2478 + UAPI_DEF_CHAIN_OBJ_TREE_NAMED( 2479 + MLX5_IB_OBJECT_FLOW_MATCHER), 2480 + UAPI_DEF_CHAIN_OBJ_TREE( 2481 + UVERBS_OBJECT_FLOW, 2482 + &mlx5_ib_fs), 2483 + UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION, 2484 + &mlx5_ib_flow_actions), 2485 + {}, 2486 + }; 2487 + 2488 + static const struct ib_device_ops flow_ops = { 2489 + .create_flow = mlx5_ib_create_flow, 2490 + .destroy_flow = mlx5_ib_destroy_flow, 2491 + .destroy_flow_action = mlx5_ib_destroy_flow_action, 2492 + }; 2493 + 2494 + static const struct ib_device_ops flow_ipsec_ops = { 2495 + .create_flow_action_esp = mlx5_ib_create_flow_action_esp, 2496 + .modify_flow_action_esp = mlx5_ib_modify_flow_action_esp, 2497 + }; 2498 + 2499 + int mlx5_ib_fs_init(struct mlx5_ib_dev *dev) 2500 + { 2501 + dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL); 2502 + 2503 + if (!dev->flow_db) 2504 + return -ENOMEM; 2505 + 2506 + mutex_init(&dev->flow_db->lock); 2507 + 2508 + ib_set_device_ops(&dev->ib_dev, &flow_ops); 2509 + if (mlx5_accel_ipsec_device_caps(dev->mdev) & 2510 + MLX5_ACCEL_IPSEC_CAP_DEVICE) 2511 + ib_set_device_ops(&dev->ib_dev, &flow_ipsec_ops); 2512 + 2513 + return 0; 2514 + }

+29

drivers/infiniband/hw/mlx5/fs.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 + /* 3 + * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved. 4 + */ 5 + 6 + #ifndef _MLX5_IB_FS_H 7 + #define _MLX5_IB_FS_H 8 + 9 + #include "mlx5_ib.h" 10 + 11 + #if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) 12 + int mlx5_ib_fs_init(struct mlx5_ib_dev *dev); 13 + #else 14 + static inline int mlx5_ib_fs_init(struct mlx5_ib_dev *dev) 15 + { 16 + dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL); 17 + 18 + if (!dev->flow_db) 19 + return -ENOMEM; 20 + 21 + mutex_init(&dev->flow_db->lock); 22 + return 0; 23 + } 24 + #endif 25 + static inline void mlx5_ib_fs_cleanup(struct mlx5_ib_dev *dev) 26 + { 27 + kfree(dev->flow_db); 28 + } 29 + #endif /* _MLX5_IB_FS_H */

+7 -1714

drivers/infiniband/hw/mlx5/main.c

··· 32 32 #include "mlx5_ib.h" 33 33 #include "ib_rep.h" 34 34 #include "cmd.h" 35 + #include "fs.h" 35 36 #include "srq.h" 36 37 #include "qp.h" 37 38 #include "wr.h" 38 39 #include "restrack.h" 39 40 #include "counters.h" 40 - #include <linux/mlx5/fs_helpers.h> 41 41 #include <linux/mlx5/accel.h> 42 42 #include <rdma/uverbs_std_types.h> 43 43 #include <rdma/mlx5_user_ioctl_verbs.h> ··· 2579 2579 mlx5_cmd_dealloc_pd(mdev->mdev, mpd->pdn, mpd->uid); 2580 2580 } 2581 2581 2582 - enum { 2583 - MATCH_CRITERIA_ENABLE_OUTER_BIT, 2584 - MATCH_CRITERIA_ENABLE_MISC_BIT, 2585 - MATCH_CRITERIA_ENABLE_INNER_BIT, 2586 - MATCH_CRITERIA_ENABLE_MISC2_BIT 2587 - }; 2588 - 2589 - #define HEADER_IS_ZERO(match_criteria, headers) \ 2590 - !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \ 2591 - 0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \ 2592 - 2593 - static u8 get_match_criteria_enable(u32 *match_criteria) 2594 - { 2595 - u8 match_criteria_enable; 2596 - 2597 - match_criteria_enable = 2598 - (!HEADER_IS_ZERO(match_criteria, outer_headers)) << 2599 - MATCH_CRITERIA_ENABLE_OUTER_BIT; 2600 - match_criteria_enable |= 2601 - (!HEADER_IS_ZERO(match_criteria, misc_parameters)) << 2602 - MATCH_CRITERIA_ENABLE_MISC_BIT; 2603 - match_criteria_enable |= 2604 - (!HEADER_IS_ZERO(match_criteria, inner_headers)) << 2605 - MATCH_CRITERIA_ENABLE_INNER_BIT; 2606 - match_criteria_enable |= 2607 - (!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) << 2608 - MATCH_CRITERIA_ENABLE_MISC2_BIT; 2609 - 2610 - return match_criteria_enable; 2611 - } 2612 - 2613 - static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val) 2614 - { 2615 - u8 entry_mask; 2616 - u8 entry_val; 2617 - int err = 0; 2618 - 2619 - if (!mask) 2620 - goto out; 2621 - 2622 - entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c, 2623 - ip_protocol); 2624 - entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v, 2625 - ip_protocol); 2626 - if (!entry_mask) { 2627 - MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask); 2628 - MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val); 2629 - goto out; 2630 - } 2631 - /* Don't override existing ip protocol */ 2632 - if (mask != entry_mask || val != entry_val) 2633 - err = -EINVAL; 2634 - out: 2635 - return err; 2636 - } 2637 - 2638 - static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val, 2639 - bool inner) 2640 - { 2641 - if (inner) { 2642 - MLX5_SET(fte_match_set_misc, 2643 - misc_c, inner_ipv6_flow_label, mask); 2644 - MLX5_SET(fte_match_set_misc, 2645 - misc_v, inner_ipv6_flow_label, val); 2646 - } else { 2647 - MLX5_SET(fte_match_set_misc, 2648 - misc_c, outer_ipv6_flow_label, mask); 2649 - MLX5_SET(fte_match_set_misc, 2650 - misc_v, outer_ipv6_flow_label, val); 2651 - } 2652 - } 2653 - 2654 - static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val) 2655 - { 2656 - MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask); 2657 - MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val); 2658 - MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2); 2659 - MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2); 2660 - } 2661 - 2662 - static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask) 2663 - { 2664 - if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) && 2665 - !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL)) 2666 - return -EOPNOTSUPP; 2667 - 2668 - if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) && 2669 - !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP)) 2670 - return -EOPNOTSUPP; 2671 - 2672 - if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) && 2673 - !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS)) 2674 - return -EOPNOTSUPP; 2675 - 2676 - if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) && 2677 - !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL)) 2678 - return -EOPNOTSUPP; 2679 - 2680 - return 0; 2681 - } 2682 - 2683 - #define LAST_ETH_FIELD vlan_tag 2684 - #define LAST_IB_FIELD sl 2685 - #define LAST_IPV4_FIELD tos 2686 - #define LAST_IPV6_FIELD traffic_class 2687 - #define LAST_TCP_UDP_FIELD src_port 2688 - #define LAST_TUNNEL_FIELD tunnel_id 2689 - #define LAST_FLOW_TAG_FIELD tag_id 2690 - #define LAST_DROP_FIELD size 2691 - #define LAST_COUNTERS_FIELD counters 2692 - 2693 - /* Field is the last supported field */ 2694 - #define FIELDS_NOT_SUPPORTED(filter, field)\ 2695 - memchr_inv((void *)&filter.field +\ 2696 - sizeof(filter.field), 0,\ 2697 - sizeof(filter) -\ 2698 - offsetof(typeof(filter), field) -\ 2699 - sizeof(filter.field)) 2700 - 2701 - int parse_flow_flow_action(struct mlx5_ib_flow_action *maction, 2702 - bool is_egress, 2703 - struct mlx5_flow_act *action) 2704 - { 2705 - 2706 - switch (maction->ib_action.type) { 2707 - case IB_FLOW_ACTION_ESP: 2708 - if (action->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT | 2709 - MLX5_FLOW_CONTEXT_ACTION_DECRYPT)) 2710 - return -EINVAL; 2711 - /* Currently only AES_GCM keymat is supported by the driver */ 2712 - action->esp_id = (uintptr_t)maction->esp_aes_gcm.ctx; 2713 - action->action |= is_egress ? 2714 - MLX5_FLOW_CONTEXT_ACTION_ENCRYPT : 2715 - MLX5_FLOW_CONTEXT_ACTION_DECRYPT; 2716 - return 0; 2717 - case IB_FLOW_ACTION_UNSPECIFIED: 2718 - if (maction->flow_action_raw.sub_type == 2719 - MLX5_IB_FLOW_ACTION_MODIFY_HEADER) { 2720 - if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 2721 - return -EINVAL; 2722 - action->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 2723 - action->modify_hdr = 2724 - maction->flow_action_raw.modify_hdr; 2725 - return 0; 2726 - } 2727 - if (maction->flow_action_raw.sub_type == 2728 - MLX5_IB_FLOW_ACTION_DECAP) { 2729 - if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP) 2730 - return -EINVAL; 2731 - action->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP; 2732 - return 0; 2733 - } 2734 - if (maction->flow_action_raw.sub_type == 2735 - MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) { 2736 - if (action->action & 2737 - MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT) 2738 - return -EINVAL; 2739 - action->action |= 2740 - MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 2741 - action->pkt_reformat = 2742 - maction->flow_action_raw.pkt_reformat; 2743 - return 0; 2744 - } 2745 - /* fall through */ 2746 - default: 2747 - return -EOPNOTSUPP; 2748 - } 2749 - } 2750 - 2751 - static int parse_flow_attr(struct mlx5_core_dev *mdev, 2752 - struct mlx5_flow_spec *spec, 2753 - const union ib_flow_spec *ib_spec, 2754 - const struct ib_flow_attr *flow_attr, 2755 - struct mlx5_flow_act *action, u32 prev_type) 2756 - { 2757 - struct mlx5_flow_context *flow_context = &spec->flow_context; 2758 - u32 *match_c = spec->match_criteria; 2759 - u32 *match_v = spec->match_value; 2760 - void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c, 2761 - misc_parameters); 2762 - void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v, 2763 - misc_parameters); 2764 - void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c, 2765 - misc_parameters_2); 2766 - void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v, 2767 - misc_parameters_2); 2768 - void *headers_c; 2769 - void *headers_v; 2770 - int match_ipv; 2771 - int ret; 2772 - 2773 - if (ib_spec->type & IB_FLOW_SPEC_INNER) { 2774 - headers_c = MLX5_ADDR_OF(fte_match_param, match_c, 2775 - inner_headers); 2776 - headers_v = MLX5_ADDR_OF(fte_match_param, match_v, 2777 - inner_headers); 2778 - match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 2779 - ft_field_support.inner_ip_version); 2780 - } else { 2781 - headers_c = MLX5_ADDR_OF(fte_match_param, match_c, 2782 - outer_headers); 2783 - headers_v = MLX5_ADDR_OF(fte_match_param, match_v, 2784 - outer_headers); 2785 - match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 2786 - ft_field_support.outer_ip_version); 2787 - } 2788 - 2789 - switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) { 2790 - case IB_FLOW_SPEC_ETH: 2791 - if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD)) 2792 - return -EOPNOTSUPP; 2793 - 2794 - ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 2795 - dmac_47_16), 2796 - ib_spec->eth.mask.dst_mac); 2797 - ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 2798 - dmac_47_16), 2799 - ib_spec->eth.val.dst_mac); 2800 - 2801 - ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 2802 - smac_47_16), 2803 - ib_spec->eth.mask.src_mac); 2804 - ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 2805 - smac_47_16), 2806 - ib_spec->eth.val.src_mac); 2807 - 2808 - if (ib_spec->eth.mask.vlan_tag) { 2809 - MLX5_SET(fte_match_set_lyr_2_4, headers_c, 2810 - cvlan_tag, 1); 2811 - MLX5_SET(fte_match_set_lyr_2_4, headers_v, 2812 - cvlan_tag, 1); 2813 - 2814 - MLX5_SET(fte_match_set_lyr_2_4, headers_c, 2815 - first_vid, ntohs(ib_spec->eth.mask.vlan_tag)); 2816 - MLX5_SET(fte_match_set_lyr_2_4, headers_v, 2817 - first_vid, ntohs(ib_spec->eth.val.vlan_tag)); 2818 - 2819 - MLX5_SET(fte_match_set_lyr_2_4, headers_c, 2820 - first_cfi, 2821 - ntohs(ib_spec->eth.mask.vlan_tag) >> 12); 2822 - MLX5_SET(fte_match_set_lyr_2_4, headers_v, 2823 - first_cfi, 2824 - ntohs(ib_spec->eth.val.vlan_tag) >> 12); 2825 - 2826 - MLX5_SET(fte_match_set_lyr_2_4, headers_c, 2827 - first_prio, 2828 - ntohs(ib_spec->eth.mask.vlan_tag) >> 13); 2829 - MLX5_SET(fte_match_set_lyr_2_4, headers_v, 2830 - first_prio, 2831 - ntohs(ib_spec->eth.val.vlan_tag) >> 13); 2832 - } 2833 - MLX5_SET(fte_match_set_lyr_2_4, headers_c, 2834 - ethertype, ntohs(ib_spec->eth.mask.ether_type)); 2835 - MLX5_SET(fte_match_set_lyr_2_4, headers_v, 2836 - ethertype, ntohs(ib_spec->eth.val.ether_type)); 2837 - break; 2838 - case IB_FLOW_SPEC_IPV4: 2839 - if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD)) 2840 - return -EOPNOTSUPP; 2841 - 2842 - if (match_ipv) { 2843 - MLX5_SET(fte_match_set_lyr_2_4, headers_c, 2844 - ip_version, 0xf); 2845 - MLX5_SET(fte_match_set_lyr_2_4, headers_v, 2846 - ip_version, MLX5_FS_IPV4_VERSION); 2847 - } else { 2848 - MLX5_SET(fte_match_set_lyr_2_4, headers_c, 2849 - ethertype, 0xffff); 2850 - MLX5_SET(fte_match_set_lyr_2_4, headers_v, 2851 - ethertype, ETH_P_IP); 2852 - } 2853 - 2854 - memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 2855 - src_ipv4_src_ipv6.ipv4_layout.ipv4), 2856 - &ib_spec->ipv4.mask.src_ip, 2857 - sizeof(ib_spec->ipv4.mask.src_ip)); 2858 - memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 2859 - src_ipv4_src_ipv6.ipv4_layout.ipv4), 2860 - &ib_spec->ipv4.val.src_ip, 2861 - sizeof(ib_spec->ipv4.val.src_ip)); 2862 - memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 2863 - dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 2864 - &ib_spec->ipv4.mask.dst_ip, 2865 - sizeof(ib_spec->ipv4.mask.dst_ip)); 2866 - memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 2867 - dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 2868 - &ib_spec->ipv4.val.dst_ip, 2869 - sizeof(ib_spec->ipv4.val.dst_ip)); 2870 - 2871 - set_tos(headers_c, headers_v, 2872 - ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos); 2873 - 2874 - if (set_proto(headers_c, headers_v, 2875 - ib_spec->ipv4.mask.proto, 2876 - ib_spec->ipv4.val.proto)) 2877 - return -EINVAL; 2878 - break; 2879 - case IB_FLOW_SPEC_IPV6: 2880 - if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD)) 2881 - return -EOPNOTSUPP; 2882 - 2883 - if (match_ipv) { 2884 - MLX5_SET(fte_match_set_lyr_2_4, headers_c, 2885 - ip_version, 0xf); 2886 - MLX5_SET(fte_match_set_lyr_2_4, headers_v, 2887 - ip_version, MLX5_FS_IPV6_VERSION); 2888 - } else { 2889 - MLX5_SET(fte_match_set_lyr_2_4, headers_c, 2890 - ethertype, 0xffff); 2891 - MLX5_SET(fte_match_set_lyr_2_4, headers_v, 2892 - ethertype, ETH_P_IPV6); 2893 - } 2894 - 2895 - memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 2896 - src_ipv4_src_ipv6.ipv6_layout.ipv6), 2897 - &ib_spec->ipv6.mask.src_ip, 2898 - sizeof(ib_spec->ipv6.mask.src_ip)); 2899 - memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 2900 - src_ipv4_src_ipv6.ipv6_layout.ipv6), 2901 - &ib_spec->ipv6.val.src_ip, 2902 - sizeof(ib_spec->ipv6.val.src_ip)); 2903 - memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 2904 - dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 2905 - &ib_spec->ipv6.mask.dst_ip, 2906 - sizeof(ib_spec->ipv6.mask.dst_ip)); 2907 - memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 2908 - dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 2909 - &ib_spec->ipv6.val.dst_ip, 2910 - sizeof(ib_spec->ipv6.val.dst_ip)); 2911 - 2912 - set_tos(headers_c, headers_v, 2913 - ib_spec->ipv6.mask.traffic_class, 2914 - ib_spec->ipv6.val.traffic_class); 2915 - 2916 - if (set_proto(headers_c, headers_v, 2917 - ib_spec->ipv6.mask.next_hdr, 2918 - ib_spec->ipv6.val.next_hdr)) 2919 - return -EINVAL; 2920 - 2921 - set_flow_label(misc_params_c, misc_params_v, 2922 - ntohl(ib_spec->ipv6.mask.flow_label), 2923 - ntohl(ib_spec->ipv6.val.flow_label), 2924 - ib_spec->type & IB_FLOW_SPEC_INNER); 2925 - break; 2926 - case IB_FLOW_SPEC_ESP: 2927 - if (ib_spec->esp.mask.seq) 2928 - return -EOPNOTSUPP; 2929 - 2930 - MLX5_SET(fte_match_set_misc, misc_params_c, outer_esp_spi, 2931 - ntohl(ib_spec->esp.mask.spi)); 2932 - MLX5_SET(fte_match_set_misc, misc_params_v, outer_esp_spi, 2933 - ntohl(ib_spec->esp.val.spi)); 2934 - break; 2935 - case IB_FLOW_SPEC_TCP: 2936 - if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, 2937 - LAST_TCP_UDP_FIELD)) 2938 - return -EOPNOTSUPP; 2939 - 2940 - if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP)) 2941 - return -EINVAL; 2942 - 2943 - MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport, 2944 - ntohs(ib_spec->tcp_udp.mask.src_port)); 2945 - MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport, 2946 - ntohs(ib_spec->tcp_udp.val.src_port)); 2947 - 2948 - MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport, 2949 - ntohs(ib_spec->tcp_udp.mask.dst_port)); 2950 - MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport, 2951 - ntohs(ib_spec->tcp_udp.val.dst_port)); 2952 - break; 2953 - case IB_FLOW_SPEC_UDP: 2954 - if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask, 2955 - LAST_TCP_UDP_FIELD)) 2956 - return -EOPNOTSUPP; 2957 - 2958 - if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP)) 2959 - return -EINVAL; 2960 - 2961 - MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport, 2962 - ntohs(ib_spec->tcp_udp.mask.src_port)); 2963 - MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, 2964 - ntohs(ib_spec->tcp_udp.val.src_port)); 2965 - 2966 - MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport, 2967 - ntohs(ib_spec->tcp_udp.mask.dst_port)); 2968 - MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, 2969 - ntohs(ib_spec->tcp_udp.val.dst_port)); 2970 - break; 2971 - case IB_FLOW_SPEC_GRE: 2972 - if (ib_spec->gre.mask.c_ks_res0_ver) 2973 - return -EOPNOTSUPP; 2974 - 2975 - if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE)) 2976 - return -EINVAL; 2977 - 2978 - MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol, 2979 - 0xff); 2980 - MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 2981 - IPPROTO_GRE); 2982 - 2983 - MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol, 2984 - ntohs(ib_spec->gre.mask.protocol)); 2985 - MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol, 2986 - ntohs(ib_spec->gre.val.protocol)); 2987 - 2988 - memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c, 2989 - gre_key.nvgre.hi), 2990 - &ib_spec->gre.mask.key, 2991 - sizeof(ib_spec->gre.mask.key)); 2992 - memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v, 2993 - gre_key.nvgre.hi), 2994 - &ib_spec->gre.val.key, 2995 - sizeof(ib_spec->gre.val.key)); 2996 - break; 2997 - case IB_FLOW_SPEC_MPLS: 2998 - switch (prev_type) { 2999 - case IB_FLOW_SPEC_UDP: 3000 - if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 3001 - ft_field_support.outer_first_mpls_over_udp), 3002 - &ib_spec->mpls.mask.tag)) 3003 - return -EOPNOTSUPP; 3004 - 3005 - memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 3006 - outer_first_mpls_over_udp), 3007 - &ib_spec->mpls.val.tag, 3008 - sizeof(ib_spec->mpls.val.tag)); 3009 - memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 3010 - outer_first_mpls_over_udp), 3011 - &ib_spec->mpls.mask.tag, 3012 - sizeof(ib_spec->mpls.mask.tag)); 3013 - break; 3014 - case IB_FLOW_SPEC_GRE: 3015 - if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 3016 - ft_field_support.outer_first_mpls_over_gre), 3017 - &ib_spec->mpls.mask.tag)) 3018 - return -EOPNOTSUPP; 3019 - 3020 - memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 3021 - outer_first_mpls_over_gre), 3022 - &ib_spec->mpls.val.tag, 3023 - sizeof(ib_spec->mpls.val.tag)); 3024 - memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 3025 - outer_first_mpls_over_gre), 3026 - &ib_spec->mpls.mask.tag, 3027 - sizeof(ib_spec->mpls.mask.tag)); 3028 - break; 3029 - default: 3030 - if (ib_spec->type & IB_FLOW_SPEC_INNER) { 3031 - if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 3032 - ft_field_support.inner_first_mpls), 3033 - &ib_spec->mpls.mask.tag)) 3034 - return -EOPNOTSUPP; 3035 - 3036 - memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 3037 - inner_first_mpls), 3038 - &ib_spec->mpls.val.tag, 3039 - sizeof(ib_spec->mpls.val.tag)); 3040 - memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 3041 - inner_first_mpls), 3042 - &ib_spec->mpls.mask.tag, 3043 - sizeof(ib_spec->mpls.mask.tag)); 3044 - } else { 3045 - if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 3046 - ft_field_support.outer_first_mpls), 3047 - &ib_spec->mpls.mask.tag)) 3048 - return -EOPNOTSUPP; 3049 - 3050 - memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v, 3051 - outer_first_mpls), 3052 - &ib_spec->mpls.val.tag, 3053 - sizeof(ib_spec->mpls.val.tag)); 3054 - memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c, 3055 - outer_first_mpls), 3056 - &ib_spec->mpls.mask.tag, 3057 - sizeof(ib_spec->mpls.mask.tag)); 3058 - } 3059 - } 3060 - break; 3061 - case IB_FLOW_SPEC_VXLAN_TUNNEL: 3062 - if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask, 3063 - LAST_TUNNEL_FIELD)) 3064 - return -EOPNOTSUPP; 3065 - 3066 - MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni, 3067 - ntohl(ib_spec->tunnel.mask.tunnel_id)); 3068 - MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni, 3069 - ntohl(ib_spec->tunnel.val.tunnel_id)); 3070 - break; 3071 - case IB_FLOW_SPEC_ACTION_TAG: 3072 - if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag, 3073 - LAST_FLOW_TAG_FIELD)) 3074 - return -EOPNOTSUPP; 3075 - if (ib_spec->flow_tag.tag_id >= BIT(24)) 3076 - return -EINVAL; 3077 - 3078 - flow_context->flow_tag = ib_spec->flow_tag.tag_id; 3079 - flow_context->flags |= FLOW_CONTEXT_HAS_TAG; 3080 - break; 3081 - case IB_FLOW_SPEC_ACTION_DROP: 3082 - if (FIELDS_NOT_SUPPORTED(ib_spec->drop, 3083 - LAST_DROP_FIELD)) 3084 - return -EOPNOTSUPP; 3085 - action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 3086 - break; 3087 - case IB_FLOW_SPEC_ACTION_HANDLE: 3088 - ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act), 3089 - flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action); 3090 - if (ret) 3091 - return ret; 3092 - break; 3093 - case IB_FLOW_SPEC_ACTION_COUNT: 3094 - if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count, 3095 - LAST_COUNTERS_FIELD)) 3096 - return -EOPNOTSUPP; 3097 - 3098 - /* for now support only one counters spec per flow */ 3099 - if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) 3100 - return -EINVAL; 3101 - 3102 - action->counters = ib_spec->flow_count.counters; 3103 - action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; 3104 - break; 3105 - default: 3106 - return -EINVAL; 3107 - } 3108 - 3109 - return 0; 3110 - } 3111 - 3112 - /* If a flow could catch both multicast and unicast packets, 3113 - * it won't fall into the multicast flow steering table and this rule 3114 - * could steal other multicast packets. 3115 - */ 3116 - static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr) 3117 - { 3118 - union ib_flow_spec *flow_spec; 3119 - 3120 - if (ib_attr->type != IB_FLOW_ATTR_NORMAL || 3121 - ib_attr->num_of_specs < 1) 3122 - return false; 3123 - 3124 - flow_spec = (union ib_flow_spec *)(ib_attr + 1); 3125 - if (flow_spec->type == IB_FLOW_SPEC_IPV4) { 3126 - struct ib_flow_spec_ipv4 *ipv4_spec; 3127 - 3128 - ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec; 3129 - if (ipv4_is_multicast(ipv4_spec->val.dst_ip)) 3130 - return true; 3131 - 3132 - return false; 3133 - } 3134 - 3135 - if (flow_spec->type == IB_FLOW_SPEC_ETH) { 3136 - struct ib_flow_spec_eth *eth_spec; 3137 - 3138 - eth_spec = (struct ib_flow_spec_eth *)flow_spec; 3139 - return is_multicast_ether_addr(eth_spec->mask.dst_mac) && 3140 - is_multicast_ether_addr(eth_spec->val.dst_mac); 3141 - } 3142 - 3143 - return false; 3144 - } 3145 - 3146 - enum valid_spec { 3147 - VALID_SPEC_INVALID, 3148 - VALID_SPEC_VALID, 3149 - VALID_SPEC_NA, 3150 - }; 3151 - 3152 - static enum valid_spec 3153 - is_valid_esp_aes_gcm(struct mlx5_core_dev *mdev, 3154 - const struct mlx5_flow_spec *spec, 3155 - const struct mlx5_flow_act *flow_act, 3156 - bool egress) 3157 - { 3158 - const u32 *match_c = spec->match_criteria; 3159 - bool is_crypto = 3160 - (flow_act->action & (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT | 3161 - MLX5_FLOW_CONTEXT_ACTION_DECRYPT)); 3162 - bool is_ipsec = mlx5_fs_is_ipsec_flow(match_c); 3163 - bool is_drop = flow_act->action & MLX5_FLOW_CONTEXT_ACTION_DROP; 3164 - 3165 - /* 3166 - * Currently only crypto is supported in egress, when regular egress 3167 - * rules would be supported, always return VALID_SPEC_NA. 3168 - */ 3169 - if (!is_crypto) 3170 - return VALID_SPEC_NA; 3171 - 3172 - return is_crypto && is_ipsec && 3173 - (!egress || (!is_drop && 3174 - !(spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG))) ? 3175 - VALID_SPEC_VALID : VALID_SPEC_INVALID; 3176 - } 3177 - 3178 - static bool is_valid_spec(struct mlx5_core_dev *mdev, 3179 - const struct mlx5_flow_spec *spec, 3180 - const struct mlx5_flow_act *flow_act, 3181 - bool egress) 3182 - { 3183 - /* We curretly only support ipsec egress flow */ 3184 - return is_valid_esp_aes_gcm(mdev, spec, flow_act, egress) != VALID_SPEC_INVALID; 3185 - } 3186 - 3187 - static bool is_valid_ethertype(struct mlx5_core_dev *mdev, 3188 - const struct ib_flow_attr *flow_attr, 3189 - bool check_inner) 3190 - { 3191 - union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1); 3192 - int match_ipv = check_inner ? 3193 - MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 3194 - ft_field_support.inner_ip_version) : 3195 - MLX5_CAP_FLOWTABLE_NIC_RX(mdev, 3196 - ft_field_support.outer_ip_version); 3197 - int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0; 3198 - bool ipv4_spec_valid, ipv6_spec_valid; 3199 - unsigned int ip_spec_type = 0; 3200 - bool has_ethertype = false; 3201 - unsigned int spec_index; 3202 - bool mask_valid = true; 3203 - u16 eth_type = 0; 3204 - bool type_valid; 3205 - 3206 - /* Validate that ethertype is correct */ 3207 - for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { 3208 - if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) && 3209 - ib_spec->eth.mask.ether_type) { 3210 - mask_valid = (ib_spec->eth.mask.ether_type == 3211 - htons(0xffff)); 3212 - has_ethertype = true; 3213 - eth_type = ntohs(ib_spec->eth.val.ether_type); 3214 - } else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) || 3215 - (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) { 3216 - ip_spec_type = ib_spec->type; 3217 - } 3218 - ib_spec = (void *)ib_spec + ib_spec->size; 3219 - } 3220 - 3221 - type_valid = (!has_ethertype) || (!ip_spec_type); 3222 - if (!type_valid && mask_valid) { 3223 - ipv4_spec_valid = (eth_type == ETH_P_IP) && 3224 - (ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit)); 3225 - ipv6_spec_valid = (eth_type == ETH_P_IPV6) && 3226 - (ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit)); 3227 - 3228 - type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) || 3229 - (((eth_type == ETH_P_MPLS_UC) || 3230 - (eth_type == ETH_P_MPLS_MC)) && match_ipv); 3231 - } 3232 - 3233 - return type_valid; 3234 - } 3235 - 3236 - static bool is_valid_attr(struct mlx5_core_dev *mdev, 3237 - const struct ib_flow_attr *flow_attr) 3238 - { 3239 - return is_valid_ethertype(mdev, flow_attr, false) && 3240 - is_valid_ethertype(mdev, flow_attr, true); 3241 - } 3242 - 3243 - static void put_flow_table(struct mlx5_ib_dev *dev, 3244 - struct mlx5_ib_flow_prio *prio, bool ft_added) 3245 - { 3246 - prio->refcount -= !!ft_added; 3247 - if (!prio->refcount) { 3248 - mlx5_destroy_flow_table(prio->flow_table); 3249 - prio->flow_table = NULL; 3250 - } 3251 - } 3252 - 3253 - static int mlx5_ib_destroy_flow(struct ib_flow *flow_id) 3254 - { 3255 - struct mlx5_ib_flow_handler *handler = container_of(flow_id, 3256 - struct mlx5_ib_flow_handler, 3257 - ibflow); 3258 - struct mlx5_ib_flow_handler *iter, *tmp; 3259 - struct mlx5_ib_dev *dev = handler->dev; 3260 - 3261 - mutex_lock(&dev->flow_db->lock); 3262 - 3263 - list_for_each_entry_safe(iter, tmp, &handler->list, list) { 3264 - mlx5_del_flow_rules(iter->rule); 3265 - put_flow_table(dev, iter->prio, true); 3266 - list_del(&iter->list); 3267 - kfree(iter); 3268 - } 3269 - 3270 - mlx5_del_flow_rules(handler->rule); 3271 - put_flow_table(dev, handler->prio, true); 3272 - mlx5_ib_counters_clear_description(handler->ibcounters); 3273 - mutex_unlock(&dev->flow_db->lock); 3274 - if (handler->flow_matcher) 3275 - atomic_dec(&handler->flow_matcher->usecnt); 3276 - kfree(handler); 3277 - 3278 - return 0; 3279 - } 3280 - 3281 - static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap) 3282 - { 3283 - priority *= 2; 3284 - if (!dont_trap) 3285 - priority++; 3286 - return priority; 3287 - } 3288 - 3289 - enum flow_table_type { 3290 - MLX5_IB_FT_RX, 3291 - MLX5_IB_FT_TX 3292 - }; 3293 - 3294 - #define MLX5_FS_MAX_TYPES 6 3295 - #define MLX5_FS_MAX_ENTRIES BIT(16) 3296 - 3297 - static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_flow_namespace *ns, 3298 - struct mlx5_ib_flow_prio *prio, 3299 - int priority, 3300 - int num_entries, int num_groups, 3301 - u32 flags) 3302 - { 3303 - struct mlx5_flow_table_attr ft_attr = {}; 3304 - struct mlx5_flow_table *ft; 3305 - 3306 - ft_attr.prio = priority; 3307 - ft_attr.max_fte = num_entries; 3308 - ft_attr.flags = flags; 3309 - ft_attr.autogroup.max_num_groups = num_groups; 3310 - ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 3311 - if (IS_ERR(ft)) 3312 - return ERR_CAST(ft); 3313 - 3314 - prio->flow_table = ft; 3315 - prio->refcount = 0; 3316 - return prio; 3317 - } 3318 - 3319 - static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, 3320 - struct ib_flow_attr *flow_attr, 3321 - enum flow_table_type ft_type) 3322 - { 3323 - bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP; 3324 - struct mlx5_flow_namespace *ns = NULL; 3325 - struct mlx5_ib_flow_prio *prio; 3326 - struct mlx5_flow_table *ft; 3327 - int max_table_size; 3328 - int num_entries; 3329 - int num_groups; 3330 - bool esw_encap; 3331 - u32 flags = 0; 3332 - int priority; 3333 - 3334 - max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 3335 - log_max_ft_size)); 3336 - esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != 3337 - DEVLINK_ESWITCH_ENCAP_MODE_NONE; 3338 - if (flow_attr->type == IB_FLOW_ATTR_NORMAL) { 3339 - enum mlx5_flow_namespace_type fn_type; 3340 - 3341 - if (flow_is_multicast_only(flow_attr) && 3342 - !dont_trap) 3343 - priority = MLX5_IB_FLOW_MCAST_PRIO; 3344 - else 3345 - priority = ib_prio_to_core_prio(flow_attr->priority, 3346 - dont_trap); 3347 - if (ft_type == MLX5_IB_FT_RX) { 3348 - fn_type = MLX5_FLOW_NAMESPACE_BYPASS; 3349 - prio = &dev->flow_db->prios[priority]; 3350 - if (!dev->is_rep && !esw_encap && 3351 - MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap)) 3352 - flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 3353 - if (!dev->is_rep && !esw_encap && 3354 - MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 3355 - reformat_l3_tunnel_to_l2)) 3356 - flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 3357 - } else { 3358 - max_table_size = 3359 - BIT(MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, 3360 - log_max_ft_size)); 3361 - fn_type = MLX5_FLOW_NAMESPACE_EGRESS; 3362 - prio = &dev->flow_db->egress_prios[priority]; 3363 - if (!dev->is_rep && !esw_encap && 3364 - MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat)) 3365 - flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 3366 - } 3367 - ns = mlx5_get_flow_namespace(dev->mdev, fn_type); 3368 - num_entries = MLX5_FS_MAX_ENTRIES; 3369 - num_groups = MLX5_FS_MAX_TYPES; 3370 - } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 3371 - flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) { 3372 - ns = mlx5_get_flow_namespace(dev->mdev, 3373 - MLX5_FLOW_NAMESPACE_LEFTOVERS); 3374 - build_leftovers_ft_param(&priority, 3375 - &num_entries, 3376 - &num_groups); 3377 - prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO]; 3378 - } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { 3379 - if (!MLX5_CAP_FLOWTABLE(dev->mdev, 3380 - allow_sniffer_and_nic_rx_shared_tir)) 3381 - return ERR_PTR(-ENOTSUPP); 3382 - 3383 - ns = mlx5_get_flow_namespace(dev->mdev, ft_type == MLX5_IB_FT_RX ? 3384 - MLX5_FLOW_NAMESPACE_SNIFFER_RX : 3385 - MLX5_FLOW_NAMESPACE_SNIFFER_TX); 3386 - 3387 - prio = &dev->flow_db->sniffer[ft_type]; 3388 - priority = 0; 3389 - num_entries = 1; 3390 - num_groups = 1; 3391 - } 3392 - 3393 - if (!ns) 3394 - return ERR_PTR(-ENOTSUPP); 3395 - 3396 - max_table_size = min_t(int, num_entries, max_table_size); 3397 - 3398 - ft = prio->flow_table; 3399 - if (!ft) 3400 - return _get_prio(ns, prio, priority, max_table_size, num_groups, 3401 - flags); 3402 - 3403 - return prio; 3404 - } 3405 - 3406 - static void set_underlay_qp(struct mlx5_ib_dev *dev, 3407 - struct mlx5_flow_spec *spec, 3408 - u32 underlay_qpn) 3409 - { 3410 - void *misc_params_c = MLX5_ADDR_OF(fte_match_param, 3411 - spec->match_criteria, 3412 - misc_parameters); 3413 - void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 3414 - misc_parameters); 3415 - 3416 - if (underlay_qpn && 3417 - MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 3418 - ft_field_support.bth_dst_qp)) { 3419 - MLX5_SET(fte_match_set_misc, 3420 - misc_params_v, bth_dst_qp, underlay_qpn); 3421 - MLX5_SET(fte_match_set_misc, 3422 - misc_params_c, bth_dst_qp, 0xffffff); 3423 - } 3424 - } 3425 - 3426 - static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev, 3427 - struct mlx5_flow_spec *spec, 3428 - struct mlx5_eswitch_rep *rep) 3429 - { 3430 - struct mlx5_eswitch *esw = dev->mdev->priv.eswitch; 3431 - void *misc; 3432 - 3433 - if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 3434 - misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 3435 - misc_parameters_2); 3436 - 3437 - MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 3438 - mlx5_eswitch_get_vport_metadata_for_match(esw, 3439 - rep->vport)); 3440 - misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 3441 - misc_parameters_2); 3442 - 3443 - MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 3444 - mlx5_eswitch_get_vport_metadata_mask()); 3445 - } else { 3446 - misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 3447 - misc_parameters); 3448 - 3449 - MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport); 3450 - 3451 - misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 3452 - misc_parameters); 3453 - 3454 - MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 3455 - } 3456 - } 3457 - 3458 - static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev, 3459 - struct mlx5_ib_flow_prio *ft_prio, 3460 - const struct ib_flow_attr *flow_attr, 3461 - struct mlx5_flow_destination *dst, 3462 - u32 underlay_qpn, 3463 - struct mlx5_ib_create_flow *ucmd) 3464 - { 3465 - struct mlx5_flow_table *ft = ft_prio->flow_table; 3466 - struct mlx5_ib_flow_handler *handler; 3467 - struct mlx5_flow_act flow_act = {}; 3468 - struct mlx5_flow_spec *spec; 3469 - struct mlx5_flow_destination dest_arr[2] = {}; 3470 - struct mlx5_flow_destination *rule_dst = dest_arr; 3471 - const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr); 3472 - unsigned int spec_index; 3473 - u32 prev_type = 0; 3474 - int err = 0; 3475 - int dest_num = 0; 3476 - bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; 3477 - 3478 - if (!is_valid_attr(dev->mdev, flow_attr)) 3479 - return ERR_PTR(-EINVAL); 3480 - 3481 - if (dev->is_rep && is_egress) 3482 - return ERR_PTR(-EINVAL); 3483 - 3484 - spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 3485 - handler = kzalloc(sizeof(*handler), GFP_KERNEL); 3486 - if (!handler || !spec) { 3487 - err = -ENOMEM; 3488 - goto free; 3489 - } 3490 - 3491 - INIT_LIST_HEAD(&handler->list); 3492 - 3493 - for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) { 3494 - err = parse_flow_attr(dev->mdev, spec, 3495 - ib_flow, flow_attr, &flow_act, 3496 - prev_type); 3497 - if (err < 0) 3498 - goto free; 3499 - 3500 - prev_type = ((union ib_flow_spec *)ib_flow)->type; 3501 - ib_flow += ((union ib_flow_spec *)ib_flow)->size; 3502 - } 3503 - 3504 - if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) { 3505 - memcpy(&dest_arr[0], dst, sizeof(*dst)); 3506 - dest_num++; 3507 - } 3508 - 3509 - if (!flow_is_multicast_only(flow_attr)) 3510 - set_underlay_qp(dev, spec, underlay_qpn); 3511 - 3512 - if (dev->is_rep) { 3513 - struct mlx5_eswitch_rep *rep; 3514 - 3515 - rep = dev->port[flow_attr->port - 1].rep; 3516 - if (!rep) { 3517 - err = -EINVAL; 3518 - goto free; 3519 - } 3520 - 3521 - mlx5_ib_set_rule_source_port(dev, spec, rep); 3522 - } 3523 - 3524 - spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria); 3525 - 3526 - if (is_egress && 3527 - !is_valid_spec(dev->mdev, spec, &flow_act, is_egress)) { 3528 - err = -EINVAL; 3529 - goto free; 3530 - } 3531 - 3532 - if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 3533 - struct mlx5_ib_mcounters *mcounters; 3534 - 3535 - err = mlx5_ib_flow_counters_set_data(flow_act.counters, ucmd); 3536 - if (err) 3537 - goto free; 3538 - 3539 - mcounters = to_mcounters(flow_act.counters); 3540 - handler->ibcounters = flow_act.counters; 3541 - dest_arr[dest_num].type = 3542 - MLX5_FLOW_DESTINATION_TYPE_COUNTER; 3543 - dest_arr[dest_num].counter_id = 3544 - mlx5_fc_id(mcounters->hw_cntrs_hndl); 3545 - dest_num++; 3546 - } 3547 - 3548 - if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) { 3549 - if (!dest_num) 3550 - rule_dst = NULL; 3551 - } else { 3552 - if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) 3553 - flow_act.action |= 3554 - MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO; 3555 - if (is_egress) 3556 - flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; 3557 - else if (dest_num) 3558 - flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 3559 - } 3560 - 3561 - if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG) && 3562 - (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 3563 - flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { 3564 - mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n", 3565 - spec->flow_context.flow_tag, flow_attr->type); 3566 - err = -EINVAL; 3567 - goto free; 3568 - } 3569 - handler->rule = mlx5_add_flow_rules(ft, spec, 3570 - &flow_act, 3571 - rule_dst, dest_num); 3572 - 3573 - if (IS_ERR(handler->rule)) { 3574 - err = PTR_ERR(handler->rule); 3575 - goto free; 3576 - } 3577 - 3578 - ft_prio->refcount++; 3579 - handler->prio = ft_prio; 3580 - handler->dev = dev; 3581 - 3582 - ft_prio->flow_table = ft; 3583 - free: 3584 - if (err && handler) { 3585 - mlx5_ib_counters_clear_description(handler->ibcounters); 3586 - kfree(handler); 3587 - } 3588 - kvfree(spec); 3589 - return err ? ERR_PTR(err) : handler; 3590 - } 3591 - 3592 - static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev, 3593 - struct mlx5_ib_flow_prio *ft_prio, 3594 - const struct ib_flow_attr *flow_attr, 3595 - struct mlx5_flow_destination *dst) 3596 - { 3597 - return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL); 3598 - } 3599 - 3600 - enum { 3601 - LEFTOVERS_MC, 3602 - LEFTOVERS_UC, 3603 - }; 3604 - 3605 - static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev, 3606 - struct mlx5_ib_flow_prio *ft_prio, 3607 - struct ib_flow_attr *flow_attr, 3608 - struct mlx5_flow_destination *dst) 3609 - { 3610 - struct mlx5_ib_flow_handler *handler_ucast = NULL; 3611 - struct mlx5_ib_flow_handler *handler = NULL; 3612 - 3613 - static struct { 3614 - struct ib_flow_attr flow_attr; 3615 - struct ib_flow_spec_eth eth_flow; 3616 - } leftovers_specs[] = { 3617 - [LEFTOVERS_MC] = { 3618 - .flow_attr = { 3619 - .num_of_specs = 1, 3620 - .size = sizeof(leftovers_specs[0]) 3621 - }, 3622 - .eth_flow = { 3623 - .type = IB_FLOW_SPEC_ETH, 3624 - .size = sizeof(struct ib_flow_spec_eth), 3625 - .mask = {.dst_mac = {0x1} }, 3626 - .val = {.dst_mac = {0x1} } 3627 - } 3628 - }, 3629 - [LEFTOVERS_UC] = { 3630 - .flow_attr = { 3631 - .num_of_specs = 1, 3632 - .size = sizeof(leftovers_specs[0]) 3633 - }, 3634 - .eth_flow = { 3635 - .type = IB_FLOW_SPEC_ETH, 3636 - .size = sizeof(struct ib_flow_spec_eth), 3637 - .mask = {.dst_mac = {0x1} }, 3638 - .val = {.dst_mac = {} } 3639 - } 3640 - } 3641 - }; 3642 - 3643 - handler = create_flow_rule(dev, ft_prio, 3644 - &leftovers_specs[LEFTOVERS_MC].flow_attr, 3645 - dst); 3646 - if (!IS_ERR(handler) && 3647 - flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) { 3648 - handler_ucast = create_flow_rule(dev, ft_prio, 3649 - &leftovers_specs[LEFTOVERS_UC].flow_attr, 3650 - dst); 3651 - if (IS_ERR(handler_ucast)) { 3652 - mlx5_del_flow_rules(handler->rule); 3653 - ft_prio->refcount--; 3654 - kfree(handler); 3655 - handler = handler_ucast; 3656 - } else { 3657 - list_add(&handler_ucast->list, &handler->list); 3658 - } 3659 - } 3660 - 3661 - return handler; 3662 - } 3663 - 3664 - static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev, 3665 - struct mlx5_ib_flow_prio *ft_rx, 3666 - struct mlx5_ib_flow_prio *ft_tx, 3667 - struct mlx5_flow_destination *dst) 3668 - { 3669 - struct mlx5_ib_flow_handler *handler_rx; 3670 - struct mlx5_ib_flow_handler *handler_tx; 3671 - int err; 3672 - static const struct ib_flow_attr flow_attr = { 3673 - .num_of_specs = 0, 3674 - .size = sizeof(flow_attr) 3675 - }; 3676 - 3677 - handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst); 3678 - if (IS_ERR(handler_rx)) { 3679 - err = PTR_ERR(handler_rx); 3680 - goto err; 3681 - } 3682 - 3683 - handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst); 3684 - if (IS_ERR(handler_tx)) { 3685 - err = PTR_ERR(handler_tx); 3686 - goto err_tx; 3687 - } 3688 - 3689 - list_add(&handler_tx->list, &handler_rx->list); 3690 - 3691 - return handler_rx; 3692 - 3693 - err_tx: 3694 - mlx5_del_flow_rules(handler_rx->rule); 3695 - ft_rx->refcount--; 3696 - kfree(handler_rx); 3697 - err: 3698 - return ERR_PTR(err); 3699 - } 3700 - 3701 - static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp, 3702 - struct ib_flow_attr *flow_attr, 3703 - int domain, 3704 - struct ib_udata *udata) 3705 - { 3706 - struct mlx5_ib_dev *dev = to_mdev(qp->device); 3707 - struct mlx5_ib_qp *mqp = to_mqp(qp); 3708 - struct mlx5_ib_flow_handler *handler = NULL; 3709 - struct mlx5_flow_destination *dst = NULL; 3710 - struct mlx5_ib_flow_prio *ft_prio_tx = NULL; 3711 - struct mlx5_ib_flow_prio *ft_prio; 3712 - bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS; 3713 - struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr; 3714 - size_t min_ucmd_sz, required_ucmd_sz; 3715 - int err; 3716 - int underlay_qpn; 3717 - 3718 - if (udata && udata->inlen) { 3719 - min_ucmd_sz = offsetof(typeof(ucmd_hdr), reserved) + 3720 - sizeof(ucmd_hdr.reserved); 3721 - if (udata->inlen < min_ucmd_sz) 3722 - return ERR_PTR(-EOPNOTSUPP); 3723 - 3724 - err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz); 3725 - if (err) 3726 - return ERR_PTR(err); 3727 - 3728 - /* currently supports only one counters data */ 3729 - if (ucmd_hdr.ncounters_data > 1) 3730 - return ERR_PTR(-EINVAL); 3731 - 3732 - required_ucmd_sz = min_ucmd_sz + 3733 - sizeof(struct mlx5_ib_flow_counters_data) * 3734 - ucmd_hdr.ncounters_data; 3735 - if (udata->inlen > required_ucmd_sz && 3736 - !ib_is_udata_cleared(udata, required_ucmd_sz, 3737 - udata->inlen - required_ucmd_sz)) 3738 - return ERR_PTR(-EOPNOTSUPP); 3739 - 3740 - ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL); 3741 - if (!ucmd) 3742 - return ERR_PTR(-ENOMEM); 3743 - 3744 - err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz); 3745 - if (err) 3746 - goto free_ucmd; 3747 - } 3748 - 3749 - if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) { 3750 - err = -ENOMEM; 3751 - goto free_ucmd; 3752 - } 3753 - 3754 - if (domain != IB_FLOW_DOMAIN_USER || 3755 - flow_attr->port > dev->num_ports || 3756 - (flow_attr->flags & ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP | 3757 - IB_FLOW_ATTR_FLAGS_EGRESS))) { 3758 - err = -EINVAL; 3759 - goto free_ucmd; 3760 - } 3761 - 3762 - if (is_egress && 3763 - (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 3764 - flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) { 3765 - err = -EINVAL; 3766 - goto free_ucmd; 3767 - } 3768 - 3769 - dst = kzalloc(sizeof(*dst), GFP_KERNEL); 3770 - if (!dst) { 3771 - err = -ENOMEM; 3772 - goto free_ucmd; 3773 - } 3774 - 3775 - mutex_lock(&dev->flow_db->lock); 3776 - 3777 - ft_prio = get_flow_table(dev, flow_attr, 3778 - is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX); 3779 - if (IS_ERR(ft_prio)) { 3780 - err = PTR_ERR(ft_prio); 3781 - goto unlock; 3782 - } 3783 - if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { 3784 - ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX); 3785 - if (IS_ERR(ft_prio_tx)) { 3786 - err = PTR_ERR(ft_prio_tx); 3787 - ft_prio_tx = NULL; 3788 - goto destroy_ft; 3789 - } 3790 - } 3791 - 3792 - if (is_egress) { 3793 - dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT; 3794 - } else { 3795 - dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR; 3796 - if (mqp->is_rss) 3797 - dst->tir_num = mqp->rss_qp.tirn; 3798 - else 3799 - dst->tir_num = mqp->raw_packet_qp.rq.tirn; 3800 - } 3801 - 3802 - if (flow_attr->type == IB_FLOW_ATTR_NORMAL) { 3803 - underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ? 3804 - mqp->underlay_qpn : 3805 - 0; 3806 - handler = _create_flow_rule(dev, ft_prio, flow_attr, dst, 3807 - underlay_qpn, ucmd); 3808 - } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT || 3809 - flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) { 3810 - handler = create_leftovers_rule(dev, ft_prio, flow_attr, 3811 - dst); 3812 - } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) { 3813 - handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst); 3814 - } else { 3815 - err = -EINVAL; 3816 - goto destroy_ft; 3817 - } 3818 - 3819 - if (IS_ERR(handler)) { 3820 - err = PTR_ERR(handler); 3821 - handler = NULL; 3822 - goto destroy_ft; 3823 - } 3824 - 3825 - mutex_unlock(&dev->flow_db->lock); 3826 - kfree(dst); 3827 - kfree(ucmd); 3828 - 3829 - return &handler->ibflow; 3830 - 3831 - destroy_ft: 3832 - put_flow_table(dev, ft_prio, false); 3833 - if (ft_prio_tx) 3834 - put_flow_table(dev, ft_prio_tx, false); 3835 - unlock: 3836 - mutex_unlock(&dev->flow_db->lock); 3837 - kfree(dst); 3838 - free_ucmd: 3839 - kfree(ucmd); 3840 - return ERR_PTR(err); 3841 - } 3842 - 3843 - static struct mlx5_ib_flow_prio * 3844 - _get_flow_table(struct mlx5_ib_dev *dev, 3845 - struct mlx5_ib_flow_matcher *fs_matcher, 3846 - bool mcast) 3847 - { 3848 - struct mlx5_flow_namespace *ns = NULL; 3849 - struct mlx5_ib_flow_prio *prio = NULL; 3850 - int max_table_size = 0; 3851 - bool esw_encap; 3852 - u32 flags = 0; 3853 - int priority; 3854 - 3855 - if (mcast) 3856 - priority = MLX5_IB_FLOW_MCAST_PRIO; 3857 - else 3858 - priority = ib_prio_to_core_prio(fs_matcher->priority, false); 3859 - 3860 - esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != 3861 - DEVLINK_ESWITCH_ENCAP_MODE_NONE; 3862 - if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) { 3863 - max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 3864 - log_max_ft_size)); 3865 - if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap) 3866 - flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 3867 - if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, 3868 - reformat_l3_tunnel_to_l2) && 3869 - !esw_encap) 3870 - flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 3871 - } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) { 3872 - max_table_size = BIT( 3873 - MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size)); 3874 - if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && !esw_encap) 3875 - flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 3876 - } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) { 3877 - max_table_size = BIT( 3878 - MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size)); 3879 - if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap) 3880 - flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; 3881 - if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, reformat_l3_tunnel_to_l2) && 3882 - esw_encap) 3883 - flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 3884 - priority = FDB_BYPASS_PATH; 3885 - } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) { 3886 - max_table_size = 3887 - BIT(MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, 3888 - log_max_ft_size)); 3889 - priority = fs_matcher->priority; 3890 - } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) { 3891 - max_table_size = 3892 - BIT(MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, 3893 - log_max_ft_size)); 3894 - priority = fs_matcher->priority; 3895 - } 3896 - 3897 - max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES); 3898 - 3899 - ns = mlx5_get_flow_namespace(dev->mdev, fs_matcher->ns_type); 3900 - if (!ns) 3901 - return ERR_PTR(-ENOTSUPP); 3902 - 3903 - if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) 3904 - prio = &dev->flow_db->prios[priority]; 3905 - else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) 3906 - prio = &dev->flow_db->egress_prios[priority]; 3907 - else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) 3908 - prio = &dev->flow_db->fdb; 3909 - else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX) 3910 - prio = &dev->flow_db->rdma_rx[priority]; 3911 - else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX) 3912 - prio = &dev->flow_db->rdma_tx[priority]; 3913 - 3914 - if (!prio) 3915 - return ERR_PTR(-EINVAL); 3916 - 3917 - if (prio->flow_table) 3918 - return prio; 3919 - 3920 - return _get_prio(ns, prio, priority, max_table_size, 3921 - MLX5_FS_MAX_TYPES, flags); 3922 - } 3923 - 3924 - static struct mlx5_ib_flow_handler * 3925 - _create_raw_flow_rule(struct mlx5_ib_dev *dev, 3926 - struct mlx5_ib_flow_prio *ft_prio, 3927 - struct mlx5_flow_destination *dst, 3928 - struct mlx5_ib_flow_matcher *fs_matcher, 3929 - struct mlx5_flow_context *flow_context, 3930 - struct mlx5_flow_act *flow_act, 3931 - void *cmd_in, int inlen, 3932 - int dst_num) 3933 - { 3934 - struct mlx5_ib_flow_handler *handler; 3935 - struct mlx5_flow_spec *spec; 3936 - struct mlx5_flow_table *ft = ft_prio->flow_table; 3937 - int err = 0; 3938 - 3939 - spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 3940 - handler = kzalloc(sizeof(*handler), GFP_KERNEL); 3941 - if (!handler || !spec) { 3942 - err = -ENOMEM; 3943 - goto free; 3944 - } 3945 - 3946 - INIT_LIST_HEAD(&handler->list); 3947 - 3948 - memcpy(spec->match_value, cmd_in, inlen); 3949 - memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params, 3950 - fs_matcher->mask_len); 3951 - spec->match_criteria_enable = fs_matcher->match_criteria_enable; 3952 - spec->flow_context = *flow_context; 3953 - 3954 - handler->rule = mlx5_add_flow_rules(ft, spec, 3955 - flow_act, dst, dst_num); 3956 - 3957 - if (IS_ERR(handler->rule)) { 3958 - err = PTR_ERR(handler->rule); 3959 - goto free; 3960 - } 3961 - 3962 - ft_prio->refcount++; 3963 - handler->prio = ft_prio; 3964 - handler->dev = dev; 3965 - ft_prio->flow_table = ft; 3966 - 3967 - free: 3968 - if (err) 3969 - kfree(handler); 3970 - kvfree(spec); 3971 - return err ? ERR_PTR(err) : handler; 3972 - } 3973 - 3974 - static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher, 3975 - void *match_v) 3976 - { 3977 - void *match_c; 3978 - void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4; 3979 - void *dmac, *dmac_mask; 3980 - void *ipv4, *ipv4_mask; 3981 - 3982 - if (!(fs_matcher->match_criteria_enable & 3983 - (1 << MATCH_CRITERIA_ENABLE_OUTER_BIT))) 3984 - return false; 3985 - 3986 - match_c = fs_matcher->matcher_mask.match_params; 3987 - match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v, 3988 - outer_headers); 3989 - match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c, 3990 - outer_headers); 3991 - 3992 - dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, 3993 - dmac_47_16); 3994 - dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, 3995 - dmac_47_16); 3996 - 3997 - if (is_multicast_ether_addr(dmac) && 3998 - is_multicast_ether_addr(dmac_mask)) 3999 - return true; 4000 - 4001 - ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4, 4002 - dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 4003 - 4004 - ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4, 4005 - dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 4006 - 4007 - if (ipv4_is_multicast(*(__be32 *)(ipv4)) && 4008 - ipv4_is_multicast(*(__be32 *)(ipv4_mask))) 4009 - return true; 4010 - 4011 - return false; 4012 - } 4013 - 4014 - struct mlx5_ib_flow_handler * 4015 - mlx5_ib_raw_fs_rule_add(struct mlx5_ib_dev *dev, 4016 - struct mlx5_ib_flow_matcher *fs_matcher, 4017 - struct mlx5_flow_context *flow_context, 4018 - struct mlx5_flow_act *flow_act, 4019 - u32 counter_id, 4020 - void *cmd_in, int inlen, int dest_id, 4021 - int dest_type) 4022 - { 4023 - struct mlx5_flow_destination *dst; 4024 - struct mlx5_ib_flow_prio *ft_prio; 4025 - struct mlx5_ib_flow_handler *handler; 4026 - int dst_num = 0; 4027 - bool mcast; 4028 - int err; 4029 - 4030 - if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL) 4031 - return ERR_PTR(-EOPNOTSUPP); 4032 - 4033 - if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO) 4034 - return ERR_PTR(-ENOMEM); 4035 - 4036 - dst = kcalloc(2, sizeof(*dst), GFP_KERNEL); 4037 - if (!dst) 4038 - return ERR_PTR(-ENOMEM); 4039 - 4040 - mcast = raw_fs_is_multicast(fs_matcher, cmd_in); 4041 - mutex_lock(&dev->flow_db->lock); 4042 - 4043 - ft_prio = _get_flow_table(dev, fs_matcher, mcast); 4044 - if (IS_ERR(ft_prio)) { 4045 - err = PTR_ERR(ft_prio); 4046 - goto unlock; 4047 - } 4048 - 4049 - if (dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR) { 4050 - dst[dst_num].type = dest_type; 4051 - dst[dst_num++].tir_num = dest_id; 4052 - flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 4053 - } else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) { 4054 - dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM; 4055 - dst[dst_num++].ft_num = dest_id; 4056 - flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 4057 - } else if (dest_type == MLX5_FLOW_DESTINATION_TYPE_PORT) { 4058 - dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT; 4059 - flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW; 4060 - } 4061 - 4062 - 4063 - if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 4064 - dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 4065 - dst[dst_num].counter_id = counter_id; 4066 - dst_num++; 4067 - } 4068 - 4069 - handler = _create_raw_flow_rule(dev, ft_prio, dst, fs_matcher, 4070 - flow_context, flow_act, 4071 - cmd_in, inlen, dst_num); 4072 - 4073 - if (IS_ERR(handler)) { 4074 - err = PTR_ERR(handler); 4075 - goto destroy_ft; 4076 - } 4077 - 4078 - mutex_unlock(&dev->flow_db->lock); 4079 - atomic_inc(&fs_matcher->usecnt); 4080 - handler->flow_matcher = fs_matcher; 4081 - 4082 - kfree(dst); 4083 - 4084 - return handler; 4085 - 4086 - destroy_ft: 4087 - put_flow_table(dev, ft_prio, false); 4088 - unlock: 4089 - mutex_unlock(&dev->flow_db->lock); 4090 - kfree(dst); 4091 - 4092 - return ERR_PTR(err); 4093 - } 4094 - 4095 - static u32 mlx5_ib_flow_action_flags_to_accel_xfrm_flags(u32 mlx5_flags) 4096 - { 4097 - u32 flags = 0; 4098 - 4099 - if (mlx5_flags & MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA) 4100 - flags |= MLX5_ACCEL_XFRM_FLAG_REQUIRE_METADATA; 4101 - 4102 - return flags; 4103 - } 4104 - 4105 - #define MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA 4106 - static struct ib_flow_action * 4107 - mlx5_ib_create_flow_action_esp(struct ib_device *device, 4108 - const struct ib_flow_action_attrs_esp *attr, 4109 - struct uverbs_attr_bundle *attrs) 4110 - { 4111 - struct mlx5_ib_dev *mdev = to_mdev(device); 4112 - struct ib_uverbs_flow_action_esp_keymat_aes_gcm *aes_gcm; 4113 - struct mlx5_accel_esp_xfrm_attrs accel_attrs = {}; 4114 - struct mlx5_ib_flow_action *action; 4115 - u64 action_flags; 4116 - u64 flags; 4117 - int err = 0; 4118 - 4119 - err = uverbs_get_flags64( 4120 - &action_flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS, 4121 - ((MLX5_FLOW_ACTION_ESP_CREATE_LAST_SUPPORTED << 1) - 1)); 4122 - if (err) 4123 - return ERR_PTR(err); 4124 - 4125 - flags = mlx5_ib_flow_action_flags_to_accel_xfrm_flags(action_flags); 4126 - 4127 - /* We current only support a subset of the standard features. Only a 4128 - * keymat of type AES_GCM, with icv_len == 16, iv_algo == SEQ and esn 4129 - * (with overlap). Full offload mode isn't supported. 4130 - */ 4131 - if (!attr->keymat || attr->replay || attr->encap || 4132 - attr->spi || attr->seq || attr->tfc_pad || 4133 - attr->hard_limit_pkts || 4134 - (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED | 4135 - IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT))) 4136 - return ERR_PTR(-EOPNOTSUPP); 4137 - 4138 - if (attr->keymat->protocol != 4139 - IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM) 4140 - return ERR_PTR(-EOPNOTSUPP); 4141 - 4142 - aes_gcm = &attr->keymat->keymat.aes_gcm; 4143 - 4144 - if (aes_gcm->icv_len != 16 || 4145 - aes_gcm->iv_algo != IB_UVERBS_FLOW_ACTION_IV_ALGO_SEQ) 4146 - return ERR_PTR(-EOPNOTSUPP); 4147 - 4148 - action = kmalloc(sizeof(*action), GFP_KERNEL); 4149 - if (!action) 4150 - return ERR_PTR(-ENOMEM); 4151 - 4152 - action->esp_aes_gcm.ib_flags = attr->flags; 4153 - memcpy(&accel_attrs.keymat.aes_gcm.aes_key, &aes_gcm->aes_key, 4154 - sizeof(accel_attrs.keymat.aes_gcm.aes_key)); 4155 - accel_attrs.keymat.aes_gcm.key_len = aes_gcm->key_len * 8; 4156 - memcpy(&accel_attrs.keymat.aes_gcm.salt, &aes_gcm->salt, 4157 - sizeof(accel_attrs.keymat.aes_gcm.salt)); 4158 - memcpy(&accel_attrs.keymat.aes_gcm.seq_iv, &aes_gcm->iv, 4159 - sizeof(accel_attrs.keymat.aes_gcm.seq_iv)); 4160 - accel_attrs.keymat.aes_gcm.icv_len = aes_gcm->icv_len * 8; 4161 - accel_attrs.keymat.aes_gcm.iv_algo = MLX5_ACCEL_ESP_AES_GCM_IV_ALGO_SEQ; 4162 - accel_attrs.keymat_type = MLX5_ACCEL_ESP_KEYMAT_AES_GCM; 4163 - 4164 - accel_attrs.esn = attr->esn; 4165 - if (attr->flags & IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED) 4166 - accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED; 4167 - if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW) 4168 - accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; 4169 - 4170 - if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT) 4171 - accel_attrs.action |= MLX5_ACCEL_ESP_ACTION_ENCRYPT; 4172 - 4173 - action->esp_aes_gcm.ctx = 4174 - mlx5_accel_esp_create_xfrm(mdev->mdev, &accel_attrs, flags); 4175 - if (IS_ERR(action->esp_aes_gcm.ctx)) { 4176 - err = PTR_ERR(action->esp_aes_gcm.ctx); 4177 - goto err_parse; 4178 - } 4179 - 4180 - action->esp_aes_gcm.ib_flags = attr->flags; 4181 - 4182 - return &action->ib_action; 4183 - 4184 - err_parse: 4185 - kfree(action); 4186 - return ERR_PTR(err); 4187 - } 4188 - 4189 - static int 4190 - mlx5_ib_modify_flow_action_esp(struct ib_flow_action *action, 4191 - const struct ib_flow_action_attrs_esp *attr, 4192 - struct uverbs_attr_bundle *attrs) 4193 - { 4194 - struct mlx5_ib_flow_action *maction = to_mflow_act(action); 4195 - struct mlx5_accel_esp_xfrm_attrs accel_attrs; 4196 - int err = 0; 4197 - 4198 - if (attr->keymat || attr->replay || attr->encap || 4199 - attr->spi || attr->seq || attr->tfc_pad || 4200 - attr->hard_limit_pkts || 4201 - (attr->flags & ~(IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED | 4202 - IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS | 4203 - IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW))) 4204 - return -EOPNOTSUPP; 4205 - 4206 - /* Only the ESN value or the MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP can 4207 - * be modified. 4208 - */ 4209 - if (!(maction->esp_aes_gcm.ib_flags & 4210 - IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED) && 4211 - attr->flags & (IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED | 4212 - IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW)) 4213 - return -EINVAL; 4214 - 4215 - memcpy(&accel_attrs, &maction->esp_aes_gcm.ctx->attrs, 4216 - sizeof(accel_attrs)); 4217 - 4218 - accel_attrs.esn = attr->esn; 4219 - if (attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW) 4220 - accel_attrs.flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; 4221 - else 4222 - accel_attrs.flags &= ~MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP; 4223 - 4224 - err = mlx5_accel_esp_modify_xfrm(maction->esp_aes_gcm.ctx, 4225 - &accel_attrs); 4226 - if (err) 4227 - return err; 4228 - 4229 - maction->esp_aes_gcm.ib_flags &= 4230 - ~IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW; 4231 - maction->esp_aes_gcm.ib_flags |= 4232 - attr->flags & IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW; 4233 - 4234 - return 0; 4235 - } 4236 - 4237 - static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action) 4238 - { 4239 - struct mlx5_ib_flow_action *maction = to_mflow_act(action); 4240 - 4241 - switch (action->type) { 4242 - case IB_FLOW_ACTION_ESP: 4243 - /* 4244 - * We only support aes_gcm by now, so we implicitly know this is 4245 - * the underline crypto. 4246 - */ 4247 - mlx5_accel_esp_destroy_xfrm(maction->esp_aes_gcm.ctx); 4248 - break; 4249 - case IB_FLOW_ACTION_UNSPECIFIED: 4250 - mlx5_ib_destroy_flow_action_raw(maction); 4251 - break; 4252 - default: 4253 - WARN_ON(true); 4254 - break; 4255 - } 4256 - 4257 - kfree(maction); 4258 - return 0; 4259 - } 4260 - 4261 2582 static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) 4262 2583 { 4263 2584 struct mlx5_ib_dev *dev = to_mdev(ibqp->device); ··· 4192 5871 return -ENOMEM; 4193 5872 } 4194 5873 4195 - static int mlx5_ib_stage_flow_db_init(struct mlx5_ib_dev *dev) 4196 - { 4197 - dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL); 4198 - 4199 - if (!dev->flow_db) 4200 - return -ENOMEM; 4201 - 4202 - mutex_init(&dev->flow_db->lock); 4203 - 4204 - return 0; 4205 - } 4206 - 4207 - static void mlx5_ib_stage_flow_db_cleanup(struct mlx5_ib_dev *dev) 4208 - { 4209 - kfree(dev->flow_db); 4210 - } 4211 - 4212 5874 static int mlx5_ib_enable_driver(struct ib_device *dev) 4213 5875 { 4214 5876 struct mlx5_ib_dev *mdev = to_mdev(dev); ··· 4218 5914 .check_mr_status = mlx5_ib_check_mr_status, 4219 5915 .create_ah = mlx5_ib_create_ah, 4220 5916 .create_cq = mlx5_ib_create_cq, 4221 - .create_flow = mlx5_ib_create_flow, 4222 5917 .create_qp = mlx5_ib_create_qp, 4223 5918 .create_srq = mlx5_ib_create_srq, 4224 5919 .dealloc_pd = mlx5_ib_dealloc_pd, ··· 4226 5923 .dereg_mr = mlx5_ib_dereg_mr, 4227 5924 .destroy_ah = mlx5_ib_destroy_ah, 4228 5925 .destroy_cq = mlx5_ib_destroy_cq, 4229 - .destroy_flow = mlx5_ib_destroy_flow, 4230 - .destroy_flow_action = mlx5_ib_destroy_flow_action, 4231 5926 .destroy_qp = mlx5_ib_destroy_qp, 4232 5927 .destroy_srq = mlx5_ib_destroy_srq, 4233 5928 .detach_mcast = mlx5_ib_mcg_detach, ··· 4268 5967 INIT_RDMA_OBJ_SIZE(ib_pd, mlx5_ib_pd, ibpd), 4269 5968 INIT_RDMA_OBJ_SIZE(ib_srq, mlx5_ib_srq, ibsrq), 4270 5969 INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx5_ib_ucontext, ibucontext), 4271 - }; 4272 - 4273 - static const struct ib_device_ops mlx5_ib_dev_flow_ipsec_ops = { 4274 - .create_flow_action_esp = mlx5_ib_create_flow_action_esp, 4275 - .modify_flow_action_esp = mlx5_ib_modify_flow_action_esp, 4276 5970 }; 4277 5971 4278 5972 static const struct ib_device_ops mlx5_ib_dev_ipoib_enhanced_ops = { ··· 4400 6104 MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM) 4401 6105 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_dm_ops); 4402 6106 4403 - if (mlx5_accel_ipsec_device_caps(dev->mdev) & 4404 - MLX5_ACCEL_IPSEC_CAP_DEVICE) 4405 - ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_flow_ipsec_ops); 4406 6107 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_ops); 4407 6108 4408 6109 if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)) ··· 4728 6435 STAGE_CREATE(MLX5_IB_STAGE_INIT, 4729 6436 mlx5_ib_stage_init_init, 4730 6437 mlx5_ib_stage_init_cleanup), 4731 - STAGE_CREATE(MLX5_IB_STAGE_FLOW_DB, 4732 - mlx5_ib_stage_flow_db_init, 4733 - mlx5_ib_stage_flow_db_cleanup), 6438 + STAGE_CREATE(MLX5_IB_STAGE_FS, 6439 + mlx5_ib_fs_init, 6440 + mlx5_ib_fs_cleanup), 4734 6441 STAGE_CREATE(MLX5_IB_STAGE_CAPS, 4735 6442 mlx5_ib_stage_caps_init, 4736 6443 mlx5_ib_stage_caps_cleanup), ··· 4791 6498 STAGE_CREATE(MLX5_IB_STAGE_INIT, 4792 6499 mlx5_ib_stage_init_init, 4793 6500 mlx5_ib_stage_init_cleanup), 4794 - STAGE_CREATE(MLX5_IB_STAGE_FLOW_DB, 4795 - mlx5_ib_stage_flow_db_init, 4796 - mlx5_ib_stage_flow_db_cleanup), 6501 + STAGE_CREATE(MLX5_IB_STAGE_FS, 6502 + mlx5_ib_fs_init, 6503 + mlx5_ib_fs_cleanup), 4797 6504 STAGE_CREATE(MLX5_IB_STAGE_CAPS, 4798 6505 mlx5_ib_stage_caps_init, 4799 6506 mlx5_ib_stage_caps_cleanup),

+1 -19

drivers/infiniband/hw/mlx5/mlx5_ib.h

··· 805 805 806 806 enum mlx5_ib_stages { 807 807 MLX5_IB_STAGE_INIT, 808 - MLX5_IB_STAGE_FLOW_DB, 808 + MLX5_IB_STAGE_FS, 809 809 MLX5_IB_STAGE_CAPS, 810 810 MLX5_IB_STAGE_NON_DEFAULT_CB, 811 811 MLX5_IB_STAGE_ROCE, ··· 1359 1359 void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid); 1360 1360 void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev); 1361 1361 void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev); 1362 - struct mlx5_ib_flow_handler *mlx5_ib_raw_fs_rule_add( 1363 - struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher, 1364 - struct mlx5_flow_context *flow_context, 1365 - struct mlx5_flow_act *flow_act, u32 counter_id, 1366 - void *cmd_in, int inlen, int dest_id, int dest_type); 1367 - bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type); 1368 - bool mlx5_ib_devx_is_flow_counter(void *obj, u32 offset, u32 *counter_id); 1369 - void mlx5_ib_destroy_flow_action_raw(struct mlx5_ib_flow_action *maction); 1370 1362 #else 1371 1363 static inline int 1372 1364 mlx5_ib_devx_create(struct mlx5_ib_dev *dev, ··· 1366 1374 static inline void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid) {} 1367 1375 static inline void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev) {} 1368 1376 static inline void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev) {} 1369 - static inline bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, 1370 - int *dest_type) 1371 - { 1372 - return false; 1373 - } 1374 - static inline void 1375 - mlx5_ib_destroy_flow_action_raw(struct mlx5_ib_flow_action *maction) 1376 - { 1377 - return; 1378 - }; 1379 1377 #endif 1380 1378 static inline void init_query_mad(struct ib_smp *mad) 1381 1379 {

Configure Feed

Configure Feed