RDMA/hns: Refactor root BT allocation for MTR

+147 -85

1 changed file

expand all

drivers

infiniband

hns

hns_roce_hem.c

+147 -85

drivers/infiniband/hw/hns/hns_roce_hem.c

··· 1053 1053 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table); 1054 1054 } 1055 1055 1056 - struct roce_hem_item { 1056 + struct hns_roce_hem_item { 1057 1057 struct list_head list; /* link all hems in the same bt level */ 1058 1058 struct list_head sibling; /* link all hems in last hop for mtt */ 1059 1059 void *addr; ··· 1063 1063 int end; /* end buf offset in this hem */ 1064 1064 }; 1065 1065 1066 - static struct roce_hem_item *hem_list_alloc_item(struct hns_roce_dev *hr_dev, 1067 - int start, int end, 1068 - int count, bool exist_bt, 1069 - int bt_level) 1066 + /* All HEM items are linked in a tree structure */ 1067 + struct hns_roce_hem_head { 1068 + struct list_head branch[HNS_ROCE_MAX_BT_REGION]; 1069 + struct list_head root; 1070 + struct list_head leaf; 1071 + }; 1072 + 1073 + static struct hns_roce_hem_item * 1074 + hem_list_alloc_item(struct hns_roce_dev *hr_dev, int start, int end, int count, 1075 + bool exist_bt, int bt_level) 1070 1076 { 1071 - struct roce_hem_item *hem; 1077 + struct hns_roce_hem_item *hem; 1072 1078 1073 1079 hem = kzalloc(sizeof(*hem), GFP_KERNEL); 1074 1080 if (!hem) ··· 1099 1093 } 1100 1094 1101 1095 static void hem_list_free_item(struct hns_roce_dev *hr_dev, 1102 - struct roce_hem_item *hem, bool exist_bt) 1096 + struct hns_roce_hem_item *hem, bool exist_bt) 1103 1097 { 1104 1098 if (exist_bt) 1105 1099 dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN, ··· 1110 1104 static void hem_list_free_all(struct hns_roce_dev *hr_dev, 1111 1105 struct list_head *head, bool exist_bt) 1112 1106 { 1113 - struct roce_hem_item *hem, *temp_hem; 1107 + struct hns_roce_hem_item *hem, *temp_hem; 1114 1108 1115 1109 list_for_each_entry_safe(hem, temp_hem, head, list) { 1116 1110 list_del(&hem->list); ··· 1126 1120 1127 1121 /* assign L0 table address to hem from root bt */ 1128 1122 static void hem_list_assign_bt(struct hns_roce_dev *hr_dev, 1129 - struct roce_hem_item *hem, void *cpu_addr, 1123 + struct hns_roce_hem_item *hem, void *cpu_addr, 1130 1124 u64 phy_addr) 1131 1125 { 1132 1126 hem->addr = cpu_addr; 1133 1127 hem->dma_addr = (dma_addr_t)phy_addr; 1134 1128 } 1135 1129 1136 - static inline bool hem_list_page_is_in_range(struct roce_hem_item *hem, 1130 + static inline bool hem_list_page_is_in_range(struct hns_roce_hem_item *hem, 1137 1131 int offset) 1138 1132 { 1139 1133 return (hem->start <= offset && offset <= hem->end); 1140 1134 } 1141 1135 1142 - static struct roce_hem_item *hem_list_search_item(struct list_head *ba_list, 1143 - int page_offset) 1136 + static struct hns_roce_hem_item *hem_list_search_item(struct list_head *ba_list, 1137 + int page_offset) 1144 1138 { 1145 - struct roce_hem_item *hem, *temp_hem; 1146 - struct roce_hem_item *found = NULL; 1139 + struct hns_roce_hem_item *hem, *temp_hem; 1140 + struct hns_roce_hem_item *found = NULL; 1147 1141 1148 1142 list_for_each_entry_safe(hem, temp_hem, ba_list, list) { 1149 1143 if (hem_list_page_is_in_range(hem, page_offset)) { ··· 1233 1227 int offset, struct list_head *mid_bt, 1234 1228 struct list_head *btm_bt) 1235 1229 { 1236 - struct roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL }; 1230 + struct hns_roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL }; 1237 1231 struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL]; 1238 - struct roce_hem_item *cur, *pre; 1232 + struct hns_roce_hem_item *cur, *pre; 1239 1233 const int hopnum = r->hopnum; 1240 1234 int start_aligned; 1241 1235 int distance; ··· 1313 1307 return ret; 1314 1308 } 1315 1309 1316 - static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev, 1317 - struct hns_roce_hem_list *hem_list, int unit, 1318 - const struct hns_roce_buf_region *regions, 1319 - int region_cnt) 1310 + static struct hns_roce_hem_item * 1311 + alloc_root_hem(struct hns_roce_dev *hr_dev, int unit, int *max_ba_num, 1312 + const struct hns_roce_buf_region *regions, int region_cnt) 1320 1313 { 1321 - struct list_head temp_list[HNS_ROCE_MAX_BT_REGION]; 1322 - struct roce_hem_item *hem, *temp_hem, *root_hem; 1323 1314 const struct hns_roce_buf_region *r; 1324 - struct list_head temp_root; 1325 - struct list_head temp_btm; 1326 - void *cpu_base; 1327 - u64 phy_base; 1328 - int ret = 0; 1315 + struct hns_roce_hem_item *hem; 1329 1316 int ba_num; 1330 1317 int offset; 1331 - int total; 1332 - int step; 1333 - int i; 1334 - 1335 - r = &regions[0]; 1336 - root_hem = hem_list_search_item(&hem_list->root_bt, r->offset); 1337 - if (root_hem) 1338 - return 0; 1339 1318 1340 1319 ba_num = hns_roce_hem_list_calc_root_ba(regions, region_cnt, unit); 1341 1320 if (ba_num < 1) 1342 - return -ENOMEM; 1321 + return ERR_PTR(-ENOMEM); 1343 1322 1344 1323 if (ba_num > unit) 1345 - return -ENOBUFS; 1324 + return ERR_PTR(-ENOBUFS); 1346 1325 1347 - ba_num = min_t(int, ba_num, unit); 1348 - INIT_LIST_HEAD(&temp_root); 1349 - offset = r->offset; 1326 + offset = regions[0].offset; 1350 1327 /* indicate to last region */ 1351 1328 r = &regions[region_cnt - 1]; 1352 - root_hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1, 1353 - ba_num, true, 0); 1329 + hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1, 1330 + ba_num, true, 0); 1331 + if (!hem) 1332 + return ERR_PTR(-ENOMEM); 1333 + 1334 + *max_ba_num = ba_num; 1335 + 1336 + return hem; 1337 + } 1338 + 1339 + static int alloc_fake_root_bt(struct hns_roce_dev *hr_dev, void *cpu_base, 1340 + u64 phy_base, const struct hns_roce_buf_region *r, 1341 + struct list_head *branch_head, 1342 + struct list_head *leaf_head) 1343 + { 1344 + struct hns_roce_hem_item *hem; 1345 + 1346 + hem = hem_list_alloc_item(hr_dev, r->offset, r->offset + r->count - 1, 1347 + r->count, false, 0); 1348 + if (!hem) 1349 + return -ENOMEM; 1350 + 1351 + hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base); 1352 + list_add(&hem->list, branch_head); 1353 + list_add(&hem->sibling, leaf_head); 1354 + 1355 + return r->count; 1356 + } 1357 + 1358 + static int setup_middle_bt(struct hns_roce_dev *hr_dev, void *cpu_base, 1359 + int unit, const struct hns_roce_buf_region *r, 1360 + const struct list_head *branch_head) 1361 + { 1362 + struct hns_roce_hem_item *hem, *temp_hem; 1363 + int total = 0; 1364 + int offset; 1365 + int step; 1366 + 1367 + step = hem_list_calc_ba_range(r->hopnum, 1, unit); 1368 + if (step < 1) 1369 + return -EINVAL; 1370 + 1371 + /* if exist mid bt, link L1 to L0 */ 1372 + list_for_each_entry_safe(hem, temp_hem, branch_head, list) { 1373 + offset = (hem->start - r->offset) / step * BA_BYTE_LEN; 1374 + hem_list_link_bt(hr_dev, cpu_base + offset, hem->dma_addr); 1375 + total++; 1376 + } 1377 + 1378 + return total; 1379 + } 1380 + 1381 + static int 1382 + setup_root_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem_list *hem_list, 1383 + int unit, int max_ba_num, struct hns_roce_hem_head *head, 1384 + const struct hns_roce_buf_region *regions, int region_cnt) 1385 + { 1386 + const struct hns_roce_buf_region *r; 1387 + struct hns_roce_hem_item *root_hem; 1388 + void *cpu_base; 1389 + u64 phy_base; 1390 + int i, total; 1391 + int ret; 1392 + 1393 + root_hem = list_first_entry(&head->root, 1394 + struct hns_roce_hem_item, list); 1354 1395 if (!root_hem) 1355 1396 return -ENOMEM; 1356 - list_add(&root_hem->list, &temp_root); 1357 - 1358 - hem_list->root_ba = root_hem->dma_addr; 1359 - 1360 - INIT_LIST_HEAD(&temp_btm); 1361 - for (i = 0; i < region_cnt; i++) 1362 - INIT_LIST_HEAD(&temp_list[i]); 1363 1397 1364 1398 total = 0; 1365 - for (i = 0; i < region_cnt && total < ba_num; i++) { 1399 + for (i = 0; i < region_cnt && total < max_ba_num; i++) { 1366 1400 r = &regions[i]; 1367 1401 if (!r->count) 1368 1402 continue; ··· 1414 1368 /* if hopnum is 0 or 1, cut a new fake hem from the root bt 1415 1369 * which's address share to all regions. 1416 1370 */ 1417 - if (hem_list_is_bottom_bt(r->hopnum, 0)) { 1418 - hem = hem_list_alloc_item(hr_dev, r->offset, 1419 - r->offset + r->count - 1, 1420 - r->count, false, 0); 1421 - if (!hem) { 1422 - ret = -ENOMEM; 1423 - goto err_exit; 1424 - } 1425 - hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base); 1426 - list_add(&hem->list, &temp_list[i]); 1427 - list_add(&hem->sibling, &temp_btm); 1428 - total += r->count; 1429 - } else { 1430 - step = hem_list_calc_ba_range(r->hopnum, 1, unit); 1431 - if (step < 1) { 1432 - ret = -EINVAL; 1433 - goto err_exit; 1434 - } 1435 - /* if exist mid bt, link L1 to L0 */ 1436 - list_for_each_entry_safe(hem, temp_hem, 1437 - &hem_list->mid_bt[i][1], list) { 1438 - offset = (hem->start - r->offset) / step * 1439 - BA_BYTE_LEN; 1440 - hem_list_link_bt(hr_dev, cpu_base + offset, 1441 - hem->dma_addr); 1442 - total++; 1443 - } 1444 - } 1371 + if (hem_list_is_bottom_bt(r->hopnum, 0)) 1372 + ret = alloc_fake_root_bt(hr_dev, cpu_base, phy_base, r, 1373 + &head->branch[i], &head->leaf); 1374 + else 1375 + ret = setup_middle_bt(hr_dev, cpu_base, unit, r, 1376 + &hem_list->mid_bt[i][1]); 1377 + 1378 + if (ret < 0) 1379 + return ret; 1380 + 1381 + total += ret; 1445 1382 } 1446 1383 1447 - list_splice(&temp_btm, &hem_list->btm_bt); 1448 - list_splice(&temp_root, &hem_list->root_bt); 1384 + list_splice(&head->leaf, &hem_list->btm_bt); 1385 + list_splice(&head->root, &hem_list->root_bt); 1449 1386 for (i = 0; i < region_cnt; i++) 1450 - list_splice(&temp_list[i], &hem_list->mid_bt[i][0]); 1387 + list_splice(&head->branch[i], &hem_list->mid_bt[i][0]); 1451 1388 1452 1389 return 0; 1390 + } 1453 1391 1454 - err_exit: 1392 + static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev, 1393 + struct hns_roce_hem_list *hem_list, int unit, 1394 + const struct hns_roce_buf_region *regions, 1395 + int region_cnt) 1396 + { 1397 + struct hns_roce_hem_item *root_hem; 1398 + struct hns_roce_hem_head head; 1399 + int max_ba_num; 1400 + int ret; 1401 + int i; 1402 + 1403 + root_hem = hem_list_search_item(&hem_list->root_bt, regions[0].offset); 1404 + if (root_hem) 1405 + return 0; 1406 + 1407 + max_ba_num = 0; 1408 + root_hem = alloc_root_hem(hr_dev, unit, &max_ba_num, regions, 1409 + region_cnt); 1410 + if (IS_ERR(root_hem)) 1411 + return PTR_ERR(root_hem); 1412 + 1413 + /* List head for storing all allocated HEM items */ 1414 + INIT_LIST_HEAD(&head.root); 1415 + INIT_LIST_HEAD(&head.leaf); 1455 1416 for (i = 0; i < region_cnt; i++) 1456 - hem_list_free_all(hr_dev, &temp_list[i], false); 1417 + INIT_LIST_HEAD(&head.branch[i]); 1457 1418 1458 - hem_list_free_all(hr_dev, &temp_root, true); 1419 + hem_list->root_ba = root_hem->dma_addr; 1420 + list_add(&root_hem->list, &head.root); 1421 + ret = setup_root_hem(hr_dev, hem_list, unit, max_ba_num, &head, regions, 1422 + region_cnt); 1423 + if (ret) { 1424 + for (i = 0; i < region_cnt; i++) 1425 + hem_list_free_all(hr_dev, &head.branch[i], false); 1426 + 1427 + hem_list_free_all(hr_dev, &head.root, true); 1428 + } 1459 1429 1460 1430 return ret; 1461 1431 } ··· 1557 1495 int offset, int *mtt_cnt, u64 *phy_addr) 1558 1496 { 1559 1497 struct list_head *head = &hem_list->btm_bt; 1560 - struct roce_hem_item *hem, *temp_hem; 1498 + struct hns_roce_hem_item *hem, *temp_hem; 1561 1499 void *cpu_base = NULL; 1562 1500 u64 phy_base = 0; 1563 1501 int nr = 0;

Configure Feed

Configure Feed