selftests/ublk: add shared memory zero-copy support in kublk

+352 -2

2 changed files

expand all

tools

testing

selftests

ublk

kublk.c

kublk.h

+338 -2

tools/testing/selftests/ublk/kublk.c

··· 4 4 */ 5 5 6 6 #include <linux/fs.h> 7 + #include <sys/un.h> 7 8 #include "kublk.h" 8 9 9 10 #define MAX_NR_TGT_ARG 64 ··· 1093 1092 } 1094 1093 1095 1094 1095 + /* 1096 + * Shared memory registration socket listener. 1097 + * 1098 + * The parent daemon context listens on a per-device unix socket at 1099 + * /run/ublk/ublkb<dev_id>.sock for shared memory registration requests 1100 + * from clients. Clients send a memfd via SCM_RIGHTS; the server 1101 + * registers it with the kernel, mmaps it, and returns the assigned index. 1102 + */ 1103 + #define UBLK_SHMEM_SOCK_DIR "/run/ublk" 1104 + 1105 + /* defined in kublk.h, shared with file_backed.c (loop target) */ 1106 + struct ublk_shmem_entry shmem_table[UBLK_BUF_MAX]; 1107 + int shmem_count; 1108 + 1109 + static void ublk_shmem_sock_path(int dev_id, char *buf, size_t len) 1110 + { 1111 + snprintf(buf, len, "%s/ublkb%d.sock", UBLK_SHMEM_SOCK_DIR, dev_id); 1112 + } 1113 + 1114 + static int ublk_shmem_sock_create(int dev_id) 1115 + { 1116 + struct sockaddr_un addr = { .sun_family = AF_UNIX }; 1117 + char path[108]; 1118 + int fd; 1119 + 1120 + mkdir(UBLK_SHMEM_SOCK_DIR, 0755); 1121 + ublk_shmem_sock_path(dev_id, path, sizeof(path)); 1122 + unlink(path); 1123 + 1124 + fd = socket(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, 0); 1125 + if (fd < 0) 1126 + return -1; 1127 + 1128 + snprintf(addr.sun_path, sizeof(addr.sun_path), "%s", path); 1129 + if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) { 1130 + close(fd); 1131 + return -1; 1132 + } 1133 + 1134 + listen(fd, 4); 1135 + ublk_dbg(UBLK_DBG_DEV, "shmem socket created: %s\n", path); 1136 + return fd; 1137 + } 1138 + 1139 + static void ublk_shmem_sock_destroy(int dev_id, int sock_fd) 1140 + { 1141 + char path[108]; 1142 + 1143 + if (sock_fd >= 0) 1144 + close(sock_fd); 1145 + ublk_shmem_sock_path(dev_id, path, sizeof(path)); 1146 + unlink(path); 1147 + } 1148 + 1149 + /* Receive a memfd from a client via SCM_RIGHTS */ 1150 + static int ublk_shmem_recv_fd(int client_fd) 1151 + { 1152 + char buf[1]; 1153 + struct iovec iov = { .iov_base = buf, .iov_len = sizeof(buf) }; 1154 + union { 1155 + char cmsg_buf[CMSG_SPACE(sizeof(int))]; 1156 + struct cmsghdr align; 1157 + } u; 1158 + struct msghdr msg = { 1159 + .msg_iov = &iov, 1160 + .msg_iovlen = 1, 1161 + .msg_control = u.cmsg_buf, 1162 + .msg_controllen = sizeof(u.cmsg_buf), 1163 + }; 1164 + struct cmsghdr *cmsg; 1165 + 1166 + if (recvmsg(client_fd, &msg, 0) <= 0) 1167 + return -1; 1168 + 1169 + cmsg = CMSG_FIRSTHDR(&msg); 1170 + if (!cmsg || cmsg->cmsg_level != SOL_SOCKET || 1171 + cmsg->cmsg_type != SCM_RIGHTS) 1172 + return -1; 1173 + 1174 + return *(int *)CMSG_DATA(cmsg); 1175 + } 1176 + 1177 + /* Register a shared memory buffer: store fd, mmap it, return index */ 1178 + static int ublk_shmem_register(int shmem_fd) 1179 + { 1180 + off_t size; 1181 + void *base; 1182 + int idx; 1183 + 1184 + if (shmem_count >= UBLK_BUF_MAX) 1185 + return -1; 1186 + 1187 + size = lseek(shmem_fd, 0, SEEK_END); 1188 + if (size <= 0) 1189 + return -1; 1190 + 1191 + base = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, 1192 + shmem_fd, 0); 1193 + if (base == MAP_FAILED) 1194 + return -1; 1195 + 1196 + idx = shmem_count++; 1197 + shmem_table[idx].fd = shmem_fd; 1198 + shmem_table[idx].mmap_base = base; 1199 + shmem_table[idx].size = size; 1200 + 1201 + ublk_dbg(UBLK_DBG_DEV, "shmem registered: index=%d fd=%d size=%zu\n", 1202 + idx, shmem_fd, (size_t)size); 1203 + return idx; 1204 + } 1205 + 1206 + static void ublk_shmem_unregister_all(void) 1207 + { 1208 + int i; 1209 + 1210 + for (i = 0; i < shmem_count; i++) { 1211 + if (shmem_table[i].mmap_base) { 1212 + munmap(shmem_table[i].mmap_base, 1213 + shmem_table[i].size); 1214 + close(shmem_table[i].fd); 1215 + shmem_table[i].mmap_base = NULL; 1216 + } 1217 + } 1218 + shmem_count = 0; 1219 + } 1220 + 1221 + static int ublk_ctrl_reg_buf(struct ublk_dev *dev, void *addr, size_t size) 1222 + { 1223 + struct ublk_shmem_buf_reg buf_reg = { 1224 + .addr = (unsigned long)addr, 1225 + .len = size, 1226 + }; 1227 + struct ublk_ctrl_cmd_data data = { 1228 + .cmd_op = UBLK_U_CMD_REG_BUF, 1229 + .flags = CTRL_CMD_HAS_BUF, 1230 + .addr = (unsigned long)&buf_reg, 1231 + .len = sizeof(buf_reg), 1232 + }; 1233 + 1234 + return __ublk_ctrl_cmd(dev, &data); 1235 + } 1236 + 1237 + /* 1238 + * Handle one client connection: receive memfd, mmap it, register 1239 + * the VA range with kernel, send back the assigned index. 1240 + */ 1241 + static void ublk_shmem_handle_client(int sock_fd, struct ublk_dev *dev) 1242 + { 1243 + int client_fd, memfd, idx, ret; 1244 + int32_t reply; 1245 + off_t size; 1246 + void *base; 1247 + 1248 + client_fd = accept(sock_fd, NULL, NULL); 1249 + if (client_fd < 0) 1250 + return; 1251 + 1252 + memfd = ublk_shmem_recv_fd(client_fd); 1253 + if (memfd < 0) { 1254 + reply = -1; 1255 + goto out; 1256 + } 1257 + 1258 + /* mmap the memfd in server address space */ 1259 + size = lseek(memfd, 0, SEEK_END); 1260 + if (size <= 0) { 1261 + reply = -1; 1262 + close(memfd); 1263 + goto out; 1264 + } 1265 + base = mmap(NULL, size, PROT_READ | PROT_WRITE, 1266 + MAP_SHARED | MAP_POPULATE, memfd, 0); 1267 + if (base == MAP_FAILED) { 1268 + reply = -1; 1269 + close(memfd); 1270 + goto out; 1271 + } 1272 + 1273 + /* Register server's VA range with kernel for PFN matching */ 1274 + ret = ublk_ctrl_reg_buf(dev, base, size); 1275 + if (ret < 0) { 1276 + ublk_dbg(UBLK_DBG_DEV, 1277 + "shmem_zc: kernel reg failed %d\n", ret); 1278 + munmap(base, size); 1279 + close(memfd); 1280 + reply = ret; 1281 + goto out; 1282 + } 1283 + 1284 + /* Store in table for I/O handling */ 1285 + idx = ublk_shmem_register(memfd); 1286 + if (idx >= 0) { 1287 + shmem_table[idx].mmap_base = base; 1288 + shmem_table[idx].size = size; 1289 + } 1290 + reply = idx; 1291 + out: 1292 + send(client_fd, &reply, sizeof(reply), 0); 1293 + close(client_fd); 1294 + } 1295 + 1296 + struct shmem_listener_info { 1297 + int dev_id; 1298 + int stop_efd; /* eventfd to signal listener to stop */ 1299 + int sock_fd; /* listener socket fd (output) */ 1300 + struct ublk_dev *dev; 1301 + }; 1302 + 1303 + /* 1304 + * Socket listener thread: runs in the parent daemon context alongside 1305 + * the I/O threads. Accepts shared memory registration requests from 1306 + * clients via SCM_RIGHTS. Exits when stop_efd is signaled. 1307 + */ 1308 + static void *ublk_shmem_listener_fn(void *data) 1309 + { 1310 + struct shmem_listener_info *info = data; 1311 + struct pollfd pfds[2]; 1312 + 1313 + info->sock_fd = ublk_shmem_sock_create(info->dev_id); 1314 + if (info->sock_fd < 0) 1315 + return NULL; 1316 + 1317 + pfds[0].fd = info->sock_fd; 1318 + pfds[0].events = POLLIN; 1319 + pfds[1].fd = info->stop_efd; 1320 + pfds[1].events = POLLIN; 1321 + 1322 + while (1) { 1323 + int ret = poll(pfds, 2, -1); 1324 + 1325 + if (ret < 0) 1326 + break; 1327 + 1328 + /* Stop signal from parent */ 1329 + if (pfds[1].revents & POLLIN) 1330 + break; 1331 + 1332 + /* Client connection */ 1333 + if (pfds[0].revents & POLLIN) 1334 + ublk_shmem_handle_client(info->sock_fd, info->dev); 1335 + } 1336 + 1337 + return NULL; 1338 + } 1339 + 1340 + static int ublk_shmem_htlb_setup(const struct dev_ctx *ctx, 1341 + struct ublk_dev *dev) 1342 + { 1343 + int fd, idx, ret; 1344 + struct stat st; 1345 + void *base; 1346 + 1347 + fd = open(ctx->htlb_path, O_RDWR); 1348 + if (fd < 0) { 1349 + ublk_err("htlb: can't open %s\n", ctx->htlb_path); 1350 + return -errno; 1351 + } 1352 + 1353 + if (fstat(fd, &st) < 0 || st.st_size <= 0) { 1354 + ublk_err("htlb: invalid file size\n"); 1355 + close(fd); 1356 + return -EINVAL; 1357 + } 1358 + 1359 + base = mmap(NULL, st.st_size, PROT_READ | PROT_WRITE, 1360 + MAP_SHARED | MAP_POPULATE, fd, 0); 1361 + if (base == MAP_FAILED) { 1362 + ublk_err("htlb: mmap failed\n"); 1363 + close(fd); 1364 + return -ENOMEM; 1365 + } 1366 + 1367 + ret = ublk_ctrl_reg_buf(dev, base, st.st_size); 1368 + if (ret < 0) { 1369 + ublk_err("htlb: reg_buf failed: %d\n", ret); 1370 + munmap(base, st.st_size); 1371 + close(fd); 1372 + return ret; 1373 + } 1374 + 1375 + if (shmem_count >= UBLK_BUF_MAX) { 1376 + munmap(base, st.st_size); 1377 + close(fd); 1378 + return -ENOMEM; 1379 + } 1380 + 1381 + idx = shmem_count++; 1382 + shmem_table[idx].fd = fd; 1383 + shmem_table[idx].mmap_base = base; 1384 + shmem_table[idx].size = st.st_size; 1385 + 1386 + ublk_dbg(UBLK_DBG_DEV, "htlb registered: index=%d size=%zu\n", 1387 + idx, (size_t)st.st_size); 1388 + return 0; 1389 + } 1390 + 1096 1391 static int ublk_start_daemon(const struct dev_ctx *ctx, struct ublk_dev *dev) 1097 1392 { 1098 1393 const struct ublksrv_ctrl_dev_info *dinfo = &dev->dev_info; 1394 + struct shmem_listener_info linfo = {}; 1099 1395 struct ublk_thread_info *tinfo; 1100 1396 unsigned long long extra_flags = 0; 1101 1397 cpu_set_t *affinity_buf; 1102 1398 unsigned char (*q_thread_map)[UBLK_MAX_QUEUES] = NULL; 1399 + uint64_t stop_val = 1; 1400 + pthread_t listener; 1103 1401 void *thread_ret; 1104 1402 sem_t ready; 1105 1403 int ret, i; ··· 1487 1187 goto fail_start; 1488 1188 } 1489 1189 1190 + if (ctx->htlb_path) { 1191 + ret = ublk_shmem_htlb_setup(ctx, dev); 1192 + if (ret < 0) { 1193 + ublk_err("htlb setup failed: %d\n", ret); 1194 + ublk_ctrl_stop_dev(dev); 1195 + goto fail_start; 1196 + } 1197 + } 1198 + 1490 1199 ublk_ctrl_get_info(dev); 1491 1200 if (ctx->fg) 1492 1201 ublk_ctrl_dump(dev); 1493 1202 else 1494 1203 ublk_send_dev_event(ctx, dev, dev->dev_info.dev_id); 1495 1204 fail_start: 1496 - /* wait until we are terminated */ 1497 - for (i = 0; i < dev->nthreads; i++) 1205 + /* 1206 + * Wait for I/O threads to exit. While waiting, a listener 1207 + * thread accepts shared memory registration requests from 1208 + * clients via a per-device unix socket (SCM_RIGHTS fd passing). 1209 + */ 1210 + linfo.dev_id = dinfo->dev_id; 1211 + linfo.dev = dev; 1212 + linfo.stop_efd = eventfd(0, 0); 1213 + if (linfo.stop_efd >= 0) 1214 + pthread_create(&listener, NULL, 1215 + ublk_shmem_listener_fn, &linfo); 1216 + 1217 + for (i = 0; i < (int)dev->nthreads; i++) 1498 1218 pthread_join(tinfo[i].thread, &thread_ret); 1219 + 1220 + /* Signal listener thread to stop and wait for it */ 1221 + if (linfo.stop_efd >= 0) { 1222 + write(linfo.stop_efd, &stop_val, sizeof(stop_val)); 1223 + pthread_join(listener, NULL); 1224 + close(linfo.stop_efd); 1225 + ublk_shmem_sock_destroy(dinfo->dev_id, linfo.sock_fd); 1226 + } 1227 + ublk_shmem_unregister_all(); 1499 1228 free(tinfo); 1500 1229 fail: 1501 1230 for (i = 0; i < dinfo->nr_hw_queues; i++) ··· 1954 1625 FEAT_NAME(UBLK_F_SAFE_STOP_DEV), 1955 1626 FEAT_NAME(UBLK_F_BATCH_IO), 1956 1627 FEAT_NAME(UBLK_F_NO_AUTO_PART_SCAN), 1628 + FEAT_NAME(UBLK_F_SHMEM_ZC), 1957 1629 }; 1958 1630 struct ublk_dev *dev; 1959 1631 __u64 features = 0; ··· 2127 1797 { "safe", 0, NULL, 0 }, 2128 1798 { "batch", 0, NULL, 'b'}, 2129 1799 { "no_auto_part_scan", 0, NULL, 0 }, 1800 + { "shmem_zc", 0, NULL, 0 }, 1801 + { "htlb", 1, NULL, 0 }, 2130 1802 { 0, 0, 0, 0 } 2131 1803 }; 2132 1804 const struct ublk_tgt_ops *ops = NULL; ··· 2244 1912 ctx.safe_stop = 1; 2245 1913 if (!strcmp(longopts[option_idx].name, "no_auto_part_scan")) 2246 1914 ctx.flags |= UBLK_F_NO_AUTO_PART_SCAN; 1915 + if (!strcmp(longopts[option_idx].name, "shmem_zc")) 1916 + ctx.flags |= UBLK_F_SHMEM_ZC; 1917 + if (!strcmp(longopts[option_idx].name, "htlb")) 1918 + ctx.htlb_path = strdup(optarg); 2247 1919 break; 2248 1920 case '?': 2249 1921 /*

+14

tools/testing/selftests/ublk/kublk.h

··· 96 96 /* for 'update_size' command */ 97 97 unsigned long long size; 98 98 99 + char *htlb_path; 100 + 99 101 union { 100 102 struct stripe_ctx stripe; 101 103 struct fault_inject_ctx fault_inject; ··· 603 601 io->result = 0; 604 602 } 605 603 } 604 + 605 + /* shared memory zero-copy support */ 606 + #define UBLK_BUF_MAX 256 607 + 608 + struct ublk_shmem_entry { 609 + int fd; 610 + void *mmap_base; 611 + size_t size; 612 + }; 613 + 614 + extern struct ublk_shmem_entry shmem_table[UBLK_BUF_MAX]; 615 + extern int shmem_count; 606 616 607 617 extern const struct ublk_tgt_ops null_tgt_ops; 608 618 extern const struct ublk_tgt_ops loop_tgt_ops;

Configure Feed

Configure Feed