matrixfurry.com / monado
The open source OpenXR runtime
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

h/mercury: Better naming

Moses Turner d7eece4c 8e3a47dd

+161 -161
+3 -3
src/xrt/tracking/hand/mercury/hg_image_math.hpp
··· 75 75 struct xrt_vec2 76 76 raycoord(ht_view *htv, struct xrt_vec2 model_out) 77 77 { 78 - model_out.x *= htv->htd->multiply_px_coord_for_undistort; 79 - model_out.y *= htv->htd->multiply_px_coord_for_undistort; 78 + model_out.x *= htv->hgt->multiply_px_coord_for_undistort; 79 + model_out.y *= htv->hgt->multiply_px_coord_for_undistort; 80 80 cv::Mat in_px_coords(1, 1, CV_32FC2); 81 81 float *write_in; 82 82 write_in = in_px_coords.ptr<float>(0); ··· 84 84 write_in[1] = model_out.y; 85 85 cv::Mat out_ray(1, 1, CV_32FC2); 86 86 87 - if (htv->htd->use_fisheye) { 87 + if (htv->hgt->use_fisheye) { 88 88 cv::fisheye::undistortPoints(in_px_coords, out_ray, htv->cameraMatrix, htv->distortion); 89 89 } else { 90 90 cv::undistortPoints(in_px_coords, out_ray, htv->cameraMatrix, htv->distortion);
+19 -19
src/xrt/tracking/hand/mercury/hg_model.hpp
··· 30 30 OrtStatus *status = wrap->api->expr; \ 31 31 if (status != nullptr) { \ 32 32 const char *msg = wrap->api->GetErrorMessage(status); \ 33 - HT_ERROR(htd, "[%s:%d]: %s\n", __FILE__, __LINE__, msg); \ 33 + HG_ERROR(hgt, "[%s:%d]: %s\n", __FILE__, __LINE__, msg); \ 34 34 wrap->api->ReleaseStatus(status); \ 35 35 assert(false); \ 36 36 } \ ··· 148 148 } 149 149 150 150 static void 151 - init_hand_detection(HandTracking *htd, onnx_wrap *wrap) 151 + init_hand_detection(HandTracking *hgt, onnx_wrap *wrap) 152 152 { 153 - std::filesystem::path path = htd->models_folder; 153 + std::filesystem::path path = hgt->models_folder; 154 154 155 155 path /= "grayscale_detection.onnx"; 156 156 ··· 204 204 XRT_TRACE_MARKER(); 205 205 206 206 ht_view *view = (ht_view *)ptr; 207 - HandTracking *htd = view->htd; 207 + HandTracking *hgt = view->hgt; 208 208 onnx_wrap *wrap = &view->detection; 209 209 cv::Mat &data_400x640 = view->run_model_on_this; 210 210 ··· 264 264 output->center = _pt; 265 265 output->size_px = size; 266 266 267 - if (htd->debug_scribble) { 267 + if (hgt->debug_scribble) { 268 268 cv::Point2i pt(_pt.x, _pt.y); 269 269 cv::rectangle(debug_frame, 270 270 cv::Rect(pt - cv::Point2i(size / 2, size / 2), cv::Size(size, size)), ··· 277 277 } 278 278 279 279 static void 280 - init_keypoint_estimation(HandTracking *htd, onnx_wrap *wrap) 280 + init_keypoint_estimation(HandTracking *hgt, onnx_wrap *wrap) 281 281 { 282 282 283 - std::filesystem::path path = htd->models_folder; 283 + std::filesystem::path path = hgt->models_folder; 284 284 285 285 path /= "grayscale_keypoint_simdr.onnx"; 286 286 ··· 337 337 keypoint_estimation_run_info *info = (keypoint_estimation_run_info *)ptr; 338 338 339 339 onnx_wrap *wrap = &info->view->keypoint[info->hand_idx]; 340 - struct HandTracking *htd = info->view->htd; 340 + struct HandTracking *hgt = info->view->hgt; 341 341 342 342 cv::Mat &debug = info->view->debug_out_to_this; 343 343 ··· 415 415 tan_space.kps[i] = raycoord(info->view, loc); 416 416 } 417 417 418 - if (htd->debug_scribble) { 418 + if (hgt->debug_scribble) { 419 419 for (int finger = 0; finger < 5; finger++) { 420 420 cv::Point last = {(int)keypoints_global[0].x, (int)keypoints_global[0].y}; 421 421 for (int joint = 0; joint < 4; joint++) { ··· 439 439 440 440 441 441 static void 442 - init_keypoint_estimation_new(HandTracking *htd, onnx_wrap *wrap) 442 + init_keypoint_estimation_new(HandTracking *hgt, onnx_wrap *wrap) 443 443 { 444 444 445 - std::filesystem::path path = htd->models_folder; 445 + std::filesystem::path path = hgt->models_folder; 446 446 447 447 path /= "grayscale_keypoint_new.onnx"; 448 448 ··· 468 468 469 469 ORT(CreateCpuMemoryInfo(OrtArenaAllocator, OrtMemTypeDefault, &wrap->meminfo)); 470 470 471 - // HT_DEBUG(this->device, "Loading hand detection model from file '%s'", path.c_str()); 471 + // HG_DEBUG(this->device, "Loading hand detection model from file '%s'", path.c_str()); 472 472 ORT(CreateSession(wrap->env, path.c_str(), opts, &wrap->session)); 473 473 assert(wrap->session != NULL); 474 474 ··· 502 502 keypoint_estimation_run_info *info = (keypoint_estimation_run_info *)ptr; 503 503 504 504 onnx_wrap *wrap = &info->view->keypoint[info->hand_idx]; 505 - struct HandTracking *htd = info->view->htd; 505 + struct HandTracking *hgt = info->view->hgt; 506 506 507 507 // Factor out starting here 508 508 ··· 594 594 tan_space.kps[i] = raycoord(info->view, loc); 595 595 } 596 596 597 - if (htd->debug_scribble) { 597 + if (hgt->debug_scribble) { 598 598 for (int finger = 0; finger < 5; finger++) { 599 599 cv::Point last = {(int)keypoints_global[0].x, (int)keypoints_global[0].y}; 600 600 for (int joint = 0; joint < 4; joint++) { ··· 665 665 { 666 666 667 667 ht_view *view = (ht_view *)ptr; 668 - HandTracking *htd = view->htd; 668 + HandTracking *hgt = view->hgt; 669 669 onnx_wrap *wrap = &view->detection; 670 670 cv::Mat &data_400x640 = view->run_model_on_this; 671 671 ··· 678 678 ncnn_option_t opt = ncnn_option_create(); 679 679 ncnn_option_set_use_vulkan_compute(opt, 1); 680 680 681 - ncnn_extractor_t ex = ncnn_extractor_create(htd->net); 681 + ncnn_extractor_t ex = ncnn_extractor_create(hgt->net); 682 682 683 683 ncnn_extractor_set_option(ex, opt); 684 684 ··· 761 761 XRT_TRACE_MARKER(); 762 762 keypoint_estimation_run_info *info = (keypoint_estimation_run_info *)ptr; 763 763 764 - struct HandTracking *htd = info->view->htd; 764 + struct HandTracking *hgt = info->view->hgt; 765 765 766 766 // Factor out starting here 767 767 ··· 816 816 ncnn_option_t opt = ncnn_option_create(); 817 817 ncnn_option_set_use_vulkan_compute(opt, 1); 818 818 819 - ncnn_extractor_t ex = ncnn_extractor_create(htd->net_keypoint); 819 + ncnn_extractor_t ex = ncnn_extractor_create(hgt->net_keypoint); 820 820 821 821 ncnn_extractor_set_option(ex, opt); 822 822 ··· 860 860 tan_space.kps[i] = raycoord(info->view, loc); 861 861 } 862 862 863 - if (htd->debug_scribble) { 863 + if (hgt->debug_scribble) { 864 864 for (int finger = 0; finger < 5; finger++) { 865 865 cv::Point last = {(int)keypoints_global[0].x, (int)keypoints_global[0].y}; 866 866 for (int joint = 0; joint < 4; joint++) {
+133 -133
src/xrt/tracking/hand/mercury/hg_sync.cpp
··· 21 21 XRT_SPACE_RELATION_POSITION_VALID_BIT | XRT_SPACE_RELATION_POSITION_TRACKED_BIT); 22 22 23 23 static void 24 - hgJointDisparityMath(struct HandTracking *htd, Hand2D *hand_in_left, Hand2D *hand_in_right, Hand3D *out_hand) 24 + hgJointDisparityMath(struct HandTracking *hgt, Hand2D *hand_in_left, Hand2D *hand_in_right, Hand3D *out_hand) 25 25 { 26 26 for (int i = 0; i < 21; i++) { 27 27 // Believe it or not, this is where the 3D stuff happens! 28 - float t = htd->baseline / (hand_in_left->kps[i].x - hand_in_right->kps[i].x); 28 + float t = hgt->baseline / (hand_in_left->kps[i].x - hand_in_right->kps[i].x); 29 29 30 30 out_hand->kps[i].z = -t; 31 31 32 32 out_hand->kps[i].x = (hand_in_left->kps[i].x * t); 33 33 out_hand->kps[i].y = -hand_in_left->kps[i].y * t; 34 34 35 - out_hand->kps[i].x += htd->baseline + (hand_in_right->kps[i].x * t); 35 + out_hand->kps[i].x += hgt->baseline + (hand_in_right->kps[i].x * t); 36 36 out_hand->kps[i].y += -hand_in_right->kps[i].y * t; 37 37 38 38 out_hand->kps[i].x *= .5; ··· 45 45 */ 46 46 47 47 static bool 48 - getCalibration(struct HandTracking *htd, t_stereo_camera_calibration *calibration) 48 + getCalibration(struct HandTracking *hgt, t_stereo_camera_calibration *calibration) 49 49 { 50 50 xrt::auxiliary::tracking::StereoCameraCalibrationWrapper wrap(calibration); 51 51 xrt_vec3 trans = {(float)wrap.camera_translation_mat(0, 0), (float)wrap.camera_translation_mat(1, 0), 52 52 (float)wrap.camera_translation_mat(2, 0)}; 53 - htd->baseline = m_vec3_len(trans); 54 - HT_DEBUG(htd, "I think the baseline is %f meters!", htd->baseline); 53 + hgt->baseline = m_vec3_len(trans); 54 + HG_DEBUG(hgt, "I think the baseline is %f meters!", hgt->baseline); 55 55 // Note, this assumes camera 0 is the left camera and camera 1 is the right camera. 56 - // If you find one with the opposite arrangement, you'll need to invert htd->baseline, and look at 56 + // If you find one with the opposite arrangement, you'll need to invert hgt->baseline, and look at 57 57 // hgJointDisparityMath 58 58 59 - htd->use_fisheye = wrap.view[0].use_fisheye; 59 + hgt->use_fisheye = wrap.view[0].use_fisheye; 60 60 61 - if (htd->use_fisheye) { 62 - HT_DEBUG(htd, "I think the cameras are fisheye!"); 61 + if (hgt->use_fisheye) { 62 + HG_DEBUG(hgt, "I think the cameras are fisheye!"); 63 63 } else { 64 - HT_DEBUG(htd, "I think the cameras are not fisheye!"); 64 + HG_DEBUG(hgt, "I think the cameras are not fisheye!"); 65 65 } 66 66 67 67 cv::Matx34d P1; ··· 70 70 cv::Matx44d Q; 71 71 72 72 // We only want R1 and R2, we don't care about anything else 73 - if (htd->use_fisheye) { 73 + if (hgt->use_fisheye) { 74 74 cv::fisheye::stereoRectify(wrap.view[0].intrinsics_mat, // cameraMatrix1 75 75 wrap.view[0].distortion_fisheye_mat, // distCoeffs1 76 76 wrap.view[1].intrinsics_mat, // cameraMatrix2 ··· 78 78 wrap.view[0].image_size_pixels_cv, // imageSize* 79 79 wrap.camera_rotation_mat, // R 80 80 wrap.camera_translation_mat, // T 81 - htd->views[0].rotate_camera_to_stereo_camera, // R1 82 - htd->views[1].rotate_camera_to_stereo_camera, // R2 81 + hgt->views[0].rotate_camera_to_stereo_camera, // R1 82 + hgt->views[1].rotate_camera_to_stereo_camera, // R2 83 83 P1, // P1 84 84 P2, // P2 85 85 Q, // Q ··· 93 93 wrap.view[0].image_size_pixels_cv, // imageSize* 94 94 wrap.camera_rotation_mat, // R 95 95 wrap.camera_translation_mat, // T 96 - htd->views[0].rotate_camera_to_stereo_camera, // R1 97 - htd->views[1].rotate_camera_to_stereo_camera, // R2 96 + hgt->views[0].rotate_camera_to_stereo_camera, // R1 97 + hgt->views[1].rotate_camera_to_stereo_camera, // R2 98 98 P1, // P1 99 99 P2, // P2 100 100 Q, // Q ··· 107 107 108 108 //* Good enough guess that view 0 and view 1 are the same size. 109 109 for (int i = 0; i < 2; i++) { 110 - htd->views[i].cameraMatrix = wrap.view[i].intrinsics_mat; 110 + hgt->views[i].cameraMatrix = wrap.view[i].intrinsics_mat; 111 111 112 - if (htd->use_fisheye) { 113 - htd->views[i].distortion = wrap.view[i].distortion_fisheye_mat; 112 + if (hgt->use_fisheye) { 113 + hgt->views[i].distortion = wrap.view[i].distortion_fisheye_mat; 114 114 } else { 115 - htd->views[i].distortion = wrap.view[i].distortion_mat; 115 + hgt->views[i].distortion = wrap.view[i].distortion_mat; 116 116 } 117 117 118 - if (htd->log_level <= U_LOGGING_DEBUG) { 119 - HT_DEBUG(htd, "R%d ->", i); 120 - std::cout << htd->views[i].rotate_camera_to_stereo_camera << std::endl; 118 + if (hgt->log_level <= U_LOGGING_DEBUG) { 119 + HG_DEBUG(hgt, "R%d ->", i); 120 + std::cout << hgt->views[i].rotate_camera_to_stereo_camera << std::endl; 121 121 122 - HT_DEBUG(htd, "K%d ->", i); 123 - std::cout << htd->views[i].cameraMatrix << std::endl; 122 + HG_DEBUG(hgt, "K%d ->", i); 123 + std::cout << hgt->views[i].cameraMatrix << std::endl; 124 124 125 - HT_DEBUG(htd, "D%d ->", i); 126 - std::cout << htd->views[i].distortion << std::endl; 125 + HG_DEBUG(hgt, "D%d ->", i); 126 + std::cout << hgt->views[i].distortion << std::endl; 127 127 } 128 128 } 129 129 130 - htd->calibration_one_view_size_px.w = wrap.view[0].image_size_pixels.w; 131 - htd->calibration_one_view_size_px.h = wrap.view[0].image_size_pixels.h; 130 + hgt->calibration_one_view_size_px.w = wrap.view[0].image_size_pixels.w; 131 + hgt->calibration_one_view_size_px.h = wrap.view[0].image_size_pixels.h; 132 132 133 - htd->last_frame_one_view_size_px = htd->calibration_one_view_size_px; 134 - htd->multiply_px_coord_for_undistort = 1.0f; 133 + hgt->last_frame_one_view_size_px = hgt->calibration_one_view_size_px; 134 + hgt->multiply_px_coord_for_undistort = 1.0f; 135 135 136 - cv::Matx33d rotate_stereo_camera_to_left_camera = htd->views[0].rotate_camera_to_stereo_camera.inv(); 136 + cv::Matx33d rotate_stereo_camera_to_left_camera = hgt->views[0].rotate_camera_to_stereo_camera.inv(); 137 137 138 138 xrt_matrix_3x3 s; 139 139 s.v[0] = rotate_stereo_camera_to_left_camera(0, 0); ··· 156 156 // have swapped row-major and col-major - remember, if you transpose a rotation matrix, you get its inverse. 157 157 // Doesn't matter that I don't understand - non-inverted looks definitely wrong, inverted looks definitely 158 158 // right. 159 - math_quat_invert(&tmp, &htd->stereo_camera_to_left_camera); 159 + math_quat_invert(&tmp, &hgt->stereo_camera_to_left_camera); 160 160 161 161 return true; 162 162 } 163 163 164 164 static void 165 - getModelsFolder(struct HandTracking *htd) 165 + getModelsFolder(struct HandTracking *hgt) 166 166 { 167 167 // Please bikeshed me on this! I don't know where is the best place to put this stuff! 168 168 #if 0 169 169 char exec_location[1024] = {}; 170 170 readlink("/proc/self/exe", exec_location, 1024); 171 171 172 - HT_DEBUG(htd, "Exec at %s\n", exec_location); 172 + HG_DEBUG(hgt, "Exec at %s\n", exec_location); 173 173 174 174 int end = 0; 175 175 while (exec_location[end] != '\0') { 176 - HT_DEBUG(htd, "%d", end); 176 + HG_DEBUG(hgt, "%d", end); 177 177 end++; 178 178 } 179 179 180 180 while (exec_location[end] != '/' && end != 0) { 181 - HT_DEBUG(htd, "%d %c", end, exec_location[end]); 181 + HG_DEBUG(hgt, "%d %c", end, exec_location[end]); 182 182 exec_location[end] = '\0'; 183 183 end--; 184 184 } 185 185 186 186 strcat(exec_location, "../share/monado/hand-tracking-models/"); 187 - strcpy(htd->startup_config.model_slug, exec_location); 187 + strcpy(hgt->startup_config.model_slug, exec_location); 188 188 #else 189 189 const char *xdg_home = getenv("XDG_CONFIG_HOME"); 190 190 const char *home = getenv("HOME"); 191 191 if (xdg_home != NULL) { 192 - strcpy(htd->models_folder, xdg_home); 192 + strcpy(hgt->models_folder, xdg_home); 193 193 } else if (home != NULL) { 194 - strcpy(htd->models_folder, home); 194 + strcpy(hgt->models_folder, home); 195 195 } else { 196 196 assert(false); 197 197 } 198 - strcat(htd->models_folder, "/.local/share/monado/hand-tracking-models/"); 198 + strcat(hgt->models_folder, "/.local/share/monado/hand-tracking-models/"); 199 199 #endif 200 200 } 201 201 ··· 227 227 } 228 228 229 229 static void 230 - applyJointWidths(struct HandTracking *htd, struct xrt_hand_joint_set *set) 230 + applyJointWidths(struct HandTracking *hgt, struct xrt_hand_joint_set *set) 231 231 { 232 232 // Thanks to Nick Klingensmith for this idea 233 233 struct xrt_hand_joint_value *gr = set->values.hand_joint_set_default; ··· 259 259 } 260 260 261 261 static bool 262 - handle_changed_image_size(HandTracking *htd, xrt_size &new_one_view_size) 262 + handle_changed_image_size(HandTracking *hgt, xrt_size &new_one_view_size) 263 263 { 264 - int gcd_calib = std::gcd(htd->calibration_one_view_size_px.h, htd->calibration_one_view_size_px.w); 264 + int gcd_calib = std::gcd(hgt->calibration_one_view_size_px.h, hgt->calibration_one_view_size_px.w); 265 265 int gcd_new = std::gcd(new_one_view_size.h, new_one_view_size.w); 266 266 267 - int lcm_h_calib = htd->calibration_one_view_size_px.h / gcd_calib; 268 - int lcm_w_calib = htd->calibration_one_view_size_px.w / gcd_calib; 267 + int lcm_h_calib = hgt->calibration_one_view_size_px.h / gcd_calib; 268 + int lcm_w_calib = hgt->calibration_one_view_size_px.w / gcd_calib; 269 269 270 270 int lcm_h_new = new_one_view_size.h / gcd_new; 271 271 int lcm_w_new = new_one_view_size.w / gcd_new; ··· 273 273 bool good = (lcm_h_calib == lcm_h_new) && (lcm_w_calib == lcm_w_new); 274 274 275 275 if (!good) { 276 - HT_WARN(htd, "Can't process this frame, wrong aspect ratio. What we wanted: %dx%d, what we got: %dx%d", 276 + HG_WARN(hgt, "Can't process this frame, wrong aspect ratio. What we wanted: %dx%d, what we got: %dx%d", 277 277 lcm_h_calib, lcm_w_calib, lcm_h_new, lcm_w_new); 278 278 return false; 279 279 } 280 280 281 - htd->multiply_px_coord_for_undistort = (float)htd->calibration_one_view_size_px.h / (float)new_one_view_size.h; 282 - htd->last_frame_one_view_size_px = new_one_view_size; 281 + hgt->multiply_px_coord_for_undistort = (float)hgt->calibration_one_view_size_px.h / (float)new_one_view_size.h; 282 + hgt->last_frame_one_view_size_px = new_one_view_size; 283 283 return true; 284 284 } 285 285 ··· 330 330 { 331 331 XRT_TRACE_MARKER(); 332 332 333 - HandTracking *htd = (struct HandTracking *)ht_sync; 333 + HandTracking *hgt = (struct HandTracking *)ht_sync; 334 334 335 - htd->current_frame_timestamp = left_frame->timestamp; 335 + hgt->current_frame_timestamp = left_frame->timestamp; 336 336 337 337 struct xrt_hand_joint_set *out_xrt_hands[2] = {out_left_hand, out_right_hand}; 338 338 ··· 347 347 const int full_height = left_frame->height; 348 348 const int full_width = left_frame->width * 2; 349 349 350 - if ((left_frame->width != (uint32_t)htd->last_frame_one_view_size_px.w) || 351 - (left_frame->height != (uint32_t)htd->last_frame_one_view_size_px.h)) { 350 + if ((left_frame->width != (uint32_t)hgt->last_frame_one_view_size_px.w) || 351 + (left_frame->height != (uint32_t)hgt->last_frame_one_view_size_px.h)) { 352 352 xrt_size new_one_view_size; 353 353 new_one_view_size.h = left_frame->height; 354 354 new_one_view_size.w = left_frame->width; 355 355 // Could be an assert, should never happen. 356 - if (!handle_changed_image_size(htd, new_one_view_size)) { 356 + if (!handle_changed_image_size(hgt, new_one_view_size)) { 357 357 return; 358 358 } 359 359 } 360 360 361 - const int view_width = htd->last_frame_one_view_size_px.w; 362 - const int view_height = htd->last_frame_one_view_size_px.h; 361 + const int view_width = hgt->last_frame_one_view_size_px.w; 362 + const int view_height = hgt->last_frame_one_view_size_px.h; 363 363 364 364 const cv::Size full_size = cv::Size(full_width, full_height); 365 365 const cv::Size view_size = cv::Size(view_width, view_height); 366 366 const cv::Point view_offsets[2] = {cv::Point(0, 0), cv::Point(view_width, 0)}; 367 367 368 - htd->views[0].run_model_on_this = cv::Mat(view_size, CV_8UC1, left_frame->data, left_frame->stride); 369 - htd->views[1].run_model_on_this = cv::Mat(view_size, CV_8UC1, right_frame->data, right_frame->stride); 368 + hgt->views[0].run_model_on_this = cv::Mat(view_size, CV_8UC1, left_frame->data, left_frame->stride); 369 + hgt->views[1].run_model_on_this = cv::Mat(view_size, CV_8UC1, right_frame->data, right_frame->stride); 370 370 371 371 372 - *out_timestamp_ns = htd->current_frame_timestamp; // No filtering, fine to do this now. Also just a reminder 372 + *out_timestamp_ns = hgt->current_frame_timestamp; // No filtering, fine to do this now. Also just a reminder 373 373 // that this took you 2 HOURS TO DEBUG THAT ONE TIME. 374 374 375 - htd->debug_scribble = u_sink_debug_is_active(&htd->debug_sink); 375 + hgt->debug_scribble = u_sink_debug_is_active(&hgt->debug_sink); 376 376 377 377 cv::Mat debug_output = {}; 378 378 xrt_frame *debug_frame = nullptr; 379 379 380 380 381 - if (htd->debug_scribble) { 381 + if (hgt->debug_scribble) { 382 382 u_frame_create_one_off(XRT_FORMAT_R8G8B8, full_width, full_height, &debug_frame); 383 - debug_frame->timestamp = htd->current_frame_timestamp; 383 + debug_frame->timestamp = hgt->current_frame_timestamp; 384 384 385 385 debug_output = cv::Mat(full_size, CV_8UC3, debug_frame->data, debug_frame->stride); 386 386 387 - cv::cvtColor(htd->views[0].run_model_on_this, debug_output(cv::Rect(view_offsets[0], view_size)), 387 + cv::cvtColor(hgt->views[0].run_model_on_this, debug_output(cv::Rect(view_offsets[0], view_size)), 388 388 cv::COLOR_GRAY2BGR); 389 - cv::cvtColor(htd->views[1].run_model_on_this, debug_output(cv::Rect(view_offsets[1], view_size)), 389 + cv::cvtColor(hgt->views[1].run_model_on_this, debug_output(cv::Rect(view_offsets[1], view_size)), 390 390 cv::COLOR_GRAY2BGR); 391 391 392 - htd->views[0].debug_out_to_this = debug_output(cv::Rect(view_offsets[0], view_size)); 393 - htd->views[1].debug_out_to_this = debug_output(cv::Rect(view_offsets[1], view_size)); 392 + hgt->views[0].debug_out_to_this = debug_output(cv::Rect(view_offsets[0], view_size)); 393 + hgt->views[1].debug_out_to_this = debug_output(cv::Rect(view_offsets[1], view_size)); 394 394 } 395 395 396 - u_worker_group_push(htd->group, run_hand_detection, &htd->views[0]); 397 - u_worker_group_push(htd->group, run_hand_detection, &htd->views[1]); 398 - u_worker_group_wait_all(htd->group); 396 + u_worker_group_push(hgt->group, run_hand_detection, &hgt->views[0]); 397 + u_worker_group_push(hgt->group, run_hand_detection, &hgt->views[1]); 398 + u_worker_group_wait_all(hgt->group); 399 399 400 400 401 401 402 402 for (int hand_idx = 0; hand_idx < 2; hand_idx++) { 403 403 404 - if (!(htd->views[0].det_outputs[hand_idx].found && htd->views[1].det_outputs[hand_idx].found)) { 404 + if (!(hgt->views[0].det_outputs[hand_idx].found && hgt->views[1].det_outputs[hand_idx].found)) { 405 405 // If we don't find this hand in *both* views 406 406 out_xrt_hands[hand_idx]->is_active = false; 407 407 continue; ··· 410 410 411 411 412 412 for (int view_idx = 0; view_idx < 2; view_idx++) { 413 - struct keypoint_estimation_run_info &inf = htd->views[view_idx].run_info[hand_idx]; 414 - inf.view = &htd->views[view_idx]; 413 + struct keypoint_estimation_run_info &inf = hgt->views[view_idx].run_info[hand_idx]; 414 + inf.view = &hgt->views[view_idx]; 415 415 inf.hand_idx = hand_idx; 416 - u_worker_group_push(htd->group, htd->keypoint_estimation_run_func, 417 - &htd->views[view_idx].run_info[hand_idx]); 416 + u_worker_group_push(hgt->group, hgt->keypoint_estimation_run_func, 417 + &hgt->views[view_idx].run_info[hand_idx]); 418 418 } 419 419 } 420 - u_worker_group_wait_all(htd->group); 420 + u_worker_group_wait_all(hgt->group); 421 421 422 422 for (int hand_idx = 0; hand_idx < 2; hand_idx++) { 423 423 if (!out_xrt_hands[hand_idx]->is_active) { 424 - htd->last_frame_hand_detected[hand_idx] = false; 424 + hgt->last_frame_hand_detected[hand_idx] = false; 425 425 continue; 426 426 } 427 - kine::kinematic_hand_4f *hand = htd->kinematic_hands[hand_idx]; 428 - if (!htd->last_frame_hand_detected[hand_idx]) { 429 - kine::init_hardcoded_statics(hand, htd->hand_size / 100.0f); 427 + kine::kinematic_hand_4f *hand = hgt->kinematic_hands[hand_idx]; 428 + if (!hgt->last_frame_hand_detected[hand_idx]) { 429 + kine::init_hardcoded_statics(hand, hgt->hand_size / 100.0f); 430 430 } 431 431 432 432 433 433 434 - Hand2D *hand_in_left_view = &htd->views[0].keypoint_outputs[hand_idx].hand_tan_space; 435 - Hand2D *hand_in_right_view = &htd->views[1].keypoint_outputs[hand_idx].hand_tan_space; 434 + Hand2D *hand_in_left_view = &hgt->views[0].keypoint_outputs[hand_idx].hand_tan_space; 435 + Hand2D *hand_in_right_view = &hgt->views[1].keypoint_outputs[hand_idx].hand_tan_space; 436 436 Hand3D hand_3d; 437 437 438 438 ··· 441 441 442 442 applyThumbIndexDrag(&hand_3d); 443 443 444 - applyJointWidths(htd, put_in_set); 444 + applyJointWidths(hgt, put_in_set); 445 445 446 - hgJointDisparityMath(htd, hand_in_left_view, hand_in_right_view, &hand_3d); 446 + hgJointDisparityMath(hgt, hand_in_left_view, hand_in_right_view, &hand_3d); 447 447 448 448 kine::optimize_new_frame(hand_3d.kps, hand, put_in_set, hand_idx); 449 449 450 450 451 451 math_pose_identity(&put_in_set->hand_pose.pose); 452 452 453 - switch (htd->output_space) { 453 + switch (hgt->output_space) { 454 454 case HT_OUTPUT_SPACE_LEFT_CAMERA: { 455 - put_in_set->hand_pose.pose.orientation = htd->stereo_camera_to_left_camera; 455 + put_in_set->hand_pose.pose.orientation = hgt->stereo_camera_to_left_camera; 456 456 } break; 457 457 case HT_OUTPUT_SPACE_CENTER_OF_STEREO_CAMERA: { 458 458 put_in_set->hand_pose.pose.orientation.w = 1.0; 459 - put_in_set->hand_pose.pose.position.x = -htd->baseline / 2; 459 + put_in_set->hand_pose.pose.position.x = -hgt->baseline / 2; 460 460 } break; 461 461 } 462 462 ··· 465 465 466 466 467 467 468 - u_frame_times_widget_push_sample(&htd->ft_widget, htd->current_frame_timestamp); 468 + u_frame_times_widget_push_sample(&hgt->ft_widget, hgt->current_frame_timestamp); 469 469 470 - if (htd->debug_scribble) { 471 - u_sink_debug_push_frame(&htd->debug_sink, debug_frame); 470 + if (hgt->debug_scribble) { 471 + u_sink_debug_push_frame(&hgt->debug_sink, debug_frame); 472 472 xrt_frame_reference(&debug_frame, NULL); 473 473 } 474 474 } ··· 495 495 { 496 496 XRT_TRACE_MARKER(); 497 497 498 - auto htd = new xrt::tracking::hand::mercury::HandTracking(); 498 + auto hgt = new xrt::tracking::hand::mercury::HandTracking(); 499 499 500 500 // Setup logging first. We like logging. 501 - htd->log_level = xrt::tracking::hand::mercury::debug_get_log_option_mercury_log(); 501 + hgt->log_level = xrt::tracking::hand::mercury::debug_get_log_option_mercury_log(); 502 502 bool use_simdr = xrt::tracking::hand::mercury::debug_get_bool_option_mercury_use_simdr_keypoint(); 503 503 504 - htd->output_space = output_space; 504 + hgt->output_space = output_space; 505 505 506 506 /* 507 507 * Get configuration 508 508 */ 509 509 510 510 assert(calib != NULL); 511 - htd->calib = NULL; 511 + hgt->calib = NULL; 512 512 // We have to reference it, getCalibration points at it. 513 - t_stereo_camera_calibration_reference(&htd->calib, calib); 514 - getCalibration(htd, calib); 515 - getModelsFolder(htd); 513 + t_stereo_camera_calibration_reference(&hgt->calib, calib); 514 + getCalibration(hgt, calib); 515 + getModelsFolder(hgt); 516 516 517 517 #ifdef USE_NCNN 518 518 { 519 - htd->net = ncnn_net_create(); 519 + hgt->net = ncnn_net_create(); 520 520 ncnn_option_t opt = ncnn_option_create(); 521 521 ncnn_option_set_use_vulkan_compute(opt, 1); 522 522 523 - ncnn_net_set_option(htd->net, opt); 524 - ncnn_net_load_param(htd->net, "/3/clones/ncnn/batch_size_2.param"); 525 - ncnn_net_load_model(htd->net, "/3/clones/ncnn/batch_size_2.bin"); 523 + ncnn_net_set_option(hgt->net, opt); 524 + ncnn_net_load_param(hgt->net, "/3/clones/ncnn/batch_size_2.param"); 525 + ncnn_net_load_model(hgt->net, "/3/clones/ncnn/batch_size_2.bin"); 526 526 } 527 527 528 528 { 529 - htd->net_keypoint = ncnn_net_create(); 529 + hgt->net_keypoint = ncnn_net_create(); 530 530 ncnn_option_t opt = ncnn_option_create(); 531 531 ncnn_option_set_use_vulkan_compute(opt, 1); 532 532 533 - ncnn_net_set_option(htd->net_keypoint, opt); 533 + ncnn_net_set_option(hgt->net_keypoint, opt); 534 534 ncnn_net_load_param( 535 - htd->net_keypoint, 535 + hgt->net_keypoint, 536 536 "/home/moses/.local/share/monado/hand-tracking-models/grayscale_keypoint_new.param"); 537 - ncnn_net_load_model(htd->net_keypoint, 537 + ncnn_net_load_model(hgt->net_keypoint, 538 538 "/home/moses/.local/share/monado/hand-tracking-models/grayscale_keypoint_new.bin"); 539 539 } 540 540 541 541 542 542 #endif 543 543 544 - htd->views[0].htd = htd; 545 - htd->views[1].htd = htd; // :) 544 + hgt->views[0].hgt = hgt; 545 + hgt->views[1].hgt = hgt; // :) 546 546 547 - init_hand_detection(htd, &htd->views[0].detection); 548 - init_hand_detection(htd, &htd->views[1].detection); 547 + init_hand_detection(hgt, &hgt->views[0].detection); 548 + init_hand_detection(hgt, &hgt->views[1].detection); 549 549 550 550 if (use_simdr) { 551 - init_keypoint_estimation(htd, &htd->views[0].keypoint[0]); 552 - init_keypoint_estimation(htd, &htd->views[0].keypoint[1]); 551 + init_keypoint_estimation(hgt, &hgt->views[0].keypoint[0]); 552 + init_keypoint_estimation(hgt, &hgt->views[0].keypoint[1]); 553 553 554 - init_keypoint_estimation(htd, &htd->views[1].keypoint[0]); 555 - init_keypoint_estimation(htd, &htd->views[1].keypoint[1]); 556 - htd->keypoint_estimation_run_func = xrt::tracking::hand::mercury::run_keypoint_estimation; 554 + init_keypoint_estimation(hgt, &hgt->views[1].keypoint[0]); 555 + init_keypoint_estimation(hgt, &hgt->views[1].keypoint[1]); 556 + hgt->keypoint_estimation_run_func = xrt::tracking::hand::mercury::run_keypoint_estimation; 557 557 } else { 558 - init_keypoint_estimation_new(htd, &htd->views[0].keypoint[0]); 559 - init_keypoint_estimation_new(htd, &htd->views[0].keypoint[1]); 558 + init_keypoint_estimation_new(hgt, &hgt->views[0].keypoint[0]); 559 + init_keypoint_estimation_new(hgt, &hgt->views[0].keypoint[1]); 560 560 561 - init_keypoint_estimation_new(htd, &htd->views[1].keypoint[0]); 562 - init_keypoint_estimation_new(htd, &htd->views[1].keypoint[1]); 563 - htd->keypoint_estimation_run_func = xrt::tracking::hand::mercury::run_keypoint_estimation_new; 561 + init_keypoint_estimation_new(hgt, &hgt->views[1].keypoint[0]); 562 + init_keypoint_estimation_new(hgt, &hgt->views[1].keypoint[1]); 563 + hgt->keypoint_estimation_run_func = xrt::tracking::hand::mercury::run_keypoint_estimation_new; 564 564 } 565 565 566 - htd->views[0].view = 0; 567 - htd->views[1].view = 1; 566 + hgt->views[0].view = 0; 567 + hgt->views[1].view = 1; 568 568 569 569 int num_threads = 4; 570 - htd->pool = u_worker_thread_pool_create(num_threads - 1, num_threads); 571 - htd->group = u_worker_group_create(htd->pool); 570 + hgt->pool = u_worker_thread_pool_create(num_threads - 1, num_threads); 571 + hgt->group = u_worker_group_create(hgt->pool); 572 572 573 - htd->hand_size = 9.5864; 574 - xrt::tracking::hand::mercury::kine::alloc_kinematic_hand(&htd->kinematic_hands[0]); 575 - xrt::tracking::hand::mercury::kine::alloc_kinematic_hand(&htd->kinematic_hands[1]); 573 + hgt->hand_size = 9.5864; 574 + xrt::tracking::hand::mercury::kine::alloc_kinematic_hand(&hgt->kinematic_hands[0]); 575 + xrt::tracking::hand::mercury::kine::alloc_kinematic_hand(&hgt->kinematic_hands[1]); 576 576 577 - u_frame_times_widget_init(&htd->ft_widget, 10.0f, 10.0f); 577 + u_frame_times_widget_init(&hgt->ft_widget, 10.0f, 10.0f); 578 578 579 - u_var_add_root(htd, "Camera-based Hand Tracker", true); 579 + u_var_add_root(hgt, "Camera-based Hand Tracker", true); 580 580 581 581 582 - u_var_add_f32(htd, &htd->hand_size, "Hand size (Centimeters between wrist and middle-proximal joint)"); 583 - u_var_add_ro_f32(htd, &htd->ft_widget.fps, "FPS!"); 584 - u_var_add_f32_timing(htd, htd->ft_widget.debug_var, "times!"); 582 + u_var_add_f32(hgt, &hgt->hand_size, "Hand size (Centimeters between wrist and middle-proximal joint)"); 583 + u_var_add_ro_f32(hgt, &hgt->ft_widget.fps, "FPS!"); 584 + u_var_add_f32_timing(hgt, hgt->ft_widget.debug_var, "times!"); 585 585 586 - u_var_add_sink_debug(htd, &htd->debug_sink, "i"); 586 + u_var_add_sink_debug(hgt, &hgt->debug_sink, "i"); 587 587 588 - HT_DEBUG(htd, "Hand Tracker initialized!"); 588 + HG_DEBUG(hgt, "Hand Tracker initialized!"); 589 589 590 - return &htd->base; 590 + return &hgt->base; 591 591 }
+6 -6
src/xrt/tracking/hand/mercury/hg_sync.hpp
··· 51 51 DEBUG_GET_ONCE_LOG_OPTION(mercury_log, "MERCURY_LOG", U_LOGGING_WARN) 52 52 DEBUG_GET_ONCE_BOOL_OPTION(mercury_use_simdr_keypoint, "MERCURY_USE_SIMDR_KEYPOINT", false) 53 53 54 - #define HT_TRACE(htd, ...) U_LOG_IFL_T(htd->log_level, __VA_ARGS__) 55 - #define HT_DEBUG(htd, ...) U_LOG_IFL_D(htd->log_level, __VA_ARGS__) 56 - #define HT_INFO(htd, ...) U_LOG_IFL_I(htd->log_level, __VA_ARGS__) 57 - #define HT_WARN(htd, ...) U_LOG_IFL_W(htd->log_level, __VA_ARGS__) 58 - #define HT_ERROR(htd, ...) U_LOG_IFL_E(htd->log_level, __VA_ARGS__) 54 + #define HG_TRACE(hgt, ...) U_LOG_IFL_T(hgt->log_level, __VA_ARGS__) 55 + #define HG_DEBUG(hgt, ...) U_LOG_IFL_D(hgt->log_level, __VA_ARGS__) 56 + #define HG_INFO(hgt, ...) U_LOG_IFL_I(hgt->log_level, __VA_ARGS__) 57 + #define HG_WARN(hgt, ...) U_LOG_IFL_W(hgt->log_level, __VA_ARGS__) 58 + #define HG_ERROR(hgt, ...) U_LOG_IFL_E(hgt->log_level, __VA_ARGS__) 59 59 60 60 #undef USE_NCNN 61 61 ··· 138 138 139 139 struct ht_view 140 140 { 141 - HandTracking *htd; 141 + HandTracking *hgt; 142 142 onnx_wrap detection; 143 143 onnx_wrap keypoint[2]; 144 144 int view;