The open source OpenXR runtime
h/mercury: Add 2D model input and output visualizers
+146
-20
+83
-12
src/xrt/tracking/hand/mercury/hg_model.cpp
+83
-12
src/xrt/tracking/hand/mercury/hg_model.cpp
···
325
325
size_t plane_size = 80 * 60;
326
326
327
327
for (int hand_idx = 0; hand_idx < 2; hand_idx++) {
328
-
const float *this_side_data = out_data + hand_idx * plane_size * 2;
328
+
float *this_side_data = out_data + hand_idx * plane_size * 2;
329
329
int max_idx = argmax(this_side_data, 4800);
330
330
331
331
hand_bounding_box *output = info->outputs[hand_idx];
···
362
362
cv::rectangle(debug_frame,
363
363
cv::Rect(pt - cv::Point2i(size / 2, size / 2), cv::Size(size, size)),
364
364
PINK, 1);
365
+
}
366
+
}
367
+
368
+
if (hgt->debug_scribble) {
369
+
// note: this will multiply the model outputs by 255, don't do anything with them after this.
370
+
int top_of_rect_y = 8; // 8 + 128 + 8 + 128 + 8;
371
+
int left_of_rect_x = 8 + ((128 + 8) * 4);
372
+
int start_y = top_of_rect_y + ((240 + 8) * view->view);
373
+
int start_x = left_of_rect_x + 8 + 320 + 8;
374
+
cv::Rect p = cv::Rect(left_of_rect_x, start_y, 320, 240);
375
+
376
+
_240x320_uint8.copyTo(hgt->visualizers.mat(p));
377
+
378
+
{
379
+
cv::Rect p = cv::Rect(start_x + (hand_idx * (80 + 8)), start_y, 80, 60);
380
+
cv::Mat start(cv::Size(80, 60), CV_32FC1, this_side_data, 80 * sizeof(float));
381
+
start *= 255.0;
382
+
start.copyTo(hgt->visualizers.mat(p));
383
+
}
384
+
385
+
{
386
+
cv::Rect p = cv::Rect(start_x + (hand_idx * (80 + 8)), start_y + ((60 + 8)), 80, 60);
387
+
cv::Mat start(cv::Size(80, 60), CV_32FC1, this_side_data + 4800, 80 * sizeof(float));
388
+
start *= 255.0;
389
+
start.copyTo(hgt->visualizers.mat(p));
365
390
}
366
391
}
367
392
}
···
647
672
}
648
673
649
674
void
675
+
make_keypoint_heatmap_output(int camera_idx, int hand_idx, int grid_pt_x, int grid_pt_y, float *plane, cv::Mat &out)
676
+
{
677
+
int root_x = 8 + ((1 + 2 * hand_idx) * (128 + 8));
678
+
int root_y = 8 + (camera_idx * (128 + 8));
679
+
680
+
int org_x = (root_x) + (grid_pt_x * 25);
681
+
int org_y = (root_y) + (grid_pt_y * 25);
682
+
cv::Rect p = cv::Rect(org_x, org_y, 22, 22);
683
+
684
+
685
+
cv::Mat start(cv::Size(22, 22), CV_32FC1, plane, 22 * sizeof(float));
686
+
// cv::Mat start(cv::Size(40, 42), CV_32FC1, plane, 40 * 42 * sizeof(float));
687
+
start *= 255.0;
688
+
689
+
start.copyTo(out(p));
690
+
}
691
+
692
+
693
+
void
650
694
run_keypoint_estimation_new(void *ptr)
651
695
{
652
696
XRT_TRACE_MARKER();
···
682
726
cv::Matx23f go_there = getAffineTransform(src_tri, dst_tri);
683
727
cv::Matx23f go_back = getAffineTransform(dst_tri, src_tri);
684
728
729
+
cv::Mat data_128x128_uint8;
730
+
685
731
{
686
732
XRT_TRACE_IDENT(transforms);
687
-
688
-
cv::Mat data_128x128_uint8;
689
733
690
734
cv::warpAffine(info->view->run_model_on_this, data_128x128_uint8, go_there, cv::Size(128, 128),
691
735
cv::INTER_LINEAR);
···
742
786
tan_space.kps[i] = raycoord(info->view, loc);
743
787
}
744
788
745
-
if (hgt->debug_scribble && hgt->tuneable_values.scribble_keypoint_model_outputs) {
789
+
if (hgt->debug_scribble) {
790
+
int data_acc_idx = 0;
791
+
792
+
int root_x = 8 + ((2 * info->hand_idx) * (128 + 8));
793
+
int root_y = 8 + (info->view->view * (128 + 8));
794
+
795
+
cv::Rect p = cv::Rect(root_x, root_y, 128, 128);
796
+
797
+
data_128x128_uint8.copyTo(hgt->visualizers.mat(p));
798
+
799
+
make_keypoint_heatmap_output(info->view->view, info->hand_idx, 0, 0,
800
+
out_data + (data_acc_idx * plane_size), hgt->visualizers.mat);
801
+
data_acc_idx++;
802
+
746
803
for (int finger = 0; finger < 5; finger++) {
747
-
cv::Point last = {(int)keypoints_global[0].x, (int)keypoints_global[0].y};
748
804
for (int joint = 0; joint < 4; joint++) {
749
-
cv::Point the_new = {(int)keypoints_global[1 + finger * 4 + joint].x,
750
-
(int)keypoints_global[1 + finger * 4 + joint].y};
751
805
752
-
cv::line(debug, last, the_new, colors[info->hand_idx]);
753
-
last = the_new;
806
+
make_keypoint_heatmap_output(info->view->view, info->hand_idx, 1 + joint, finger,
807
+
out_data + (data_acc_idx * plane_size),
808
+
hgt->visualizers.mat);
809
+
data_acc_idx++;
754
810
}
755
811
}
756
812
757
-
for (int i = 0; i < 21; i++) {
758
-
xrt_vec2 loc = keypoints_global[i];
759
-
handDot(debug, loc, 2, (float)(i) / 21.0, 1, 2);
813
+
814
+
815
+
if (hgt->tuneable_values.scribble_keypoint_model_outputs) {
816
+
for (int finger = 0; finger < 5; finger++) {
817
+
cv::Point last = {(int)keypoints_global[0].x, (int)keypoints_global[0].y};
818
+
for (int joint = 0; joint < 4; joint++) {
819
+
cv::Point the_new = {(int)keypoints_global[1 + finger * 4 + joint].x,
820
+
(int)keypoints_global[1 + finger * 4 + joint].y};
821
+
822
+
cv::line(debug, last, the_new, colors[info->hand_idx]);
823
+
last = the_new;
824
+
}
825
+
}
826
+
827
+
for (int i = 0; i < 21; i++) {
828
+
xrt_vec2 loc = keypoints_global[i];
829
+
handDot(debug, loc, 2, (float)(i) / 21.0, 1, 2);
830
+
}
760
831
}
761
832
}
762
833
+47
-5
src/xrt/tracking/hand/mercury/hg_sync.cpp
+47
-5
src/xrt/tracking/hand/mercury/hg_sync.cpp
···
14
14
#include "util/u_hand_tracking.h"
15
15
#include "math/m_vec2.h"
16
16
#include "util/u_misc.h"
17
+
#include "xrt/xrt_frame.h"
17
18
18
19
19
20
#include <numeric>
···
590
591
{
591
592
this->base.process = &HandTracking::cCallbackProcess;
592
593
this->base.destroy = &HandTracking::cCallbackDestroy;
593
-
u_sink_debug_init(&this->debug_sink);
594
+
u_sink_debug_init(&this->debug_sink_ann);
595
+
u_sink_debug_init(&this->debug_sink_model);
594
596
}
595
597
596
598
HandTracking::~HandTracking()
597
599
{
598
-
u_sink_debug_destroy(&this->debug_sink);
600
+
u_sink_debug_destroy(&this->debug_sink_ann);
601
+
u_sink_debug_destroy(&this->debug_sink_model);
602
+
603
+
xrt_frame_reference(&this->visualizers.old_frame, NULL);
599
604
600
605
release_onnx_wrap(&this->views[0].keypoint[0]);
601
606
release_onnx_wrap(&this->views[0].keypoint[1]);
···
672
677
*out_timestamp_ns = hgt->current_frame_timestamp; // No filtering, fine to do this now. Also just a reminder
673
678
// that this took you 2 HOURS TO DEBUG THAT ONE TIME.
674
679
675
-
hgt->debug_scribble = u_sink_debug_is_active(&hgt->debug_sink);
680
+
hgt->debug_scribble =
681
+
u_sink_debug_is_active(&hgt->debug_sink_ann) && u_sink_debug_is_active(&hgt->debug_sink_model);
676
682
677
683
cv::Mat debug_output = {};
678
684
xrt_frame *debug_frame = nullptr;
···
692
698
hgt->views[0].debug_out_to_this = debug_output(cv::Rect(view_offsets[0], view_size));
693
699
hgt->views[1].debug_out_to_this = debug_output(cv::Rect(view_offsets[1], view_size));
694
700
scribble_image_boundary(hgt);
701
+
702
+
//
703
+
704
+
struct xrt_frame *new_model_inputs_and_outputs = NULL;
705
+
706
+
// Let's check that the collage size is actually as big as we think it is
707
+
static_assert(1064 == (8 + ((128 + 8) * 4) + ((320 + 8)) + ((80 + 8) * 2) + 8));
708
+
static_assert(504 == (240 + 240 + 8 + 8 + 8));
709
+
710
+
const int w = 1064;
711
+
const int h = 504;
712
+
713
+
u_frame_create_one_off(XRT_FORMAT_L8, w, h, &hgt->visualizers.xrtframe);
714
+
hgt->visualizers.xrtframe->timestamp = hgt->current_frame_timestamp;
715
+
716
+
cv::Size size = cv::Size(w, h);
717
+
718
+
hgt->visualizers.mat =
719
+
cv::Mat(size, CV_8U, hgt->visualizers.xrtframe->data, hgt->visualizers.xrtframe->stride);
720
+
721
+
if (hgt->visualizers.old_frame == NULL) {
722
+
// There wasn't a previous frame so let's setup the background
723
+
hgt->visualizers.mat = 255;
724
+
} else {
725
+
// They had better be the same size.
726
+
memcpy(hgt->visualizers.xrtframe->data, hgt->visualizers.old_frame->data,
727
+
hgt->visualizers.old_frame->size);
728
+
xrt_frame_reference(&hgt->visualizers.old_frame, NULL);
729
+
}
695
730
}
696
731
697
732
check_new_user_event(hgt);
···
885
920
886
921
// If the debug UI is active, push our debug frame
887
922
if (hgt->debug_scribble) {
888
-
u_sink_debug_push_frame(&hgt->debug_sink, debug_frame);
923
+
u_sink_debug_push_frame(&hgt->debug_sink_ann, debug_frame);
889
924
xrt_frame_reference(&debug_frame, NULL);
925
+
926
+
// We don't dereference the model inputs/outputs frame here; we make a copy of it next frame and
927
+
// dereference it then.
928
+
u_sink_debug_push_frame(&hgt->debug_sink_model, hgt->visualizers.xrtframe);
929
+
xrt_frame_reference(&hgt->visualizers.old_frame, hgt->visualizers.xrtframe);
930
+
xrt_frame_reference(&hgt->visualizers.xrtframe, NULL);
890
931
}
891
932
892
933
// done!
···
1043
1084
"Use IK optimizer (may put tracking in unexpected state, use with care)");
1044
1085
1045
1086
1046
-
u_var_add_sink_debug(hgt, &hgt->debug_sink, "i");
1087
+
u_var_add_sink_debug(hgt, &hgt->debug_sink_ann, "Annotated camera feeds");
1088
+
u_var_add_sink_debug(hgt, &hgt->debug_sink_model, "Model inputs and outputs");
1047
1089
1048
1090
HG_DEBUG(hgt, "Hand Tracker initialized!");
1049
1091
+16
-3
src/xrt/tracking/hand/mercury/hg_sync.hpp
+16
-3
src/xrt/tracking/hand/mercury/hg_sync.hpp
···
172
172
float hand_size_refinement_schedule_y = 0;
173
173
};
174
174
175
+
struct model_output_visualizers
176
+
{
177
+
// After setup, these reference the same piece of memory.
178
+
cv::Mat mat;
179
+
xrt_frame *xrtframe = NULL;
180
+
181
+
// After pushing to the debug UI, we reference the frame here so that we can copy memory out of it for next
182
+
// frame.
183
+
xrt_frame *old_frame = NULL;
184
+
};
185
+
175
186
/*!
176
187
* Main class of Mercury hand tracking.
177
188
*
···
183
194
// Base thing, has to be first.
184
195
t_hand_tracking_sync base = {};
185
196
186
-
struct u_sink_debug debug_sink = {};
197
+
struct u_sink_debug debug_sink_ann = {};
198
+
struct u_sink_debug debug_sink_model = {};
187
199
188
200
float multiply_px_coord_for_undistort;
189
201
···
203
215
204
216
struct ht_view views[2] = {};
205
217
218
+
struct model_output_visualizers visualizers;
219
+
206
220
u_worker_thread_pool *pool;
207
221
208
222
u_worker_group *group;
···
213
227
214
228
uint64_t current_frame_timestamp = {};
215
229
216
-
// Change this whenever you want
217
-
volatile bool debug_scribble = true;
230
+
bool debug_scribble = false;
218
231
219
232
char models_folder[1024];
220
233