The open source OpenXR runtime
d/ht: remove old ht_models API
+2
-750
-1
src/xrt/drivers/CMakeLists.txt
-1
src/xrt/drivers/CMakeLists.txt
+2
src/xrt/drivers/ht/ht_algorithm.cpp
+2
src/xrt/drivers/ht/ht_algorithm.cpp
···
20
20
#include "ht_model.hpp"
21
21
#include "templates/NaivePermutationSort.hpp"
22
22
23
+
#include <future>
24
+
23
25
// Flags to tell state tracker that these are indeed valid joints
24
26
static const enum xrt_space_relation_flags valid_flags_ht = (enum xrt_space_relation_flags)(
25
27
XRT_SPACE_RELATION_ORIENTATION_VALID_BIT | XRT_SPACE_RELATION_ORIENTATION_TRACKED_BIT |
-4
src/xrt/drivers/ht/ht_driver.cpp
-4
src/xrt/drivers/ht/ht_driver.cpp
···
41
41
42
42
#include "ht_algorithm.hpp"
43
43
#include "ht_model.hpp"
44
-
#include "ht_models.hpp"
45
44
46
45
#include <cjson/cJSON.h>
47
46
#include <opencv2/core/mat.hpp>
···
605
604
606
605
// Lock this mutex so we don't try to free things as they're being used on the last iteration
607
606
os_mutex_lock(&htd->unlocked_between_frames);
608
-
destroyOnnx(htd);
609
607
// Remove the variable tracking.
610
608
u_var_remove_root(htd);
611
609
···
683
681
684
682
htd->views[0].view = 0;
685
683
htd->views[1].view = 1;
686
-
687
-
initOnnx(htd);
688
684
689
685
htd->base.tracking_origin = &htd->tracking_origin;
690
686
htd->base.tracking_origin->type = XRT_TRACKING_TYPE_RGB;
-30
src/xrt/drivers/ht/ht_driver.hpp
-30
src/xrt/drivers/ht/ht_driver.hpp
···
36
36
37
37
#include <opencv2/opencv.hpp>
38
38
39
-
#include "core/session/onnxruntime_c_api.h"
40
-
41
-
#include <future>
42
39
#include <vector>
43
40
44
41
using namespace xrt::auxiliary::util;
···
203
200
bool htAlgorithm_approves = false;
204
201
};
205
202
206
-
207
-
struct ModelInfo
208
-
{
209
-
OrtSession *session = nullptr;
210
-
OrtMemoryInfo *memoryInfo = nullptr;
211
-
// std::vector's don't make too much sense here, but they're oh so easy
212
-
std::vector<int64_t> input_shape;
213
-
size_t input_size_bytes;
214
-
std::vector<const char *> output_names;
215
-
std::vector<const char *> input_names;
216
-
};
217
-
218
-
219
203
// Forward declaration for ht_view
220
204
struct ht_device;
221
205
···
234
218
cv::Mat debug_out_to_this;
235
219
236
220
std::vector<HandHistory2DBBox> bbox_histories;
237
-
238
-
struct ModelInfo detection_model;
239
-
std::vector<Palm7KP> (*run_detection_model)(struct ht_view *htv, cv::Mat &img);
240
-
241
-
struct ModelInfo keypoint_model;
242
-
// The cv::mat is passed by value, *not* passed by reference or by pointer;
243
-
// in the tight loop that sets these off we reuse that cv::Mat; changing the data pointer as all the models are
244
-
// running is... going to wreak havoc let's say that.
245
-
Hand2D (*run_keypoint_model)(struct ht_view *htv, cv::Mat img);
246
221
};
247
222
248
223
enum ht_detection_scribble
···
324
299
cJSON *output_array;
325
300
} gst;
326
301
#endif
327
-
328
-
329
-
330
-
const OrtApi *ort_api;
331
-
OrtEnv *ort_env;
332
302
333
303
struct xrt_frame *frame_for_process;
334
304
cv::Mat *mat_for_process;
-695
src/xrt/drivers/ht/ht_models.cpp
-695
src/xrt/drivers/ht/ht_models.cpp
···
1
-
// Copyright 2021, Collabora, Ltd.
2
-
// SPDX-License-Identifier: BSL-1.0
3
-
/*!
4
-
* @file
5
-
* @brief Code to run machine learning models for camera-based hand tracker.
6
-
* @author Moses Turner <moses@collabora.com>
7
-
* @author Marcus Edel <marcus.edel@collabora.com>
8
-
* @ingroup drv_ht
9
-
*/
10
-
11
-
// Many C api things were stolen from here (MIT license):
12
-
// https://github.com/microsoft/onnxruntime-inference-examples/blob/main/c_cxx/fns_candy_style_transfer/fns_candy_style_transfer.c
13
-
14
-
#include "ht_driver.hpp"
15
-
#include "ht_image_math.hpp"
16
-
#include "ht_models.hpp"
17
-
#include "ht_nms.hpp"
18
-
19
-
#include <core/session/onnxruntime_c_api.h>
20
-
21
-
#include <opencv2/core/mat.hpp>
22
-
23
-
#define ORT_CHECK(g_ort, expr) \
24
-
do { \
25
-
OrtStatus *onnx_status = (expr); \
26
-
if (onnx_status != nullptr) { \
27
-
const char *msg = g_ort->GetErrorMessage(onnx_status); \
28
-
U_LOG_E("at %s:%d: %s\n", __FILE__, __LINE__, msg); \
29
-
g_ort->ReleaseStatus(onnx_status); \
30
-
assert(false); \
31
-
} \
32
-
} while (0);
33
-
34
-
static Hand2D
35
-
runKeypointEstimator(struct ht_view *htv, cv::Mat img)
36
-
{
37
-
constexpr size_t lix = 224;
38
-
constexpr size_t liy = 224;
39
-
constexpr size_t nb_planes = 3;
40
-
cv::Mat planes[nb_planes];
41
-
42
-
constexpr size_t size = lix * liy * nb_planes;
43
-
44
-
std::vector<uint8_t> combined_planes(size);
45
-
planarize(img, combined_planes.data());
46
-
47
-
// Normalize - supposedly, the keypoint estimator wants keypoints in [0,1]
48
-
std::vector<float> real_thing(size);
49
-
for (size_t i = 0; i < size; i++) {
50
-
real_thing[i] = (float)combined_planes[i] / 255.0;
51
-
}
52
-
53
-
const OrtApi *g_ort = htv->htd->ort_api;
54
-
struct ModelInfo *model = &htv->keypoint_model;
55
-
56
-
OrtValue *input_tensor = nullptr;
57
-
58
-
ORT_CHECK(g_ort, g_ort->CreateTensorWithDataAsOrtValue(
59
-
model->memoryInfo, real_thing.data(), model->input_size_bytes, model->input_shape.data(),
60
-
model->input_shape.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &input_tensor));
61
-
62
-
// Cargo-culted
63
-
assert(input_tensor != nullptr);
64
-
int is_tensor;
65
-
ORT_CHECK(g_ort, g_ort->IsTensor(input_tensor, &is_tensor));
66
-
assert(is_tensor);
67
-
68
-
const char *output_names[] = {"Identity", "Identity_2", "Identity_2"};
69
-
70
-
// If any of these are non-NULL, ONNX will explode and won't tell you why! Be extremely paranoid!
71
-
OrtValue *output_tensor[3] = {nullptr, nullptr, nullptr};
72
-
ORT_CHECK(g_ort, g_ort->Run(model->session, nullptr, model->input_names.data(), &input_tensor, 1, output_names,
73
-
3, output_tensor));
74
-
75
-
ORT_CHECK(g_ort, g_ort->IsTensor(output_tensor[0], &is_tensor));
76
-
assert(is_tensor);
77
-
78
-
float *landmarks = nullptr;
79
-
80
-
// Should give a pointer to data that is freed on g_ort->ReleaseValue(output_tensor[0]);.
81
-
ORT_CHECK(g_ort, g_ort->GetTensorMutableData(output_tensor[0], (void **)&landmarks));
82
-
83
-
int stride = 3;
84
-
Hand2D dumb;
85
-
for (size_t i = 0; i < 21; i++) {
86
-
int rt = i * stride;
87
-
float x = landmarks[rt];
88
-
float y = landmarks[rt + 1];
89
-
float z = landmarks[rt + 2];
90
-
dumb.kps[i].x = x;
91
-
dumb.kps[i].y = y;
92
-
dumb.kps[i].z = z;
93
-
}
94
-
95
-
// We have to get to here, or else we leak a whole lot! If you need to return early, make this into a goto!
96
-
g_ort->ReleaseValue(output_tensor[0]);
97
-
g_ort->ReleaseValue(output_tensor[1]);
98
-
g_ort->ReleaseValue(output_tensor[2]);
99
-
g_ort->ReleaseValue(input_tensor);
100
-
return dumb;
101
-
}
102
-
103
-
#undef HEAVY_SCRIBBLE
104
-
105
-
/*
106
-
* Anchors data taken from mediapipe's hands detection, used for single-shot
107
-
* detector model.
108
-
*
109
-
* See:
110
-
* https://google.github.io/mediapipe/solutions/hands.html#palm-detection-model
111
-
* https://github.com/google/mediapipe/blob/v0.8.8/mediapipe/calculators/tflite/ssd_anchors_calculator.cc#L101
112
-
* https://github.com/google/mediapipe/blob/v0.8.8/mediapipe/modules/palm_detection/palm_detection_cpu.pbtxt#L60
113
-
*/
114
-
struct anchor
115
-
{
116
-
float x, y;
117
-
};
118
-
119
-
static const struct anchor anchors[896]{
120
-
{0.031250, 0.031250}, {0.031250, 0.031250}, {0.093750, 0.031250}, {0.093750, 0.031250}, //
121
-
{0.156250, 0.031250}, {0.156250, 0.031250}, {0.218750, 0.031250}, {0.218750, 0.031250}, //
122
-
{0.281250, 0.031250}, {0.281250, 0.031250}, {0.343750, 0.031250}, {0.343750, 0.031250}, //
123
-
{0.406250, 0.031250}, {0.406250, 0.031250}, {0.468750, 0.031250}, {0.468750, 0.031250}, //
124
-
{0.531250, 0.031250}, {0.531250, 0.031250}, {0.593750, 0.031250}, {0.593750, 0.031250}, //
125
-
{0.656250, 0.031250}, {0.656250, 0.031250}, {0.718750, 0.031250}, {0.718750, 0.031250}, //
126
-
{0.781250, 0.031250}, {0.781250, 0.031250}, {0.843750, 0.031250}, {0.843750, 0.031250}, //
127
-
{0.906250, 0.031250}, {0.906250, 0.031250}, {0.968750, 0.031250}, {0.968750, 0.031250}, //
128
-
{0.031250, 0.093750}, {0.031250, 0.093750}, {0.093750, 0.093750}, {0.093750, 0.093750}, //
129
-
{0.156250, 0.093750}, {0.156250, 0.093750}, {0.218750, 0.093750}, {0.218750, 0.093750}, //
130
-
{0.281250, 0.093750}, {0.281250, 0.093750}, {0.343750, 0.093750}, {0.343750, 0.093750}, //
131
-
{0.406250, 0.093750}, {0.406250, 0.093750}, {0.468750, 0.093750}, {0.468750, 0.093750}, //
132
-
{0.531250, 0.093750}, {0.531250, 0.093750}, {0.593750, 0.093750}, {0.593750, 0.093750}, //
133
-
{0.656250, 0.093750}, {0.656250, 0.093750}, {0.718750, 0.093750}, {0.718750, 0.093750}, //
134
-
{0.781250, 0.093750}, {0.781250, 0.093750}, {0.843750, 0.093750}, {0.843750, 0.093750}, //
135
-
{0.906250, 0.093750}, {0.906250, 0.093750}, {0.968750, 0.093750}, {0.968750, 0.093750}, //
136
-
{0.031250, 0.156250}, {0.031250, 0.156250}, {0.093750, 0.156250}, {0.093750, 0.156250}, //
137
-
{0.156250, 0.156250}, {0.156250, 0.156250}, {0.218750, 0.156250}, {0.218750, 0.156250}, //
138
-
{0.281250, 0.156250}, {0.281250, 0.156250}, {0.343750, 0.156250}, {0.343750, 0.156250}, //
139
-
{0.406250, 0.156250}, {0.406250, 0.156250}, {0.468750, 0.156250}, {0.468750, 0.156250}, //
140
-
{0.531250, 0.156250}, {0.531250, 0.156250}, {0.593750, 0.156250}, {0.593750, 0.156250}, //
141
-
{0.656250, 0.156250}, {0.656250, 0.156250}, {0.718750, 0.156250}, {0.718750, 0.156250}, //
142
-
{0.781250, 0.156250}, {0.781250, 0.156250}, {0.843750, 0.156250}, {0.843750, 0.156250}, //
143
-
{0.906250, 0.156250}, {0.906250, 0.156250}, {0.968750, 0.156250}, {0.968750, 0.156250}, //
144
-
{0.031250, 0.218750}, {0.031250, 0.218750}, {0.093750, 0.218750}, {0.093750, 0.218750}, //
145
-
{0.156250, 0.218750}, {0.156250, 0.218750}, {0.218750, 0.218750}, {0.218750, 0.218750}, //
146
-
{0.281250, 0.218750}, {0.281250, 0.218750}, {0.343750, 0.218750}, {0.343750, 0.218750}, //
147
-
{0.406250, 0.218750}, {0.406250, 0.218750}, {0.468750, 0.218750}, {0.468750, 0.218750}, //
148
-
{0.531250, 0.218750}, {0.531250, 0.218750}, {0.593750, 0.218750}, {0.593750, 0.218750}, //
149
-
{0.656250, 0.218750}, {0.656250, 0.218750}, {0.718750, 0.218750}, {0.718750, 0.218750}, //
150
-
{0.781250, 0.218750}, {0.781250, 0.218750}, {0.843750, 0.218750}, {0.843750, 0.218750}, //
151
-
{0.906250, 0.218750}, {0.906250, 0.218750}, {0.968750, 0.218750}, {0.968750, 0.218750}, //
152
-
{0.031250, 0.281250}, {0.031250, 0.281250}, {0.093750, 0.281250}, {0.093750, 0.281250}, //
153
-
{0.156250, 0.281250}, {0.156250, 0.281250}, {0.218750, 0.281250}, {0.218750, 0.281250}, //
154
-
{0.281250, 0.281250}, {0.281250, 0.281250}, {0.343750, 0.281250}, {0.343750, 0.281250}, //
155
-
{0.406250, 0.281250}, {0.406250, 0.281250}, {0.468750, 0.281250}, {0.468750, 0.281250}, //
156
-
{0.531250, 0.281250}, {0.531250, 0.281250}, {0.593750, 0.281250}, {0.593750, 0.281250}, //
157
-
{0.656250, 0.281250}, {0.656250, 0.281250}, {0.718750, 0.281250}, {0.718750, 0.281250}, //
158
-
{0.781250, 0.281250}, {0.781250, 0.281250}, {0.843750, 0.281250}, {0.843750, 0.281250}, //
159
-
{0.906250, 0.281250}, {0.906250, 0.281250}, {0.968750, 0.281250}, {0.968750, 0.281250}, //
160
-
{0.031250, 0.343750}, {0.031250, 0.343750}, {0.093750, 0.343750}, {0.093750, 0.343750}, //
161
-
{0.156250, 0.343750}, {0.156250, 0.343750}, {0.218750, 0.343750}, {0.218750, 0.343750}, //
162
-
{0.281250, 0.343750}, {0.281250, 0.343750}, {0.343750, 0.343750}, {0.343750, 0.343750}, //
163
-
{0.406250, 0.343750}, {0.406250, 0.343750}, {0.468750, 0.343750}, {0.468750, 0.343750}, //
164
-
{0.531250, 0.343750}, {0.531250, 0.343750}, {0.593750, 0.343750}, {0.593750, 0.343750}, //
165
-
{0.656250, 0.343750}, {0.656250, 0.343750}, {0.718750, 0.343750}, {0.718750, 0.343750}, //
166
-
{0.781250, 0.343750}, {0.781250, 0.343750}, {0.843750, 0.343750}, {0.843750, 0.343750}, //
167
-
{0.906250, 0.343750}, {0.906250, 0.343750}, {0.968750, 0.343750}, {0.968750, 0.343750}, //
168
-
{0.031250, 0.406250}, {0.031250, 0.406250}, {0.093750, 0.406250}, {0.093750, 0.406250}, //
169
-
{0.156250, 0.406250}, {0.156250, 0.406250}, {0.218750, 0.406250}, {0.218750, 0.406250}, //
170
-
{0.281250, 0.406250}, {0.281250, 0.406250}, {0.343750, 0.406250}, {0.343750, 0.406250}, //
171
-
{0.406250, 0.406250}, {0.406250, 0.406250}, {0.468750, 0.406250}, {0.468750, 0.406250}, //
172
-
{0.531250, 0.406250}, {0.531250, 0.406250}, {0.593750, 0.406250}, {0.593750, 0.406250}, //
173
-
{0.656250, 0.406250}, {0.656250, 0.406250}, {0.718750, 0.406250}, {0.718750, 0.406250}, //
174
-
{0.781250, 0.406250}, {0.781250, 0.406250}, {0.843750, 0.406250}, {0.843750, 0.406250}, //
175
-
{0.906250, 0.406250}, {0.906250, 0.406250}, {0.968750, 0.406250}, {0.968750, 0.406250}, //
176
-
{0.031250, 0.468750}, {0.031250, 0.468750}, {0.093750, 0.468750}, {0.093750, 0.468750}, //
177
-
{0.156250, 0.468750}, {0.156250, 0.468750}, {0.218750, 0.468750}, {0.218750, 0.468750}, //
178
-
{0.281250, 0.468750}, {0.281250, 0.468750}, {0.343750, 0.468750}, {0.343750, 0.468750}, //
179
-
{0.406250, 0.468750}, {0.406250, 0.468750}, {0.468750, 0.468750}, {0.468750, 0.468750}, //
180
-
{0.531250, 0.468750}, {0.531250, 0.468750}, {0.593750, 0.468750}, {0.593750, 0.468750}, //
181
-
{0.656250, 0.468750}, {0.656250, 0.468750}, {0.718750, 0.468750}, {0.718750, 0.468750}, //
182
-
{0.781250, 0.468750}, {0.781250, 0.468750}, {0.843750, 0.468750}, {0.843750, 0.468750}, //
183
-
{0.906250, 0.468750}, {0.906250, 0.468750}, {0.968750, 0.468750}, {0.968750, 0.468750}, //
184
-
{0.031250, 0.531250}, {0.031250, 0.531250}, {0.093750, 0.531250}, {0.093750, 0.531250}, //
185
-
{0.156250, 0.531250}, {0.156250, 0.531250}, {0.218750, 0.531250}, {0.218750, 0.531250}, //
186
-
{0.281250, 0.531250}, {0.281250, 0.531250}, {0.343750, 0.531250}, {0.343750, 0.531250}, //
187
-
{0.406250, 0.531250}, {0.406250, 0.531250}, {0.468750, 0.531250}, {0.468750, 0.531250}, //
188
-
{0.531250, 0.531250}, {0.531250, 0.531250}, {0.593750, 0.531250}, {0.593750, 0.531250}, //
189
-
{0.656250, 0.531250}, {0.656250, 0.531250}, {0.718750, 0.531250}, {0.718750, 0.531250}, //
190
-
{0.781250, 0.531250}, {0.781250, 0.531250}, {0.843750, 0.531250}, {0.843750, 0.531250}, //
191
-
{0.906250, 0.531250}, {0.906250, 0.531250}, {0.968750, 0.531250}, {0.968750, 0.531250}, //
192
-
{0.031250, 0.593750}, {0.031250, 0.593750}, {0.093750, 0.593750}, {0.093750, 0.593750}, //
193
-
{0.156250, 0.593750}, {0.156250, 0.593750}, {0.218750, 0.593750}, {0.218750, 0.593750}, //
194
-
{0.281250, 0.593750}, {0.281250, 0.593750}, {0.343750, 0.593750}, {0.343750, 0.593750}, //
195
-
{0.406250, 0.593750}, {0.406250, 0.593750}, {0.468750, 0.593750}, {0.468750, 0.593750}, //
196
-
{0.531250, 0.593750}, {0.531250, 0.593750}, {0.593750, 0.593750}, {0.593750, 0.593750}, //
197
-
{0.656250, 0.593750}, {0.656250, 0.593750}, {0.718750, 0.593750}, {0.718750, 0.593750}, //
198
-
{0.781250, 0.593750}, {0.781250, 0.593750}, {0.843750, 0.593750}, {0.843750, 0.593750}, //
199
-
{0.906250, 0.593750}, {0.906250, 0.593750}, {0.968750, 0.593750}, {0.968750, 0.593750}, //
200
-
{0.031250, 0.656250}, {0.031250, 0.656250}, {0.093750, 0.656250}, {0.093750, 0.656250}, //
201
-
{0.156250, 0.656250}, {0.156250, 0.656250}, {0.218750, 0.656250}, {0.218750, 0.656250}, //
202
-
{0.281250, 0.656250}, {0.281250, 0.656250}, {0.343750, 0.656250}, {0.343750, 0.656250}, //
203
-
{0.406250, 0.656250}, {0.406250, 0.656250}, {0.468750, 0.656250}, {0.468750, 0.656250}, //
204
-
{0.531250, 0.656250}, {0.531250, 0.656250}, {0.593750, 0.656250}, {0.593750, 0.656250}, //
205
-
{0.656250, 0.656250}, {0.656250, 0.656250}, {0.718750, 0.656250}, {0.718750, 0.656250}, //
206
-
{0.781250, 0.656250}, {0.781250, 0.656250}, {0.843750, 0.656250}, {0.843750, 0.656250}, //
207
-
{0.906250, 0.656250}, {0.906250, 0.656250}, {0.968750, 0.656250}, {0.968750, 0.656250}, //
208
-
{0.031250, 0.718750}, {0.031250, 0.718750}, {0.093750, 0.718750}, {0.093750, 0.718750}, //
209
-
{0.156250, 0.718750}, {0.156250, 0.718750}, {0.218750, 0.718750}, {0.218750, 0.718750}, //
210
-
{0.281250, 0.718750}, {0.281250, 0.718750}, {0.343750, 0.718750}, {0.343750, 0.718750}, //
211
-
{0.406250, 0.718750}, {0.406250, 0.718750}, {0.468750, 0.718750}, {0.468750, 0.718750}, //
212
-
{0.531250, 0.718750}, {0.531250, 0.718750}, {0.593750, 0.718750}, {0.593750, 0.718750}, //
213
-
{0.656250, 0.718750}, {0.656250, 0.718750}, {0.718750, 0.718750}, {0.718750, 0.718750}, //
214
-
{0.781250, 0.718750}, {0.781250, 0.718750}, {0.843750, 0.718750}, {0.843750, 0.718750}, //
215
-
{0.906250, 0.718750}, {0.906250, 0.718750}, {0.968750, 0.718750}, {0.968750, 0.718750}, //
216
-
{0.031250, 0.781250}, {0.031250, 0.781250}, {0.093750, 0.781250}, {0.093750, 0.781250}, //
217
-
{0.156250, 0.781250}, {0.156250, 0.781250}, {0.218750, 0.781250}, {0.218750, 0.781250}, //
218
-
{0.281250, 0.781250}, {0.281250, 0.781250}, {0.343750, 0.781250}, {0.343750, 0.781250}, //
219
-
{0.406250, 0.781250}, {0.406250, 0.781250}, {0.468750, 0.781250}, {0.468750, 0.781250}, //
220
-
{0.531250, 0.781250}, {0.531250, 0.781250}, {0.593750, 0.781250}, {0.593750, 0.781250}, //
221
-
{0.656250, 0.781250}, {0.656250, 0.781250}, {0.718750, 0.781250}, {0.718750, 0.781250}, //
222
-
{0.781250, 0.781250}, {0.781250, 0.781250}, {0.843750, 0.781250}, {0.843750, 0.781250}, //
223
-
{0.906250, 0.781250}, {0.906250, 0.781250}, {0.968750, 0.781250}, {0.968750, 0.781250}, //
224
-
{0.031250, 0.843750}, {0.031250, 0.843750}, {0.093750, 0.843750}, {0.093750, 0.843750}, //
225
-
{0.156250, 0.843750}, {0.156250, 0.843750}, {0.218750, 0.843750}, {0.218750, 0.843750}, //
226
-
{0.281250, 0.843750}, {0.281250, 0.843750}, {0.343750, 0.843750}, {0.343750, 0.843750}, //
227
-
{0.406250, 0.843750}, {0.406250, 0.843750}, {0.468750, 0.843750}, {0.468750, 0.843750}, //
228
-
{0.531250, 0.843750}, {0.531250, 0.843750}, {0.593750, 0.843750}, {0.593750, 0.843750}, //
229
-
{0.656250, 0.843750}, {0.656250, 0.843750}, {0.718750, 0.843750}, {0.718750, 0.843750}, //
230
-
{0.781250, 0.843750}, {0.781250, 0.843750}, {0.843750, 0.843750}, {0.843750, 0.843750}, //
231
-
{0.906250, 0.843750}, {0.906250, 0.843750}, {0.968750, 0.843750}, {0.968750, 0.843750}, //
232
-
{0.031250, 0.906250}, {0.031250, 0.906250}, {0.093750, 0.906250}, {0.093750, 0.906250}, //
233
-
{0.156250, 0.906250}, {0.156250, 0.906250}, {0.218750, 0.906250}, {0.218750, 0.906250}, //
234
-
{0.281250, 0.906250}, {0.281250, 0.906250}, {0.343750, 0.906250}, {0.343750, 0.906250}, //
235
-
{0.406250, 0.906250}, {0.406250, 0.906250}, {0.468750, 0.906250}, {0.468750, 0.906250}, //
236
-
{0.531250, 0.906250}, {0.531250, 0.906250}, {0.593750, 0.906250}, {0.593750, 0.906250}, //
237
-
{0.656250, 0.906250}, {0.656250, 0.906250}, {0.718750, 0.906250}, {0.718750, 0.906250}, //
238
-
{0.781250, 0.906250}, {0.781250, 0.906250}, {0.843750, 0.906250}, {0.843750, 0.906250}, //
239
-
{0.906250, 0.906250}, {0.906250, 0.906250}, {0.968750, 0.906250}, {0.968750, 0.906250}, //
240
-
{0.031250, 0.968750}, {0.031250, 0.968750}, {0.093750, 0.968750}, {0.093750, 0.968750}, //
241
-
{0.156250, 0.968750}, {0.156250, 0.968750}, {0.218750, 0.968750}, {0.218750, 0.968750}, //
242
-
{0.281250, 0.968750}, {0.281250, 0.968750}, {0.343750, 0.968750}, {0.343750, 0.968750}, //
243
-
{0.406250, 0.968750}, {0.406250, 0.968750}, {0.468750, 0.968750}, {0.468750, 0.968750}, //
244
-
{0.531250, 0.968750}, {0.531250, 0.968750}, {0.593750, 0.968750}, {0.593750, 0.968750}, //
245
-
{0.656250, 0.968750}, {0.656250, 0.968750}, {0.718750, 0.968750}, {0.718750, 0.968750}, //
246
-
{0.781250, 0.968750}, {0.781250, 0.968750}, {0.843750, 0.968750}, {0.843750, 0.968750}, //
247
-
{0.906250, 0.968750}, {0.906250, 0.968750}, {0.968750, 0.968750}, {0.968750, 0.968750}, //
248
-
{0.062500, 0.062500}, {0.062500, 0.062500}, {0.062500, 0.062500}, {0.062500, 0.062500}, //
249
-
{0.062500, 0.062500}, {0.062500, 0.062500}, {0.187500, 0.062500}, {0.187500, 0.062500}, //
250
-
{0.187500, 0.062500}, {0.187500, 0.062500}, {0.187500, 0.062500}, {0.187500, 0.062500}, //
251
-
{0.312500, 0.062500}, {0.312500, 0.062500}, {0.312500, 0.062500}, {0.312500, 0.062500}, //
252
-
{0.312500, 0.062500}, {0.312500, 0.062500}, {0.437500, 0.062500}, {0.437500, 0.062500}, //
253
-
{0.437500, 0.062500}, {0.437500, 0.062500}, {0.437500, 0.062500}, {0.437500, 0.062500}, //
254
-
{0.562500, 0.062500}, {0.562500, 0.062500}, {0.562500, 0.062500}, {0.562500, 0.062500}, //
255
-
{0.562500, 0.062500}, {0.562500, 0.062500}, {0.687500, 0.062500}, {0.687500, 0.062500}, //
256
-
{0.687500, 0.062500}, {0.687500, 0.062500}, {0.687500, 0.062500}, {0.687500, 0.062500}, //
257
-
{0.812500, 0.062500}, {0.812500, 0.062500}, {0.812500, 0.062500}, {0.812500, 0.062500}, //
258
-
{0.812500, 0.062500}, {0.812500, 0.062500}, {0.937500, 0.062500}, {0.937500, 0.062500}, //
259
-
{0.937500, 0.062500}, {0.937500, 0.062500}, {0.937500, 0.062500}, {0.937500, 0.062500}, //
260
-
{0.062500, 0.187500}, {0.062500, 0.187500}, {0.062500, 0.187500}, {0.062500, 0.187500}, //
261
-
{0.062500, 0.187500}, {0.062500, 0.187500}, {0.187500, 0.187500}, {0.187500, 0.187500}, //
262
-
{0.187500, 0.187500}, {0.187500, 0.187500}, {0.187500, 0.187500}, {0.187500, 0.187500}, //
263
-
{0.312500, 0.187500}, {0.312500, 0.187500}, {0.312500, 0.187500}, {0.312500, 0.187500}, //
264
-
{0.312500, 0.187500}, {0.312500, 0.187500}, {0.437500, 0.187500}, {0.437500, 0.187500}, //
265
-
{0.437500, 0.187500}, {0.437500, 0.187500}, {0.437500, 0.187500}, {0.437500, 0.187500}, //
266
-
{0.562500, 0.187500}, {0.562500, 0.187500}, {0.562500, 0.187500}, {0.562500, 0.187500}, //
267
-
{0.562500, 0.187500}, {0.562500, 0.187500}, {0.687500, 0.187500}, {0.687500, 0.187500}, //
268
-
{0.687500, 0.187500}, {0.687500, 0.187500}, {0.687500, 0.187500}, {0.687500, 0.187500}, //
269
-
{0.812500, 0.187500}, {0.812500, 0.187500}, {0.812500, 0.187500}, {0.812500, 0.187500}, //
270
-
{0.812500, 0.187500}, {0.812500, 0.187500}, {0.937500, 0.187500}, {0.937500, 0.187500}, //
271
-
{0.937500, 0.187500}, {0.937500, 0.187500}, {0.937500, 0.187500}, {0.937500, 0.187500}, //
272
-
{0.062500, 0.312500}, {0.062500, 0.312500}, {0.062500, 0.312500}, {0.062500, 0.312500}, //
273
-
{0.062500, 0.312500}, {0.062500, 0.312500}, {0.187500, 0.312500}, {0.187500, 0.312500}, //
274
-
{0.187500, 0.312500}, {0.187500, 0.312500}, {0.187500, 0.312500}, {0.187500, 0.312500}, //
275
-
{0.312500, 0.312500}, {0.312500, 0.312500}, {0.312500, 0.312500}, {0.312500, 0.312500}, //
276
-
{0.312500, 0.312500}, {0.312500, 0.312500}, {0.437500, 0.312500}, {0.437500, 0.312500}, //
277
-
{0.437500, 0.312500}, {0.437500, 0.312500}, {0.437500, 0.312500}, {0.437500, 0.312500}, //
278
-
{0.562500, 0.312500}, {0.562500, 0.312500}, {0.562500, 0.312500}, {0.562500, 0.312500}, //
279
-
{0.562500, 0.312500}, {0.562500, 0.312500}, {0.687500, 0.312500}, {0.687500, 0.312500}, //
280
-
{0.687500, 0.312500}, {0.687500, 0.312500}, {0.687500, 0.312500}, {0.687500, 0.312500}, //
281
-
{0.812500, 0.312500}, {0.812500, 0.312500}, {0.812500, 0.312500}, {0.812500, 0.312500}, //
282
-
{0.812500, 0.312500}, {0.812500, 0.312500}, {0.937500, 0.312500}, {0.937500, 0.312500}, //
283
-
{0.937500, 0.312500}, {0.937500, 0.312500}, {0.937500, 0.312500}, {0.937500, 0.312500}, //
284
-
{0.062500, 0.437500}, {0.062500, 0.437500}, {0.062500, 0.437500}, {0.062500, 0.437500}, //
285
-
{0.062500, 0.437500}, {0.062500, 0.437500}, {0.187500, 0.437500}, {0.187500, 0.437500}, //
286
-
{0.187500, 0.437500}, {0.187500, 0.437500}, {0.187500, 0.437500}, {0.187500, 0.437500}, //
287
-
{0.312500, 0.437500}, {0.312500, 0.437500}, {0.312500, 0.437500}, {0.312500, 0.437500}, //
288
-
{0.312500, 0.437500}, {0.312500, 0.437500}, {0.437500, 0.437500}, {0.437500, 0.437500}, //
289
-
{0.437500, 0.437500}, {0.437500, 0.437500}, {0.437500, 0.437500}, {0.437500, 0.437500}, //
290
-
{0.562500, 0.437500}, {0.562500, 0.437500}, {0.562500, 0.437500}, {0.562500, 0.437500}, //
291
-
{0.562500, 0.437500}, {0.562500, 0.437500}, {0.687500, 0.437500}, {0.687500, 0.437500}, //
292
-
{0.687500, 0.437500}, {0.687500, 0.437500}, {0.687500, 0.437500}, {0.687500, 0.437500}, //
293
-
{0.812500, 0.437500}, {0.812500, 0.437500}, {0.812500, 0.437500}, {0.812500, 0.437500}, //
294
-
{0.812500, 0.437500}, {0.812500, 0.437500}, {0.937500, 0.437500}, {0.937500, 0.437500}, //
295
-
{0.937500, 0.437500}, {0.937500, 0.437500}, {0.937500, 0.437500}, {0.937500, 0.437500}, //
296
-
{0.062500, 0.562500}, {0.062500, 0.562500}, {0.062500, 0.562500}, {0.062500, 0.562500}, //
297
-
{0.062500, 0.562500}, {0.062500, 0.562500}, {0.187500, 0.562500}, {0.187500, 0.562500}, //
298
-
{0.187500, 0.562500}, {0.187500, 0.562500}, {0.187500, 0.562500}, {0.187500, 0.562500}, //
299
-
{0.312500, 0.562500}, {0.312500, 0.562500}, {0.312500, 0.562500}, {0.312500, 0.562500}, //
300
-
{0.312500, 0.562500}, {0.312500, 0.562500}, {0.437500, 0.562500}, {0.437500, 0.562500}, //
301
-
{0.437500, 0.562500}, {0.437500, 0.562500}, {0.437500, 0.562500}, {0.437500, 0.562500}, //
302
-
{0.562500, 0.562500}, {0.562500, 0.562500}, {0.562500, 0.562500}, {0.562500, 0.562500}, //
303
-
{0.562500, 0.562500}, {0.562500, 0.562500}, {0.687500, 0.562500}, {0.687500, 0.562500}, //
304
-
{0.687500, 0.562500}, {0.687500, 0.562500}, {0.687500, 0.562500}, {0.687500, 0.562500}, //
305
-
{0.812500, 0.562500}, {0.812500, 0.562500}, {0.812500, 0.562500}, {0.812500, 0.562500}, //
306
-
{0.812500, 0.562500}, {0.812500, 0.562500}, {0.937500, 0.562500}, {0.937500, 0.562500}, //
307
-
{0.937500, 0.562500}, {0.937500, 0.562500}, {0.937500, 0.562500}, {0.937500, 0.562500}, //
308
-
{0.062500, 0.687500}, {0.062500, 0.687500}, {0.062500, 0.687500}, {0.062500, 0.687500}, //
309
-
{0.062500, 0.687500}, {0.062500, 0.687500}, {0.187500, 0.687500}, {0.187500, 0.687500}, //
310
-
{0.187500, 0.687500}, {0.187500, 0.687500}, {0.187500, 0.687500}, {0.187500, 0.687500}, //
311
-
{0.312500, 0.687500}, {0.312500, 0.687500}, {0.312500, 0.687500}, {0.312500, 0.687500}, //
312
-
{0.312500, 0.687500}, {0.312500, 0.687500}, {0.437500, 0.687500}, {0.437500, 0.687500}, //
313
-
{0.437500, 0.687500}, {0.437500, 0.687500}, {0.437500, 0.687500}, {0.437500, 0.687500}, //
314
-
{0.562500, 0.687500}, {0.562500, 0.687500}, {0.562500, 0.687500}, {0.562500, 0.687500}, //
315
-
{0.562500, 0.687500}, {0.562500, 0.687500}, {0.687500, 0.687500}, {0.687500, 0.687500}, //
316
-
{0.687500, 0.687500}, {0.687500, 0.687500}, {0.687500, 0.687500}, {0.687500, 0.687500}, //
317
-
{0.812500, 0.687500}, {0.812500, 0.687500}, {0.812500, 0.687500}, {0.812500, 0.687500}, //
318
-
{0.812500, 0.687500}, {0.812500, 0.687500}, {0.937500, 0.687500}, {0.937500, 0.687500}, //
319
-
{0.937500, 0.687500}, {0.937500, 0.687500}, {0.937500, 0.687500}, {0.937500, 0.687500}, //
320
-
{0.062500, 0.812500}, {0.062500, 0.812500}, {0.062500, 0.812500}, {0.062500, 0.812500}, //
321
-
{0.062500, 0.812500}, {0.062500, 0.812500}, {0.187500, 0.812500}, {0.187500, 0.812500}, //
322
-
{0.187500, 0.812500}, {0.187500, 0.812500}, {0.187500, 0.812500}, {0.187500, 0.812500}, //
323
-
{0.312500, 0.812500}, {0.312500, 0.812500}, {0.312500, 0.812500}, {0.312500, 0.812500}, //
324
-
{0.312500, 0.812500}, {0.312500, 0.812500}, {0.437500, 0.812500}, {0.437500, 0.812500}, //
325
-
{0.437500, 0.812500}, {0.437500, 0.812500}, {0.437500, 0.812500}, {0.437500, 0.812500}, //
326
-
{0.562500, 0.812500}, {0.562500, 0.812500}, {0.562500, 0.812500}, {0.562500, 0.812500}, //
327
-
{0.562500, 0.812500}, {0.562500, 0.812500}, {0.687500, 0.812500}, {0.687500, 0.812500}, //
328
-
{0.687500, 0.812500}, {0.687500, 0.812500}, {0.687500, 0.812500}, {0.687500, 0.812500}, //
329
-
{0.812500, 0.812500}, {0.812500, 0.812500}, {0.812500, 0.812500}, {0.812500, 0.812500}, //
330
-
{0.812500, 0.812500}, {0.812500, 0.812500}, {0.937500, 0.812500}, {0.937500, 0.812500}, //
331
-
{0.937500, 0.812500}, {0.937500, 0.812500}, {0.937500, 0.812500}, {0.937500, 0.812500}, //
332
-
{0.062500, 0.937500}, {0.062500, 0.937500}, {0.062500, 0.937500}, {0.062500, 0.937500}, //
333
-
{0.062500, 0.937500}, {0.062500, 0.937500}, {0.187500, 0.937500}, {0.187500, 0.937500}, //
334
-
{0.187500, 0.937500}, {0.187500, 0.937500}, {0.187500, 0.937500}, {0.187500, 0.937500}, //
335
-
{0.312500, 0.937500}, {0.312500, 0.937500}, {0.312500, 0.937500}, {0.312500, 0.937500}, //
336
-
{0.312500, 0.937500}, {0.312500, 0.937500}, {0.437500, 0.937500}, {0.437500, 0.937500}, //
337
-
{0.437500, 0.937500}, {0.437500, 0.937500}, {0.437500, 0.937500}, {0.437500, 0.937500}, //
338
-
{0.562500, 0.937500}, {0.562500, 0.937500}, {0.562500, 0.937500}, {0.562500, 0.937500}, //
339
-
{0.562500, 0.937500}, {0.562500, 0.937500}, {0.687500, 0.937500}, {0.687500, 0.937500}, //
340
-
{0.687500, 0.937500}, {0.687500, 0.937500}, {0.687500, 0.937500}, {0.687500, 0.937500}, //
341
-
{0.812500, 0.937500}, {0.812500, 0.937500}, {0.812500, 0.937500}, {0.812500, 0.937500}, //
342
-
{0.812500, 0.937500}, {0.812500, 0.937500}, {0.937500, 0.937500}, {0.937500, 0.937500}, //
343
-
{0.937500, 0.937500}, {0.937500, 0.937500}, {0.937500, 0.937500}, {0.937500, 0.937500}, //
344
-
};
345
-
346
-
static std::vector<Palm7KP>
347
-
runHandDetector(struct ht_view *htv, cv::Mat &raw_input)
348
-
{
349
-
const OrtApi *g_ort = htv->htd->ort_api;
350
-
351
-
cv::Mat img;
352
-
353
-
constexpr int hd_size = 128;
354
-
constexpr size_t nb_planes = 3;
355
-
constexpr size_t size = hd_size * hd_size * nb_planes;
356
-
constexpr int inputTensorSize = size * sizeof(float);
357
-
358
-
cv::Matx23f back_from_blackbar = blackbar(raw_input, img, {hd_size, hd_size});
359
-
360
-
float scale_factor = back_from_blackbar(0, 0); // 960/128
361
-
assert(img.isContinuous());
362
-
constexpr float mean = 128.0f;
363
-
constexpr float std = 128.0f;
364
-
365
-
std::vector<float> real_thing(size);
366
-
367
-
if (htv->htd->startup_config.palm_detection_use_mediapipe) {
368
-
std::vector<uint8_t> combined_planes(size);
369
-
planarize(img, combined_planes.data());
370
-
for (size_t i = 0; i < size; i++) {
371
-
float val = (float)combined_planes[i];
372
-
real_thing[i] = (val - mean) / std;
373
-
}
374
-
} else {
375
-
376
-
assert(img.isContinuous());
377
-
378
-
for (size_t i = 0; i < size; i++) {
379
-
int val = img.data[i];
380
-
381
-
real_thing[i] = (val - mean) / std;
382
-
}
383
-
}
384
-
385
-
// Convenience
386
-
struct ModelInfo *model_hd = &htv->detection_model;
387
-
388
-
// If this is non-NULL, ONNX will explode and won't tell you why!
389
-
OrtValue *input_tensor = nullptr;
390
-
391
-
ORT_CHECK(g_ort, g_ort->CreateTensorWithDataAsOrtValue(
392
-
model_hd->memoryInfo, real_thing.data(), inputTensorSize, model_hd->input_shape.data(),
393
-
model_hd->input_shape.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &input_tensor));
394
-
395
-
// Cargo-culted
396
-
assert(input_tensor != nullptr);
397
-
int is_tensor;
398
-
ORT_CHECK(g_ort, g_ort->IsTensor(input_tensor, &is_tensor));
399
-
assert(is_tensor);
400
-
401
-
// If any of these are non-NULL, ONNX will explode and won't tell you why! Be extremely paranoid!
402
-
OrtValue *output_tensor[2] = {nullptr, nullptr};
403
-
ORT_CHECK(g_ort, g_ort->Run(model_hd->session, nullptr, model_hd->input_names.data(),
404
-
(const OrtValue *const *)&input_tensor, model_hd->input_names.size(),
405
-
model_hd->output_names.data(), model_hd->output_names.size(), output_tensor));
406
-
407
-
ORT_CHECK(g_ort, g_ort->IsTensor(output_tensor[0], &is_tensor));
408
-
assert(is_tensor);
409
-
410
-
float *classificators = nullptr;
411
-
float *regressors = nullptr;
412
-
413
-
ORT_CHECK(g_ort,
414
-
g_ort->GetTensorMutableData(output_tensor[0], (void **)&classificators)); // Totally won't segfault!
415
-
ORT_CHECK(g_ort, g_ort->GetTensorMutableData(output_tensor[1], (void **)®ressors));
416
-
// We need to free these!
417
-
418
-
float *rg = regressors; // shorter name
419
-
420
-
std::vector<NMSPalm> detections;
421
-
int count = 0;
422
-
423
-
std::vector<Palm7KP> output;
424
-
std::vector<NMSPalm> nms_palms;
425
-
426
-
OrtTensorTypeAndShapeInfo *classificators_info;
427
-
ORT_CHECK(g_ort, g_ort->GetTensorTypeAndShape(output_tensor[0], &classificators_info));
428
-
429
-
size_t classificators_size;
430
-
ORT_CHECK(g_ort, g_ort->GetTensorShapeElementCount(classificators_info, &classificators_size));
431
-
assert(classificators_size == 896);
432
-
433
-
OrtTensorTypeAndShapeInfo *regressors_info;
434
-
ORT_CHECK(g_ort, g_ort->GetTensorTypeAndShape(output_tensor[1], ®ressors_info));
435
-
436
-
size_t regressors_size;
437
-
ORT_CHECK(g_ort, g_ort->GetTensorShapeElementCount(regressors_info, ®ressors_size));
438
-
assert(regressors_size == 896 * 18);
439
-
440
-
g_ort->ReleaseTensorTypeAndShapeInfo(classificators_info);
441
-
g_ort->ReleaseTensorTypeAndShapeInfo(regressors_info);
442
-
443
-
for (size_t i = 0; i < classificators_size; ++i) {
444
-
const float score = 1.0 / (1.0 + exp(-classificators[i]));
445
-
446
-
// Let a lot of detections in - they'll be slowly rejected later
447
-
if (score <= htv->htd->dynamic_config.nms_threshold.val) {
448
-
continue;
449
-
}
450
-
451
-
const struct anchor *anchor = &anchors[i];
452
-
453
-
// Boundary box.
454
-
NMSPalm det;
455
-
456
-
float anchx = anchor->x * 128;
457
-
float anchy = anchor->y * 128;
458
-
459
-
float shiftx = regressors[i * 18];
460
-
float shifty = regressors[i * 18 + 1];
461
-
462
-
float w = regressors[i * 18 + 2];
463
-
float h = regressors[i * 18 + 3];
464
-
465
-
float cx = shiftx + anchx;
466
-
float cy = shifty + anchy;
467
-
468
-
struct xrt_vec2 *kps = det.keypoints;
469
-
470
-
kps[0] = {rg[i * 18 + 4], rg[i * 18 + 5]};
471
-
kps[1] = {rg[i * 18 + 6], rg[i * 18 + 7]};
472
-
kps[2] = {rg[i * 18 + 8], rg[i * 18 + 9]};
473
-
kps[3] = {rg[i * 18 + 10], rg[i * 18 + 11]};
474
-
kps[4] = {rg[i * 18 + 12], rg[i * 18 + 13]};
475
-
kps[5] = {rg[i * 18 + 14], rg[i * 18 + 15]};
476
-
kps[6] = {rg[i * 18 + 16], rg[i * 18 + 17]};
477
-
478
-
479
-
for (int i = 0; i < 7; i++) {
480
-
struct xrt_vec2 *b = &kps[i];
481
-
b->x += anchx;
482
-
b->y += anchy;
483
-
}
484
-
485
-
det.bbox.w = w;
486
-
det.bbox.h = h;
487
-
det.bbox.cx = cx;
488
-
det.bbox.cy = cy;
489
-
det.confidence = score;
490
-
detections.push_back(det);
491
-
count++;
492
-
493
-
if (htv->htd->debug_scribble && (htv->htd->dynamic_config.scribble_raw_detections)) {
494
-
xrt_vec2 center = transformVecBy2x3(xrt_vec2{cx, cy}, back_from_blackbar);
495
-
496
-
float sz = det.bbox.w * scale_factor;
497
-
498
-
cv::rectangle(htv->debug_out_to_this,
499
-
{(int)(center.x - (sz / 2)), (int)(center.y - (sz / 2)), (int)sz, (int)sz},
500
-
hsv2rgb(0.0f, math_map_ranges(det.confidence, 0.0f, 1.0f, 1.5f, -0.1f),
501
-
math_map_ranges(det.confidence, 0.0f, 1.0f, 0.2f, 1.4f)),
502
-
1);
503
-
504
-
for (int i = 0; i < 7; i++) {
505
-
handDot(htv->debug_out_to_this, transformVecBy2x3(kps[i], back_from_blackbar),
506
-
det.confidence * 7, ((float)i) * (360.0f / 7.0f), det.confidence, 1);
507
-
}
508
-
}
509
-
510
-
511
-
512
-
int square = fmax(w, h);
513
-
514
-
square = square / scale_factor;
515
-
}
516
-
517
-
if (count == 0) {
518
-
goto cleanup;
519
-
}
520
-
521
-
nms_palms = filterBoxesWeightedAvg(detections, htv->htd->dynamic_config.nms_iou.val);
522
-
523
-
524
-
525
-
for (NMSPalm cooler : nms_palms) {
526
-
527
-
// Display box
528
-
529
-
struct xrt_vec2 tl = {cooler.bbox.cx - cooler.bbox.w / 2, cooler.bbox.cy - cooler.bbox.h / 2};
530
-
struct xrt_vec2 bob = transformVecBy2x3(tl, back_from_blackbar);
531
-
float sz = cooler.bbox.w * scale_factor;
532
-
533
-
if (htv->htd->debug_scribble && htv->htd->dynamic_config.scribble_nms_detections) {
534
-
cv::rectangle(htv->debug_out_to_this, {(int)bob.x, (int)bob.y, (int)sz, (int)sz},
535
-
hsv2rgb(180.0f, math_map_ranges(cooler.confidence, 0.0f, 1.0f, 0.8f, -0.1f),
536
-
math_map_ranges(cooler.confidence, 0.0f, 1.0f, 0.2f, 1.4f)),
537
-
2);
538
-
for (int i = 0; i < 7; i++) {
539
-
handDot(htv->debug_out_to_this,
540
-
transformVecBy2x3(cooler.keypoints[i], back_from_blackbar),
541
-
cooler.confidence * 14, ((float)i) * (360.0f / 7.0f), cooler.confidence, 3);
542
-
}
543
-
}
544
-
545
-
546
-
Palm7KP this_element;
547
-
548
-
for (int i = 0; i < 7; i++) {
549
-
struct xrt_vec2 b = cooler.keypoints[i];
550
-
this_element.kps[i] = transformVecBy2x3(b, back_from_blackbar);
551
-
}
552
-
this_element.confidence = cooler.confidence;
553
-
554
-
output.push_back(this_element);
555
-
}
556
-
557
-
cleanup:
558
-
g_ort->ReleaseValue(output_tensor[0]);
559
-
g_ort->ReleaseValue(output_tensor[1]);
560
-
g_ort->ReleaseValue(input_tensor);
561
-
return output;
562
-
}
563
-
564
-
565
-
static void
566
-
addSlug(struct ht_device *htd, const char *suffix, char *out)
567
-
{
568
-
strcpy(out, htd->startup_config.model_slug);
569
-
strcat(out, suffix);
570
-
}
571
-
572
-
static void
573
-
initKeypointEstimator(struct ht_device *htd, ht_view *htv)
574
-
{
575
-
struct ModelInfo *model_ke = &htv->keypoint_model;
576
-
const OrtApi *g_ort = htd->ort_api;
577
-
OrtSessionOptions *opts = nullptr;
578
-
579
-
ORT_CHECK(g_ort, g_ort->CreateSessionOptions(&opts));
580
-
581
-
ORT_CHECK(g_ort, g_ort->SetSessionGraphOptimizationLevel(opts, ORT_ENABLE_ALL));
582
-
ORT_CHECK(g_ort, g_ort->SetIntraOpNumThreads(opts, 1));
583
-
584
-
char modelLocation[1024];
585
-
if (htd->startup_config.keypoint_estimation_use_mediapipe) {
586
-
addSlug(htd, "hand_landmark_MEDIAPIPE.onnx", modelLocation);
587
-
} else {
588
-
addSlug(htd, "hand_landmark_COLLABORA.onnx", modelLocation);
589
-
}
590
-
ORT_CHECK(g_ort, g_ort->CreateSession(htd->ort_env, modelLocation, opts, &model_ke->session));
591
-
g_ort->ReleaseSessionOptions(opts);
592
-
593
-
model_ke->input_shape.push_back(1);
594
-
model_ke->input_shape.push_back(3);
595
-
model_ke->input_shape.push_back(224);
596
-
model_ke->input_shape.push_back(224);
597
-
598
-
model_ke->input_names.push_back("input_1");
599
-
600
-
model_ke->output_names.push_back("Identity");
601
-
model_ke->output_names.push_back("Identity_1");
602
-
model_ke->output_names.push_back("Identity_2");
603
-
604
-
model_ke->input_size_bytes = 224 * 224 * 3 * sizeof(float);
605
-
606
-
ORT_CHECK(g_ort, g_ort->CreateCpuMemoryInfo(OrtArenaAllocator, OrtMemTypeDefault, &model_ke->memoryInfo));
607
-
608
-
htv->run_keypoint_model = runKeypointEstimator;
609
-
}
610
-
611
-
static void
612
-
initHandDetector(struct ht_device *htd, ht_view *htv)
613
-
{
614
-
struct ModelInfo *model_hd = &htv->detection_model;
615
-
memset(model_hd, 0, sizeof(struct ModelInfo));
616
-
617
-
const OrtApi *g_ort = htd->ort_api;
618
-
OrtSessionOptions *opts = nullptr;
619
-
ORT_CHECK(g_ort, g_ort->CreateSessionOptions(&opts));
620
-
621
-
ORT_CHECK(g_ort, g_ort->SetSessionGraphOptimizationLevel(opts, ORT_ENABLE_ALL));
622
-
ORT_CHECK(g_ort, g_ort->SetIntraOpNumThreads(opts, 1));
623
-
624
-
char modelLocation[1024];
625
-
626
-
// Hard-coded. Even though you can use the ONNX runtime's API to dynamically figure these out, that doesn't make
627
-
// any sense because these don't change between runs, and if you are swapping models you have to do much more
628
-
// than just change the input/output names.
629
-
if (htd->startup_config.palm_detection_use_mediapipe) {
630
-
addSlug(htd, "palm_detection_MEDIAPIPE.onnx", modelLocation);
631
-
model_hd->input_shape.push_back(1);
632
-
model_hd->input_shape.push_back(3);
633
-
model_hd->input_shape.push_back(128);
634
-
model_hd->input_shape.push_back(128);
635
-
636
-
model_hd->input_names.push_back("input");
637
-
} else {
638
-
addSlug(htd, "palm_detection_COLLABORA.onnx", modelLocation);
639
-
640
-
model_hd->input_shape.push_back(1);
641
-
model_hd->input_shape.push_back(128);
642
-
model_hd->input_shape.push_back(128);
643
-
model_hd->input_shape.push_back(3);
644
-
645
-
model_hd->input_names.push_back("input:0");
646
-
}
647
-
648
-
ORT_CHECK(g_ort, g_ort->CreateSession(htd->ort_env, modelLocation, opts, &model_hd->session));
649
-
g_ort->ReleaseSessionOptions(opts);
650
-
651
-
652
-
653
-
model_hd->output_names.push_back("classificators");
654
-
model_hd->output_names.push_back("regressors");
655
-
656
-
ORT_CHECK(g_ort, g_ort->CreateCpuMemoryInfo(OrtArenaAllocator, OrtMemTypeDefault, &model_hd->memoryInfo));
657
-
658
-
htv->run_detection_model = runHandDetector;
659
-
}
660
-
661
-
void
662
-
initOnnx(struct ht_device *htd)
663
-
{
664
-
htd->ort_api = OrtGetApiBase()->GetApi(ORT_API_VERSION);
665
-
ORT_CHECK(htd->ort_api, htd->ort_api->CreateEnv(ORT_LOGGING_LEVEL_WARNING, "moses", &htd->ort_env));
666
-
667
-
initHandDetector(htd, &htd->views[0]);
668
-
initHandDetector(htd, &htd->views[1]);
669
-
670
-
initKeypointEstimator(htd, &htd->views[0]);
671
-
initKeypointEstimator(htd, &htd->views[1]);
672
-
}
673
-
674
-
static void
675
-
destroyModelInfo(struct ht_device *htd, ModelInfo *info)
676
-
{
677
-
const OrtApi *g_ort = htd->ort_api;
678
-
g_ort->ReleaseSession(info->session);
679
-
g_ort->ReleaseMemoryInfo(info->memoryInfo);
680
-
// Same deal as in ht_device - I'm mixing C and C++ idioms, so sometimes it's easier to just manually call their
681
-
// destructors instead of figuring out some way to convince C++ to call them implicitly.
682
-
info->output_names.~vector();
683
-
info->input_names.~vector();
684
-
info->input_shape.~vector();
685
-
}
686
-
687
-
void
688
-
destroyOnnx(struct ht_device *htd)
689
-
{
690
-
destroyModelInfo(htd, &htd->views[0].keypoint_model);
691
-
destroyModelInfo(htd, &htd->views[1].keypoint_model);
692
-
destroyModelInfo(htd, &htd->views[0].detection_model);
693
-
destroyModelInfo(htd, &htd->views[1].detection_model);
694
-
htd->ort_api->ReleaseEnv(htd->ort_env);
695
-
}
-19
src/xrt/drivers/ht/ht_models.hpp
-19
src/xrt/drivers/ht/ht_models.hpp
···
1
-
// Copyright 2021, Collabora, Ltd.
2
-
// SPDX-License-Identifier: BSL-1.0
3
-
/*!
4
-
* @file
5
-
* @brief Code to run machine learning models for camera-based hand tracker.
6
-
* @author Moses Turner <moses@collabora.com>
7
-
* @author Marcus Edel <marcus.edel@collabora.com>
8
-
* @ingroup drv_ht
9
-
*/
10
-
11
-
#pragma once
12
-
13
-
struct ht_device;
14
-
15
-
void
16
-
initOnnx(struct ht_device *htd);
17
-
18
-
void
19
-
destroyOnnx(struct ht_device *htd);