The open source OpenXR runtime
d/ht: split ht_models into source and header
authored by
Simon Zeni
and committed by
Moses Turner
39bc1295
7618a7e3
+702
-713
+1
-1
src/xrt/drivers/CMakeLists.txt
+1
-1
src/xrt/drivers/CMakeLists.txt
+1
src/xrt/drivers/ht/ht_algorithm.hpp
+1
src/xrt/drivers/ht/ht_algorithm.hpp
+695
src/xrt/drivers/ht/ht_models.cpp
+695
src/xrt/drivers/ht/ht_models.cpp
···
1
+
// Copyright 2021, Collabora, Ltd.
2
+
// SPDX-License-Identifier: BSL-1.0
3
+
/*!
4
+
* @file
5
+
* @brief Code to run machine learning models for camera-based hand tracker.
6
+
* @author Moses Turner <moses@collabora.com>
7
+
* @author Marcus Edel <marcus.edel@collabora.com>
8
+
* @ingroup drv_ht
9
+
*/
10
+
11
+
// Many C api things were stolen from here (MIT license):
12
+
// https://github.com/microsoft/onnxruntime-inference-examples/blob/main/c_cxx/fns_candy_style_transfer/fns_candy_style_transfer.c
13
+
14
+
#include "ht_driver.hpp"
15
+
#include "ht_image_math.hpp"
16
+
#include "ht_models.hpp"
17
+
#include "ht_nms.hpp"
18
+
19
+
#include <core/session/onnxruntime_c_api.h>
20
+
21
+
#include <opencv2/core/mat.hpp>
22
+
23
+
#define ORT_CHECK(g_ort, expr) \
24
+
do { \
25
+
OrtStatus *onnx_status = (expr); \
26
+
if (onnx_status != nullptr) { \
27
+
const char *msg = g_ort->GetErrorMessage(onnx_status); \
28
+
U_LOG_E("at %s:%d: %s\n", __FILE__, __LINE__, msg); \
29
+
g_ort->ReleaseStatus(onnx_status); \
30
+
assert(false); \
31
+
} \
32
+
} while (0);
33
+
34
+
static Hand2D
35
+
runKeypointEstimator(struct ht_view *htv, cv::Mat img)
36
+
{
37
+
constexpr size_t lix = 224;
38
+
constexpr size_t liy = 224;
39
+
constexpr size_t nb_planes = 3;
40
+
cv::Mat planes[nb_planes];
41
+
42
+
constexpr size_t size = lix * liy * nb_planes;
43
+
44
+
std::vector<uint8_t> combined_planes(size);
45
+
planarize(img, combined_planes.data());
46
+
47
+
// Normalize - supposedly, the keypoint estimator wants keypoints in [0,1]
48
+
std::vector<float> real_thing(size);
49
+
for (size_t i = 0; i < size; i++) {
50
+
real_thing[i] = (float)combined_planes[i] / 255.0;
51
+
}
52
+
53
+
const OrtApi *g_ort = htv->htd->ort_api;
54
+
struct ModelInfo *model = &htv->keypoint_model;
55
+
56
+
OrtValue *input_tensor = nullptr;
57
+
58
+
ORT_CHECK(g_ort, g_ort->CreateTensorWithDataAsOrtValue(
59
+
model->memoryInfo, real_thing.data(), model->input_size_bytes, model->input_shape.data(),
60
+
model->input_shape.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &input_tensor));
61
+
62
+
// Cargo-culted
63
+
assert(input_tensor != nullptr);
64
+
int is_tensor;
65
+
ORT_CHECK(g_ort, g_ort->IsTensor(input_tensor, &is_tensor));
66
+
assert(is_tensor);
67
+
68
+
const char *output_names[] = {"Identity", "Identity_2", "Identity_2"};
69
+
70
+
// If any of these are non-NULL, ONNX will explode and won't tell you why! Be extremely paranoid!
71
+
OrtValue *output_tensor[3] = {nullptr, nullptr, nullptr};
72
+
ORT_CHECK(g_ort, g_ort->Run(model->session, nullptr, model->input_names.data(), &input_tensor, 1, output_names,
73
+
3, output_tensor));
74
+
75
+
ORT_CHECK(g_ort, g_ort->IsTensor(output_tensor[0], &is_tensor));
76
+
assert(is_tensor);
77
+
78
+
float *landmarks = nullptr;
79
+
80
+
// Should give a pointer to data that is freed on g_ort->ReleaseValue(output_tensor[0]);.
81
+
ORT_CHECK(g_ort, g_ort->GetTensorMutableData(output_tensor[0], (void **)&landmarks));
82
+
83
+
int stride = 3;
84
+
Hand2D dumb;
85
+
for (size_t i = 0; i < 21; i++) {
86
+
int rt = i * stride;
87
+
float x = landmarks[rt];
88
+
float y = landmarks[rt + 1];
89
+
float z = landmarks[rt + 2];
90
+
dumb.kps[i].x = x;
91
+
dumb.kps[i].y = y;
92
+
dumb.kps[i].z = z;
93
+
}
94
+
95
+
// We have to get to here, or else we leak a whole lot! If you need to return early, make this into a goto!
96
+
g_ort->ReleaseValue(output_tensor[0]);
97
+
g_ort->ReleaseValue(output_tensor[1]);
98
+
g_ort->ReleaseValue(output_tensor[2]);
99
+
g_ort->ReleaseValue(input_tensor);
100
+
return dumb;
101
+
}
102
+
103
+
#undef HEAVY_SCRIBBLE
104
+
105
+
/*
106
+
* Anchors data taken from mediapipe's hands detection, used for single-shot
107
+
* detector model.
108
+
*
109
+
* See:
110
+
* https://google.github.io/mediapipe/solutions/hands.html#palm-detection-model
111
+
* https://github.com/google/mediapipe/blob/v0.8.8/mediapipe/calculators/tflite/ssd_anchors_calculator.cc#L101
112
+
* https://github.com/google/mediapipe/blob/v0.8.8/mediapipe/modules/palm_detection/palm_detection_cpu.pbtxt#L60
113
+
*/
114
+
struct anchor
115
+
{
116
+
float x, y;
117
+
};
118
+
119
+
static const struct anchor anchors[896]{
120
+
{0.031250, 0.031250}, {0.031250, 0.031250}, {0.093750, 0.031250}, {0.093750, 0.031250}, //
121
+
{0.156250, 0.031250}, {0.156250, 0.031250}, {0.218750, 0.031250}, {0.218750, 0.031250}, //
122
+
{0.281250, 0.031250}, {0.281250, 0.031250}, {0.343750, 0.031250}, {0.343750, 0.031250}, //
123
+
{0.406250, 0.031250}, {0.406250, 0.031250}, {0.468750, 0.031250}, {0.468750, 0.031250}, //
124
+
{0.531250, 0.031250}, {0.531250, 0.031250}, {0.593750, 0.031250}, {0.593750, 0.031250}, //
125
+
{0.656250, 0.031250}, {0.656250, 0.031250}, {0.718750, 0.031250}, {0.718750, 0.031250}, //
126
+
{0.781250, 0.031250}, {0.781250, 0.031250}, {0.843750, 0.031250}, {0.843750, 0.031250}, //
127
+
{0.906250, 0.031250}, {0.906250, 0.031250}, {0.968750, 0.031250}, {0.968750, 0.031250}, //
128
+
{0.031250, 0.093750}, {0.031250, 0.093750}, {0.093750, 0.093750}, {0.093750, 0.093750}, //
129
+
{0.156250, 0.093750}, {0.156250, 0.093750}, {0.218750, 0.093750}, {0.218750, 0.093750}, //
130
+
{0.281250, 0.093750}, {0.281250, 0.093750}, {0.343750, 0.093750}, {0.343750, 0.093750}, //
131
+
{0.406250, 0.093750}, {0.406250, 0.093750}, {0.468750, 0.093750}, {0.468750, 0.093750}, //
132
+
{0.531250, 0.093750}, {0.531250, 0.093750}, {0.593750, 0.093750}, {0.593750, 0.093750}, //
133
+
{0.656250, 0.093750}, {0.656250, 0.093750}, {0.718750, 0.093750}, {0.718750, 0.093750}, //
134
+
{0.781250, 0.093750}, {0.781250, 0.093750}, {0.843750, 0.093750}, {0.843750, 0.093750}, //
135
+
{0.906250, 0.093750}, {0.906250, 0.093750}, {0.968750, 0.093750}, {0.968750, 0.093750}, //
136
+
{0.031250, 0.156250}, {0.031250, 0.156250}, {0.093750, 0.156250}, {0.093750, 0.156250}, //
137
+
{0.156250, 0.156250}, {0.156250, 0.156250}, {0.218750, 0.156250}, {0.218750, 0.156250}, //
138
+
{0.281250, 0.156250}, {0.281250, 0.156250}, {0.343750, 0.156250}, {0.343750, 0.156250}, //
139
+
{0.406250, 0.156250}, {0.406250, 0.156250}, {0.468750, 0.156250}, {0.468750, 0.156250}, //
140
+
{0.531250, 0.156250}, {0.531250, 0.156250}, {0.593750, 0.156250}, {0.593750, 0.156250}, //
141
+
{0.656250, 0.156250}, {0.656250, 0.156250}, {0.718750, 0.156250}, {0.718750, 0.156250}, //
142
+
{0.781250, 0.156250}, {0.781250, 0.156250}, {0.843750, 0.156250}, {0.843750, 0.156250}, //
143
+
{0.906250, 0.156250}, {0.906250, 0.156250}, {0.968750, 0.156250}, {0.968750, 0.156250}, //
144
+
{0.031250, 0.218750}, {0.031250, 0.218750}, {0.093750, 0.218750}, {0.093750, 0.218750}, //
145
+
{0.156250, 0.218750}, {0.156250, 0.218750}, {0.218750, 0.218750}, {0.218750, 0.218750}, //
146
+
{0.281250, 0.218750}, {0.281250, 0.218750}, {0.343750, 0.218750}, {0.343750, 0.218750}, //
147
+
{0.406250, 0.218750}, {0.406250, 0.218750}, {0.468750, 0.218750}, {0.468750, 0.218750}, //
148
+
{0.531250, 0.218750}, {0.531250, 0.218750}, {0.593750, 0.218750}, {0.593750, 0.218750}, //
149
+
{0.656250, 0.218750}, {0.656250, 0.218750}, {0.718750, 0.218750}, {0.718750, 0.218750}, //
150
+
{0.781250, 0.218750}, {0.781250, 0.218750}, {0.843750, 0.218750}, {0.843750, 0.218750}, //
151
+
{0.906250, 0.218750}, {0.906250, 0.218750}, {0.968750, 0.218750}, {0.968750, 0.218750}, //
152
+
{0.031250, 0.281250}, {0.031250, 0.281250}, {0.093750, 0.281250}, {0.093750, 0.281250}, //
153
+
{0.156250, 0.281250}, {0.156250, 0.281250}, {0.218750, 0.281250}, {0.218750, 0.281250}, //
154
+
{0.281250, 0.281250}, {0.281250, 0.281250}, {0.343750, 0.281250}, {0.343750, 0.281250}, //
155
+
{0.406250, 0.281250}, {0.406250, 0.281250}, {0.468750, 0.281250}, {0.468750, 0.281250}, //
156
+
{0.531250, 0.281250}, {0.531250, 0.281250}, {0.593750, 0.281250}, {0.593750, 0.281250}, //
157
+
{0.656250, 0.281250}, {0.656250, 0.281250}, {0.718750, 0.281250}, {0.718750, 0.281250}, //
158
+
{0.781250, 0.281250}, {0.781250, 0.281250}, {0.843750, 0.281250}, {0.843750, 0.281250}, //
159
+
{0.906250, 0.281250}, {0.906250, 0.281250}, {0.968750, 0.281250}, {0.968750, 0.281250}, //
160
+
{0.031250, 0.343750}, {0.031250, 0.343750}, {0.093750, 0.343750}, {0.093750, 0.343750}, //
161
+
{0.156250, 0.343750}, {0.156250, 0.343750}, {0.218750, 0.343750}, {0.218750, 0.343750}, //
162
+
{0.281250, 0.343750}, {0.281250, 0.343750}, {0.343750, 0.343750}, {0.343750, 0.343750}, //
163
+
{0.406250, 0.343750}, {0.406250, 0.343750}, {0.468750, 0.343750}, {0.468750, 0.343750}, //
164
+
{0.531250, 0.343750}, {0.531250, 0.343750}, {0.593750, 0.343750}, {0.593750, 0.343750}, //
165
+
{0.656250, 0.343750}, {0.656250, 0.343750}, {0.718750, 0.343750}, {0.718750, 0.343750}, //
166
+
{0.781250, 0.343750}, {0.781250, 0.343750}, {0.843750, 0.343750}, {0.843750, 0.343750}, //
167
+
{0.906250, 0.343750}, {0.906250, 0.343750}, {0.968750, 0.343750}, {0.968750, 0.343750}, //
168
+
{0.031250, 0.406250}, {0.031250, 0.406250}, {0.093750, 0.406250}, {0.093750, 0.406250}, //
169
+
{0.156250, 0.406250}, {0.156250, 0.406250}, {0.218750, 0.406250}, {0.218750, 0.406250}, //
170
+
{0.281250, 0.406250}, {0.281250, 0.406250}, {0.343750, 0.406250}, {0.343750, 0.406250}, //
171
+
{0.406250, 0.406250}, {0.406250, 0.406250}, {0.468750, 0.406250}, {0.468750, 0.406250}, //
172
+
{0.531250, 0.406250}, {0.531250, 0.406250}, {0.593750, 0.406250}, {0.593750, 0.406250}, //
173
+
{0.656250, 0.406250}, {0.656250, 0.406250}, {0.718750, 0.406250}, {0.718750, 0.406250}, //
174
+
{0.781250, 0.406250}, {0.781250, 0.406250}, {0.843750, 0.406250}, {0.843750, 0.406250}, //
175
+
{0.906250, 0.406250}, {0.906250, 0.406250}, {0.968750, 0.406250}, {0.968750, 0.406250}, //
176
+
{0.031250, 0.468750}, {0.031250, 0.468750}, {0.093750, 0.468750}, {0.093750, 0.468750}, //
177
+
{0.156250, 0.468750}, {0.156250, 0.468750}, {0.218750, 0.468750}, {0.218750, 0.468750}, //
178
+
{0.281250, 0.468750}, {0.281250, 0.468750}, {0.343750, 0.468750}, {0.343750, 0.468750}, //
179
+
{0.406250, 0.468750}, {0.406250, 0.468750}, {0.468750, 0.468750}, {0.468750, 0.468750}, //
180
+
{0.531250, 0.468750}, {0.531250, 0.468750}, {0.593750, 0.468750}, {0.593750, 0.468750}, //
181
+
{0.656250, 0.468750}, {0.656250, 0.468750}, {0.718750, 0.468750}, {0.718750, 0.468750}, //
182
+
{0.781250, 0.468750}, {0.781250, 0.468750}, {0.843750, 0.468750}, {0.843750, 0.468750}, //
183
+
{0.906250, 0.468750}, {0.906250, 0.468750}, {0.968750, 0.468750}, {0.968750, 0.468750}, //
184
+
{0.031250, 0.531250}, {0.031250, 0.531250}, {0.093750, 0.531250}, {0.093750, 0.531250}, //
185
+
{0.156250, 0.531250}, {0.156250, 0.531250}, {0.218750, 0.531250}, {0.218750, 0.531250}, //
186
+
{0.281250, 0.531250}, {0.281250, 0.531250}, {0.343750, 0.531250}, {0.343750, 0.531250}, //
187
+
{0.406250, 0.531250}, {0.406250, 0.531250}, {0.468750, 0.531250}, {0.468750, 0.531250}, //
188
+
{0.531250, 0.531250}, {0.531250, 0.531250}, {0.593750, 0.531250}, {0.593750, 0.531250}, //
189
+
{0.656250, 0.531250}, {0.656250, 0.531250}, {0.718750, 0.531250}, {0.718750, 0.531250}, //
190
+
{0.781250, 0.531250}, {0.781250, 0.531250}, {0.843750, 0.531250}, {0.843750, 0.531250}, //
191
+
{0.906250, 0.531250}, {0.906250, 0.531250}, {0.968750, 0.531250}, {0.968750, 0.531250}, //
192
+
{0.031250, 0.593750}, {0.031250, 0.593750}, {0.093750, 0.593750}, {0.093750, 0.593750}, //
193
+
{0.156250, 0.593750}, {0.156250, 0.593750}, {0.218750, 0.593750}, {0.218750, 0.593750}, //
194
+
{0.281250, 0.593750}, {0.281250, 0.593750}, {0.343750, 0.593750}, {0.343750, 0.593750}, //
195
+
{0.406250, 0.593750}, {0.406250, 0.593750}, {0.468750, 0.593750}, {0.468750, 0.593750}, //
196
+
{0.531250, 0.593750}, {0.531250, 0.593750}, {0.593750, 0.593750}, {0.593750, 0.593750}, //
197
+
{0.656250, 0.593750}, {0.656250, 0.593750}, {0.718750, 0.593750}, {0.718750, 0.593750}, //
198
+
{0.781250, 0.593750}, {0.781250, 0.593750}, {0.843750, 0.593750}, {0.843750, 0.593750}, //
199
+
{0.906250, 0.593750}, {0.906250, 0.593750}, {0.968750, 0.593750}, {0.968750, 0.593750}, //
200
+
{0.031250, 0.656250}, {0.031250, 0.656250}, {0.093750, 0.656250}, {0.093750, 0.656250}, //
201
+
{0.156250, 0.656250}, {0.156250, 0.656250}, {0.218750, 0.656250}, {0.218750, 0.656250}, //
202
+
{0.281250, 0.656250}, {0.281250, 0.656250}, {0.343750, 0.656250}, {0.343750, 0.656250}, //
203
+
{0.406250, 0.656250}, {0.406250, 0.656250}, {0.468750, 0.656250}, {0.468750, 0.656250}, //
204
+
{0.531250, 0.656250}, {0.531250, 0.656250}, {0.593750, 0.656250}, {0.593750, 0.656250}, //
205
+
{0.656250, 0.656250}, {0.656250, 0.656250}, {0.718750, 0.656250}, {0.718750, 0.656250}, //
206
+
{0.781250, 0.656250}, {0.781250, 0.656250}, {0.843750, 0.656250}, {0.843750, 0.656250}, //
207
+
{0.906250, 0.656250}, {0.906250, 0.656250}, {0.968750, 0.656250}, {0.968750, 0.656250}, //
208
+
{0.031250, 0.718750}, {0.031250, 0.718750}, {0.093750, 0.718750}, {0.093750, 0.718750}, //
209
+
{0.156250, 0.718750}, {0.156250, 0.718750}, {0.218750, 0.718750}, {0.218750, 0.718750}, //
210
+
{0.281250, 0.718750}, {0.281250, 0.718750}, {0.343750, 0.718750}, {0.343750, 0.718750}, //
211
+
{0.406250, 0.718750}, {0.406250, 0.718750}, {0.468750, 0.718750}, {0.468750, 0.718750}, //
212
+
{0.531250, 0.718750}, {0.531250, 0.718750}, {0.593750, 0.718750}, {0.593750, 0.718750}, //
213
+
{0.656250, 0.718750}, {0.656250, 0.718750}, {0.718750, 0.718750}, {0.718750, 0.718750}, //
214
+
{0.781250, 0.718750}, {0.781250, 0.718750}, {0.843750, 0.718750}, {0.843750, 0.718750}, //
215
+
{0.906250, 0.718750}, {0.906250, 0.718750}, {0.968750, 0.718750}, {0.968750, 0.718750}, //
216
+
{0.031250, 0.781250}, {0.031250, 0.781250}, {0.093750, 0.781250}, {0.093750, 0.781250}, //
217
+
{0.156250, 0.781250}, {0.156250, 0.781250}, {0.218750, 0.781250}, {0.218750, 0.781250}, //
218
+
{0.281250, 0.781250}, {0.281250, 0.781250}, {0.343750, 0.781250}, {0.343750, 0.781250}, //
219
+
{0.406250, 0.781250}, {0.406250, 0.781250}, {0.468750, 0.781250}, {0.468750, 0.781250}, //
220
+
{0.531250, 0.781250}, {0.531250, 0.781250}, {0.593750, 0.781250}, {0.593750, 0.781250}, //
221
+
{0.656250, 0.781250}, {0.656250, 0.781250}, {0.718750, 0.781250}, {0.718750, 0.781250}, //
222
+
{0.781250, 0.781250}, {0.781250, 0.781250}, {0.843750, 0.781250}, {0.843750, 0.781250}, //
223
+
{0.906250, 0.781250}, {0.906250, 0.781250}, {0.968750, 0.781250}, {0.968750, 0.781250}, //
224
+
{0.031250, 0.843750}, {0.031250, 0.843750}, {0.093750, 0.843750}, {0.093750, 0.843750}, //
225
+
{0.156250, 0.843750}, {0.156250, 0.843750}, {0.218750, 0.843750}, {0.218750, 0.843750}, //
226
+
{0.281250, 0.843750}, {0.281250, 0.843750}, {0.343750, 0.843750}, {0.343750, 0.843750}, //
227
+
{0.406250, 0.843750}, {0.406250, 0.843750}, {0.468750, 0.843750}, {0.468750, 0.843750}, //
228
+
{0.531250, 0.843750}, {0.531250, 0.843750}, {0.593750, 0.843750}, {0.593750, 0.843750}, //
229
+
{0.656250, 0.843750}, {0.656250, 0.843750}, {0.718750, 0.843750}, {0.718750, 0.843750}, //
230
+
{0.781250, 0.843750}, {0.781250, 0.843750}, {0.843750, 0.843750}, {0.843750, 0.843750}, //
231
+
{0.906250, 0.843750}, {0.906250, 0.843750}, {0.968750, 0.843750}, {0.968750, 0.843750}, //
232
+
{0.031250, 0.906250}, {0.031250, 0.906250}, {0.093750, 0.906250}, {0.093750, 0.906250}, //
233
+
{0.156250, 0.906250}, {0.156250, 0.906250}, {0.218750, 0.906250}, {0.218750, 0.906250}, //
234
+
{0.281250, 0.906250}, {0.281250, 0.906250}, {0.343750, 0.906250}, {0.343750, 0.906250}, //
235
+
{0.406250, 0.906250}, {0.406250, 0.906250}, {0.468750, 0.906250}, {0.468750, 0.906250}, //
236
+
{0.531250, 0.906250}, {0.531250, 0.906250}, {0.593750, 0.906250}, {0.593750, 0.906250}, //
237
+
{0.656250, 0.906250}, {0.656250, 0.906250}, {0.718750, 0.906250}, {0.718750, 0.906250}, //
238
+
{0.781250, 0.906250}, {0.781250, 0.906250}, {0.843750, 0.906250}, {0.843750, 0.906250}, //
239
+
{0.906250, 0.906250}, {0.906250, 0.906250}, {0.968750, 0.906250}, {0.968750, 0.906250}, //
240
+
{0.031250, 0.968750}, {0.031250, 0.968750}, {0.093750, 0.968750}, {0.093750, 0.968750}, //
241
+
{0.156250, 0.968750}, {0.156250, 0.968750}, {0.218750, 0.968750}, {0.218750, 0.968750}, //
242
+
{0.281250, 0.968750}, {0.281250, 0.968750}, {0.343750, 0.968750}, {0.343750, 0.968750}, //
243
+
{0.406250, 0.968750}, {0.406250, 0.968750}, {0.468750, 0.968750}, {0.468750, 0.968750}, //
244
+
{0.531250, 0.968750}, {0.531250, 0.968750}, {0.593750, 0.968750}, {0.593750, 0.968750}, //
245
+
{0.656250, 0.968750}, {0.656250, 0.968750}, {0.718750, 0.968750}, {0.718750, 0.968750}, //
246
+
{0.781250, 0.968750}, {0.781250, 0.968750}, {0.843750, 0.968750}, {0.843750, 0.968750}, //
247
+
{0.906250, 0.968750}, {0.906250, 0.968750}, {0.968750, 0.968750}, {0.968750, 0.968750}, //
248
+
{0.062500, 0.062500}, {0.062500, 0.062500}, {0.062500, 0.062500}, {0.062500, 0.062500}, //
249
+
{0.062500, 0.062500}, {0.062500, 0.062500}, {0.187500, 0.062500}, {0.187500, 0.062500}, //
250
+
{0.187500, 0.062500}, {0.187500, 0.062500}, {0.187500, 0.062500}, {0.187500, 0.062500}, //
251
+
{0.312500, 0.062500}, {0.312500, 0.062500}, {0.312500, 0.062500}, {0.312500, 0.062500}, //
252
+
{0.312500, 0.062500}, {0.312500, 0.062500}, {0.437500, 0.062500}, {0.437500, 0.062500}, //
253
+
{0.437500, 0.062500}, {0.437500, 0.062500}, {0.437500, 0.062500}, {0.437500, 0.062500}, //
254
+
{0.562500, 0.062500}, {0.562500, 0.062500}, {0.562500, 0.062500}, {0.562500, 0.062500}, //
255
+
{0.562500, 0.062500}, {0.562500, 0.062500}, {0.687500, 0.062500}, {0.687500, 0.062500}, //
256
+
{0.687500, 0.062500}, {0.687500, 0.062500}, {0.687500, 0.062500}, {0.687500, 0.062500}, //
257
+
{0.812500, 0.062500}, {0.812500, 0.062500}, {0.812500, 0.062500}, {0.812500, 0.062500}, //
258
+
{0.812500, 0.062500}, {0.812500, 0.062500}, {0.937500, 0.062500}, {0.937500, 0.062500}, //
259
+
{0.937500, 0.062500}, {0.937500, 0.062500}, {0.937500, 0.062500}, {0.937500, 0.062500}, //
260
+
{0.062500, 0.187500}, {0.062500, 0.187500}, {0.062500, 0.187500}, {0.062500, 0.187500}, //
261
+
{0.062500, 0.187500}, {0.062500, 0.187500}, {0.187500, 0.187500}, {0.187500, 0.187500}, //
262
+
{0.187500, 0.187500}, {0.187500, 0.187500}, {0.187500, 0.187500}, {0.187500, 0.187500}, //
263
+
{0.312500, 0.187500}, {0.312500, 0.187500}, {0.312500, 0.187500}, {0.312500, 0.187500}, //
264
+
{0.312500, 0.187500}, {0.312500, 0.187500}, {0.437500, 0.187500}, {0.437500, 0.187500}, //
265
+
{0.437500, 0.187500}, {0.437500, 0.187500}, {0.437500, 0.187500}, {0.437500, 0.187500}, //
266
+
{0.562500, 0.187500}, {0.562500, 0.187500}, {0.562500, 0.187500}, {0.562500, 0.187500}, //
267
+
{0.562500, 0.187500}, {0.562500, 0.187500}, {0.687500, 0.187500}, {0.687500, 0.187500}, //
268
+
{0.687500, 0.187500}, {0.687500, 0.187500}, {0.687500, 0.187500}, {0.687500, 0.187500}, //
269
+
{0.812500, 0.187500}, {0.812500, 0.187500}, {0.812500, 0.187500}, {0.812500, 0.187500}, //
270
+
{0.812500, 0.187500}, {0.812500, 0.187500}, {0.937500, 0.187500}, {0.937500, 0.187500}, //
271
+
{0.937500, 0.187500}, {0.937500, 0.187500}, {0.937500, 0.187500}, {0.937500, 0.187500}, //
272
+
{0.062500, 0.312500}, {0.062500, 0.312500}, {0.062500, 0.312500}, {0.062500, 0.312500}, //
273
+
{0.062500, 0.312500}, {0.062500, 0.312500}, {0.187500, 0.312500}, {0.187500, 0.312500}, //
274
+
{0.187500, 0.312500}, {0.187500, 0.312500}, {0.187500, 0.312500}, {0.187500, 0.312500}, //
275
+
{0.312500, 0.312500}, {0.312500, 0.312500}, {0.312500, 0.312500}, {0.312500, 0.312500}, //
276
+
{0.312500, 0.312500}, {0.312500, 0.312500}, {0.437500, 0.312500}, {0.437500, 0.312500}, //
277
+
{0.437500, 0.312500}, {0.437500, 0.312500}, {0.437500, 0.312500}, {0.437500, 0.312500}, //
278
+
{0.562500, 0.312500}, {0.562500, 0.312500}, {0.562500, 0.312500}, {0.562500, 0.312500}, //
279
+
{0.562500, 0.312500}, {0.562500, 0.312500}, {0.687500, 0.312500}, {0.687500, 0.312500}, //
280
+
{0.687500, 0.312500}, {0.687500, 0.312500}, {0.687500, 0.312500}, {0.687500, 0.312500}, //
281
+
{0.812500, 0.312500}, {0.812500, 0.312500}, {0.812500, 0.312500}, {0.812500, 0.312500}, //
282
+
{0.812500, 0.312500}, {0.812500, 0.312500}, {0.937500, 0.312500}, {0.937500, 0.312500}, //
283
+
{0.937500, 0.312500}, {0.937500, 0.312500}, {0.937500, 0.312500}, {0.937500, 0.312500}, //
284
+
{0.062500, 0.437500}, {0.062500, 0.437500}, {0.062500, 0.437500}, {0.062500, 0.437500}, //
285
+
{0.062500, 0.437500}, {0.062500, 0.437500}, {0.187500, 0.437500}, {0.187500, 0.437500}, //
286
+
{0.187500, 0.437500}, {0.187500, 0.437500}, {0.187500, 0.437500}, {0.187500, 0.437500}, //
287
+
{0.312500, 0.437500}, {0.312500, 0.437500}, {0.312500, 0.437500}, {0.312500, 0.437500}, //
288
+
{0.312500, 0.437500}, {0.312500, 0.437500}, {0.437500, 0.437500}, {0.437500, 0.437500}, //
289
+
{0.437500, 0.437500}, {0.437500, 0.437500}, {0.437500, 0.437500}, {0.437500, 0.437500}, //
290
+
{0.562500, 0.437500}, {0.562500, 0.437500}, {0.562500, 0.437500}, {0.562500, 0.437500}, //
291
+
{0.562500, 0.437500}, {0.562500, 0.437500}, {0.687500, 0.437500}, {0.687500, 0.437500}, //
292
+
{0.687500, 0.437500}, {0.687500, 0.437500}, {0.687500, 0.437500}, {0.687500, 0.437500}, //
293
+
{0.812500, 0.437500}, {0.812500, 0.437500}, {0.812500, 0.437500}, {0.812500, 0.437500}, //
294
+
{0.812500, 0.437500}, {0.812500, 0.437500}, {0.937500, 0.437500}, {0.937500, 0.437500}, //
295
+
{0.937500, 0.437500}, {0.937500, 0.437500}, {0.937500, 0.437500}, {0.937500, 0.437500}, //
296
+
{0.062500, 0.562500}, {0.062500, 0.562500}, {0.062500, 0.562500}, {0.062500, 0.562500}, //
297
+
{0.062500, 0.562500}, {0.062500, 0.562500}, {0.187500, 0.562500}, {0.187500, 0.562500}, //
298
+
{0.187500, 0.562500}, {0.187500, 0.562500}, {0.187500, 0.562500}, {0.187500, 0.562500}, //
299
+
{0.312500, 0.562500}, {0.312500, 0.562500}, {0.312500, 0.562500}, {0.312500, 0.562500}, //
300
+
{0.312500, 0.562500}, {0.312500, 0.562500}, {0.437500, 0.562500}, {0.437500, 0.562500}, //
301
+
{0.437500, 0.562500}, {0.437500, 0.562500}, {0.437500, 0.562500}, {0.437500, 0.562500}, //
302
+
{0.562500, 0.562500}, {0.562500, 0.562500}, {0.562500, 0.562500}, {0.562500, 0.562500}, //
303
+
{0.562500, 0.562500}, {0.562500, 0.562500}, {0.687500, 0.562500}, {0.687500, 0.562500}, //
304
+
{0.687500, 0.562500}, {0.687500, 0.562500}, {0.687500, 0.562500}, {0.687500, 0.562500}, //
305
+
{0.812500, 0.562500}, {0.812500, 0.562500}, {0.812500, 0.562500}, {0.812500, 0.562500}, //
306
+
{0.812500, 0.562500}, {0.812500, 0.562500}, {0.937500, 0.562500}, {0.937500, 0.562500}, //
307
+
{0.937500, 0.562500}, {0.937500, 0.562500}, {0.937500, 0.562500}, {0.937500, 0.562500}, //
308
+
{0.062500, 0.687500}, {0.062500, 0.687500}, {0.062500, 0.687500}, {0.062500, 0.687500}, //
309
+
{0.062500, 0.687500}, {0.062500, 0.687500}, {0.187500, 0.687500}, {0.187500, 0.687500}, //
310
+
{0.187500, 0.687500}, {0.187500, 0.687500}, {0.187500, 0.687500}, {0.187500, 0.687500}, //
311
+
{0.312500, 0.687500}, {0.312500, 0.687500}, {0.312500, 0.687500}, {0.312500, 0.687500}, //
312
+
{0.312500, 0.687500}, {0.312500, 0.687500}, {0.437500, 0.687500}, {0.437500, 0.687500}, //
313
+
{0.437500, 0.687500}, {0.437500, 0.687500}, {0.437500, 0.687500}, {0.437500, 0.687500}, //
314
+
{0.562500, 0.687500}, {0.562500, 0.687500}, {0.562500, 0.687500}, {0.562500, 0.687500}, //
315
+
{0.562500, 0.687500}, {0.562500, 0.687500}, {0.687500, 0.687500}, {0.687500, 0.687500}, //
316
+
{0.687500, 0.687500}, {0.687500, 0.687500}, {0.687500, 0.687500}, {0.687500, 0.687500}, //
317
+
{0.812500, 0.687500}, {0.812500, 0.687500}, {0.812500, 0.687500}, {0.812500, 0.687500}, //
318
+
{0.812500, 0.687500}, {0.812500, 0.687500}, {0.937500, 0.687500}, {0.937500, 0.687500}, //
319
+
{0.937500, 0.687500}, {0.937500, 0.687500}, {0.937500, 0.687500}, {0.937500, 0.687500}, //
320
+
{0.062500, 0.812500}, {0.062500, 0.812500}, {0.062500, 0.812500}, {0.062500, 0.812500}, //
321
+
{0.062500, 0.812500}, {0.062500, 0.812500}, {0.187500, 0.812500}, {0.187500, 0.812500}, //
322
+
{0.187500, 0.812500}, {0.187500, 0.812500}, {0.187500, 0.812500}, {0.187500, 0.812500}, //
323
+
{0.312500, 0.812500}, {0.312500, 0.812500}, {0.312500, 0.812500}, {0.312500, 0.812500}, //
324
+
{0.312500, 0.812500}, {0.312500, 0.812500}, {0.437500, 0.812500}, {0.437500, 0.812500}, //
325
+
{0.437500, 0.812500}, {0.437500, 0.812500}, {0.437500, 0.812500}, {0.437500, 0.812500}, //
326
+
{0.562500, 0.812500}, {0.562500, 0.812500}, {0.562500, 0.812500}, {0.562500, 0.812500}, //
327
+
{0.562500, 0.812500}, {0.562500, 0.812500}, {0.687500, 0.812500}, {0.687500, 0.812500}, //
328
+
{0.687500, 0.812500}, {0.687500, 0.812500}, {0.687500, 0.812500}, {0.687500, 0.812500}, //
329
+
{0.812500, 0.812500}, {0.812500, 0.812500}, {0.812500, 0.812500}, {0.812500, 0.812500}, //
330
+
{0.812500, 0.812500}, {0.812500, 0.812500}, {0.937500, 0.812500}, {0.937500, 0.812500}, //
331
+
{0.937500, 0.812500}, {0.937500, 0.812500}, {0.937500, 0.812500}, {0.937500, 0.812500}, //
332
+
{0.062500, 0.937500}, {0.062500, 0.937500}, {0.062500, 0.937500}, {0.062500, 0.937500}, //
333
+
{0.062500, 0.937500}, {0.062500, 0.937500}, {0.187500, 0.937500}, {0.187500, 0.937500}, //
334
+
{0.187500, 0.937500}, {0.187500, 0.937500}, {0.187500, 0.937500}, {0.187500, 0.937500}, //
335
+
{0.312500, 0.937500}, {0.312500, 0.937500}, {0.312500, 0.937500}, {0.312500, 0.937500}, //
336
+
{0.312500, 0.937500}, {0.312500, 0.937500}, {0.437500, 0.937500}, {0.437500, 0.937500}, //
337
+
{0.437500, 0.937500}, {0.437500, 0.937500}, {0.437500, 0.937500}, {0.437500, 0.937500}, //
338
+
{0.562500, 0.937500}, {0.562500, 0.937500}, {0.562500, 0.937500}, {0.562500, 0.937500}, //
339
+
{0.562500, 0.937500}, {0.562500, 0.937500}, {0.687500, 0.937500}, {0.687500, 0.937500}, //
340
+
{0.687500, 0.937500}, {0.687500, 0.937500}, {0.687500, 0.937500}, {0.687500, 0.937500}, //
341
+
{0.812500, 0.937500}, {0.812500, 0.937500}, {0.812500, 0.937500}, {0.812500, 0.937500}, //
342
+
{0.812500, 0.937500}, {0.812500, 0.937500}, {0.937500, 0.937500}, {0.937500, 0.937500}, //
343
+
{0.937500, 0.937500}, {0.937500, 0.937500}, {0.937500, 0.937500}, {0.937500, 0.937500}, //
344
+
};
345
+
346
+
static std::vector<Palm7KP>
347
+
runHandDetector(struct ht_view *htv, cv::Mat &raw_input)
348
+
{
349
+
const OrtApi *g_ort = htv->htd->ort_api;
350
+
351
+
cv::Mat img;
352
+
353
+
constexpr int hd_size = 128;
354
+
constexpr size_t nb_planes = 3;
355
+
constexpr size_t size = hd_size * hd_size * nb_planes;
356
+
constexpr int inputTensorSize = size * sizeof(float);
357
+
358
+
cv::Matx23f back_from_blackbar = blackbar(raw_input, img, {hd_size, hd_size});
359
+
360
+
float scale_factor = back_from_blackbar(0, 0); // 960/128
361
+
assert(img.isContinuous());
362
+
constexpr float mean = 128.0f;
363
+
constexpr float std = 128.0f;
364
+
365
+
std::vector<float> real_thing(size);
366
+
367
+
if (htv->htd->startup_config.palm_detection_use_mediapipe) {
368
+
std::vector<uint8_t> combined_planes(size);
369
+
planarize(img, combined_planes.data());
370
+
for (size_t i = 0; i < size; i++) {
371
+
float val = (float)combined_planes[i];
372
+
real_thing[i] = (val - mean) / std;
373
+
}
374
+
} else {
375
+
376
+
assert(img.isContinuous());
377
+
378
+
for (size_t i = 0; i < size; i++) {
379
+
int val = img.data[i];
380
+
381
+
real_thing[i] = (val - mean) / std;
382
+
}
383
+
}
384
+
385
+
// Convenience
386
+
struct ModelInfo *model_hd = &htv->detection_model;
387
+
388
+
// If this is non-NULL, ONNX will explode and won't tell you why!
389
+
OrtValue *input_tensor = nullptr;
390
+
391
+
ORT_CHECK(g_ort, g_ort->CreateTensorWithDataAsOrtValue(
392
+
model_hd->memoryInfo, real_thing.data(), inputTensorSize, model_hd->input_shape.data(),
393
+
model_hd->input_shape.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &input_tensor));
394
+
395
+
// Cargo-culted
396
+
assert(input_tensor != nullptr);
397
+
int is_tensor;
398
+
ORT_CHECK(g_ort, g_ort->IsTensor(input_tensor, &is_tensor));
399
+
assert(is_tensor);
400
+
401
+
// If any of these are non-NULL, ONNX will explode and won't tell you why! Be extremely paranoid!
402
+
OrtValue *output_tensor[2] = {nullptr, nullptr};
403
+
ORT_CHECK(g_ort, g_ort->Run(model_hd->session, nullptr, model_hd->input_names.data(),
404
+
(const OrtValue *const *)&input_tensor, model_hd->input_names.size(),
405
+
model_hd->output_names.data(), model_hd->output_names.size(), output_tensor));
406
+
407
+
ORT_CHECK(g_ort, g_ort->IsTensor(output_tensor[0], &is_tensor));
408
+
assert(is_tensor);
409
+
410
+
float *classificators = nullptr;
411
+
float *regressors = nullptr;
412
+
413
+
ORT_CHECK(g_ort,
414
+
g_ort->GetTensorMutableData(output_tensor[0], (void **)&classificators)); // Totally won't segfault!
415
+
ORT_CHECK(g_ort, g_ort->GetTensorMutableData(output_tensor[1], (void **)®ressors));
416
+
// We need to free these!
417
+
418
+
float *rg = regressors; // shorter name
419
+
420
+
std::vector<NMSPalm> detections;
421
+
int count = 0;
422
+
423
+
std::vector<Palm7KP> output;
424
+
std::vector<NMSPalm> nms_palms;
425
+
426
+
OrtTensorTypeAndShapeInfo *classificators_info;
427
+
ORT_CHECK(g_ort, g_ort->GetTensorTypeAndShape(output_tensor[0], &classificators_info));
428
+
429
+
size_t classificators_size;
430
+
ORT_CHECK(g_ort, g_ort->GetTensorShapeElementCount(classificators_info, &classificators_size));
431
+
assert(classificators_size == 896);
432
+
433
+
OrtTensorTypeAndShapeInfo *regressors_info;
434
+
ORT_CHECK(g_ort, g_ort->GetTensorTypeAndShape(output_tensor[1], ®ressors_info));
435
+
436
+
size_t regressors_size;
437
+
ORT_CHECK(g_ort, g_ort->GetTensorShapeElementCount(regressors_info, ®ressors_size));
438
+
assert(regressors_size == 896 * 18);
439
+
440
+
g_ort->ReleaseTensorTypeAndShapeInfo(classificators_info);
441
+
g_ort->ReleaseTensorTypeAndShapeInfo(regressors_info);
442
+
443
+
for (size_t i = 0; i < classificators_size; ++i) {
444
+
const float score = 1.0 / (1.0 + exp(-classificators[i]));
445
+
446
+
// Let a lot of detections in - they'll be slowly rejected later
447
+
if (score <= htv->htd->dynamic_config.nms_threshold.val) {
448
+
continue;
449
+
}
450
+
451
+
const struct anchor *anchor = &anchors[i];
452
+
453
+
// Boundary box.
454
+
NMSPalm det;
455
+
456
+
float anchx = anchor->x * 128;
457
+
float anchy = anchor->y * 128;
458
+
459
+
float shiftx = regressors[i * 18];
460
+
float shifty = regressors[i * 18 + 1];
461
+
462
+
float w = regressors[i * 18 + 2];
463
+
float h = regressors[i * 18 + 3];
464
+
465
+
float cx = shiftx + anchx;
466
+
float cy = shifty + anchy;
467
+
468
+
struct xrt_vec2 *kps = det.keypoints;
469
+
470
+
kps[0] = {rg[i * 18 + 4], rg[i * 18 + 5]};
471
+
kps[1] = {rg[i * 18 + 6], rg[i * 18 + 7]};
472
+
kps[2] = {rg[i * 18 + 8], rg[i * 18 + 9]};
473
+
kps[3] = {rg[i * 18 + 10], rg[i * 18 + 11]};
474
+
kps[4] = {rg[i * 18 + 12], rg[i * 18 + 13]};
475
+
kps[5] = {rg[i * 18 + 14], rg[i * 18 + 15]};
476
+
kps[6] = {rg[i * 18 + 16], rg[i * 18 + 17]};
477
+
478
+
479
+
for (int i = 0; i < 7; i++) {
480
+
struct xrt_vec2 *b = &kps[i];
481
+
b->x += anchx;
482
+
b->y += anchy;
483
+
}
484
+
485
+
det.bbox.w = w;
486
+
det.bbox.h = h;
487
+
det.bbox.cx = cx;
488
+
det.bbox.cy = cy;
489
+
det.confidence = score;
490
+
detections.push_back(det);
491
+
count++;
492
+
493
+
if (htv->htd->debug_scribble && (htv->htd->dynamic_config.scribble_raw_detections)) {
494
+
xrt_vec2 center = transformVecBy2x3(xrt_vec2{cx, cy}, back_from_blackbar);
495
+
496
+
float sz = det.bbox.w * scale_factor;
497
+
498
+
cv::rectangle(htv->debug_out_to_this,
499
+
{(int)(center.x - (sz / 2)), (int)(center.y - (sz / 2)), (int)sz, (int)sz},
500
+
hsv2rgb(0.0f, math_map_ranges(det.confidence, 0.0f, 1.0f, 1.5f, -0.1f),
501
+
math_map_ranges(det.confidence, 0.0f, 1.0f, 0.2f, 1.4f)),
502
+
1);
503
+
504
+
for (int i = 0; i < 7; i++) {
505
+
handDot(htv->debug_out_to_this, transformVecBy2x3(kps[i], back_from_blackbar),
506
+
det.confidence * 7, ((float)i) * (360.0f / 7.0f), det.confidence, 1);
507
+
}
508
+
}
509
+
510
+
511
+
512
+
int square = fmax(w, h);
513
+
514
+
square = square / scale_factor;
515
+
}
516
+
517
+
if (count == 0) {
518
+
goto cleanup;
519
+
}
520
+
521
+
nms_palms = filterBoxesWeightedAvg(detections, htv->htd->dynamic_config.nms_iou.val);
522
+
523
+
524
+
525
+
for (NMSPalm cooler : nms_palms) {
526
+
527
+
// Display box
528
+
529
+
struct xrt_vec2 tl = {cooler.bbox.cx - cooler.bbox.w / 2, cooler.bbox.cy - cooler.bbox.h / 2};
530
+
struct xrt_vec2 bob = transformVecBy2x3(tl, back_from_blackbar);
531
+
float sz = cooler.bbox.w * scale_factor;
532
+
533
+
if (htv->htd->debug_scribble && htv->htd->dynamic_config.scribble_nms_detections) {
534
+
cv::rectangle(htv->debug_out_to_this, {(int)bob.x, (int)bob.y, (int)sz, (int)sz},
535
+
hsv2rgb(180.0f, math_map_ranges(cooler.confidence, 0.0f, 1.0f, 0.8f, -0.1f),
536
+
math_map_ranges(cooler.confidence, 0.0f, 1.0f, 0.2f, 1.4f)),
537
+
2);
538
+
for (int i = 0; i < 7; i++) {
539
+
handDot(htv->debug_out_to_this,
540
+
transformVecBy2x3(cooler.keypoints[i], back_from_blackbar),
541
+
cooler.confidence * 14, ((float)i) * (360.0f / 7.0f), cooler.confidence, 3);
542
+
}
543
+
}
544
+
545
+
546
+
Palm7KP this_element;
547
+
548
+
for (int i = 0; i < 7; i++) {
549
+
struct xrt_vec2 b = cooler.keypoints[i];
550
+
this_element.kps[i] = transformVecBy2x3(b, back_from_blackbar);
551
+
}
552
+
this_element.confidence = cooler.confidence;
553
+
554
+
output.push_back(this_element);
555
+
}
556
+
557
+
cleanup:
558
+
g_ort->ReleaseValue(output_tensor[0]);
559
+
g_ort->ReleaseValue(output_tensor[1]);
560
+
g_ort->ReleaseValue(input_tensor);
561
+
return output;
562
+
}
563
+
564
+
565
+
static void
566
+
addSlug(struct ht_device *htd, const char *suffix, char *out)
567
+
{
568
+
strcpy(out, htd->startup_config.model_slug);
569
+
strcat(out, suffix);
570
+
}
571
+
572
+
static void
573
+
initKeypointEstimator(struct ht_device *htd, ht_view *htv)
574
+
{
575
+
struct ModelInfo *model_ke = &htv->keypoint_model;
576
+
const OrtApi *g_ort = htd->ort_api;
577
+
OrtSessionOptions *opts = nullptr;
578
+
579
+
ORT_CHECK(g_ort, g_ort->CreateSessionOptions(&opts));
580
+
581
+
ORT_CHECK(g_ort, g_ort->SetSessionGraphOptimizationLevel(opts, ORT_ENABLE_ALL));
582
+
ORT_CHECK(g_ort, g_ort->SetIntraOpNumThreads(opts, 1));
583
+
584
+
char modelLocation[1024];
585
+
if (htd->startup_config.keypoint_estimation_use_mediapipe) {
586
+
addSlug(htd, "hand_landmark_MEDIAPIPE.onnx", modelLocation);
587
+
} else {
588
+
addSlug(htd, "hand_landmark_COLLABORA.onnx", modelLocation);
589
+
}
590
+
ORT_CHECK(g_ort, g_ort->CreateSession(htd->ort_env, modelLocation, opts, &model_ke->session));
591
+
g_ort->ReleaseSessionOptions(opts);
592
+
593
+
model_ke->input_shape.push_back(1);
594
+
model_ke->input_shape.push_back(3);
595
+
model_ke->input_shape.push_back(224);
596
+
model_ke->input_shape.push_back(224);
597
+
598
+
model_ke->input_names.push_back("input_1");
599
+
600
+
model_ke->output_names.push_back("Identity");
601
+
model_ke->output_names.push_back("Identity_1");
602
+
model_ke->output_names.push_back("Identity_2");
603
+
604
+
model_ke->input_size_bytes = 224 * 224 * 3 * sizeof(float);
605
+
606
+
ORT_CHECK(g_ort, g_ort->CreateCpuMemoryInfo(OrtArenaAllocator, OrtMemTypeDefault, &model_ke->memoryInfo));
607
+
608
+
htv->run_keypoint_model = runKeypointEstimator;
609
+
}
610
+
611
+
static void
612
+
initHandDetector(struct ht_device *htd, ht_view *htv)
613
+
{
614
+
struct ModelInfo *model_hd = &htv->detection_model;
615
+
memset(model_hd, 0, sizeof(struct ModelInfo));
616
+
617
+
const OrtApi *g_ort = htd->ort_api;
618
+
OrtSessionOptions *opts = nullptr;
619
+
ORT_CHECK(g_ort, g_ort->CreateSessionOptions(&opts));
620
+
621
+
ORT_CHECK(g_ort, g_ort->SetSessionGraphOptimizationLevel(opts, ORT_ENABLE_ALL));
622
+
ORT_CHECK(g_ort, g_ort->SetIntraOpNumThreads(opts, 1));
623
+
624
+
char modelLocation[1024];
625
+
626
+
// Hard-coded. Even though you can use the ONNX runtime's API to dynamically figure these out, that doesn't make
627
+
// any sense because these don't change between runs, and if you are swapping models you have to do much more
628
+
// than just change the input/output names.
629
+
if (htd->startup_config.palm_detection_use_mediapipe) {
630
+
addSlug(htd, "palm_detection_MEDIAPIPE.onnx", modelLocation);
631
+
model_hd->input_shape.push_back(1);
632
+
model_hd->input_shape.push_back(3);
633
+
model_hd->input_shape.push_back(128);
634
+
model_hd->input_shape.push_back(128);
635
+
636
+
model_hd->input_names.push_back("input");
637
+
} else {
638
+
addSlug(htd, "palm_detection_COLLABORA.onnx", modelLocation);
639
+
640
+
model_hd->input_shape.push_back(1);
641
+
model_hd->input_shape.push_back(128);
642
+
model_hd->input_shape.push_back(128);
643
+
model_hd->input_shape.push_back(3);
644
+
645
+
model_hd->input_names.push_back("input:0");
646
+
}
647
+
648
+
ORT_CHECK(g_ort, g_ort->CreateSession(htd->ort_env, modelLocation, opts, &model_hd->session));
649
+
g_ort->ReleaseSessionOptions(opts);
650
+
651
+
652
+
653
+
model_hd->output_names.push_back("classificators");
654
+
model_hd->output_names.push_back("regressors");
655
+
656
+
ORT_CHECK(g_ort, g_ort->CreateCpuMemoryInfo(OrtArenaAllocator, OrtMemTypeDefault, &model_hd->memoryInfo));
657
+
658
+
htv->run_detection_model = runHandDetector;
659
+
}
660
+
661
+
void
662
+
initOnnx(struct ht_device *htd)
663
+
{
664
+
htd->ort_api = OrtGetApiBase()->GetApi(ORT_API_VERSION);
665
+
ORT_CHECK(htd->ort_api, htd->ort_api->CreateEnv(ORT_LOGGING_LEVEL_WARNING, "moses", &htd->ort_env));
666
+
667
+
initHandDetector(htd, &htd->views[0]);
668
+
initHandDetector(htd, &htd->views[1]);
669
+
670
+
initKeypointEstimator(htd, &htd->views[0]);
671
+
initKeypointEstimator(htd, &htd->views[1]);
672
+
}
673
+
674
+
static void
675
+
destroyModelInfo(struct ht_device *htd, ModelInfo *info)
676
+
{
677
+
const OrtApi *g_ort = htd->ort_api;
678
+
g_ort->ReleaseSession(info->session);
679
+
g_ort->ReleaseMemoryInfo(info->memoryInfo);
680
+
// Same deal as in ht_device - I'm mixing C and C++ idioms, so sometimes it's easier to just manually call their
681
+
// destructors instead of figuring out some way to convince C++ to call them implicitly.
682
+
info->output_names.~vector();
683
+
info->input_names.~vector();
684
+
info->input_shape.~vector();
685
+
}
686
+
687
+
void
688
+
destroyOnnx(struct ht_device *htd)
689
+
{
690
+
destroyModelInfo(htd, &htd->views[0].keypoint_model);
691
+
destroyModelInfo(htd, &htd->views[1].keypoint_model);
692
+
destroyModelInfo(htd, &htd->views[0].detection_model);
693
+
destroyModelInfo(htd, &htd->views[1].detection_model);
694
+
htd->ort_api->ReleaseEnv(htd->ort_env);
695
+
}
+4
-711
src/xrt/drivers/ht/ht_models.hpp
+4
-711
src/xrt/drivers/ht/ht_models.hpp
···
8
8
* @ingroup drv_ht
9
9
*/
10
10
11
-
// Many C api things were stolen from here (MIT license):
12
-
// https://github.com/microsoft/onnxruntime-inference-examples/blob/main/c_cxx/fns_candy_style_transfer/fns_candy_style_transfer.c
13
-
14
11
#pragma once
15
12
16
-
#include "os/os_time.h"
17
-
#include "os/os_threading.h"
18
-
19
-
#include "math/m_api.h"
20
-
#include "math/m_vec2.h"
21
-
#include "math/m_vec3.h"
22
-
23
-
#include "util/u_json.h"
24
-
#include "util/u_time.h"
25
-
#include "util/u_logging.h"
26
-
#include "util/u_trace_marker.h"
27
-
28
-
#include "ht_nms.hpp"
29
-
#include "ht_driver.hpp"
30
-
#include "ht_image_math.hpp"
31
-
32
-
#include <core/session/onnxruntime_c_api.h>
33
-
34
-
#include <opencv2/core/types.hpp>
35
-
#include <opencv2/calib3d.hpp>
36
-
#include <opencv2/imgproc.hpp>
37
-
38
-
#include <math.h>
39
-
#include <stdio.h>
40
-
#include <string.h>
41
-
42
-
#include <cmath>
43
-
#include <cstdlib>
44
-
45
-
#include <vector>
46
-
#include <limits>
47
-
#include <numeric>
48
-
#include <sstream>
49
-
#include <fstream>
50
-
#include <iostream>
51
-
#include <exception>
52
-
53
-
#define ORT_CHECK(g_ort, expr) \
54
-
do { \
55
-
OrtStatus *onnx_status = (expr); \
56
-
if (onnx_status != nullptr) { \
57
-
const char *msg = g_ort->GetErrorMessage(onnx_status); \
58
-
U_LOG_E("at %s:%d: %s\n", __FILE__, __LINE__, msg); \
59
-
g_ort->ReleaseStatus(onnx_status); \
60
-
assert(false); \
61
-
} \
62
-
} while (0);
13
+
struct ht_device;
63
14
64
-
static Hand2D
65
-
runKeypointEstimator(struct ht_view *htv, cv::Mat img)
66
-
{
67
-
constexpr size_t lix = 224;
68
-
constexpr size_t liy = 224;
69
-
constexpr size_t nb_planes = 3;
70
-
cv::Mat planes[nb_planes];
71
-
72
-
constexpr size_t size = lix * liy * nb_planes;
73
-
74
-
std::vector<uint8_t> combined_planes(size);
75
-
planarize(img, combined_planes.data());
76
-
77
-
// Normalize - supposedly, the keypoint estimator wants keypoints in [0,1]
78
-
std::vector<float> real_thing(size);
79
-
for (size_t i = 0; i < size; i++) {
80
-
real_thing[i] = (float)combined_planes[i] / 255.0;
81
-
}
82
-
83
-
const OrtApi *g_ort = htv->htd->ort_api;
84
-
struct ModelInfo *model = &htv->keypoint_model;
85
-
86
-
OrtValue *input_tensor = nullptr;
87
-
88
-
ORT_CHECK(g_ort, g_ort->CreateTensorWithDataAsOrtValue(
89
-
model->memoryInfo, real_thing.data(), model->input_size_bytes, model->input_shape.data(),
90
-
model->input_shape.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &input_tensor));
91
-
92
-
// Cargo-culted
93
-
assert(input_tensor != nullptr);
94
-
int is_tensor;
95
-
ORT_CHECK(g_ort, g_ort->IsTensor(input_tensor, &is_tensor));
96
-
assert(is_tensor);
97
-
98
-
const char *output_names[] = {"Identity", "Identity_2", "Identity_2"};
99
-
100
-
// If any of these are non-NULL, ONNX will explode and won't tell you why! Be extremely paranoid!
101
-
OrtValue *output_tensor[3] = {nullptr, nullptr, nullptr};
102
-
ORT_CHECK(g_ort, g_ort->Run(model->session, nullptr, model->input_names.data(), &input_tensor, 1, output_names,
103
-
3, output_tensor));
104
-
105
-
ORT_CHECK(g_ort, g_ort->IsTensor(output_tensor[0], &is_tensor));
106
-
assert(is_tensor);
107
-
108
-
float *landmarks = nullptr;
109
-
110
-
// Should give a pointer to data that is freed on g_ort->ReleaseValue(output_tensor[0]);.
111
-
ORT_CHECK(g_ort, g_ort->GetTensorMutableData(output_tensor[0], (void **)&landmarks));
112
-
113
-
int stride = 3;
114
-
Hand2D dumb;
115
-
for (size_t i = 0; i < 21; i++) {
116
-
int rt = i * stride;
117
-
float x = landmarks[rt];
118
-
float y = landmarks[rt + 1];
119
-
float z = landmarks[rt + 2];
120
-
dumb.kps[i].x = x;
121
-
dumb.kps[i].y = y;
122
-
dumb.kps[i].z = z;
123
-
}
124
-
125
-
// We have to get to here, or else we leak a whole lot! If you need to return early, make this into a goto!
126
-
g_ort->ReleaseValue(output_tensor[0]);
127
-
g_ort->ReleaseValue(output_tensor[1]);
128
-
g_ort->ReleaseValue(output_tensor[2]);
129
-
g_ort->ReleaseValue(input_tensor);
130
-
return dumb;
131
-
}
132
-
133
-
#undef HEAVY_SCRIBBLE
134
-
135
-
/*
136
-
* Anchors data taken from mediapipe's hands detection, used for single-shot
137
-
* detector model.
138
-
*
139
-
* See:
140
-
* https://google.github.io/mediapipe/solutions/hands.html#palm-detection-model
141
-
* https://github.com/google/mediapipe/blob/v0.8.8/mediapipe/calculators/tflite/ssd_anchors_calculator.cc#L101
142
-
* https://github.com/google/mediapipe/blob/v0.8.8/mediapipe/modules/palm_detection/palm_detection_cpu.pbtxt#L60
143
-
*/
144
-
struct anchor
145
-
{
146
-
float x, y;
147
-
};
148
-
149
-
static const struct anchor anchors[896]{
150
-
{0.031250, 0.031250}, {0.031250, 0.031250}, {0.093750, 0.031250}, {0.093750, 0.031250}, //
151
-
{0.156250, 0.031250}, {0.156250, 0.031250}, {0.218750, 0.031250}, {0.218750, 0.031250}, //
152
-
{0.281250, 0.031250}, {0.281250, 0.031250}, {0.343750, 0.031250}, {0.343750, 0.031250}, //
153
-
{0.406250, 0.031250}, {0.406250, 0.031250}, {0.468750, 0.031250}, {0.468750, 0.031250}, //
154
-
{0.531250, 0.031250}, {0.531250, 0.031250}, {0.593750, 0.031250}, {0.593750, 0.031250}, //
155
-
{0.656250, 0.031250}, {0.656250, 0.031250}, {0.718750, 0.031250}, {0.718750, 0.031250}, //
156
-
{0.781250, 0.031250}, {0.781250, 0.031250}, {0.843750, 0.031250}, {0.843750, 0.031250}, //
157
-
{0.906250, 0.031250}, {0.906250, 0.031250}, {0.968750, 0.031250}, {0.968750, 0.031250}, //
158
-
{0.031250, 0.093750}, {0.031250, 0.093750}, {0.093750, 0.093750}, {0.093750, 0.093750}, //
159
-
{0.156250, 0.093750}, {0.156250, 0.093750}, {0.218750, 0.093750}, {0.218750, 0.093750}, //
160
-
{0.281250, 0.093750}, {0.281250, 0.093750}, {0.343750, 0.093750}, {0.343750, 0.093750}, //
161
-
{0.406250, 0.093750}, {0.406250, 0.093750}, {0.468750, 0.093750}, {0.468750, 0.093750}, //
162
-
{0.531250, 0.093750}, {0.531250, 0.093750}, {0.593750, 0.093750}, {0.593750, 0.093750}, //
163
-
{0.656250, 0.093750}, {0.656250, 0.093750}, {0.718750, 0.093750}, {0.718750, 0.093750}, //
164
-
{0.781250, 0.093750}, {0.781250, 0.093750}, {0.843750, 0.093750}, {0.843750, 0.093750}, //
165
-
{0.906250, 0.093750}, {0.906250, 0.093750}, {0.968750, 0.093750}, {0.968750, 0.093750}, //
166
-
{0.031250, 0.156250}, {0.031250, 0.156250}, {0.093750, 0.156250}, {0.093750, 0.156250}, //
167
-
{0.156250, 0.156250}, {0.156250, 0.156250}, {0.218750, 0.156250}, {0.218750, 0.156250}, //
168
-
{0.281250, 0.156250}, {0.281250, 0.156250}, {0.343750, 0.156250}, {0.343750, 0.156250}, //
169
-
{0.406250, 0.156250}, {0.406250, 0.156250}, {0.468750, 0.156250}, {0.468750, 0.156250}, //
170
-
{0.531250, 0.156250}, {0.531250, 0.156250}, {0.593750, 0.156250}, {0.593750, 0.156250}, //
171
-
{0.656250, 0.156250}, {0.656250, 0.156250}, {0.718750, 0.156250}, {0.718750, 0.156250}, //
172
-
{0.781250, 0.156250}, {0.781250, 0.156250}, {0.843750, 0.156250}, {0.843750, 0.156250}, //
173
-
{0.906250, 0.156250}, {0.906250, 0.156250}, {0.968750, 0.156250}, {0.968750, 0.156250}, //
174
-
{0.031250, 0.218750}, {0.031250, 0.218750}, {0.093750, 0.218750}, {0.093750, 0.218750}, //
175
-
{0.156250, 0.218750}, {0.156250, 0.218750}, {0.218750, 0.218750}, {0.218750, 0.218750}, //
176
-
{0.281250, 0.218750}, {0.281250, 0.218750}, {0.343750, 0.218750}, {0.343750, 0.218750}, //
177
-
{0.406250, 0.218750}, {0.406250, 0.218750}, {0.468750, 0.218750}, {0.468750, 0.218750}, //
178
-
{0.531250, 0.218750}, {0.531250, 0.218750}, {0.593750, 0.218750}, {0.593750, 0.218750}, //
179
-
{0.656250, 0.218750}, {0.656250, 0.218750}, {0.718750, 0.218750}, {0.718750, 0.218750}, //
180
-
{0.781250, 0.218750}, {0.781250, 0.218750}, {0.843750, 0.218750}, {0.843750, 0.218750}, //
181
-
{0.906250, 0.218750}, {0.906250, 0.218750}, {0.968750, 0.218750}, {0.968750, 0.218750}, //
182
-
{0.031250, 0.281250}, {0.031250, 0.281250}, {0.093750, 0.281250}, {0.093750, 0.281250}, //
183
-
{0.156250, 0.281250}, {0.156250, 0.281250}, {0.218750, 0.281250}, {0.218750, 0.281250}, //
184
-
{0.281250, 0.281250}, {0.281250, 0.281250}, {0.343750, 0.281250}, {0.343750, 0.281250}, //
185
-
{0.406250, 0.281250}, {0.406250, 0.281250}, {0.468750, 0.281250}, {0.468750, 0.281250}, //
186
-
{0.531250, 0.281250}, {0.531250, 0.281250}, {0.593750, 0.281250}, {0.593750, 0.281250}, //
187
-
{0.656250, 0.281250}, {0.656250, 0.281250}, {0.718750, 0.281250}, {0.718750, 0.281250}, //
188
-
{0.781250, 0.281250}, {0.781250, 0.281250}, {0.843750, 0.281250}, {0.843750, 0.281250}, //
189
-
{0.906250, 0.281250}, {0.906250, 0.281250}, {0.968750, 0.281250}, {0.968750, 0.281250}, //
190
-
{0.031250, 0.343750}, {0.031250, 0.343750}, {0.093750, 0.343750}, {0.093750, 0.343750}, //
191
-
{0.156250, 0.343750}, {0.156250, 0.343750}, {0.218750, 0.343750}, {0.218750, 0.343750}, //
192
-
{0.281250, 0.343750}, {0.281250, 0.343750}, {0.343750, 0.343750}, {0.343750, 0.343750}, //
193
-
{0.406250, 0.343750}, {0.406250, 0.343750}, {0.468750, 0.343750}, {0.468750, 0.343750}, //
194
-
{0.531250, 0.343750}, {0.531250, 0.343750}, {0.593750, 0.343750}, {0.593750, 0.343750}, //
195
-
{0.656250, 0.343750}, {0.656250, 0.343750}, {0.718750, 0.343750}, {0.718750, 0.343750}, //
196
-
{0.781250, 0.343750}, {0.781250, 0.343750}, {0.843750, 0.343750}, {0.843750, 0.343750}, //
197
-
{0.906250, 0.343750}, {0.906250, 0.343750}, {0.968750, 0.343750}, {0.968750, 0.343750}, //
198
-
{0.031250, 0.406250}, {0.031250, 0.406250}, {0.093750, 0.406250}, {0.093750, 0.406250}, //
199
-
{0.156250, 0.406250}, {0.156250, 0.406250}, {0.218750, 0.406250}, {0.218750, 0.406250}, //
200
-
{0.281250, 0.406250}, {0.281250, 0.406250}, {0.343750, 0.406250}, {0.343750, 0.406250}, //
201
-
{0.406250, 0.406250}, {0.406250, 0.406250}, {0.468750, 0.406250}, {0.468750, 0.406250}, //
202
-
{0.531250, 0.406250}, {0.531250, 0.406250}, {0.593750, 0.406250}, {0.593750, 0.406250}, //
203
-
{0.656250, 0.406250}, {0.656250, 0.406250}, {0.718750, 0.406250}, {0.718750, 0.406250}, //
204
-
{0.781250, 0.406250}, {0.781250, 0.406250}, {0.843750, 0.406250}, {0.843750, 0.406250}, //
205
-
{0.906250, 0.406250}, {0.906250, 0.406250}, {0.968750, 0.406250}, {0.968750, 0.406250}, //
206
-
{0.031250, 0.468750}, {0.031250, 0.468750}, {0.093750, 0.468750}, {0.093750, 0.468750}, //
207
-
{0.156250, 0.468750}, {0.156250, 0.468750}, {0.218750, 0.468750}, {0.218750, 0.468750}, //
208
-
{0.281250, 0.468750}, {0.281250, 0.468750}, {0.343750, 0.468750}, {0.343750, 0.468750}, //
209
-
{0.406250, 0.468750}, {0.406250, 0.468750}, {0.468750, 0.468750}, {0.468750, 0.468750}, //
210
-
{0.531250, 0.468750}, {0.531250, 0.468750}, {0.593750, 0.468750}, {0.593750, 0.468750}, //
211
-
{0.656250, 0.468750}, {0.656250, 0.468750}, {0.718750, 0.468750}, {0.718750, 0.468750}, //
212
-
{0.781250, 0.468750}, {0.781250, 0.468750}, {0.843750, 0.468750}, {0.843750, 0.468750}, //
213
-
{0.906250, 0.468750}, {0.906250, 0.468750}, {0.968750, 0.468750}, {0.968750, 0.468750}, //
214
-
{0.031250, 0.531250}, {0.031250, 0.531250}, {0.093750, 0.531250}, {0.093750, 0.531250}, //
215
-
{0.156250, 0.531250}, {0.156250, 0.531250}, {0.218750, 0.531250}, {0.218750, 0.531250}, //
216
-
{0.281250, 0.531250}, {0.281250, 0.531250}, {0.343750, 0.531250}, {0.343750, 0.531250}, //
217
-
{0.406250, 0.531250}, {0.406250, 0.531250}, {0.468750, 0.531250}, {0.468750, 0.531250}, //
218
-
{0.531250, 0.531250}, {0.531250, 0.531250}, {0.593750, 0.531250}, {0.593750, 0.531250}, //
219
-
{0.656250, 0.531250}, {0.656250, 0.531250}, {0.718750, 0.531250}, {0.718750, 0.531250}, //
220
-
{0.781250, 0.531250}, {0.781250, 0.531250}, {0.843750, 0.531250}, {0.843750, 0.531250}, //
221
-
{0.906250, 0.531250}, {0.906250, 0.531250}, {0.968750, 0.531250}, {0.968750, 0.531250}, //
222
-
{0.031250, 0.593750}, {0.031250, 0.593750}, {0.093750, 0.593750}, {0.093750, 0.593750}, //
223
-
{0.156250, 0.593750}, {0.156250, 0.593750}, {0.218750, 0.593750}, {0.218750, 0.593750}, //
224
-
{0.281250, 0.593750}, {0.281250, 0.593750}, {0.343750, 0.593750}, {0.343750, 0.593750}, //
225
-
{0.406250, 0.593750}, {0.406250, 0.593750}, {0.468750, 0.593750}, {0.468750, 0.593750}, //
226
-
{0.531250, 0.593750}, {0.531250, 0.593750}, {0.593750, 0.593750}, {0.593750, 0.593750}, //
227
-
{0.656250, 0.593750}, {0.656250, 0.593750}, {0.718750, 0.593750}, {0.718750, 0.593750}, //
228
-
{0.781250, 0.593750}, {0.781250, 0.593750}, {0.843750, 0.593750}, {0.843750, 0.593750}, //
229
-
{0.906250, 0.593750}, {0.906250, 0.593750}, {0.968750, 0.593750}, {0.968750, 0.593750}, //
230
-
{0.031250, 0.656250}, {0.031250, 0.656250}, {0.093750, 0.656250}, {0.093750, 0.656250}, //
231
-
{0.156250, 0.656250}, {0.156250, 0.656250}, {0.218750, 0.656250}, {0.218750, 0.656250}, //
232
-
{0.281250, 0.656250}, {0.281250, 0.656250}, {0.343750, 0.656250}, {0.343750, 0.656250}, //
233
-
{0.406250, 0.656250}, {0.406250, 0.656250}, {0.468750, 0.656250}, {0.468750, 0.656250}, //
234
-
{0.531250, 0.656250}, {0.531250, 0.656250}, {0.593750, 0.656250}, {0.593750, 0.656250}, //
235
-
{0.656250, 0.656250}, {0.656250, 0.656250}, {0.718750, 0.656250}, {0.718750, 0.656250}, //
236
-
{0.781250, 0.656250}, {0.781250, 0.656250}, {0.843750, 0.656250}, {0.843750, 0.656250}, //
237
-
{0.906250, 0.656250}, {0.906250, 0.656250}, {0.968750, 0.656250}, {0.968750, 0.656250}, //
238
-
{0.031250, 0.718750}, {0.031250, 0.718750}, {0.093750, 0.718750}, {0.093750, 0.718750}, //
239
-
{0.156250, 0.718750}, {0.156250, 0.718750}, {0.218750, 0.718750}, {0.218750, 0.718750}, //
240
-
{0.281250, 0.718750}, {0.281250, 0.718750}, {0.343750, 0.718750}, {0.343750, 0.718750}, //
241
-
{0.406250, 0.718750}, {0.406250, 0.718750}, {0.468750, 0.718750}, {0.468750, 0.718750}, //
242
-
{0.531250, 0.718750}, {0.531250, 0.718750}, {0.593750, 0.718750}, {0.593750, 0.718750}, //
243
-
{0.656250, 0.718750}, {0.656250, 0.718750}, {0.718750, 0.718750}, {0.718750, 0.718750}, //
244
-
{0.781250, 0.718750}, {0.781250, 0.718750}, {0.843750, 0.718750}, {0.843750, 0.718750}, //
245
-
{0.906250, 0.718750}, {0.906250, 0.718750}, {0.968750, 0.718750}, {0.968750, 0.718750}, //
246
-
{0.031250, 0.781250}, {0.031250, 0.781250}, {0.093750, 0.781250}, {0.093750, 0.781250}, //
247
-
{0.156250, 0.781250}, {0.156250, 0.781250}, {0.218750, 0.781250}, {0.218750, 0.781250}, //
248
-
{0.281250, 0.781250}, {0.281250, 0.781250}, {0.343750, 0.781250}, {0.343750, 0.781250}, //
249
-
{0.406250, 0.781250}, {0.406250, 0.781250}, {0.468750, 0.781250}, {0.468750, 0.781250}, //
250
-
{0.531250, 0.781250}, {0.531250, 0.781250}, {0.593750, 0.781250}, {0.593750, 0.781250}, //
251
-
{0.656250, 0.781250}, {0.656250, 0.781250}, {0.718750, 0.781250}, {0.718750, 0.781250}, //
252
-
{0.781250, 0.781250}, {0.781250, 0.781250}, {0.843750, 0.781250}, {0.843750, 0.781250}, //
253
-
{0.906250, 0.781250}, {0.906250, 0.781250}, {0.968750, 0.781250}, {0.968750, 0.781250}, //
254
-
{0.031250, 0.843750}, {0.031250, 0.843750}, {0.093750, 0.843750}, {0.093750, 0.843750}, //
255
-
{0.156250, 0.843750}, {0.156250, 0.843750}, {0.218750, 0.843750}, {0.218750, 0.843750}, //
256
-
{0.281250, 0.843750}, {0.281250, 0.843750}, {0.343750, 0.843750}, {0.343750, 0.843750}, //
257
-
{0.406250, 0.843750}, {0.406250, 0.843750}, {0.468750, 0.843750}, {0.468750, 0.843750}, //
258
-
{0.531250, 0.843750}, {0.531250, 0.843750}, {0.593750, 0.843750}, {0.593750, 0.843750}, //
259
-
{0.656250, 0.843750}, {0.656250, 0.843750}, {0.718750, 0.843750}, {0.718750, 0.843750}, //
260
-
{0.781250, 0.843750}, {0.781250, 0.843750}, {0.843750, 0.843750}, {0.843750, 0.843750}, //
261
-
{0.906250, 0.843750}, {0.906250, 0.843750}, {0.968750, 0.843750}, {0.968750, 0.843750}, //
262
-
{0.031250, 0.906250}, {0.031250, 0.906250}, {0.093750, 0.906250}, {0.093750, 0.906250}, //
263
-
{0.156250, 0.906250}, {0.156250, 0.906250}, {0.218750, 0.906250}, {0.218750, 0.906250}, //
264
-
{0.281250, 0.906250}, {0.281250, 0.906250}, {0.343750, 0.906250}, {0.343750, 0.906250}, //
265
-
{0.406250, 0.906250}, {0.406250, 0.906250}, {0.468750, 0.906250}, {0.468750, 0.906250}, //
266
-
{0.531250, 0.906250}, {0.531250, 0.906250}, {0.593750, 0.906250}, {0.593750, 0.906250}, //
267
-
{0.656250, 0.906250}, {0.656250, 0.906250}, {0.718750, 0.906250}, {0.718750, 0.906250}, //
268
-
{0.781250, 0.906250}, {0.781250, 0.906250}, {0.843750, 0.906250}, {0.843750, 0.906250}, //
269
-
{0.906250, 0.906250}, {0.906250, 0.906250}, {0.968750, 0.906250}, {0.968750, 0.906250}, //
270
-
{0.031250, 0.968750}, {0.031250, 0.968750}, {0.093750, 0.968750}, {0.093750, 0.968750}, //
271
-
{0.156250, 0.968750}, {0.156250, 0.968750}, {0.218750, 0.968750}, {0.218750, 0.968750}, //
272
-
{0.281250, 0.968750}, {0.281250, 0.968750}, {0.343750, 0.968750}, {0.343750, 0.968750}, //
273
-
{0.406250, 0.968750}, {0.406250, 0.968750}, {0.468750, 0.968750}, {0.468750, 0.968750}, //
274
-
{0.531250, 0.968750}, {0.531250, 0.968750}, {0.593750, 0.968750}, {0.593750, 0.968750}, //
275
-
{0.656250, 0.968750}, {0.656250, 0.968750}, {0.718750, 0.968750}, {0.718750, 0.968750}, //
276
-
{0.781250, 0.968750}, {0.781250, 0.968750}, {0.843750, 0.968750}, {0.843750, 0.968750}, //
277
-
{0.906250, 0.968750}, {0.906250, 0.968750}, {0.968750, 0.968750}, {0.968750, 0.968750}, //
278
-
{0.062500, 0.062500}, {0.062500, 0.062500}, {0.062500, 0.062500}, {0.062500, 0.062500}, //
279
-
{0.062500, 0.062500}, {0.062500, 0.062500}, {0.187500, 0.062500}, {0.187500, 0.062500}, //
280
-
{0.187500, 0.062500}, {0.187500, 0.062500}, {0.187500, 0.062500}, {0.187500, 0.062500}, //
281
-
{0.312500, 0.062500}, {0.312500, 0.062500}, {0.312500, 0.062500}, {0.312500, 0.062500}, //
282
-
{0.312500, 0.062500}, {0.312500, 0.062500}, {0.437500, 0.062500}, {0.437500, 0.062500}, //
283
-
{0.437500, 0.062500}, {0.437500, 0.062500}, {0.437500, 0.062500}, {0.437500, 0.062500}, //
284
-
{0.562500, 0.062500}, {0.562500, 0.062500}, {0.562500, 0.062500}, {0.562500, 0.062500}, //
285
-
{0.562500, 0.062500}, {0.562500, 0.062500}, {0.687500, 0.062500}, {0.687500, 0.062500}, //
286
-
{0.687500, 0.062500}, {0.687500, 0.062500}, {0.687500, 0.062500}, {0.687500, 0.062500}, //
287
-
{0.812500, 0.062500}, {0.812500, 0.062500}, {0.812500, 0.062500}, {0.812500, 0.062500}, //
288
-
{0.812500, 0.062500}, {0.812500, 0.062500}, {0.937500, 0.062500}, {0.937500, 0.062500}, //
289
-
{0.937500, 0.062500}, {0.937500, 0.062500}, {0.937500, 0.062500}, {0.937500, 0.062500}, //
290
-
{0.062500, 0.187500}, {0.062500, 0.187500}, {0.062500, 0.187500}, {0.062500, 0.187500}, //
291
-
{0.062500, 0.187500}, {0.062500, 0.187500}, {0.187500, 0.187500}, {0.187500, 0.187500}, //
292
-
{0.187500, 0.187500}, {0.187500, 0.187500}, {0.187500, 0.187500}, {0.187500, 0.187500}, //
293
-
{0.312500, 0.187500}, {0.312500, 0.187500}, {0.312500, 0.187500}, {0.312500, 0.187500}, //
294
-
{0.312500, 0.187500}, {0.312500, 0.187500}, {0.437500, 0.187500}, {0.437500, 0.187500}, //
295
-
{0.437500, 0.187500}, {0.437500, 0.187500}, {0.437500, 0.187500}, {0.437500, 0.187500}, //
296
-
{0.562500, 0.187500}, {0.562500, 0.187500}, {0.562500, 0.187500}, {0.562500, 0.187500}, //
297
-
{0.562500, 0.187500}, {0.562500, 0.187500}, {0.687500, 0.187500}, {0.687500, 0.187500}, //
298
-
{0.687500, 0.187500}, {0.687500, 0.187500}, {0.687500, 0.187500}, {0.687500, 0.187500}, //
299
-
{0.812500, 0.187500}, {0.812500, 0.187500}, {0.812500, 0.187500}, {0.812500, 0.187500}, //
300
-
{0.812500, 0.187500}, {0.812500, 0.187500}, {0.937500, 0.187500}, {0.937500, 0.187500}, //
301
-
{0.937500, 0.187500}, {0.937500, 0.187500}, {0.937500, 0.187500}, {0.937500, 0.187500}, //
302
-
{0.062500, 0.312500}, {0.062500, 0.312500}, {0.062500, 0.312500}, {0.062500, 0.312500}, //
303
-
{0.062500, 0.312500}, {0.062500, 0.312500}, {0.187500, 0.312500}, {0.187500, 0.312500}, //
304
-
{0.187500, 0.312500}, {0.187500, 0.312500}, {0.187500, 0.312500}, {0.187500, 0.312500}, //
305
-
{0.312500, 0.312500}, {0.312500, 0.312500}, {0.312500, 0.312500}, {0.312500, 0.312500}, //
306
-
{0.312500, 0.312500}, {0.312500, 0.312500}, {0.437500, 0.312500}, {0.437500, 0.312500}, //
307
-
{0.437500, 0.312500}, {0.437500, 0.312500}, {0.437500, 0.312500}, {0.437500, 0.312500}, //
308
-
{0.562500, 0.312500}, {0.562500, 0.312500}, {0.562500, 0.312500}, {0.562500, 0.312500}, //
309
-
{0.562500, 0.312500}, {0.562500, 0.312500}, {0.687500, 0.312500}, {0.687500, 0.312500}, //
310
-
{0.687500, 0.312500}, {0.687500, 0.312500}, {0.687500, 0.312500}, {0.687500, 0.312500}, //
311
-
{0.812500, 0.312500}, {0.812500, 0.312500}, {0.812500, 0.312500}, {0.812500, 0.312500}, //
312
-
{0.812500, 0.312500}, {0.812500, 0.312500}, {0.937500, 0.312500}, {0.937500, 0.312500}, //
313
-
{0.937500, 0.312500}, {0.937500, 0.312500}, {0.937500, 0.312500}, {0.937500, 0.312500}, //
314
-
{0.062500, 0.437500}, {0.062500, 0.437500}, {0.062500, 0.437500}, {0.062500, 0.437500}, //
315
-
{0.062500, 0.437500}, {0.062500, 0.437500}, {0.187500, 0.437500}, {0.187500, 0.437500}, //
316
-
{0.187500, 0.437500}, {0.187500, 0.437500}, {0.187500, 0.437500}, {0.187500, 0.437500}, //
317
-
{0.312500, 0.437500}, {0.312500, 0.437500}, {0.312500, 0.437500}, {0.312500, 0.437500}, //
318
-
{0.312500, 0.437500}, {0.312500, 0.437500}, {0.437500, 0.437500}, {0.437500, 0.437500}, //
319
-
{0.437500, 0.437500}, {0.437500, 0.437500}, {0.437500, 0.437500}, {0.437500, 0.437500}, //
320
-
{0.562500, 0.437500}, {0.562500, 0.437500}, {0.562500, 0.437500}, {0.562500, 0.437500}, //
321
-
{0.562500, 0.437500}, {0.562500, 0.437500}, {0.687500, 0.437500}, {0.687500, 0.437500}, //
322
-
{0.687500, 0.437500}, {0.687500, 0.437500}, {0.687500, 0.437500}, {0.687500, 0.437500}, //
323
-
{0.812500, 0.437500}, {0.812500, 0.437500}, {0.812500, 0.437500}, {0.812500, 0.437500}, //
324
-
{0.812500, 0.437500}, {0.812500, 0.437500}, {0.937500, 0.437500}, {0.937500, 0.437500}, //
325
-
{0.937500, 0.437500}, {0.937500, 0.437500}, {0.937500, 0.437500}, {0.937500, 0.437500}, //
326
-
{0.062500, 0.562500}, {0.062500, 0.562500}, {0.062500, 0.562500}, {0.062500, 0.562500}, //
327
-
{0.062500, 0.562500}, {0.062500, 0.562500}, {0.187500, 0.562500}, {0.187500, 0.562500}, //
328
-
{0.187500, 0.562500}, {0.187500, 0.562500}, {0.187500, 0.562500}, {0.187500, 0.562500}, //
329
-
{0.312500, 0.562500}, {0.312500, 0.562500}, {0.312500, 0.562500}, {0.312500, 0.562500}, //
330
-
{0.312500, 0.562500}, {0.312500, 0.562500}, {0.437500, 0.562500}, {0.437500, 0.562500}, //
331
-
{0.437500, 0.562500}, {0.437500, 0.562500}, {0.437500, 0.562500}, {0.437500, 0.562500}, //
332
-
{0.562500, 0.562500}, {0.562500, 0.562500}, {0.562500, 0.562500}, {0.562500, 0.562500}, //
333
-
{0.562500, 0.562500}, {0.562500, 0.562500}, {0.687500, 0.562500}, {0.687500, 0.562500}, //
334
-
{0.687500, 0.562500}, {0.687500, 0.562500}, {0.687500, 0.562500}, {0.687500, 0.562500}, //
335
-
{0.812500, 0.562500}, {0.812500, 0.562500}, {0.812500, 0.562500}, {0.812500, 0.562500}, //
336
-
{0.812500, 0.562500}, {0.812500, 0.562500}, {0.937500, 0.562500}, {0.937500, 0.562500}, //
337
-
{0.937500, 0.562500}, {0.937500, 0.562500}, {0.937500, 0.562500}, {0.937500, 0.562500}, //
338
-
{0.062500, 0.687500}, {0.062500, 0.687500}, {0.062500, 0.687500}, {0.062500, 0.687500}, //
339
-
{0.062500, 0.687500}, {0.062500, 0.687500}, {0.187500, 0.687500}, {0.187500, 0.687500}, //
340
-
{0.187500, 0.687500}, {0.187500, 0.687500}, {0.187500, 0.687500}, {0.187500, 0.687500}, //
341
-
{0.312500, 0.687500}, {0.312500, 0.687500}, {0.312500, 0.687500}, {0.312500, 0.687500}, //
342
-
{0.312500, 0.687500}, {0.312500, 0.687500}, {0.437500, 0.687500}, {0.437500, 0.687500}, //
343
-
{0.437500, 0.687500}, {0.437500, 0.687500}, {0.437500, 0.687500}, {0.437500, 0.687500}, //
344
-
{0.562500, 0.687500}, {0.562500, 0.687500}, {0.562500, 0.687500}, {0.562500, 0.687500}, //
345
-
{0.562500, 0.687500}, {0.562500, 0.687500}, {0.687500, 0.687500}, {0.687500, 0.687500}, //
346
-
{0.687500, 0.687500}, {0.687500, 0.687500}, {0.687500, 0.687500}, {0.687500, 0.687500}, //
347
-
{0.812500, 0.687500}, {0.812500, 0.687500}, {0.812500, 0.687500}, {0.812500, 0.687500}, //
348
-
{0.812500, 0.687500}, {0.812500, 0.687500}, {0.937500, 0.687500}, {0.937500, 0.687500}, //
349
-
{0.937500, 0.687500}, {0.937500, 0.687500}, {0.937500, 0.687500}, {0.937500, 0.687500}, //
350
-
{0.062500, 0.812500}, {0.062500, 0.812500}, {0.062500, 0.812500}, {0.062500, 0.812500}, //
351
-
{0.062500, 0.812500}, {0.062500, 0.812500}, {0.187500, 0.812500}, {0.187500, 0.812500}, //
352
-
{0.187500, 0.812500}, {0.187500, 0.812500}, {0.187500, 0.812500}, {0.187500, 0.812500}, //
353
-
{0.312500, 0.812500}, {0.312500, 0.812500}, {0.312500, 0.812500}, {0.312500, 0.812500}, //
354
-
{0.312500, 0.812500}, {0.312500, 0.812500}, {0.437500, 0.812500}, {0.437500, 0.812500}, //
355
-
{0.437500, 0.812500}, {0.437500, 0.812500}, {0.437500, 0.812500}, {0.437500, 0.812500}, //
356
-
{0.562500, 0.812500}, {0.562500, 0.812500}, {0.562500, 0.812500}, {0.562500, 0.812500}, //
357
-
{0.562500, 0.812500}, {0.562500, 0.812500}, {0.687500, 0.812500}, {0.687500, 0.812500}, //
358
-
{0.687500, 0.812500}, {0.687500, 0.812500}, {0.687500, 0.812500}, {0.687500, 0.812500}, //
359
-
{0.812500, 0.812500}, {0.812500, 0.812500}, {0.812500, 0.812500}, {0.812500, 0.812500}, //
360
-
{0.812500, 0.812500}, {0.812500, 0.812500}, {0.937500, 0.812500}, {0.937500, 0.812500}, //
361
-
{0.937500, 0.812500}, {0.937500, 0.812500}, {0.937500, 0.812500}, {0.937500, 0.812500}, //
362
-
{0.062500, 0.937500}, {0.062500, 0.937500}, {0.062500, 0.937500}, {0.062500, 0.937500}, //
363
-
{0.062500, 0.937500}, {0.062500, 0.937500}, {0.187500, 0.937500}, {0.187500, 0.937500}, //
364
-
{0.187500, 0.937500}, {0.187500, 0.937500}, {0.187500, 0.937500}, {0.187500, 0.937500}, //
365
-
{0.312500, 0.937500}, {0.312500, 0.937500}, {0.312500, 0.937500}, {0.312500, 0.937500}, //
366
-
{0.312500, 0.937500}, {0.312500, 0.937500}, {0.437500, 0.937500}, {0.437500, 0.937500}, //
367
-
{0.437500, 0.937500}, {0.437500, 0.937500}, {0.437500, 0.937500}, {0.437500, 0.937500}, //
368
-
{0.562500, 0.937500}, {0.562500, 0.937500}, {0.562500, 0.937500}, {0.562500, 0.937500}, //
369
-
{0.562500, 0.937500}, {0.562500, 0.937500}, {0.687500, 0.937500}, {0.687500, 0.937500}, //
370
-
{0.687500, 0.937500}, {0.687500, 0.937500}, {0.687500, 0.937500}, {0.687500, 0.937500}, //
371
-
{0.812500, 0.937500}, {0.812500, 0.937500}, {0.812500, 0.937500}, {0.812500, 0.937500}, //
372
-
{0.812500, 0.937500}, {0.812500, 0.937500}, {0.937500, 0.937500}, {0.937500, 0.937500}, //
373
-
{0.937500, 0.937500}, {0.937500, 0.937500}, {0.937500, 0.937500}, {0.937500, 0.937500}, //
374
-
};
375
-
376
-
static std::vector<Palm7KP>
377
-
runHandDetector(struct ht_view *htv, cv::Mat &raw_input)
378
-
{
379
-
const OrtApi *g_ort = htv->htd->ort_api;
380
-
381
-
cv::Mat img;
382
-
383
-
constexpr int hd_size = 128;
384
-
constexpr size_t nb_planes = 3;
385
-
constexpr size_t size = hd_size * hd_size * nb_planes;
386
-
constexpr int inputTensorSize = size * sizeof(float);
387
-
388
-
cv::Matx23f back_from_blackbar = blackbar(raw_input, img, {hd_size, hd_size});
389
-
390
-
float scale_factor = back_from_blackbar(0, 0); // 960/128
391
-
assert(img.isContinuous());
392
-
constexpr float mean = 128.0f;
393
-
constexpr float std = 128.0f;
394
-
395
-
std::vector<float> real_thing(size);
396
-
397
-
if (htv->htd->startup_config.palm_detection_use_mediapipe) {
398
-
std::vector<uint8_t> combined_planes(size);
399
-
planarize(img, combined_planes.data());
400
-
for (size_t i = 0; i < size; i++) {
401
-
float val = (float)combined_planes[i];
402
-
real_thing[i] = (val - mean) / std;
403
-
}
404
-
} else {
405
-
406
-
assert(img.isContinuous());
407
-
408
-
for (size_t i = 0; i < size; i++) {
409
-
int val = img.data[i];
410
-
411
-
real_thing[i] = (val - mean) / std;
412
-
}
413
-
}
414
-
415
-
// Convenience
416
-
struct ModelInfo *model_hd = &htv->detection_model;
417
-
418
-
// If this is non-NULL, ONNX will explode and won't tell you why!
419
-
OrtValue *input_tensor = nullptr;
420
-
421
-
ORT_CHECK(g_ort, g_ort->CreateTensorWithDataAsOrtValue(
422
-
model_hd->memoryInfo, real_thing.data(), inputTensorSize, model_hd->input_shape.data(),
423
-
model_hd->input_shape.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &input_tensor));
424
-
425
-
// Cargo-culted
426
-
assert(input_tensor != nullptr);
427
-
int is_tensor;
428
-
ORT_CHECK(g_ort, g_ort->IsTensor(input_tensor, &is_tensor));
429
-
assert(is_tensor);
430
-
431
-
// If any of these are non-NULL, ONNX will explode and won't tell you why! Be extremely paranoid!
432
-
OrtValue *output_tensor[2] = {nullptr, nullptr};
433
-
ORT_CHECK(g_ort, g_ort->Run(model_hd->session, nullptr, model_hd->input_names.data(),
434
-
(const OrtValue *const *)&input_tensor, model_hd->input_names.size(),
435
-
model_hd->output_names.data(), model_hd->output_names.size(), output_tensor));
436
-
437
-
ORT_CHECK(g_ort, g_ort->IsTensor(output_tensor[0], &is_tensor));
438
-
assert(is_tensor);
439
-
440
-
float *classificators = nullptr;
441
-
float *regressors = nullptr;
442
-
443
-
ORT_CHECK(g_ort,
444
-
g_ort->GetTensorMutableData(output_tensor[0], (void **)&classificators)); // Totally won't segfault!
445
-
ORT_CHECK(g_ort, g_ort->GetTensorMutableData(output_tensor[1], (void **)®ressors));
446
-
// We need to free these!
447
-
448
-
float *rg = regressors; // shorter name
449
-
450
-
std::vector<NMSPalm> detections;
451
-
int count = 0;
452
-
453
-
std::vector<Palm7KP> output;
454
-
std::vector<NMSPalm> nms_palms;
455
-
456
-
OrtTensorTypeAndShapeInfo *classificators_info;
457
-
ORT_CHECK(g_ort, g_ort->GetTensorTypeAndShape(output_tensor[0], &classificators_info));
458
-
459
-
size_t classificators_size;
460
-
ORT_CHECK(g_ort, g_ort->GetTensorShapeElementCount(classificators_info, &classificators_size));
461
-
assert(classificators_size == 896);
462
-
463
-
OrtTensorTypeAndShapeInfo *regressors_info;
464
-
ORT_CHECK(g_ort, g_ort->GetTensorTypeAndShape(output_tensor[1], ®ressors_info));
465
-
466
-
size_t regressors_size;
467
-
ORT_CHECK(g_ort, g_ort->GetTensorShapeElementCount(regressors_info, ®ressors_size));
468
-
assert(regressors_size == 896 * 18);
469
-
470
-
g_ort->ReleaseTensorTypeAndShapeInfo(classificators_info);
471
-
g_ort->ReleaseTensorTypeAndShapeInfo(regressors_info);
472
-
473
-
for (size_t i = 0; i < classificators_size; ++i) {
474
-
const float score = 1.0 / (1.0 + exp(-classificators[i]));
475
-
476
-
// Let a lot of detections in - they'll be slowly rejected later
477
-
if (score <= htv->htd->dynamic_config.nms_threshold.val) {
478
-
continue;
479
-
}
480
-
481
-
const struct anchor *anchor = &anchors[i];
482
-
483
-
// Boundary box.
484
-
NMSPalm det;
485
-
486
-
float anchx = anchor->x * 128;
487
-
float anchy = anchor->y * 128;
488
-
489
-
float shiftx = regressors[i * 18];
490
-
float shifty = regressors[i * 18 + 1];
491
-
492
-
float w = regressors[i * 18 + 2];
493
-
float h = regressors[i * 18 + 3];
494
-
495
-
float cx = shiftx + anchx;
496
-
float cy = shifty + anchy;
497
-
498
-
struct xrt_vec2 *kps = det.keypoints;
499
-
500
-
kps[0] = {rg[i * 18 + 4], rg[i * 18 + 5]};
501
-
kps[1] = {rg[i * 18 + 6], rg[i * 18 + 7]};
502
-
kps[2] = {rg[i * 18 + 8], rg[i * 18 + 9]};
503
-
kps[3] = {rg[i * 18 + 10], rg[i * 18 + 11]};
504
-
kps[4] = {rg[i * 18 + 12], rg[i * 18 + 13]};
505
-
kps[5] = {rg[i * 18 + 14], rg[i * 18 + 15]};
506
-
kps[6] = {rg[i * 18 + 16], rg[i * 18 + 17]};
507
-
508
-
509
-
for (int i = 0; i < 7; i++) {
510
-
struct xrt_vec2 *b = &kps[i];
511
-
b->x += anchx;
512
-
b->y += anchy;
513
-
}
514
-
515
-
det.bbox.w = w;
516
-
det.bbox.h = h;
517
-
det.bbox.cx = cx;
518
-
det.bbox.cy = cy;
519
-
det.confidence = score;
520
-
detections.push_back(det);
521
-
count++;
522
-
523
-
if (htv->htd->debug_scribble && (htv->htd->dynamic_config.scribble_raw_detections)) {
524
-
xrt_vec2 center = transformVecBy2x3(xrt_vec2{cx, cy}, back_from_blackbar);
525
-
526
-
float sz = det.bbox.w * scale_factor;
527
-
528
-
cv::rectangle(htv->debug_out_to_this,
529
-
{(int)(center.x - (sz / 2)), (int)(center.y - (sz / 2)), (int)sz, (int)sz},
530
-
hsv2rgb(0.0f, math_map_ranges(det.confidence, 0.0f, 1.0f, 1.5f, -0.1f),
531
-
math_map_ranges(det.confidence, 0.0f, 1.0f, 0.2f, 1.4f)),
532
-
1);
533
-
534
-
for (int i = 0; i < 7; i++) {
535
-
handDot(htv->debug_out_to_this, transformVecBy2x3(kps[i], back_from_blackbar),
536
-
det.confidence * 7, ((float)i) * (360.0f / 7.0f), det.confidence, 1);
537
-
}
538
-
}
539
-
540
-
541
-
542
-
int square = fmax(w, h);
543
-
544
-
square = square / scale_factor;
545
-
}
546
-
547
-
if (count == 0) {
548
-
goto cleanup;
549
-
}
550
-
551
-
nms_palms = filterBoxesWeightedAvg(detections, htv->htd->dynamic_config.nms_iou.val);
552
-
553
-
554
-
555
-
for (NMSPalm cooler : nms_palms) {
556
-
557
-
// Display box
558
-
559
-
struct xrt_vec2 tl = {cooler.bbox.cx - cooler.bbox.w / 2, cooler.bbox.cy - cooler.bbox.h / 2};
560
-
struct xrt_vec2 bob = transformVecBy2x3(tl, back_from_blackbar);
561
-
float sz = cooler.bbox.w * scale_factor;
562
-
563
-
if (htv->htd->debug_scribble && htv->htd->dynamic_config.scribble_nms_detections) {
564
-
cv::rectangle(htv->debug_out_to_this, {(int)bob.x, (int)bob.y, (int)sz, (int)sz},
565
-
hsv2rgb(180.0f, math_map_ranges(cooler.confidence, 0.0f, 1.0f, 0.8f, -0.1f),
566
-
math_map_ranges(cooler.confidence, 0.0f, 1.0f, 0.2f, 1.4f)),
567
-
2);
568
-
for (int i = 0; i < 7; i++) {
569
-
handDot(htv->debug_out_to_this,
570
-
transformVecBy2x3(cooler.keypoints[i], back_from_blackbar),
571
-
cooler.confidence * 14, ((float)i) * (360.0f / 7.0f), cooler.confidence, 3);
572
-
}
573
-
}
574
-
575
-
576
-
Palm7KP this_element;
577
-
578
-
for (int i = 0; i < 7; i++) {
579
-
struct xrt_vec2 b = cooler.keypoints[i];
580
-
this_element.kps[i] = transformVecBy2x3(b, back_from_blackbar);
581
-
}
582
-
this_element.confidence = cooler.confidence;
583
-
584
-
output.push_back(this_element);
585
-
}
586
-
587
-
cleanup:
588
-
g_ort->ReleaseValue(output_tensor[0]);
589
-
g_ort->ReleaseValue(output_tensor[1]);
590
-
g_ort->ReleaseValue(input_tensor);
591
-
return output;
592
-
}
593
-
594
-
595
-
static void
596
-
addSlug(struct ht_device *htd, const char *suffix, char *out)
597
-
{
598
-
strcpy(out, htd->startup_config.model_slug);
599
-
strcat(out, suffix);
600
-
}
601
-
602
-
static void
603
-
initKeypointEstimator(struct ht_device *htd, ht_view *htv)
604
-
{
605
-
struct ModelInfo *model_ke = &htv->keypoint_model;
606
-
const OrtApi *g_ort = htd->ort_api;
607
-
OrtSessionOptions *opts = nullptr;
608
-
609
-
ORT_CHECK(g_ort, g_ort->CreateSessionOptions(&opts));
610
-
611
-
ORT_CHECK(g_ort, g_ort->SetSessionGraphOptimizationLevel(opts, ORT_ENABLE_ALL));
612
-
ORT_CHECK(g_ort, g_ort->SetIntraOpNumThreads(opts, 1));
613
-
614
-
char modelLocation[1024];
615
-
if (htd->startup_config.keypoint_estimation_use_mediapipe) {
616
-
addSlug(htd, "hand_landmark_MEDIAPIPE.onnx", modelLocation);
617
-
} else {
618
-
addSlug(htd, "hand_landmark_COLLABORA.onnx", modelLocation);
619
-
}
620
-
ORT_CHECK(g_ort, g_ort->CreateSession(htd->ort_env, modelLocation, opts, &model_ke->session));
621
-
g_ort->ReleaseSessionOptions(opts);
622
-
623
-
model_ke->input_shape.push_back(1);
624
-
model_ke->input_shape.push_back(3);
625
-
model_ke->input_shape.push_back(224);
626
-
model_ke->input_shape.push_back(224);
627
-
628
-
model_ke->input_names.push_back("input_1");
629
-
630
-
model_ke->output_names.push_back("Identity");
631
-
model_ke->output_names.push_back("Identity_1");
632
-
model_ke->output_names.push_back("Identity_2");
633
-
634
-
model_ke->input_size_bytes = 224 * 224 * 3 * sizeof(float);
635
-
636
-
ORT_CHECK(g_ort, g_ort->CreateCpuMemoryInfo(OrtArenaAllocator, OrtMemTypeDefault, &model_ke->memoryInfo));
637
-
638
-
htv->run_keypoint_model = runKeypointEstimator;
639
-
}
640
-
641
-
static void
642
-
initHandDetector(struct ht_device *htd, ht_view *htv)
643
-
{
644
-
struct ModelInfo *model_hd = &htv->detection_model;
645
-
memset(model_hd, 0, sizeof(struct ModelInfo));
646
-
647
-
const OrtApi *g_ort = htd->ort_api;
648
-
OrtSessionOptions *opts = nullptr;
649
-
ORT_CHECK(g_ort, g_ort->CreateSessionOptions(&opts));
650
-
651
-
ORT_CHECK(g_ort, g_ort->SetSessionGraphOptimizationLevel(opts, ORT_ENABLE_ALL));
652
-
ORT_CHECK(g_ort, g_ort->SetIntraOpNumThreads(opts, 1));
653
-
654
-
char modelLocation[1024];
655
-
656
-
// Hard-coded. Even though you can use the ONNX runtime's API to dynamically figure these out, that doesn't make
657
-
// any sense because these don't change between runs, and if you are swapping models you have to do much more
658
-
// than just change the input/output names.
659
-
if (htd->startup_config.palm_detection_use_mediapipe) {
660
-
addSlug(htd, "palm_detection_MEDIAPIPE.onnx", modelLocation);
661
-
model_hd->input_shape.push_back(1);
662
-
model_hd->input_shape.push_back(3);
663
-
model_hd->input_shape.push_back(128);
664
-
model_hd->input_shape.push_back(128);
665
-
666
-
model_hd->input_names.push_back("input");
667
-
} else {
668
-
addSlug(htd, "palm_detection_COLLABORA.onnx", modelLocation);
669
-
670
-
model_hd->input_shape.push_back(1);
671
-
model_hd->input_shape.push_back(128);
672
-
model_hd->input_shape.push_back(128);
673
-
model_hd->input_shape.push_back(3);
674
-
675
-
model_hd->input_names.push_back("input:0");
676
-
}
677
-
678
-
ORT_CHECK(g_ort, g_ort->CreateSession(htd->ort_env, modelLocation, opts, &model_hd->session));
679
-
g_ort->ReleaseSessionOptions(opts);
680
-
681
-
682
-
683
-
model_hd->output_names.push_back("classificators");
684
-
model_hd->output_names.push_back("regressors");
685
-
686
-
ORT_CHECK(g_ort, g_ort->CreateCpuMemoryInfo(OrtArenaAllocator, OrtMemTypeDefault, &model_hd->memoryInfo));
687
-
688
-
htv->run_detection_model = runHandDetector;
689
-
}
690
-
691
-
692
-
static void
693
-
initOnnx(struct ht_device *htd)
694
-
{
695
-
htd->ort_api = OrtGetApiBase()->GetApi(ORT_API_VERSION);
696
-
ORT_CHECK(htd->ort_api, htd->ort_api->CreateEnv(ORT_LOGGING_LEVEL_WARNING, "moses", &htd->ort_env));
697
-
698
-
initHandDetector(htd, &htd->views[0]);
699
-
initHandDetector(htd, &htd->views[1]);
700
-
701
-
initKeypointEstimator(htd, &htd->views[0]);
702
-
initKeypointEstimator(htd, &htd->views[1]);
703
-
}
704
-
705
-
static void
706
-
destroyModelInfo(struct ht_device *htd, ModelInfo *info)
707
-
{
708
-
const OrtApi *g_ort = htd->ort_api;
709
-
g_ort->ReleaseSession(info->session);
710
-
g_ort->ReleaseMemoryInfo(info->memoryInfo);
711
-
// Same deal as in ht_device - I'm mixing C and C++ idioms, so sometimes it's easier to just manually call their
712
-
// destructors instead of figuring out some way to convince C++ to call them implicitly.
713
-
info->output_names.~vector();
714
-
info->input_names.~vector();
715
-
info->input_shape.~vector();
716
-
}
15
+
void
16
+
initOnnx(struct ht_device *htd);
717
17
718
18
void
719
-
destroyOnnx(struct ht_device *htd)
720
-
{
721
-
destroyModelInfo(htd, &htd->views[0].keypoint_model);
722
-
destroyModelInfo(htd, &htd->views[1].keypoint_model);
723
-
destroyModelInfo(htd, &htd->views[0].detection_model);
724
-
destroyModelInfo(htd, &htd->views[1].detection_model);
725
-
htd->ort_api->ReleaseEnv(htd->ort_env);
726
-
}
19
+
destroyOnnx(struct ht_device *htd);