The open source OpenXR runtime
d/ht: split ht_nms into source and header
authored by
Simon Zeni
and committed by
Moses Turner
7618a7e3
83630efc
+146
-136
+1
-1
src/xrt/drivers/CMakeLists.txt
+1
-1
src/xrt/drivers/CMakeLists.txt
···
218
218
ht/ht_models.hpp
219
219
ht/ht_hand_math.cpp
220
220
ht/ht_image_math.cpp
221
-
ht/ht_nms.hpp
221
+
ht/ht_nms.cpp
222
222
ht/templates/NaivePermutationSort.hpp)
223
223
target_link_libraries(drv_ht PRIVATE xrt-interfaces aux_os aux_util aux_math aux_gstreamer ONNXRuntime::ONNXRuntime ${OpenCV_LIBRARIES})
224
224
target_include_directories(drv_ht PRIVATE ${OpenCV_INCLUDE_DIRS} ${EIGEN3_INCLUDE_DIR})
+141
src/xrt/drivers/ht/ht_nms.cpp
+141
src/xrt/drivers/ht/ht_nms.cpp
···
1
+
// Copyright 2021, Collabora, Ltd.
2
+
// SPDX-License-Identifier: BSL-1.0
3
+
/*!
4
+
* @file
5
+
* @brief Code to deal with bounding boxes for camera-based hand-tracking.
6
+
* @author Moses Turner <moses@collabora.com>
7
+
* @author Marcus Edel <marcus.edel@collabora.com>
8
+
* @ingroup drv_ht
9
+
*/
10
+
11
+
#include "ht_nms.hpp"
12
+
13
+
#include <math.h>
14
+
15
+
static float
16
+
overlap(float x1, float w1, float x2, float w2)
17
+
{
18
+
float l1 = x1 - w1 / 2;
19
+
float l2 = x2 - w2 / 2;
20
+
float left = l1 > l2 ? l1 : l2;
21
+
22
+
float r1 = x1 + w1 / 2;
23
+
float r2 = x2 + w2 / 2;
24
+
float right = r1 < r2 ? r1 : r2;
25
+
26
+
return right - left;
27
+
}
28
+
29
+
static float
30
+
boxIntersection(const Box &a, const Box &b)
31
+
{
32
+
float w = overlap(a.cx, a.w, b.cx, b.w);
33
+
float h = overlap(a.cy, a.h, b.cy, b.h);
34
+
35
+
if (w < 0 || h < 0)
36
+
return 0;
37
+
38
+
return w * h;
39
+
}
40
+
41
+
static float
42
+
boxUnion(const Box &a, const Box &b)
43
+
{
44
+
return a.w * a.h + b.w * b.h - boxIntersection(a, b);
45
+
}
46
+
47
+
static float
48
+
boxIOU(const Box &a, const Box &b)
49
+
{
50
+
return boxIntersection(a, b) / boxUnion(a, b);
51
+
}
52
+
53
+
static NMSPalm
54
+
weightedAvgBoxes(const std::vector<NMSPalm> &detections)
55
+
{
56
+
float weight = 0.0f; // or, sum_confidences.
57
+
float cx = 0.0f;
58
+
float cy = 0.0f;
59
+
float size = 0.0f;
60
+
NMSPalm out = {};
61
+
62
+
for (const NMSPalm &detection : detections) {
63
+
weight += detection.confidence;
64
+
cx += detection.bbox.cx * detection.confidence;
65
+
cy += detection.bbox.cy * detection.confidence;
66
+
size += detection.bbox.w * .5 * detection.confidence;
67
+
size += detection.bbox.h * .5 * detection.confidence;
68
+
69
+
for (int i = 0; i < 7; i++) {
70
+
out.keypoints[i].x += detection.keypoints[i].x * detection.confidence;
71
+
out.keypoints[i].y += detection.keypoints[i].y * detection.confidence;
72
+
}
73
+
}
74
+
cx /= weight;
75
+
cy /= weight;
76
+
size /= weight;
77
+
for (int i = 0; i < 7; i++) {
78
+
out.keypoints[i].x /= weight;
79
+
out.keypoints[i].y /= weight;
80
+
}
81
+
82
+
83
+
float bare_confidence = weight / detections.size();
84
+
85
+
// desmos \frac{1}{1+e^{-.5x}}-.5
86
+
87
+
float steep = 0.2;
88
+
float cent = 0.5;
89
+
90
+
float exp = detections.size();
91
+
92
+
float sigmoid_addendum = (1.0f / (1.0f + pow(M_E, (-steep * exp)))) - cent;
93
+
94
+
float diff_bare_to_one = 1.0f - bare_confidence;
95
+
96
+
out.confidence = bare_confidence + (sigmoid_addendum * diff_bare_to_one);
97
+
98
+
// U_LOG_E("Bare %f num %f sig %f diff %f out %f", bare_confidence, exp, sigmoid_addendum, diff_bare_to_one,
99
+
// out.confidence);
100
+
101
+
out.bbox.cx = cx;
102
+
out.bbox.cy = cy;
103
+
out.bbox.w = size;
104
+
out.bbox.h = size;
105
+
return out;
106
+
}
107
+
108
+
std::vector<NMSPalm>
109
+
filterBoxesWeightedAvg(const std::vector<NMSPalm> &detections, float min_iou)
110
+
{
111
+
std::vector<std::vector<NMSPalm>> overlaps;
112
+
std::vector<NMSPalm> outs;
113
+
114
+
// U_LOG_D("\n\nStarting filtering boxes. There are %zu boxes to look at.\n", detections.size());
115
+
for (const NMSPalm &detection : detections) {
116
+
// U_LOG_D("Starting looking at one detection\n");
117
+
bool foundAHome = false;
118
+
for (size_t i = 0; i < outs.size(); i++) {
119
+
float iou = boxIOU(outs[i].bbox, detection.bbox);
120
+
// U_LOG_D("IOU is %f\n", iou);
121
+
// U_LOG_D("Outs box is %f %f %f %f", outs[i].bbox.cx, outs[i].bbox.cy, outs[i].bbox.w,
122
+
// outs[i].bbox.h)
123
+
if (iou > min_iou) {
124
+
// This one intersects with the whole thing
125
+
overlaps[i].push_back(detection);
126
+
outs[i] = weightedAvgBoxes(overlaps[i]);
127
+
foundAHome = true;
128
+
break;
129
+
}
130
+
}
131
+
if (!foundAHome) {
132
+
// U_LOG_D("No home\n");
133
+
overlaps.push_back({detection});
134
+
outs.push_back({detection});
135
+
} else {
136
+
// U_LOG_D("Found a home!\n");
137
+
}
138
+
}
139
+
// U_LOG_D("Sizeeeeeeeeeeeeeeeeeeeee is %zu\n", outs.size());
140
+
return outs;
141
+
}
+3
-134
src/xrt/drivers/ht/ht_nms.hpp
+3
-134
src/xrt/drivers/ht/ht_nms.hpp
···
11
11
#pragma once
12
12
13
13
#include "xrt/xrt_defines.h"
14
-
#include "ht_driver.hpp"
15
-
16
-
#include <math.h>
17
-
#include <stdio.h>
18
14
19
15
#include <vector>
20
-
21
16
22
17
struct Box
23
18
{
···
30
25
struct NMSPalm
31
26
{
32
27
Box bbox;
33
-
xrt_vec2 keypoints[7];
28
+
struct xrt_vec2 keypoints[7];
34
29
float confidence;
35
30
};
36
31
37
-
static float
38
-
overlap(float x1, float w1, float x2, float w2)
39
-
{
40
-
float l1 = x1 - w1 / 2;
41
-
float l2 = x2 - w2 / 2;
42
-
float left = l1 > l2 ? l1 : l2;
43
-
44
-
float r1 = x1 + w1 / 2;
45
-
float r2 = x2 + w2 / 2;
46
-
float right = r1 < r2 ? r1 : r2;
47
-
48
-
return right - left;
49
-
}
50
-
51
-
static float
52
-
boxIntersection(const Box &a, const Box &b)
53
-
{
54
-
float w = overlap(a.cx, a.w, b.cx, b.w);
55
-
float h = overlap(a.cy, a.h, b.cy, b.h);
56
-
57
-
if (w < 0 || h < 0)
58
-
return 0;
59
-
60
-
return w * h;
61
-
}
62
-
63
-
static float
64
-
boxUnion(const Box &a, const Box &b)
65
-
{
66
-
return a.w * a.h + b.w * b.h - boxIntersection(a, b);
67
-
}
68
-
69
-
static float
70
-
boxIOU(const Box &a, const Box &b)
71
-
{
72
-
return boxIntersection(a, b) / boxUnion(a, b);
73
-
}
74
-
75
-
static NMSPalm
76
-
weightedAvgBoxes(std::vector<NMSPalm> &detections)
77
-
{
78
-
float weight = 0.0f; // or, sum_confidences.
79
-
float cx = 0.0f;
80
-
float cy = 0.0f;
81
-
float size = 0.0f;
82
-
NMSPalm out = {};
83
-
84
-
for (NMSPalm &detection : detections) {
85
-
weight += detection.confidence;
86
-
cx += detection.bbox.cx * detection.confidence;
87
-
cy += detection.bbox.cy * detection.confidence;
88
-
size += detection.bbox.w * .5 * detection.confidence;
89
-
size += detection.bbox.h * .5 * detection.confidence;
90
-
91
-
for (int i = 0; i < 7; i++) {
92
-
out.keypoints[i].x += detection.keypoints[i].x * detection.confidence;
93
-
out.keypoints[i].y += detection.keypoints[i].y * detection.confidence;
94
-
}
95
-
}
96
-
cx /= weight;
97
-
cy /= weight;
98
-
size /= weight;
99
-
for (int i = 0; i < 7; i++) {
100
-
out.keypoints[i].x /= weight;
101
-
out.keypoints[i].y /= weight;
102
-
}
103
-
104
-
105
-
float bare_confidence = weight / detections.size();
106
-
107
-
// desmos \frac{1}{1+e^{-.5x}}-.5
108
-
109
-
float steep = 0.2;
110
-
float cent = 0.5;
111
-
112
-
float exp = detections.size();
113
-
114
-
float sigmoid_addendum = (1.0f / (1.0f + pow(M_E, (-steep * exp)))) - cent;
115
-
116
-
float diff_bare_to_one = 1.0f - bare_confidence;
117
-
118
-
out.confidence = bare_confidence + (sigmoid_addendum * diff_bare_to_one);
119
-
120
-
// U_LOG_E("Bare %f num %f sig %f diff %f out %f", bare_confidence, exp, sigmoid_addendum, diff_bare_to_one,
121
-
// out.confidence);
122
-
123
-
out.bbox.cx = cx;
124
-
out.bbox.cy = cy;
125
-
out.bbox.w = size;
126
-
out.bbox.h = size;
127
-
return out;
128
-
}
129
-
130
-
static std::vector<NMSPalm>
131
-
filterBoxesWeightedAvg(std::vector<NMSPalm> &detections, float min_iou = 0.1f)
132
-
{
133
-
std::vector<std::vector<NMSPalm>> overlaps;
134
-
std::vector<NMSPalm> outs;
135
-
136
-
137
-
// U_LOG_D("\n\nStarting filtering boxes. There are %zu boxes to look at.\n", detections.size());
138
-
for (NMSPalm &detection : detections) {
139
-
// U_LOG_D("Starting looking at one detection\n");
140
-
bool foundAHome = false;
141
-
for (size_t i = 0; i < outs.size(); i++) {
142
-
float iou = boxIOU(outs[i].bbox, detection.bbox);
143
-
// U_LOG_D("IOU is %f\n", iou);
144
-
// U_LOG_D("Outs box is %f %f %f %f", outs[i].bbox.cx, outs[i].bbox.cy, outs[i].bbox.w,
145
-
// outs[i].bbox.h)
146
-
if (iou > min_iou) {
147
-
// This one intersects with the whole thing
148
-
overlaps[i].push_back(detection);
149
-
outs[i] = weightedAvgBoxes(overlaps[i]);
150
-
foundAHome = true;
151
-
break;
152
-
}
153
-
}
154
-
if (!foundAHome) {
155
-
// U_LOG_D("No home\n");
156
-
overlaps.push_back({detection});
157
-
outs.push_back({detection});
158
-
} else {
159
-
// U_LOG_D("Found a home!\n");
160
-
}
161
-
}
162
-
// U_LOG_D("Sizeeeeeeeeeeeeeeeeeeeee is %zu\n", outs.size());
163
-
return outs;
164
-
}
32
+
std::vector<NMSPalm>
33
+
filterBoxesWeightedAvg(const std::vector<NMSPalm> &detections, float min_iou = 0.1f);