The open source OpenXR runtime
u/aeg: Add utility for automatic exposure and gain computation
authored by
Mateo de Mayo
and committed by
Jakob Bornecrantz
77139b02
7ea7cce9
+367
+2
src/xrt/auxiliary/CMakeLists.txt
+2
src/xrt/auxiliary/CMakeLists.txt
+303
src/xrt/auxiliary/util/u_autoexpgain.c
+303
src/xrt/auxiliary/util/u_autoexpgain.c
···
1
+
// Copyright 2022, Collabora, Ltd.
2
+
// SPDX-License-Identifier: BSL-1.0
3
+
/*!
4
+
* @file
5
+
* @brief Automatically compute exposure and gain values from an image stream
6
+
* @author Mateo de Mayo <mateo.demayo@collabora.com>
7
+
* @ingroup aux_util
8
+
*/
9
+
10
+
#include "util/u_autoexpgain.h"
11
+
#include "util/u_format.h"
12
+
#include "util/u_misc.h"
13
+
#include "util/u_var.h"
14
+
15
+
#include <assert.h>
16
+
#include <math.h>
17
+
#include <stdint.h>
18
+
19
+
#define MIN(a, b) ((a) < (b) ? (a) : (b))
20
+
#define MAX(a, b) ((a) > (b) ? (a) : (b))
21
+
#define CLAMP(X, A, B) (MIN(MAX((X), (A)), (B)))
22
+
23
+
#define LEVELS 256 //!< Possible pixel intensity values, only 8-bit supported
24
+
#define INITIAL_BRIGHTNESS 0.5
25
+
#define INITIAL_MAX_BRIGHTNES_STEP 0.05 //!< 0.1 is faster but introduces oscilations more often
26
+
#define INITIAL_THRESHOLD 0.1
27
+
#define GRID_COLS 40 //!< Amount of columns for the histogram sample grid
28
+
29
+
//! Auto exposure and gain (AEG) adjustment algorithm state.
30
+
struct u_autoexpgain
31
+
{
32
+
bool enable; //!< Whether to enable auto exposure and gain adjustment
33
+
34
+
//! Algorithm strategy that affects how score and brightness are computed
35
+
enum u_aeg_strategy strategy;
36
+
struct u_var_combo strategy_combo; //!< UI combo box for selecting `strategy`
37
+
38
+
float histogram[LEVELS]; //!< Pixel intensity histogram
39
+
struct u_var_histogram_f32 histogram_ui; //!< UI for `histogram`
40
+
41
+
//! This is a made up scalar that lives in the [0, 1] range. 0 maps to minimum
42
+
//! exp/gain values while 1 to their maximums. An autoexposure strategy limits
43
+
//! itself to modify this value. The mapping between the scalar and the
44
+
//! respective exp/gain values is provided by `brightness_to_expgain`.
45
+
struct u_var_draggable_f32 brightness;
46
+
float last_brightness; //!< Triggers a exp/gain update when it differs
47
+
float max_brightness_step; //!< Max `brightness` step for each update
48
+
49
+
//! The AEG score lives in the [-1, +1] range and represents how dark or
50
+
//! bright this image is. Values close to zero (by `threshold`) represent
51
+
//! images with a good enough `brightness` value.
52
+
float current_score;
53
+
54
+
//! Scores further than `threshold` from zero will trigger a `brightness` update.
55
+
float threshold;
56
+
57
+
uint32_t frame_counter; //!< Number of frames received
58
+
59
+
//! Every how many frames should we update `brightness`. Some cameras take a
60
+
//! couple of frames until the new exposure/gain sets in and a new score can
61
+
//! be recomputed properly.
62
+
uint8_t update_every;
63
+
64
+
float exposure; //!< Currently computed exposure value to use
65
+
float gain; //!< Currently computed gain value to use
66
+
};
67
+
68
+
//! Maps a `brightness` in [0, 1] to a pair of exposure and gain values based on
69
+
//! a piecewise function.
70
+
static void
71
+
brightness_to_expgain(struct u_autoexpgain *aeg, float brightness, float *out_exposure, float *out_gain)
72
+
{
73
+
74
+
//! These are steps for constructing a piecewise linear function that maps
75
+
//! brightness into (exposure, gain) pairs.
76
+
struct step
77
+
{
78
+
float b; //!< Brightness
79
+
float e; //!< Exposure
80
+
float g; //!< Gain
81
+
};
82
+
83
+
// These tables were tuned over WMR cameras such that increasing
84
+
// brightness increases the histogram range more or less linearly.
85
+
struct step steps_t[] = {{0, 120, 16}, {0.15, 4500, 16}, {0.5, 4500, 127},
86
+
{0.55, 6000, 127}, {0.9, 6000, 255}, {1, 9000, 255}};
87
+
struct step steps_dr[] = {{0, 120, 16}, {0.3, 9000, 16}, {1.0, 9000, 255}};
88
+
89
+
// Select the steps table to use based on our strategy/objective
90
+
struct step *steps = NULL;
91
+
int steps_count = 0;
92
+
if (aeg->strategy == U_AEG_STRATEGY_TRACKING) {
93
+
steps = steps_t;
94
+
steps_count = sizeof(steps_t) / sizeof(struct step);
95
+
} else if (aeg->strategy == U_AEG_STRATEGY_DYNAMIC_RANGE) {
96
+
steps = steps_dr;
97
+
steps_count = sizeof(steps_dr) / sizeof(struct step);
98
+
} else {
99
+
assert(false && "unexpected branch taken");
100
+
}
101
+
102
+
// Other simpler tables that might work for WMR are:
103
+
// {{0, 120, 16}, {0.2, 6000, 16}, {1.0, 6000, 255}};
104
+
// {{0, 120, 16}, {0.2, 6000, 16}, {0.9, 6000, 255}, {1.0, 9000, 255}};
105
+
106
+
// Assertions
107
+
assert(steps_count >= 2 && "Exoected at least two steps");
108
+
assert(steps[0].b == 0 && "First step should be at b=0");
109
+
assert(steps[steps_count - 1].b == 1 && "Last step should be at b=1");
110
+
assert(brightness >= 0 && brightness <= 1);
111
+
112
+
// Compute the piecewise function result from `steps`
113
+
float exposure = 0;
114
+
float gain = 0;
115
+
for (int i = 1; i < steps_count; i++) {
116
+
struct step s0 = steps[i - 1];
117
+
struct step s1 = steps[i];
118
+
119
+
float lower_b = s0.b;
120
+
float higher_b = s1.b;
121
+
122
+
if (brightness >= lower_b && brightness <= higher_b) {
123
+
exposure = s0.e + ((brightness - lower_b) / (higher_b - lower_b)) * (s1.e - s0.e);
124
+
gain = s0.g + ((brightness - lower_b) / (higher_b - lower_b)) * (s1.g - s0.g);
125
+
break;
126
+
}
127
+
}
128
+
*out_exposure = exposure;
129
+
*out_gain = gain;
130
+
}
131
+
132
+
//! Update `exposure` and `gain` based on current `brightness` value.
133
+
static void
134
+
update_expgain(struct u_autoexpgain *aeg)
135
+
{
136
+
float brightness = aeg->brightness.val;
137
+
if (aeg->last_brightness == brightness) {
138
+
return;
139
+
}
140
+
aeg->last_brightness = brightness;
141
+
142
+
float exposure = -1;
143
+
float gain = -1;
144
+
brightness_to_expgain(aeg, brightness, &exposure, &gain);
145
+
aeg->exposure = (uint16_t)exposure;
146
+
aeg->gain = (uint8_t)gain;
147
+
}
148
+
149
+
//! Returns a value in the range [-1, 1] describing how dark-bright the image
150
+
//! is, 0 means it's alright.
151
+
static float
152
+
get_score(struct u_autoexpgain *aeg, struct xrt_frame *xf)
153
+
{
154
+
uint32_t w = xf->width;
155
+
uint32_t h = xf->height;
156
+
uint32_t s = w / GRID_COLS; // Grid cell size
157
+
158
+
// Compute histogram (PDF)
159
+
int histogram[LEVELS] = {0};
160
+
int samples_count = 0;
161
+
size_t pixel_size = u_format_block_size(xf->format);
162
+
for (uint32_t y = 0; y < h; y += s) {
163
+
for (uint32_t x = 0; x < w; x += s) {
164
+
// Note that for multichannel images only the first channel is in use.
165
+
uint8_t intensity = xf->data[y * xf->stride + x * pixel_size];
166
+
histogram[intensity]++;
167
+
samples_count++;
168
+
}
169
+
}
170
+
171
+
// Draw histogram
172
+
for (int i = 0; i < LEVELS; i++) {
173
+
aeg->histogram[i] = histogram[i];
174
+
}
175
+
176
+
// Compute mean
177
+
float mean = 0;
178
+
for (int i = 0; i < LEVELS; i++) {
179
+
mean += (float)i * histogram[i];
180
+
}
181
+
mean /= samples_count;
182
+
183
+
float score = 0;
184
+
185
+
// Score that tries to make the mean reach TARGET_MEAN.
186
+
187
+
float target_mean = -1;
188
+
if (aeg->strategy == U_AEG_STRATEGY_TRACKING) {
189
+
// We are not that much interested in using the full dynamic range for tracking
190
+
// so we prefer a darkish image because that reduces exposure and gain.
191
+
target_mean = LEVELS / 4;
192
+
} else if (aeg->strategy == U_AEG_STRATEGY_DYNAMIC_RANGE) {
193
+
target_mean = LEVELS / 2;
194
+
}
195
+
196
+
const float range_size = mean < target_mean ? target_mean : (LEVELS - target_mean);
197
+
score = (mean - target_mean) / range_size;
198
+
score = CLAMP(score, -1, 1);
199
+
200
+
return score;
201
+
}
202
+
203
+
204
+
static void
205
+
update_brightness(struct u_autoexpgain *aeg, struct xrt_frame *xf)
206
+
{
207
+
float score = get_score(aeg, xf);
208
+
aeg->current_score = score;
209
+
210
+
if (!aeg->enable) {
211
+
return;
212
+
}
213
+
214
+
aeg->frame_counter++;
215
+
if (aeg->frame_counter % aeg->update_every != 0) {
216
+
return;
217
+
}
218
+
219
+
bool score_is_high = fabsf(score) > aeg->threshold;
220
+
if (!score_is_high) {
221
+
return;
222
+
}
223
+
224
+
float max_step = aeg->max_brightness_step;
225
+
float step = CLAMP(max_step * score, -max_step, max_step);
226
+
aeg->brightness.val -= step;
227
+
aeg->brightness.val = CLAMP(aeg->brightness.val, 0, 1);
228
+
}
229
+
230
+
/*
231
+
*
232
+
* Exported functions
233
+
*
234
+
*/
235
+
236
+
struct u_autoexpgain *
237
+
u_autoexpgain_create(enum u_aeg_strategy strategy, bool enabled_from_start, uint8_t update_every)
238
+
{
239
+
struct u_autoexpgain *aeg = U_TYPED_CALLOC(struct u_autoexpgain);
240
+
241
+
aeg->enable = enabled_from_start;
242
+
aeg->strategy = strategy;
243
+
aeg->strategy_combo.count = U_AEG_STRATEGY_COUNT;
244
+
aeg->strategy_combo.options = "Tracking\0Dynamic Range\0\0";
245
+
aeg->strategy_combo.value = (int *)&aeg->strategy;
246
+
247
+
aeg->histogram_ui.values = aeg->histogram;
248
+
aeg->histogram_ui.count = LEVELS;
249
+
250
+
aeg->brightness.max = 1;
251
+
aeg->brightness.min = 0;
252
+
aeg->brightness.step = 0.002;
253
+
aeg->brightness.val = INITIAL_BRIGHTNESS;
254
+
aeg->last_brightness = INITIAL_BRIGHTNESS;
255
+
aeg->max_brightness_step = INITIAL_MAX_BRIGHTNES_STEP;
256
+
257
+
aeg->threshold = INITIAL_THRESHOLD;
258
+
aeg->frame_counter = 0;
259
+
aeg->update_every = update_every;
260
+
261
+
brightness_to_expgain(aeg, INITIAL_BRIGHTNESS, &aeg->exposure, &aeg->gain);
262
+
263
+
return aeg;
264
+
}
265
+
266
+
void
267
+
u_autoexpgain_add_vars(struct u_autoexpgain *aeg, void *root)
268
+
{
269
+
u_var_add_bool(root, &aeg->enable, "Enable AEG");
270
+
u_var_add_u8(root, &aeg->update_every, "Update every X frames");
271
+
u_var_add_combo(root, &aeg->strategy_combo, "Strategy");
272
+
u_var_add_draggable_f32(root, &aeg->brightness, "Brightness");
273
+
u_var_add_f32(root, &aeg->threshold, "Score threshold");
274
+
u_var_add_f32(root, &aeg->max_brightness_step, "Max brightness step");
275
+
u_var_add_ro_f32(root, &aeg->current_score, "Image score");
276
+
u_var_add_histogram_f32(root, &aeg->histogram_ui, "Intensity histogram");
277
+
}
278
+
279
+
void
280
+
u_autoexpgain_update(struct u_autoexpgain *aeg, struct xrt_frame *xf)
281
+
{
282
+
update_brightness(aeg, xf);
283
+
update_expgain(aeg);
284
+
}
285
+
286
+
float
287
+
u_autoexpgain_get_exposure(struct u_autoexpgain *aeg)
288
+
{
289
+
return aeg->exposure;
290
+
}
291
+
292
+
float
293
+
u_autoexpgain_get_gain(struct u_autoexpgain *aeg)
294
+
{
295
+
return aeg->gain;
296
+
}
297
+
298
+
void
299
+
u_autoexpgain_destroy(struct u_autoexpgain **aeg)
300
+
{
301
+
free(*aeg);
302
+
*aeg = NULL;
303
+
}
+62
src/xrt/auxiliary/util/u_autoexpgain.h
+62
src/xrt/auxiliary/util/u_autoexpgain.h
···
1
+
// Copyright 2022, Collabora, Ltd.
2
+
// SPDX-License-Identifier: BSL-1.0
3
+
/*!
4
+
* @file
5
+
* @brief Automatically compute exposure and gain values from an image stream
6
+
* @author Mateo de Mayo <mateo.demayo@collabora.com>
7
+
* @ingroup aux_util
8
+
*/
9
+
10
+
#pragma once
11
+
12
+
#include "xrt/xrt_frame.h"
13
+
#include <stdint.h>
14
+
15
+
#ifdef __cplusplus
16
+
extern "C" {
17
+
#endif
18
+
19
+
//! An auto exposure/gain strategy tunes the algorithm for specific objectives.
20
+
enum u_aeg_strategy
21
+
{
22
+
U_AEG_STRATEGY_TRACKING = 0, //!< Lower exposure and gain at the cost of darker images.
23
+
U_AEG_STRATEGY_DYNAMIC_RANGE, //!< Tries to maximize the image information
24
+
U_AEG_STRATEGY_COUNT
25
+
};
26
+
27
+
struct u_autoexpgain;
28
+
29
+
/*!
30
+
* Create auto exposure and gain (AEG) algorithm object.
31
+
*
32
+
* @param strategy What objective is preferred for the algorithm.
33
+
* @param enabled_from_start Update exposure/gain from the start.
34
+
* @param update_every Every how many frames should we update exposure/gain
35
+
* @return struct u_autoexpgain* Created object
36
+
*/
37
+
struct u_autoexpgain *
38
+
u_autoexpgain_create(enum u_aeg_strategy strategy, bool enabled_from_start, uint8_t update_every);
39
+
40
+
//! Setup UI for the AEG algorithm
41
+
void
42
+
u_autoexpgain_add_vars(struct u_autoexpgain *aeg, void *root);
43
+
44
+
//! Update the AEG with a frame
45
+
void
46
+
u_autoexpgain_update(struct u_autoexpgain *aeg, struct xrt_frame *xf);
47
+
48
+
//! Get currently computed exposure value in usecs.
49
+
float
50
+
u_autoexpgain_get_exposure(struct u_autoexpgain *aeg);
51
+
52
+
//! Get currently computed gain value in the [0, 255] range.
53
+
float
54
+
u_autoexpgain_get_gain(struct u_autoexpgain *aeg);
55
+
56
+
//! Destroy AEG object
57
+
void
58
+
u_autoexpgain_destroy(struct u_autoexpgain **aeg);
59
+
60
+
#ifdef __cplusplus
61
+
}
62
+
#endif