···11+// Copyright 2023, Collabora, Ltd.
22+// SPDX-License-Identifier: BSL-1.0
33+/*!
44+ * @file
55+ * @brief Visual-Intertial Tracking interface header.
66+ * @author Mateo de Mayo <mateo.demayo@collabora.com>
77+ * @author Simon Zeni <simon.zeni@collabora.com>
88+ * @author Jakob Bornecrantz <jakob@collabora.com>
99+ *
1010+ * This file contains the declaration of the @ref vit_tracker struct. This
1111+ * header is intended to appear in both Monado and external SLAM systems. The
1212+ * implementation of `vit_interface` is provided by the external system.
1313+ * Additional data types are declared for the communication between Monado and
1414+ * the system.
1515+ */
1616+1717+#pragma once
1818+1919+#if defined(__cplusplus)
2020+extern "C" {
2121+#endif
2222+2323+#include <stdint.h>
2424+#include <stdbool.h>
2525+2626+// For implementation: same as IMPLEMENTATION_VERSION_*
2727+// For user: expected IMPLEMENTATION_VERSION_*. Should be checked in runtime.
2828+#define VIT_HEADER_VERSION_MAJOR 1 //!< API Breakages
2929+#define VIT_HEADER_VERSION_MINOR 0 //!< Backwards compatible API changes
3030+#define VIT_HEADER_VERSION_PATCH 0 //!< Backw. comp. .h-implemented changes
3131+3232+/*!
3333+ * Result type used by the VIT system
3434+ *
3535+ * 0 is @ref VIT_SUCCESS, positive values are non fatal results and negative values are errors.
3636+ */
3737+typedef enum vit_result
3838+{
3939+ /*!
4040+ * Operation suceeded
4141+ */
4242+ VIT_SUCCESS = 0,
4343+4444+ /*!
4545+ * The operation is not available on the current version
4646+ */
4747+ VIT_ERROR_INVALID_VERSION = -1,
4848+4949+ /*!
5050+ * The operation received an invalid value.
5151+ */
5252+ VIT_ERROR_INVALID_VALUE = -2,
5353+5454+ /*!
5555+ * The operation was not able to allocate memory to pursue the operation.
5656+ */
5757+ VIT_ERROR_ALLOCATION_FAILURE = -3,
5858+5959+ /*!
6060+ * The operation requires a capability that is not supported.
6161+ */
6262+ VIT_ERROR_NOT_SUPPORTED = -4,
6363+6464+ /*!
6565+ * The operation requires a capability that is not enabled.
6666+ */
6767+ VIT_ERROR_NOT_ENABLED = -5,
6868+} vit_result_t;
6969+7070+/*!
7171+ * Image formats.
7272+ */
7373+typedef enum vit_image_format
7474+{
7575+ //! 8-bit luminance
7676+ VIT_IMAGE_FORMAT_L8 = 1,
7777+ //! 16-bit luminance
7878+ VIT_IMAGE_FORMAT_L16 = 2,
7979+ //! 24-bit rgb, tightly packed.
8080+ VIT_IMAGE_FORMAT_R8G8B8 = 3,
8181+} vit_image_format_t;
8282+8383+/*!
8484+ * Camera calibration types.
8585+ */
8686+typedef enum vit_camera_distortion
8787+{
8888+ //! No distortion (pre-disotorted).
8989+ VIT_CAMERA_DISTORTION_NONE,
9090+ //! Distortion radial-tangential (OpenCV), 4 parameters.
9191+ VIT_CAMERA_DISTORTION_RT4,
9292+ //! Distortion radial-tangential (OpenCV), 5 parameters.
9393+ VIT_CAMERA_DISTORTION_RT5,
9494+ //! Distortion radial-tangential (OpenCV), 8 parameters.
9595+ VIT_CAMERA_DISTORTION_RT8,
9696+ //! Distortion Kannala-Brandt (OpenCV fisheye), 4 parameters.
9797+ VIT_CAMERA_DISTORTION_KB4,
9898+} vit_camera_distortion_t;
9999+100100+/*!
101101+ * Capabilities of the tracker.
102102+ */
103103+typedef enum vit_tracker_capability
104104+{
105105+ //! Does the tracker support per pose (frame) timing data.
106106+ VIT_TRACKER_CAPABILITY_CAMERA_CALIBRATION = 1 << 0,
107107+ //! Does the tracker support per pose (frame) and per camera features.
108108+ VIT_TRACKER_CAPABILITY_IMU_CALIBRATION = 1 << 1,
109109+} vit_tracker_capability_t;
110110+111111+/*!
112112+ * Capabilities of the poses that this tracker produces.
113113+ */
114114+typedef enum vit_tracker_pose_capability
115115+{
116116+ //! Does the tracker support per pose (frame) timing data.
117117+ VIT_TRACKER_POSE_CAPABILITY_TIMING = 1 << 0,
118118+ //! Does the tracker support per pose (frame) and per camera features.
119119+ VIT_TRACKER_POSE_CAPABILITY_FEATURES = 1 << 1,
120120+} vit_tracker_pose_capability_t;
121121+122122+/*!
123123+ * @brief Visual-Inertial Tracking interface, opaque type.
124124+ */
125125+struct vit_tracker;
126126+typedef struct vit_tracker vit_tracker_t;
127127+128128+/*!
129129+ * @brief Pose interface, opaque type.
130130+ */
131131+struct vit_pose;
132132+typedef struct vit_pose vit_pose_t;
133133+134134+/*!
135135+ * @brief Names of the timestamps returned by `vit_pose_get_timings`
136136+ */
137137+typedef struct vit_tracker_timing_titles
138138+{
139139+ uint32_t count; //! Number of titles
140140+ const char **titles; //! Names of the measures timestamps
141141+} vit_tracker_timing_titles;
142142+143143+/*!
144144+ * @brief Parameters for creating the system pipeline.
145145+ */
146146+typedef struct vit_config {
147147+ //! Path to a implementation-specific config file. If null, use defaults.
148148+ const char *file;
149149+150150+ //! Number of cameras to use. Required.
151151+ uint32_t cam_count;
152152+153153+ //! Number of IMU to use. Required.
154154+ uint32_t imu_count;
155155+156156+ //! If supported, whether to open the system's UI.
157157+ bool show_ui;
158158+} vit_config_t;
159159+160160+/*!
161161+ * @brief IMU sample type feed into VIT tracker
162162+ */
163163+typedef struct vit_imu_sample {
164164+ //! In nanoseconds
165165+ int64_t timestamp;
166166+167167+ //! Acceleration in meters per second squared (m/s²)
168168+ float ax, ay, az;
169169+170170+ //! Gyro in radians per second (rad/s)
171171+ float wx, wy, wz;
172172+} vit_imu_sample_t;
173173+174174+/*!
175175+ * Region in image space that this mask covers.
176176+ */
177177+typedef struct vit_mask
178178+{
179179+ //! In pixels.
180180+ float x, y, w, h;
181181+} vit_mask_t;
182182+183183+/*!
184184+ * @brief Image sample type feed into VIT tracker
185185+ *
186186+ * Can easily be converted into an OpenCV Matrix for processing.
187187+ */
188188+typedef struct vit_img_sample {
189189+ uint32_t cam_index;
190190+191191+ //! In nanoseconds, must increase monotonically.
192192+ int64_t timestamp;
193193+194194+ // !Image data
195195+ uint8_t *data;
196196+ uint32_t width, height;
197197+ uint32_t stride, size;
198198+ vit_image_format_t format;
199199+200200+ //! Regions to ignore
201201+ uint32_t mask_count;
202202+ vit_mask_t *masks;
203203+} vit_img_sample_t;
204204+205205+/*!
206206+ * Data that is always returned from tracker.
207207+ */
208208+typedef struct vit_pose_data {
209209+ //! In nanoseconds, must increase monotonically.
210210+ int64_t timestamp;
211211+212212+ //! Position vector.
213213+ float px, py, pz;
214214+215215+ //! Orientation quaternion.
216216+ float ox, oy, oz, ow;
217217+218218+ //! Linear velocity.
219219+ float vx, vy, vz;
220220+} vit_pose_data_t;
221221+222222+/*!
223223+ * Result of pose timing request function.
224224+ */
225225+typedef struct vit_pose_timing {
226226+ uint32_t count;
227227+ const int64_t *timestamps;
228228+} vit_pose_timing_t;
229229+230230+/*!
231231+ * One single feature, element of @ref vit_pose_features result.
232232+ */
233233+typedef struct vit_pose_feature {
234234+ int64_t id;
235235+ float u, v, depth;
236236+} vit_pose_feature_t;
237237+238238+/*!
239239+ * Result of pose feature request function.
240240+ */
241241+typedef struct vit_pose_features {
242242+ uint32_t count;
243243+ const struct vit_pose_feature *features;
244244+} vit_pose_features_t;
245245+246246+/*!
247247+ * Container of parameters for a pinhole camera calibration (fx, fy, cx, cy)
248248+ * with an optional distortion.
249249+ *
250250+ *`distortion_model` and its corresponding `distortion` parameters are not
251251+ * standardized in this struct to facilitate implementation prototyping.
252252+ */
253253+typedef struct vit_camera_calibration {
254254+ uint32_t camera_index; // <! For multi-camera setup. For stereo 0 ~ left, 1 ~ right.
255255+256256+ int width, height; //<! Resolution
257257+ double frequency; //<! Frames per second
258258+ double fx, fy; //<! Focal point
259259+ double cx, cy; //<! Principal point
260260+ enum vit_camera_distortion model;
261261+ uint32_t distortion_count;
262262+ double distortion[32]; //!< Parameters for the distortion model
263263+ double transform[16]; //!< Row-major 4x4 camera transform w.r.t. the IMU (i.e., T_imu_cam)
264264+} vit_camera_calibration_t;
265265+266266+typedef struct vit_inertial_calibration {
267267+ // Calibration intrinsics to apply to each raw measurement.
268268+269269+ //! Row major 3x3 linear transformation for raw measurements alignment and scaling.
270270+ double transform[9];
271271+272272+ //! Offset to add to raw measurements; called bias in other contexts.
273273+ double offset[3];
274274+275275+ // Parameters for the random processes that model this IMU. See section "2.1
276276+ // Gyro Noise Model" of N. Trawny and S. I. Roumeliotis, "Indirect Kalman
277277+ // Filter for 3D Attitude Estimation". Analogous for accelerometers.
278278+ // http://mars.cs.umn.edu/tr/reports/Trawny05b.pdf#page=15
279279+280280+ //! IMU internal bias ~ wiener process with steps N(0, σ²); this field is σ;
281281+ //! [σ] = U / sqrt(sec³) with U = rad if gyroscope, U = m/s if accelerometer.
282282+ double bias_std[3];
283283+284284+ //! IMU measurement noise ~ N(0, σ²); this field is σ.
285285+ //! [σ] = U / sqrt(sec) with U = rad if gyroscope, U = m/s if accelerometer.
286286+ double noise_std[3];
287287+} vit_inertial_calibration_t;
288288+289289+/*!
290290+ * Calibration for one IMU.
291291+ */
292292+typedef struct vit_imu_calibration {
293293+ uint32_t imu_index; //!< For multi-imu setup, usually just 0.
294294+295295+ double frequency; //!< Samples per second
296296+ struct vit_inertial_calibration accel;
297297+ struct vit_inertial_calibration gyro;
298298+} vit_imu_calibration_t;
299299+300300+301301+/*
302302+ *
303303+ * Function prototypes.
304304+ *
305305+ */
306306+307307+typedef vit_result_t (*PFN_vit_api_get_version)(uint32_t *out_major, uint32_t *out_minor, uint32_t *out_patch);
308308+typedef vit_result_t (*PFN_vit_tracker_create)(const vit_config_t *config, vit_tracker_t **out_tracker);
309309+typedef void (*PFN_vit_tracker_destroy)(vit_tracker_t *tracker);
310310+typedef vit_result_t (*PFN_vit_tracker_has_image_format)(const vit_tracker_t *tracker, const vit_image_format_t image_format, bool *out_supported);
311311+typedef vit_result_t (*PFN_vit_tracker_get_capabilities)(const vit_tracker_t *tracker, vit_tracker_capability_t *out_caps);
312312+typedef vit_result_t (*PFN_vit_tracker_get_pose_capabilities)(const vit_tracker_t *tracker, vit_tracker_pose_capability_t *out_caps);
313313+typedef vit_result_t (*PFN_vit_tracker_set_pose_capabilities)(vit_tracker_t *tracker, const vit_tracker_pose_capability_t caps, bool value);
314314+typedef vit_result_t (*PFN_vit_tracker_start)(vit_tracker_t *tracker);
315315+typedef vit_result_t (*PFN_vit_tracker_stop)(vit_tracker_t *tracker);
316316+typedef vit_result_t (*PFN_vit_tracker_reset)(vit_tracker_t *tracker);
317317+typedef vit_result_t (*PFN_vit_tracker_is_running)(const vit_tracker_t *tracker, bool *out_bool);
318318+typedef vit_result_t (*PFN_vit_tracker_push_imu_sample)(vit_tracker_t *tracker, const vit_imu_sample_t *sample);
319319+typedef vit_result_t (*PFN_vit_tracker_push_img_sample)(vit_tracker_t *tracker, const vit_img_sample_t *sample);
320320+typedef vit_result_t (*PFN_vit_tracker_add_imu_calibration)(vit_tracker_t *tracker, const vit_imu_calibration_t *calibration);
321321+typedef vit_result_t (*PFN_vit_tracker_add_camera_calibration)(vit_tracker_t *tracker, const vit_camera_calibration_t *calibration);
322322+typedef vit_result_t (*PFN_vit_tracker_pop_pose)(vit_tracker_t *tracker, vit_pose_t **out_pose);
323323+typedef vit_result_t (*PFN_vit_tracker_get_timing_titles)(const vit_tracker_t *tracker, vit_tracker_timing_titles *out_titles);
324324+typedef void (*PFN_vit_pose_destroy)(vit_pose_t *pose);
325325+typedef vit_result_t (*PFN_vit_pose_get_data)(const vit_pose_t *pose, vit_pose_data_t *out_data);
326326+typedef vit_result_t (*PFN_vit_pose_get_timing)(const vit_pose_t *pose, vit_pose_timing_t *out_timing);
327327+typedef vit_result_t (*PFN_vit_pose_get_features)(const vit_pose_t *pose, uint32_t camera_index, vit_pose_features_t *out_features);
328328+329329+330330+/*
331331+ *
332332+ * Functions.
333333+ *
334334+ */
335335+336336+#ifdef VIT_INTERFACE_IMPLEMENTATION
337337+338338+/*!
339339+ * @brief Returns the API version implemented by the VIT system.
340340+ */
341341+vit_result_t
342342+vit_api_get_version(uint32_t *out_major, uint32_t *out_minor, uint32_t *out_patch);
343343+344344+/*!
345345+ * @brief Creates a new VIT tracker. The caller is responsible of destroying it when done.
346346+ */
347347+vit_result_t
348348+vit_tracker_create(const vit_config_t *config, vit_tracker_t **out_tracker);
349349+350350+/*!
351351+ * @brief Destroys the VIT tracker and free all resources allocated.
352352+ */
353353+void
354354+vit_tracker_destroy(vit_tracker_t *tracker);
355355+356356+/*!
357357+ * @brief Verifies if the tracker supports a given `vit_image_format_t`.
358358+ */
359359+vit_result_t
360360+vit_tracker_has_image_format(const vit_tracker_t *tracker, const vit_image_format_t image_format, bool *out_supported);
361361+362362+/*!
363363+ * @brief Returns a bitfield of capabilities supported by the tracker.
364364+ *
365365+ * @see vit_tracker_capability_t
366366+ */
367367+368368+vit_result_t
369369+vit_tracker_get_capabilities(const vit_tracker_t *tracker, vit_tracker_capability_t *out_caps);
370370+371371+/*!
372372+ * @brief Returns a bitfield of pose capabilities supported by the tracker.
373373+ *
374374+ * @see vit_tracker_pose_capability_t
375375+ */
376376+vit_result_t
377377+vit_tracker_get_pose_capabilities(const vit_tracker_t *tracker, vit_tracker_pose_capability_t *out_caps);
378378+379379+/*!
380380+ * @brief Enables or disables multiple tracker pose capabilities.
381381+ *
382382+ * @p caps can be a bitfield of `vit_tracker_pose_capability_t`.
383383+ *
384384+ * @see vit_tracker_pose_capability_t
385385+ */
386386+vit_result_t
387387+vit_tracker_set_pose_capabilities(vit_tracker_t *tracker, const vit_tracker_pose_capability_t caps, bool value);
388388+389389+/*!
390390+ * @brief Starts the VIT tracker. Image and IMU samples can be pushed and pose can be retrieved.
391391+ *
392392+ * This function must be non blocking. The VIT system implementing it is expected to start its own event loop.
393393+ */
394394+vit_result_t
395395+vit_tracker_start(vit_tracker_t *tracker);
396396+397397+/*!
398398+ * @brief Stops the VIT tracker. The tracker wont accept image and IMU samples, and will not return poses.
399399+ */
400400+vit_result_t
401401+vit_tracker_stop(vit_tracker_t *tracker);
402402+403403+/*!
404404+ * @brief Resets the VIT tracker. The tracker internal state will be set to its original state.
405405+ */
406406+vit_result_t
407407+vit_tracker_reset(vit_tracker_t *tracker);
408408+409409+/*!
410410+ * @brief Verifies if the tracker is running.
411411+ */
412412+vit_result_t
413413+vit_tracker_is_running(const vit_tracker_t *tracker, bool *out_bool);
414414+415415+/*!
416416+ * @brief Push an IMU sample into the tracker.
417417+ *
418418+ * There must be a single producer thread pushing samples.
419419+ * Samples must have monotonically increasing timestamps.
420420+ * The implementation must be non-blocking.
421421+ * Thus, a separate consumer thread should process the samples.
422422+ */
423423+vit_result_t
424424+vit_tracker_push_imu_sample(vit_tracker_t *tracker, const vit_imu_sample_t *sample);
425425+426426+/*!
427427+ * @brief Push an image sample into the tracker.
428428+ *
429429+ * Same conditions as @ref push_imu_sample apply.
430430+ * When using N>1 cameras, the N frames must be pushed following @ref cam_index order.
431431+ * The bundle of N frames must have the same timestamps.
432432+ */
433433+vit_result_t
434434+vit_tracker_push_img_sample(vit_tracker_t *tracker, const vit_img_sample_t *sample);
435435+436436+/*!
437437+ * @brief Adds an inertial measurement unit calibration to the tracker. The tracker must not be started.
438438+ *
439439+ * Returns `VIT_ERROR_NOT_SUPPORTED` if the tracker doesn't offer the capability.
440440+ *
441441+ * @see vit_tracker_get_capabilities
442442+ * @see vit_tracker_capability_t
443443+ */
444444+vit_result_t
445445+vit_tracker_add_imu_calibration(vit_tracker_t *tracker, const vit_imu_calibration_t *calibration);
446446+447447+/*!
448448+ * @brief Adds a camera calibration to the tracker. The tracker must not be started.
449449+ *
450450+ * Returns `VIT_ERROR_NOT_SUPPORTED` if the tracker doesn't offer the capability.
451451+ *
452452+ * @see vit_tracker_get_capabilities
453453+ * @see vit_tracker_capability_t
454454+ */
455455+vit_result_t
456456+vit_tracker_add_camera_calibration(vit_tracker_t *tracker, const vit_camera_calibration_t *calibration);
457457+458458+/*!
459459+ * @brief Get the pose from the front of the tracking queue from the VIT tracker
460460+ *
461461+ * This function must be non-blocking and consumed by a single consummer.
462462+ *
463463+ * If @p out_pose is NULL the pose will be immediately destroyed.
464464+ *
465465+ * @param[out] out_pose Pose returned to the caller, NULL if there is not pose available or on error.
466466+ */
467467+vit_result_t
468468+vit_tracker_pop_pose(vit_tracker_t *tracker, vit_pose_t **out_pose);
469469+470470+/*!
471471+ * @brief Get the titles of the timestamps measured by the pose timings.
472472+ *
473473+ * Returns `VIT_ERROR_NOT_SUPPORTED` if the tracker doesn't offer the pose timing capability.
474474+ */
475475+vit_result_t
476476+vit_tracker_get_timing_titles(const vit_tracker_t *tracker, vit_tracker_timing_titles *out_titles);
477477+478478+/*!
479479+ * @brief Destroys a pose. All of the data, timing and features associated to it will be invalidated.
480480+ */
481481+void
482482+vit_pose_destroy(vit_pose_t *pose);
483483+484484+/*!
485485+ * @brief Gets the data form a given `vit_pose_t`.
486486+ *
487487+ * The data becomes invalid when the associated pose gets destroyed.
488488+ */
489489+vit_result_t
490490+vit_pose_get_data(const vit_pose_t *pose, vit_pose_data_t *out_data);
491491+492492+/*!
493493+ * @brief Gets the timing form a given `vit_pose_t`.
494494+ *
495495+ * The timing data becomes invalid when the associated pose gets destroyed.
496496+ */
497497+vit_result_t
498498+vit_pose_get_timing(const vit_pose_t *pose, vit_pose_timing_t *out_timing);
499499+500500+/*!
501501+ * @brief Gets the features form a given `vit_pose_t`.
502502+ *
503503+ * The features data becomes invalid when the associated pose gets destroyed.
504504+ */
505505+vit_result_t
506506+vit_pose_get_features(const vit_pose_t *pose, uint32_t camera_index, vit_pose_features_t *out_features);
507507+508508+#endif
509509+510510+#if defined(__cplusplus)
511511+}
512512+#endif