The open source OpenXR runtime
xrt: add support for FB_face_tracking2 xrt-devices
Part-of: <https://gitlab.freedesktop.org/monado/monado/-/merge_requests/2299>
+523
-1
+2
CMakeLists.txt
+2
CMakeLists.txt
···
353
353
option(XRT_FEATURE_OPENXR_VULKAN_SWAPCHAIN_FORMAT_LIST "Enable XR_KHR_vulkan_swapchain_format_list" ON)
354
354
option(XRT_FEATURE_OPENXR_FACIAL_TRACKING_HTC "Enable XR_HTC_facial_tracking" OFF)
355
355
option(XRT_FEATURE_OPENXR_BODY_TRACKING_FB "Enable XR_FB_body_tracking" OFF)
356
+
option(XRT_FEATURE_OPENXR_FACE_TRACKING2_FB "Enable XR_FB_face_tracking2" OFF)
356
357
option(XRT_FEATURE_OPENXR_XDEV_SPACE "Enable XR_MNDX_xdev_space" ON)
357
358
358
359
# Interaction extension support.
···
578
579
message(STATUS "# FEATURE_DEBUG_GUI: ${XRT_FEATURE_DEBUG_GUI}")
579
580
message(STATUS "# FEATURE_OPENXR: ${XRT_FEATURE_OPENXR}")
580
581
message(STATUS "# FEATURE_OPENXR_BODY_TRACKING_FB: ${XRT_FEATURE_OPENXR_BODY_TRACKING_FB}")
582
+
message(STATUS "# FEATURE_OPENXR_FACE_TRACKING2_FB: ${XRT_FEATURE_OPENXR_FACE_TRACKING2_FB}")
581
583
message(STATUS "# FEATURE_OPENXR_DEBUG_UTILS: ${XRT_FEATURE_OPENXR_DEBUG_UTILS}")
582
584
message(STATUS "# FEATURE_OPENXR_DISPLAY_REFRESH_RATE: ${XRT_FEATURE_OPENXR_DISPLAY_REFRESH_RATE}")
583
585
message(STATUS "# FEATURE_OPENXR_FACIAL_TRACKING_HTC: ${XRT_FEATURE_OPENXR_FACIAL_TRACKING_HTC}")
+1
scripts/generate_oxr_ext_support.py
+1
scripts/generate_oxr_ext_support.py
···
73
73
['XR_FB_composition_layer_image_layout', 'XRT_FEATURE_OPENXR_LAYER_FB_IMAGE_LAYOUT'],
74
74
['XR_FB_composition_layer_settings', 'XRT_FEATURE_OPENXR_LAYER_FB_SETTINGS'],
75
75
['XR_FB_composition_layer_depth_test', 'XRT_FEATURE_OPENXR_LAYER_FB_DEPTH_TEST'],
76
+
['XR_FB_face_tracking2', 'XRT_FEATURE_OPENXR_FACE_TRACKING2_FB'],
76
77
['XR_FB_display_refresh_rate', 'XRT_FEATURE_OPENXR_DISPLAY_REFRESH_RATE'],
77
78
['XR_FB_passthrough', 'XRT_FEATURE_OPENXR_LAYER_PASSTHROUGH'],
78
79
['XR_FB_touch_controller_pro', 'XRT_FEATURE_OPENXR_INTERACTION_TOUCH_PRO'],
+1
scripts/mapping.imp
+1
scripts/mapping.imp
···
36
36
{ symbol: ["XRT_FEATURE_OPENXR_DEBUG_UTILS", "public", "\"xrt/xrt_config_build.h\"", "public"] },
37
37
{ symbol: ["XRT_FEATURE_OPENXR_DISPLAY_REFRESH_RATE", "public", "\"xrt/xrt_config_build.h\"", "public"] },
38
38
{ symbol: ["XRT_FEATURE_OPENXR_FORCE_FEEDBACK_CURL", "public", "\"xrt/xrt_config_build.h\"", "public"] },
39
+
{ symbol: ["XRT_FEATURE_OPENXR_FACE_TRACKING2_FB", "public", "\"xrt/xrt_config_build.h\"", "public"] },
39
40
{ symbol: ["XRT_FEATURE_OPENXR_FACIAL_TRACKING_HTC", "public", "\"xrt/xrt_config_build.h\"", "public"] },
40
41
{ symbol: ["XRT_FEATURE_OPENXR_HEADLESS", "public", "\"xrt/xrt_config_build.h\"", "public"] },
41
42
{ symbol: ["XRT_FEATURE_OPENXR_INTERACTION_EXT_EYE_GAZE", "public", "\"xrt/xrt_config_build.h\"", "public"] },
+1
src/xrt/compositor/multi/comp_multi_system.c
+1
src/xrt/compositor/multi/comp_multi_system.c
+1
src/xrt/include/xrt/xrt_compositor.h
+1
src/xrt/include/xrt/xrt_compositor.h
+1
src/xrt/include/xrt/xrt_config_build.h.cmake_in
+1
src/xrt/include/xrt/xrt_config_build.h.cmake_in
···
33
33
#cmakedefine XRT_FEATURE_OPENXR_DEBUG_UTILS
34
34
#cmakedefine XRT_FEATURE_OPENXR_DISPLAY_REFRESH_RATE
35
35
#cmakedefine XRT_FEATURE_OPENXR_FORCE_FEEDBACK_CURL
36
+
#cmakedefine XRT_FEATURE_OPENXR_FACE_TRACKING2_FB
36
37
#cmakedefine XRT_FEATURE_OPENXR_FACIAL_TRACKING_HTC
37
38
#cmakedefine XRT_FEATURE_OPENXR_HEADLESS
38
39
#cmakedefine XRT_FEATURE_OPENXR_INTERACTION_EXT_EYE_GAZE
+115
src/xrt/include/xrt/xrt_defines.h
+115
src/xrt/include/xrt/xrt_defines.h
···
759
759
//! XR_FB_body_tracking
760
760
XRT_DEVICE_FB_BODY_TRACKING,
761
761
762
+
//! XR_FB_face_tracking2
763
+
XRT_DEVICE_FB_FACE_TRACKING2,
764
+
762
765
// added in OpenXR 1.1
763
766
XRT_DEVICE_PICO_NEO3_CONTROLLER,
764
767
XRT_DEVICE_PICO4_CONTROLLER,
···
1138
1141
\
1139
1142
_(XRT_INPUT_HTC_EYE_FACE_TRACKING , XRT_INPUT_NAME(0x0601, FACE_TRACKING)) \
1140
1143
_(XRT_INPUT_HTC_LIP_FACE_TRACKING , XRT_INPUT_NAME(0x0602, FACE_TRACKING)) \
1144
+
_(XRT_INPUT_FB_FACE_TRACKING2_AUDIO , XRT_INPUT_NAME(0x0603, FACE_TRACKING)) \
1145
+
_(XRT_INPUT_FB_FACE_TRACKING2_VISUAL , XRT_INPUT_NAME(0x0604, FACE_TRACKING)) \
1141
1146
\
1142
1147
_(XRT_INPUT_GENERIC_BODY_TRACKING , XRT_INPUT_NAME(0x0700, BODY_TRACKING)) \
1143
1148
_(XRT_INPUT_FB_BODY_TRACKING , XRT_INPUT_NAME(0x0701, BODY_TRACKING)) \
···
1442
1447
1443
1448
#define XRT_OUTPUT_NAME(id, type) ((UINT32_C(id) << XRT_OUTPUT_TYPE_BITWIDTH) | (uint32_t)XRT_OUTPUT_TYPE_##type)
1444
1449
1450
+
enum xrt_face_expression2_fb
1451
+
{
1452
+
XRT_FACE_EXPRESSION2_BROW_LOWERER_L_FB = 0,
1453
+
XRT_FACE_EXPRESSION2_BROW_LOWERER_R_FB = 1,
1454
+
XRT_FACE_EXPRESSION2_CHEEK_PUFF_L_FB = 2,
1455
+
XRT_FACE_EXPRESSION2_CHEEK_PUFF_R_FB = 3,
1456
+
XRT_FACE_EXPRESSION2_CHEEK_RAISER_L_FB = 4,
1457
+
XRT_FACE_EXPRESSION2_CHEEK_RAISER_R_FB = 5,
1458
+
XRT_FACE_EXPRESSION2_CHEEK_SUCK_L_FB = 6,
1459
+
XRT_FACE_EXPRESSION2_CHEEK_SUCK_R_FB = 7,
1460
+
XRT_FACE_EXPRESSION2_CHIN_RAISER_B_FB = 8,
1461
+
XRT_FACE_EXPRESSION2_CHIN_RAISER_T_FB = 9,
1462
+
XRT_FACE_EXPRESSION2_DIMPLER_L_FB = 10,
1463
+
XRT_FACE_EXPRESSION2_DIMPLER_R_FB = 11,
1464
+
XRT_FACE_EXPRESSION2_EYES_CLOSED_L_FB = 12,
1465
+
XRT_FACE_EXPRESSION2_EYES_CLOSED_R_FB = 13,
1466
+
XRT_FACE_EXPRESSION2_EYES_LOOK_DOWN_L_FB = 14,
1467
+
XRT_FACE_EXPRESSION2_EYES_LOOK_DOWN_R_FB = 15,
1468
+
XRT_FACE_EXPRESSION2_EYES_LOOK_LEFT_L_FB = 16,
1469
+
XRT_FACE_EXPRESSION2_EYES_LOOK_LEFT_R_FB = 17,
1470
+
XRT_FACE_EXPRESSION2_EYES_LOOK_RIGHT_L_FB = 18,
1471
+
XRT_FACE_EXPRESSION2_EYES_LOOK_RIGHT_R_FB = 19,
1472
+
XRT_FACE_EXPRESSION2_EYES_LOOK_UP_L_FB = 20,
1473
+
XRT_FACE_EXPRESSION2_EYES_LOOK_UP_R_FB = 21,
1474
+
XRT_FACE_EXPRESSION2_INNER_BROW_RAISER_L_FB = 22,
1475
+
XRT_FACE_EXPRESSION2_INNER_BROW_RAISER_R_FB = 23,
1476
+
XRT_FACE_EXPRESSION2_JAW_DROP_FB = 24,
1477
+
XRT_FACE_EXPRESSION2_JAW_SIDEWAYS_LEFT_FB = 25,
1478
+
XRT_FACE_EXPRESSION2_JAW_SIDEWAYS_RIGHT_FB = 26,
1479
+
XRT_FACE_EXPRESSION2_JAW_THRUST_FB = 27,
1480
+
XRT_FACE_EXPRESSION2_LID_TIGHTENER_L_FB = 28,
1481
+
XRT_FACE_EXPRESSION2_LID_TIGHTENER_R_FB = 29,
1482
+
XRT_FACE_EXPRESSION2_LIP_CORNER_DEPRESSOR_L_FB = 30,
1483
+
XRT_FACE_EXPRESSION2_LIP_CORNER_DEPRESSOR_R_FB = 31,
1484
+
XRT_FACE_EXPRESSION2_LIP_CORNER_PULLER_L_FB = 32,
1485
+
XRT_FACE_EXPRESSION2_LIP_CORNER_PULLER_R_FB = 33,
1486
+
XRT_FACE_EXPRESSION2_LIP_FUNNELER_LB_FB = 34,
1487
+
XRT_FACE_EXPRESSION2_LIP_FUNNELER_LT_FB = 35,
1488
+
XRT_FACE_EXPRESSION2_LIP_FUNNELER_RB_FB = 36,
1489
+
XRT_FACE_EXPRESSION2_LIP_FUNNELER_RT_FB = 37,
1490
+
XRT_FACE_EXPRESSION2_LIP_PRESSOR_L_FB = 38,
1491
+
XRT_FACE_EXPRESSION2_LIP_PRESSOR_R_FB = 39,
1492
+
XRT_FACE_EXPRESSION2_LIP_PUCKER_L_FB = 40,
1493
+
XRT_FACE_EXPRESSION2_LIP_PUCKER_R_FB = 41,
1494
+
XRT_FACE_EXPRESSION2_LIP_STRETCHER_L_FB = 42,
1495
+
XRT_FACE_EXPRESSION2_LIP_STRETCHER_R_FB = 43,
1496
+
XRT_FACE_EXPRESSION2_LIP_SUCK_LB_FB = 44,
1497
+
XRT_FACE_EXPRESSION2_LIP_SUCK_LT_FB = 45,
1498
+
XRT_FACE_EXPRESSION2_LIP_SUCK_RB_FB = 46,
1499
+
XRT_FACE_EXPRESSION2_LIP_SUCK_RT_FB = 47,
1500
+
XRT_FACE_EXPRESSION2_LIP_TIGHTENER_L_FB = 48,
1501
+
XRT_FACE_EXPRESSION2_LIP_TIGHTENER_R_FB = 49,
1502
+
XRT_FACE_EXPRESSION2_LIPS_TOWARD_FB = 50,
1503
+
XRT_FACE_EXPRESSION2_LOWER_LIP_DEPRESSOR_L_FB = 51,
1504
+
XRT_FACE_EXPRESSION2_LOWER_LIP_DEPRESSOR_R_FB = 52,
1505
+
XRT_FACE_EXPRESSION2_MOUTH_LEFT_FB = 53,
1506
+
XRT_FACE_EXPRESSION2_MOUTH_RIGHT_FB = 54,
1507
+
XRT_FACE_EXPRESSION2_NOSE_WRINKLER_L_FB = 55,
1508
+
XRT_FACE_EXPRESSION2_NOSE_WRINKLER_R_FB = 56,
1509
+
XRT_FACE_EXPRESSION2_OUTER_BROW_RAISER_L_FB = 57,
1510
+
XRT_FACE_EXPRESSION2_OUTER_BROW_RAISER_R_FB = 58,
1511
+
XRT_FACE_EXPRESSION2_UPPER_LID_RAISER_L_FB = 59,
1512
+
XRT_FACE_EXPRESSION2_UPPER_LID_RAISER_R_FB = 60,
1513
+
XRT_FACE_EXPRESSION2_UPPER_LIP_RAISER_L_FB = 61,
1514
+
XRT_FACE_EXPRESSION2_UPPER_LIP_RAISER_R_FB = 62,
1515
+
XRT_FACE_EXPRESSION2_TONGUE_TIP_INTERDENTAL_FB = 63,
1516
+
XRT_FACE_EXPRESSION2_TONGUE_TIP_ALVEOLAR_FB = 64,
1517
+
XRT_FACE_EXPRESSION2_TONGUE_FRONT_DORSAL_PALATE_FB = 65,
1518
+
XRT_FACE_EXPRESSION2_TONGUE_MID_DORSAL_PALATE_FB = 66,
1519
+
XRT_FACE_EXPRESSION2_TONGUE_BACK_DORSAL_VELAR_FB = 67,
1520
+
XRT_FACE_EXPRESSION2_TONGUE_OUT_FB = 68,
1521
+
XRT_FACE_EXPRESSION2_TONGUE_RETREAT_FB = 69,
1522
+
XRT_FACE_EXPRESSION2_COUNT_FB = 70,
1523
+
XRT_FACE_EXPRESSION_2FB_MAX_ENUM_FB = 0x7FFFFFFF
1524
+
};
1525
+
1526
+
enum xrt_face_confidence2_fb
1527
+
{
1528
+
XRT_FACE_CONFIDENCE2_LOWER_FACE_FB = 0,
1529
+
XRT_FACE_CONFIDENCE2_UPPER_FACE_FB = 1,
1530
+
XRT_FACE_CONFIDENCE2_COUNT_FB = 2,
1531
+
XRT_FACE_CONFIDENCE_2FB_MAX_ENUM_FB = 0x7FFFFFFF
1532
+
};
1533
+
1534
+
enum xrt_face_expression_set2_fb
1535
+
{
1536
+
XRT_FACE_EXPRESSION_SET2_DEFAULT_FB = 0,
1537
+
XRT_FACE_EXPRESSION_SET_2FB_MAX_ENUM_FB = 0x7FFFFFFF
1538
+
};
1539
+
1540
+
enum xrt_face_tracking_data_source2_fb
1541
+
{
1542
+
XRT_FACE_TRACKING_DATA_SOURCE2_VISUAL_FB = 0,
1543
+
XRT_FACE_TRACKING_DATA_SOURCE2_AUDIO_FB = 1,
1544
+
XRT_FACE_TRACKING_DATA_SOURCE_2FB_MAX_ENUM_FB = 0x7FFFFFFF
1545
+
};
1546
+
1445
1547
enum xrt_eye_expression_htc
1446
1548
{
1447
1549
XRT_EYE_EXPRESSION_LEFT_BLINK_HTC = 0,
···
1530
1632
float expression_weights[XRT_FACIAL_EXPRESSION_LIP_COUNT_HTC];
1531
1633
};
1532
1634
1635
+
struct xrt_facial_expression_set2_fb
1636
+
{
1637
+
float weights[XRT_FACE_EXPRESSION2_COUNT_FB];
1638
+
float confidences[XRT_FACE_CONFIDENCE2_COUNT_FB];
1639
+
1640
+
enum xrt_face_tracking_data_source2_fb data_source;
1641
+
uint64_t sample_time_ns;
1642
+
1643
+
bool is_valid;
1644
+
bool is_eye_following_blendshapes_valid;
1645
+
};
1646
+
1533
1647
struct xrt_facial_expression_set
1534
1648
{
1535
1649
union {
1536
1650
struct xrt_facial_base_expression_set_htc base_expression_set_htc;
1537
1651
struct xrt_facial_eye_expression_set_htc eye_expression_set_htc;
1538
1652
struct xrt_facial_lip_expression_set_htc lip_expression_set_htc;
1653
+
struct xrt_facial_expression_set2_fb face_expression_set2_fb;
1539
1654
};
1540
1655
};
1541
1656
+1
src/xrt/include/xrt/xrt_instance.h
+1
src/xrt/include/xrt/xrt_instance.h
+16
src/xrt/ipc/client/ipc_client_hmd.c
+16
src/xrt/ipc/client/ipc_client_hmd.c
···
216
216
}
217
217
}
218
218
219
+
static xrt_result_t
220
+
ipc_client_hmd_get_face_tracking(struct xrt_device *xdev,
221
+
enum xrt_input_name facial_expression_type,
222
+
struct xrt_facial_expression_set *out_value)
223
+
{
224
+
ipc_client_hmd_t *icd = ipc_client_hmd(xdev);
225
+
226
+
xrt_result_t xret = ipc_call_device_get_face_tracking( //
227
+
icd->ipc_c, //
228
+
icd->device_id, //
229
+
facial_expression_type, //
230
+
out_value); //
231
+
IPC_CHK_ALWAYS_RET(icd->ipc_c, xret, "ipc_call_device_get_face_tracking");
232
+
}
233
+
219
234
static bool
220
235
ipc_client_hmd_compute_distortion(
221
236
struct xrt_device *xdev, uint32_t view, float u, float v, struct xrt_uv_triplet *out_result)
···
312
327
ich->device_id = device_id;
313
328
ich->base.update_inputs = ipc_client_hmd_update_inputs;
314
329
ich->base.get_tracked_pose = ipc_client_hmd_get_tracked_pose;
330
+
ich->base.get_face_tracking = ipc_client_hmd_get_face_tracking;
315
331
ich->base.get_view_poses = ipc_client_hmd_get_view_poses;
316
332
ich->base.compute_distortion = ipc_client_hmd_compute_distortion;
317
333
ich->base.destroy = ipc_client_hmd_destroy;
+4
src/xrt/ipc/server/ipc_server_handler.c
+4
src/xrt/ipc/server/ipc_server_handler.c
···
280
280
#ifdef OXR_HAVE_FB_body_tracking
281
281
EXT(fb_body_tracking_enabled);
282
282
#endif
283
+
#ifdef OXR_HAVE_FB_face_tracking2
284
+
EXT(fb_face_tracking2_enabled);
285
+
#endif
283
286
284
287
#undef EXT
285
288
#undef PTT
···
373
376
.ext_hand_interaction_enabled = ics->client_state.info.ext_hand_interaction_enabled,
374
377
.htc_facial_tracking_enabled = ics->client_state.info.htc_facial_tracking_enabled,
375
378
.fb_body_tracking_enabled = ics->client_state.info.fb_body_tracking_enabled,
379
+
.fb_face_tracking2_enabled = ics->client_state.info.fb_face_tracking2_enabled,
376
380
};
377
381
378
382
return xrt_comp_begin_session(ics->xc, &begin_session_info);
+4
src/xrt/state_trackers/oxr/CMakeLists.txt
+4
src/xrt/state_trackers/oxr/CMakeLists.txt
···
110
110
target_sources(st_oxr PRIVATE oxr_api_face_tracking.c oxr_face_tracking.c)
111
111
endif()
112
112
113
+
if(XRT_FEATURE_OPENXR_FACE_TRACKING2_FB)
114
+
target_sources(st_oxr PRIVATE oxr_api_face_tracking2_fb.c oxr_face_tracking2_fb.c)
115
+
endif()
116
+
113
117
target_link_libraries(
114
118
st_oxr
115
119
PRIVATE
+82
src/xrt/state_trackers/oxr/oxr_api_face_tracking2_fb.c
+82
src/xrt/state_trackers/oxr/oxr_api_face_tracking2_fb.c
···
1
+
// Copyright 2024, Collabora, Ltd.
2
+
// SPDX-License-Identifier: BSL-1.0
3
+
/*!
4
+
* @file
5
+
* @brief face tracking related API entrypoint functions.
6
+
* @author galister <galister@librevr.org>
7
+
* @ingroup oxr_api
8
+
*/
9
+
10
+
#include <stdio.h>
11
+
#include <stdlib.h>
12
+
#include <string.h>
13
+
#include <inttypes.h>
14
+
15
+
#include "openxr/openxr.h"
16
+
#include "util/u_trace_marker.h"
17
+
18
+
#include "oxr_objects.h"
19
+
#include "oxr_logger.h"
20
+
21
+
#include "oxr_api_funcs.h"
22
+
#include "oxr_api_verify.h"
23
+
#include "oxr_handle.h"
24
+
25
+
XRAPI_ATTR XrResult XRAPI_CALL
26
+
oxr_xrCreateFaceTracker2FB(XrSession session,
27
+
const XrFaceTrackerCreateInfo2FB *createInfo,
28
+
XrFaceTracker2FB *faceTracker)
29
+
{
30
+
OXR_TRACE_MARKER();
31
+
32
+
struct oxr_logger log;
33
+
XrResult ret = XR_SUCCESS;
34
+
struct oxr_session *sess = NULL;
35
+
struct oxr_face_tracker2_fb *face_tracker2_fb = NULL;
36
+
OXR_VERIFY_SESSION_AND_INIT_LOG(&log, session, sess, "xrCreateFaceTracker2FB");
37
+
OXR_VERIFY_SESSION_NOT_LOST(&log, sess);
38
+
OXR_VERIFY_ARG_TYPE_AND_NOT_NULL(&log, createInfo, XR_TYPE_FACE_TRACKER_CREATE_INFO2_FB);
39
+
OXR_VERIFY_EXTENSION(&log, sess->sys->inst, FB_face_tracking2);
40
+
41
+
ret = oxr_face_tracker2_fb_create(&log, sess, createInfo, &face_tracker2_fb);
42
+
if (ret != XR_SUCCESS) {
43
+
return ret;
44
+
}
45
+
46
+
OXR_VERIFY_ARG_NOT_NULL(&log, face_tracker2_fb);
47
+
*faceTracker = oxr_face_tracker2_fb_to_openxr(face_tracker2_fb);
48
+
49
+
return XR_SUCCESS;
50
+
}
51
+
52
+
XRAPI_ATTR XrResult XRAPI_CALL
53
+
oxr_xrDestroyFaceTracker2FB(XrFaceTracker2FB faceTracker)
54
+
{
55
+
OXR_TRACE_MARKER();
56
+
57
+
struct oxr_logger log;
58
+
struct oxr_face_tracker2_fb *face_tracker2_fb = NULL;
59
+
OXR_VERIFY_FACE_TRACKER2_FB_AND_INIT_LOG(&log, faceTracker, face_tracker2_fb, "xrDestroyFaceTracker2FB");
60
+
61
+
return oxr_handle_destroy(&log, &face_tracker2_fb->handle);
62
+
}
63
+
64
+
XRAPI_ATTR XrResult XRAPI_CALL
65
+
oxr_xrGetFaceExpressionWeights2FB(XrFaceTracker2FB faceTracker,
66
+
const XrFaceExpressionInfo2FB *expressionInfo,
67
+
XrFaceExpressionWeights2FB *expressionWeights)
68
+
{
69
+
OXR_TRACE_MARKER();
70
+
71
+
struct oxr_logger log;
72
+
struct oxr_face_tracker2_fb *face_tracker2_fb = NULL;
73
+
OXR_VERIFY_FACE_TRACKER2_FB_AND_INIT_LOG(&log, faceTracker, face_tracker2_fb, "xrGetFaceExpressionWeights2FB");
74
+
OXR_VERIFY_SESSION_NOT_LOST(&log, face_tracker2_fb->sess);
75
+
OXR_VERIFY_ARG_NOT_NULL(&log, face_tracker2_fb->xdev);
76
+
OXR_VERIFY_ARG_TYPE_AND_NOT_NULL(&log, expressionInfo, XR_TYPE_FACE_EXPRESSION_INFO2_FB);
77
+
OXR_VERIFY_ARG_TYPE_AND_NOT_NULL(&log, expressionWeights, XR_TYPE_FACE_EXPRESSION_WEIGHTS2_FB);
78
+
OXR_VERIFY_ARG_NOT_NULL(&log, expressionWeights->weights);
79
+
OXR_VERIFY_ARG_NOT_NULL(&log, expressionWeights->confidences);
80
+
81
+
return oxr_get_face_expression_weights2_fb(&log, face_tracker2_fb, expressionInfo, expressionWeights);
82
+
}
+17
src/xrt/state_trackers/oxr/oxr_api_funcs.h
+17
src/xrt/state_trackers/oxr/oxr_api_funcs.h
···
655
655
XrBodyJointLocationsFB *locations);
656
656
#endif
657
657
658
+
#ifdef OXR_HAVE_FB_face_tracking2
659
+
//! OpenXR API function @ep{xrCreateFaceTracker2FB}
660
+
XRAPI_ATTR XrResult XRAPI_CALL
661
+
oxr_xrCreateFaceTracker2FB(XrSession session,
662
+
const XrFaceTrackerCreateInfo2FB *createInfo,
663
+
XrFaceTracker2FB *faceTracker);
664
+
665
+
//! OpenXR API function @ep{xrDestroyFaceTracker2FB}
666
+
XRAPI_ATTR XrResult XRAPI_CALL
667
+
oxr_xrDestroyFaceTracker2FB(XrFaceTracker2FB faceTracker);
668
+
669
+
//! OpenXR API function @ep{xrGetFaceExpressionWeights2FB}
670
+
XRAPI_ATTR XrResult XRAPI_CALL
671
+
oxr_xrGetFaceExpressionWeights2FB(XrFaceTracker2FB faceTracker,
672
+
const XrFaceExpressionInfo2FB *expressionInfo,
673
+
XrFaceExpressionWeights2FB *expressionWeights);
674
+
#endif
658
675
659
676
/*
660
677
*
+6
src/xrt/state_trackers/oxr/oxr_api_negotiate.c
+6
src/xrt/state_trackers/oxr/oxr_api_negotiate.c
···
332
332
ENTRY_IF_EXT(xrLocateBodyJointsFB, FB_body_tracking);
333
333
#endif
334
334
335
+
#ifdef OXR_HAVE_FB_face_tracking2
336
+
ENTRY_IF_EXT(xrCreateFaceTracker2FB, FB_face_tracking2);
337
+
ENTRY_IF_EXT(xrDestroyFaceTracker2FB, FB_face_tracking2);
338
+
ENTRY_IF_EXT(xrGetFaceExpressionWeights2FB, FB_face_tracking2);
339
+
#endif
340
+
335
341
#ifdef OXR_HAVE_MNDX_xdev_space
336
342
ENTRY_IF_EXT(xrCreateXDevListMNDX, MNDX_xdev_space);
337
343
ENTRY_IF_EXT(xrGetXDevListGenerationNumberMNDX, MNDX_xdev_space);
+2
src/xrt/state_trackers/oxr/oxr_api_verify.h
+2
src/xrt/state_trackers/oxr/oxr_api_verify.h
···
74
74
OXR_VERIFY_AND_SET_AND_INIT(log, thing, new_thing, oxr_passthrough_layer, PASSTHROUGH_LAYER, name, new_thing->sess->sys->inst)
75
75
#define OXR_VERIFY_FACE_TRACKER_HTC_AND_INIT_LOG(log, thing, new_thing, name) \
76
76
OXR_VERIFY_AND_SET_AND_INIT(log, thing, new_thing, oxr_facial_tracker_htc, FTRACKER, name, new_thing->sess->sys->inst)
77
+
#define OXR_VERIFY_FACE_TRACKER2_FB_AND_INIT_LOG(log, thing, new_thing, name) \
78
+
OXR_VERIFY_AND_SET_AND_INIT(log, thing, new_thing, oxr_face_tracker2_fb, FTRACKER, name, new_thing->sess->sys->inst)
77
79
#define OXR_VERIFY_BODY_TRACKER_FB_AND_INIT_LOG(log, thing, new_thing, name) \
78
80
OXR_VERIFY_AND_SET_AND_INIT(log, thing, new_thing, oxr_body_tracker_fb, BTRACKER, name, new_thing->sess->sys->inst)
79
81
#define OXR_VERIFY_XDEVLIST_AND_INIT_LOG(log, thing, new_thing, name) \
+12
src/xrt/state_trackers/oxr/oxr_extension_support.h
+12
src/xrt/state_trackers/oxr/oxr_extension_support.h
···
428
428
429
429
430
430
/*
431
+
* XR_FB_face_tracking2
432
+
*/
433
+
#if defined(XR_FB_face_tracking2) && defined(XRT_FEATURE_OPENXR_FACE_TRACKING2_FB)
434
+
#define OXR_HAVE_FB_face_tracking2
435
+
#define OXR_EXTENSION_SUPPORT_FB_face_tracking2(_) _(FB_face_tracking2, FB_FACE_TRACKING2)
436
+
#else
437
+
#define OXR_EXTENSION_SUPPORT_FB_face_tracking2(_)
438
+
#endif
439
+
440
+
441
+
/*
431
442
* XR_FB_composition_layer_alpha_blend
432
443
*/
433
444
#if defined(XR_FB_composition_layer_alpha_blend) && defined(XRT_FEATURE_OPENXR_LAYER_FB_ALPHA_BLEND)
···
784
795
OXR_EXTENSION_SUPPORT_EXT_samsung_odyssey_controller(_) \
785
796
OXR_EXTENSION_SUPPORT_BD_controller_interaction(_) \
786
797
OXR_EXTENSION_SUPPORT_FB_body_tracking(_) \
798
+
OXR_EXTENSION_SUPPORT_FB_face_tracking2(_) \
787
799
OXR_EXTENSION_SUPPORT_FB_composition_layer_alpha_blend(_) \
788
800
OXR_EXTENSION_SUPPORT_FB_composition_layer_image_layout(_) \
789
801
OXR_EXTENSION_SUPPORT_FB_composition_layer_settings(_) \
+150
src/xrt/state_trackers/oxr/oxr_face_tracking2_fb.c
+150
src/xrt/state_trackers/oxr/oxr_face_tracking2_fb.c
···
1
+
// Copyright 2024, Collabora, Ltd.
2
+
// SPDX-License-Identifier: BSL-1.0
3
+
/*!
4
+
* @file
5
+
* @brief face tracking related API entrypoint functions.
6
+
* @author galister <galister@librevr.org>
7
+
* @ingroup oxr_main
8
+
*/
9
+
10
+
#include <stdio.h>
11
+
#include <stdlib.h>
12
+
#include <string.h>
13
+
#include <inttypes.h>
14
+
15
+
#include "oxr_objects.h"
16
+
#include "oxr_logger.h"
17
+
#include "oxr_handle.h"
18
+
#include "xrt/xrt_defines.h"
19
+
20
+
static XrResult
21
+
oxr_face_tracker2_fb_destroy_cb(struct oxr_logger *log, struct oxr_handle_base *hb)
22
+
{
23
+
struct oxr_face_tracker2_fb *face_tracker2_fb = (struct oxr_face_tracker2_fb *)hb;
24
+
free(face_tracker2_fb);
25
+
return XR_SUCCESS;
26
+
}
27
+
28
+
XrResult
29
+
oxr_face_tracker2_fb_create(struct oxr_logger *log,
30
+
struct oxr_session *sess,
31
+
const XrFaceTrackerCreateInfo2FB *createInfo,
32
+
struct oxr_face_tracker2_fb **out_face_tracker2_fb)
33
+
{
34
+
if (createInfo->faceExpressionSet != XR_FACE_EXPRESSION_SET2_DEFAULT_FB) {
35
+
return oxr_error(log, XR_ERROR_FEATURE_UNSUPPORTED, "Unsupported expression set");
36
+
}
37
+
38
+
struct xrt_device *xdev = GET_XDEV_BY_ROLE(sess->sys, face);
39
+
if (xdev == NULL) {
40
+
return oxr_error(log, XR_ERROR_FEATURE_UNSUPPORTED, "No device found for face tracking role");
41
+
}
42
+
43
+
if (!xdev->face_tracking_supported || xdev->get_face_tracking == NULL) {
44
+
return oxr_error(log, XR_ERROR_FEATURE_UNSUPPORTED, "Device does not support FB2 face tracking");
45
+
}
46
+
47
+
bool supports_audio, supports_visual;
48
+
49
+
oxr_system_get_face_tracking2_fb_support(log, sess->sys->inst, &supports_audio, &supports_visual);
50
+
51
+
if (!supports_audio && !supports_visual) {
52
+
return oxr_error(log, XR_ERROR_FEATURE_UNSUPPORTED, "System does not support FB2 face tracking");
53
+
}
54
+
55
+
bool want_audio = false;
56
+
bool want_visual = false;
57
+
58
+
// spec: the runtime must ensure no duplicates in requestedDataSources
59
+
for (uint32_t i = 0; i < createInfo->requestedDataSourceCount; i++) {
60
+
if (createInfo->requestedDataSources[i] == XR_FACE_TRACKING_DATA_SOURCE2_AUDIO_FB) {
61
+
if (!supports_audio) {
62
+
return oxr_error(log, XR_ERROR_FEATURE_UNSUPPORTED, "Audio source not supported");
63
+
}
64
+
if (want_audio) {
65
+
return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, "Duplicate entry for data source");
66
+
}
67
+
want_audio = true;
68
+
} else if (createInfo->requestedDataSources[i] == XR_FACE_TRACKING_DATA_SOURCE2_VISUAL_FB) {
69
+
if (!supports_visual) {
70
+
return oxr_error(log, XR_ERROR_FEATURE_UNSUPPORTED, "Visual source not supported");
71
+
}
72
+
if (want_visual) {
73
+
return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, "Duplicate entry for data source");
74
+
}
75
+
want_visual = true;
76
+
} else {
77
+
return oxr_error(log, XR_ERROR_FEATURE_UNSUPPORTED, "Unsupported data source");
78
+
}
79
+
}
80
+
81
+
// spec: if no data source is requested, select the highest fidelity available
82
+
if (!want_audio && !want_visual) {
83
+
if (supports_visual) {
84
+
want_visual = true;
85
+
} else {
86
+
want_audio = true;
87
+
}
88
+
}
89
+
90
+
struct oxr_face_tracker2_fb *face_tracker2_fb = NULL;
91
+
OXR_ALLOCATE_HANDLE_OR_RETURN(log, face_tracker2_fb, OXR_XR_DEBUG_FTRACKER, oxr_face_tracker2_fb_destroy_cb,
92
+
&sess->handle);
93
+
94
+
face_tracker2_fb->sess = sess;
95
+
face_tracker2_fb->xdev = xdev;
96
+
face_tracker2_fb->audio_enabled = want_audio;
97
+
face_tracker2_fb->visual_enabled = want_visual;
98
+
99
+
*out_face_tracker2_fb = face_tracker2_fb;
100
+
101
+
return XR_SUCCESS;
102
+
}
103
+
104
+
XrResult
105
+
oxr_get_face_expression_weights2_fb(struct oxr_logger *log,
106
+
struct oxr_face_tracker2_fb *face_tracker2_fb,
107
+
const XrFaceExpressionInfo2FB *expression_info,
108
+
XrFaceExpressionWeights2FB *expression_weights)
109
+
{
110
+
if (expression_weights->weightCount < XRT_FACE_EXPRESSION2_COUNT_FB) {
111
+
return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, "weightCount != XR_FACE_EXPRESSION2_COUNT_FB");
112
+
}
113
+
if (expression_weights->confidenceCount < XRT_FACE_CONFIDENCE2_COUNT_FB) {
114
+
return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, "confidenceCount != XR_FACE_CONFIDENCE2_COUNT_FB");
115
+
}
116
+
struct xrt_facial_expression_set result = {0};
117
+
118
+
// spec: visual is allowed to use both camera and audio
119
+
enum xrt_input_name ft_input_name =
120
+
face_tracker2_fb->visual_enabled ? XRT_INPUT_FB_FACE_TRACKING2_VISUAL : XRT_INPUT_FB_FACE_TRACKING2_AUDIO;
121
+
122
+
xrt_result_t xres = xrt_device_get_face_tracking(face_tracker2_fb->xdev, ft_input_name, &result);
123
+
if (xres != XRT_SUCCESS) {
124
+
return XR_ERROR_RUNTIME_FAILURE;
125
+
}
126
+
127
+
expression_weights->isValid = result.face_expression_set2_fb.is_valid;
128
+
if (!expression_weights->isValid) {
129
+
return XR_SUCCESS;
130
+
}
131
+
132
+
expression_weights->isEyeFollowingBlendshapesValid =
133
+
result.face_expression_set2_fb.is_eye_following_blendshapes_valid;
134
+
135
+
const struct oxr_instance *inst = face_tracker2_fb->sess->sys->inst;
136
+
expression_weights->time =
137
+
time_state_monotonic_to_ts_ns(inst->timekeeping, result.face_expression_set2_fb.sample_time_ns);
138
+
139
+
expression_weights->dataSource = (XrFaceTrackingDataSource2FB)result.face_expression_set2_fb.data_source;
140
+
141
+
expression_weights->weightCount = XRT_FACE_EXPRESSION2_COUNT_FB;
142
+
expression_weights->confidenceCount = XRT_FACE_CONFIDENCE2_COUNT_FB;
143
+
144
+
memcpy(expression_weights->weights, result.face_expression_set2_fb.weights,
145
+
sizeof(float) * XRT_FACE_EXPRESSION2_COUNT_FB);
146
+
memcpy(expression_weights->confidences, result.face_expression_set2_fb.confidences,
147
+
sizeof(float) * XRT_FACE_CONFIDENCE2_COUNT_FB);
148
+
149
+
return XR_SUCCESS;
150
+
}
+3
src/xrt/state_trackers/oxr/oxr_instance.c
+3
src/xrt/state_trackers/oxr/oxr_instance.c
···
279
279
#ifdef OXR_HAVE_FB_body_tracking
280
280
.fb_body_tracking_enabled = extensions->FB_body_tracking,
281
281
#endif
282
+
#ifdef OXR_HAVE_FB_face_tracking2
283
+
.fb_face_tracking2_enabled = extensions->FB_face_tracking2,
284
+
#endif
282
285
};
283
286
snprintf(i_info.application_name, sizeof(inst->xinst->instance_info.application_name), "%s",
284
287
createInfo->applicationInfo.applicationName);
+56
-1
src/xrt/state_trackers/oxr/oxr_objects.h
+56
-1
src/xrt/state_trackers/oxr/oxr_objects.h
···
122
122
struct oxr_action_ref;
123
123
struct oxr_hand_tracker;
124
124
struct oxr_facial_tracker_htc;
125
-
struct oxr_facial_tracker_fb;
125
+
struct oxr_face_tracker2_fb;
126
126
struct oxr_body_tracker_fb;
127
127
struct oxr_xdev_list;
128
128
···
413
413
}
414
414
#endif
415
415
416
+
#ifdef OXR_HAVE_FB_face_tracking2
417
+
/*!
418
+
* To go back to a OpenXR object.
419
+
*
420
+
* @relates oxr_face_tracker2_fb
421
+
*/
422
+
static inline XrFaceTracker2FB
423
+
oxr_face_tracker2_fb_to_openxr(struct oxr_face_tracker2_fb *face_tracker2_fb)
424
+
{
425
+
return XRT_CAST_PTR_TO_OXR_HANDLE(XrFaceTracker2FB, face_tracker2_fb);
426
+
}
427
+
#endif
416
428
/*!
417
429
*
418
430
* @name oxr_input.c
···
1034
1046
struct oxr_instance *inst,
1035
1047
bool *supports_eye,
1036
1048
bool *supports_lip);
1049
+
1050
+
void
1051
+
oxr_system_get_face_tracking2_fb_support(struct oxr_logger *log,
1052
+
struct oxr_instance *inst,
1053
+
bool *supports_audio,
1054
+
bool *supports_visual);
1037
1055
1038
1056
bool
1039
1057
oxr_system_get_body_tracking_fb_support(struct oxr_logger *log, struct oxr_instance *inst);
···
2742
2760
struct oxr_space *base_spc,
2743
2761
const XrBodyJointsLocateInfoFB *locateInfo,
2744
2762
XrBodyJointLocationsFB *locations);
2763
+
#endif
2764
+
2765
+
#ifdef OXR_HAVE_FB_face_tracking2
2766
+
/*!
2767
+
* FB specific Face tracker2.
2768
+
*
2769
+
* Parent type/handle is @ref oxr_instance
2770
+
*
2771
+
* @obj{XrFaceTracker2FB}
2772
+
* @extends oxr_handle_base
2773
+
*/
2774
+
struct oxr_face_tracker2_fb
2775
+
{
2776
+
//! Common structure for things referred to by OpenXR handles.
2777
+
struct oxr_handle_base handle;
2778
+
2779
+
//! Owner of this face tracker.
2780
+
struct oxr_session *sess;
2781
+
2782
+
//! xrt_device backing this face tracker
2783
+
struct xrt_device *xdev;
2784
+
2785
+
bool audio_enabled;
2786
+
bool visual_enabled;
2787
+
};
2788
+
2789
+
XrResult
2790
+
oxr_face_tracker2_fb_create(struct oxr_logger *log,
2791
+
struct oxr_session *sess,
2792
+
const XrFaceTrackerCreateInfo2FB *createInfo,
2793
+
struct oxr_face_tracker2_fb **out_face_tracker2_fb);
2794
+
2795
+
XrResult
2796
+
oxr_get_face_expression_weights2_fb(struct oxr_logger *log,
2797
+
struct oxr_face_tracker2_fb *face_tracker2_fb,
2798
+
const XrFaceExpressionInfo2FB *expression_info,
2799
+
XrFaceExpressionWeights2FB *expression_weights);
2745
2800
#endif
2746
2801
2747
2802
#ifdef OXR_HAVE_MNDX_xdev_space
+3
src/xrt/state_trackers/oxr/oxr_session.c
+3
src/xrt/state_trackers/oxr/oxr_session.c
···
237
237
#ifdef OXR_HAVE_FB_body_tracking
238
238
.fb_body_tracking_enabled = extensions->FB_body_tracking,
239
239
#endif
240
+
#ifdef OXR_HAVE_FB_face_tracking2
241
+
.fb_face_tracking2_enabled = extensions->FB_face_tracking2,
242
+
#endif
240
243
};
241
244
242
245
xrt_result_t xret = xrt_comp_begin_session(xc, &begin_session_info);
+45
src/xrt/state_trackers/oxr/oxr_system.c
+45
src/xrt/state_trackers/oxr/oxr_system.c
···
317
317
}
318
318
}
319
319
320
+
void
321
+
oxr_system_get_face_tracking2_fb_support(struct oxr_logger *log,
322
+
struct oxr_instance *inst,
323
+
bool *supports_audio,
324
+
bool *supports_visual)
325
+
{
326
+
if (supports_audio != NULL)
327
+
*supports_audio = false;
328
+
329
+
if (supports_visual != NULL)
330
+
*supports_visual = false;
331
+
332
+
struct oxr_system *sys = &inst->system;
333
+
struct xrt_device *face_xdev = GET_XDEV_BY_ROLE(sys, face);
334
+
335
+
if (face_xdev == NULL || !face_xdev->face_tracking_supported || face_xdev->inputs == NULL) {
336
+
return;
337
+
}
338
+
339
+
for (size_t input_idx = 0; input_idx < face_xdev->input_count; ++input_idx) {
340
+
const struct xrt_input *input = &face_xdev->inputs[input_idx];
341
+
if (input->name == XRT_INPUT_FB_FACE_TRACKING2_AUDIO && supports_audio != NULL) {
342
+
*supports_audio = true;
343
+
} else if (input->name == XRT_INPUT_FB_FACE_TRACKING2_VISUAL && supports_visual != NULL) {
344
+
*supports_visual = true;
345
+
}
346
+
}
347
+
return;
348
+
}
349
+
320
350
static bool
321
351
oxr_system_get_body_tracking_support(struct oxr_logger *log,
322
352
struct oxr_instance *inst,
···
448
478
body_tracking_fb_props->supportsBodyTracking = oxr_system_get_body_tracking_fb_support(log, sys->inst);
449
479
}
450
480
#endif // OXR_HAVE_FB_body_tracking
481
+
482
+
#ifdef OXR_HAVE_FB_face_tracking2
483
+
XrSystemFaceTrackingProperties2FB *face_tracking2_fb_props = NULL;
484
+
if (sys->inst->extensions.FB_face_tracking2) {
485
+
face_tracking2_fb_props = OXR_GET_OUTPUT_FROM_CHAIN(
486
+
properties, XR_TYPE_SYSTEM_FACE_TRACKING_PROPERTIES2_FB, XrSystemFaceTrackingProperties2FB);
487
+
}
488
+
489
+
if (face_tracking2_fb_props) {
490
+
bool supports_audio, supports_visual;
491
+
oxr_system_get_face_tracking2_fb_support(log, sys->inst, &supports_audio, &supports_visual);
492
+
face_tracking2_fb_props->supportsAudioFaceTracking = supports_audio;
493
+
face_tracking2_fb_props->supportsVisualFaceTracking = supports_visual;
494
+
}
495
+
#endif // OXR_HAVE_FB_face_tracking2
451
496
452
497
453
498
#ifdef OXR_HAVE_MNDX_xdev_space