The open source OpenXR runtime
c/main: Hook up layer squasher in main compositor
Co-authored-by: Jakob Bornecrantz <jakob@collabora.com>
Co-authored-by: Fernando Velazquez Innella <finnella@magicleap.com>
authored by
Christoph Haag
Jakob Bornecrantz
Fernando Velazquez Innella
and committed by
Jakob Bornecrantz
f6821402
95fb034b
+370
-7
+370
-7
src/xrt/compositor/main/comp_renderer.c
+370
-7
src/xrt/compositor/main/comp_renderer.c
···
10
10
* @ingroup comp_main
11
11
*/
12
12
13
+
#include "render/render_interface.h"
13
14
#include "xrt/xrt_defines.h"
14
15
#include "xrt/xrt_frame.h"
15
16
#include "xrt/xrt_compositor.h"
···
970
971
*/
971
972
972
973
static void
973
-
get_view_poses(struct comp_renderer *r, struct xrt_pose out_results[2])
974
+
get_view_poses(struct comp_renderer *r, struct xrt_pose out_world[2], struct xrt_pose out_eye[2])
974
975
{
975
976
COMP_TRACE_MARKER();
976
977
···
1007
1008
m_relation_chain_push_pose_if_not_identity(&xrc, &base_space_pose);
1008
1009
m_relation_chain_resolve(&xrc, &result);
1009
1010
1010
-
out_results[i] = result.pose;
1011
+
out_eye[i] = eye_pose;
1012
+
out_world[i] = result.pose;
1013
+
}
1014
+
}
1015
+
1016
+
static void
1017
+
ensure_scratch_image(struct comp_renderer *r,
1018
+
struct render_viewport_data *out_l_viewport_data,
1019
+
struct render_viewport_data *out_r_viewport_data)
1020
+
{
1021
+
struct xrt_view *l_v = &r->c->xdev->hmd->views[0];
1022
+
struct xrt_view *r_v = &r->c->xdev->hmd->views[1];
1023
+
1024
+
uint32_t w = MAX(l_v->viewport.w_pixels, r_v->viewport.w_pixels);
1025
+
uint32_t h = MAX(l_v->viewport.h_pixels, r_v->viewport.h_pixels);
1026
+
1027
+
// Adjust size to be bigger, 140%, to match default recommended viewport size.
1028
+
//! @todo Make this match fully, or even match app provided layers.
1029
+
w = (uint32_t)(w * 1.4f);
1030
+
h = (uint32_t)(h * 1.4f);
1031
+
1032
+
struct render_viewport_data l_viewport_data = {
1033
+
.w = w,
1034
+
.h = h,
1035
+
.x = 0,
1036
+
.y = 0,
1037
+
};
1038
+
1039
+
struct render_viewport_data r_viewport_data = {
1040
+
.w = w,
1041
+
.h = h,
1042
+
.x = w,
1043
+
.y = 0,
1044
+
};
1045
+
1046
+
VkExtent2D extent = {w * 2, h};
1047
+
1048
+
if (!render_ensure_scratch_image(&r->c->nr, extent)) {
1049
+
U_LOG_E("Failed to create scratch image!");
1050
+
assert(false);
1011
1051
}
1052
+
1053
+
*out_l_viewport_data = l_viewport_data;
1054
+
*out_r_viewport_data = r_viewport_data;
1055
+
}
1056
+
1057
+
static void
1058
+
do_layers(struct comp_renderer *r,
1059
+
struct render_compute *crc,
1060
+
const struct comp_layer *layers,
1061
+
const uint32_t layer_count)
1062
+
{
1063
+
struct render_viewport_data views[2];
1064
+
1065
+
// Create scratch image and get target views.
1066
+
ensure_scratch_image(r, &views[0], &views[1]);
1067
+
1068
+
struct render_compute_layer_ubo_data *ubo_data =
1069
+
(struct render_compute_layer_ubo_data *)crc->r->compute.layer.ubo.mapped;
1070
+
1071
+
for (uint32_t i = 0; i < 2; i++) {
1072
+
ubo_data->views[i] = views[i];
1073
+
}
1074
+
1075
+
ubo_data->pre_transforms[0] = crc->r->distortion.uv_to_tanangle[0];
1076
+
ubo_data->pre_transforms[1] = crc->r->distortion.uv_to_tanangle[1];
1077
+
1078
+
VkImage target_image = crc->r->scratch.color.image;
1079
+
VkImageView target_image_view = crc->r->scratch.color.unorm_view; // Have to write in linear
1080
+
1081
+
struct xrt_pose world_poses[2], eye_poses[2];
1082
+
get_view_poses(r, world_poses, eye_poses);
1083
+
1084
+
// Not the transform of the views, but the inverse: actual view matrices.
1085
+
struct xrt_matrix_4x4 world_view_mats[2], eye_view_mats[2];
1086
+
math_matrix_4x4_view_from_pose(&world_poses[0], &world_view_mats[0]);
1087
+
math_matrix_4x4_view_from_pose(&world_poses[1], &world_view_mats[1]);
1088
+
math_matrix_4x4_view_from_pose(&eye_poses[0], &eye_view_mats[0]);
1089
+
math_matrix_4x4_view_from_pose(&eye_poses[1], &eye_view_mats[1]);
1090
+
1091
+
// Tightly pack color and optional depth images.
1092
+
uint32_t cur_image = 0;
1093
+
VkSampler src_samplers[COMP_MAX_IMAGES];
1094
+
VkImageView src_image_views[COMP_MAX_IMAGES];
1095
+
1096
+
for (uint32_t layer_i = 0; layer_i < layer_count; layer_i++) {
1097
+
const struct xrt_layer_data *data = &layers[layer_i].data;
1098
+
const struct comp_layer *layer = &layers[layer_i];
1099
+
1100
+
ubo_data->layer_type[layer_i].val = data->type;
1101
+
ubo_data->layer_type[layer_i].unpremultiplied =
1102
+
(data->flags & XRT_LAYER_COMPOSITION_UNPREMULTIPLIED_ALPHA_BIT) != 0;
1103
+
1104
+
// Base index into arrays that have a value per view & per layer.
1105
+
uint32_t view_index_for_layer = layer_i * COMP_VIEWS_PER_LAYER;
1106
+
1107
+
//! Stop compositing layers if device's sampled image limit is reached.
1108
+
//! This is necessary until composition can be split in multiple passes.
1109
+
//! @todo: remove this after multi-pass composition is implemented.
1110
+
uint32_t required_image_samplers;
1111
+
switch (data->type) {
1112
+
case XRT_LAYER_STEREO_PROJECTION: required_image_samplers = 2; break;
1113
+
case XRT_LAYER_STEREO_PROJECTION_DEPTH: required_image_samplers = 4; break;
1114
+
case XRT_LAYER_QUAD: required_image_samplers = 1; break;
1115
+
default: required_image_samplers = 0;
1116
+
}
1117
+
//! Exit loop if shader cannot receive more image samplers
1118
+
if (cur_image + required_image_samplers >
1119
+
crc->r->vk->features.max_per_stage_descriptor_sampled_images) {
1120
+
for (uint32_t i = layer_i; i < layer_count; i++) {
1121
+
ubo_data->layer_type[i].val = UINT32_MAX; //! @todo make this not needed.
1122
+
}
1123
+
break;
1124
+
}
1125
+
1126
+
1127
+
switch (data->type) {
1128
+
case XRT_LAYER_STEREO_PROJECTION_DEPTH:
1129
+
case XRT_LAYER_STEREO_PROJECTION: {
1130
+
const struct xrt_layer_projection_view_data *lvd = NULL;
1131
+
const struct xrt_layer_projection_view_data *rvd = NULL;
1132
+
const struct xrt_layer_depth_data *l_dvd = NULL;
1133
+
const struct xrt_layer_depth_data *r_dvd = NULL;
1134
+
1135
+
if (data->type == XRT_LAYER_STEREO_PROJECTION) {
1136
+
const struct xrt_layer_stereo_projection_data *stereo = &layer->data.stereo;
1137
+
lvd = &stereo->l;
1138
+
rvd = &stereo->r;
1139
+
} else {
1140
+
const struct xrt_layer_stereo_projection_depth_data *stereo = &layer->data.stereo_depth;
1141
+
lvd = &stereo->l;
1142
+
rvd = &stereo->r;
1143
+
l_dvd = &stereo->l_d;
1144
+
r_dvd = &stereo->r_d;
1145
+
}
1146
+
1147
+
uint32_t left_array_index = lvd->sub.array_index;
1148
+
uint32_t right_array_index = rvd->sub.array_index;
1149
+
const struct comp_swapchain_image *left = &layer->sc_array[0]->images[lvd->sub.image_index];
1150
+
const struct comp_swapchain_image *right = &layer->sc_array[1]->images[rvd->sub.image_index];
1151
+
1152
+
// Left
1153
+
src_samplers[cur_image] = left->sampler;
1154
+
src_image_views[cur_image] = get_image_view(left, data->flags, left_array_index);
1155
+
ubo_data->images_samplers[view_index_for_layer + 0].images[0] = cur_image++;
1156
+
1157
+
// Right
1158
+
src_samplers[cur_image] = right->sampler;
1159
+
src_image_views[cur_image] = get_image_view(right, data->flags, right_array_index);
1160
+
ubo_data->images_samplers[view_index_for_layer + 1].images[0] = cur_image++;
1161
+
1162
+
// Depth
1163
+
if (data->type == XRT_LAYER_STEREO_PROJECTION_DEPTH) {
1164
+
uint32_t d_left_array_index = lvd->sub.array_index;
1165
+
uint32_t d_right_array_index = rvd->sub.array_index;
1166
+
const struct comp_swapchain_image *d_left =
1167
+
&layer->sc_array[2]->images[l_dvd->sub.image_index];
1168
+
const struct comp_swapchain_image *d_right =
1169
+
&layer->sc_array[3]->images[r_dvd->sub.image_index];
1170
+
1171
+
// Depth left
1172
+
src_samplers[cur_image] = d_left->sampler;
1173
+
src_image_views[cur_image] = get_image_view(d_left, data->flags, d_left_array_index);
1174
+
ubo_data->images_samplers[view_index_for_layer + 0].images[1] = cur_image++;
1175
+
1176
+
// Depth right
1177
+
src_samplers[cur_image] = d_right->sampler;
1178
+
src_image_views[cur_image] = get_image_view(d_right, data->flags, d_right_array_index);
1179
+
ubo_data->images_samplers[view_index_for_layer + 1].images[1] = cur_image++;
1180
+
}
1181
+
1182
+
struct xrt_normalized_rect *post_transforms = &ubo_data->post_transforms[view_index_for_layer];
1183
+
post_transforms[0] = lvd->sub.norm_rect;
1184
+
post_transforms[1] = rvd->sub.norm_rect;
1185
+
if (data->flip_y) {
1186
+
post_transforms[0].h = -post_transforms[0].h;
1187
+
post_transforms[0].y = 1.0f + post_transforms[0].y;
1188
+
post_transforms[1].h = -post_transforms[1].h;
1189
+
post_transforms[1].y = 1.0f + post_transforms[1].y;
1190
+
}
1191
+
1192
+
// unused if timewarp is off
1193
+
if (!r->c->debug.atw_off) {
1194
+
render_calc_time_warp_matrix( //
1195
+
&lvd->pose, //
1196
+
&lvd->fov, //
1197
+
&world_poses[0], //
1198
+
&ubo_data->transforms[view_index_for_layer + 0]); //
1199
+
render_calc_time_warp_matrix( //
1200
+
&rvd->pose, //
1201
+
&rvd->fov, //
1202
+
&world_poses[1], //
1203
+
&ubo_data->transforms[view_index_for_layer + 1]); //
1204
+
}
1205
+
1206
+
} break;
1207
+
case XRT_LAYER_QUAD: {
1208
+
const struct xrt_layer_quad_data *q = &layer->data.quad;
1209
+
const struct comp_swapchain_image *image = &layer->sc_array[0]->images[q->sub.image_index];
1210
+
uint32_t array_index = q->sub.array_index;
1211
+
1212
+
// Same image for both views
1213
+
src_samplers[cur_image] = image->sampler;
1214
+
src_image_views[cur_image] = get_image_view(image, layer->data.flags, array_index);
1215
+
ubo_data->images_samplers[view_index_for_layer + 0].images[0] = cur_image;
1216
+
ubo_data->images_samplers[view_index_for_layer + 1].images[0] = cur_image;
1217
+
cur_image++;
1218
+
1219
+
1220
+
struct xrt_normalized_rect *post_transforms = &ubo_data->post_transforms[view_index_for_layer];
1221
+
1222
+
// Same image for both views
1223
+
post_transforms[0] = q->sub.norm_rect;
1224
+
post_transforms[1] = q->sub.norm_rect;
1225
+
1226
+
// quad layers calculated in flipped space than projection layers.
1227
+
// Note: different y flip logic compared to projection layers.
1228
+
if (!data->flip_y) {
1229
+
post_transforms[0].h = -post_transforms[0].h;
1230
+
post_transforms[0].y = post_transforms[0].y - post_transforms[0].h;
1231
+
post_transforms[1].h = -post_transforms[1].h;
1232
+
post_transforms[1].y = post_transforms[1].y - post_transforms[1].h;
1233
+
}
1234
+
1235
+
ubo_data->quad_extent[layer_i].val = data->quad.size;
1236
+
1237
+
// Is this layer viewspace or not.
1238
+
const struct xrt_matrix_4x4 *view_mats =
1239
+
(layer->data.flags & XRT_LAYER_COMPOSITION_VIEW_SPACE_BIT) ? eye_view_mats
1240
+
: world_view_mats;
1241
+
1242
+
for (uint32_t view_i = 0; view_i < 2; view_i++) {
1243
+
// transform quad pose into view space for each view
1244
+
math_matrix_4x4_transform_vec3(
1245
+
&view_mats[view_i], &data->quad.pose.position,
1246
+
&ubo_data->quad_position[view_index_for_layer + view_i].val);
1247
+
1248
+
// neutral quad layer faces +z, towards the user
1249
+
struct xrt_vec3 normal = (struct xrt_vec3){.x = 0, .y = 0, .z = 1};
1250
+
1251
+
// rotation of the quad normal in world space
1252
+
struct xrt_quat rotation = data->quad.pose.orientation;
1253
+
math_quat_rotate_vec3(&rotation, &normal, &normal);
1254
+
1255
+
/*
1256
+
* normal is a vector that originates on the plane, not on the origin.
1257
+
* Instead of using the inverse quad transform to transform it into view space we can
1258
+
* simply add up vectors:
1259
+
*
1260
+
* combined_normal [in world space] = plane_origin [in world space] + normal [in plane
1261
+
* space] [with plane in world space]
1262
+
*
1263
+
* Then combined_normal can be transformed to view space via view matrix and a new
1264
+
* normal_view_space retrieved:
1265
+
*
1266
+
* normal_view_space = combined_normal [in view space] - plane_origin [in view space]
1267
+
*/
1268
+
struct xrt_vec3 normal_view_space = normal;
1269
+
math_vec3_accum(&data->quad.pose.position, &normal_view_space);
1270
+
math_matrix_4x4_transform_vec3(&view_mats[view_i], &normal_view_space,
1271
+
&normal_view_space);
1272
+
math_vec3_subtract(&ubo_data->quad_position[view_index_for_layer + view_i].val,
1273
+
&normal_view_space);
1274
+
ubo_data->quad_normal[view_index_for_layer + view_i].val = normal_view_space;
1275
+
1276
+
1277
+
struct xrt_vec3 scale = {1.f, 1.f, 1.f};
1278
+
struct xrt_matrix_4x4 plane_transform_view_space;
1279
+
math_matrix_4x4_model(&data->quad.pose, &scale, &plane_transform_view_space);
1280
+
math_matrix_4x4_multiply(&view_mats[view_i], &plane_transform_view_space,
1281
+
&plane_transform_view_space);
1282
+
math_matrix_4x4_inverse(
1283
+
&plane_transform_view_space,
1284
+
&ubo_data->inverse_quad_transform[view_index_for_layer + view_i]);
1285
+
}
1286
+
1287
+
// hide a quad layer by pointing its normal away from the camera in view space
1288
+
struct xrt_vec3 hidden_normal = {.x = 0, .y = 0, .z = -1};
1289
+
switch (q->visibility) {
1290
+
case XRT_LAYER_EYE_VISIBILITY_NONE:
1291
+
ubo_data->quad_normal[view_index_for_layer + 0].val = hidden_normal;
1292
+
ubo_data->quad_normal[view_index_for_layer + 1].val = hidden_normal;
1293
+
break;
1294
+
case XRT_LAYER_EYE_VISIBILITY_LEFT_BIT:
1295
+
ubo_data->quad_normal[view_index_for_layer + 1].val = hidden_normal;
1296
+
break;
1297
+
case XRT_LAYER_EYE_VISIBILITY_RIGHT_BIT:
1298
+
ubo_data->quad_normal[view_index_for_layer + 0].val = hidden_normal;
1299
+
break;
1300
+
case XRT_LAYER_EYE_VISIBILITY_BOTH: break;
1301
+
}
1302
+
1303
+
} break;
1304
+
default:
1305
+
COMP_ERROR(r->c, "Layer type %d not supported by compute shader, skipping", data->type);
1306
+
ubo_data->layer_type[layer_i].val = UINT32_MAX;
1307
+
}
1308
+
}
1309
+
1310
+
for (uint32_t i = layer_count; i < COMP_MAX_LAYERS; i++) {
1311
+
ubo_data->layer_type[i].val = UINT32_MAX;
1312
+
}
1313
+
1314
+
//! @todo: If Vulkan 1.2, use VK_DESCRIPTOR_BINDING_PARTIALLY_BOUND_BIT and skip this
1315
+
while (cur_image < crc->r->compute.layer.image_array_size) {
1316
+
src_samplers[cur_image] = crc->r->compute.default_sampler;
1317
+
src_image_views[cur_image] = crc->r->mock.color.image_view;
1318
+
cur_image++;
1319
+
}
1320
+
1321
+
render_compute_layers( //
1322
+
crc, //
1323
+
src_samplers, //
1324
+
src_image_views, //
1325
+
cur_image, //
1326
+
target_image, //
1327
+
target_image_view, //
1328
+
VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, //
1329
+
!r->c->debug.atw_off); //
1330
+
}
1331
+
1332
+
static void
1333
+
do_distortion(struct comp_renderer *r, struct render_compute *crc, const struct render_viewport_data views[2])
1334
+
{
1335
+
VkImage target_image = r->c->target->images[r->acquired_buffer].handle;
1336
+
VkImageView target_image_view = r->c->target->images[r->acquired_buffer].view;
1337
+
1338
+
VkImageView view = crc->r->scratch.color.srgb_view; // Read with gamma curve.
1339
+
VkSampler sampler = crc->r->compute.default_sampler;
1340
+
1341
+
VkImageView src_image_views[2] = {view, view};
1342
+
VkSampler src_samplers[2] = {sampler, sampler};
1343
+
1344
+
struct xrt_normalized_rect src_norm_rects[2] = {
1345
+
{
1346
+
// Left, takes up half the screen.
1347
+
.x = 0.0f,
1348
+
.y = 0.0f,
1349
+
.w = 0.5f,
1350
+
.h = 1.0f,
1351
+
},
1352
+
{
1353
+
// Right, takes up half the screen.
1354
+
.x = 0.5f,
1355
+
.y = 0.0f,
1356
+
.w = 0.5f,
1357
+
.h = 1.0f,
1358
+
},
1359
+
};
1360
+
1361
+
render_compute_projection( //
1362
+
crc, //
1363
+
src_samplers, //
1364
+
src_image_views, //
1365
+
src_norm_rects, //
1366
+
target_image, //
1367
+
target_image_view, //
1368
+
views //
1369
+
);
1012
1370
}
1013
1371
1014
1372
static void
···
1030
1388
VkImage target_image = r->c->target->images[r->acquired_buffer].handle;
1031
1389
VkImageView target_image_view = r->c->target->images[r->acquired_buffer].view;
1032
1390
1033
-
struct xrt_pose new_view_poses[2];
1034
-
get_view_poses(r, new_view_poses);
1391
+
struct xrt_pose new_world_poses[2];
1392
+
struct xrt_pose unused[2]; // New eye poses, unused.
1393
+
get_view_poses(r, new_world_poses, unused);
1035
1394
1036
1395
VkSampler src_samplers[2] = {
1037
1396
left->sampler,
···
1078
1437
src_norm_rects, //
1079
1438
src_poses, //
1080
1439
src_fovs, //
1081
-
new_view_poses, //
1440
+
new_world_poses, //
1082
1441
target_image, //
1083
1442
target_image_view, //
1084
1443
views); //
···
1103
1462
VkImageView target_image_view = r->c->target->images[r->acquired_buffer].view;
1104
1463
1105
1464
uint32_t layer_count = c->base.slot.layer_count;
1106
-
if (layer_count > 0 && c->base.slot.layers[0].data.type == XRT_LAYER_STEREO_PROJECTION) {
1465
+
if (layer_count == 1 && c->base.slot.layers[0].data.type == XRT_LAYER_STEREO_PROJECTION) {
1107
1466
int i = 0;
1108
1467
const struct comp_layer *layer = &c->base.slot.layers[i];
1109
1468
const struct xrt_layer_stereo_projection_data *stereo = &layer->data.stereo;
···
1111
1470
const struct xrt_layer_projection_view_data *rvd = &stereo->r;
1112
1471
1113
1472
do_projection_layers(r, crc, layer, lvd, rvd);
1114
-
} else if (layer_count > 0 && c->base.slot.layers[0].data.type == XRT_LAYER_STEREO_PROJECTION_DEPTH) {
1473
+
} else if (layer_count == 1 && c->base.slot.layers[0].data.type == XRT_LAYER_STEREO_PROJECTION_DEPTH) {
1115
1474
int i = 0;
1116
1475
const struct comp_layer *layer = &c->base.slot.layers[i];
1117
1476
const struct xrt_layer_stereo_projection_depth_data *stereo = &layer->data.stereo_depth;
···
1119
1478
const struct xrt_layer_projection_view_data *rvd = &stereo->r;
1120
1479
1121
1480
do_projection_layers(r, crc, layer, lvd, rvd);
1481
+
} else if (layer_count > 0) {
1482
+
do_layers(r, crc, c->base.slot.layers, layer_count);
1483
+
1484
+
do_distortion(r, crc, views);
1122
1485
} else {
1123
1486
render_compute_clear( //
1124
1487
crc, //