personal memory agent
Add ArUco mask visualization to transcripts viewer
Pass aruco data from backend to frontend and render black polygon
overlay on thumbnails and full-frame viewer when masked=true.
- routes.py: Include aruco field in source_ref
- workspace.html: Add _computeArucoMaskPolygon() to extract polygon
from corner tag markers (IDs 2,4,6,7)
- Apply mask overlay in both drawThumbnail() and drawFull()
- Pass aruco data through render functions via data attributes
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
+66
-6
+1
apps/transcripts/routes.py
+1
apps/transcripts/routes.py
+65
-6
apps/transcripts/workspace.html
+65
-6
apps/transcripts/workspace.html
···
975
975
976
976
// Draw thumbnail to canvas with optional annotations
977
977
async drawThumbnail(canvas, videoUrl, frameId, options = {}) {
978
-
const { width, height } = options;
978
+
const { width, height, aruco } = options;
979
979
980
980
try {
981
981
const thumb = await this.captureThumbnail(videoUrl, frameId, width, height);
···
990
990
const ctx = canvas.getContext('2d');
991
991
ctx.drawImage(thumb, 0, 0, canvas.width, canvas.height);
992
992
993
+
// Apply aruco mask overlay if present
994
+
// Note: thumb is at target size, so sourceWidth/Height = canvas dimensions
995
+
if (aruco) {
996
+
// Get original video dimensions for proper scaling
997
+
const entry = await this.loadVideo(videoUrl);
998
+
this._applyOverlays(ctx, canvas, entry.width, entry.height, { aruco });
999
+
}
1000
+
993
1001
canvas.classList.remove('loading');
994
1002
return true;
995
1003
} catch (err) {
···
1001
1009
1002
1010
// Draw full-resolution frame to canvas (no caching)
1003
1011
async drawFull(canvas, videoUrl, frameId, options = {}) {
1004
-
const { boxCoords, participants } = options;
1012
+
const { boxCoords, participants, aruco } = options;
1005
1013
1006
1014
try {
1007
1015
const bitmap = await this.captureFullFrame(videoUrl, frameId);
···
1016
1024
const ctx = canvas.getContext('2d');
1017
1025
ctx.drawImage(bitmap, 0, 0, canvas.width, canvas.height);
1018
1026
1019
-
this._applyOverlays(ctx, canvas, bitmap.width, bitmap.height, boxCoords, participants);
1027
+
this._applyOverlays(ctx, canvas, bitmap.width, bitmap.height, { boxCoords, participants, aruco });
1020
1028
1021
1029
if (bitmap && typeof bitmap.close === 'function') {
1022
1030
bitmap.close();
···
1030
1038
}
1031
1039
}
1032
1040
1033
-
_applyOverlays(ctx, canvas, sourceWidth, sourceHeight, boxCoords, participants) {
1041
+
// Compute mask polygon from ArUco corner tag markers
1042
+
// Corner tag IDs: 6=TL, 7=TR, 2=BR, 4=BL
1043
+
// Each marker has corners in order [TL, TR, BR, BL]
1044
+
_computeArucoMaskPolygon(aruco) {
1045
+
if (!aruco || !aruco.masked || !aruco.markers) return null;
1046
+
1047
+
const cornerTagIds = { 6: 0, 7: 1, 2: 2, 4: 3 }; // id -> which corner to use
1048
+
const tagCorners = {};
1049
+
1050
+
for (const marker of aruco.markers) {
1051
+
if (marker.id in cornerTagIds && marker.corners?.length === 4) {
1052
+
const cornerIdx = cornerTagIds[marker.id];
1053
+
tagCorners[marker.id] = marker.corners[cornerIdx];
1054
+
}
1055
+
}
1056
+
1057
+
// Need all 4 corner tags
1058
+
if (!(6 in tagCorners && 7 in tagCorners && 2 in tagCorners && 4 in tagCorners)) {
1059
+
return null;
1060
+
}
1061
+
1062
+
// Return polygon: TL, TR, BR, BL
1063
+
return [tagCorners[6], tagCorners[7], tagCorners[2], tagCorners[4]];
1064
+
}
1065
+
1066
+
_applyOverlays(ctx, canvas, sourceWidth, sourceHeight, options = {}) {
1067
+
const { boxCoords, participants, aruco } = options;
1068
+
1069
+
// Apply ArUco mask first (so other overlays draw on top)
1070
+
const maskPolygon = this._computeArucoMaskPolygon(aruco);
1071
+
if (maskPolygon) {
1072
+
const scaleX = canvas.width / sourceWidth;
1073
+
const scaleY = canvas.height / sourceHeight;
1074
+
1075
+
ctx.fillStyle = '#000000';
1076
+
ctx.beginPath();
1077
+
ctx.moveTo(maskPolygon[0][0] * scaleX, maskPolygon[0][1] * scaleY);
1078
+
for (let i = 1; i < maskPolygon.length; i++) {
1079
+
ctx.lineTo(maskPolygon[i][0] * scaleX, maskPolygon[i][1] * scaleY);
1080
+
}
1081
+
ctx.closePath();
1082
+
ctx.fill();
1083
+
}
1084
+
1034
1085
if (boxCoords && boxCoords.length === 4) {
1035
1086
const [xMin, yMin, xMax, yMax] = boxCoords;
1036
1087
const scaleX = canvas.width / sourceWidth;
···
1519
1570
const frameId = parseInt(canvas.dataset.frameId, 10);
1520
1571
const boxCoordsStr = canvas.dataset.boxCoords;
1521
1572
const boxCoords = boxCoordsStr ? JSON.parse(boxCoordsStr) : null;
1573
+
const arucoStr = canvas.dataset.aruco;
1574
+
const aruco = arucoStr ? JSON.parse(arucoStr) : null;
1522
1575
1523
1576
if (!videoUrl || isNaN(frameId)) {
1524
1577
canvas.classList.remove('loading');
···
1528
1581
// Draw at thumbnail size (120x68)
1529
1582
frameCapture.drawThumbnail(canvas, videoUrl, frameId, {
1530
1583
boxCoords,
1584
+
aruco,
1531
1585
width: 120,
1532
1586
height: 68
1533
1587
});
···
1606
1660
const videoUrl = getVideoUrlForChunk(entry);
1607
1661
const frameId = entry.source_ref?.frame_id;
1608
1662
const boxCoords = entry.source_ref?.box_2d;
1663
+
const aruco = entry.source_ref?.aruco;
1609
1664
const analysis = entry.source_ref?.analysis || {};
1610
1665
const category = analysis.primary || 'unknown';
1611
1666
const description = analysis.visual_description || category;
···
1615
1670
html += `<div class="tr-group-item" data-frame-idx="${frameIdx}" title="${escapeHtml(description)}">`;
1616
1671
html += `<canvas class="loading" data-video-url="${escapeHtml(videoUrl)}" data-frame-id="${frameId}"`;
1617
1672
if (boxCoords) html += ` data-box-coords='${JSON.stringify(boxCoords)}'`;
1673
+
if (aruco) html += ` data-aruco='${JSON.stringify(aruco)}'`;
1618
1674
html += `></canvas>`;
1619
1675
html += `<span class="tr-group-item-badge">${escapeHtml(category)}</span>`;
1620
1676
html += '</div>';
···
1632
1688
const videoUrl = getVideoUrlForChunk(chunk);
1633
1689
const frameId = chunk.source_ref?.frame_id;
1634
1690
const boxCoords = chunk.source_ref?.box_2d;
1691
+
const aruco = chunk.source_ref?.aruco;
1635
1692
const frameIdx = findFrameIndex(chunk);
1636
1693
1637
1694
let html = `<div class="tr-entry tr-entry-screen" data-idx="${idx}" data-frame-idx="${frameIdx}" data-type="screen">`;
···
1641
1698
if (videoUrl && frameId) {
1642
1699
html += `<canvas class="tr-entry-thumb loading" data-video-url="${escapeHtml(videoUrl)}" data-frame-id="${frameId}"`;
1643
1700
if (boxCoords) html += ` data-box-coords='${JSON.stringify(boxCoords)}'`;
1701
+
if (aruco) html += ` data-aruco='${JSON.stringify(aruco)}'`;
1644
1702
html += `></canvas>`;
1645
1703
}
1646
1704
···
1712
1770
const videoUrl = getVideoUrlForChunk(f);
1713
1771
const frameId = f.source_ref?.frame_id;
1714
1772
const boxCoords = f.source_ref?.box_2d;
1773
+
const aruco = f.source_ref?.aruco;
1715
1774
1716
1775
if (videoUrl && frameId) {
1717
1776
frameCapture.drawFull(canvas, videoUrl, frameId, {
1718
1777
boxCoords,
1719
-
participants
1720
-
// No width/height - use full resolution
1778
+
participants,
1779
+
aruco
1721
1780
});
1722
1781
} else {
1723
1782
canvas.classList.remove('loading');