A loose federation of distributed, typed datasets
1<!DOCTYPE html>
2<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
3
4<meta charset="utf-8">
5<meta name="generator" content="quarto-1.7.34">
6
7<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
8
9<meta name="description" content="Bidirectional transformations between sample types">
10
11<title>Lenses – atdata</title>
12<style>
13code{white-space: pre-wrap;}
14span.smallcaps{font-variant: small-caps;}
15div.columns{display: flex; gap: min(4vw, 1.5em);}
16div.column{flex: auto; overflow-x: auto;}
17div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
18ul.task-list{list-style: none;}
19ul.task-list li input[type="checkbox"] {
20 width: 0.8em;
21 margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
22 vertical-align: middle;
23}
24/* CSS for syntax highlighting */
25html { -webkit-text-size-adjust: 100%; }
26pre > code.sourceCode { white-space: pre; position: relative; }
27pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
28pre > code.sourceCode > span:empty { height: 1.2em; }
29.sourceCode { overflow: visible; }
30code.sourceCode > span { color: inherit; text-decoration: inherit; }
31div.sourceCode { margin: 1em 0; }
32pre.sourceCode { margin: 0; }
33@media screen {
34div.sourceCode { overflow: auto; }
35}
36@media print {
37pre > code.sourceCode { white-space: pre-wrap; }
38pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
39}
40pre.numberSource code
41 { counter-reset: source-line 0; }
42pre.numberSource code > span
43 { position: relative; left: -4em; counter-increment: source-line; }
44pre.numberSource code > span > a:first-child::before
45 { content: counter(source-line);
46 position: relative; left: -1em; text-align: right; vertical-align: baseline;
47 border: none; display: inline-block;
48 -webkit-touch-callout: none; -webkit-user-select: none;
49 -khtml-user-select: none; -moz-user-select: none;
50 -ms-user-select: none; user-select: none;
51 padding: 0 4px; width: 4em;
52 }
53pre.numberSource { margin-left: 3em; padding-left: 4px; }
54div.sourceCode
55 { }
56@media screen {
57pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
58}
59</style>
60
61
62<script src="../site_libs/quarto-nav/quarto-nav.js"></script>
63<script src="../site_libs/quarto-nav/headroom.min.js"></script>
64<script src="../site_libs/clipboard/clipboard.min.js"></script>
65<script src="../site_libs/quarto-search/autocomplete.umd.js"></script>
66<script src="../site_libs/quarto-search/fuse.min.js"></script>
67<script src="../site_libs/quarto-search/quarto-search.js"></script>
68<meta name="quarto:offset" content="../">
69<script src="../site_libs/quarto-html/quarto.js" type="module"></script>
70<script src="../site_libs/quarto-html/tabsets/tabsets.js" type="module"></script>
71<script src="../site_libs/quarto-html/popper.min.js"></script>
72<script src="../site_libs/quarto-html/tippy.umd.min.js"></script>
73<script src="../site_libs/quarto-html/anchor.min.js"></script>
74<link href="../site_libs/quarto-html/tippy.css" rel="stylesheet">
75<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme" id="quarto-text-highlighting-styles">
76<link href="../site_libs/quarto-html/quarto-syntax-highlighting-dark-8dcd8563ea6803ab7cbb3d71ca5772e1.css" rel="stylesheet" class="quarto-color-scheme quarto-color-alternate" id="quarto-text-highlighting-styles">
77<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme-extra" id="quarto-text-highlighting-styles">
78<script src="../site_libs/bootstrap/bootstrap.min.js"></script>
79<link href="../site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
80<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme" id="quarto-bootstrap" data-mode="light">
81<link href="../site_libs/bootstrap/bootstrap-dark-7964ffd8887b0991fe8d71c6c8bc75d6.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme quarto-color-alternate" id="quarto-bootstrap" data-mode="dark">
82<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme-extra" id="quarto-bootstrap" data-mode="light">
83<script id="quarto-search-options" type="application/json">{
84 "location": "navbar",
85 "copy-button": false,
86 "collapse-after": 3,
87 "panel-placement": "end",
88 "type": "overlay",
89 "limit": 50,
90 "keyboard-shortcut": [
91 "f",
92 "/",
93 "s"
94 ],
95 "show-item-context": false,
96 "language": {
97 "search-no-results-text": "No results",
98 "search-matching-documents-text": "matching documents",
99 "search-copy-link-title": "Copy link to search",
100 "search-hide-matches-text": "Hide additional matches",
101 "search-more-match-text": "more match in this document",
102 "search-more-matches-text": "more matches in this document",
103 "search-clear-button-title": "Clear",
104 "search-text-placeholder": "",
105 "search-detached-cancel-button-title": "Cancel",
106 "search-submit-button-title": "Submit",
107 "search-label": "Search"
108 }
109}</script>
110
111
112<link rel="stylesheet" href="../assets/styles.css">
113</head>
114
115<body class="nav-sidebar docked nav-fixed quarto-light"><script id="quarto-html-before-body" type="application/javascript">
116 const toggleBodyColorMode = (bsSheetEl) => {
117 const mode = bsSheetEl.getAttribute("data-mode");
118 const bodyEl = window.document.querySelector("body");
119 if (mode === "dark") {
120 bodyEl.classList.add("quarto-dark");
121 bodyEl.classList.remove("quarto-light");
122 } else {
123 bodyEl.classList.add("quarto-light");
124 bodyEl.classList.remove("quarto-dark");
125 }
126 }
127 const toggleBodyColorPrimary = () => {
128 const bsSheetEl = window.document.querySelector("link#quarto-bootstrap:not([rel=disabled-stylesheet])");
129 if (bsSheetEl) {
130 toggleBodyColorMode(bsSheetEl);
131 }
132 }
133 const setColorSchemeToggle = (alternate) => {
134 const toggles = window.document.querySelectorAll('.quarto-color-scheme-toggle');
135 for (let i=0; i < toggles.length; i++) {
136 const toggle = toggles[i];
137 if (toggle) {
138 if (alternate) {
139 toggle.classList.add("alternate");
140 } else {
141 toggle.classList.remove("alternate");
142 }
143 }
144 }
145 };
146 const toggleColorMode = (alternate) => {
147 // Switch the stylesheets
148 const primaryStylesheets = window.document.querySelectorAll('link.quarto-color-scheme:not(.quarto-color-alternate)');
149 const alternateStylesheets = window.document.querySelectorAll('link.quarto-color-scheme.quarto-color-alternate');
150 manageTransitions('#quarto-margin-sidebar .nav-link', false);
151 if (alternate) {
152 // note: dark is layered on light, we don't disable primary!
153 enableStylesheet(alternateStylesheets);
154 for (const sheetNode of alternateStylesheets) {
155 if (sheetNode.id === "quarto-bootstrap") {
156 toggleBodyColorMode(sheetNode);
157 }
158 }
159 } else {
160 disableStylesheet(alternateStylesheets);
161 enableStylesheet(primaryStylesheets)
162 toggleBodyColorPrimary();
163 }
164 manageTransitions('#quarto-margin-sidebar .nav-link', true);
165 // Switch the toggles
166 setColorSchemeToggle(alternate)
167 // Hack to workaround the fact that safari doesn't
168 // properly recolor the scrollbar when toggling (#1455)
169 if (navigator.userAgent.indexOf('Safari') > 0 && navigator.userAgent.indexOf('Chrome') == -1) {
170 manageTransitions("body", false);
171 window.scrollTo(0, 1);
172 setTimeout(() => {
173 window.scrollTo(0, 0);
174 manageTransitions("body", true);
175 }, 40);
176 }
177 }
178 const disableStylesheet = (stylesheets) => {
179 for (let i=0; i < stylesheets.length; i++) {
180 const stylesheet = stylesheets[i];
181 stylesheet.rel = 'disabled-stylesheet';
182 }
183 }
184 const enableStylesheet = (stylesheets) => {
185 for (let i=0; i < stylesheets.length; i++) {
186 const stylesheet = stylesheets[i];
187 if(stylesheet.rel !== 'stylesheet') { // for Chrome, which will still FOUC without this check
188 stylesheet.rel = 'stylesheet';
189 }
190 }
191 }
192 const manageTransitions = (selector, allowTransitions) => {
193 const els = window.document.querySelectorAll(selector);
194 for (let i=0; i < els.length; i++) {
195 const el = els[i];
196 if (allowTransitions) {
197 el.classList.remove('notransition');
198 } else {
199 el.classList.add('notransition');
200 }
201 }
202 }
203 const isFileUrl = () => {
204 return window.location.protocol === 'file:';
205 }
206 const hasAlternateSentinel = () => {
207 let styleSentinel = getColorSchemeSentinel();
208 if (styleSentinel !== null) {
209 return styleSentinel === "alternate";
210 } else {
211 return false;
212 }
213 }
214 const setStyleSentinel = (alternate) => {
215 const value = alternate ? "alternate" : "default";
216 if (!isFileUrl()) {
217 window.localStorage.setItem("quarto-color-scheme", value);
218 } else {
219 localAlternateSentinel = value;
220 }
221 }
222 const getColorSchemeSentinel = () => {
223 if (!isFileUrl()) {
224 const storageValue = window.localStorage.getItem("quarto-color-scheme");
225 return storageValue != null ? storageValue : localAlternateSentinel;
226 } else {
227 return localAlternateSentinel;
228 }
229 }
230 const toggleGiscusIfUsed = (isAlternate, darkModeDefault) => {
231 const baseTheme = document.querySelector('#giscus-base-theme')?.value ?? 'light';
232 const alternateTheme = document.querySelector('#giscus-alt-theme')?.value ?? 'dark';
233 let newTheme = '';
234 if(authorPrefersDark) {
235 newTheme = isAlternate ? baseTheme : alternateTheme;
236 } else {
237 newTheme = isAlternate ? alternateTheme : baseTheme;
238 }
239 const changeGiscusTheme = () => {
240 // From: https://github.com/giscus/giscus/issues/336
241 const sendMessage = (message) => {
242 const iframe = document.querySelector('iframe.giscus-frame');
243 if (!iframe) return;
244 iframe.contentWindow.postMessage({ giscus: message }, 'https://giscus.app');
245 }
246 sendMessage({
247 setConfig: {
248 theme: newTheme
249 }
250 });
251 }
252 const isGiscussLoaded = window.document.querySelector('iframe.giscus-frame') !== null;
253 if (isGiscussLoaded) {
254 changeGiscusTheme();
255 }
256 };
257 const authorPrefersDark = false;
258 const darkModeDefault = authorPrefersDark;
259 document.querySelector('link#quarto-text-highlighting-styles.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
260 document.querySelector('link#quarto-bootstrap.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
261 let localAlternateSentinel = darkModeDefault ? 'alternate' : 'default';
262 // Dark / light mode switch
263 window.quartoToggleColorScheme = () => {
264 // Read the current dark / light value
265 let toAlternate = !hasAlternateSentinel();
266 toggleColorMode(toAlternate);
267 setStyleSentinel(toAlternate);
268 toggleGiscusIfUsed(toAlternate, darkModeDefault);
269 window.dispatchEvent(new Event('resize'));
270 };
271 // Switch to dark mode if need be
272 if (hasAlternateSentinel()) {
273 toggleColorMode(true);
274 } else {
275 toggleColorMode(false);
276 }
277 </script>
278
279<div id="quarto-search-results"></div>
280 <header id="quarto-header" class="headroom fixed-top">
281 <nav class="navbar navbar-expand-lg " data-bs-theme="dark">
282 <div class="navbar-container container-fluid">
283 <div class="navbar-brand-container mx-auto">
284 <a class="navbar-brand" href="../index.html">
285 <span class="navbar-title">atdata</span>
286 </a>
287 </div>
288 <div id="quarto-search" class="" title="Search"></div>
289 <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarCollapse" aria-controls="navbarCollapse" role="menu" aria-expanded="false" aria-label="Toggle navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
290 <span class="navbar-toggler-icon"></span>
291</button>
292 <div class="collapse navbar-collapse" id="navbarCollapse">
293 <ul class="navbar-nav navbar-nav-scroll me-auto">
294 <li class="nav-item">
295 <a class="nav-link active" href="../index.html" aria-current="page">
296<span class="menu-text">Guide</span></a>
297 </li>
298 <li class="nav-item dropdown ">
299 <a class="nav-link dropdown-toggle" href="#" id="nav-menu-tutorials" role="link" data-bs-toggle="dropdown" aria-expanded="false">
300 <span class="menu-text">Tutorials</span>
301 </a>
302 <ul class="dropdown-menu" aria-labelledby="nav-menu-tutorials">
303 <li>
304 <a class="dropdown-item" href="../tutorials/quickstart.html">
305 <span class="dropdown-text">Quick Start</span></a>
306 </li>
307 <li>
308 <a class="dropdown-item" href="../tutorials/local-workflow.html">
309 <span class="dropdown-text">Local Workflow</span></a>
310 </li>
311 <li>
312 <a class="dropdown-item" href="../tutorials/atmosphere.html">
313 <span class="dropdown-text">Atmosphere Publishing</span></a>
314 </li>
315 <li>
316 <a class="dropdown-item" href="../tutorials/promotion.html">
317 <span class="dropdown-text">Promotion Workflow</span></a>
318 </li>
319 </ul>
320 </li>
321 <li class="nav-item dropdown ">
322 <a class="nav-link dropdown-toggle" href="#" id="nav-menu-reference" role="link" data-bs-toggle="dropdown" aria-expanded="false">
323 <span class="menu-text">Reference</span>
324 </a>
325 <ul class="dropdown-menu" aria-labelledby="nav-menu-reference">
326 <li>
327 <a class="dropdown-item" href="../reference/architecture.html">
328 <span class="dropdown-text">Architecture Overview</span></a>
329 </li>
330 <li>
331 <a class="dropdown-item" href="../reference/packable-samples.html">
332 <span class="dropdown-text">Packable Samples</span></a>
333 </li>
334 <li>
335 <a class="dropdown-item" href="../reference/datasets.html">
336 <span class="dropdown-text">Datasets</span></a>
337 </li>
338 <li>
339 <a class="dropdown-item" href="../reference/lenses.html">
340 <span class="dropdown-text">Lenses</span></a>
341 </li>
342 <li>
343 <a class="dropdown-item" href="../reference/local-storage.html">
344 <span class="dropdown-text">Local Storage</span></a>
345 </li>
346 <li>
347 <a class="dropdown-item" href="../reference/atmosphere.html">
348 <span class="dropdown-text">Atmosphere</span></a>
349 </li>
350 <li>
351 <a class="dropdown-item" href="../reference/promotion.html">
352 <span class="dropdown-text">Promotion</span></a>
353 </li>
354 <li>
355 <a class="dropdown-item" href="../reference/load-dataset.html">
356 <span class="dropdown-text">load_dataset API</span></a>
357 </li>
358 <li>
359 <a class="dropdown-item" href="../reference/protocols.html">
360 <span class="dropdown-text">Protocols</span></a>
361 </li>
362 <li>
363 <a class="dropdown-item" href="../reference/uri-spec.html">
364 <span class="dropdown-text">URI Specification</span></a>
365 </li>
366 <li>
367 <a class="dropdown-item" href="../reference/troubleshooting.html">
368 <span class="dropdown-text">Troubleshooting & FAQ</span></a>
369 </li>
370 <li>
371 <a class="dropdown-item" href="../reference/deployment.html">
372 <span class="dropdown-text">Deployment Guide</span></a>
373 </li>
374 </ul>
375 </li>
376 <li class="nav-item">
377 <a class="nav-link" href="../api/index.html">
378<span class="menu-text">API</span></a>
379 </li>
380</ul>
381 <ul class="navbar-nav navbar-nav-scroll ms-auto">
382 <li class="nav-item compact">
383 <a class="nav-link" href="https://github.com/your-org/atdata"> <i class="bi bi-github" role="img">
384</i>
385<span class="menu-text"></span></a>
386 </li>
387</ul>
388 </div> <!-- /navcollapse -->
389 <div class="quarto-navbar-tools">
390 <a href="" class="quarto-color-scheme-toggle quarto-navigation-tool px-1" onclick="window.quartoToggleColorScheme(); return false;" title="Toggle dark mode"><i class="bi"></i></a>
391</div>
392 </div> <!-- /container-fluid -->
393 </nav>
394 <nav class="quarto-secondary-nav">
395 <div class="container-fluid d-flex">
396 <button type="button" class="quarto-btn-toggle btn" data-bs-toggle="collapse" role="button" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
397 <i class="bi bi-layout-text-sidebar-reverse"></i>
398 </button>
399 <nav class="quarto-page-breadcrumbs" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item"><a href="../reference/architecture.html">Reference</a></li><li class="breadcrumb-item"><a href="../reference/lenses.html">Lenses</a></li></ol></nav>
400 <a class="flex-grow-1" role="navigation" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
401 </a>
402 </div>
403 </nav>
404</header>
405<!-- content -->
406<div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-article page-navbar">
407<!-- sidebar -->
408 <nav id="quarto-sidebar" class="sidebar collapse collapse-horizontal quarto-sidebar-collapse-item sidebar-navigation docked overflow-auto">
409 <div class="sidebar-menu-container">
410 <ul class="list-unstyled mt-1">
411 <li class="sidebar-item">
412 <div class="sidebar-item-container">
413 <a href="../index.html" class="sidebar-item-text sidebar-link">
414 <span class="menu-text">atdata</span></a>
415 </div>
416</li>
417 <li class="sidebar-item sidebar-item-section">
418 <div class="sidebar-item-container">
419 <a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" role="navigation" aria-expanded="true">
420 <span class="menu-text">Getting Started</span></a>
421 <a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" role="navigation" aria-expanded="true" aria-label="Toggle section">
422 <i class="bi bi-chevron-right ms-2"></i>
423 </a>
424 </div>
425 <ul id="quarto-sidebar-section-1" class="collapse list-unstyled sidebar-section depth1 show">
426 <li class="sidebar-item">
427 <div class="sidebar-item-container">
428 <a href="../tutorials/quickstart.html" class="sidebar-item-text sidebar-link">
429 <span class="menu-text">Quick Start</span></a>
430 </div>
431</li>
432 <li class="sidebar-item">
433 <div class="sidebar-item-container">
434 <a href="../tutorials/local-workflow.html" class="sidebar-item-text sidebar-link">
435 <span class="menu-text">Local Workflow</span></a>
436 </div>
437</li>
438 <li class="sidebar-item">
439 <div class="sidebar-item-container">
440 <a href="../tutorials/atmosphere.html" class="sidebar-item-text sidebar-link">
441 <span class="menu-text">Atmosphere Publishing</span></a>
442 </div>
443</li>
444 <li class="sidebar-item">
445 <div class="sidebar-item-container">
446 <a href="../tutorials/promotion.html" class="sidebar-item-text sidebar-link">
447 <span class="menu-text">Promotion Workflow</span></a>
448 </div>
449</li>
450 </ul>
451 </li>
452 <li class="sidebar-item sidebar-item-section">
453 <div class="sidebar-item-container">
454 <a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" role="navigation" aria-expanded="true">
455 <span class="menu-text">Reference</span></a>
456 <a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" role="navigation" aria-expanded="true" aria-label="Toggle section">
457 <i class="bi bi-chevron-right ms-2"></i>
458 </a>
459 </div>
460 <ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
461 <li class="sidebar-item">
462 <div class="sidebar-item-container">
463 <a href="../reference/architecture.html" class="sidebar-item-text sidebar-link">
464 <span class="menu-text">Architecture Overview</span></a>
465 </div>
466</li>
467 <li class="sidebar-item">
468 <div class="sidebar-item-container">
469 <a href="../reference/packable-samples.html" class="sidebar-item-text sidebar-link">
470 <span class="menu-text">Packable Samples</span></a>
471 </div>
472</li>
473 <li class="sidebar-item">
474 <div class="sidebar-item-container">
475 <a href="../reference/datasets.html" class="sidebar-item-text sidebar-link">
476 <span class="menu-text">Datasets</span></a>
477 </div>
478</li>
479 <li class="sidebar-item">
480 <div class="sidebar-item-container">
481 <a href="../reference/lenses.html" class="sidebar-item-text sidebar-link active">
482 <span class="menu-text">Lenses</span></a>
483 </div>
484</li>
485 <li class="sidebar-item">
486 <div class="sidebar-item-container">
487 <a href="../reference/local-storage.html" class="sidebar-item-text sidebar-link">
488 <span class="menu-text">Local Storage</span></a>
489 </div>
490</li>
491 <li class="sidebar-item">
492 <div class="sidebar-item-container">
493 <a href="../reference/atmosphere.html" class="sidebar-item-text sidebar-link">
494 <span class="menu-text">Atmosphere (ATProto Integration)</span></a>
495 </div>
496</li>
497 <li class="sidebar-item">
498 <div class="sidebar-item-container">
499 <a href="../reference/promotion.html" class="sidebar-item-text sidebar-link">
500 <span class="menu-text">Promotion Workflow</span></a>
501 </div>
502</li>
503 <li class="sidebar-item">
504 <div class="sidebar-item-container">
505 <a href="../reference/load-dataset.html" class="sidebar-item-text sidebar-link">
506 <span class="menu-text">load_dataset API</span></a>
507 </div>
508</li>
509 <li class="sidebar-item">
510 <div class="sidebar-item-container">
511 <a href="../reference/protocols.html" class="sidebar-item-text sidebar-link">
512 <span class="menu-text">Protocols</span></a>
513 </div>
514</li>
515 <li class="sidebar-item">
516 <div class="sidebar-item-container">
517 <a href="../reference/uri-spec.html" class="sidebar-item-text sidebar-link">
518 <span class="menu-text">URI Specification</span></a>
519 </div>
520</li>
521 <li class="sidebar-item">
522 <div class="sidebar-item-container">
523 <a href="../reference/troubleshooting.html" class="sidebar-item-text sidebar-link">
524 <span class="menu-text">Troubleshooting & FAQ</span></a>
525 </div>
526</li>
527 <li class="sidebar-item">
528 <div class="sidebar-item-container">
529 <a href="../reference/deployment.html" class="sidebar-item-text sidebar-link">
530 <span class="menu-text">Deployment Guide</span></a>
531 </div>
532</li>
533 </ul>
534 </li>
535 </ul>
536 </div>
537</nav>
538<div id="quarto-sidebar-glass" class="quarto-sidebar-collapse-item" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item"></div>
539<!-- margin-sidebar -->
540 <div id="quarto-margin-sidebar" class="sidebar margin-sidebar">
541 <nav id="TOC" role="doc-toc" class="toc-active">
542 <h2 id="toc-title">On this page</h2>
543
544 <ul>
545 <li><a href="#overview" id="toc-overview" class="nav-link active" data-scroll-target="#overview">Overview</a></li>
546 <li><a href="#creating-a-lens" id="toc-creating-a-lens" class="nav-link" data-scroll-target="#creating-a-lens">Creating a Lens</a></li>
547 <li><a href="#adding-a-putter" id="toc-adding-a-putter" class="nav-link" data-scroll-target="#adding-a-putter">Adding a Putter</a></li>
548 <li><a href="#using-lenses-with-datasets" id="toc-using-lenses-with-datasets" class="nav-link" data-scroll-target="#using-lenses-with-datasets">Using Lenses with Datasets</a></li>
549 <li><a href="#direct-lens-usage" id="toc-direct-lens-usage" class="nav-link" data-scroll-target="#direct-lens-usage">Direct Lens Usage</a></li>
550 <li><a href="#lens-laws" id="toc-lens-laws" class="nav-link" data-scroll-target="#lens-laws">Lens Laws</a></li>
551 <li><a href="#trivial-putter" id="toc-trivial-putter" class="nav-link" data-scroll-target="#trivial-putter">Trivial Putter</a></li>
552 <li><a href="#lensnetwork-registry" id="toc-lensnetwork-registry" class="nav-link" data-scroll-target="#lensnetwork-registry">LensNetwork Registry</a></li>
553 <li><a href="#example-feature-extraction" id="toc-example-feature-extraction" class="nav-link" data-scroll-target="#example-feature-extraction">Example: Feature Extraction</a></li>
554 <li><a href="#related" id="toc-related" class="nav-link" data-scroll-target="#related">Related</a></li>
555 </ul>
556<div class="toc-actions"><ul><li><a href="https://github.com/your-org/atdata/edit/main/reference/lenses.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></nav>
557 </div>
558<!-- main -->
559<main class="content" id="quarto-document-content">
560
561
562<header id="title-block-header" class="quarto-title-block default"><nav class="quarto-page-breadcrumbs quarto-title-breadcrumbs d-none d-lg-block" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item"><a href="../reference/architecture.html">Reference</a></li><li class="breadcrumb-item"><a href="../reference/lenses.html">Lenses</a></li></ol></nav>
563<div class="quarto-title">
564<h1 class="title">Lenses</h1>
565</div>
566
567<div>
568 <div class="description">
569 Bidirectional transformations between sample types
570 </div>
571</div>
572
573
574<div class="quarto-title-meta">
575
576
577
578
579 </div>
580
581
582
583</header>
584
585
586<p>Lenses provide bidirectional transformations between sample types, enabling datasets to be viewed through different schemas without duplicating data.</p>
587<section id="overview" class="level2">
588<h2 class="anchored" data-anchor-id="overview">Overview</h2>
589<p>A lens consists of:</p>
590<ul>
591<li><strong>Getter</strong>: Transforms source type <code>S</code> to view type <code>V</code></li>
592<li><strong>Putter</strong>: Updates source based on a modified view (optional)</li>
593</ul>
594</section>
595<section id="creating-a-lens" class="level2">
596<h2 class="anchored" data-anchor-id="creating-a-lens">Creating a Lens</h2>
597<p>Use the <code>@lens</code> decorator to define a getter:</p>
598<div id="390e6ab5" class="cell">
599<div class="sourceCode cell-code" id="cb1"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> atdata</span>
600<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> numpy.typing <span class="im">import</span> NDArray</span>
601<span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a></span>
602<span id="cb1-4"><a href="#cb1-4" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
603<span id="cb1-5"><a href="#cb1-5" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> FullSample:</span>
604<span id="cb1-6"><a href="#cb1-6" aria-hidden="true" tabindex="-1"></a> image: NDArray</span>
605<span id="cb1-7"><a href="#cb1-7" aria-hidden="true" tabindex="-1"></a> label: <span class="bu">str</span></span>
606<span id="cb1-8"><a href="#cb1-8" aria-hidden="true" tabindex="-1"></a> confidence: <span class="bu">float</span></span>
607<span id="cb1-9"><a href="#cb1-9" aria-hidden="true" tabindex="-1"></a> metadata: <span class="bu">dict</span></span>
608<span id="cb1-10"><a href="#cb1-10" aria-hidden="true" tabindex="-1"></a></span>
609<span id="cb1-11"><a href="#cb1-11" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
610<span id="cb1-12"><a href="#cb1-12" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> SimpleSample:</span>
611<span id="cb1-13"><a href="#cb1-13" aria-hidden="true" tabindex="-1"></a> label: <span class="bu">str</span></span>
612<span id="cb1-14"><a href="#cb1-14" aria-hidden="true" tabindex="-1"></a> confidence: <span class="bu">float</span></span>
613<span id="cb1-15"><a href="#cb1-15" aria-hidden="true" tabindex="-1"></a></span>
614<span id="cb1-16"><a href="#cb1-16" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.lens</span></span>
615<span id="cb1-17"><a href="#cb1-17" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> simplify(src: FullSample) <span class="op">-></span> SimpleSample:</span>
616<span id="cb1-18"><a href="#cb1-18" aria-hidden="true" tabindex="-1"></a> <span class="cf">return</span> SimpleSample(label<span class="op">=</span>src.label, confidence<span class="op">=</span>src.confidence)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
617</div>
618<p>The decorator:</p>
619<ol type="1">
620<li>Creates a <code>Lens</code> object from the getter function</li>
621<li>Registers it in the global <code>LensNetwork</code> registry</li>
622<li>Extracts source/view types from annotations</li>
623</ol>
624</section>
625<section id="adding-a-putter" class="level2">
626<h2 class="anchored" data-anchor-id="adding-a-putter">Adding a Putter</h2>
627<p>To enable bidirectional updates, add a putter:</p>
628<div id="858a66a9" class="cell">
629<div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="at">@simplify.putter</span></span>
630<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> simplify_put(view: SimpleSample, source: FullSample) <span class="op">-></span> FullSample:</span>
631<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a> <span class="cf">return</span> FullSample(</span>
632<span id="cb2-4"><a href="#cb2-4" aria-hidden="true" tabindex="-1"></a> image<span class="op">=</span>source.image,</span>
633<span id="cb2-5"><a href="#cb2-5" aria-hidden="true" tabindex="-1"></a> label<span class="op">=</span>view.label,</span>
634<span id="cb2-6"><a href="#cb2-6" aria-hidden="true" tabindex="-1"></a> confidence<span class="op">=</span>view.confidence,</span>
635<span id="cb2-7"><a href="#cb2-7" aria-hidden="true" tabindex="-1"></a> metadata<span class="op">=</span>source.metadata,</span>
636<span id="cb2-8"><a href="#cb2-8" aria-hidden="true" tabindex="-1"></a> )</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
637</div>
638<p>The putter receives:</p>
639<ul>
640<li><code>view</code>: The modified view value</li>
641<li><code>source</code>: The original source value</li>
642</ul>
643<p>It returns an updated source that reflects changes from the view.</p>
644</section>
645<section id="using-lenses-with-datasets" class="level2">
646<h2 class="anchored" data-anchor-id="using-lenses-with-datasets">Using Lenses with Datasets</h2>
647<p>Lenses integrate with <code>Dataset.as_type()</code>:</p>
648<div id="d9989ec0" class="cell">
649<div class="sourceCode cell-code" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a>dataset <span class="op">=</span> atdata.Dataset[FullSample](<span class="st">"data-{000000..000009}.tar"</span>)</span>
650<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a></span>
651<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a><span class="co"># View through a different type</span></span>
652<span id="cb3-4"><a href="#cb3-4" aria-hidden="true" tabindex="-1"></a>simple_ds <span class="op">=</span> dataset.as_type(SimpleSample)</span>
653<span id="cb3-5"><a href="#cb3-5" aria-hidden="true" tabindex="-1"></a></span>
654<span id="cb3-6"><a href="#cb3-6" aria-hidden="true" tabindex="-1"></a><span class="cf">for</span> batch <span class="kw">in</span> simple_ds.ordered(batch_size<span class="op">=</span><span class="dv">32</span>):</span>
655<span id="cb3-7"><a href="#cb3-7" aria-hidden="true" tabindex="-1"></a> <span class="co"># Only SimpleSample fields available</span></span>
656<span id="cb3-8"><a href="#cb3-8" aria-hidden="true" tabindex="-1"></a> labels <span class="op">=</span> batch.label</span>
657<span id="cb3-9"><a href="#cb3-9" aria-hidden="true" tabindex="-1"></a> scores <span class="op">=</span> batch.confidence</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
658</div>
659</section>
660<section id="direct-lens-usage" class="level2">
661<h2 class="anchored" data-anchor-id="direct-lens-usage">Direct Lens Usage</h2>
662<p>Lenses can also be called directly:</p>
663<div id="33deda47" class="cell">
664<div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
665<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a></span>
666<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a>full <span class="op">=</span> FullSample(</span>
667<span id="cb4-4"><a href="#cb4-4" aria-hidden="true" tabindex="-1"></a> image<span class="op">=</span>np.zeros((<span class="dv">224</span>, <span class="dv">224</span>, <span class="dv">3</span>)),</span>
668<span id="cb4-5"><a href="#cb4-5" aria-hidden="true" tabindex="-1"></a> label<span class="op">=</span><span class="st">"cat"</span>,</span>
669<span id="cb4-6"><a href="#cb4-6" aria-hidden="true" tabindex="-1"></a> confidence<span class="op">=</span><span class="fl">0.95</span>,</span>
670<span id="cb4-7"><a href="#cb4-7" aria-hidden="true" tabindex="-1"></a> metadata<span class="op">=</span>{<span class="st">"source"</span>: <span class="st">"training"</span>}</span>
671<span id="cb4-8"><a href="#cb4-8" aria-hidden="true" tabindex="-1"></a>)</span>
672<span id="cb4-9"><a href="#cb4-9" aria-hidden="true" tabindex="-1"></a></span>
673<span id="cb4-10"><a href="#cb4-10" aria-hidden="true" tabindex="-1"></a><span class="co"># Apply getter</span></span>
674<span id="cb4-11"><a href="#cb4-11" aria-hidden="true" tabindex="-1"></a>simple <span class="op">=</span> simplify(full)</span>
675<span id="cb4-12"><a href="#cb4-12" aria-hidden="true" tabindex="-1"></a><span class="co"># Or: simple = simplify.get(full)</span></span>
676<span id="cb4-13"><a href="#cb4-13" aria-hidden="true" tabindex="-1"></a></span>
677<span id="cb4-14"><a href="#cb4-14" aria-hidden="true" tabindex="-1"></a><span class="co"># Apply putter</span></span>
678<span id="cb4-15"><a href="#cb4-15" aria-hidden="true" tabindex="-1"></a>modified_simple <span class="op">=</span> SimpleSample(label<span class="op">=</span><span class="st">"dog"</span>, confidence<span class="op">=</span><span class="fl">0.87</span>)</span>
679<span id="cb4-16"><a href="#cb4-16" aria-hidden="true" tabindex="-1"></a>updated_full <span class="op">=</span> simplify.put(modified_simple, full)</span>
680<span id="cb4-17"><a href="#cb4-17" aria-hidden="true" tabindex="-1"></a><span class="co"># updated_full has label="dog", confidence=0.87, but retains</span></span>
681<span id="cb4-18"><a href="#cb4-18" aria-hidden="true" tabindex="-1"></a><span class="co"># original image and metadata</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
682</div>
683</section>
684<section id="lens-laws" class="level2">
685<h2 class="anchored" data-anchor-id="lens-laws">Lens Laws</h2>
686<p>Well-behaved lenses should satisfy these properties:</p>
687<div class="tabset-margin-container"></div><div class="panel-tabset">
688<ul class="nav nav-tabs" role="tablist"><li class="nav-item" role="presentation"><a class="nav-link active" id="tabset-1-1-tab" data-bs-toggle="tab" data-bs-target="#tabset-1-1" role="tab" aria-controls="tabset-1-1" aria-selected="true">GetPut</a></li><li class="nav-item" role="presentation"><a class="nav-link" id="tabset-1-2-tab" data-bs-toggle="tab" data-bs-target="#tabset-1-2" role="tab" aria-controls="tabset-1-2" aria-selected="false">PutGet</a></li><li class="nav-item" role="presentation"><a class="nav-link" id="tabset-1-3-tab" data-bs-toggle="tab" data-bs-target="#tabset-1-3" role="tab" aria-controls="tabset-1-3" aria-selected="false">PutPut</a></li></ul>
689<div class="tab-content">
690<div id="tabset-1-1" class="tab-pane active" role="tabpanel" aria-labelledby="tabset-1-1-tab">
691<p>If you get a view and immediately put it back, the source is unchanged:</p>
692<div id="fba0d404" class="cell">
693<div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a>view <span class="op">=</span> lens.get(source)</span>
694<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a><span class="cf">assert</span> lens.put(view, source) <span class="op">==</span> source</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
695</div>
696</div>
697<div id="tabset-1-2" class="tab-pane" role="tabpanel" aria-labelledby="tabset-1-2-tab">
698<p>If you put a view, getting it back yields that view:</p>
699<div id="00a23052" class="cell">
700<div class="sourceCode cell-code" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a>updated <span class="op">=</span> lens.put(view, source)</span>
701<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a><span class="cf">assert</span> lens.get(updated) <span class="op">==</span> view</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
702</div>
703</div>
704<div id="tabset-1-3" class="tab-pane" role="tabpanel" aria-labelledby="tabset-1-3-tab">
705<p>Putting twice is equivalent to putting once with the final value:</p>
706<div id="2ca3d0ae" class="cell">
707<div class="sourceCode cell-code" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a>result1 <span class="op">=</span> lens.put(v2, lens.put(v1, source))</span>
708<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a>result2 <span class="op">=</span> lens.put(v2, source)</span>
709<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a><span class="cf">assert</span> result1 <span class="op">==</span> result2</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
710</div>
711</div>
712</div>
713</div>
714</section>
715<section id="trivial-putter" class="level2">
716<h2 class="anchored" data-anchor-id="trivial-putter">Trivial Putter</h2>
717<p>If no putter is defined, a trivial putter is used that ignores view updates:</p>
718<div id="e91f979c" class="cell">
719<div class="sourceCode cell-code" id="cb8"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.lens</span></span>
720<span id="cb8-2"><a href="#cb8-2" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> extract_label(src: FullSample) <span class="op">-></span> SimpleSample:</span>
721<span id="cb8-3"><a href="#cb8-3" aria-hidden="true" tabindex="-1"></a> <span class="cf">return</span> SimpleSample(label<span class="op">=</span>src.label, confidence<span class="op">=</span>src.confidence)</span>
722<span id="cb8-4"><a href="#cb8-4" aria-hidden="true" tabindex="-1"></a></span>
723<span id="cb8-5"><a href="#cb8-5" aria-hidden="true" tabindex="-1"></a><span class="co"># Without a putter, put() returns the original source unchanged</span></span>
724<span id="cb8-6"><a href="#cb8-6" aria-hidden="true" tabindex="-1"></a>view <span class="op">=</span> SimpleSample(label<span class="op">=</span><span class="st">"modified"</span>, confidence<span class="op">=</span><span class="fl">0.5</span>)</span>
725<span id="cb8-7"><a href="#cb8-7" aria-hidden="true" tabindex="-1"></a>updated <span class="op">=</span> extract_label.put(view, original)</span>
726<span id="cb8-8"><a href="#cb8-8" aria-hidden="true" tabindex="-1"></a><span class="cf">assert</span> updated <span class="op">==</span> original <span class="co"># No changes applied</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
727</div>
728</section>
729<section id="lensnetwork-registry" class="level2">
730<h2 class="anchored" data-anchor-id="lensnetwork-registry">LensNetwork Registry</h2>
731<p>The <code>LensNetwork</code> is a singleton that stores all registered lenses:</p>
732<div id="fa3c962e" class="cell">
733<div class="sourceCode cell-code" id="cb9"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb9-1"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.lens <span class="im">import</span> LensNetwork</span>
734<span id="cb9-2"><a href="#cb9-2" aria-hidden="true" tabindex="-1"></a></span>
735<span id="cb9-3"><a href="#cb9-3" aria-hidden="true" tabindex="-1"></a>network <span class="op">=</span> LensNetwork()</span>
736<span id="cb9-4"><a href="#cb9-4" aria-hidden="true" tabindex="-1"></a></span>
737<span id="cb9-5"><a href="#cb9-5" aria-hidden="true" tabindex="-1"></a><span class="co"># Look up a specific lens</span></span>
738<span id="cb9-6"><a href="#cb9-6" aria-hidden="true" tabindex="-1"></a>lens <span class="op">=</span> network.transform(FullSample, SimpleSample)</span>
739<span id="cb9-7"><a href="#cb9-7" aria-hidden="true" tabindex="-1"></a></span>
740<span id="cb9-8"><a href="#cb9-8" aria-hidden="true" tabindex="-1"></a><span class="co"># Raises ValueError if no lens exists</span></span>
741<span id="cb9-9"><a href="#cb9-9" aria-hidden="true" tabindex="-1"></a><span class="cf">try</span>:</span>
742<span id="cb9-10"><a href="#cb9-10" aria-hidden="true" tabindex="-1"></a> lens <span class="op">=</span> network.transform(TypeA, TypeB)</span>
743<span id="cb9-11"><a href="#cb9-11" aria-hidden="true" tabindex="-1"></a><span class="cf">except</span> <span class="pp">ValueError</span>:</span>
744<span id="cb9-12"><a href="#cb9-12" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="st">"No lens registered for TypeA -> TypeB"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
745</div>
746</section>
747<section id="example-feature-extraction" class="level2">
748<h2 class="anchored" data-anchor-id="example-feature-extraction">Example: Feature Extraction</h2>
749<div id="4af8323d" class="cell">
750<div class="sourceCode cell-code" id="cb10"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
751<span id="cb10-2"><a href="#cb10-2" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> RawSample:</span>
752<span id="cb10-3"><a href="#cb10-3" aria-hidden="true" tabindex="-1"></a> audio: NDArray</span>
753<span id="cb10-4"><a href="#cb10-4" aria-hidden="true" tabindex="-1"></a> text: <span class="bu">str</span></span>
754<span id="cb10-5"><a href="#cb10-5" aria-hidden="true" tabindex="-1"></a> speaker_id: <span class="bu">int</span></span>
755<span id="cb10-6"><a href="#cb10-6" aria-hidden="true" tabindex="-1"></a></span>
756<span id="cb10-7"><a href="#cb10-7" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
757<span id="cb10-8"><a href="#cb10-8" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> TextFeatures:</span>
758<span id="cb10-9"><a href="#cb10-9" aria-hidden="true" tabindex="-1"></a> text: <span class="bu">str</span></span>
759<span id="cb10-10"><a href="#cb10-10" aria-hidden="true" tabindex="-1"></a> word_count: <span class="bu">int</span></span>
760<span id="cb10-11"><a href="#cb10-11" aria-hidden="true" tabindex="-1"></a></span>
761<span id="cb10-12"><a href="#cb10-12" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.lens</span></span>
762<span id="cb10-13"><a href="#cb10-13" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> extract_text(src: RawSample) <span class="op">-></span> TextFeatures:</span>
763<span id="cb10-14"><a href="#cb10-14" aria-hidden="true" tabindex="-1"></a> <span class="cf">return</span> TextFeatures(</span>
764<span id="cb10-15"><a href="#cb10-15" aria-hidden="true" tabindex="-1"></a> text<span class="op">=</span>src.text,</span>
765<span id="cb10-16"><a href="#cb10-16" aria-hidden="true" tabindex="-1"></a> word_count<span class="op">=</span><span class="bu">len</span>(src.text.split())</span>
766<span id="cb10-17"><a href="#cb10-17" aria-hidden="true" tabindex="-1"></a> )</span>
767<span id="cb10-18"><a href="#cb10-18" aria-hidden="true" tabindex="-1"></a></span>
768<span id="cb10-19"><a href="#cb10-19" aria-hidden="true" tabindex="-1"></a><span class="at">@extract_text.putter</span></span>
769<span id="cb10-20"><a href="#cb10-20" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> extract_text_put(view: TextFeatures, source: RawSample) <span class="op">-></span> RawSample:</span>
770<span id="cb10-21"><a href="#cb10-21" aria-hidden="true" tabindex="-1"></a> <span class="cf">return</span> RawSample(</span>
771<span id="cb10-22"><a href="#cb10-22" aria-hidden="true" tabindex="-1"></a> audio<span class="op">=</span>source.audio,</span>
772<span id="cb10-23"><a href="#cb10-23" aria-hidden="true" tabindex="-1"></a> text<span class="op">=</span>view.text,</span>
773<span id="cb10-24"><a href="#cb10-24" aria-hidden="true" tabindex="-1"></a> speaker_id<span class="op">=</span>source.speaker_id</span>
774<span id="cb10-25"><a href="#cb10-25" aria-hidden="true" tabindex="-1"></a> )</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
775</div>
776</section>
777<section id="related" class="level2">
778<h2 class="anchored" data-anchor-id="related">Related</h2>
779<ul>
780<li><a href="../reference/datasets.html">Datasets</a> - Using lenses with Dataset.as_type()</li>
781<li><a href="../reference/packable-samples.html">Packable Samples</a> - Defining sample types</li>
782<li><a href="../reference/atmosphere.html">Atmosphere</a> - Publishing lenses to ATProto federation</li>
783</ul>
784
785
786</section>
787
788</main> <!-- /main -->
789<script id="quarto-html-after-body" type="application/javascript">
790 window.document.addEventListener("DOMContentLoaded", function (event) {
791 // Ensure there is a toggle, if there isn't float one in the top right
792 if (window.document.querySelector('.quarto-color-scheme-toggle') === null) {
793 const a = window.document.createElement('a');
794 a.classList.add('top-right');
795 a.classList.add('quarto-color-scheme-toggle');
796 a.href = "";
797 a.onclick = function() { try { window.quartoToggleColorScheme(); } catch {} return false; };
798 const i = window.document.createElement("i");
799 i.classList.add('bi');
800 a.appendChild(i);
801 window.document.body.appendChild(a);
802 }
803 setColorSchemeToggle(hasAlternateSentinel())
804 const icon = "";
805 const anchorJS = new window.AnchorJS();
806 anchorJS.options = {
807 placement: 'right',
808 icon: icon
809 };
810 anchorJS.add('.anchored');
811 const isCodeAnnotation = (el) => {
812 for (const clz of el.classList) {
813 if (clz.startsWith('code-annotation-')) {
814 return true;
815 }
816 }
817 return false;
818 }
819 const onCopySuccess = function(e) {
820 // button target
821 const button = e.trigger;
822 // don't keep focus
823 button.blur();
824 // flash "checked"
825 button.classList.add('code-copy-button-checked');
826 var currentTitle = button.getAttribute("title");
827 button.setAttribute("title", "Copied!");
828 let tooltip;
829 if (window.bootstrap) {
830 button.setAttribute("data-bs-toggle", "tooltip");
831 button.setAttribute("data-bs-placement", "left");
832 button.setAttribute("data-bs-title", "Copied!");
833 tooltip = new bootstrap.Tooltip(button,
834 { trigger: "manual",
835 customClass: "code-copy-button-tooltip",
836 offset: [0, -8]});
837 tooltip.show();
838 }
839 setTimeout(function() {
840 if (tooltip) {
841 tooltip.hide();
842 button.removeAttribute("data-bs-title");
843 button.removeAttribute("data-bs-toggle");
844 button.removeAttribute("data-bs-placement");
845 }
846 button.setAttribute("title", currentTitle);
847 button.classList.remove('code-copy-button-checked');
848 }, 1000);
849 // clear code selection
850 e.clearSelection();
851 }
852 const getTextToCopy = function(trigger) {
853 const codeEl = trigger.previousElementSibling.cloneNode(true);
854 for (const childEl of codeEl.children) {
855 if (isCodeAnnotation(childEl)) {
856 childEl.remove();
857 }
858 }
859 return codeEl.innerText;
860 }
861 const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
862 text: getTextToCopy
863 });
864 clipboard.on('success', onCopySuccess);
865 if (window.document.getElementById('quarto-embedded-source-code-modal')) {
866 const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', {
867 text: getTextToCopy,
868 container: window.document.getElementById('quarto-embedded-source-code-modal')
869 });
870 clipboardModal.on('success', onCopySuccess);
871 }
872 var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//);
873 var mailtoRegex = new RegExp(/^mailto:/);
874 var filterRegex = new RegExp("https:\/\/github\.com\/your-org\/atdata");
875 var isInternal = (href) => {
876 return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href);
877 }
878 // Inspect non-navigation links and adorn them if external
879 var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)');
880 for (var i=0; i<links.length; i++) {
881 const link = links[i];
882 if (!isInternal(link.href)) {
883 // undo the damage that might have been done by quarto-nav.js in the case of
884 // links that we want to consider external
885 if (link.dataset.originalHref !== undefined) {
886 link.href = link.dataset.originalHref;
887 }
888 }
889 }
890 function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
891 const config = {
892 allowHTML: true,
893 maxWidth: 500,
894 delay: 100,
895 arrow: false,
896 appendTo: function(el) {
897 return el.parentElement;
898 },
899 interactive: true,
900 interactiveBorder: 10,
901 theme: 'quarto',
902 placement: 'bottom-start',
903 };
904 if (contentFn) {
905 config.content = contentFn;
906 }
907 if (onTriggerFn) {
908 config.onTrigger = onTriggerFn;
909 }
910 if (onUntriggerFn) {
911 config.onUntrigger = onUntriggerFn;
912 }
913 window.tippy(el, config);
914 }
915 const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
916 for (var i=0; i<noterefs.length; i++) {
917 const ref = noterefs[i];
918 tippyHover(ref, function() {
919 // use id or data attribute instead here
920 let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
921 try { href = new URL(href).hash; } catch {}
922 const id = href.replace(/^#\/?/, "");
923 const note = window.document.getElementById(id);
924 if (note) {
925 return note.innerHTML;
926 } else {
927 return "";
928 }
929 });
930 }
931 const xrefs = window.document.querySelectorAll('a.quarto-xref');
932 const processXRef = (id, note) => {
933 // Strip column container classes
934 const stripColumnClz = (el) => {
935 el.classList.remove("page-full", "page-columns");
936 if (el.children) {
937 for (const child of el.children) {
938 stripColumnClz(child);
939 }
940 }
941 }
942 stripColumnClz(note)
943 if (id === null || id.startsWith('sec-')) {
944 // Special case sections, only their first couple elements
945 const container = document.createElement("div");
946 if (note.children && note.children.length > 2) {
947 container.appendChild(note.children[0].cloneNode(true));
948 for (let i = 1; i < note.children.length; i++) {
949 const child = note.children[i];
950 if (child.tagName === "P" && child.innerText === "") {
951 continue;
952 } else {
953 container.appendChild(child.cloneNode(true));
954 break;
955 }
956 }
957 if (window.Quarto?.typesetMath) {
958 window.Quarto.typesetMath(container);
959 }
960 return container.innerHTML
961 } else {
962 if (window.Quarto?.typesetMath) {
963 window.Quarto.typesetMath(note);
964 }
965 return note.innerHTML;
966 }
967 } else {
968 // Remove any anchor links if they are present
969 const anchorLink = note.querySelector('a.anchorjs-link');
970 if (anchorLink) {
971 anchorLink.remove();
972 }
973 if (window.Quarto?.typesetMath) {
974 window.Quarto.typesetMath(note);
975 }
976 if (note.classList.contains("callout")) {
977 return note.outerHTML;
978 } else {
979 return note.innerHTML;
980 }
981 }
982 }
983 for (var i=0; i<xrefs.length; i++) {
984 const xref = xrefs[i];
985 tippyHover(xref, undefined, function(instance) {
986 instance.disable();
987 let url = xref.getAttribute('href');
988 let hash = undefined;
989 if (url.startsWith('#')) {
990 hash = url;
991 } else {
992 try { hash = new URL(url).hash; } catch {}
993 }
994 if (hash) {
995 const id = hash.replace(/^#\/?/, "");
996 const note = window.document.getElementById(id);
997 if (note !== null) {
998 try {
999 const html = processXRef(id, note.cloneNode(true));
1000 instance.setContent(html);
1001 } finally {
1002 instance.enable();
1003 instance.show();
1004 }
1005 } else {
1006 // See if we can fetch this
1007 fetch(url.split('#')[0])
1008 .then(res => res.text())
1009 .then(html => {
1010 const parser = new DOMParser();
1011 const htmlDoc = parser.parseFromString(html, "text/html");
1012 const note = htmlDoc.getElementById(id);
1013 if (note !== null) {
1014 const html = processXRef(id, note);
1015 instance.setContent(html);
1016 }
1017 }).finally(() => {
1018 instance.enable();
1019 instance.show();
1020 });
1021 }
1022 } else {
1023 // See if we can fetch a full url (with no hash to target)
1024 // This is a special case and we should probably do some content thinning / targeting
1025 fetch(url)
1026 .then(res => res.text())
1027 .then(html => {
1028 const parser = new DOMParser();
1029 const htmlDoc = parser.parseFromString(html, "text/html");
1030 const note = htmlDoc.querySelector('main.content');
1031 if (note !== null) {
1032 // This should only happen for chapter cross references
1033 // (since there is no id in the URL)
1034 // remove the first header
1035 if (note.children.length > 0 && note.children[0].tagName === "HEADER") {
1036 note.children[0].remove();
1037 }
1038 const html = processXRef(null, note);
1039 instance.setContent(html);
1040 }
1041 }).finally(() => {
1042 instance.enable();
1043 instance.show();
1044 });
1045 }
1046 }, function(instance) {
1047 });
1048 }
1049 let selectedAnnoteEl;
1050 const selectorForAnnotation = ( cell, annotation) => {
1051 let cellAttr = 'data-code-cell="' + cell + '"';
1052 let lineAttr = 'data-code-annotation="' + annotation + '"';
1053 const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
1054 return selector;
1055 }
1056 const selectCodeLines = (annoteEl) => {
1057 const doc = window.document;
1058 const targetCell = annoteEl.getAttribute("data-target-cell");
1059 const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
1060 const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
1061 const lines = annoteSpan.getAttribute("data-code-lines").split(",");
1062 const lineIds = lines.map((line) => {
1063 return targetCell + "-" + line;
1064 })
1065 let top = null;
1066 let height = null;
1067 let parent = null;
1068 if (lineIds.length > 0) {
1069 //compute the position of the single el (top and bottom and make a div)
1070 const el = window.document.getElementById(lineIds[0]);
1071 top = el.offsetTop;
1072 height = el.offsetHeight;
1073 parent = el.parentElement.parentElement;
1074 if (lineIds.length > 1) {
1075 const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
1076 const bottom = lastEl.offsetTop + lastEl.offsetHeight;
1077 height = bottom - top;
1078 }
1079 if (top !== null && height !== null && parent !== null) {
1080 // cook up a div (if necessary) and position it
1081 let div = window.document.getElementById("code-annotation-line-highlight");
1082 if (div === null) {
1083 div = window.document.createElement("div");
1084 div.setAttribute("id", "code-annotation-line-highlight");
1085 div.style.position = 'absolute';
1086 parent.appendChild(div);
1087 }
1088 div.style.top = top - 2 + "px";
1089 div.style.height = height + 4 + "px";
1090 div.style.left = 0;
1091 let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
1092 if (gutterDiv === null) {
1093 gutterDiv = window.document.createElement("div");
1094 gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
1095 gutterDiv.style.position = 'absolute';
1096 const codeCell = window.document.getElementById(targetCell);
1097 const gutter = codeCell.querySelector('.code-annotation-gutter');
1098 gutter.appendChild(gutterDiv);
1099 }
1100 gutterDiv.style.top = top - 2 + "px";
1101 gutterDiv.style.height = height + 4 + "px";
1102 }
1103 selectedAnnoteEl = annoteEl;
1104 }
1105 };
1106 const unselectCodeLines = () => {
1107 const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
1108 elementsIds.forEach((elId) => {
1109 const div = window.document.getElementById(elId);
1110 if (div) {
1111 div.remove();
1112 }
1113 });
1114 selectedAnnoteEl = undefined;
1115 };
1116 // Handle positioning of the toggle
1117 window.addEventListener(
1118 "resize",
1119 throttle(() => {
1120 elRect = undefined;
1121 if (selectedAnnoteEl) {
1122 selectCodeLines(selectedAnnoteEl);
1123 }
1124 }, 10)
1125 );
1126 function throttle(fn, ms) {
1127 let throttle = false;
1128 let timer;
1129 return (...args) => {
1130 if(!throttle) { // first call gets through
1131 fn.apply(this, args);
1132 throttle = true;
1133 } else { // all the others get throttled
1134 if(timer) clearTimeout(timer); // cancel #2
1135 timer = setTimeout(() => {
1136 fn.apply(this, args);
1137 timer = throttle = false;
1138 }, ms);
1139 }
1140 };
1141 }
1142 // Attach click handler to the DT
1143 const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
1144 for (const annoteDlNode of annoteDls) {
1145 annoteDlNode.addEventListener('click', (event) => {
1146 const clickedEl = event.target;
1147 if (clickedEl !== selectedAnnoteEl) {
1148 unselectCodeLines();
1149 const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
1150 if (activeEl) {
1151 activeEl.classList.remove('code-annotation-active');
1152 }
1153 selectCodeLines(clickedEl);
1154 clickedEl.classList.add('code-annotation-active');
1155 } else {
1156 // Unselect the line
1157 unselectCodeLines();
1158 clickedEl.classList.remove('code-annotation-active');
1159 }
1160 });
1161 }
1162 const findCites = (el) => {
1163 const parentEl = el.parentElement;
1164 if (parentEl) {
1165 const cites = parentEl.dataset.cites;
1166 if (cites) {
1167 return {
1168 el,
1169 cites: cites.split(' ')
1170 };
1171 } else {
1172 return findCites(el.parentElement)
1173 }
1174 } else {
1175 return undefined;
1176 }
1177 };
1178 var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
1179 for (var i=0; i<bibliorefs.length; i++) {
1180 const ref = bibliorefs[i];
1181 const citeInfo = findCites(ref);
1182 if (citeInfo) {
1183 tippyHover(citeInfo.el, function() {
1184 var popup = window.document.createElement('div');
1185 citeInfo.cites.forEach(function(cite) {
1186 var citeDiv = window.document.createElement('div');
1187 citeDiv.classList.add('hanging-indent');
1188 citeDiv.classList.add('csl-entry');
1189 var biblioDiv = window.document.getElementById('ref-' + cite);
1190 if (biblioDiv) {
1191 citeDiv.innerHTML = biblioDiv.innerHTML;
1192 }
1193 popup.appendChild(citeDiv);
1194 });
1195 return popup.innerHTML;
1196 });
1197 }
1198 }
1199 });
1200 </script>
1201</div> <!-- /content -->
1202<footer class="footer">
1203 <div class="nav-footer">
1204 <div class="nav-footer-left">
1205<p>Built with <a href="https://quarto.org/">Quarto</a></p>
1206</div>
1207 <div class="nav-footer-center">
1208
1209 <div class="toc-actions d-sm-block d-md-none"><ul><li><a href="https://github.com/your-org/atdata/edit/main/reference/lenses.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></div>
1210 <div class="nav-footer-right">
1211<p>MIT License</p>
1212</div>
1213 </div>
1214</footer>
1215
1216
1217
1218
1219</body></html>