A loose federation of distributed, typed datasets
1<!DOCTYPE html>
2<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
3
4<meta charset="utf-8">
5<meta name="generator" content="quarto-1.7.34">
6
7<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
8
9<meta name="description" content="Understanding the design and components of atdata">
10
11<title>Architecture Overview – atdata</title>
12<style>
13code{white-space: pre-wrap;}
14span.smallcaps{font-variant: small-caps;}
15div.columns{display: flex; gap: min(4vw, 1.5em);}
16div.column{flex: auto; overflow-x: auto;}
17div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
18ul.task-list{list-style: none;}
19ul.task-list li input[type="checkbox"] {
20 width: 0.8em;
21 margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
22 vertical-align: middle;
23}
24/* CSS for syntax highlighting */
25html { -webkit-text-size-adjust: 100%; }
26pre > code.sourceCode { white-space: pre; position: relative; }
27pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
28pre > code.sourceCode > span:empty { height: 1.2em; }
29.sourceCode { overflow: visible; }
30code.sourceCode > span { color: inherit; text-decoration: inherit; }
31div.sourceCode { margin: 1em 0; }
32pre.sourceCode { margin: 0; }
33@media screen {
34div.sourceCode { overflow: auto; }
35}
36@media print {
37pre > code.sourceCode { white-space: pre-wrap; }
38pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
39}
40pre.numberSource code
41 { counter-reset: source-line 0; }
42pre.numberSource code > span
43 { position: relative; left: -4em; counter-increment: source-line; }
44pre.numberSource code > span > a:first-child::before
45 { content: counter(source-line);
46 position: relative; left: -1em; text-align: right; vertical-align: baseline;
47 border: none; display: inline-block;
48 -webkit-touch-callout: none; -webkit-user-select: none;
49 -khtml-user-select: none; -moz-user-select: none;
50 -ms-user-select: none; user-select: none;
51 padding: 0 4px; width: 4em;
52 }
53pre.numberSource { margin-left: 3em; padding-left: 4px; }
54div.sourceCode
55 { }
56@media screen {
57pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
58}
59</style>
60
61
62<script src="../site_libs/quarto-nav/quarto-nav.js"></script>
63<script src="../site_libs/quarto-nav/headroom.min.js"></script>
64<script src="../site_libs/clipboard/clipboard.min.js"></script>
65<script src="../site_libs/quarto-search/autocomplete.umd.js"></script>
66<script src="../site_libs/quarto-search/fuse.min.js"></script>
67<script src="../site_libs/quarto-search/quarto-search.js"></script>
68<meta name="quarto:offset" content="../">
69<script src="../site_libs/quarto-html/quarto.js" type="module"></script>
70<script src="../site_libs/quarto-html/tabsets/tabsets.js" type="module"></script>
71<script src="../site_libs/quarto-html/popper.min.js"></script>
72<script src="../site_libs/quarto-html/tippy.umd.min.js"></script>
73<script src="../site_libs/quarto-html/anchor.min.js"></script>
74<link href="../site_libs/quarto-html/tippy.css" rel="stylesheet">
75<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme" id="quarto-text-highlighting-styles">
76<link href="../site_libs/quarto-html/quarto-syntax-highlighting-dark-8dcd8563ea6803ab7cbb3d71ca5772e1.css" rel="stylesheet" class="quarto-color-scheme quarto-color-alternate" id="quarto-text-highlighting-styles">
77<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme-extra" id="quarto-text-highlighting-styles">
78<script src="../site_libs/bootstrap/bootstrap.min.js"></script>
79<link href="../site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
80<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme" id="quarto-bootstrap" data-mode="light">
81<link href="../site_libs/bootstrap/bootstrap-dark-7964ffd8887b0991fe8d71c6c8bc75d6.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme quarto-color-alternate" id="quarto-bootstrap" data-mode="dark">
82<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme-extra" id="quarto-bootstrap" data-mode="light">
83<script id="quarto-search-options" type="application/json">{
84 "location": "navbar",
85 "copy-button": false,
86 "collapse-after": 3,
87 "panel-placement": "end",
88 "type": "overlay",
89 "limit": 50,
90 "keyboard-shortcut": [
91 "f",
92 "/",
93 "s"
94 ],
95 "show-item-context": false,
96 "language": {
97 "search-no-results-text": "No results",
98 "search-matching-documents-text": "matching documents",
99 "search-copy-link-title": "Copy link to search",
100 "search-hide-matches-text": "Hide additional matches",
101 "search-more-match-text": "more match in this document",
102 "search-more-matches-text": "more matches in this document",
103 "search-clear-button-title": "Clear",
104 "search-text-placeholder": "",
105 "search-detached-cancel-button-title": "Cancel",
106 "search-submit-button-title": "Submit",
107 "search-label": "Search"
108 }
109}</script>
110
111
112<link rel="stylesheet" href="../assets/styles.css">
113</head>
114
115<body class="nav-sidebar docked nav-fixed quarto-light"><script id="quarto-html-before-body" type="application/javascript">
116 const toggleBodyColorMode = (bsSheetEl) => {
117 const mode = bsSheetEl.getAttribute("data-mode");
118 const bodyEl = window.document.querySelector("body");
119 if (mode === "dark") {
120 bodyEl.classList.add("quarto-dark");
121 bodyEl.classList.remove("quarto-light");
122 } else {
123 bodyEl.classList.add("quarto-light");
124 bodyEl.classList.remove("quarto-dark");
125 }
126 }
127 const toggleBodyColorPrimary = () => {
128 const bsSheetEl = window.document.querySelector("link#quarto-bootstrap:not([rel=disabled-stylesheet])");
129 if (bsSheetEl) {
130 toggleBodyColorMode(bsSheetEl);
131 }
132 }
133 const setColorSchemeToggle = (alternate) => {
134 const toggles = window.document.querySelectorAll('.quarto-color-scheme-toggle');
135 for (let i=0; i < toggles.length; i++) {
136 const toggle = toggles[i];
137 if (toggle) {
138 if (alternate) {
139 toggle.classList.add("alternate");
140 } else {
141 toggle.classList.remove("alternate");
142 }
143 }
144 }
145 };
146 const toggleColorMode = (alternate) => {
147 // Switch the stylesheets
148 const primaryStylesheets = window.document.querySelectorAll('link.quarto-color-scheme:not(.quarto-color-alternate)');
149 const alternateStylesheets = window.document.querySelectorAll('link.quarto-color-scheme.quarto-color-alternate');
150 manageTransitions('#quarto-margin-sidebar .nav-link', false);
151 if (alternate) {
152 // note: dark is layered on light, we don't disable primary!
153 enableStylesheet(alternateStylesheets);
154 for (const sheetNode of alternateStylesheets) {
155 if (sheetNode.id === "quarto-bootstrap") {
156 toggleBodyColorMode(sheetNode);
157 }
158 }
159 } else {
160 disableStylesheet(alternateStylesheets);
161 enableStylesheet(primaryStylesheets)
162 toggleBodyColorPrimary();
163 }
164 manageTransitions('#quarto-margin-sidebar .nav-link', true);
165 // Switch the toggles
166 setColorSchemeToggle(alternate)
167 // Hack to workaround the fact that safari doesn't
168 // properly recolor the scrollbar when toggling (#1455)
169 if (navigator.userAgent.indexOf('Safari') > 0 && navigator.userAgent.indexOf('Chrome') == -1) {
170 manageTransitions("body", false);
171 window.scrollTo(0, 1);
172 setTimeout(() => {
173 window.scrollTo(0, 0);
174 manageTransitions("body", true);
175 }, 40);
176 }
177 }
178 const disableStylesheet = (stylesheets) => {
179 for (let i=0; i < stylesheets.length; i++) {
180 const stylesheet = stylesheets[i];
181 stylesheet.rel = 'disabled-stylesheet';
182 }
183 }
184 const enableStylesheet = (stylesheets) => {
185 for (let i=0; i < stylesheets.length; i++) {
186 const stylesheet = stylesheets[i];
187 if(stylesheet.rel !== 'stylesheet') { // for Chrome, which will still FOUC without this check
188 stylesheet.rel = 'stylesheet';
189 }
190 }
191 }
192 const manageTransitions = (selector, allowTransitions) => {
193 const els = window.document.querySelectorAll(selector);
194 for (let i=0; i < els.length; i++) {
195 const el = els[i];
196 if (allowTransitions) {
197 el.classList.remove('notransition');
198 } else {
199 el.classList.add('notransition');
200 }
201 }
202 }
203 const isFileUrl = () => {
204 return window.location.protocol === 'file:';
205 }
206 const hasAlternateSentinel = () => {
207 let styleSentinel = getColorSchemeSentinel();
208 if (styleSentinel !== null) {
209 return styleSentinel === "alternate";
210 } else {
211 return false;
212 }
213 }
214 const setStyleSentinel = (alternate) => {
215 const value = alternate ? "alternate" : "default";
216 if (!isFileUrl()) {
217 window.localStorage.setItem("quarto-color-scheme", value);
218 } else {
219 localAlternateSentinel = value;
220 }
221 }
222 const getColorSchemeSentinel = () => {
223 if (!isFileUrl()) {
224 const storageValue = window.localStorage.getItem("quarto-color-scheme");
225 return storageValue != null ? storageValue : localAlternateSentinel;
226 } else {
227 return localAlternateSentinel;
228 }
229 }
230 const toggleGiscusIfUsed = (isAlternate, darkModeDefault) => {
231 const baseTheme = document.querySelector('#giscus-base-theme')?.value ?? 'light';
232 const alternateTheme = document.querySelector('#giscus-alt-theme')?.value ?? 'dark';
233 let newTheme = '';
234 if(authorPrefersDark) {
235 newTheme = isAlternate ? baseTheme : alternateTheme;
236 } else {
237 newTheme = isAlternate ? alternateTheme : baseTheme;
238 }
239 const changeGiscusTheme = () => {
240 // From: https://github.com/giscus/giscus/issues/336
241 const sendMessage = (message) => {
242 const iframe = document.querySelector('iframe.giscus-frame');
243 if (!iframe) return;
244 iframe.contentWindow.postMessage({ giscus: message }, 'https://giscus.app');
245 }
246 sendMessage({
247 setConfig: {
248 theme: newTheme
249 }
250 });
251 }
252 const isGiscussLoaded = window.document.querySelector('iframe.giscus-frame') !== null;
253 if (isGiscussLoaded) {
254 changeGiscusTheme();
255 }
256 };
257 const authorPrefersDark = false;
258 const darkModeDefault = authorPrefersDark;
259 document.querySelector('link#quarto-text-highlighting-styles.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
260 document.querySelector('link#quarto-bootstrap.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
261 let localAlternateSentinel = darkModeDefault ? 'alternate' : 'default';
262 // Dark / light mode switch
263 window.quartoToggleColorScheme = () => {
264 // Read the current dark / light value
265 let toAlternate = !hasAlternateSentinel();
266 toggleColorMode(toAlternate);
267 setStyleSentinel(toAlternate);
268 toggleGiscusIfUsed(toAlternate, darkModeDefault);
269 window.dispatchEvent(new Event('resize'));
270 };
271 // Switch to dark mode if need be
272 if (hasAlternateSentinel()) {
273 toggleColorMode(true);
274 } else {
275 toggleColorMode(false);
276 }
277 </script>
278
279<div id="quarto-search-results"></div>
280 <header id="quarto-header" class="headroom fixed-top">
281 <nav class="navbar navbar-expand-lg " data-bs-theme="dark">
282 <div class="navbar-container container-fluid">
283 <div class="navbar-brand-container mx-auto">
284 <a class="navbar-brand" href="../index.html">
285 <span class="navbar-title">atdata</span>
286 </a>
287 </div>
288 <div id="quarto-search" class="" title="Search"></div>
289 <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarCollapse" aria-controls="navbarCollapse" role="menu" aria-expanded="false" aria-label="Toggle navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
290 <span class="navbar-toggler-icon"></span>
291</button>
292 <div class="collapse navbar-collapse" id="navbarCollapse">
293 <ul class="navbar-nav navbar-nav-scroll me-auto">
294 <li class="nav-item">
295 <a class="nav-link active" href="../index.html" aria-current="page">
296<span class="menu-text">Guide</span></a>
297 </li>
298 <li class="nav-item dropdown ">
299 <a class="nav-link dropdown-toggle" href="#" id="nav-menu-tutorials" role="link" data-bs-toggle="dropdown" aria-expanded="false">
300 <span class="menu-text">Tutorials</span>
301 </a>
302 <ul class="dropdown-menu" aria-labelledby="nav-menu-tutorials">
303 <li>
304 <a class="dropdown-item" href="../tutorials/quickstart.html">
305 <span class="dropdown-text">Quick Start</span></a>
306 </li>
307 <li>
308 <a class="dropdown-item" href="../tutorials/local-workflow.html">
309 <span class="dropdown-text">Local Workflow</span></a>
310 </li>
311 <li>
312 <a class="dropdown-item" href="../tutorials/atmosphere.html">
313 <span class="dropdown-text">Atmosphere Publishing</span></a>
314 </li>
315 <li>
316 <a class="dropdown-item" href="../tutorials/promotion.html">
317 <span class="dropdown-text">Promotion Workflow</span></a>
318 </li>
319 </ul>
320 </li>
321 <li class="nav-item dropdown ">
322 <a class="nav-link dropdown-toggle" href="#" id="nav-menu-reference" role="link" data-bs-toggle="dropdown" aria-expanded="false">
323 <span class="menu-text">Reference</span>
324 </a>
325 <ul class="dropdown-menu" aria-labelledby="nav-menu-reference">
326 <li>
327 <a class="dropdown-item" href="../reference/architecture.html">
328 <span class="dropdown-text">Architecture Overview</span></a>
329 </li>
330 <li>
331 <a class="dropdown-item" href="../reference/packable-samples.html">
332 <span class="dropdown-text">Packable Samples</span></a>
333 </li>
334 <li>
335 <a class="dropdown-item" href="../reference/datasets.html">
336 <span class="dropdown-text">Datasets</span></a>
337 </li>
338 <li>
339 <a class="dropdown-item" href="../reference/lenses.html">
340 <span class="dropdown-text">Lenses</span></a>
341 </li>
342 <li>
343 <a class="dropdown-item" href="../reference/local-storage.html">
344 <span class="dropdown-text">Local Storage</span></a>
345 </li>
346 <li>
347 <a class="dropdown-item" href="../reference/atmosphere.html">
348 <span class="dropdown-text">Atmosphere</span></a>
349 </li>
350 <li>
351 <a class="dropdown-item" href="../reference/promotion.html">
352 <span class="dropdown-text">Promotion</span></a>
353 </li>
354 <li>
355 <a class="dropdown-item" href="../reference/load-dataset.html">
356 <span class="dropdown-text">load_dataset API</span></a>
357 </li>
358 <li>
359 <a class="dropdown-item" href="../reference/protocols.html">
360 <span class="dropdown-text">Protocols</span></a>
361 </li>
362 <li>
363 <a class="dropdown-item" href="../reference/uri-spec.html">
364 <span class="dropdown-text">URI Specification</span></a>
365 </li>
366 <li>
367 <a class="dropdown-item" href="../reference/troubleshooting.html">
368 <span class="dropdown-text">Troubleshooting & FAQ</span></a>
369 </li>
370 <li>
371 <a class="dropdown-item" href="../reference/deployment.html">
372 <span class="dropdown-text">Deployment Guide</span></a>
373 </li>
374 </ul>
375 </li>
376 <li class="nav-item">
377 <a class="nav-link" href="../api/index.html">
378<span class="menu-text">API</span></a>
379 </li>
380</ul>
381 <ul class="navbar-nav navbar-nav-scroll ms-auto">
382 <li class="nav-item compact">
383 <a class="nav-link" href="https://github.com/your-org/atdata"> <i class="bi bi-github" role="img">
384</i>
385<span class="menu-text"></span></a>
386 </li>
387</ul>
388 </div> <!-- /navcollapse -->
389 <div class="quarto-navbar-tools">
390 <a href="" class="quarto-color-scheme-toggle quarto-navigation-tool px-1" onclick="window.quartoToggleColorScheme(); return false;" title="Toggle dark mode"><i class="bi"></i></a>
391</div>
392 </div> <!-- /container-fluid -->
393 </nav>
394 <nav class="quarto-secondary-nav">
395 <div class="container-fluid d-flex">
396 <button type="button" class="quarto-btn-toggle btn" data-bs-toggle="collapse" role="button" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
397 <i class="bi bi-layout-text-sidebar-reverse"></i>
398 </button>
399 <nav class="quarto-page-breadcrumbs" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item"><a href="../reference/architecture.html">Reference</a></li><li class="breadcrumb-item"><a href="../reference/architecture.html">Architecture Overview</a></li></ol></nav>
400 <a class="flex-grow-1" role="navigation" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
401 </a>
402 </div>
403 </nav>
404</header>
405<!-- content -->
406<div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-article page-navbar">
407<!-- sidebar -->
408 <nav id="quarto-sidebar" class="sidebar collapse collapse-horizontal quarto-sidebar-collapse-item sidebar-navigation docked overflow-auto">
409 <div class="sidebar-menu-container">
410 <ul class="list-unstyled mt-1">
411 <li class="sidebar-item">
412 <div class="sidebar-item-container">
413 <a href="../index.html" class="sidebar-item-text sidebar-link">
414 <span class="menu-text">atdata</span></a>
415 </div>
416</li>
417 <li class="sidebar-item sidebar-item-section">
418 <div class="sidebar-item-container">
419 <a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" role="navigation" aria-expanded="true">
420 <span class="menu-text">Getting Started</span></a>
421 <a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" role="navigation" aria-expanded="true" aria-label="Toggle section">
422 <i class="bi bi-chevron-right ms-2"></i>
423 </a>
424 </div>
425 <ul id="quarto-sidebar-section-1" class="collapse list-unstyled sidebar-section depth1 show">
426 <li class="sidebar-item">
427 <div class="sidebar-item-container">
428 <a href="../tutorials/quickstart.html" class="sidebar-item-text sidebar-link">
429 <span class="menu-text">Quick Start</span></a>
430 </div>
431</li>
432 <li class="sidebar-item">
433 <div class="sidebar-item-container">
434 <a href="../tutorials/local-workflow.html" class="sidebar-item-text sidebar-link">
435 <span class="menu-text">Local Workflow</span></a>
436 </div>
437</li>
438 <li class="sidebar-item">
439 <div class="sidebar-item-container">
440 <a href="../tutorials/atmosphere.html" class="sidebar-item-text sidebar-link">
441 <span class="menu-text">Atmosphere Publishing</span></a>
442 </div>
443</li>
444 <li class="sidebar-item">
445 <div class="sidebar-item-container">
446 <a href="../tutorials/promotion.html" class="sidebar-item-text sidebar-link">
447 <span class="menu-text">Promotion Workflow</span></a>
448 </div>
449</li>
450 </ul>
451 </li>
452 <li class="sidebar-item sidebar-item-section">
453 <div class="sidebar-item-container">
454 <a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" role="navigation" aria-expanded="true">
455 <span class="menu-text">Reference</span></a>
456 <a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" role="navigation" aria-expanded="true" aria-label="Toggle section">
457 <i class="bi bi-chevron-right ms-2"></i>
458 </a>
459 </div>
460 <ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
461 <li class="sidebar-item">
462 <div class="sidebar-item-container">
463 <a href="../reference/architecture.html" class="sidebar-item-text sidebar-link active">
464 <span class="menu-text">Architecture Overview</span></a>
465 </div>
466</li>
467 <li class="sidebar-item">
468 <div class="sidebar-item-container">
469 <a href="../reference/packable-samples.html" class="sidebar-item-text sidebar-link">
470 <span class="menu-text">Packable Samples</span></a>
471 </div>
472</li>
473 <li class="sidebar-item">
474 <div class="sidebar-item-container">
475 <a href="../reference/datasets.html" class="sidebar-item-text sidebar-link">
476 <span class="menu-text">Datasets</span></a>
477 </div>
478</li>
479 <li class="sidebar-item">
480 <div class="sidebar-item-container">
481 <a href="../reference/lenses.html" class="sidebar-item-text sidebar-link">
482 <span class="menu-text">Lenses</span></a>
483 </div>
484</li>
485 <li class="sidebar-item">
486 <div class="sidebar-item-container">
487 <a href="../reference/local-storage.html" class="sidebar-item-text sidebar-link">
488 <span class="menu-text">Local Storage</span></a>
489 </div>
490</li>
491 <li class="sidebar-item">
492 <div class="sidebar-item-container">
493 <a href="../reference/atmosphere.html" class="sidebar-item-text sidebar-link">
494 <span class="menu-text">Atmosphere (ATProto Integration)</span></a>
495 </div>
496</li>
497 <li class="sidebar-item">
498 <div class="sidebar-item-container">
499 <a href="../reference/promotion.html" class="sidebar-item-text sidebar-link">
500 <span class="menu-text">Promotion Workflow</span></a>
501 </div>
502</li>
503 <li class="sidebar-item">
504 <div class="sidebar-item-container">
505 <a href="../reference/load-dataset.html" class="sidebar-item-text sidebar-link">
506 <span class="menu-text">load_dataset API</span></a>
507 </div>
508</li>
509 <li class="sidebar-item">
510 <div class="sidebar-item-container">
511 <a href="../reference/protocols.html" class="sidebar-item-text sidebar-link">
512 <span class="menu-text">Protocols</span></a>
513 </div>
514</li>
515 <li class="sidebar-item">
516 <div class="sidebar-item-container">
517 <a href="../reference/uri-spec.html" class="sidebar-item-text sidebar-link">
518 <span class="menu-text">URI Specification</span></a>
519 </div>
520</li>
521 <li class="sidebar-item">
522 <div class="sidebar-item-container">
523 <a href="../reference/troubleshooting.html" class="sidebar-item-text sidebar-link">
524 <span class="menu-text">Troubleshooting & FAQ</span></a>
525 </div>
526</li>
527 <li class="sidebar-item">
528 <div class="sidebar-item-container">
529 <a href="../reference/deployment.html" class="sidebar-item-text sidebar-link">
530 <span class="menu-text">Deployment Guide</span></a>
531 </div>
532</li>
533 </ul>
534 </li>
535 </ul>
536 </div>
537</nav>
538<div id="quarto-sidebar-glass" class="quarto-sidebar-collapse-item" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item"></div>
539<!-- margin-sidebar -->
540 <div id="quarto-margin-sidebar" class="sidebar margin-sidebar">
541 <nav id="TOC" role="doc-toc" class="toc-active">
542 <h2 id="toc-title">On this page</h2>
543
544 <ul>
545 <li><a href="#design-philosophy" id="toc-design-philosophy" class="nav-link active" data-scroll-target="#design-philosophy">Design Philosophy</a>
546 <ul class="collapse">
547 <li><a href="#the-problem" id="toc-the-problem" class="nav-link" data-scroll-target="#the-problem">The Problem</a></li>
548 <li><a href="#the-solution" id="toc-the-solution" class="nav-link" data-scroll-target="#the-solution">The Solution</a></li>
549 </ul></li>
550 <li><a href="#core-components" id="toc-core-components" class="nav-link" data-scroll-target="#core-components">Core Components</a>
551 <ul class="collapse">
552 <li><a href="#packablesample-the-foundation" id="toc-packablesample-the-foundation" class="nav-link" data-scroll-target="#packablesample-the-foundation">PackableSample: The Foundation</a></li>
553 <li><a href="#dataset-typed-iteration" id="toc-dataset-typed-iteration" class="nav-link" data-scroll-target="#dataset-typed-iteration">Dataset: Typed Iteration</a></li>
554 <li><a href="#samplebatch-automatic-aggregation" id="toc-samplebatch-automatic-aggregation" class="nav-link" data-scroll-target="#samplebatch-automatic-aggregation">SampleBatch: Automatic Aggregation</a></li>
555 <li><a href="#lens-schema-transformations" id="toc-lens-schema-transformations" class="nav-link" data-scroll-target="#lens-schema-transformations">Lens: Schema Transformations</a></li>
556 </ul></li>
557 <li><a href="#storage-backends" id="toc-storage-backends" class="nav-link" data-scroll-target="#storage-backends">Storage Backends</a>
558 <ul class="collapse">
559 <li><a href="#local-index-redis-s3" id="toc-local-index-redis-s3" class="nav-link" data-scroll-target="#local-index-redis-s3">Local Index (Redis + S3)</a></li>
560 <li><a href="#atmosphere-index-atproto" id="toc-atmosphere-index-atproto" class="nav-link" data-scroll-target="#atmosphere-index-atproto">Atmosphere Index (ATProto)</a></li>
561 </ul></li>
562 <li><a href="#protocol-abstractions" id="toc-protocol-abstractions" class="nav-link" data-scroll-target="#protocol-abstractions">Protocol Abstractions</a>
563 <ul class="collapse">
564 <li><a href="#abstractindex" id="toc-abstractindex" class="nav-link" data-scroll-target="#abstractindex">AbstractIndex</a></li>
565 <li><a href="#abstractdatastore" id="toc-abstractdatastore" class="nav-link" data-scroll-target="#abstractdatastore">AbstractDataStore</a></li>
566 <li><a href="#datasource" id="toc-datasource" class="nav-link" data-scroll-target="#datasource">DataSource</a></li>
567 </ul></li>
568 <li><a href="#data-flow-local-to-federation" id="toc-data-flow-local-to-federation" class="nav-link" data-scroll-target="#data-flow-local-to-federation">Data Flow: Local to Federation</a>
569 <ul class="collapse">
570 <li><a href="#stage-1-local-development" id="toc-stage-1-local-development" class="nav-link" data-scroll-target="#stage-1-local-development">Stage 1: Local Development</a></li>
571 <li><a href="#stage-2-team-storage" id="toc-stage-2-team-storage" class="nav-link" data-scroll-target="#stage-2-team-storage">Stage 2: Team Storage</a></li>
572 <li><a href="#stage-3-federation" id="toc-stage-3-federation" class="nav-link" data-scroll-target="#stage-3-federation">Stage 3: Federation</a></li>
573 </ul></li>
574 <li><a href="#content-addressing" id="toc-content-addressing" class="nav-link" data-scroll-target="#content-addressing">Content Addressing</a></li>
575 <li><a href="#extension-points" id="toc-extension-points" class="nav-link" data-scroll-target="#extension-points">Extension Points</a>
576 <ul class="collapse">
577 <li><a href="#custom-datasources" id="toc-custom-datasources" class="nav-link" data-scroll-target="#custom-datasources">Custom DataSources</a></li>
578 <li><a href="#custom-lenses" id="toc-custom-lenses" class="nav-link" data-scroll-target="#custom-lenses">Custom Lenses</a></li>
579 <li><a href="#schema-extensions" id="toc-schema-extensions" class="nav-link" data-scroll-target="#schema-extensions">Schema Extensions</a></li>
580 </ul></li>
581 <li><a href="#summary" id="toc-summary" class="nav-link" data-scroll-target="#summary">Summary</a></li>
582 <li><a href="#related" id="toc-related" class="nav-link" data-scroll-target="#related">Related</a></li>
583 </ul>
584<div class="toc-actions"><ul><li><a href="https://github.com/your-org/atdata/edit/main/reference/architecture.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></nav>
585 </div>
586<!-- main -->
587<main class="content" id="quarto-document-content">
588
589
590<header id="title-block-header" class="quarto-title-block default"><nav class="quarto-page-breadcrumbs quarto-title-breadcrumbs d-none d-lg-block" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item"><a href="../reference/architecture.html">Reference</a></li><li class="breadcrumb-item"><a href="../reference/architecture.html">Architecture Overview</a></li></ol></nav>
591<div class="quarto-title">
592<h1 class="title">Architecture Overview</h1>
593</div>
594
595<div>
596 <div class="description">
597 Understanding the design and components of atdata
598 </div>
599</div>
600
601
602<div class="quarto-title-meta">
603
604
605
606
607 </div>
608
609
610
611</header>
612
613
614<p>atdata is designed around a simple but powerful idea: <strong>typed, serializable samples</strong> that can flow seamlessly between local development, team storage, and a federated network. This page explains the architectural decisions and how the components work together.</p>
615<section id="design-philosophy" class="level2">
616<h2 class="anchored" data-anchor-id="design-philosophy">Design Philosophy</h2>
617<section id="the-problem" class="level3">
618<h3 class="anchored" data-anchor-id="the-problem">The Problem</h3>
619<p>Machine learning workflows involve datasets at every stage—training data, validation sets, embeddings, features, and model outputs. These datasets are often:</p>
620<ul>
621<li><strong>Untyped</strong>: Raw files with implicit schemas, leading to runtime errors</li>
622<li><strong>Siloed</strong>: Stuck in one location (local disk, team bucket, or cloud storage)</li>
623<li><strong>Undiscoverable</strong>: No standard way to find and share datasets across teams or organizations</li>
624</ul>
625</section>
626<section id="the-solution" class="level3">
627<h3 class="anchored" data-anchor-id="the-solution">The Solution</h3>
628<p>atdata provides a three-layer architecture that addresses each problem:</p>
629<pre><code>┌─────────────────────────────────────────────────────────────┐
630│ Layer 3: Federation (ATProto Atmosphere) │
631│ - Decentralized discovery and sharing │
632│ - Content-addressable identifiers │
633│ - Cross-organization dataset federation │
634└─────────────────────────────────────────────────────────────┘
635 ↑
636 Promotion
637 ↑
638┌─────────────────────────────────────────────────────────────┐
639│ Layer 2: Team Storage (Redis + S3) │
640│ - Shared index for team discovery │
641│ - Scalable object storage for data │
642│ - Schema registry for type consistency │
643└─────────────────────────────────────────────────────────────┘
644 ↑
645 Insert
646 ↑
647┌─────────────────────────────────────────────────────────────┐
648│ Layer 1: Local Development │
649│ - Typed samples with automatic serialization │
650│ - WebDataset tar files for efficient storage │
651│ - Lens transformations for schema flexibility │
652└─────────────────────────────────────────────────────────────┘</code></pre>
653</section>
654</section>
655<section id="core-components" class="level2">
656<h2 class="anchored" data-anchor-id="core-components">Core Components</h2>
657<section id="packablesample-the-foundation" class="level3">
658<h3 class="anchored" data-anchor-id="packablesample-the-foundation">PackableSample: The Foundation</h3>
659<p>Everything in atdata starts with <strong>PackableSample</strong>—a base class that makes Python dataclasses serializable with msgpack:</p>
660<div id="935164ef" class="cell">
661<div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
662<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> ImageSample:</span>
663<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a> image: NDArray <span class="co"># Automatically converted to/from bytes</span></span>
664<span id="cb2-4"><a href="#cb2-4" aria-hidden="true" tabindex="-1"></a> label: <span class="bu">str</span> <span class="co"># Standard msgpack serialization</span></span>
665<span id="cb2-5"><a href="#cb2-5" aria-hidden="true" tabindex="-1"></a> confidence: <span class="bu">float</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
666</div>
667<p>Key features:</p>
668<ul>
669<li><strong>Automatic NDArray handling</strong>: Numpy arrays are serialized efficiently</li>
670<li><strong>Type safety</strong>: Field types are preserved and validated</li>
671<li><strong>Round-trip fidelity</strong>: Serialize → deserialize always produces identical data</li>
672</ul>
673<p>The <code>@packable</code> decorator is syntactic sugar that:</p>
674<ol type="1">
675<li>Converts your class to a dataclass</li>
676<li>Adds <code>PackableSample</code> as a base class</li>
677<li>Registers a lens from <code>DictSample</code> for flexible loading</li>
678</ol>
679</section>
680<section id="dataset-typed-iteration" class="level3">
681<h3 class="anchored" data-anchor-id="dataset-typed-iteration">Dataset: Typed Iteration</h3>
682<p>The <code>Dataset[T]</code> class wraps WebDataset tar archives with type information:</p>
683<div id="cb4b1f5d" class="cell">
684<div class="sourceCode cell-code" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a>dataset <span class="op">=</span> atdata.Dataset[ImageSample](<span class="st">"data-{000000..000009}.tar"</span>)</span>
685<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a></span>
686<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a><span class="cf">for</span> batch <span class="kw">in</span> dataset.shuffled(batch_size<span class="op">=</span><span class="dv">32</span>):</span>
687<span id="cb3-4"><a href="#cb3-4" aria-hidden="true" tabindex="-1"></a> images <span class="op">=</span> batch.image <span class="co"># Stacked NDArray: (32, H, W, C)</span></span>
688<span id="cb3-5"><a href="#cb3-5" aria-hidden="true" tabindex="-1"></a> labels <span class="op">=</span> batch.label <span class="co"># List of 32 strings</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
689</div>
690<p><strong>Why WebDataset?</strong></p>
691<p>WebDataset is a battle-tested format for large-scale ML training:</p>
692<ul>
693<li><strong>Streaming</strong>: No need to download entire datasets</li>
694<li><strong>Sharding</strong>: Data split across multiple tar files for parallelism</li>
695<li><strong>Shuffling</strong>: Two-level shuffling (shard + sample) for training</li>
696</ul>
697<p>atdata adds:</p>
698<ul>
699<li><strong>Type safety</strong>: Know the schema at compile time</li>
700<li><strong>Batch aggregation</strong>: NDArrays are automatically stacked</li>
701<li><strong>Lens transformations</strong>: View data through different schemas</li>
702</ul>
703</section>
704<section id="samplebatch-automatic-aggregation" class="level3">
705<h3 class="anchored" data-anchor-id="samplebatch-automatic-aggregation">SampleBatch: Automatic Aggregation</h3>
706<p>When iterating with <code>batch_size</code>, atdata returns <code>SampleBatch[T]</code> objects that aggregate sample attributes:</p>
707<div id="dac750c2" class="cell">
708<div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a>batch <span class="op">=</span> SampleBatch[ImageSample](samples)</span>
709<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a></span>
710<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a><span class="co"># NDArray fields → stacked numpy array with batch dimension</span></span>
711<span id="cb4-4"><a href="#cb4-4" aria-hidden="true" tabindex="-1"></a>batch.image.shape <span class="co"># (batch_size, H, W, C)</span></span>
712<span id="cb4-5"><a href="#cb4-5" aria-hidden="true" tabindex="-1"></a></span>
713<span id="cb4-6"><a href="#cb4-6" aria-hidden="true" tabindex="-1"></a><span class="co"># Other fields → list</span></span>
714<span id="cb4-7"><a href="#cb4-7" aria-hidden="true" tabindex="-1"></a>batch.label <span class="co"># ["cat", "dog", "bird", ...]</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
715</div>
716<p>This eliminates boilerplate collation code and works automatically for any <code>PackableSample</code> type.</p>
717</section>
718<section id="lens-schema-transformations" class="level3">
719<h3 class="anchored" data-anchor-id="lens-schema-transformations">Lens: Schema Transformations</h3>
720<p>Lenses enable viewing datasets through different schemas without duplicating data:</p>
721<div id="06cb19a2" class="cell">
722<div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
723<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> SimplifiedSample:</span>
724<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a> label: <span class="bu">str</span></span>
725<span id="cb5-4"><a href="#cb5-4" aria-hidden="true" tabindex="-1"></a></span>
726<span id="cb5-5"><a href="#cb5-5" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.lens</span></span>
727<span id="cb5-6"><a href="#cb5-6" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> simplify(src: ImageSample) <span class="op">-></span> SimplifiedSample:</span>
728<span id="cb5-7"><a href="#cb5-7" aria-hidden="true" tabindex="-1"></a> <span class="cf">return</span> SimplifiedSample(label<span class="op">=</span>src.label)</span>
729<span id="cb5-8"><a href="#cb5-8" aria-hidden="true" tabindex="-1"></a></span>
730<span id="cb5-9"><a href="#cb5-9" aria-hidden="true" tabindex="-1"></a><span class="co"># View dataset through simplified schema</span></span>
731<span id="cb5-10"><a href="#cb5-10" aria-hidden="true" tabindex="-1"></a>simple_ds <span class="op">=</span> dataset.as_type(SimplifiedSample)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
732</div>
733<p><strong>When to use lenses:</strong></p>
734<ul>
735<li><strong>Reducing fields</strong>: Drop unnecessary data for specific tasks</li>
736<li><strong>Transforming data</strong>: Compute derived fields on-the-fly</li>
737<li><strong>Schema migration</strong>: Handle version differences between datasets</li>
738</ul>
739<p>Lenses are registered globally in a <code>LensNetwork</code>, enabling automatic discovery of transformation paths.</p>
740</section>
741</section>
742<section id="storage-backends" class="level2">
743<h2 class="anchored" data-anchor-id="storage-backends">Storage Backends</h2>
744<section id="local-index-redis-s3" class="level3">
745<h3 class="anchored" data-anchor-id="local-index-redis-s3">Local Index (Redis + S3)</h3>
746<p>For team-scale usage, atdata provides a two-component storage system:</p>
747<p><strong>Redis Index</strong>: Stores metadata and enables fast lookups</p>
748<ul>
749<li>Dataset entries (name, schema, URLs, metadata)</li>
750<li>Schema registry (type definitions)</li>
751<li>CID-based content addressing</li>
752</ul>
753<p><strong>S3 DataStore</strong>: Stores actual data files</p>
754<ul>
755<li>WebDataset tar shards</li>
756<li>Any S3-compatible storage (AWS, MinIO, Cloudflare R2)</li>
757</ul>
758<div id="046d8bbf" class="cell">
759<div class="sourceCode cell-code" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a>store <span class="op">=</span> S3DataStore(credentials<span class="op">=</span>creds, bucket<span class="op">=</span><span class="st">"datasets"</span>)</span>
760<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a>index <span class="op">=</span> LocalIndex(data_store<span class="op">=</span>store)</span>
761<span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a></span>
762<span id="cb6-4"><a href="#cb6-4" aria-hidden="true" tabindex="-1"></a><span class="co"># Insert dataset: writes to S3, indexes in Redis</span></span>
763<span id="cb6-5"><a href="#cb6-5" aria-hidden="true" tabindex="-1"></a>entry <span class="op">=</span> index.insert_dataset(dataset, name<span class="op">=</span><span class="st">"training-v1"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
764</div>
765<p><strong>Why this split?</strong></p>
766<ul>
767<li><strong>Separation of concerns</strong>: Metadata queries don’t touch data storage</li>
768<li><strong>Flexibility</strong>: Use any S3-compatible storage</li>
769<li><strong>Scalability</strong>: Redis handles high-throughput lookups; S3 handles large files</li>
770</ul>
771</section>
772<section id="atmosphere-index-atproto" class="level3">
773<h3 class="anchored" data-anchor-id="atmosphere-index-atproto">Atmosphere Index (ATProto)</h3>
774<p>For public or cross-organization sharing, atdata integrates with the AT Protocol:</p>
775<p><strong>ATProto PDS</strong>: Your Personal Data Server stores records</p>
776<ul>
777<li>Schema definitions</li>
778<li>Dataset index records</li>
779<li>Lens transformation records</li>
780</ul>
781<p><strong>PDSBlobStore</strong>: Optional blob storage on your PDS</p>
782<ul>
783<li>Store actual data shards as ATProto blobs</li>
784<li>Fully decentralized—no external dependencies</li>
785</ul>
786<div id="eb3771a4" class="cell">
787<div class="sourceCode cell-code" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a>client <span class="op">=</span> AtmosphereClient()</span>
788<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a>client.login(<span class="st">"handle.bsky.social"</span>, <span class="st">"app-password"</span>)</span>
789<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a></span>
790<span id="cb7-4"><a href="#cb7-4" aria-hidden="true" tabindex="-1"></a>store <span class="op">=</span> PDSBlobStore(client)</span>
791<span id="cb7-5"><a href="#cb7-5" aria-hidden="true" tabindex="-1"></a>index <span class="op">=</span> AtmosphereIndex(client, data_store<span class="op">=</span>store)</span>
792<span id="cb7-6"><a href="#cb7-6" aria-hidden="true" tabindex="-1"></a></span>
793<span id="cb7-7"><a href="#cb7-7" aria-hidden="true" tabindex="-1"></a><span class="co"># Publish: creates ATProto records, uploads blobs</span></span>
794<span id="cb7-8"><a href="#cb7-8" aria-hidden="true" tabindex="-1"></a>entry <span class="op">=</span> index.insert_dataset(dataset, name<span class="op">=</span><span class="st">"public-features"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
795</div>
796</section>
797</section>
798<section id="protocol-abstractions" class="level2">
799<h2 class="anchored" data-anchor-id="protocol-abstractions">Protocol Abstractions</h2>
800<p>atdata uses <strong>protocols</strong> (structural typing) to enable backend interoperability:</p>
801<section id="abstractindex" class="level3">
802<h3 class="anchored" data-anchor-id="abstractindex">AbstractIndex</h3>
803<p>Common interface for both <code>LocalIndex</code> and <code>AtmosphereIndex</code>:</p>
804<div id="d95e2c44" class="cell">
805<div class="sourceCode cell-code" id="cb8"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> process_dataset(index: AbstractIndex, name: <span class="bu">str</span>):</span>
806<span id="cb8-2"><a href="#cb8-2" aria-hidden="true" tabindex="-1"></a> entry <span class="op">=</span> index.get_dataset(name)</span>
807<span id="cb8-3"><a href="#cb8-3" aria-hidden="true" tabindex="-1"></a> schema <span class="op">=</span> index.decode_schema(entry.schema_ref)</span>
808<span id="cb8-4"><a href="#cb8-4" aria-hidden="true" tabindex="-1"></a> <span class="co"># Works with either LocalIndex or AtmosphereIndex</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
809</div>
810<p>Key methods:</p>
811<ul>
812<li><code>insert_dataset()</code> / <code>get_dataset()</code>: Dataset CRUD</li>
813<li><code>publish_schema()</code> / <code>decode_schema()</code>: Schema management</li>
814<li><code>list_datasets()</code> / <code>list_schemas()</code>: Discovery</li>
815</ul>
816</section>
817<section id="abstractdatastore" class="level3">
818<h3 class="anchored" data-anchor-id="abstractdatastore">AbstractDataStore</h3>
819<p>Common interface for <code>S3DataStore</code> and <code>PDSBlobStore</code>:</p>
820<div id="d5c8d1d6" class="cell">
821<div class="sourceCode cell-code" id="cb9"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb9-1"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> write_to_store(store: AbstractDataStore, dataset: Dataset):</span>
822<span id="cb9-2"><a href="#cb9-2" aria-hidden="true" tabindex="-1"></a> urls <span class="op">=</span> store.write_shards(dataset, prefix<span class="op">=</span><span class="st">"data/v1"</span>)</span>
823<span id="cb9-3"><a href="#cb9-3" aria-hidden="true" tabindex="-1"></a> <span class="co"># Works with S3 or PDS blob storage</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
824</div>
825</section>
826<section id="datasource" class="level3">
827<h3 class="anchored" data-anchor-id="datasource">DataSource</h3>
828<p>Common interface for data streaming:</p>
829<ul>
830<li><code>URLSource</code>: WebDataset-compatible URLs</li>
831<li><code>S3Source</code>: S3 with explicit credentials</li>
832<li><code>BlobSource</code>: ATProto PDS blobs</li>
833</ul>
834</section>
835</section>
836<section id="data-flow-local-to-federation" class="level2">
837<h2 class="anchored" data-anchor-id="data-flow-local-to-federation">Data Flow: Local to Federation</h2>
838<p>A typical workflow progresses through three stages:</p>
839<section id="stage-1-local-development" class="level3">
840<h3 class="anchored" data-anchor-id="stage-1-local-development">Stage 1: Local Development</h3>
841<div id="d075eb1f" class="cell">
842<div class="sourceCode cell-code" id="cb10"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Define type and create samples</span></span>
843<span id="cb10-2"><a href="#cb10-2" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
844<span id="cb10-3"><a href="#cb10-3" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> MySample:</span>
845<span id="cb10-4"><a href="#cb10-4" aria-hidden="true" tabindex="-1"></a> features: NDArray</span>
846<span id="cb10-5"><a href="#cb10-5" aria-hidden="true" tabindex="-1"></a> label: <span class="bu">str</span></span>
847<span id="cb10-6"><a href="#cb10-6" aria-hidden="true" tabindex="-1"></a></span>
848<span id="cb10-7"><a href="#cb10-7" aria-hidden="true" tabindex="-1"></a><span class="co"># Write to local tar</span></span>
849<span id="cb10-8"><a href="#cb10-8" aria-hidden="true" tabindex="-1"></a><span class="cf">with</span> wds.writer.TarWriter(<span class="st">"data.tar"</span>) <span class="im">as</span> sink:</span>
850<span id="cb10-9"><a href="#cb10-9" aria-hidden="true" tabindex="-1"></a> <span class="cf">for</span> sample <span class="kw">in</span> samples:</span>
851<span id="cb10-10"><a href="#cb10-10" aria-hidden="true" tabindex="-1"></a> sink.write(sample.as_wds)</span>
852<span id="cb10-11"><a href="#cb10-11" aria-hidden="true" tabindex="-1"></a></span>
853<span id="cb10-12"><a href="#cb10-12" aria-hidden="true" tabindex="-1"></a><span class="co"># Iterate locally</span></span>
854<span id="cb10-13"><a href="#cb10-13" aria-hidden="true" tabindex="-1"></a>dataset <span class="op">=</span> atdata.Dataset[MySample](<span class="st">"data.tar"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
855</div>
856</section>
857<section id="stage-2-team-storage" class="level3">
858<h3 class="anchored" data-anchor-id="stage-2-team-storage">Stage 2: Team Storage</h3>
859<div id="006a8036" class="cell">
860<div class="sourceCode cell-code" id="cb11"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb11-1"><a href="#cb11-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Set up team storage</span></span>
861<span id="cb11-2"><a href="#cb11-2" aria-hidden="true" tabindex="-1"></a>store <span class="op">=</span> S3DataStore(credentials<span class="op">=</span>team_creds, bucket<span class="op">=</span><span class="st">"team-datasets"</span>)</span>
862<span id="cb11-3"><a href="#cb11-3" aria-hidden="true" tabindex="-1"></a>index <span class="op">=</span> LocalIndex(data_store<span class="op">=</span>store)</span>
863<span id="cb11-4"><a href="#cb11-4" aria-hidden="true" tabindex="-1"></a></span>
864<span id="cb11-5"><a href="#cb11-5" aria-hidden="true" tabindex="-1"></a><span class="co"># Publish schema and insert</span></span>
865<span id="cb11-6"><a href="#cb11-6" aria-hidden="true" tabindex="-1"></a>index.publish_schema(MySample, version<span class="op">=</span><span class="st">"1.0.0"</span>)</span>
866<span id="cb11-7"><a href="#cb11-7" aria-hidden="true" tabindex="-1"></a>entry <span class="op">=</span> index.insert_dataset(dataset, name<span class="op">=</span><span class="st">"my-features"</span>)</span>
867<span id="cb11-8"><a href="#cb11-8" aria-hidden="true" tabindex="-1"></a></span>
868<span id="cb11-9"><a href="#cb11-9" aria-hidden="true" tabindex="-1"></a><span class="co"># Team members can now load via index</span></span>
869<span id="cb11-10"><a href="#cb11-10" aria-hidden="true" tabindex="-1"></a>ds <span class="op">=</span> load_dataset(<span class="st">"@local/my-features"</span>, index<span class="op">=</span>index)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
870</div>
871</section>
872<section id="stage-3-federation" class="level3">
873<h3 class="anchored" data-anchor-id="stage-3-federation">Stage 3: Federation</h3>
874<div id="8964a63d" class="cell">
875<div class="sourceCode cell-code" id="cb12"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb12-1"><a href="#cb12-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Promote to atmosphere</span></span>
876<span id="cb12-2"><a href="#cb12-2" aria-hidden="true" tabindex="-1"></a>client <span class="op">=</span> AtmosphereClient()</span>
877<span id="cb12-3"><a href="#cb12-3" aria-hidden="true" tabindex="-1"></a>client.login(<span class="st">"handle.bsky.social"</span>, <span class="st">"app-password"</span>)</span>
878<span id="cb12-4"><a href="#cb12-4" aria-hidden="true" tabindex="-1"></a></span>
879<span id="cb12-5"><a href="#cb12-5" aria-hidden="true" tabindex="-1"></a>at_uri <span class="op">=</span> promote_to_atmosphere(entry, index, client)</span>
880<span id="cb12-6"><a href="#cb12-6" aria-hidden="true" tabindex="-1"></a></span>
881<span id="cb12-7"><a href="#cb12-7" aria-hidden="true" tabindex="-1"></a><span class="co"># Anyone can now discover and load</span></span>
882<span id="cb12-8"><a href="#cb12-8" aria-hidden="true" tabindex="-1"></a><span class="co"># ds = load_dataset("@handle.bsky.social/my-features")</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
883</div>
884</section>
885</section>
886<section id="content-addressing" class="level2">
887<h2 class="anchored" data-anchor-id="content-addressing">Content Addressing</h2>
888<p>atdata uses <strong>CIDs</strong> (Content Identifiers) for content-addressable storage:</p>
889<ul>
890<li><strong>Schema CIDs</strong>: Hash of schema definition</li>
891<li><strong>Entry CIDs</strong>: Hash of (schema_ref, data_urls)</li>
892<li><strong>Blob CIDs</strong>: Hash of data content</li>
893</ul>
894<p>Benefits:</p>
895<ul>
896<li><strong>Deduplication</strong>: Identical content has identical CID</li>
897<li><strong>Integrity</strong>: Verify data matches expected hash</li>
898<li><strong>ATProto compatibility</strong>: CIDs are native to the AT Protocol</li>
899</ul>
900</section>
901<section id="extension-points" class="level2">
902<h2 class="anchored" data-anchor-id="extension-points">Extension Points</h2>
903<p>atdata is designed for extensibility:</p>
904<section id="custom-datasources" class="level3">
905<h3 class="anchored" data-anchor-id="custom-datasources">Custom DataSources</h3>
906<p>Implement the <code>DataSource</code> protocol to add new storage backends:</p>
907<div id="2aa480fc" class="cell">
908<div class="sourceCode cell-code" id="cb13"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb13-1"><a href="#cb13-1" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> MyCustomSource:</span>
909<span id="cb13-2"><a href="#cb13-2" aria-hidden="true" tabindex="-1"></a> <span class="kw">def</span> list_shards(<span class="va">self</span>) <span class="op">-></span> <span class="bu">list</span>[<span class="bu">str</span>]: ...</span>
910<span id="cb13-3"><a href="#cb13-3" aria-hidden="true" tabindex="-1"></a> <span class="kw">def</span> open_shard(<span class="va">self</span>, shard_id: <span class="bu">str</span>) <span class="op">-></span> IO[<span class="bu">bytes</span>]: ...</span>
911<span id="cb13-4"><a href="#cb13-4" aria-hidden="true" tabindex="-1"></a></span>
912<span id="cb13-5"><a href="#cb13-5" aria-hidden="true" tabindex="-1"></a> <span class="at">@property</span></span>
913<span id="cb13-6"><a href="#cb13-6" aria-hidden="true" tabindex="-1"></a> <span class="kw">def</span> shards(<span class="va">self</span>) <span class="op">-></span> Iterator[<span class="bu">tuple</span>[<span class="bu">str</span>, IO[<span class="bu">bytes</span>]]]: ...</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
914</div>
915</section>
916<section id="custom-lenses" class="level3">
917<h3 class="anchored" data-anchor-id="custom-lenses">Custom Lenses</h3>
918<p>Register transformations between any PackableSample types:</p>
919<div id="d65d4dda" class="cell">
920<div class="sourceCode cell-code" id="cb14"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb14-1"><a href="#cb14-1" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.lens</span></span>
921<span id="cb14-2"><a href="#cb14-2" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> my_transform(src: SourceType) <span class="op">-></span> TargetType:</span>
922<span id="cb14-3"><a href="#cb14-3" aria-hidden="true" tabindex="-1"></a> <span class="cf">return</span> TargetType(...)</span>
923<span id="cb14-4"><a href="#cb14-4" aria-hidden="true" tabindex="-1"></a></span>
924<span id="cb14-5"><a href="#cb14-5" aria-hidden="true" tabindex="-1"></a><span class="at">@my_transform.putter</span></span>
925<span id="cb14-6"><a href="#cb14-6" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> my_transform_put(view: TargetType, src: SourceType) <span class="op">-></span> SourceType:</span>
926<span id="cb14-7"><a href="#cb14-7" aria-hidden="true" tabindex="-1"></a> <span class="cf">return</span> SourceType(...)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
927</div>
928</section>
929<section id="schema-extensions" class="level3">
930<h3 class="anchored" data-anchor-id="schema-extensions">Schema Extensions</h3>
931<p>The schema format supports custom metadata for domain-specific needs:</p>
932<div id="e2b76dd7" class="cell">
933<div class="sourceCode cell-code" id="cb15"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb15-1"><a href="#cb15-1" aria-hidden="true" tabindex="-1"></a>index.publish_schema(</span>
934<span id="cb15-2"><a href="#cb15-2" aria-hidden="true" tabindex="-1"></a> MySample,</span>
935<span id="cb15-3"><a href="#cb15-3" aria-hidden="true" tabindex="-1"></a> version<span class="op">=</span><span class="st">"1.0.0"</span>,</span>
936<span id="cb15-4"><a href="#cb15-4" aria-hidden="true" tabindex="-1"></a> metadata<span class="op">=</span>{<span class="st">"domain"</span>: <span class="st">"chemistry"</span>, <span class="st">"units"</span>: <span class="st">"mol/L"</span>},</span>
937<span id="cb15-5"><a href="#cb15-5" aria-hidden="true" tabindex="-1"></a>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
938</div>
939</section>
940</section>
941<section id="summary" class="level2">
942<h2 class="anchored" data-anchor-id="summary">Summary</h2>
943<table class="caption-top table">
944<colgroup>
945<col style="width: 33%">
946<col style="width: 27%">
947<col style="width: 39%">
948</colgroup>
949<thead>
950<tr class="header">
951<th>Component</th>
952<th>Purpose</th>
953<th>Key Classes</th>
954</tr>
955</thead>
956<tbody>
957<tr class="odd">
958<td><strong>Samples</strong></td>
959<td>Typed, serializable data</td>
960<td><code>PackableSample</code>, <code>@packable</code></td>
961</tr>
962<tr class="even">
963<td><strong>Datasets</strong></td>
964<td>Typed iteration over WebDataset</td>
965<td><code>Dataset[T]</code>, <code>SampleBatch[T]</code></td>
966</tr>
967<tr class="odd">
968<td><strong>Lenses</strong></td>
969<td>Schema transformations</td>
970<td><code>Lens</code>, <code>@lens</code>, <code>LensNetwork</code></td>
971</tr>
972<tr class="even">
973<td><strong>Local Storage</strong></td>
974<td>Team-scale index + data</td>
975<td><code>LocalIndex</code>, <code>S3DataStore</code></td>
976</tr>
977<tr class="odd">
978<td><strong>Atmosphere</strong></td>
979<td>Federated sharing</td>
980<td><code>AtmosphereIndex</code>, <code>PDSBlobStore</code></td>
981</tr>
982<tr class="even">
983<td><strong>Protocols</strong></td>
984<td>Backend abstraction</td>
985<td><code>AbstractIndex</code>, <code>AbstractDataStore</code>, <code>DataSource</code></td>
986</tr>
987</tbody>
988</table>
989<p>The architecture enables a smooth progression from local experimentation to team collaboration to public federation, all while maintaining type safety and efficient data handling.</p>
990</section>
991<section id="related" class="level2">
992<h2 class="anchored" data-anchor-id="related">Related</h2>
993<ul>
994<li><a href="../reference/packable-samples.html">Packable Samples</a> - Defining sample types</li>
995<li><a href="../reference/datasets.html">Datasets</a> - Dataset iteration and batching</li>
996<li><a href="../reference/local-storage.html">Local Storage</a> - Redis + S3 backend</li>
997<li><a href="../reference/atmosphere.html">Atmosphere</a> - ATProto federation</li>
998<li><a href="../reference/protocols.html">Protocols</a> - Abstract interfaces</li>
999</ul>
1000
1001
1002</section>
1003
1004</main> <!-- /main -->
1005<script id="quarto-html-after-body" type="application/javascript">
1006 window.document.addEventListener("DOMContentLoaded", function (event) {
1007 // Ensure there is a toggle, if there isn't float one in the top right
1008 if (window.document.querySelector('.quarto-color-scheme-toggle') === null) {
1009 const a = window.document.createElement('a');
1010 a.classList.add('top-right');
1011 a.classList.add('quarto-color-scheme-toggle');
1012 a.href = "";
1013 a.onclick = function() { try { window.quartoToggleColorScheme(); } catch {} return false; };
1014 const i = window.document.createElement("i");
1015 i.classList.add('bi');
1016 a.appendChild(i);
1017 window.document.body.appendChild(a);
1018 }
1019 setColorSchemeToggle(hasAlternateSentinel())
1020 const icon = "";
1021 const anchorJS = new window.AnchorJS();
1022 anchorJS.options = {
1023 placement: 'right',
1024 icon: icon
1025 };
1026 anchorJS.add('.anchored');
1027 const isCodeAnnotation = (el) => {
1028 for (const clz of el.classList) {
1029 if (clz.startsWith('code-annotation-')) {
1030 return true;
1031 }
1032 }
1033 return false;
1034 }
1035 const onCopySuccess = function(e) {
1036 // button target
1037 const button = e.trigger;
1038 // don't keep focus
1039 button.blur();
1040 // flash "checked"
1041 button.classList.add('code-copy-button-checked');
1042 var currentTitle = button.getAttribute("title");
1043 button.setAttribute("title", "Copied!");
1044 let tooltip;
1045 if (window.bootstrap) {
1046 button.setAttribute("data-bs-toggle", "tooltip");
1047 button.setAttribute("data-bs-placement", "left");
1048 button.setAttribute("data-bs-title", "Copied!");
1049 tooltip = new bootstrap.Tooltip(button,
1050 { trigger: "manual",
1051 customClass: "code-copy-button-tooltip",
1052 offset: [0, -8]});
1053 tooltip.show();
1054 }
1055 setTimeout(function() {
1056 if (tooltip) {
1057 tooltip.hide();
1058 button.removeAttribute("data-bs-title");
1059 button.removeAttribute("data-bs-toggle");
1060 button.removeAttribute("data-bs-placement");
1061 }
1062 button.setAttribute("title", currentTitle);
1063 button.classList.remove('code-copy-button-checked');
1064 }, 1000);
1065 // clear code selection
1066 e.clearSelection();
1067 }
1068 const getTextToCopy = function(trigger) {
1069 const codeEl = trigger.previousElementSibling.cloneNode(true);
1070 for (const childEl of codeEl.children) {
1071 if (isCodeAnnotation(childEl)) {
1072 childEl.remove();
1073 }
1074 }
1075 return codeEl.innerText;
1076 }
1077 const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
1078 text: getTextToCopy
1079 });
1080 clipboard.on('success', onCopySuccess);
1081 if (window.document.getElementById('quarto-embedded-source-code-modal')) {
1082 const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', {
1083 text: getTextToCopy,
1084 container: window.document.getElementById('quarto-embedded-source-code-modal')
1085 });
1086 clipboardModal.on('success', onCopySuccess);
1087 }
1088 var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//);
1089 var mailtoRegex = new RegExp(/^mailto:/);
1090 var filterRegex = new RegExp("https:\/\/github\.com\/your-org\/atdata");
1091 var isInternal = (href) => {
1092 return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href);
1093 }
1094 // Inspect non-navigation links and adorn them if external
1095 var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)');
1096 for (var i=0; i<links.length; i++) {
1097 const link = links[i];
1098 if (!isInternal(link.href)) {
1099 // undo the damage that might have been done by quarto-nav.js in the case of
1100 // links that we want to consider external
1101 if (link.dataset.originalHref !== undefined) {
1102 link.href = link.dataset.originalHref;
1103 }
1104 }
1105 }
1106 function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
1107 const config = {
1108 allowHTML: true,
1109 maxWidth: 500,
1110 delay: 100,
1111 arrow: false,
1112 appendTo: function(el) {
1113 return el.parentElement;
1114 },
1115 interactive: true,
1116 interactiveBorder: 10,
1117 theme: 'quarto',
1118 placement: 'bottom-start',
1119 };
1120 if (contentFn) {
1121 config.content = contentFn;
1122 }
1123 if (onTriggerFn) {
1124 config.onTrigger = onTriggerFn;
1125 }
1126 if (onUntriggerFn) {
1127 config.onUntrigger = onUntriggerFn;
1128 }
1129 window.tippy(el, config);
1130 }
1131 const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
1132 for (var i=0; i<noterefs.length; i++) {
1133 const ref = noterefs[i];
1134 tippyHover(ref, function() {
1135 // use id or data attribute instead here
1136 let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
1137 try { href = new URL(href).hash; } catch {}
1138 const id = href.replace(/^#\/?/, "");
1139 const note = window.document.getElementById(id);
1140 if (note) {
1141 return note.innerHTML;
1142 } else {
1143 return "";
1144 }
1145 });
1146 }
1147 const xrefs = window.document.querySelectorAll('a.quarto-xref');
1148 const processXRef = (id, note) => {
1149 // Strip column container classes
1150 const stripColumnClz = (el) => {
1151 el.classList.remove("page-full", "page-columns");
1152 if (el.children) {
1153 for (const child of el.children) {
1154 stripColumnClz(child);
1155 }
1156 }
1157 }
1158 stripColumnClz(note)
1159 if (id === null || id.startsWith('sec-')) {
1160 // Special case sections, only their first couple elements
1161 const container = document.createElement("div");
1162 if (note.children && note.children.length > 2) {
1163 container.appendChild(note.children[0].cloneNode(true));
1164 for (let i = 1; i < note.children.length; i++) {
1165 const child = note.children[i];
1166 if (child.tagName === "P" && child.innerText === "") {
1167 continue;
1168 } else {
1169 container.appendChild(child.cloneNode(true));
1170 break;
1171 }
1172 }
1173 if (window.Quarto?.typesetMath) {
1174 window.Quarto.typesetMath(container);
1175 }
1176 return container.innerHTML
1177 } else {
1178 if (window.Quarto?.typesetMath) {
1179 window.Quarto.typesetMath(note);
1180 }
1181 return note.innerHTML;
1182 }
1183 } else {
1184 // Remove any anchor links if they are present
1185 const anchorLink = note.querySelector('a.anchorjs-link');
1186 if (anchorLink) {
1187 anchorLink.remove();
1188 }
1189 if (window.Quarto?.typesetMath) {
1190 window.Quarto.typesetMath(note);
1191 }
1192 if (note.classList.contains("callout")) {
1193 return note.outerHTML;
1194 } else {
1195 return note.innerHTML;
1196 }
1197 }
1198 }
1199 for (var i=0; i<xrefs.length; i++) {
1200 const xref = xrefs[i];
1201 tippyHover(xref, undefined, function(instance) {
1202 instance.disable();
1203 let url = xref.getAttribute('href');
1204 let hash = undefined;
1205 if (url.startsWith('#')) {
1206 hash = url;
1207 } else {
1208 try { hash = new URL(url).hash; } catch {}
1209 }
1210 if (hash) {
1211 const id = hash.replace(/^#\/?/, "");
1212 const note = window.document.getElementById(id);
1213 if (note !== null) {
1214 try {
1215 const html = processXRef(id, note.cloneNode(true));
1216 instance.setContent(html);
1217 } finally {
1218 instance.enable();
1219 instance.show();
1220 }
1221 } else {
1222 // See if we can fetch this
1223 fetch(url.split('#')[0])
1224 .then(res => res.text())
1225 .then(html => {
1226 const parser = new DOMParser();
1227 const htmlDoc = parser.parseFromString(html, "text/html");
1228 const note = htmlDoc.getElementById(id);
1229 if (note !== null) {
1230 const html = processXRef(id, note);
1231 instance.setContent(html);
1232 }
1233 }).finally(() => {
1234 instance.enable();
1235 instance.show();
1236 });
1237 }
1238 } else {
1239 // See if we can fetch a full url (with no hash to target)
1240 // This is a special case and we should probably do some content thinning / targeting
1241 fetch(url)
1242 .then(res => res.text())
1243 .then(html => {
1244 const parser = new DOMParser();
1245 const htmlDoc = parser.parseFromString(html, "text/html");
1246 const note = htmlDoc.querySelector('main.content');
1247 if (note !== null) {
1248 // This should only happen for chapter cross references
1249 // (since there is no id in the URL)
1250 // remove the first header
1251 if (note.children.length > 0 && note.children[0].tagName === "HEADER") {
1252 note.children[0].remove();
1253 }
1254 const html = processXRef(null, note);
1255 instance.setContent(html);
1256 }
1257 }).finally(() => {
1258 instance.enable();
1259 instance.show();
1260 });
1261 }
1262 }, function(instance) {
1263 });
1264 }
1265 let selectedAnnoteEl;
1266 const selectorForAnnotation = ( cell, annotation) => {
1267 let cellAttr = 'data-code-cell="' + cell + '"';
1268 let lineAttr = 'data-code-annotation="' + annotation + '"';
1269 const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
1270 return selector;
1271 }
1272 const selectCodeLines = (annoteEl) => {
1273 const doc = window.document;
1274 const targetCell = annoteEl.getAttribute("data-target-cell");
1275 const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
1276 const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
1277 const lines = annoteSpan.getAttribute("data-code-lines").split(",");
1278 const lineIds = lines.map((line) => {
1279 return targetCell + "-" + line;
1280 })
1281 let top = null;
1282 let height = null;
1283 let parent = null;
1284 if (lineIds.length > 0) {
1285 //compute the position of the single el (top and bottom and make a div)
1286 const el = window.document.getElementById(lineIds[0]);
1287 top = el.offsetTop;
1288 height = el.offsetHeight;
1289 parent = el.parentElement.parentElement;
1290 if (lineIds.length > 1) {
1291 const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
1292 const bottom = lastEl.offsetTop + lastEl.offsetHeight;
1293 height = bottom - top;
1294 }
1295 if (top !== null && height !== null && parent !== null) {
1296 // cook up a div (if necessary) and position it
1297 let div = window.document.getElementById("code-annotation-line-highlight");
1298 if (div === null) {
1299 div = window.document.createElement("div");
1300 div.setAttribute("id", "code-annotation-line-highlight");
1301 div.style.position = 'absolute';
1302 parent.appendChild(div);
1303 }
1304 div.style.top = top - 2 + "px";
1305 div.style.height = height + 4 + "px";
1306 div.style.left = 0;
1307 let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
1308 if (gutterDiv === null) {
1309 gutterDiv = window.document.createElement("div");
1310 gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
1311 gutterDiv.style.position = 'absolute';
1312 const codeCell = window.document.getElementById(targetCell);
1313 const gutter = codeCell.querySelector('.code-annotation-gutter');
1314 gutter.appendChild(gutterDiv);
1315 }
1316 gutterDiv.style.top = top - 2 + "px";
1317 gutterDiv.style.height = height + 4 + "px";
1318 }
1319 selectedAnnoteEl = annoteEl;
1320 }
1321 };
1322 const unselectCodeLines = () => {
1323 const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
1324 elementsIds.forEach((elId) => {
1325 const div = window.document.getElementById(elId);
1326 if (div) {
1327 div.remove();
1328 }
1329 });
1330 selectedAnnoteEl = undefined;
1331 };
1332 // Handle positioning of the toggle
1333 window.addEventListener(
1334 "resize",
1335 throttle(() => {
1336 elRect = undefined;
1337 if (selectedAnnoteEl) {
1338 selectCodeLines(selectedAnnoteEl);
1339 }
1340 }, 10)
1341 );
1342 function throttle(fn, ms) {
1343 let throttle = false;
1344 let timer;
1345 return (...args) => {
1346 if(!throttle) { // first call gets through
1347 fn.apply(this, args);
1348 throttle = true;
1349 } else { // all the others get throttled
1350 if(timer) clearTimeout(timer); // cancel #2
1351 timer = setTimeout(() => {
1352 fn.apply(this, args);
1353 timer = throttle = false;
1354 }, ms);
1355 }
1356 };
1357 }
1358 // Attach click handler to the DT
1359 const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
1360 for (const annoteDlNode of annoteDls) {
1361 annoteDlNode.addEventListener('click', (event) => {
1362 const clickedEl = event.target;
1363 if (clickedEl !== selectedAnnoteEl) {
1364 unselectCodeLines();
1365 const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
1366 if (activeEl) {
1367 activeEl.classList.remove('code-annotation-active');
1368 }
1369 selectCodeLines(clickedEl);
1370 clickedEl.classList.add('code-annotation-active');
1371 } else {
1372 // Unselect the line
1373 unselectCodeLines();
1374 clickedEl.classList.remove('code-annotation-active');
1375 }
1376 });
1377 }
1378 const findCites = (el) => {
1379 const parentEl = el.parentElement;
1380 if (parentEl) {
1381 const cites = parentEl.dataset.cites;
1382 if (cites) {
1383 return {
1384 el,
1385 cites: cites.split(' ')
1386 };
1387 } else {
1388 return findCites(el.parentElement)
1389 }
1390 } else {
1391 return undefined;
1392 }
1393 };
1394 var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
1395 for (var i=0; i<bibliorefs.length; i++) {
1396 const ref = bibliorefs[i];
1397 const citeInfo = findCites(ref);
1398 if (citeInfo) {
1399 tippyHover(citeInfo.el, function() {
1400 var popup = window.document.createElement('div');
1401 citeInfo.cites.forEach(function(cite) {
1402 var citeDiv = window.document.createElement('div');
1403 citeDiv.classList.add('hanging-indent');
1404 citeDiv.classList.add('csl-entry');
1405 var biblioDiv = window.document.getElementById('ref-' + cite);
1406 if (biblioDiv) {
1407 citeDiv.innerHTML = biblioDiv.innerHTML;
1408 }
1409 popup.appendChild(citeDiv);
1410 });
1411 return popup.innerHTML;
1412 });
1413 }
1414 }
1415 });
1416 </script>
1417</div> <!-- /content -->
1418<footer class="footer">
1419 <div class="nav-footer">
1420 <div class="nav-footer-left">
1421<p>Built with <a href="https://quarto.org/">Quarto</a></p>
1422</div>
1423 <div class="nav-footer-center">
1424
1425 <div class="toc-actions d-sm-block d-md-none"><ul><li><a href="https://github.com/your-org/atdata/edit/main/reference/architecture.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></div>
1426 <div class="nav-footer-right">
1427<p>MIT License</p>
1428</div>
1429 </div>
1430</footer>
1431
1432
1433
1434
1435</body></html>