A loose federation of distributed, typed datasets
1<!DOCTYPE html>
2<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
3
4<meta charset="utf-8">
5<meta name="generator" content="quarto-1.7.34">
6
7<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
8
9<meta name="description" content="Abstract interfaces for index backends, data stores, and data sources">
10
11<title>Protocols – atdata</title>
12<style>
13code{white-space: pre-wrap;}
14span.smallcaps{font-variant: small-caps;}
15div.columns{display: flex; gap: min(4vw, 1.5em);}
16div.column{flex: auto; overflow-x: auto;}
17div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
18ul.task-list{list-style: none;}
19ul.task-list li input[type="checkbox"] {
20 width: 0.8em;
21 margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
22 vertical-align: middle;
23}
24/* CSS for syntax highlighting */
25html { -webkit-text-size-adjust: 100%; }
26pre > code.sourceCode { white-space: pre; position: relative; }
27pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
28pre > code.sourceCode > span:empty { height: 1.2em; }
29.sourceCode { overflow: visible; }
30code.sourceCode > span { color: inherit; text-decoration: inherit; }
31div.sourceCode { margin: 1em 0; }
32pre.sourceCode { margin: 0; }
33@media screen {
34div.sourceCode { overflow: auto; }
35}
36@media print {
37pre > code.sourceCode { white-space: pre-wrap; }
38pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
39}
40pre.numberSource code
41 { counter-reset: source-line 0; }
42pre.numberSource code > span
43 { position: relative; left: -4em; counter-increment: source-line; }
44pre.numberSource code > span > a:first-child::before
45 { content: counter(source-line);
46 position: relative; left: -1em; text-align: right; vertical-align: baseline;
47 border: none; display: inline-block;
48 -webkit-touch-callout: none; -webkit-user-select: none;
49 -khtml-user-select: none; -moz-user-select: none;
50 -ms-user-select: none; user-select: none;
51 padding: 0 4px; width: 4em;
52 }
53pre.numberSource { margin-left: 3em; padding-left: 4px; }
54div.sourceCode
55 { }
56@media screen {
57pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
58}
59</style>
60
61
62<script src="../site_libs/quarto-nav/quarto-nav.js"></script>
63<script src="../site_libs/quarto-nav/headroom.min.js"></script>
64<script src="../site_libs/clipboard/clipboard.min.js"></script>
65<script src="../site_libs/quarto-search/autocomplete.umd.js"></script>
66<script src="../site_libs/quarto-search/fuse.min.js"></script>
67<script src="../site_libs/quarto-search/quarto-search.js"></script>
68<meta name="quarto:offset" content="../">
69<script src="../site_libs/quarto-html/quarto.js" type="module"></script>
70<script src="../site_libs/quarto-html/tabsets/tabsets.js" type="module"></script>
71<script src="../site_libs/quarto-html/popper.min.js"></script>
72<script src="../site_libs/quarto-html/tippy.umd.min.js"></script>
73<script src="../site_libs/quarto-html/anchor.min.js"></script>
74<link href="../site_libs/quarto-html/tippy.css" rel="stylesheet">
75<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme" id="quarto-text-highlighting-styles">
76<link href="../site_libs/quarto-html/quarto-syntax-highlighting-dark-8dcd8563ea6803ab7cbb3d71ca5772e1.css" rel="stylesheet" class="quarto-color-scheme quarto-color-alternate" id="quarto-text-highlighting-styles">
77<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme-extra" id="quarto-text-highlighting-styles">
78<script src="../site_libs/bootstrap/bootstrap.min.js"></script>
79<link href="../site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
80<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme" id="quarto-bootstrap" data-mode="light">
81<link href="../site_libs/bootstrap/bootstrap-dark-7964ffd8887b0991fe8d71c6c8bc75d6.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme quarto-color-alternate" id="quarto-bootstrap" data-mode="dark">
82<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme-extra" id="quarto-bootstrap" data-mode="light">
83<script id="quarto-search-options" type="application/json">{
84 "location": "navbar",
85 "copy-button": false,
86 "collapse-after": 3,
87 "panel-placement": "end",
88 "type": "overlay",
89 "limit": 50,
90 "keyboard-shortcut": [
91 "f",
92 "/",
93 "s"
94 ],
95 "show-item-context": false,
96 "language": {
97 "search-no-results-text": "No results",
98 "search-matching-documents-text": "matching documents",
99 "search-copy-link-title": "Copy link to search",
100 "search-hide-matches-text": "Hide additional matches",
101 "search-more-match-text": "more match in this document",
102 "search-more-matches-text": "more matches in this document",
103 "search-clear-button-title": "Clear",
104 "search-text-placeholder": "",
105 "search-detached-cancel-button-title": "Cancel",
106 "search-submit-button-title": "Submit",
107 "search-label": "Search"
108 }
109}</script>
110
111
112<link rel="stylesheet" href="../assets/styles.css">
113</head>
114
115<body class="nav-sidebar docked nav-fixed quarto-light"><script id="quarto-html-before-body" type="application/javascript">
116 const toggleBodyColorMode = (bsSheetEl) => {
117 const mode = bsSheetEl.getAttribute("data-mode");
118 const bodyEl = window.document.querySelector("body");
119 if (mode === "dark") {
120 bodyEl.classList.add("quarto-dark");
121 bodyEl.classList.remove("quarto-light");
122 } else {
123 bodyEl.classList.add("quarto-light");
124 bodyEl.classList.remove("quarto-dark");
125 }
126 }
127 const toggleBodyColorPrimary = () => {
128 const bsSheetEl = window.document.querySelector("link#quarto-bootstrap:not([rel=disabled-stylesheet])");
129 if (bsSheetEl) {
130 toggleBodyColorMode(bsSheetEl);
131 }
132 }
133 const setColorSchemeToggle = (alternate) => {
134 const toggles = window.document.querySelectorAll('.quarto-color-scheme-toggle');
135 for (let i=0; i < toggles.length; i++) {
136 const toggle = toggles[i];
137 if (toggle) {
138 if (alternate) {
139 toggle.classList.add("alternate");
140 } else {
141 toggle.classList.remove("alternate");
142 }
143 }
144 }
145 };
146 const toggleColorMode = (alternate) => {
147 // Switch the stylesheets
148 const primaryStylesheets = window.document.querySelectorAll('link.quarto-color-scheme:not(.quarto-color-alternate)');
149 const alternateStylesheets = window.document.querySelectorAll('link.quarto-color-scheme.quarto-color-alternate');
150 manageTransitions('#quarto-margin-sidebar .nav-link', false);
151 if (alternate) {
152 // note: dark is layered on light, we don't disable primary!
153 enableStylesheet(alternateStylesheets);
154 for (const sheetNode of alternateStylesheets) {
155 if (sheetNode.id === "quarto-bootstrap") {
156 toggleBodyColorMode(sheetNode);
157 }
158 }
159 } else {
160 disableStylesheet(alternateStylesheets);
161 enableStylesheet(primaryStylesheets)
162 toggleBodyColorPrimary();
163 }
164 manageTransitions('#quarto-margin-sidebar .nav-link', true);
165 // Switch the toggles
166 setColorSchemeToggle(alternate)
167 // Hack to workaround the fact that safari doesn't
168 // properly recolor the scrollbar when toggling (#1455)
169 if (navigator.userAgent.indexOf('Safari') > 0 && navigator.userAgent.indexOf('Chrome') == -1) {
170 manageTransitions("body", false);
171 window.scrollTo(0, 1);
172 setTimeout(() => {
173 window.scrollTo(0, 0);
174 manageTransitions("body", true);
175 }, 40);
176 }
177 }
178 const disableStylesheet = (stylesheets) => {
179 for (let i=0; i < stylesheets.length; i++) {
180 const stylesheet = stylesheets[i];
181 stylesheet.rel = 'disabled-stylesheet';
182 }
183 }
184 const enableStylesheet = (stylesheets) => {
185 for (let i=0; i < stylesheets.length; i++) {
186 const stylesheet = stylesheets[i];
187 if(stylesheet.rel !== 'stylesheet') { // for Chrome, which will still FOUC without this check
188 stylesheet.rel = 'stylesheet';
189 }
190 }
191 }
192 const manageTransitions = (selector, allowTransitions) => {
193 const els = window.document.querySelectorAll(selector);
194 for (let i=0; i < els.length; i++) {
195 const el = els[i];
196 if (allowTransitions) {
197 el.classList.remove('notransition');
198 } else {
199 el.classList.add('notransition');
200 }
201 }
202 }
203 const isFileUrl = () => {
204 return window.location.protocol === 'file:';
205 }
206 const hasAlternateSentinel = () => {
207 let styleSentinel = getColorSchemeSentinel();
208 if (styleSentinel !== null) {
209 return styleSentinel === "alternate";
210 } else {
211 return false;
212 }
213 }
214 const setStyleSentinel = (alternate) => {
215 const value = alternate ? "alternate" : "default";
216 if (!isFileUrl()) {
217 window.localStorage.setItem("quarto-color-scheme", value);
218 } else {
219 localAlternateSentinel = value;
220 }
221 }
222 const getColorSchemeSentinel = () => {
223 if (!isFileUrl()) {
224 const storageValue = window.localStorage.getItem("quarto-color-scheme");
225 return storageValue != null ? storageValue : localAlternateSentinel;
226 } else {
227 return localAlternateSentinel;
228 }
229 }
230 const toggleGiscusIfUsed = (isAlternate, darkModeDefault) => {
231 const baseTheme = document.querySelector('#giscus-base-theme')?.value ?? 'light';
232 const alternateTheme = document.querySelector('#giscus-alt-theme')?.value ?? 'dark';
233 let newTheme = '';
234 if(authorPrefersDark) {
235 newTheme = isAlternate ? baseTheme : alternateTheme;
236 } else {
237 newTheme = isAlternate ? alternateTheme : baseTheme;
238 }
239 const changeGiscusTheme = () => {
240 // From: https://github.com/giscus/giscus/issues/336
241 const sendMessage = (message) => {
242 const iframe = document.querySelector('iframe.giscus-frame');
243 if (!iframe) return;
244 iframe.contentWindow.postMessage({ giscus: message }, 'https://giscus.app');
245 }
246 sendMessage({
247 setConfig: {
248 theme: newTheme
249 }
250 });
251 }
252 const isGiscussLoaded = window.document.querySelector('iframe.giscus-frame') !== null;
253 if (isGiscussLoaded) {
254 changeGiscusTheme();
255 }
256 };
257 const authorPrefersDark = false;
258 const darkModeDefault = authorPrefersDark;
259 document.querySelector('link#quarto-text-highlighting-styles.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
260 document.querySelector('link#quarto-bootstrap.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
261 let localAlternateSentinel = darkModeDefault ? 'alternate' : 'default';
262 // Dark / light mode switch
263 window.quartoToggleColorScheme = () => {
264 // Read the current dark / light value
265 let toAlternate = !hasAlternateSentinel();
266 toggleColorMode(toAlternate);
267 setStyleSentinel(toAlternate);
268 toggleGiscusIfUsed(toAlternate, darkModeDefault);
269 window.dispatchEvent(new Event('resize'));
270 };
271 // Switch to dark mode if need be
272 if (hasAlternateSentinel()) {
273 toggleColorMode(true);
274 } else {
275 toggleColorMode(false);
276 }
277 </script>
278
279<div id="quarto-search-results"></div>
280 <header id="quarto-header" class="headroom fixed-top">
281 <nav class="navbar navbar-expand-lg " data-bs-theme="dark">
282 <div class="navbar-container container-fluid">
283 <div class="navbar-brand-container mx-auto">
284 <a class="navbar-brand" href="../index.html">
285 <span class="navbar-title">atdata</span>
286 </a>
287 </div>
288 <div id="quarto-search" class="" title="Search"></div>
289 <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarCollapse" aria-controls="navbarCollapse" role="menu" aria-expanded="false" aria-label="Toggle navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
290 <span class="navbar-toggler-icon"></span>
291</button>
292 <div class="collapse navbar-collapse" id="navbarCollapse">
293 <ul class="navbar-nav navbar-nav-scroll me-auto">
294 <li class="nav-item">
295 <a class="nav-link active" href="../index.html" aria-current="page">
296<span class="menu-text">Guide</span></a>
297 </li>
298 <li class="nav-item dropdown ">
299 <a class="nav-link dropdown-toggle" href="#" id="nav-menu-tutorials" role="link" data-bs-toggle="dropdown" aria-expanded="false">
300 <span class="menu-text">Tutorials</span>
301 </a>
302 <ul class="dropdown-menu" aria-labelledby="nav-menu-tutorials">
303 <li>
304 <a class="dropdown-item" href="../tutorials/quickstart.html">
305 <span class="dropdown-text">Quick Start</span></a>
306 </li>
307 <li>
308 <a class="dropdown-item" href="../tutorials/local-workflow.html">
309 <span class="dropdown-text">Local Workflow</span></a>
310 </li>
311 <li>
312 <a class="dropdown-item" href="../tutorials/atmosphere.html">
313 <span class="dropdown-text">Atmosphere Publishing</span></a>
314 </li>
315 <li>
316 <a class="dropdown-item" href="../tutorials/promotion.html">
317 <span class="dropdown-text">Promotion Workflow</span></a>
318 </li>
319 </ul>
320 </li>
321 <li class="nav-item dropdown ">
322 <a class="nav-link dropdown-toggle" href="#" id="nav-menu-reference" role="link" data-bs-toggle="dropdown" aria-expanded="false">
323 <span class="menu-text">Reference</span>
324 </a>
325 <ul class="dropdown-menu" aria-labelledby="nav-menu-reference">
326 <li>
327 <a class="dropdown-item" href="../reference/architecture.html">
328 <span class="dropdown-text">Architecture Overview</span></a>
329 </li>
330 <li>
331 <a class="dropdown-item" href="../reference/packable-samples.html">
332 <span class="dropdown-text">Packable Samples</span></a>
333 </li>
334 <li>
335 <a class="dropdown-item" href="../reference/datasets.html">
336 <span class="dropdown-text">Datasets</span></a>
337 </li>
338 <li>
339 <a class="dropdown-item" href="../reference/lenses.html">
340 <span class="dropdown-text">Lenses</span></a>
341 </li>
342 <li>
343 <a class="dropdown-item" href="../reference/local-storage.html">
344 <span class="dropdown-text">Local Storage</span></a>
345 </li>
346 <li>
347 <a class="dropdown-item" href="../reference/atmosphere.html">
348 <span class="dropdown-text">Atmosphere</span></a>
349 </li>
350 <li>
351 <a class="dropdown-item" href="../reference/promotion.html">
352 <span class="dropdown-text">Promotion</span></a>
353 </li>
354 <li>
355 <a class="dropdown-item" href="../reference/load-dataset.html">
356 <span class="dropdown-text">load_dataset API</span></a>
357 </li>
358 <li>
359 <a class="dropdown-item" href="../reference/protocols.html">
360 <span class="dropdown-text">Protocols</span></a>
361 </li>
362 <li>
363 <a class="dropdown-item" href="../reference/uri-spec.html">
364 <span class="dropdown-text">URI Specification</span></a>
365 </li>
366 <li>
367 <a class="dropdown-item" href="../reference/troubleshooting.html">
368 <span class="dropdown-text">Troubleshooting & FAQ</span></a>
369 </li>
370 <li>
371 <a class="dropdown-item" href="../reference/deployment.html">
372 <span class="dropdown-text">Deployment Guide</span></a>
373 </li>
374 </ul>
375 </li>
376 <li class="nav-item">
377 <a class="nav-link" href="../api/index.html">
378<span class="menu-text">API</span></a>
379 </li>
380</ul>
381 <ul class="navbar-nav navbar-nav-scroll ms-auto">
382 <li class="nav-item compact">
383 <a class="nav-link" href="https://github.com/your-org/atdata"> <i class="bi bi-github" role="img">
384</i>
385<span class="menu-text"></span></a>
386 </li>
387</ul>
388 </div> <!-- /navcollapse -->
389 <div class="quarto-navbar-tools">
390 <a href="" class="quarto-color-scheme-toggle quarto-navigation-tool px-1" onclick="window.quartoToggleColorScheme(); return false;" title="Toggle dark mode"><i class="bi"></i></a>
391</div>
392 </div> <!-- /container-fluid -->
393 </nav>
394 <nav class="quarto-secondary-nav">
395 <div class="container-fluid d-flex">
396 <button type="button" class="quarto-btn-toggle btn" data-bs-toggle="collapse" role="button" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
397 <i class="bi bi-layout-text-sidebar-reverse"></i>
398 </button>
399 <nav class="quarto-page-breadcrumbs" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item"><a href="../reference/architecture.html">Reference</a></li><li class="breadcrumb-item"><a href="../reference/protocols.html">Protocols</a></li></ol></nav>
400 <a class="flex-grow-1" role="navigation" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
401 </a>
402 </div>
403 </nav>
404</header>
405<!-- content -->
406<div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-article page-navbar">
407<!-- sidebar -->
408 <nav id="quarto-sidebar" class="sidebar collapse collapse-horizontal quarto-sidebar-collapse-item sidebar-navigation docked overflow-auto">
409 <div class="sidebar-menu-container">
410 <ul class="list-unstyled mt-1">
411 <li class="sidebar-item">
412 <div class="sidebar-item-container">
413 <a href="../index.html" class="sidebar-item-text sidebar-link">
414 <span class="menu-text">atdata</span></a>
415 </div>
416</li>
417 <li class="sidebar-item sidebar-item-section">
418 <div class="sidebar-item-container">
419 <a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" role="navigation" aria-expanded="true">
420 <span class="menu-text">Getting Started</span></a>
421 <a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" role="navigation" aria-expanded="true" aria-label="Toggle section">
422 <i class="bi bi-chevron-right ms-2"></i>
423 </a>
424 </div>
425 <ul id="quarto-sidebar-section-1" class="collapse list-unstyled sidebar-section depth1 show">
426 <li class="sidebar-item">
427 <div class="sidebar-item-container">
428 <a href="../tutorials/quickstart.html" class="sidebar-item-text sidebar-link">
429 <span class="menu-text">Quick Start</span></a>
430 </div>
431</li>
432 <li class="sidebar-item">
433 <div class="sidebar-item-container">
434 <a href="../tutorials/local-workflow.html" class="sidebar-item-text sidebar-link">
435 <span class="menu-text">Local Workflow</span></a>
436 </div>
437</li>
438 <li class="sidebar-item">
439 <div class="sidebar-item-container">
440 <a href="../tutorials/atmosphere.html" class="sidebar-item-text sidebar-link">
441 <span class="menu-text">Atmosphere Publishing</span></a>
442 </div>
443</li>
444 <li class="sidebar-item">
445 <div class="sidebar-item-container">
446 <a href="../tutorials/promotion.html" class="sidebar-item-text sidebar-link">
447 <span class="menu-text">Promotion Workflow</span></a>
448 </div>
449</li>
450 </ul>
451 </li>
452 <li class="sidebar-item sidebar-item-section">
453 <div class="sidebar-item-container">
454 <a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" role="navigation" aria-expanded="true">
455 <span class="menu-text">Reference</span></a>
456 <a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" role="navigation" aria-expanded="true" aria-label="Toggle section">
457 <i class="bi bi-chevron-right ms-2"></i>
458 </a>
459 </div>
460 <ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
461 <li class="sidebar-item">
462 <div class="sidebar-item-container">
463 <a href="../reference/architecture.html" class="sidebar-item-text sidebar-link">
464 <span class="menu-text">Architecture Overview</span></a>
465 </div>
466</li>
467 <li class="sidebar-item">
468 <div class="sidebar-item-container">
469 <a href="../reference/packable-samples.html" class="sidebar-item-text sidebar-link">
470 <span class="menu-text">Packable Samples</span></a>
471 </div>
472</li>
473 <li class="sidebar-item">
474 <div class="sidebar-item-container">
475 <a href="../reference/datasets.html" class="sidebar-item-text sidebar-link">
476 <span class="menu-text">Datasets</span></a>
477 </div>
478</li>
479 <li class="sidebar-item">
480 <div class="sidebar-item-container">
481 <a href="../reference/lenses.html" class="sidebar-item-text sidebar-link">
482 <span class="menu-text">Lenses</span></a>
483 </div>
484</li>
485 <li class="sidebar-item">
486 <div class="sidebar-item-container">
487 <a href="../reference/local-storage.html" class="sidebar-item-text sidebar-link">
488 <span class="menu-text">Local Storage</span></a>
489 </div>
490</li>
491 <li class="sidebar-item">
492 <div class="sidebar-item-container">
493 <a href="../reference/atmosphere.html" class="sidebar-item-text sidebar-link">
494 <span class="menu-text">Atmosphere (ATProto Integration)</span></a>
495 </div>
496</li>
497 <li class="sidebar-item">
498 <div class="sidebar-item-container">
499 <a href="../reference/promotion.html" class="sidebar-item-text sidebar-link">
500 <span class="menu-text">Promotion Workflow</span></a>
501 </div>
502</li>
503 <li class="sidebar-item">
504 <div class="sidebar-item-container">
505 <a href="../reference/load-dataset.html" class="sidebar-item-text sidebar-link">
506 <span class="menu-text">load_dataset API</span></a>
507 </div>
508</li>
509 <li class="sidebar-item">
510 <div class="sidebar-item-container">
511 <a href="../reference/protocols.html" class="sidebar-item-text sidebar-link active">
512 <span class="menu-text">Protocols</span></a>
513 </div>
514</li>
515 <li class="sidebar-item">
516 <div class="sidebar-item-container">
517 <a href="../reference/uri-spec.html" class="sidebar-item-text sidebar-link">
518 <span class="menu-text">URI Specification</span></a>
519 </div>
520</li>
521 <li class="sidebar-item">
522 <div class="sidebar-item-container">
523 <a href="../reference/troubleshooting.html" class="sidebar-item-text sidebar-link">
524 <span class="menu-text">Troubleshooting & FAQ</span></a>
525 </div>
526</li>
527 <li class="sidebar-item">
528 <div class="sidebar-item-container">
529 <a href="../reference/deployment.html" class="sidebar-item-text sidebar-link">
530 <span class="menu-text">Deployment Guide</span></a>
531 </div>
532</li>
533 </ul>
534 </li>
535 </ul>
536 </div>
537</nav>
538<div id="quarto-sidebar-glass" class="quarto-sidebar-collapse-item" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item"></div>
539<!-- margin-sidebar -->
540 <div id="quarto-margin-sidebar" class="sidebar margin-sidebar">
541 <nav id="TOC" role="doc-toc" class="toc-active">
542 <h2 id="toc-title">On this page</h2>
543
544 <ul>
545 <li><a href="#overview" id="toc-overview" class="nav-link active" data-scroll-target="#overview">Overview</a></li>
546 <li><a href="#indexentry-protocol" id="toc-indexentry-protocol" class="nav-link" data-scroll-target="#indexentry-protocol">IndexEntry Protocol</a>
547 <ul class="collapse">
548 <li><a href="#properties" id="toc-properties" class="nav-link" data-scroll-target="#properties">Properties</a></li>
549 <li><a href="#implementations" id="toc-implementations" class="nav-link" data-scroll-target="#implementations">Implementations</a></li>
550 </ul></li>
551 <li><a href="#abstractindex-protocol" id="toc-abstractindex-protocol" class="nav-link" data-scroll-target="#abstractindex-protocol">AbstractIndex Protocol</a>
552 <ul class="collapse">
553 <li><a href="#dataset-operations" id="toc-dataset-operations" class="nav-link" data-scroll-target="#dataset-operations">Dataset Operations</a></li>
554 <li><a href="#schema-operations" id="toc-schema-operations" class="nav-link" data-scroll-target="#schema-operations">Schema Operations</a></li>
555 <li><a href="#implementations-1" id="toc-implementations-1" class="nav-link" data-scroll-target="#implementations-1">Implementations</a></li>
556 </ul></li>
557 <li><a href="#abstractdatastore-protocol" id="toc-abstractdatastore-protocol" class="nav-link" data-scroll-target="#abstractdatastore-protocol">AbstractDataStore Protocol</a>
558 <ul class="collapse">
559 <li><a href="#methods" id="toc-methods" class="nav-link" data-scroll-target="#methods">Methods</a></li>
560 <li><a href="#implementations-2" id="toc-implementations-2" class="nav-link" data-scroll-target="#implementations-2">Implementations</a></li>
561 </ul></li>
562 <li><a href="#datasource-protocol" id="toc-datasource-protocol" class="nav-link" data-scroll-target="#datasource-protocol">DataSource Protocol</a>
563 <ul class="collapse">
564 <li><a href="#methods-1" id="toc-methods-1" class="nav-link" data-scroll-target="#methods-1">Methods</a></li>
565 <li><a href="#implementations-3" id="toc-implementations-3" class="nav-link" data-scroll-target="#implementations-3">Implementations</a></li>
566 <li><a href="#creating-custom-data-sources" id="toc-creating-custom-data-sources" class="nav-link" data-scroll-target="#creating-custom-data-sources">Creating Custom Data Sources</a></li>
567 </ul></li>
568 <li><a href="#using-protocols-for-polymorphism" id="toc-using-protocols-for-polymorphism" class="nav-link" data-scroll-target="#using-protocols-for-polymorphism">Using Protocols for Polymorphism</a></li>
569 <li><a href="#schema-reference-formats" id="toc-schema-reference-formats" class="nav-link" data-scroll-target="#schema-reference-formats">Schema Reference Formats</a></li>
570 <li><a href="#type-checking" id="toc-type-checking" class="nav-link" data-scroll-target="#type-checking">Type Checking</a></li>
571 <li><a href="#complete-example" id="toc-complete-example" class="nav-link" data-scroll-target="#complete-example">Complete Example</a></li>
572 <li><a href="#related" id="toc-related" class="nav-link" data-scroll-target="#related">Related</a></li>
573 </ul>
574<div class="toc-actions"><ul><li><a href="https://github.com/your-org/atdata/edit/main/reference/protocols.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></nav>
575 </div>
576<!-- main -->
577<main class="content" id="quarto-document-content">
578
579
580<header id="title-block-header" class="quarto-title-block default"><nav class="quarto-page-breadcrumbs quarto-title-breadcrumbs d-none d-lg-block" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item"><a href="../reference/architecture.html">Reference</a></li><li class="breadcrumb-item"><a href="../reference/protocols.html">Protocols</a></li></ol></nav>
581<div class="quarto-title">
582<h1 class="title">Protocols</h1>
583</div>
584
585<div>
586 <div class="description">
587 Abstract interfaces for index backends, data stores, and data sources
588 </div>
589</div>
590
591
592<div class="quarto-title-meta">
593
594
595
596
597 </div>
598
599
600
601</header>
602
603
604<p>The protocols module defines abstract interfaces that enable interchangeable index backends (local Redis vs ATProto), data stores (S3 vs PDS blobs), and data sources (URL, S3, etc.).</p>
605<section id="overview" class="level2">
606<h2 class="anchored" data-anchor-id="overview">Overview</h2>
607<p>Both local and atmosphere implementations solve the same problem: indexed dataset storage with external data URLs. These protocols formalize that common interface:</p>
608<ul>
609<li><strong>IndexEntry</strong>: Common interface for dataset index entries</li>
610<li><strong>AbstractIndex</strong>: Protocol for index operations</li>
611<li><strong>AbstractDataStore</strong>: Protocol for data storage operations</li>
612<li><strong>DataSource</strong>: Protocol for streaming data from various backends</li>
613</ul>
614</section>
615<section id="indexentry-protocol" class="level2">
616<h2 class="anchored" data-anchor-id="indexentry-protocol">IndexEntry Protocol</h2>
617<p>Represents a dataset entry in any index:</p>
618<div id="fe6ad9a5" class="cell">
619<div class="sourceCode cell-code" id="cb1"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata._protocols <span class="im">import</span> IndexEntry</span>
620<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a></span>
621<span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> process_entry(entry: IndexEntry) <span class="op">-></span> <span class="va">None</span>:</span>
622<span id="cb1-4"><a href="#cb1-4" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f"Name: </span><span class="sc">{</span>entry<span class="sc">.</span>name<span class="sc">}</span><span class="ss">"</span>)</span>
623<span id="cb1-5"><a href="#cb1-5" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f"Schema: </span><span class="sc">{</span>entry<span class="sc">.</span>schema_ref<span class="sc">}</span><span class="ss">"</span>)</span>
624<span id="cb1-6"><a href="#cb1-6" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f"URLs: </span><span class="sc">{</span>entry<span class="sc">.</span>data_urls<span class="sc">}</span><span class="ss">"</span>)</span>
625<span id="cb1-7"><a href="#cb1-7" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f"Metadata: </span><span class="sc">{</span>entry<span class="sc">.</span>metadata<span class="sc">}</span><span class="ss">"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
626</div>
627<section id="properties" class="level3">
628<h3 class="anchored" data-anchor-id="properties">Properties</h3>
629<table class="caption-top table">
630<thead>
631<tr class="header">
632<th>Property</th>
633<th>Type</th>
634<th>Description</th>
635</tr>
636</thead>
637<tbody>
638<tr class="odd">
639<td><code>name</code></td>
640<td><code>str</code></td>
641<td>Human-readable dataset name</td>
642</tr>
643<tr class="even">
644<td><code>schema_ref</code></td>
645<td><code>str</code></td>
646<td>Schema reference (local:// or at://)</td>
647</tr>
648<tr class="odd">
649<td><code>data_urls</code></td>
650<td><code>list[str]</code></td>
651<td>WebDataset URLs for the data</td>
652</tr>
653<tr class="even">
654<td><code>metadata</code></td>
655<td><code>dict \| None</code></td>
656<td>Arbitrary metadata dictionary</td>
657</tr>
658</tbody>
659</table>
660</section>
661<section id="implementations" class="level3">
662<h3 class="anchored" data-anchor-id="implementations">Implementations</h3>
663<ul>
664<li><code>LocalDatasetEntry</code> (from <code>atdata.local</code>)</li>
665<li><code>AtmosphereIndexEntry</code> (from <code>atdata.atmosphere</code>)</li>
666</ul>
667</section>
668</section>
669<section id="abstractindex-protocol" class="level2">
670<h2 class="anchored" data-anchor-id="abstractindex-protocol">AbstractIndex Protocol</h2>
671<p>Defines operations for managing schemas and datasets:</p>
672<div id="439d68b1" class="cell">
673<div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata._protocols <span class="im">import</span> AbstractIndex</span>
674<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a></span>
675<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> list_all_datasets(index: AbstractIndex) <span class="op">-></span> <span class="va">None</span>:</span>
676<span id="cb2-4"><a href="#cb2-4" aria-hidden="true" tabindex="-1"></a> <span class="co">"""Works with LocalIndex or AtmosphereIndex."""</span></span>
677<span id="cb2-5"><a href="#cb2-5" aria-hidden="true" tabindex="-1"></a> <span class="cf">for</span> entry <span class="kw">in</span> index.list_datasets():</span>
678<span id="cb2-6"><a href="#cb2-6" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f"</span><span class="sc">{</span>entry<span class="sc">.</span>name<span class="sc">}</span><span class="ss">: </span><span class="sc">{</span>entry<span class="sc">.</span>schema_ref<span class="sc">}</span><span class="ss">"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
679</div>
680<section id="dataset-operations" class="level3">
681<h3 class="anchored" data-anchor-id="dataset-operations">Dataset Operations</h3>
682<div id="8647c740" class="cell">
683<div class="sourceCode cell-code" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Insert a dataset</span></span>
684<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a>entry <span class="op">=</span> index.insert_dataset(</span>
685<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a> dataset,</span>
686<span id="cb3-4"><a href="#cb3-4" aria-hidden="true" tabindex="-1"></a> name<span class="op">=</span><span class="st">"my-dataset"</span>,</span>
687<span id="cb3-5"><a href="#cb3-5" aria-hidden="true" tabindex="-1"></a> schema_ref<span class="op">=</span><span class="st">"local://schemas/MySample@1.0.0"</span>, <span class="co"># optional</span></span>
688<span id="cb3-6"><a href="#cb3-6" aria-hidden="true" tabindex="-1"></a>)</span>
689<span id="cb3-7"><a href="#cb3-7" aria-hidden="true" tabindex="-1"></a></span>
690<span id="cb3-8"><a href="#cb3-8" aria-hidden="true" tabindex="-1"></a><span class="co"># Get by name/reference</span></span>
691<span id="cb3-9"><a href="#cb3-9" aria-hidden="true" tabindex="-1"></a>entry <span class="op">=</span> index.get_dataset(<span class="st">"my-dataset"</span>)</span>
692<span id="cb3-10"><a href="#cb3-10" aria-hidden="true" tabindex="-1"></a></span>
693<span id="cb3-11"><a href="#cb3-11" aria-hidden="true" tabindex="-1"></a><span class="co"># List all datasets</span></span>
694<span id="cb3-12"><a href="#cb3-12" aria-hidden="true" tabindex="-1"></a><span class="cf">for</span> entry <span class="kw">in</span> index.list_datasets():</span>
695<span id="cb3-13"><a href="#cb3-13" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(entry.name)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
696</div>
697</section>
698<section id="schema-operations" class="level3">
699<h3 class="anchored" data-anchor-id="schema-operations">Schema Operations</h3>
700<div id="ed91bcba" class="cell">
701<div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Publish a schema</span></span>
702<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a>schema_ref <span class="op">=</span> index.publish_schema(</span>
703<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a> MySample,</span>
704<span id="cb4-4"><a href="#cb4-4" aria-hidden="true" tabindex="-1"></a> version<span class="op">=</span><span class="st">"1.0.0"</span>,</span>
705<span id="cb4-5"><a href="#cb4-5" aria-hidden="true" tabindex="-1"></a>)</span>
706<span id="cb4-6"><a href="#cb4-6" aria-hidden="true" tabindex="-1"></a></span>
707<span id="cb4-7"><a href="#cb4-7" aria-hidden="true" tabindex="-1"></a><span class="co"># Get schema record</span></span>
708<span id="cb4-8"><a href="#cb4-8" aria-hidden="true" tabindex="-1"></a>schema <span class="op">=</span> index.get_schema(schema_ref)</span>
709<span id="cb4-9"><a href="#cb4-9" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(schema[<span class="st">"name"</span>], schema[<span class="st">"version"</span>])</span>
710<span id="cb4-10"><a href="#cb4-10" aria-hidden="true" tabindex="-1"></a></span>
711<span id="cb4-11"><a href="#cb4-11" aria-hidden="true" tabindex="-1"></a><span class="co"># List all schemas</span></span>
712<span id="cb4-12"><a href="#cb4-12" aria-hidden="true" tabindex="-1"></a><span class="cf">for</span> schema <span class="kw">in</span> index.list_schemas():</span>
713<span id="cb4-13"><a href="#cb4-13" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f"</span><span class="sc">{</span>schema[<span class="st">'name'</span>]<span class="sc">}</span><span class="ss">@</span><span class="sc">{</span>schema[<span class="st">'version'</span>]<span class="sc">}</span><span class="ss">"</span>)</span>
714<span id="cb4-14"><a href="#cb4-14" aria-hidden="true" tabindex="-1"></a></span>
715<span id="cb4-15"><a href="#cb4-15" aria-hidden="true" tabindex="-1"></a><span class="co"># Decode schema to Python type</span></span>
716<span id="cb4-16"><a href="#cb4-16" aria-hidden="true" tabindex="-1"></a>SampleType <span class="op">=</span> index.decode_schema(schema_ref)</span>
717<span id="cb4-17"><a href="#cb4-17" aria-hidden="true" tabindex="-1"></a>dataset <span class="op">=</span> atdata.Dataset[SampleType](entry.data_urls[<span class="dv">0</span>])</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
718</div>
719</section>
720<section id="implementations-1" class="level3">
721<h3 class="anchored" data-anchor-id="implementations-1">Implementations</h3>
722<ul>
723<li><code>LocalIndex</code> / <code>Index</code> (from <code>atdata.local</code>)</li>
724<li><code>AtmosphereIndex</code> (from <code>atdata.atmosphere</code>)</li>
725</ul>
726</section>
727</section>
728<section id="abstractdatastore-protocol" class="level2">
729<h2 class="anchored" data-anchor-id="abstractdatastore-protocol">AbstractDataStore Protocol</h2>
730<p>Abstracts over different storage backends:</p>
731<div id="ec8edb2e" class="cell">
732<div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata._protocols <span class="im">import</span> AbstractDataStore</span>
733<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a></span>
734<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> write_dataset(store: AbstractDataStore, dataset) <span class="op">-></span> <span class="bu">list</span>[<span class="bu">str</span>]:</span>
735<span id="cb5-4"><a href="#cb5-4" aria-hidden="true" tabindex="-1"></a> <span class="co">"""Works with S3DataStore or future PDS blob store."""</span></span>
736<span id="cb5-5"><a href="#cb5-5" aria-hidden="true" tabindex="-1"></a> urls <span class="op">=</span> store.write_shards(dataset, prefix<span class="op">=</span><span class="st">"datasets/v1"</span>)</span>
737<span id="cb5-6"><a href="#cb5-6" aria-hidden="true" tabindex="-1"></a> <span class="cf">return</span> urls</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
738</div>
739<section id="methods" class="level3">
740<h3 class="anchored" data-anchor-id="methods">Methods</h3>
741<div id="f3a03a91" class="cell">
742<div class="sourceCode cell-code" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Write dataset shards</span></span>
743<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a>urls <span class="op">=</span> store.write_shards(</span>
744<span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a> dataset,</span>
745<span id="cb6-4"><a href="#cb6-4" aria-hidden="true" tabindex="-1"></a> prefix<span class="op">=</span><span class="st">"datasets/mnist/v1"</span>,</span>
746<span id="cb6-5"><a href="#cb6-5" aria-hidden="true" tabindex="-1"></a> maxcount<span class="op">=</span><span class="dv">10000</span>, <span class="co"># samples per shard</span></span>
747<span id="cb6-6"><a href="#cb6-6" aria-hidden="true" tabindex="-1"></a>)</span>
748<span id="cb6-7"><a href="#cb6-7" aria-hidden="true" tabindex="-1"></a></span>
749<span id="cb6-8"><a href="#cb6-8" aria-hidden="true" tabindex="-1"></a><span class="co"># Resolve URL for reading</span></span>
750<span id="cb6-9"><a href="#cb6-9" aria-hidden="true" tabindex="-1"></a>readable_url <span class="op">=</span> store.read_url(<span class="st">"s3://bucket/path.tar"</span>)</span>
751<span id="cb6-10"><a href="#cb6-10" aria-hidden="true" tabindex="-1"></a></span>
752<span id="cb6-11"><a href="#cb6-11" aria-hidden="true" tabindex="-1"></a><span class="co"># Check streaming support</span></span>
753<span id="cb6-12"><a href="#cb6-12" aria-hidden="true" tabindex="-1"></a><span class="cf">if</span> store.supports_streaming():</span>
754<span id="cb6-13"><a href="#cb6-13" aria-hidden="true" tabindex="-1"></a> <span class="co"># Can stream directly</span></span>
755<span id="cb6-14"><a href="#cb6-14" aria-hidden="true" tabindex="-1"></a> <span class="cf">pass</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
756</div>
757</section>
758<section id="implementations-2" class="level3">
759<h3 class="anchored" data-anchor-id="implementations-2">Implementations</h3>
760<ul>
761<li><code>S3DataStore</code> (from <code>atdata.local</code>)</li>
762</ul>
763</section>
764</section>
765<section id="datasource-protocol" class="level2">
766<h2 class="anchored" data-anchor-id="datasource-protocol">DataSource Protocol</h2>
767<p>Abstracts over different data source backends for streaming dataset shards:</p>
768<div id="51efd4bf" class="cell">
769<div class="sourceCode cell-code" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata._protocols <span class="im">import</span> DataSource</span>
770<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a></span>
771<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> load_from_source(source: DataSource) <span class="op">-></span> <span class="va">None</span>:</span>
772<span id="cb7-4"><a href="#cb7-4" aria-hidden="true" tabindex="-1"></a> <span class="co">"""Works with URLSource, S3Source, or custom implementations."""</span></span>
773<span id="cb7-5"><a href="#cb7-5" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f"Shards: </span><span class="sc">{</span>source<span class="sc">.</span>shard_list<span class="sc">}</span><span class="ss">"</span>)</span>
774<span id="cb7-6"><a href="#cb7-6" aria-hidden="true" tabindex="-1"></a></span>
775<span id="cb7-7"><a href="#cb7-7" aria-hidden="true" tabindex="-1"></a> <span class="cf">for</span> shard_id, stream <span class="kw">in</span> source.shards():</span>
776<span id="cb7-8"><a href="#cb7-8" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f"Reading </span><span class="sc">{</span>shard_id<span class="sc">}</span><span class="ss">"</span>)</span>
777<span id="cb7-9"><a href="#cb7-9" aria-hidden="true" tabindex="-1"></a> <span class="co"># stream is a file-like object</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
778</div>
779<section id="methods-1" class="level3">
780<h3 class="anchored" data-anchor-id="methods-1">Methods</h3>
781<div id="03a780ee" class="cell">
782<div class="sourceCode cell-code" id="cb8"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Get list of shard identifiers</span></span>
783<span id="cb8-2"><a href="#cb8-2" aria-hidden="true" tabindex="-1"></a>shard_ids <span class="op">=</span> source.shard_list <span class="co"># ['data-000000.tar', 'data-000001.tar', ...]</span></span>
784<span id="cb8-3"><a href="#cb8-3" aria-hidden="true" tabindex="-1"></a></span>
785<span id="cb8-4"><a href="#cb8-4" aria-hidden="true" tabindex="-1"></a><span class="co"># Iterate over all shards with streams</span></span>
786<span id="cb8-5"><a href="#cb8-5" aria-hidden="true" tabindex="-1"></a><span class="cf">for</span> shard_id, stream <span class="kw">in</span> source.shards():</span>
787<span id="cb8-6"><a href="#cb8-6" aria-hidden="true" tabindex="-1"></a> <span class="co"># stream is IO[bytes], can be passed to tar reader</span></span>
788<span id="cb8-7"><a href="#cb8-7" aria-hidden="true" tabindex="-1"></a> process_shard(stream)</span>
789<span id="cb8-8"><a href="#cb8-8" aria-hidden="true" tabindex="-1"></a></span>
790<span id="cb8-9"><a href="#cb8-9" aria-hidden="true" tabindex="-1"></a><span class="co"># Open a specific shard</span></span>
791<span id="cb8-10"><a href="#cb8-10" aria-hidden="true" tabindex="-1"></a>stream <span class="op">=</span> source.open_shard(<span class="st">"data-000001.tar"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
792</div>
793</section>
794<section id="implementations-3" class="level3">
795<h3 class="anchored" data-anchor-id="implementations-3">Implementations</h3>
796<ul>
797<li><code>URLSource</code> (from <code>atdata</code>) - WebDataset-compatible URLs (local, HTTP, etc.)</li>
798<li><code>S3Source</code> (from <code>atdata</code>) - S3 and S3-compatible storage with boto3</li>
799</ul>
800</section>
801<section id="creating-custom-data-sources" class="level3">
802<h3 class="anchored" data-anchor-id="creating-custom-data-sources">Creating Custom Data Sources</h3>
803<p>Implement the <code>DataSource</code> protocol for custom backends:</p>
804<div id="594f4fdb" class="cell">
805<div class="sourceCode cell-code" id="cb9"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb9-1"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> typing <span class="im">import</span> Iterator, IO</span>
806<span id="cb9-2"><a href="#cb9-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata._protocols <span class="im">import</span> DataSource</span>
807<span id="cb9-3"><a href="#cb9-3" aria-hidden="true" tabindex="-1"></a></span>
808<span id="cb9-4"><a href="#cb9-4" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> MyCustomSource:</span>
809<span id="cb9-5"><a href="#cb9-5" aria-hidden="true" tabindex="-1"></a> <span class="co">"""Custom data source for proprietary storage."""</span></span>
810<span id="cb9-6"><a href="#cb9-6" aria-hidden="true" tabindex="-1"></a></span>
811<span id="cb9-7"><a href="#cb9-7" aria-hidden="true" tabindex="-1"></a> <span class="kw">def</span> <span class="fu">__init__</span>(<span class="va">self</span>, config: <span class="bu">dict</span>):</span>
812<span id="cb9-8"><a href="#cb9-8" aria-hidden="true" tabindex="-1"></a> <span class="va">self</span>._config <span class="op">=</span> config</span>
813<span id="cb9-9"><a href="#cb9-9" aria-hidden="true" tabindex="-1"></a> <span class="va">self</span>._shards <span class="op">=</span> [<span class="st">"shard-001.tar"</span>, <span class="st">"shard-002.tar"</span>]</span>
814<span id="cb9-10"><a href="#cb9-10" aria-hidden="true" tabindex="-1"></a></span>
815<span id="cb9-11"><a href="#cb9-11" aria-hidden="true" tabindex="-1"></a> <span class="at">@property</span></span>
816<span id="cb9-12"><a href="#cb9-12" aria-hidden="true" tabindex="-1"></a> <span class="kw">def</span> shard_list(<span class="va">self</span>) <span class="op">-></span> <span class="bu">list</span>[<span class="bu">str</span>]:</span>
817<span id="cb9-13"><a href="#cb9-13" aria-hidden="true" tabindex="-1"></a> <span class="cf">return</span> <span class="va">self</span>._shards</span>
818<span id="cb9-14"><a href="#cb9-14" aria-hidden="true" tabindex="-1"></a></span>
819<span id="cb9-15"><a href="#cb9-15" aria-hidden="true" tabindex="-1"></a> <span class="kw">def</span> shards(<span class="va">self</span>) <span class="op">-></span> Iterator[<span class="bu">tuple</span>[<span class="bu">str</span>, IO[<span class="bu">bytes</span>]]]:</span>
820<span id="cb9-16"><a href="#cb9-16" aria-hidden="true" tabindex="-1"></a> <span class="cf">for</span> shard_id <span class="kw">in</span> <span class="va">self</span>._shards:</span>
821<span id="cb9-17"><a href="#cb9-17" aria-hidden="true" tabindex="-1"></a> stream <span class="op">=</span> <span class="va">self</span>._open(shard_id)</span>
822<span id="cb9-18"><a href="#cb9-18" aria-hidden="true" tabindex="-1"></a> <span class="cf">yield</span> shard_id, stream</span>
823<span id="cb9-19"><a href="#cb9-19" aria-hidden="true" tabindex="-1"></a></span>
824<span id="cb9-20"><a href="#cb9-20" aria-hidden="true" tabindex="-1"></a> <span class="kw">def</span> open_shard(<span class="va">self</span>, shard_id: <span class="bu">str</span>) <span class="op">-></span> IO[<span class="bu">bytes</span>]:</span>
825<span id="cb9-21"><a href="#cb9-21" aria-hidden="true" tabindex="-1"></a> <span class="cf">if</span> shard_id <span class="kw">not</span> <span class="kw">in</span> <span class="va">self</span>._shards:</span>
826<span id="cb9-22"><a href="#cb9-22" aria-hidden="true" tabindex="-1"></a> <span class="cf">raise</span> <span class="pp">KeyError</span>(<span class="ss">f"Shard not found: </span><span class="sc">{</span>shard_id<span class="sc">}</span><span class="ss">"</span>)</span>
827<span id="cb9-23"><a href="#cb9-23" aria-hidden="true" tabindex="-1"></a> <span class="cf">return</span> <span class="va">self</span>._open(shard_id)</span>
828<span id="cb9-24"><a href="#cb9-24" aria-hidden="true" tabindex="-1"></a></span>
829<span id="cb9-25"><a href="#cb9-25" aria-hidden="true" tabindex="-1"></a> <span class="kw">def</span> _open(<span class="va">self</span>, shard_id: <span class="bu">str</span>) <span class="op">-></span> IO[<span class="bu">bytes</span>]:</span>
830<span id="cb9-26"><a href="#cb9-26" aria-hidden="true" tabindex="-1"></a> <span class="co"># Implementation-specific logic</span></span>
831<span id="cb9-27"><a href="#cb9-27" aria-hidden="true" tabindex="-1"></a> ...</span>
832<span id="cb9-28"><a href="#cb9-28" aria-hidden="true" tabindex="-1"></a></span>
833<span id="cb9-29"><a href="#cb9-29" aria-hidden="true" tabindex="-1"></a><span class="co"># Use with Dataset</span></span>
834<span id="cb9-30"><a href="#cb9-30" aria-hidden="true" tabindex="-1"></a>source <span class="op">=</span> MyCustomSource({<span class="st">"endpoint"</span>: <span class="st">"..."</span>})</span>
835<span id="cb9-31"><a href="#cb9-31" aria-hidden="true" tabindex="-1"></a>dataset <span class="op">=</span> atdata.Dataset[MySample](source)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
836</div>
837</section>
838</section>
839<section id="using-protocols-for-polymorphism" class="level2">
840<h2 class="anchored" data-anchor-id="using-protocols-for-polymorphism">Using Protocols for Polymorphism</h2>
841<p>Write code that works with any backend:</p>
842<div id="1bffcf78" class="cell">
843<div class="sourceCode cell-code" id="cb10"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata._protocols <span class="im">import</span> AbstractIndex, IndexEntry</span>
844<span id="cb10-2"><a href="#cb10-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata <span class="im">import</span> Dataset</span>
845<span id="cb10-3"><a href="#cb10-3" aria-hidden="true" tabindex="-1"></a></span>
846<span id="cb10-4"><a href="#cb10-4" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> backup_all_datasets(</span>
847<span id="cb10-5"><a href="#cb10-5" aria-hidden="true" tabindex="-1"></a> source: AbstractIndex,</span>
848<span id="cb10-6"><a href="#cb10-6" aria-hidden="true" tabindex="-1"></a> target: AbstractIndex,</span>
849<span id="cb10-7"><a href="#cb10-7" aria-hidden="true" tabindex="-1"></a>) <span class="op">-></span> <span class="va">None</span>:</span>
850<span id="cb10-8"><a href="#cb10-8" aria-hidden="true" tabindex="-1"></a> <span class="co">"""Copy all datasets from source index to target."""</span></span>
851<span id="cb10-9"><a href="#cb10-9" aria-hidden="true" tabindex="-1"></a> <span class="cf">for</span> entry <span class="kw">in</span> source.list_datasets():</span>
852<span id="cb10-10"><a href="#cb10-10" aria-hidden="true" tabindex="-1"></a> <span class="co"># Decode schema from source</span></span>
853<span id="cb10-11"><a href="#cb10-11" aria-hidden="true" tabindex="-1"></a> SampleType <span class="op">=</span> source.decode_schema(entry.schema_ref)</span>
854<span id="cb10-12"><a href="#cb10-12" aria-hidden="true" tabindex="-1"></a></span>
855<span id="cb10-13"><a href="#cb10-13" aria-hidden="true" tabindex="-1"></a> <span class="co"># Publish schema to target</span></span>
856<span id="cb10-14"><a href="#cb10-14" aria-hidden="true" tabindex="-1"></a> target_schema <span class="op">=</span> target.publish_schema(SampleType)</span>
857<span id="cb10-15"><a href="#cb10-15" aria-hidden="true" tabindex="-1"></a></span>
858<span id="cb10-16"><a href="#cb10-16" aria-hidden="true" tabindex="-1"></a> <span class="co"># Load and re-insert dataset</span></span>
859<span id="cb10-17"><a href="#cb10-17" aria-hidden="true" tabindex="-1"></a> ds <span class="op">=</span> Dataset[SampleType](entry.data_urls[<span class="dv">0</span>])</span>
860<span id="cb10-18"><a href="#cb10-18" aria-hidden="true" tabindex="-1"></a> target.insert_dataset(</span>
861<span id="cb10-19"><a href="#cb10-19" aria-hidden="true" tabindex="-1"></a> ds,</span>
862<span id="cb10-20"><a href="#cb10-20" aria-hidden="true" tabindex="-1"></a> name<span class="op">=</span>entry.name,</span>
863<span id="cb10-21"><a href="#cb10-21" aria-hidden="true" tabindex="-1"></a> schema_ref<span class="op">=</span>target_schema,</span>
864<span id="cb10-22"><a href="#cb10-22" aria-hidden="true" tabindex="-1"></a> )</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
865</div>
866</section>
867<section id="schema-reference-formats" class="level2">
868<h2 class="anchored" data-anchor-id="schema-reference-formats">Schema Reference Formats</h2>
869<p>Schema references vary by backend:</p>
870<table class="caption-top table">
871<colgroup>
872<col style="width: 34%">
873<col style="width: 30%">
874<col style="width: 34%">
875</colgroup>
876<thead>
877<tr class="header">
878<th>Backend</th>
879<th>Format</th>
880<th>Example</th>
881</tr>
882</thead>
883<tbody>
884<tr class="odd">
885<td>Local</td>
886<td><code>atdata://local/sampleSchema/{Class}@{version}</code></td>
887<td><code>atdata://local/sampleSchema/ImageSample@1.0.0</code></td>
888</tr>
889<tr class="even">
890<td>Atmosphere</td>
891<td><code>at://{did}/{collection}/{rkey}</code></td>
892<td><code>at://did:plc:abc123/ac.foundation.dataset.sampleSchema/xyz</code></td>
893</tr>
894</tbody>
895</table>
896<div class="callout callout-style-default callout-note callout-titled">
897<div class="callout-header d-flex align-content-center">
898<div class="callout-icon-container">
899<i class="callout-icon"></i>
900</div>
901<div class="callout-title-container flex-fill">
902Note
903</div>
904</div>
905<div class="callout-body-container callout-body">
906<p>Legacy <code>local://schemas/</code> URIs are still supported for backward compatibility.</p>
907</div>
908</div>
909</section>
910<section id="type-checking" class="level2">
911<h2 class="anchored" data-anchor-id="type-checking">Type Checking</h2>
912<p>Protocols are runtime-checkable:</p>
913<div id="7712ed61" class="cell">
914<div class="sourceCode cell-code" id="cb11"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb11-1"><a href="#cb11-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata._protocols <span class="im">import</span> IndexEntry, AbstractIndex</span>
915<span id="cb11-2"><a href="#cb11-2" aria-hidden="true" tabindex="-1"></a></span>
916<span id="cb11-3"><a href="#cb11-3" aria-hidden="true" tabindex="-1"></a><span class="co"># Check if object implements protocol</span></span>
917<span id="cb11-4"><a href="#cb11-4" aria-hidden="true" tabindex="-1"></a>entry <span class="op">=</span> index.get_dataset(<span class="st">"test"</span>)</span>
918<span id="cb11-5"><a href="#cb11-5" aria-hidden="true" tabindex="-1"></a><span class="cf">assert</span> <span class="bu">isinstance</span>(entry, IndexEntry)</span>
919<span id="cb11-6"><a href="#cb11-6" aria-hidden="true" tabindex="-1"></a></span>
920<span id="cb11-7"><a href="#cb11-7" aria-hidden="true" tabindex="-1"></a><span class="co"># Type hints work with protocols</span></span>
921<span id="cb11-8"><a href="#cb11-8" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> process(index: AbstractIndex) <span class="op">-></span> <span class="va">None</span>:</span>
922<span id="cb11-9"><a href="#cb11-9" aria-hidden="true" tabindex="-1"></a> ... <span class="co"># IDE provides autocomplete</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
923</div>
924</section>
925<section id="complete-example" class="level2">
926<h2 class="anchored" data-anchor-id="complete-example">Complete Example</h2>
927<div id="b6ead38d" class="cell">
928<div class="sourceCode cell-code" id="cb12"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb12-1"><a href="#cb12-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> atdata</span>
929<span id="cb12-2"><a href="#cb12-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.local <span class="im">import</span> LocalIndex, S3DataStore</span>
930<span id="cb12-3"><a href="#cb12-3" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.atmosphere <span class="im">import</span> AtmosphereClient, AtmosphereIndex</span>
931<span id="cb12-4"><a href="#cb12-4" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata._protocols <span class="im">import</span> AbstractIndex</span>
932<span id="cb12-5"><a href="#cb12-5" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
933<span id="cb12-6"><a href="#cb12-6" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> numpy.typing <span class="im">import</span> NDArray</span>
934<span id="cb12-7"><a href="#cb12-7" aria-hidden="true" tabindex="-1"></a></span>
935<span id="cb12-8"><a href="#cb12-8" aria-hidden="true" tabindex="-1"></a><span class="co"># Define sample type</span></span>
936<span id="cb12-9"><a href="#cb12-9" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
937<span id="cb12-10"><a href="#cb12-10" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> FeatureSample:</span>
938<span id="cb12-11"><a href="#cb12-11" aria-hidden="true" tabindex="-1"></a> features: NDArray</span>
939<span id="cb12-12"><a href="#cb12-12" aria-hidden="true" tabindex="-1"></a> label: <span class="bu">int</span></span>
940<span id="cb12-13"><a href="#cb12-13" aria-hidden="true" tabindex="-1"></a></span>
941<span id="cb12-14"><a href="#cb12-14" aria-hidden="true" tabindex="-1"></a><span class="co"># Function works with any index</span></span>
942<span id="cb12-15"><a href="#cb12-15" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> count_datasets(index: AbstractIndex) <span class="op">-></span> <span class="bu">int</span>:</span>
943<span id="cb12-16"><a href="#cb12-16" aria-hidden="true" tabindex="-1"></a> <span class="cf">return</span> <span class="bu">sum</span>(<span class="dv">1</span> <span class="cf">for</span> _ <span class="kw">in</span> index.list_datasets())</span>
944<span id="cb12-17"><a href="#cb12-17" aria-hidden="true" tabindex="-1"></a></span>
945<span id="cb12-18"><a href="#cb12-18" aria-hidden="true" tabindex="-1"></a><span class="co"># Use with local index</span></span>
946<span id="cb12-19"><a href="#cb12-19" aria-hidden="true" tabindex="-1"></a>local_index <span class="op">=</span> LocalIndex()</span>
947<span id="cb12-20"><a href="#cb12-20" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Local datasets: </span><span class="sc">{</span>count_datasets(local_index)<span class="sc">}</span><span class="ss">"</span>)</span>
948<span id="cb12-21"><a href="#cb12-21" aria-hidden="true" tabindex="-1"></a></span>
949<span id="cb12-22"><a href="#cb12-22" aria-hidden="true" tabindex="-1"></a><span class="co"># Use with atmosphere index</span></span>
950<span id="cb12-23"><a href="#cb12-23" aria-hidden="true" tabindex="-1"></a>client <span class="op">=</span> AtmosphereClient()</span>
951<span id="cb12-24"><a href="#cb12-24" aria-hidden="true" tabindex="-1"></a>client.login(<span class="st">"handle.bsky.social"</span>, <span class="st">"app-password"</span>)</span>
952<span id="cb12-25"><a href="#cb12-25" aria-hidden="true" tabindex="-1"></a>atm_index <span class="op">=</span> AtmosphereIndex(client)</span>
953<span id="cb12-26"><a href="#cb12-26" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Atmosphere datasets: </span><span class="sc">{</span>count_datasets(atm_index)<span class="sc">}</span><span class="ss">"</span>)</span>
954<span id="cb12-27"><a href="#cb12-27" aria-hidden="true" tabindex="-1"></a></span>
955<span id="cb12-28"><a href="#cb12-28" aria-hidden="true" tabindex="-1"></a><span class="co"># Migrate from local to atmosphere</span></span>
956<span id="cb12-29"><a href="#cb12-29" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> migrate_dataset(</span>
957<span id="cb12-30"><a href="#cb12-30" aria-hidden="true" tabindex="-1"></a> name: <span class="bu">str</span>,</span>
958<span id="cb12-31"><a href="#cb12-31" aria-hidden="true" tabindex="-1"></a> source: AbstractIndex,</span>
959<span id="cb12-32"><a href="#cb12-32" aria-hidden="true" tabindex="-1"></a> target: AbstractIndex,</span>
960<span id="cb12-33"><a href="#cb12-33" aria-hidden="true" tabindex="-1"></a>) <span class="op">-></span> <span class="va">None</span>:</span>
961<span id="cb12-34"><a href="#cb12-34" aria-hidden="true" tabindex="-1"></a> entry <span class="op">=</span> source.get_dataset(name)</span>
962<span id="cb12-35"><a href="#cb12-35" aria-hidden="true" tabindex="-1"></a> SampleType <span class="op">=</span> source.decode_schema(entry.schema_ref)</span>
963<span id="cb12-36"><a href="#cb12-36" aria-hidden="true" tabindex="-1"></a></span>
964<span id="cb12-37"><a href="#cb12-37" aria-hidden="true" tabindex="-1"></a> <span class="co"># Publish schema</span></span>
965<span id="cb12-38"><a href="#cb12-38" aria-hidden="true" tabindex="-1"></a> schema_ref <span class="op">=</span> target.publish_schema(SampleType)</span>
966<span id="cb12-39"><a href="#cb12-39" aria-hidden="true" tabindex="-1"></a></span>
967<span id="cb12-40"><a href="#cb12-40" aria-hidden="true" tabindex="-1"></a> <span class="co"># Create dataset and insert</span></span>
968<span id="cb12-41"><a href="#cb12-41" aria-hidden="true" tabindex="-1"></a> ds <span class="op">=</span> atdata.Dataset[SampleType](entry.data_urls[<span class="dv">0</span>])</span>
969<span id="cb12-42"><a href="#cb12-42" aria-hidden="true" tabindex="-1"></a> target.insert_dataset(ds, name<span class="op">=</span>name, schema_ref<span class="op">=</span>schema_ref)</span>
970<span id="cb12-43"><a href="#cb12-43" aria-hidden="true" tabindex="-1"></a></span>
971<span id="cb12-44"><a href="#cb12-44" aria-hidden="true" tabindex="-1"></a>migrate_dataset(<span class="st">"my-features"</span>, local_index, atm_index)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
972</div>
973</section>
974<section id="related" class="level2">
975<h2 class="anchored" data-anchor-id="related">Related</h2>
976<ul>
977<li><a href="../reference/local-storage.html">Local Storage</a> - LocalIndex and S3DataStore</li>
978<li><a href="../reference/atmosphere.html">Atmosphere</a> - AtmosphereIndex</li>
979<li><a href="../reference/promotion.html">Promotion</a> - Local to atmosphere migration</li>
980<li><a href="../reference/load-dataset.html">load_dataset</a> - Using indexes with load_dataset()</li>
981</ul>
982
983
984</section>
985
986</main> <!-- /main -->
987<script id="quarto-html-after-body" type="application/javascript">
988 window.document.addEventListener("DOMContentLoaded", function (event) {
989 // Ensure there is a toggle, if there isn't float one in the top right
990 if (window.document.querySelector('.quarto-color-scheme-toggle') === null) {
991 const a = window.document.createElement('a');
992 a.classList.add('top-right');
993 a.classList.add('quarto-color-scheme-toggle');
994 a.href = "";
995 a.onclick = function() { try { window.quartoToggleColorScheme(); } catch {} return false; };
996 const i = window.document.createElement("i");
997 i.classList.add('bi');
998 a.appendChild(i);
999 window.document.body.appendChild(a);
1000 }
1001 setColorSchemeToggle(hasAlternateSentinel())
1002 const icon = "";
1003 const anchorJS = new window.AnchorJS();
1004 anchorJS.options = {
1005 placement: 'right',
1006 icon: icon
1007 };
1008 anchorJS.add('.anchored');
1009 const isCodeAnnotation = (el) => {
1010 for (const clz of el.classList) {
1011 if (clz.startsWith('code-annotation-')) {
1012 return true;
1013 }
1014 }
1015 return false;
1016 }
1017 const onCopySuccess = function(e) {
1018 // button target
1019 const button = e.trigger;
1020 // don't keep focus
1021 button.blur();
1022 // flash "checked"
1023 button.classList.add('code-copy-button-checked');
1024 var currentTitle = button.getAttribute("title");
1025 button.setAttribute("title", "Copied!");
1026 let tooltip;
1027 if (window.bootstrap) {
1028 button.setAttribute("data-bs-toggle", "tooltip");
1029 button.setAttribute("data-bs-placement", "left");
1030 button.setAttribute("data-bs-title", "Copied!");
1031 tooltip = new bootstrap.Tooltip(button,
1032 { trigger: "manual",
1033 customClass: "code-copy-button-tooltip",
1034 offset: [0, -8]});
1035 tooltip.show();
1036 }
1037 setTimeout(function() {
1038 if (tooltip) {
1039 tooltip.hide();
1040 button.removeAttribute("data-bs-title");
1041 button.removeAttribute("data-bs-toggle");
1042 button.removeAttribute("data-bs-placement");
1043 }
1044 button.setAttribute("title", currentTitle);
1045 button.classList.remove('code-copy-button-checked');
1046 }, 1000);
1047 // clear code selection
1048 e.clearSelection();
1049 }
1050 const getTextToCopy = function(trigger) {
1051 const codeEl = trigger.previousElementSibling.cloneNode(true);
1052 for (const childEl of codeEl.children) {
1053 if (isCodeAnnotation(childEl)) {
1054 childEl.remove();
1055 }
1056 }
1057 return codeEl.innerText;
1058 }
1059 const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
1060 text: getTextToCopy
1061 });
1062 clipboard.on('success', onCopySuccess);
1063 if (window.document.getElementById('quarto-embedded-source-code-modal')) {
1064 const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', {
1065 text: getTextToCopy,
1066 container: window.document.getElementById('quarto-embedded-source-code-modal')
1067 });
1068 clipboardModal.on('success', onCopySuccess);
1069 }
1070 var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//);
1071 var mailtoRegex = new RegExp(/^mailto:/);
1072 var filterRegex = new RegExp("https:\/\/github\.com\/your-org\/atdata");
1073 var isInternal = (href) => {
1074 return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href);
1075 }
1076 // Inspect non-navigation links and adorn them if external
1077 var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)');
1078 for (var i=0; i<links.length; i++) {
1079 const link = links[i];
1080 if (!isInternal(link.href)) {
1081 // undo the damage that might have been done by quarto-nav.js in the case of
1082 // links that we want to consider external
1083 if (link.dataset.originalHref !== undefined) {
1084 link.href = link.dataset.originalHref;
1085 }
1086 }
1087 }
1088 function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
1089 const config = {
1090 allowHTML: true,
1091 maxWidth: 500,
1092 delay: 100,
1093 arrow: false,
1094 appendTo: function(el) {
1095 return el.parentElement;
1096 },
1097 interactive: true,
1098 interactiveBorder: 10,
1099 theme: 'quarto',
1100 placement: 'bottom-start',
1101 };
1102 if (contentFn) {
1103 config.content = contentFn;
1104 }
1105 if (onTriggerFn) {
1106 config.onTrigger = onTriggerFn;
1107 }
1108 if (onUntriggerFn) {
1109 config.onUntrigger = onUntriggerFn;
1110 }
1111 window.tippy(el, config);
1112 }
1113 const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
1114 for (var i=0; i<noterefs.length; i++) {
1115 const ref = noterefs[i];
1116 tippyHover(ref, function() {
1117 // use id or data attribute instead here
1118 let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
1119 try { href = new URL(href).hash; } catch {}
1120 const id = href.replace(/^#\/?/, "");
1121 const note = window.document.getElementById(id);
1122 if (note) {
1123 return note.innerHTML;
1124 } else {
1125 return "";
1126 }
1127 });
1128 }
1129 const xrefs = window.document.querySelectorAll('a.quarto-xref');
1130 const processXRef = (id, note) => {
1131 // Strip column container classes
1132 const stripColumnClz = (el) => {
1133 el.classList.remove("page-full", "page-columns");
1134 if (el.children) {
1135 for (const child of el.children) {
1136 stripColumnClz(child);
1137 }
1138 }
1139 }
1140 stripColumnClz(note)
1141 if (id === null || id.startsWith('sec-')) {
1142 // Special case sections, only their first couple elements
1143 const container = document.createElement("div");
1144 if (note.children && note.children.length > 2) {
1145 container.appendChild(note.children[0].cloneNode(true));
1146 for (let i = 1; i < note.children.length; i++) {
1147 const child = note.children[i];
1148 if (child.tagName === "P" && child.innerText === "") {
1149 continue;
1150 } else {
1151 container.appendChild(child.cloneNode(true));
1152 break;
1153 }
1154 }
1155 if (window.Quarto?.typesetMath) {
1156 window.Quarto.typesetMath(container);
1157 }
1158 return container.innerHTML
1159 } else {
1160 if (window.Quarto?.typesetMath) {
1161 window.Quarto.typesetMath(note);
1162 }
1163 return note.innerHTML;
1164 }
1165 } else {
1166 // Remove any anchor links if they are present
1167 const anchorLink = note.querySelector('a.anchorjs-link');
1168 if (anchorLink) {
1169 anchorLink.remove();
1170 }
1171 if (window.Quarto?.typesetMath) {
1172 window.Quarto.typesetMath(note);
1173 }
1174 if (note.classList.contains("callout")) {
1175 return note.outerHTML;
1176 } else {
1177 return note.innerHTML;
1178 }
1179 }
1180 }
1181 for (var i=0; i<xrefs.length; i++) {
1182 const xref = xrefs[i];
1183 tippyHover(xref, undefined, function(instance) {
1184 instance.disable();
1185 let url = xref.getAttribute('href');
1186 let hash = undefined;
1187 if (url.startsWith('#')) {
1188 hash = url;
1189 } else {
1190 try { hash = new URL(url).hash; } catch {}
1191 }
1192 if (hash) {
1193 const id = hash.replace(/^#\/?/, "");
1194 const note = window.document.getElementById(id);
1195 if (note !== null) {
1196 try {
1197 const html = processXRef(id, note.cloneNode(true));
1198 instance.setContent(html);
1199 } finally {
1200 instance.enable();
1201 instance.show();
1202 }
1203 } else {
1204 // See if we can fetch this
1205 fetch(url.split('#')[0])
1206 .then(res => res.text())
1207 .then(html => {
1208 const parser = new DOMParser();
1209 const htmlDoc = parser.parseFromString(html, "text/html");
1210 const note = htmlDoc.getElementById(id);
1211 if (note !== null) {
1212 const html = processXRef(id, note);
1213 instance.setContent(html);
1214 }
1215 }).finally(() => {
1216 instance.enable();
1217 instance.show();
1218 });
1219 }
1220 } else {
1221 // See if we can fetch a full url (with no hash to target)
1222 // This is a special case and we should probably do some content thinning / targeting
1223 fetch(url)
1224 .then(res => res.text())
1225 .then(html => {
1226 const parser = new DOMParser();
1227 const htmlDoc = parser.parseFromString(html, "text/html");
1228 const note = htmlDoc.querySelector('main.content');
1229 if (note !== null) {
1230 // This should only happen for chapter cross references
1231 // (since there is no id in the URL)
1232 // remove the first header
1233 if (note.children.length > 0 && note.children[0].tagName === "HEADER") {
1234 note.children[0].remove();
1235 }
1236 const html = processXRef(null, note);
1237 instance.setContent(html);
1238 }
1239 }).finally(() => {
1240 instance.enable();
1241 instance.show();
1242 });
1243 }
1244 }, function(instance) {
1245 });
1246 }
1247 let selectedAnnoteEl;
1248 const selectorForAnnotation = ( cell, annotation) => {
1249 let cellAttr = 'data-code-cell="' + cell + '"';
1250 let lineAttr = 'data-code-annotation="' + annotation + '"';
1251 const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
1252 return selector;
1253 }
1254 const selectCodeLines = (annoteEl) => {
1255 const doc = window.document;
1256 const targetCell = annoteEl.getAttribute("data-target-cell");
1257 const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
1258 const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
1259 const lines = annoteSpan.getAttribute("data-code-lines").split(",");
1260 const lineIds = lines.map((line) => {
1261 return targetCell + "-" + line;
1262 })
1263 let top = null;
1264 let height = null;
1265 let parent = null;
1266 if (lineIds.length > 0) {
1267 //compute the position of the single el (top and bottom and make a div)
1268 const el = window.document.getElementById(lineIds[0]);
1269 top = el.offsetTop;
1270 height = el.offsetHeight;
1271 parent = el.parentElement.parentElement;
1272 if (lineIds.length > 1) {
1273 const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
1274 const bottom = lastEl.offsetTop + lastEl.offsetHeight;
1275 height = bottom - top;
1276 }
1277 if (top !== null && height !== null && parent !== null) {
1278 // cook up a div (if necessary) and position it
1279 let div = window.document.getElementById("code-annotation-line-highlight");
1280 if (div === null) {
1281 div = window.document.createElement("div");
1282 div.setAttribute("id", "code-annotation-line-highlight");
1283 div.style.position = 'absolute';
1284 parent.appendChild(div);
1285 }
1286 div.style.top = top - 2 + "px";
1287 div.style.height = height + 4 + "px";
1288 div.style.left = 0;
1289 let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
1290 if (gutterDiv === null) {
1291 gutterDiv = window.document.createElement("div");
1292 gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
1293 gutterDiv.style.position = 'absolute';
1294 const codeCell = window.document.getElementById(targetCell);
1295 const gutter = codeCell.querySelector('.code-annotation-gutter');
1296 gutter.appendChild(gutterDiv);
1297 }
1298 gutterDiv.style.top = top - 2 + "px";
1299 gutterDiv.style.height = height + 4 + "px";
1300 }
1301 selectedAnnoteEl = annoteEl;
1302 }
1303 };
1304 const unselectCodeLines = () => {
1305 const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
1306 elementsIds.forEach((elId) => {
1307 const div = window.document.getElementById(elId);
1308 if (div) {
1309 div.remove();
1310 }
1311 });
1312 selectedAnnoteEl = undefined;
1313 };
1314 // Handle positioning of the toggle
1315 window.addEventListener(
1316 "resize",
1317 throttle(() => {
1318 elRect = undefined;
1319 if (selectedAnnoteEl) {
1320 selectCodeLines(selectedAnnoteEl);
1321 }
1322 }, 10)
1323 );
1324 function throttle(fn, ms) {
1325 let throttle = false;
1326 let timer;
1327 return (...args) => {
1328 if(!throttle) { // first call gets through
1329 fn.apply(this, args);
1330 throttle = true;
1331 } else { // all the others get throttled
1332 if(timer) clearTimeout(timer); // cancel #2
1333 timer = setTimeout(() => {
1334 fn.apply(this, args);
1335 timer = throttle = false;
1336 }, ms);
1337 }
1338 };
1339 }
1340 // Attach click handler to the DT
1341 const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
1342 for (const annoteDlNode of annoteDls) {
1343 annoteDlNode.addEventListener('click', (event) => {
1344 const clickedEl = event.target;
1345 if (clickedEl !== selectedAnnoteEl) {
1346 unselectCodeLines();
1347 const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
1348 if (activeEl) {
1349 activeEl.classList.remove('code-annotation-active');
1350 }
1351 selectCodeLines(clickedEl);
1352 clickedEl.classList.add('code-annotation-active');
1353 } else {
1354 // Unselect the line
1355 unselectCodeLines();
1356 clickedEl.classList.remove('code-annotation-active');
1357 }
1358 });
1359 }
1360 const findCites = (el) => {
1361 const parentEl = el.parentElement;
1362 if (parentEl) {
1363 const cites = parentEl.dataset.cites;
1364 if (cites) {
1365 return {
1366 el,
1367 cites: cites.split(' ')
1368 };
1369 } else {
1370 return findCites(el.parentElement)
1371 }
1372 } else {
1373 return undefined;
1374 }
1375 };
1376 var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
1377 for (var i=0; i<bibliorefs.length; i++) {
1378 const ref = bibliorefs[i];
1379 const citeInfo = findCites(ref);
1380 if (citeInfo) {
1381 tippyHover(citeInfo.el, function() {
1382 var popup = window.document.createElement('div');
1383 citeInfo.cites.forEach(function(cite) {
1384 var citeDiv = window.document.createElement('div');
1385 citeDiv.classList.add('hanging-indent');
1386 citeDiv.classList.add('csl-entry');
1387 var biblioDiv = window.document.getElementById('ref-' + cite);
1388 if (biblioDiv) {
1389 citeDiv.innerHTML = biblioDiv.innerHTML;
1390 }
1391 popup.appendChild(citeDiv);
1392 });
1393 return popup.innerHTML;
1394 });
1395 }
1396 }
1397 });
1398 </script>
1399</div> <!-- /content -->
1400<footer class="footer">
1401 <div class="nav-footer">
1402 <div class="nav-footer-left">
1403<p>Built with <a href="https://quarto.org/">Quarto</a></p>
1404</div>
1405 <div class="nav-footer-center">
1406
1407 <div class="toc-actions d-sm-block d-md-none"><ul><li><a href="https://github.com/your-org/atdata/edit/main/reference/protocols.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></div>
1408 <div class="nav-footer-right">
1409<p>MIT License</p>
1410</div>
1411 </div>
1412</footer>
1413
1414
1415
1416
1417</body></html>