A loose federation of distributed, typed datasets
1<!DOCTYPE html>
2<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
3
4<meta charset="utf-8">
5<meta name="generator" content="quarto-1.7.34">
6
7<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
8
9<meta name="description" content="Publish and discover datasets on the ATProto network">
10
11<title>Atmosphere Publishing – atdata</title>
12<style>
13code{white-space: pre-wrap;}
14span.smallcaps{font-variant: small-caps;}
15div.columns{display: flex; gap: min(4vw, 1.5em);}
16div.column{flex: auto; overflow-x: auto;}
17div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
18ul.task-list{list-style: none;}
19ul.task-list li input[type="checkbox"] {
20 width: 0.8em;
21 margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
22 vertical-align: middle;
23}
24/* CSS for syntax highlighting */
25html { -webkit-text-size-adjust: 100%; }
26pre > code.sourceCode { white-space: pre; position: relative; }
27pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
28pre > code.sourceCode > span:empty { height: 1.2em; }
29.sourceCode { overflow: visible; }
30code.sourceCode > span { color: inherit; text-decoration: inherit; }
31div.sourceCode { margin: 1em 0; }
32pre.sourceCode { margin: 0; }
33@media screen {
34div.sourceCode { overflow: auto; }
35}
36@media print {
37pre > code.sourceCode { white-space: pre-wrap; }
38pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
39}
40pre.numberSource code
41 { counter-reset: source-line 0; }
42pre.numberSource code > span
43 { position: relative; left: -4em; counter-increment: source-line; }
44pre.numberSource code > span > a:first-child::before
45 { content: counter(source-line);
46 position: relative; left: -1em; text-align: right; vertical-align: baseline;
47 border: none; display: inline-block;
48 -webkit-touch-callout: none; -webkit-user-select: none;
49 -khtml-user-select: none; -moz-user-select: none;
50 -ms-user-select: none; user-select: none;
51 padding: 0 4px; width: 4em;
52 }
53pre.numberSource { margin-left: 3em; padding-left: 4px; }
54div.sourceCode
55 { }
56@media screen {
57pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
58}
59</style>
60
61
62<script src="../site_libs/quarto-nav/quarto-nav.js"></script>
63<script src="../site_libs/quarto-nav/headroom.min.js"></script>
64<script src="../site_libs/clipboard/clipboard.min.js"></script>
65<script src="../site_libs/quarto-search/autocomplete.umd.js"></script>
66<script src="../site_libs/quarto-search/fuse.min.js"></script>
67<script src="../site_libs/quarto-search/quarto-search.js"></script>
68<meta name="quarto:offset" content="../">
69<script src="../site_libs/quarto-html/quarto.js" type="module"></script>
70<script src="../site_libs/quarto-html/tabsets/tabsets.js" type="module"></script>
71<script src="../site_libs/quarto-html/popper.min.js"></script>
72<script src="../site_libs/quarto-html/tippy.umd.min.js"></script>
73<script src="../site_libs/quarto-html/anchor.min.js"></script>
74<link href="../site_libs/quarto-html/tippy.css" rel="stylesheet">
75<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme" id="quarto-text-highlighting-styles">
76<link href="../site_libs/quarto-html/quarto-syntax-highlighting-dark-8dcd8563ea6803ab7cbb3d71ca5772e1.css" rel="stylesheet" class="quarto-color-scheme quarto-color-alternate" id="quarto-text-highlighting-styles">
77<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme-extra" id="quarto-text-highlighting-styles">
78<script src="../site_libs/bootstrap/bootstrap.min.js"></script>
79<link href="../site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
80<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme" id="quarto-bootstrap" data-mode="light">
81<link href="../site_libs/bootstrap/bootstrap-dark-7964ffd8887b0991fe8d71c6c8bc75d6.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme quarto-color-alternate" id="quarto-bootstrap" data-mode="dark">
82<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme-extra" id="quarto-bootstrap" data-mode="light">
83<script id="quarto-search-options" type="application/json">{
84 "location": "navbar",
85 "copy-button": false,
86 "collapse-after": 3,
87 "panel-placement": "end",
88 "type": "overlay",
89 "limit": 50,
90 "keyboard-shortcut": [
91 "f",
92 "/",
93 "s"
94 ],
95 "show-item-context": false,
96 "language": {
97 "search-no-results-text": "No results",
98 "search-matching-documents-text": "matching documents",
99 "search-copy-link-title": "Copy link to search",
100 "search-hide-matches-text": "Hide additional matches",
101 "search-more-match-text": "more match in this document",
102 "search-more-matches-text": "more matches in this document",
103 "search-clear-button-title": "Clear",
104 "search-text-placeholder": "",
105 "search-detached-cancel-button-title": "Cancel",
106 "search-submit-button-title": "Submit",
107 "search-label": "Search"
108 }
109}</script>
110
111
112<link rel="stylesheet" href="../assets/styles.css">
113</head>
114
115<body class="nav-sidebar docked nav-fixed quarto-light"><script id="quarto-html-before-body" type="application/javascript">
116 const toggleBodyColorMode = (bsSheetEl) => {
117 const mode = bsSheetEl.getAttribute("data-mode");
118 const bodyEl = window.document.querySelector("body");
119 if (mode === "dark") {
120 bodyEl.classList.add("quarto-dark");
121 bodyEl.classList.remove("quarto-light");
122 } else {
123 bodyEl.classList.add("quarto-light");
124 bodyEl.classList.remove("quarto-dark");
125 }
126 }
127 const toggleBodyColorPrimary = () => {
128 const bsSheetEl = window.document.querySelector("link#quarto-bootstrap:not([rel=disabled-stylesheet])");
129 if (bsSheetEl) {
130 toggleBodyColorMode(bsSheetEl);
131 }
132 }
133 const setColorSchemeToggle = (alternate) => {
134 const toggles = window.document.querySelectorAll('.quarto-color-scheme-toggle');
135 for (let i=0; i < toggles.length; i++) {
136 const toggle = toggles[i];
137 if (toggle) {
138 if (alternate) {
139 toggle.classList.add("alternate");
140 } else {
141 toggle.classList.remove("alternate");
142 }
143 }
144 }
145 };
146 const toggleColorMode = (alternate) => {
147 // Switch the stylesheets
148 const primaryStylesheets = window.document.querySelectorAll('link.quarto-color-scheme:not(.quarto-color-alternate)');
149 const alternateStylesheets = window.document.querySelectorAll('link.quarto-color-scheme.quarto-color-alternate');
150 manageTransitions('#quarto-margin-sidebar .nav-link', false);
151 if (alternate) {
152 // note: dark is layered on light, we don't disable primary!
153 enableStylesheet(alternateStylesheets);
154 for (const sheetNode of alternateStylesheets) {
155 if (sheetNode.id === "quarto-bootstrap") {
156 toggleBodyColorMode(sheetNode);
157 }
158 }
159 } else {
160 disableStylesheet(alternateStylesheets);
161 enableStylesheet(primaryStylesheets)
162 toggleBodyColorPrimary();
163 }
164 manageTransitions('#quarto-margin-sidebar .nav-link', true);
165 // Switch the toggles
166 setColorSchemeToggle(alternate)
167 // Hack to workaround the fact that safari doesn't
168 // properly recolor the scrollbar when toggling (#1455)
169 if (navigator.userAgent.indexOf('Safari') > 0 && navigator.userAgent.indexOf('Chrome') == -1) {
170 manageTransitions("body", false);
171 window.scrollTo(0, 1);
172 setTimeout(() => {
173 window.scrollTo(0, 0);
174 manageTransitions("body", true);
175 }, 40);
176 }
177 }
178 const disableStylesheet = (stylesheets) => {
179 for (let i=0; i < stylesheets.length; i++) {
180 const stylesheet = stylesheets[i];
181 stylesheet.rel = 'disabled-stylesheet';
182 }
183 }
184 const enableStylesheet = (stylesheets) => {
185 for (let i=0; i < stylesheets.length; i++) {
186 const stylesheet = stylesheets[i];
187 if(stylesheet.rel !== 'stylesheet') { // for Chrome, which will still FOUC without this check
188 stylesheet.rel = 'stylesheet';
189 }
190 }
191 }
192 const manageTransitions = (selector, allowTransitions) => {
193 const els = window.document.querySelectorAll(selector);
194 for (let i=0; i < els.length; i++) {
195 const el = els[i];
196 if (allowTransitions) {
197 el.classList.remove('notransition');
198 } else {
199 el.classList.add('notransition');
200 }
201 }
202 }
203 const isFileUrl = () => {
204 return window.location.protocol === 'file:';
205 }
206 const hasAlternateSentinel = () => {
207 let styleSentinel = getColorSchemeSentinel();
208 if (styleSentinel !== null) {
209 return styleSentinel === "alternate";
210 } else {
211 return false;
212 }
213 }
214 const setStyleSentinel = (alternate) => {
215 const value = alternate ? "alternate" : "default";
216 if (!isFileUrl()) {
217 window.localStorage.setItem("quarto-color-scheme", value);
218 } else {
219 localAlternateSentinel = value;
220 }
221 }
222 const getColorSchemeSentinel = () => {
223 if (!isFileUrl()) {
224 const storageValue = window.localStorage.getItem("quarto-color-scheme");
225 return storageValue != null ? storageValue : localAlternateSentinel;
226 } else {
227 return localAlternateSentinel;
228 }
229 }
230 const toggleGiscusIfUsed = (isAlternate, darkModeDefault) => {
231 const baseTheme = document.querySelector('#giscus-base-theme')?.value ?? 'light';
232 const alternateTheme = document.querySelector('#giscus-alt-theme')?.value ?? 'dark';
233 let newTheme = '';
234 if(authorPrefersDark) {
235 newTheme = isAlternate ? baseTheme : alternateTheme;
236 } else {
237 newTheme = isAlternate ? alternateTheme : baseTheme;
238 }
239 const changeGiscusTheme = () => {
240 // From: https://github.com/giscus/giscus/issues/336
241 const sendMessage = (message) => {
242 const iframe = document.querySelector('iframe.giscus-frame');
243 if (!iframe) return;
244 iframe.contentWindow.postMessage({ giscus: message }, 'https://giscus.app');
245 }
246 sendMessage({
247 setConfig: {
248 theme: newTheme
249 }
250 });
251 }
252 const isGiscussLoaded = window.document.querySelector('iframe.giscus-frame') !== null;
253 if (isGiscussLoaded) {
254 changeGiscusTheme();
255 }
256 };
257 const authorPrefersDark = false;
258 const darkModeDefault = authorPrefersDark;
259 document.querySelector('link#quarto-text-highlighting-styles.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
260 document.querySelector('link#quarto-bootstrap.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
261 let localAlternateSentinel = darkModeDefault ? 'alternate' : 'default';
262 // Dark / light mode switch
263 window.quartoToggleColorScheme = () => {
264 // Read the current dark / light value
265 let toAlternate = !hasAlternateSentinel();
266 toggleColorMode(toAlternate);
267 setStyleSentinel(toAlternate);
268 toggleGiscusIfUsed(toAlternate, darkModeDefault);
269 window.dispatchEvent(new Event('resize'));
270 };
271 // Switch to dark mode if need be
272 if (hasAlternateSentinel()) {
273 toggleColorMode(true);
274 } else {
275 toggleColorMode(false);
276 }
277 </script>
278
279<div id="quarto-search-results"></div>
280 <header id="quarto-header" class="headroom fixed-top">
281 <nav class="navbar navbar-expand-lg " data-bs-theme="dark">
282 <div class="navbar-container container-fluid">
283 <div class="navbar-brand-container mx-auto">
284 <a class="navbar-brand" href="../index.html">
285 <span class="navbar-title">atdata</span>
286 </a>
287 </div>
288 <div id="quarto-search" class="" title="Search"></div>
289 <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarCollapse" aria-controls="navbarCollapse" role="menu" aria-expanded="false" aria-label="Toggle navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
290 <span class="navbar-toggler-icon"></span>
291</button>
292 <div class="collapse navbar-collapse" id="navbarCollapse">
293 <ul class="navbar-nav navbar-nav-scroll me-auto">
294 <li class="nav-item">
295 <a class="nav-link active" href="../index.html" aria-current="page">
296<span class="menu-text">Guide</span></a>
297 </li>
298 <li class="nav-item dropdown ">
299 <a class="nav-link dropdown-toggle" href="#" id="nav-menu-tutorials" role="link" data-bs-toggle="dropdown" aria-expanded="false">
300 <span class="menu-text">Tutorials</span>
301 </a>
302 <ul class="dropdown-menu" aria-labelledby="nav-menu-tutorials">
303 <li>
304 <a class="dropdown-item" href="../tutorials/quickstart.html">
305 <span class="dropdown-text">Quick Start</span></a>
306 </li>
307 <li>
308 <a class="dropdown-item" href="../tutorials/local-workflow.html">
309 <span class="dropdown-text">Local Workflow</span></a>
310 </li>
311 <li>
312 <a class="dropdown-item" href="../tutorials/atmosphere.html">
313 <span class="dropdown-text">Atmosphere Publishing</span></a>
314 </li>
315 <li>
316 <a class="dropdown-item" href="../tutorials/promotion.html">
317 <span class="dropdown-text">Promotion Workflow</span></a>
318 </li>
319 </ul>
320 </li>
321 <li class="nav-item dropdown ">
322 <a class="nav-link dropdown-toggle" href="#" id="nav-menu-reference" role="link" data-bs-toggle="dropdown" aria-expanded="false">
323 <span class="menu-text">Reference</span>
324 </a>
325 <ul class="dropdown-menu" aria-labelledby="nav-menu-reference">
326 <li>
327 <a class="dropdown-item" href="../reference/architecture.html">
328 <span class="dropdown-text">Architecture Overview</span></a>
329 </li>
330 <li>
331 <a class="dropdown-item" href="../reference/packable-samples.html">
332 <span class="dropdown-text">Packable Samples</span></a>
333 </li>
334 <li>
335 <a class="dropdown-item" href="../reference/datasets.html">
336 <span class="dropdown-text">Datasets</span></a>
337 </li>
338 <li>
339 <a class="dropdown-item" href="../reference/lenses.html">
340 <span class="dropdown-text">Lenses</span></a>
341 </li>
342 <li>
343 <a class="dropdown-item" href="../reference/local-storage.html">
344 <span class="dropdown-text">Local Storage</span></a>
345 </li>
346 <li>
347 <a class="dropdown-item" href="../reference/atmosphere.html">
348 <span class="dropdown-text">Atmosphere</span></a>
349 </li>
350 <li>
351 <a class="dropdown-item" href="../reference/promotion.html">
352 <span class="dropdown-text">Promotion</span></a>
353 </li>
354 <li>
355 <a class="dropdown-item" href="../reference/load-dataset.html">
356 <span class="dropdown-text">load_dataset API</span></a>
357 </li>
358 <li>
359 <a class="dropdown-item" href="../reference/protocols.html">
360 <span class="dropdown-text">Protocols</span></a>
361 </li>
362 <li>
363 <a class="dropdown-item" href="../reference/uri-spec.html">
364 <span class="dropdown-text">URI Specification</span></a>
365 </li>
366 <li>
367 <a class="dropdown-item" href="../reference/troubleshooting.html">
368 <span class="dropdown-text">Troubleshooting & FAQ</span></a>
369 </li>
370 <li>
371 <a class="dropdown-item" href="../reference/deployment.html">
372 <span class="dropdown-text">Deployment Guide</span></a>
373 </li>
374 </ul>
375 </li>
376 <li class="nav-item">
377 <a class="nav-link" href="../api/index.html">
378<span class="menu-text">API</span></a>
379 </li>
380</ul>
381 <ul class="navbar-nav navbar-nav-scroll ms-auto">
382 <li class="nav-item compact">
383 <a class="nav-link" href="https://github.com/your-org/atdata"> <i class="bi bi-github" role="img">
384</i>
385<span class="menu-text"></span></a>
386 </li>
387</ul>
388 </div> <!-- /navcollapse -->
389 <div class="quarto-navbar-tools">
390 <a href="" class="quarto-color-scheme-toggle quarto-navigation-tool px-1" onclick="window.quartoToggleColorScheme(); return false;" title="Toggle dark mode"><i class="bi"></i></a>
391</div>
392 </div> <!-- /container-fluid -->
393 </nav>
394 <nav class="quarto-secondary-nav">
395 <div class="container-fluid d-flex">
396 <button type="button" class="quarto-btn-toggle btn" data-bs-toggle="collapse" role="button" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
397 <i class="bi bi-layout-text-sidebar-reverse"></i>
398 </button>
399 <nav class="quarto-page-breadcrumbs" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item"><a href="../tutorials/quickstart.html">Getting Started</a></li><li class="breadcrumb-item"><a href="../tutorials/atmosphere.html">Atmosphere Publishing</a></li></ol></nav>
400 <a class="flex-grow-1" role="navigation" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
401 </a>
402 </div>
403 </nav>
404</header>
405<!-- content -->
406<div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-article page-navbar">
407<!-- sidebar -->
408 <nav id="quarto-sidebar" class="sidebar collapse collapse-horizontal quarto-sidebar-collapse-item sidebar-navigation docked overflow-auto">
409 <div class="sidebar-menu-container">
410 <ul class="list-unstyled mt-1">
411 <li class="sidebar-item">
412 <div class="sidebar-item-container">
413 <a href="../index.html" class="sidebar-item-text sidebar-link">
414 <span class="menu-text">atdata</span></a>
415 </div>
416</li>
417 <li class="sidebar-item sidebar-item-section">
418 <div class="sidebar-item-container">
419 <a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" role="navigation" aria-expanded="true">
420 <span class="menu-text">Getting Started</span></a>
421 <a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" role="navigation" aria-expanded="true" aria-label="Toggle section">
422 <i class="bi bi-chevron-right ms-2"></i>
423 </a>
424 </div>
425 <ul id="quarto-sidebar-section-1" class="collapse list-unstyled sidebar-section depth1 show">
426 <li class="sidebar-item">
427 <div class="sidebar-item-container">
428 <a href="../tutorials/quickstart.html" class="sidebar-item-text sidebar-link">
429 <span class="menu-text">Quick Start</span></a>
430 </div>
431</li>
432 <li class="sidebar-item">
433 <div class="sidebar-item-container">
434 <a href="../tutorials/local-workflow.html" class="sidebar-item-text sidebar-link">
435 <span class="menu-text">Local Workflow</span></a>
436 </div>
437</li>
438 <li class="sidebar-item">
439 <div class="sidebar-item-container">
440 <a href="../tutorials/atmosphere.html" class="sidebar-item-text sidebar-link active">
441 <span class="menu-text">Atmosphere Publishing</span></a>
442 </div>
443</li>
444 <li class="sidebar-item">
445 <div class="sidebar-item-container">
446 <a href="../tutorials/promotion.html" class="sidebar-item-text sidebar-link">
447 <span class="menu-text">Promotion Workflow</span></a>
448 </div>
449</li>
450 </ul>
451 </li>
452 <li class="sidebar-item sidebar-item-section">
453 <div class="sidebar-item-container">
454 <a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" role="navigation" aria-expanded="true">
455 <span class="menu-text">Reference</span></a>
456 <a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" role="navigation" aria-expanded="true" aria-label="Toggle section">
457 <i class="bi bi-chevron-right ms-2"></i>
458 </a>
459 </div>
460 <ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
461 <li class="sidebar-item">
462 <div class="sidebar-item-container">
463 <a href="../reference/architecture.html" class="sidebar-item-text sidebar-link">
464 <span class="menu-text">Architecture Overview</span></a>
465 </div>
466</li>
467 <li class="sidebar-item">
468 <div class="sidebar-item-container">
469 <a href="../reference/packable-samples.html" class="sidebar-item-text sidebar-link">
470 <span class="menu-text">Packable Samples</span></a>
471 </div>
472</li>
473 <li class="sidebar-item">
474 <div class="sidebar-item-container">
475 <a href="../reference/datasets.html" class="sidebar-item-text sidebar-link">
476 <span class="menu-text">Datasets</span></a>
477 </div>
478</li>
479 <li class="sidebar-item">
480 <div class="sidebar-item-container">
481 <a href="../reference/lenses.html" class="sidebar-item-text sidebar-link">
482 <span class="menu-text">Lenses</span></a>
483 </div>
484</li>
485 <li class="sidebar-item">
486 <div class="sidebar-item-container">
487 <a href="../reference/local-storage.html" class="sidebar-item-text sidebar-link">
488 <span class="menu-text">Local Storage</span></a>
489 </div>
490</li>
491 <li class="sidebar-item">
492 <div class="sidebar-item-container">
493 <a href="../reference/atmosphere.html" class="sidebar-item-text sidebar-link">
494 <span class="menu-text">Atmosphere (ATProto Integration)</span></a>
495 </div>
496</li>
497 <li class="sidebar-item">
498 <div class="sidebar-item-container">
499 <a href="../reference/promotion.html" class="sidebar-item-text sidebar-link">
500 <span class="menu-text">Promotion Workflow</span></a>
501 </div>
502</li>
503 <li class="sidebar-item">
504 <div class="sidebar-item-container">
505 <a href="../reference/load-dataset.html" class="sidebar-item-text sidebar-link">
506 <span class="menu-text">load_dataset API</span></a>
507 </div>
508</li>
509 <li class="sidebar-item">
510 <div class="sidebar-item-container">
511 <a href="../reference/protocols.html" class="sidebar-item-text sidebar-link">
512 <span class="menu-text">Protocols</span></a>
513 </div>
514</li>
515 <li class="sidebar-item">
516 <div class="sidebar-item-container">
517 <a href="../reference/uri-spec.html" class="sidebar-item-text sidebar-link">
518 <span class="menu-text">URI Specification</span></a>
519 </div>
520</li>
521 <li class="sidebar-item">
522 <div class="sidebar-item-container">
523 <a href="../reference/troubleshooting.html" class="sidebar-item-text sidebar-link">
524 <span class="menu-text">Troubleshooting & FAQ</span></a>
525 </div>
526</li>
527 <li class="sidebar-item">
528 <div class="sidebar-item-container">
529 <a href="../reference/deployment.html" class="sidebar-item-text sidebar-link">
530 <span class="menu-text">Deployment Guide</span></a>
531 </div>
532</li>
533 </ul>
534 </li>
535 </ul>
536 </div>
537</nav>
538<div id="quarto-sidebar-glass" class="quarto-sidebar-collapse-item" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item"></div>
539<!-- margin-sidebar -->
540 <div id="quarto-margin-sidebar" class="sidebar margin-sidebar">
541 <nav id="TOC" role="doc-toc" class="toc-active">
542 <h2 id="toc-title">On this page</h2>
543
544 <ul>
545 <li><a href="#why-federation" id="toc-why-federation" class="nav-link active" data-scroll-target="#why-federation">Why Federation?</a></li>
546 <li><a href="#prerequisites" id="toc-prerequisites" class="nav-link" data-scroll-target="#prerequisites">Prerequisites</a></li>
547 <li><a href="#setup" id="toc-setup" class="nav-link" data-scroll-target="#setup">Setup</a></li>
548 <li><a href="#define-sample-types" id="toc-define-sample-types" class="nav-link" data-scroll-target="#define-sample-types">Define Sample Types</a></li>
549 <li><a href="#type-introspection" id="toc-type-introspection" class="nav-link" data-scroll-target="#type-introspection">Type Introspection</a></li>
550 <li><a href="#at-uri-parsing" id="toc-at-uri-parsing" class="nav-link" data-scroll-target="#at-uri-parsing">AT URI Parsing</a></li>
551 <li><a href="#authentication" id="toc-authentication" class="nav-link" data-scroll-target="#authentication">Authentication</a></li>
552 <li><a href="#publish-a-schema" id="toc-publish-a-schema" class="nav-link" data-scroll-target="#publish-a-schema">Publish a Schema</a></li>
553 <li><a href="#list-your-schemas" id="toc-list-your-schemas" class="nav-link" data-scroll-target="#list-your-schemas">List Your Schemas</a></li>
554 <li><a href="#publish-a-dataset" id="toc-publish-a-dataset" class="nav-link" data-scroll-target="#publish-a-dataset">Publish a Dataset</a>
555 <ul class="collapse">
556 <li><a href="#with-external-urls" id="toc-with-external-urls" class="nav-link" data-scroll-target="#with-external-urls">With External URLs</a></li>
557 <li><a href="#with-pds-blob-storage-recommended" id="toc-with-pds-blob-storage-recommended" class="nav-link" data-scroll-target="#with-pds-blob-storage-recommended">With PDS Blob Storage (Recommended)</a></li>
558 <li><a href="#with-external-urls-1" id="toc-with-external-urls-1" class="nav-link" data-scroll-target="#with-external-urls-1">With External URLs</a></li>
559 </ul></li>
560 <li><a href="#list-and-load-datasets" id="toc-list-and-load-datasets" class="nav-link" data-scroll-target="#list-and-load-datasets">List and Load Datasets</a></li>
561 <li><a href="#load-a-dataset" id="toc-load-a-dataset" class="nav-link" data-scroll-target="#load-a-dataset">Load a Dataset</a></li>
562 <li><a href="#complete-publishing-workflow" id="toc-complete-publishing-workflow" class="nav-link" data-scroll-target="#complete-publishing-workflow">Complete Publishing Workflow</a></li>
563 <li><a href="#what-youve-learned" id="toc-what-youve-learned" class="nav-link" data-scroll-target="#what-youve-learned">What You’ve Learned</a></li>
564 <li><a href="#the-full-picture" id="toc-the-full-picture" class="nav-link" data-scroll-target="#the-full-picture">The Full Picture</a></li>
565 <li><a href="#next-steps" id="toc-next-steps" class="nav-link" data-scroll-target="#next-steps">Next Steps</a></li>
566 </ul>
567<div class="toc-actions"><ul><li><a href="https://github.com/your-org/atdata/edit/main/tutorials/atmosphere.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></nav>
568 </div>
569<!-- main -->
570<main class="content" id="quarto-document-content">
571
572
573<header id="title-block-header" class="quarto-title-block default"><nav class="quarto-page-breadcrumbs quarto-title-breadcrumbs d-none d-lg-block" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item"><a href="../tutorials/quickstart.html">Getting Started</a></li><li class="breadcrumb-item"><a href="../tutorials/atmosphere.html">Atmosphere Publishing</a></li></ol></nav>
574<div class="quarto-title">
575<h1 class="title">Atmosphere Publishing</h1>
576</div>
577
578<div>
579 <div class="description">
580 Publish and discover datasets on the ATProto network
581 </div>
582</div>
583
584
585<div class="quarto-title-meta">
586
587
588
589
590 </div>
591
592
593
594</header>
595
596
597<p>This tutorial demonstrates how to use the atmosphere module to publish datasets to the AT Protocol network, enabling federated discovery and sharing. This is <strong>Layer 3</strong> of atdata’s architecture—decentralized federation that enables cross-organization dataset sharing.</p>
598<section id="why-federation" class="level2">
599<h2 class="anchored" data-anchor-id="why-federation">Why Federation?</h2>
600<p>Team storage (Redis + S3) works well within an organization, but sharing across organizations introduces new challenges:</p>
601<ul>
602<li><strong>Discovery</strong>: How do researchers find relevant datasets across institutions?</li>
603<li><strong>Trust</strong>: How do you verify a dataset is what it claims to be?</li>
604<li><strong>Durability</strong>: What happens if the original publisher goes offline?</li>
605</ul>
606<p>The <strong>AT Protocol</strong> (ATProto), developed by Bluesky, provides a foundation for decentralized social applications. atdata leverages ATProto’s infrastructure for dataset federation:</p>
607<table class="caption-top table">
608<colgroup>
609<col style="width: 53%">
610<col style="width: 46%">
611</colgroup>
612<thead>
613<tr class="header">
614<th>ATProto Feature</th>
615<th>atdata Usage</th>
616</tr>
617</thead>
618<tbody>
619<tr class="odd">
620<td><strong>DIDs</strong> (Decentralized Identifiers)</td>
621<td>Publisher identity verification</td>
622</tr>
623<tr class="even">
624<td><strong>Lexicons</strong></td>
625<td>Dataset/schema record schemas</td>
626</tr>
627<tr class="odd">
628<td><strong>PDSes</strong> (Personal Data Servers)</td>
629<td>Storage for records and blobs</td>
630</tr>
631<tr class="even">
632<td><strong>Relays & AppViews</strong></td>
633<td>Discovery and aggregation</td>
634</tr>
635</tbody>
636</table>
637<p>The key insight: your Bluesky identity (<code>@handle.bsky.social</code>) becomes your dataset publisher identity. Anyone can verify that a dataset was published by you, and can discover your datasets through the federated network.</p>
638</section>
639<section id="prerequisites" class="level2">
640<h2 class="anchored" data-anchor-id="prerequisites">Prerequisites</h2>
641<ul>
642<li><code>pip install atdata[atmosphere]</code></li>
643<li>A Bluesky account with an <a href="https://bsky.app/settings/app-passwords">app-specific password</a></li>
644</ul>
645<div class="callout callout-style-default callout-warning callout-titled">
646<div class="callout-header d-flex align-content-center">
647<div class="callout-icon-container">
648<i class="callout-icon"></i>
649</div>
650<div class="callout-title-container flex-fill">
651Warning
652</div>
653</div>
654<div class="callout-body-container callout-body">
655<p>Always use an app-specific password, not your main Bluesky password.</p>
656</div>
657</div>
658</section>
659<section id="setup" class="level2">
660<h2 class="anchored" data-anchor-id="setup">Setup</h2>
661<div id="be2b35d7" class="cell">
662<div class="sourceCode cell-code" id="cb1"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
663<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> numpy.typing <span class="im">import</span> NDArray</span>
664<span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> atdata</span>
665<span id="cb1-4"><a href="#cb1-4" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.atmosphere <span class="im">import</span> (</span>
666<span id="cb1-5"><a href="#cb1-5" aria-hidden="true" tabindex="-1"></a> AtmosphereClient,</span>
667<span id="cb1-6"><a href="#cb1-6" aria-hidden="true" tabindex="-1"></a> AtmosphereIndex,</span>
668<span id="cb1-7"><a href="#cb1-7" aria-hidden="true" tabindex="-1"></a> PDSBlobStore,</span>
669<span id="cb1-8"><a href="#cb1-8" aria-hidden="true" tabindex="-1"></a> SchemaPublisher,</span>
670<span id="cb1-9"><a href="#cb1-9" aria-hidden="true" tabindex="-1"></a> SchemaLoader,</span>
671<span id="cb1-10"><a href="#cb1-10" aria-hidden="true" tabindex="-1"></a> DatasetPublisher,</span>
672<span id="cb1-11"><a href="#cb1-11" aria-hidden="true" tabindex="-1"></a> DatasetLoader,</span>
673<span id="cb1-12"><a href="#cb1-12" aria-hidden="true" tabindex="-1"></a> AtUri,</span>
674<span id="cb1-13"><a href="#cb1-13" aria-hidden="true" tabindex="-1"></a>)</span>
675<span id="cb1-14"><a href="#cb1-14" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata <span class="im">import</span> BlobSource</span>
676<span id="cb1-15"><a href="#cb1-15" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> webdataset <span class="im">as</span> wds</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
677</div>
678</section>
679<section id="define-sample-types" class="level2">
680<h2 class="anchored" data-anchor-id="define-sample-types">Define Sample Types</h2>
681<div id="f472e384" class="cell">
682<div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
683<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> ImageSample:</span>
684<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a> <span class="co">"""A sample containing image data with metadata."""</span></span>
685<span id="cb2-4"><a href="#cb2-4" aria-hidden="true" tabindex="-1"></a> image: NDArray</span>
686<span id="cb2-5"><a href="#cb2-5" aria-hidden="true" tabindex="-1"></a> label: <span class="bu">str</span></span>
687<span id="cb2-6"><a href="#cb2-6" aria-hidden="true" tabindex="-1"></a> confidence: <span class="bu">float</span></span>
688<span id="cb2-7"><a href="#cb2-7" aria-hidden="true" tabindex="-1"></a></span>
689<span id="cb2-8"><a href="#cb2-8" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
690<span id="cb2-9"><a href="#cb2-9" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> TextEmbeddingSample:</span>
691<span id="cb2-10"><a href="#cb2-10" aria-hidden="true" tabindex="-1"></a> <span class="co">"""A sample containing text with embedding vectors."""</span></span>
692<span id="cb2-11"><a href="#cb2-11" aria-hidden="true" tabindex="-1"></a> text: <span class="bu">str</span></span>
693<span id="cb2-12"><a href="#cb2-12" aria-hidden="true" tabindex="-1"></a> embedding: NDArray</span>
694<span id="cb2-13"><a href="#cb2-13" aria-hidden="true" tabindex="-1"></a> source: <span class="bu">str</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
695</div>
696</section>
697<section id="type-introspection" class="level2">
698<h2 class="anchored" data-anchor-id="type-introspection">Type Introspection</h2>
699<p>See what information is available from a PackableSample type:</p>
700<div id="ffea79d0" class="cell">
701<div class="sourceCode cell-code" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> dataclasses <span class="im">import</span> fields, is_dataclass</span>
702<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a></span>
703<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Sample type: </span><span class="sc">{</span>ImageSample<span class="sc">.</span><span class="va">__name__</span><span class="sc">}</span><span class="ss">"</span>)</span>
704<span id="cb3-4"><a href="#cb3-4" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Is dataclass: </span><span class="sc">{</span>is_dataclass(ImageSample)<span class="sc">}</span><span class="ss">"</span>)</span>
705<span id="cb3-5"><a href="#cb3-5" aria-hidden="true" tabindex="-1"></a></span>
706<span id="cb3-6"><a href="#cb3-6" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="st">"</span><span class="ch">\n</span><span class="st">Fields:"</span>)</span>
707<span id="cb3-7"><a href="#cb3-7" aria-hidden="true" tabindex="-1"></a><span class="cf">for</span> field <span class="kw">in</span> fields(ImageSample):</span>
708<span id="cb3-8"><a href="#cb3-8" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f" - </span><span class="sc">{</span>field<span class="sc">.</span>name<span class="sc">}</span><span class="ss">: </span><span class="sc">{</span>field<span class="sc">.</span><span class="bu">type</span><span class="sc">}</span><span class="ss">"</span>)</span>
709<span id="cb3-9"><a href="#cb3-9" aria-hidden="true" tabindex="-1"></a></span>
710<span id="cb3-10"><a href="#cb3-10" aria-hidden="true" tabindex="-1"></a><span class="co"># Create and serialize a sample</span></span>
711<span id="cb3-11"><a href="#cb3-11" aria-hidden="true" tabindex="-1"></a>sample <span class="op">=</span> ImageSample(</span>
712<span id="cb3-12"><a href="#cb3-12" aria-hidden="true" tabindex="-1"></a> image<span class="op">=</span>np.random.rand(<span class="dv">224</span>, <span class="dv">224</span>, <span class="dv">3</span>).astype(np.float32),</span>
713<span id="cb3-13"><a href="#cb3-13" aria-hidden="true" tabindex="-1"></a> label<span class="op">=</span><span class="st">"cat"</span>,</span>
714<span id="cb3-14"><a href="#cb3-14" aria-hidden="true" tabindex="-1"></a> confidence<span class="op">=</span><span class="fl">0.95</span>,</span>
715<span id="cb3-15"><a href="#cb3-15" aria-hidden="true" tabindex="-1"></a>)</span>
716<span id="cb3-16"><a href="#cb3-16" aria-hidden="true" tabindex="-1"></a></span>
717<span id="cb3-17"><a href="#cb3-17" aria-hidden="true" tabindex="-1"></a>packed <span class="op">=</span> sample.packed</span>
718<span id="cb3-18"><a href="#cb3-18" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"</span><span class="ch">\n</span><span class="ss">Serialized size: </span><span class="sc">{</span><span class="bu">len</span>(packed)<span class="sc">:,}</span><span class="ss"> bytes"</span>)</span>
719<span id="cb3-19"><a href="#cb3-19" aria-hidden="true" tabindex="-1"></a></span>
720<span id="cb3-20"><a href="#cb3-20" aria-hidden="true" tabindex="-1"></a><span class="co"># Round-trip</span></span>
721<span id="cb3-21"><a href="#cb3-21" aria-hidden="true" tabindex="-1"></a>restored <span class="op">=</span> ImageSample.from_bytes(packed)</span>
722<span id="cb3-22"><a href="#cb3-22" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Round-trip successful: </span><span class="sc">{</span>np<span class="sc">.</span>allclose(sample.image, restored.image)<span class="sc">}</span><span class="ss">"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
723</div>
724</section>
725<section id="at-uri-parsing" class="level2">
726<h2 class="anchored" data-anchor-id="at-uri-parsing">AT URI Parsing</h2>
727<p>Every record in ATProto is identified by an <strong>AT URI</strong>, which encodes:</p>
728<ul>
729<li><strong>Authority</strong>: The DID or handle of the record owner</li>
730<li><strong>Collection</strong>: The Lexicon type (like a table name)</li>
731<li><strong>Rkey</strong>: The record key (unique within the collection)</li>
732</ul>
733<p>Understanding AT URIs is essential for working with atmosphere datasets, as they’re how you reference schemas, datasets, and lenses.</p>
734<p>ATProto records are identified by AT URIs:</p>
735<div id="dfdc208c" class="cell">
736<div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a>uris <span class="op">=</span> [</span>
737<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a> <span class="st">"at://did:plc:abc123/ac.foundation.dataset.sampleSchema/xyz789"</span>,</span>
738<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a> <span class="st">"at://alice.bsky.social/ac.foundation.dataset.record/my-dataset"</span>,</span>
739<span id="cb4-4"><a href="#cb4-4" aria-hidden="true" tabindex="-1"></a>]</span>
740<span id="cb4-5"><a href="#cb4-5" aria-hidden="true" tabindex="-1"></a></span>
741<span id="cb4-6"><a href="#cb4-6" aria-hidden="true" tabindex="-1"></a><span class="cf">for</span> uri_str <span class="kw">in</span> uris:</span>
742<span id="cb4-7"><a href="#cb4-7" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f"</span><span class="ch">\n</span><span class="ss">Parsing: </span><span class="sc">{</span>uri_str<span class="sc">}</span><span class="ss">"</span>)</span>
743<span id="cb4-8"><a href="#cb4-8" aria-hidden="true" tabindex="-1"></a> uri <span class="op">=</span> AtUri.parse(uri_str)</span>
744<span id="cb4-9"><a href="#cb4-9" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f" Authority: </span><span class="sc">{</span>uri<span class="sc">.</span>authority<span class="sc">}</span><span class="ss">"</span>)</span>
745<span id="cb4-10"><a href="#cb4-10" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f" Collection: </span><span class="sc">{</span>uri<span class="sc">.</span>collection<span class="sc">}</span><span class="ss">"</span>)</span>
746<span id="cb4-11"><a href="#cb4-11" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f" Rkey: </span><span class="sc">{</span>uri<span class="sc">.</span>rkey<span class="sc">}</span><span class="ss">"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
747</div>
748</section>
749<section id="authentication" class="level2">
750<h2 class="anchored" data-anchor-id="authentication">Authentication</h2>
751<p>The <code>AtmosphereClient</code> handles ATProto authentication. When you authenticate, you’re proving ownership of your decentralized identity (DID), which gives you permission to create and modify records in your Personal Data Server (PDS).</p>
752<p>Connect to ATProto:</p>
753<div id="a4f475f4" class="cell">
754<div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a>client <span class="op">=</span> AtmosphereClient()</span>
755<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a>client.login(<span class="st">"your.handle.social"</span>, <span class="st">"your-app-password"</span>)</span>
756<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a></span>
757<span id="cb5-4"><a href="#cb5-4" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Authenticated as: </span><span class="sc">{</span>client<span class="sc">.</span>handle<span class="sc">}</span><span class="ss">"</span>)</span>
758<span id="cb5-5"><a href="#cb5-5" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"DID: </span><span class="sc">{</span>client<span class="sc">.</span>did<span class="sc">}</span><span class="ss">"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
759</div>
760</section>
761<section id="publish-a-schema" class="level2">
762<h2 class="anchored" data-anchor-id="publish-a-schema">Publish a Schema</h2>
763<p>When you publish a schema to ATProto, it becomes a <strong>public, immutable record</strong> that others can reference. The schema CID ensures that anyone can verify they’re using exactly the same type definition you published.</p>
764<div id="96200176" class="cell">
765<div class="sourceCode cell-code" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a>schema_publisher <span class="op">=</span> SchemaPublisher(client)</span>
766<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a>schema_uri <span class="op">=</span> schema_publisher.publish(</span>
767<span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a> ImageSample,</span>
768<span id="cb6-4"><a href="#cb6-4" aria-hidden="true" tabindex="-1"></a> name<span class="op">=</span><span class="st">"ImageSample"</span>,</span>
769<span id="cb6-5"><a href="#cb6-5" aria-hidden="true" tabindex="-1"></a> version<span class="op">=</span><span class="st">"1.0.0"</span>,</span>
770<span id="cb6-6"><a href="#cb6-6" aria-hidden="true" tabindex="-1"></a> description<span class="op">=</span><span class="st">"Demo: Image sample with label and confidence"</span>,</span>
771<span id="cb6-7"><a href="#cb6-7" aria-hidden="true" tabindex="-1"></a>)</span>
772<span id="cb6-8"><a href="#cb6-8" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Schema URI: </span><span class="sc">{</span>schema_uri<span class="sc">}</span><span class="ss">"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
773</div>
774</section>
775<section id="list-your-schemas" class="level2">
776<h2 class="anchored" data-anchor-id="list-your-schemas">List Your Schemas</h2>
777<div id="076dac8a" class="cell">
778<div class="sourceCode cell-code" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a>schema_loader <span class="op">=</span> SchemaLoader(client)</span>
779<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a>schemas <span class="op">=</span> schema_loader.list_all(limit<span class="op">=</span><span class="dv">10</span>)</span>
780<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Found </span><span class="sc">{</span><span class="bu">len</span>(schemas)<span class="sc">}</span><span class="ss"> schema(s)"</span>)</span>
781<span id="cb7-4"><a href="#cb7-4" aria-hidden="true" tabindex="-1"></a></span>
782<span id="cb7-5"><a href="#cb7-5" aria-hidden="true" tabindex="-1"></a><span class="cf">for</span> schema <span class="kw">in</span> schemas:</span>
783<span id="cb7-6"><a href="#cb7-6" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f" - </span><span class="sc">{</span>schema<span class="sc">.</span>get(<span class="st">'name'</span>, <span class="st">'Unknown'</span>)<span class="sc">}</span><span class="ss">: v</span><span class="sc">{</span>schema<span class="sc">.</span>get(<span class="st">'version'</span>, <span class="st">'?'</span>)<span class="sc">}</span><span class="ss">"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
784</div>
785</section>
786<section id="publish-a-dataset" class="level2">
787<h2 class="anchored" data-anchor-id="publish-a-dataset">Publish a Dataset</h2>
788<section id="with-external-urls" class="level3">
789<h3 class="anchored" data-anchor-id="with-external-urls">With External URLs</h3>
790<div id="6b04e0fa" class="cell">
791<div class="sourceCode cell-code" id="cb8"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a>dataset_publisher <span class="op">=</span> DatasetPublisher(client)</span>
792<span id="cb8-2"><a href="#cb8-2" aria-hidden="true" tabindex="-1"></a>dataset_uri <span class="op">=</span> dataset_publisher.publish_with_urls(</span>
793<span id="cb8-3"><a href="#cb8-3" aria-hidden="true" tabindex="-1"></a> urls<span class="op">=</span>[<span class="st">"s3://example-bucket/demo-data-{000000..000009}.tar"</span>],</span>
794<span id="cb8-4"><a href="#cb8-4" aria-hidden="true" tabindex="-1"></a> schema_uri<span class="op">=</span><span class="bu">str</span>(schema_uri),</span>
795<span id="cb8-5"><a href="#cb8-5" aria-hidden="true" tabindex="-1"></a> name<span class="op">=</span><span class="st">"Demo Image Dataset"</span>,</span>
796<span id="cb8-6"><a href="#cb8-6" aria-hidden="true" tabindex="-1"></a> description<span class="op">=</span><span class="st">"Example dataset demonstrating atmosphere publishing"</span>,</span>
797<span id="cb8-7"><a href="#cb8-7" aria-hidden="true" tabindex="-1"></a> tags<span class="op">=</span>[<span class="st">"demo"</span>, <span class="st">"images"</span>, <span class="st">"atdata"</span>],</span>
798<span id="cb8-8"><a href="#cb8-8" aria-hidden="true" tabindex="-1"></a> license<span class="op">=</span><span class="st">"MIT"</span>,</span>
799<span id="cb8-9"><a href="#cb8-9" aria-hidden="true" tabindex="-1"></a>)</span>
800<span id="cb8-10"><a href="#cb8-10" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Dataset URI: </span><span class="sc">{</span>dataset_uri<span class="sc">}</span><span class="ss">"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
801</div>
802</section>
803<section id="with-pds-blob-storage-recommended" class="level3">
804<h3 class="anchored" data-anchor-id="with-pds-blob-storage-recommended">With PDS Blob Storage (Recommended)</h3>
805<p>The <code>PDSBlobStore</code> is the <strong>fully decentralized</strong> option: your dataset shards are stored as ATProto blobs directly in your PDS, alongside your other ATProto records. This means:</p>
806<ul>
807<li><strong>No external dependencies</strong>: Data lives in the same infrastructure as your identity</li>
808<li><strong>Content-addressed</strong>: Blobs are identified by their CID, ensuring integrity</li>
809<li><strong>Federated replication</strong>: Relays can mirror your blobs for availability</li>
810</ul>
811<p>For fully decentralized storage, use <code>PDSBlobStore</code> to store dataset shards directly as ATProto blobs in your PDS:</p>
812<div id="703c6bab" class="cell">
813<div class="sourceCode cell-code" id="cb9"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb9-1"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Create store and index with blob storage</span></span>
814<span id="cb9-2"><a href="#cb9-2" aria-hidden="true" tabindex="-1"></a>store <span class="op">=</span> PDSBlobStore(client)</span>
815<span id="cb9-3"><a href="#cb9-3" aria-hidden="true" tabindex="-1"></a>index <span class="op">=</span> AtmosphereIndex(client, data_store<span class="op">=</span>store)</span>
816<span id="cb9-4"><a href="#cb9-4" aria-hidden="true" tabindex="-1"></a></span>
817<span id="cb9-5"><a href="#cb9-5" aria-hidden="true" tabindex="-1"></a><span class="co"># Define sample type</span></span>
818<span id="cb9-6"><a href="#cb9-6" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
819<span id="cb9-7"><a href="#cb9-7" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> FeatureSample:</span>
820<span id="cb9-8"><a href="#cb9-8" aria-hidden="true" tabindex="-1"></a> features: NDArray</span>
821<span id="cb9-9"><a href="#cb9-9" aria-hidden="true" tabindex="-1"></a> label: <span class="bu">int</span></span>
822<span id="cb9-10"><a href="#cb9-10" aria-hidden="true" tabindex="-1"></a></span>
823<span id="cb9-11"><a href="#cb9-11" aria-hidden="true" tabindex="-1"></a><span class="co"># Create dataset in memory or from existing tar</span></span>
824<span id="cb9-12"><a href="#cb9-12" aria-hidden="true" tabindex="-1"></a>samples <span class="op">=</span> [FeatureSample(features<span class="op">=</span>np.random.randn(<span class="dv">64</span>).astype(np.float32), label<span class="op">=</span>i <span class="op">%</span> <span class="dv">10</span>) <span class="cf">for</span> i <span class="kw">in</span> <span class="bu">range</span>(<span class="dv">100</span>)]</span>
825<span id="cb9-13"><a href="#cb9-13" aria-hidden="true" tabindex="-1"></a></span>
826<span id="cb9-14"><a href="#cb9-14" aria-hidden="true" tabindex="-1"></a><span class="co"># Write to temporary tar</span></span>
827<span id="cb9-15"><a href="#cb9-15" aria-hidden="true" tabindex="-1"></a><span class="cf">with</span> wds.writer.TarWriter(<span class="st">"temp.tar"</span>) <span class="im">as</span> sink:</span>
828<span id="cb9-16"><a href="#cb9-16" aria-hidden="true" tabindex="-1"></a> <span class="cf">for</span> i, s <span class="kw">in</span> <span class="bu">enumerate</span>(samples):</span>
829<span id="cb9-17"><a href="#cb9-17" aria-hidden="true" tabindex="-1"></a> sink.write({<span class="op">**</span>s.as_wds, <span class="st">"__key__"</span>: <span class="ss">f"</span><span class="sc">{</span>i<span class="sc">:06d}</span><span class="ss">"</span>})</span>
830<span id="cb9-18"><a href="#cb9-18" aria-hidden="true" tabindex="-1"></a></span>
831<span id="cb9-19"><a href="#cb9-19" aria-hidden="true" tabindex="-1"></a>dataset <span class="op">=</span> atdata.Dataset[FeatureSample](<span class="st">"temp.tar"</span>)</span>
832<span id="cb9-20"><a href="#cb9-20" aria-hidden="true" tabindex="-1"></a></span>
833<span id="cb9-21"><a href="#cb9-21" aria-hidden="true" tabindex="-1"></a><span class="co"># Publish - shards are uploaded as blobs automatically</span></span>
834<span id="cb9-22"><a href="#cb9-22" aria-hidden="true" tabindex="-1"></a>schema_uri <span class="op">=</span> index.publish_schema(FeatureSample, version<span class="op">=</span><span class="st">"1.0.0"</span>)</span>
835<span id="cb9-23"><a href="#cb9-23" aria-hidden="true" tabindex="-1"></a>entry <span class="op">=</span> index.insert_dataset(</span>
836<span id="cb9-24"><a href="#cb9-24" aria-hidden="true" tabindex="-1"></a> dataset,</span>
837<span id="cb9-25"><a href="#cb9-25" aria-hidden="true" tabindex="-1"></a> name<span class="op">=</span><span class="st">"blob-stored-features"</span>,</span>
838<span id="cb9-26"><a href="#cb9-26" aria-hidden="true" tabindex="-1"></a> schema_ref<span class="op">=</span>schema_uri,</span>
839<span id="cb9-27"><a href="#cb9-27" aria-hidden="true" tabindex="-1"></a> description<span class="op">=</span><span class="st">"Features stored as PDS blobs"</span>,</span>
840<span id="cb9-28"><a href="#cb9-28" aria-hidden="true" tabindex="-1"></a>)</span>
841<span id="cb9-29"><a href="#cb9-29" aria-hidden="true" tabindex="-1"></a></span>
842<span id="cb9-30"><a href="#cb9-30" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Dataset URI: </span><span class="sc">{</span>entry<span class="sc">.</span>uri<span class="sc">}</span><span class="ss">"</span>)</span>
843<span id="cb9-31"><a href="#cb9-31" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Blob URLs: </span><span class="sc">{</span>entry<span class="sc">.</span>data_urls<span class="sc">}</span><span class="ss">"</span>) <span class="co"># at://did/blob/cid format</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
844</div>
845<div class="callout callout-style-default callout-tip callout-titled">
846<div class="callout-header d-flex align-content-center">
847<div class="callout-icon-container">
848<i class="callout-icon"></i>
849</div>
850<div class="callout-title-container flex-fill">
851Reading Blob-Stored Datasets
852</div>
853</div>
854<div class="callout-body-container callout-body">
855<p>Use <code>BlobSource</code> to stream directly from PDS blobs:</p>
856<div id="930a5ae0" class="cell">
857<div class="sourceCode cell-code" id="cb10"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Create source from the blob URLs</span></span>
858<span id="cb10-2"><a href="#cb10-2" aria-hidden="true" tabindex="-1"></a>source <span class="op">=</span> store.create_source(entry.data_urls)</span>
859<span id="cb10-3"><a href="#cb10-3" aria-hidden="true" tabindex="-1"></a></span>
860<span id="cb10-4"><a href="#cb10-4" aria-hidden="true" tabindex="-1"></a><span class="co"># Or manually from blob references</span></span>
861<span id="cb10-5"><a href="#cb10-5" aria-hidden="true" tabindex="-1"></a>source <span class="op">=</span> BlobSource.from_refs([</span>
862<span id="cb10-6"><a href="#cb10-6" aria-hidden="true" tabindex="-1"></a> {<span class="st">"did"</span>: client.did, <span class="st">"cid"</span>: <span class="st">"bafyrei..."</span>},</span>
863<span id="cb10-7"><a href="#cb10-7" aria-hidden="true" tabindex="-1"></a>])</span>
864<span id="cb10-8"><a href="#cb10-8" aria-hidden="true" tabindex="-1"></a></span>
865<span id="cb10-9"><a href="#cb10-9" aria-hidden="true" tabindex="-1"></a><span class="co"># Load and iterate</span></span>
866<span id="cb10-10"><a href="#cb10-10" aria-hidden="true" tabindex="-1"></a>ds <span class="op">=</span> atdata.Dataset[FeatureSample](source)</span>
867<span id="cb10-11"><a href="#cb10-11" aria-hidden="true" tabindex="-1"></a><span class="cf">for</span> batch <span class="kw">in</span> ds.ordered(batch_size<span class="op">=</span><span class="dv">32</span>):</span>
868<span id="cb10-12"><a href="#cb10-12" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(batch.features.shape)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
869</div>
870</div>
871</div>
872</section>
873<section id="with-external-urls-1" class="level3">
874<h3 class="anchored" data-anchor-id="with-external-urls-1">With External URLs</h3>
875<p>For larger datasets that exceed PDS blob limits, or when you already have data in object storage, you can publish a dataset record that references external URLs. The ATProto record serves as the <strong>index entry</strong> while the actual data lives elsewhere.</p>
876<p>For larger datasets or when using existing object storage:</p>
877<div id="f5218175" class="cell">
878<div class="sourceCode cell-code" id="cb11"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb11-1"><a href="#cb11-1" aria-hidden="true" tabindex="-1"></a>dataset_publisher <span class="op">=</span> DatasetPublisher(client)</span>
879<span id="cb11-2"><a href="#cb11-2" aria-hidden="true" tabindex="-1"></a>dataset_uri <span class="op">=</span> dataset_publisher.publish_with_urls(</span>
880<span id="cb11-3"><a href="#cb11-3" aria-hidden="true" tabindex="-1"></a> urls<span class="op">=</span>[<span class="st">"s3://example-bucket/demo-data-{000000..000009}.tar"</span>],</span>
881<span id="cb11-4"><a href="#cb11-4" aria-hidden="true" tabindex="-1"></a> schema_uri<span class="op">=</span><span class="bu">str</span>(schema_uri),</span>
882<span id="cb11-5"><a href="#cb11-5" aria-hidden="true" tabindex="-1"></a> name<span class="op">=</span><span class="st">"Demo Image Dataset"</span>,</span>
883<span id="cb11-6"><a href="#cb11-6" aria-hidden="true" tabindex="-1"></a> description<span class="op">=</span><span class="st">"Example dataset demonstrating atmosphere publishing"</span>,</span>
884<span id="cb11-7"><a href="#cb11-7" aria-hidden="true" tabindex="-1"></a> tags<span class="op">=</span>[<span class="st">"demo"</span>, <span class="st">"images"</span>, <span class="st">"atdata"</span>],</span>
885<span id="cb11-8"><a href="#cb11-8" aria-hidden="true" tabindex="-1"></a> license<span class="op">=</span><span class="st">"MIT"</span>,</span>
886<span id="cb11-9"><a href="#cb11-9" aria-hidden="true" tabindex="-1"></a>)</span>
887<span id="cb11-10"><a href="#cb11-10" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Dataset URI: </span><span class="sc">{</span>dataset_uri<span class="sc">}</span><span class="ss">"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
888</div>
889</section>
890</section>
891<section id="list-and-load-datasets" class="level2">
892<h2 class="anchored" data-anchor-id="list-and-load-datasets">List and Load Datasets</h2>
893<div id="be94f99c" class="cell">
894<div class="sourceCode cell-code" id="cb12"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb12-1"><a href="#cb12-1" aria-hidden="true" tabindex="-1"></a>dataset_loader <span class="op">=</span> DatasetLoader(client)</span>
895<span id="cb12-2"><a href="#cb12-2" aria-hidden="true" tabindex="-1"></a>datasets <span class="op">=</span> dataset_loader.list_all(limit<span class="op">=</span><span class="dv">10</span>)</span>
896<span id="cb12-3"><a href="#cb12-3" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Found </span><span class="sc">{</span><span class="bu">len</span>(datasets)<span class="sc">}</span><span class="ss"> dataset(s)"</span>)</span>
897<span id="cb12-4"><a href="#cb12-4" aria-hidden="true" tabindex="-1"></a></span>
898<span id="cb12-5"><a href="#cb12-5" aria-hidden="true" tabindex="-1"></a><span class="cf">for</span> ds <span class="kw">in</span> datasets:</span>
899<span id="cb12-6"><a href="#cb12-6" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f" - </span><span class="sc">{</span>ds<span class="sc">.</span>get(<span class="st">'name'</span>, <span class="st">'Unknown'</span>)<span class="sc">}</span><span class="ss">"</span>)</span>
900<span id="cb12-7"><a href="#cb12-7" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f" Schema: </span><span class="sc">{</span>ds<span class="sc">.</span>get(<span class="st">'schemaRef'</span>, <span class="st">'N/A'</span>)<span class="sc">}</span><span class="ss">"</span>)</span>
901<span id="cb12-8"><a href="#cb12-8" aria-hidden="true" tabindex="-1"></a> tags <span class="op">=</span> ds.get(<span class="st">'tags'</span>, [])</span>
902<span id="cb12-9"><a href="#cb12-9" aria-hidden="true" tabindex="-1"></a> <span class="cf">if</span> tags:</span>
903<span id="cb12-10"><a href="#cb12-10" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f" Tags: </span><span class="sc">{</span><span class="st">', '</span><span class="sc">.</span>join(tags)<span class="sc">}</span><span class="ss">"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
904</div>
905</section>
906<section id="load-a-dataset" class="level2">
907<h2 class="anchored" data-anchor-id="load-a-dataset">Load a Dataset</h2>
908<div id="7b4bc5bb" class="cell">
909<div class="sourceCode cell-code" id="cb13"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb13-1"><a href="#cb13-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Check storage type</span></span>
910<span id="cb13-2"><a href="#cb13-2" aria-hidden="true" tabindex="-1"></a>storage_type <span class="op">=</span> dataset_loader.get_storage_type(<span class="bu">str</span>(blob_dataset_uri))</span>
911<span id="cb13-3"><a href="#cb13-3" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Storage type: </span><span class="sc">{</span>storage_type<span class="sc">}</span><span class="ss">"</span>)</span>
912<span id="cb13-4"><a href="#cb13-4" aria-hidden="true" tabindex="-1"></a></span>
913<span id="cb13-5"><a href="#cb13-5" aria-hidden="true" tabindex="-1"></a><span class="cf">if</span> storage_type <span class="op">==</span> <span class="st">"blobs"</span>:</span>
914<span id="cb13-6"><a href="#cb13-6" aria-hidden="true" tabindex="-1"></a> blob_urls <span class="op">=</span> dataset_loader.get_blob_urls(<span class="bu">str</span>(blob_dataset_uri))</span>
915<span id="cb13-7"><a href="#cb13-7" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f"Blob URLs: </span><span class="sc">{</span><span class="bu">len</span>(blob_urls)<span class="sc">}</span><span class="ss"> blob(s)"</span>)</span>
916<span id="cb13-8"><a href="#cb13-8" aria-hidden="true" tabindex="-1"></a></span>
917<span id="cb13-9"><a href="#cb13-9" aria-hidden="true" tabindex="-1"></a><span class="co"># Load and iterate (works for both storage types)</span></span>
918<span id="cb13-10"><a href="#cb13-10" aria-hidden="true" tabindex="-1"></a>ds <span class="op">=</span> dataset_loader.to_dataset(<span class="bu">str</span>(blob_dataset_uri), DemoSample)</span>
919<span id="cb13-11"><a href="#cb13-11" aria-hidden="true" tabindex="-1"></a><span class="cf">for</span> batch <span class="kw">in</span> ds.ordered():</span>
920<span id="cb13-12"><a href="#cb13-12" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f"Sample id=</span><span class="sc">{</span>batch<span class="sc">.</span><span class="bu">id</span><span class="sc">}</span><span class="ss">, text=</span><span class="sc">{</span>batch<span class="sc">.</span>text<span class="sc">}</span><span class="ss">"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
921</div>
922</section>
923<section id="complete-publishing-workflow" class="level2">
924<h2 class="anchored" data-anchor-id="complete-publishing-workflow">Complete Publishing Workflow</h2>
925<p>Here’s the end-to-end workflow for publishing a dataset to the atmosphere:</p>
926<ol type="1">
927<li><strong>Define your sample type</strong> using <code>@packable</code></li>
928<li><strong>Create samples and write to tar</strong> (same as local workflow)</li>
929<li><strong>Authenticate</strong> with your ATProto identity</li>
930<li><strong>Create index with blob storage</strong> (<code>AtmosphereIndex</code> + <code>PDSBlobStore</code>)</li>
931<li><strong>Publish schema</strong> (creates ATProto record)</li>
932<li><strong>Insert dataset</strong> (uploads blobs, creates dataset record)</li>
933</ol>
934<p>Notice how similar this is to the local workflow—the same sample types and patterns, just with a different storage backend.</p>
935<p>This example shows the recommended workflow using <code>PDSBlobStore</code> for fully decentralized storage:</p>
936<div id="48092185" class="cell">
937<div class="sourceCode cell-code" id="cb14"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb14-1"><a href="#cb14-1" aria-hidden="true" tabindex="-1"></a><span class="co"># 1. Define and create samples</span></span>
938<span id="cb14-2"><a href="#cb14-2" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
939<span id="cb14-3"><a href="#cb14-3" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> FeatureSample:</span>
940<span id="cb14-4"><a href="#cb14-4" aria-hidden="true" tabindex="-1"></a> features: NDArray</span>
941<span id="cb14-5"><a href="#cb14-5" aria-hidden="true" tabindex="-1"></a> label: <span class="bu">int</span></span>
942<span id="cb14-6"><a href="#cb14-6" aria-hidden="true" tabindex="-1"></a> source: <span class="bu">str</span></span>
943<span id="cb14-7"><a href="#cb14-7" aria-hidden="true" tabindex="-1"></a></span>
944<span id="cb14-8"><a href="#cb14-8" aria-hidden="true" tabindex="-1"></a>samples <span class="op">=</span> [</span>
945<span id="cb14-9"><a href="#cb14-9" aria-hidden="true" tabindex="-1"></a> FeatureSample(</span>
946<span id="cb14-10"><a href="#cb14-10" aria-hidden="true" tabindex="-1"></a> features<span class="op">=</span>np.random.randn(<span class="dv">128</span>).astype(np.float32),</span>
947<span id="cb14-11"><a href="#cb14-11" aria-hidden="true" tabindex="-1"></a> label<span class="op">=</span>i <span class="op">%</span> <span class="dv">10</span>,</span>
948<span id="cb14-12"><a href="#cb14-12" aria-hidden="true" tabindex="-1"></a> source<span class="op">=</span><span class="st">"synthetic"</span>,</span>
949<span id="cb14-13"><a href="#cb14-13" aria-hidden="true" tabindex="-1"></a> )</span>
950<span id="cb14-14"><a href="#cb14-14" aria-hidden="true" tabindex="-1"></a> <span class="cf">for</span> i <span class="kw">in</span> <span class="bu">range</span>(<span class="dv">1000</span>)</span>
951<span id="cb14-15"><a href="#cb14-15" aria-hidden="true" tabindex="-1"></a>]</span>
952<span id="cb14-16"><a href="#cb14-16" aria-hidden="true" tabindex="-1"></a></span>
953<span id="cb14-17"><a href="#cb14-17" aria-hidden="true" tabindex="-1"></a><span class="co"># 2. Write to tar</span></span>
954<span id="cb14-18"><a href="#cb14-18" aria-hidden="true" tabindex="-1"></a><span class="cf">with</span> wds.writer.TarWriter(<span class="st">"features.tar"</span>) <span class="im">as</span> sink:</span>
955<span id="cb14-19"><a href="#cb14-19" aria-hidden="true" tabindex="-1"></a> <span class="cf">for</span> i, s <span class="kw">in</span> <span class="bu">enumerate</span>(samples):</span>
956<span id="cb14-20"><a href="#cb14-20" aria-hidden="true" tabindex="-1"></a> sink.write({<span class="op">**</span>s.as_wds, <span class="st">"__key__"</span>: <span class="ss">f"</span><span class="sc">{</span>i<span class="sc">:06d}</span><span class="ss">"</span>})</span>
957<span id="cb14-21"><a href="#cb14-21" aria-hidden="true" tabindex="-1"></a></span>
958<span id="cb14-22"><a href="#cb14-22" aria-hidden="true" tabindex="-1"></a><span class="co"># 3. Authenticate and create index with blob storage</span></span>
959<span id="cb14-23"><a href="#cb14-23" aria-hidden="true" tabindex="-1"></a>client <span class="op">=</span> AtmosphereClient()</span>
960<span id="cb14-24"><a href="#cb14-24" aria-hidden="true" tabindex="-1"></a>client.login(<span class="st">"myhandle.bsky.social"</span>, <span class="st">"app-password"</span>)</span>
961<span id="cb14-25"><a href="#cb14-25" aria-hidden="true" tabindex="-1"></a></span>
962<span id="cb14-26"><a href="#cb14-26" aria-hidden="true" tabindex="-1"></a>store <span class="op">=</span> PDSBlobStore(client)</span>
963<span id="cb14-27"><a href="#cb14-27" aria-hidden="true" tabindex="-1"></a>index <span class="op">=</span> AtmosphereIndex(client, data_store<span class="op">=</span>store)</span>
964<span id="cb14-28"><a href="#cb14-28" aria-hidden="true" tabindex="-1"></a></span>
965<span id="cb14-29"><a href="#cb14-29" aria-hidden="true" tabindex="-1"></a><span class="co"># 4. Publish schema</span></span>
966<span id="cb14-30"><a href="#cb14-30" aria-hidden="true" tabindex="-1"></a>schema_uri <span class="op">=</span> index.publish_schema(</span>
967<span id="cb14-31"><a href="#cb14-31" aria-hidden="true" tabindex="-1"></a> FeatureSample,</span>
968<span id="cb14-32"><a href="#cb14-32" aria-hidden="true" tabindex="-1"></a> version<span class="op">=</span><span class="st">"1.0.0"</span>,</span>
969<span id="cb14-33"><a href="#cb14-33" aria-hidden="true" tabindex="-1"></a> description<span class="op">=</span><span class="st">"Feature vectors with labels"</span>,</span>
970<span id="cb14-34"><a href="#cb14-34" aria-hidden="true" tabindex="-1"></a>)</span>
971<span id="cb14-35"><a href="#cb14-35" aria-hidden="true" tabindex="-1"></a></span>
972<span id="cb14-36"><a href="#cb14-36" aria-hidden="true" tabindex="-1"></a><span class="co"># 5. Publish dataset (shards uploaded as blobs automatically)</span></span>
973<span id="cb14-37"><a href="#cb14-37" aria-hidden="true" tabindex="-1"></a>dataset <span class="op">=</span> atdata.Dataset[FeatureSample](<span class="st">"features.tar"</span>)</span>
974<span id="cb14-38"><a href="#cb14-38" aria-hidden="true" tabindex="-1"></a>entry <span class="op">=</span> index.insert_dataset(</span>
975<span id="cb14-39"><a href="#cb14-39" aria-hidden="true" tabindex="-1"></a> dataset,</span>
976<span id="cb14-40"><a href="#cb14-40" aria-hidden="true" tabindex="-1"></a> name<span class="op">=</span><span class="st">"synthetic-features-v1"</span>,</span>
977<span id="cb14-41"><a href="#cb14-41" aria-hidden="true" tabindex="-1"></a> schema_ref<span class="op">=</span>schema_uri,</span>
978<span id="cb14-42"><a href="#cb14-42" aria-hidden="true" tabindex="-1"></a> tags<span class="op">=</span>[<span class="st">"features"</span>, <span class="st">"synthetic"</span>],</span>
979<span id="cb14-43"><a href="#cb14-43" aria-hidden="true" tabindex="-1"></a>)</span>
980<span id="cb14-44"><a href="#cb14-44" aria-hidden="true" tabindex="-1"></a></span>
981<span id="cb14-45"><a href="#cb14-45" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Published: </span><span class="sc">{</span>entry<span class="sc">.</span>uri<span class="sc">}</span><span class="ss">"</span>)</span>
982<span id="cb14-46"><a href="#cb14-46" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Data stored at: </span><span class="sc">{</span>entry<span class="sc">.</span>data_urls<span class="sc">}</span><span class="ss">"</span>) <span class="co"># at://did/blob/cid URLs</span></span>
983<span id="cb14-47"><a href="#cb14-47" aria-hidden="true" tabindex="-1"></a></span>
984<span id="cb14-48"><a href="#cb14-48" aria-hidden="true" tabindex="-1"></a><span class="co"># 6. Later: load from blobs</span></span>
985<span id="cb14-49"><a href="#cb14-49" aria-hidden="true" tabindex="-1"></a>source <span class="op">=</span> store.create_source(entry.data_urls)</span>
986<span id="cb14-50"><a href="#cb14-50" aria-hidden="true" tabindex="-1"></a>ds <span class="op">=</span> atdata.Dataset[FeatureSample](source)</span>
987<span id="cb14-51"><a href="#cb14-51" aria-hidden="true" tabindex="-1"></a><span class="cf">for</span> batch <span class="kw">in</span> ds.ordered(batch_size<span class="op">=</span><span class="dv">32</span>):</span>
988<span id="cb14-52"><a href="#cb14-52" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f"Loaded batch with </span><span class="sc">{</span><span class="bu">len</span>(batch.label)<span class="sc">}</span><span class="ss"> samples"</span>)</span>
989<span id="cb14-53"><a href="#cb14-53" aria-hidden="true" tabindex="-1"></a> <span class="cf">break</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
990</div>
991</section>
992<section id="what-youve-learned" class="level2">
993<h2 class="anchored" data-anchor-id="what-youve-learned">What You’ve Learned</h2>
994<p>You now understand federated dataset publishing in atdata:</p>
995<table class="caption-top table">
996<thead>
997<tr class="header">
998<th>Concept</th>
999<th>Purpose</th>
1000</tr>
1001</thead>
1002<tbody>
1003<tr class="odd">
1004<td><code>AtmosphereClient</code></td>
1005<td>ATProto authentication and record management</td>
1006</tr>
1007<tr class="even">
1008<td><code>AtmosphereIndex</code></td>
1009<td>Federated index implementing <code>AbstractIndex</code></td>
1010</tr>
1011<tr class="odd">
1012<td><code>PDSBlobStore</code></td>
1013<td>PDS blob storage implementing <code>AbstractDataStore</code></td>
1014</tr>
1015<tr class="even">
1016<td><code>BlobSource</code></td>
1017<td>Stream datasets from PDS blobs</td>
1018</tr>
1019<tr class="odd">
1020<td>AT URIs</td>
1021<td>Universal identifiers for schemas and datasets</td>
1022</tr>
1023</tbody>
1024</table>
1025<p>The protocol abstractions (<code>AbstractIndex</code>, <code>AbstractDataStore</code>, <code>DataSource</code>) ensure your code works across all three layers of atdata—local files, team storage, and federated sharing.</p>
1026</section>
1027<section id="the-full-picture" class="level2">
1028<h2 class="anchored" data-anchor-id="the-full-picture">The Full Picture</h2>
1029<p>You’ve now seen atdata’s complete architecture:</p>
1030<pre><code>Local Development Team Storage Federation
1031───────────────── ──────────── ──────────
1032tar files Redis + S3 ATProto PDS
1033Dataset[T] LocalIndex AtmosphereIndex
1034 S3DataStore PDSBlobStore</code></pre>
1035<p>The same <code>@packable</code> sample types, the same <code>Dataset[T]</code> iteration patterns, and the same lens transformations work at every layer. Only the storage backend changes.</p>
1036</section>
1037<section id="next-steps" class="level2">
1038<h2 class="anchored" data-anchor-id="next-steps">Next Steps</h2>
1039<div class="callout callout-style-default callout-tip callout-titled">
1040<div class="callout-header d-flex align-content-center">
1041<div class="callout-icon-container">
1042<i class="callout-icon"></i>
1043</div>
1044<div class="callout-title-container flex-fill">
1045Already Have Local Datasets?
1046</div>
1047</div>
1048<div class="callout-body-container callout-body">
1049<p>The <a href="../tutorials/promotion.html">Promotion Workflow</a> tutorial shows how to migrate existing datasets from local storage to the atmosphere without re-processing your data.</p>
1050</div>
1051</div>
1052<ul>
1053<li><strong><a href="../tutorials/promotion.html">Promotion Workflow</a></strong> - Migrate from local storage to atmosphere</li>
1054<li><strong><a href="../reference/atmosphere.html">Atmosphere Reference</a></strong> - Complete API reference</li>
1055<li><strong><a href="../reference/protocols.html">Protocols</a></strong> - Abstract interfaces</li>
1056</ul>
1057
1058
1059</section>
1060
1061</main> <!-- /main -->
1062<script id="quarto-html-after-body" type="application/javascript">
1063 window.document.addEventListener("DOMContentLoaded", function (event) {
1064 // Ensure there is a toggle, if there isn't float one in the top right
1065 if (window.document.querySelector('.quarto-color-scheme-toggle') === null) {
1066 const a = window.document.createElement('a');
1067 a.classList.add('top-right');
1068 a.classList.add('quarto-color-scheme-toggle');
1069 a.href = "";
1070 a.onclick = function() { try { window.quartoToggleColorScheme(); } catch {} return false; };
1071 const i = window.document.createElement("i");
1072 i.classList.add('bi');
1073 a.appendChild(i);
1074 window.document.body.appendChild(a);
1075 }
1076 setColorSchemeToggle(hasAlternateSentinel())
1077 const icon = "";
1078 const anchorJS = new window.AnchorJS();
1079 anchorJS.options = {
1080 placement: 'right',
1081 icon: icon
1082 };
1083 anchorJS.add('.anchored');
1084 const isCodeAnnotation = (el) => {
1085 for (const clz of el.classList) {
1086 if (clz.startsWith('code-annotation-')) {
1087 return true;
1088 }
1089 }
1090 return false;
1091 }
1092 const onCopySuccess = function(e) {
1093 // button target
1094 const button = e.trigger;
1095 // don't keep focus
1096 button.blur();
1097 // flash "checked"
1098 button.classList.add('code-copy-button-checked');
1099 var currentTitle = button.getAttribute("title");
1100 button.setAttribute("title", "Copied!");
1101 let tooltip;
1102 if (window.bootstrap) {
1103 button.setAttribute("data-bs-toggle", "tooltip");
1104 button.setAttribute("data-bs-placement", "left");
1105 button.setAttribute("data-bs-title", "Copied!");
1106 tooltip = new bootstrap.Tooltip(button,
1107 { trigger: "manual",
1108 customClass: "code-copy-button-tooltip",
1109 offset: [0, -8]});
1110 tooltip.show();
1111 }
1112 setTimeout(function() {
1113 if (tooltip) {
1114 tooltip.hide();
1115 button.removeAttribute("data-bs-title");
1116 button.removeAttribute("data-bs-toggle");
1117 button.removeAttribute("data-bs-placement");
1118 }
1119 button.setAttribute("title", currentTitle);
1120 button.classList.remove('code-copy-button-checked');
1121 }, 1000);
1122 // clear code selection
1123 e.clearSelection();
1124 }
1125 const getTextToCopy = function(trigger) {
1126 const codeEl = trigger.previousElementSibling.cloneNode(true);
1127 for (const childEl of codeEl.children) {
1128 if (isCodeAnnotation(childEl)) {
1129 childEl.remove();
1130 }
1131 }
1132 return codeEl.innerText;
1133 }
1134 const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
1135 text: getTextToCopy
1136 });
1137 clipboard.on('success', onCopySuccess);
1138 if (window.document.getElementById('quarto-embedded-source-code-modal')) {
1139 const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', {
1140 text: getTextToCopy,
1141 container: window.document.getElementById('quarto-embedded-source-code-modal')
1142 });
1143 clipboardModal.on('success', onCopySuccess);
1144 }
1145 var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//);
1146 var mailtoRegex = new RegExp(/^mailto:/);
1147 var filterRegex = new RegExp("https:\/\/github\.com\/your-org\/atdata");
1148 var isInternal = (href) => {
1149 return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href);
1150 }
1151 // Inspect non-navigation links and adorn them if external
1152 var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)');
1153 for (var i=0; i<links.length; i++) {
1154 const link = links[i];
1155 if (!isInternal(link.href)) {
1156 // undo the damage that might have been done by quarto-nav.js in the case of
1157 // links that we want to consider external
1158 if (link.dataset.originalHref !== undefined) {
1159 link.href = link.dataset.originalHref;
1160 }
1161 }
1162 }
1163 function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
1164 const config = {
1165 allowHTML: true,
1166 maxWidth: 500,
1167 delay: 100,
1168 arrow: false,
1169 appendTo: function(el) {
1170 return el.parentElement;
1171 },
1172 interactive: true,
1173 interactiveBorder: 10,
1174 theme: 'quarto',
1175 placement: 'bottom-start',
1176 };
1177 if (contentFn) {
1178 config.content = contentFn;
1179 }
1180 if (onTriggerFn) {
1181 config.onTrigger = onTriggerFn;
1182 }
1183 if (onUntriggerFn) {
1184 config.onUntrigger = onUntriggerFn;
1185 }
1186 window.tippy(el, config);
1187 }
1188 const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
1189 for (var i=0; i<noterefs.length; i++) {
1190 const ref = noterefs[i];
1191 tippyHover(ref, function() {
1192 // use id or data attribute instead here
1193 let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
1194 try { href = new URL(href).hash; } catch {}
1195 const id = href.replace(/^#\/?/, "");
1196 const note = window.document.getElementById(id);
1197 if (note) {
1198 return note.innerHTML;
1199 } else {
1200 return "";
1201 }
1202 });
1203 }
1204 const xrefs = window.document.querySelectorAll('a.quarto-xref');
1205 const processXRef = (id, note) => {
1206 // Strip column container classes
1207 const stripColumnClz = (el) => {
1208 el.classList.remove("page-full", "page-columns");
1209 if (el.children) {
1210 for (const child of el.children) {
1211 stripColumnClz(child);
1212 }
1213 }
1214 }
1215 stripColumnClz(note)
1216 if (id === null || id.startsWith('sec-')) {
1217 // Special case sections, only their first couple elements
1218 const container = document.createElement("div");
1219 if (note.children && note.children.length > 2) {
1220 container.appendChild(note.children[0].cloneNode(true));
1221 for (let i = 1; i < note.children.length; i++) {
1222 const child = note.children[i];
1223 if (child.tagName === "P" && child.innerText === "") {
1224 continue;
1225 } else {
1226 container.appendChild(child.cloneNode(true));
1227 break;
1228 }
1229 }
1230 if (window.Quarto?.typesetMath) {
1231 window.Quarto.typesetMath(container);
1232 }
1233 return container.innerHTML
1234 } else {
1235 if (window.Quarto?.typesetMath) {
1236 window.Quarto.typesetMath(note);
1237 }
1238 return note.innerHTML;
1239 }
1240 } else {
1241 // Remove any anchor links if they are present
1242 const anchorLink = note.querySelector('a.anchorjs-link');
1243 if (anchorLink) {
1244 anchorLink.remove();
1245 }
1246 if (window.Quarto?.typesetMath) {
1247 window.Quarto.typesetMath(note);
1248 }
1249 if (note.classList.contains("callout")) {
1250 return note.outerHTML;
1251 } else {
1252 return note.innerHTML;
1253 }
1254 }
1255 }
1256 for (var i=0; i<xrefs.length; i++) {
1257 const xref = xrefs[i];
1258 tippyHover(xref, undefined, function(instance) {
1259 instance.disable();
1260 let url = xref.getAttribute('href');
1261 let hash = undefined;
1262 if (url.startsWith('#')) {
1263 hash = url;
1264 } else {
1265 try { hash = new URL(url).hash; } catch {}
1266 }
1267 if (hash) {
1268 const id = hash.replace(/^#\/?/, "");
1269 const note = window.document.getElementById(id);
1270 if (note !== null) {
1271 try {
1272 const html = processXRef(id, note.cloneNode(true));
1273 instance.setContent(html);
1274 } finally {
1275 instance.enable();
1276 instance.show();
1277 }
1278 } else {
1279 // See if we can fetch this
1280 fetch(url.split('#')[0])
1281 .then(res => res.text())
1282 .then(html => {
1283 const parser = new DOMParser();
1284 const htmlDoc = parser.parseFromString(html, "text/html");
1285 const note = htmlDoc.getElementById(id);
1286 if (note !== null) {
1287 const html = processXRef(id, note);
1288 instance.setContent(html);
1289 }
1290 }).finally(() => {
1291 instance.enable();
1292 instance.show();
1293 });
1294 }
1295 } else {
1296 // See if we can fetch a full url (with no hash to target)
1297 // This is a special case and we should probably do some content thinning / targeting
1298 fetch(url)
1299 .then(res => res.text())
1300 .then(html => {
1301 const parser = new DOMParser();
1302 const htmlDoc = parser.parseFromString(html, "text/html");
1303 const note = htmlDoc.querySelector('main.content');
1304 if (note !== null) {
1305 // This should only happen for chapter cross references
1306 // (since there is no id in the URL)
1307 // remove the first header
1308 if (note.children.length > 0 && note.children[0].tagName === "HEADER") {
1309 note.children[0].remove();
1310 }
1311 const html = processXRef(null, note);
1312 instance.setContent(html);
1313 }
1314 }).finally(() => {
1315 instance.enable();
1316 instance.show();
1317 });
1318 }
1319 }, function(instance) {
1320 });
1321 }
1322 let selectedAnnoteEl;
1323 const selectorForAnnotation = ( cell, annotation) => {
1324 let cellAttr = 'data-code-cell="' + cell + '"';
1325 let lineAttr = 'data-code-annotation="' + annotation + '"';
1326 const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
1327 return selector;
1328 }
1329 const selectCodeLines = (annoteEl) => {
1330 const doc = window.document;
1331 const targetCell = annoteEl.getAttribute("data-target-cell");
1332 const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
1333 const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
1334 const lines = annoteSpan.getAttribute("data-code-lines").split(",");
1335 const lineIds = lines.map((line) => {
1336 return targetCell + "-" + line;
1337 })
1338 let top = null;
1339 let height = null;
1340 let parent = null;
1341 if (lineIds.length > 0) {
1342 //compute the position of the single el (top and bottom and make a div)
1343 const el = window.document.getElementById(lineIds[0]);
1344 top = el.offsetTop;
1345 height = el.offsetHeight;
1346 parent = el.parentElement.parentElement;
1347 if (lineIds.length > 1) {
1348 const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
1349 const bottom = lastEl.offsetTop + lastEl.offsetHeight;
1350 height = bottom - top;
1351 }
1352 if (top !== null && height !== null && parent !== null) {
1353 // cook up a div (if necessary) and position it
1354 let div = window.document.getElementById("code-annotation-line-highlight");
1355 if (div === null) {
1356 div = window.document.createElement("div");
1357 div.setAttribute("id", "code-annotation-line-highlight");
1358 div.style.position = 'absolute';
1359 parent.appendChild(div);
1360 }
1361 div.style.top = top - 2 + "px";
1362 div.style.height = height + 4 + "px";
1363 div.style.left = 0;
1364 let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
1365 if (gutterDiv === null) {
1366 gutterDiv = window.document.createElement("div");
1367 gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
1368 gutterDiv.style.position = 'absolute';
1369 const codeCell = window.document.getElementById(targetCell);
1370 const gutter = codeCell.querySelector('.code-annotation-gutter');
1371 gutter.appendChild(gutterDiv);
1372 }
1373 gutterDiv.style.top = top - 2 + "px";
1374 gutterDiv.style.height = height + 4 + "px";
1375 }
1376 selectedAnnoteEl = annoteEl;
1377 }
1378 };
1379 const unselectCodeLines = () => {
1380 const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
1381 elementsIds.forEach((elId) => {
1382 const div = window.document.getElementById(elId);
1383 if (div) {
1384 div.remove();
1385 }
1386 });
1387 selectedAnnoteEl = undefined;
1388 };
1389 // Handle positioning of the toggle
1390 window.addEventListener(
1391 "resize",
1392 throttle(() => {
1393 elRect = undefined;
1394 if (selectedAnnoteEl) {
1395 selectCodeLines(selectedAnnoteEl);
1396 }
1397 }, 10)
1398 );
1399 function throttle(fn, ms) {
1400 let throttle = false;
1401 let timer;
1402 return (...args) => {
1403 if(!throttle) { // first call gets through
1404 fn.apply(this, args);
1405 throttle = true;
1406 } else { // all the others get throttled
1407 if(timer) clearTimeout(timer); // cancel #2
1408 timer = setTimeout(() => {
1409 fn.apply(this, args);
1410 timer = throttle = false;
1411 }, ms);
1412 }
1413 };
1414 }
1415 // Attach click handler to the DT
1416 const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
1417 for (const annoteDlNode of annoteDls) {
1418 annoteDlNode.addEventListener('click', (event) => {
1419 const clickedEl = event.target;
1420 if (clickedEl !== selectedAnnoteEl) {
1421 unselectCodeLines();
1422 const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
1423 if (activeEl) {
1424 activeEl.classList.remove('code-annotation-active');
1425 }
1426 selectCodeLines(clickedEl);
1427 clickedEl.classList.add('code-annotation-active');
1428 } else {
1429 // Unselect the line
1430 unselectCodeLines();
1431 clickedEl.classList.remove('code-annotation-active');
1432 }
1433 });
1434 }
1435 const findCites = (el) => {
1436 const parentEl = el.parentElement;
1437 if (parentEl) {
1438 const cites = parentEl.dataset.cites;
1439 if (cites) {
1440 return {
1441 el,
1442 cites: cites.split(' ')
1443 };
1444 } else {
1445 return findCites(el.parentElement)
1446 }
1447 } else {
1448 return undefined;
1449 }
1450 };
1451 var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
1452 for (var i=0; i<bibliorefs.length; i++) {
1453 const ref = bibliorefs[i];
1454 const citeInfo = findCites(ref);
1455 if (citeInfo) {
1456 tippyHover(citeInfo.el, function() {
1457 var popup = window.document.createElement('div');
1458 citeInfo.cites.forEach(function(cite) {
1459 var citeDiv = window.document.createElement('div');
1460 citeDiv.classList.add('hanging-indent');
1461 citeDiv.classList.add('csl-entry');
1462 var biblioDiv = window.document.getElementById('ref-' + cite);
1463 if (biblioDiv) {
1464 citeDiv.innerHTML = biblioDiv.innerHTML;
1465 }
1466 popup.appendChild(citeDiv);
1467 });
1468 return popup.innerHTML;
1469 });
1470 }
1471 }
1472 });
1473 </script>
1474</div> <!-- /content -->
1475<footer class="footer">
1476 <div class="nav-footer">
1477 <div class="nav-footer-left">
1478<p>Built with <a href="https://quarto.org/">Quarto</a></p>
1479</div>
1480 <div class="nav-footer-center">
1481
1482 <div class="toc-actions d-sm-block d-md-none"><ul><li><a href="https://github.com/your-org/atdata/edit/main/tutorials/atmosphere.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></div>
1483 <div class="nav-footer-right">
1484<p>MIT License</p>
1485</div>
1486 </div>
1487</footer>
1488
1489
1490
1491
1492</body></html>