A loose federation of distributed, typed datasets
1<!DOCTYPE html>
2<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
3
4<meta charset="utf-8">
5<meta name="generator" content="quarto-1.7.34">
6
7<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
8
9<meta name="description" content="Get up and running with atdata in 5 minutes">
10
11<title>Quick Start – atdata</title>
12<style>
13code{white-space: pre-wrap;}
14span.smallcaps{font-variant: small-caps;}
15div.columns{display: flex; gap: min(4vw, 1.5em);}
16div.column{flex: auto; overflow-x: auto;}
17div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
18ul.task-list{list-style: none;}
19ul.task-list li input[type="checkbox"] {
20 width: 0.8em;
21 margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
22 vertical-align: middle;
23}
24/* CSS for syntax highlighting */
25html { -webkit-text-size-adjust: 100%; }
26pre > code.sourceCode { white-space: pre; position: relative; }
27pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
28pre > code.sourceCode > span:empty { height: 1.2em; }
29.sourceCode { overflow: visible; }
30code.sourceCode > span { color: inherit; text-decoration: inherit; }
31div.sourceCode { margin: 1em 0; }
32pre.sourceCode { margin: 0; }
33@media screen {
34div.sourceCode { overflow: auto; }
35}
36@media print {
37pre > code.sourceCode { white-space: pre-wrap; }
38pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
39}
40pre.numberSource code
41 { counter-reset: source-line 0; }
42pre.numberSource code > span
43 { position: relative; left: -4em; counter-increment: source-line; }
44pre.numberSource code > span > a:first-child::before
45 { content: counter(source-line);
46 position: relative; left: -1em; text-align: right; vertical-align: baseline;
47 border: none; display: inline-block;
48 -webkit-touch-callout: none; -webkit-user-select: none;
49 -khtml-user-select: none; -moz-user-select: none;
50 -ms-user-select: none; user-select: none;
51 padding: 0 4px; width: 4em;
52 }
53pre.numberSource { margin-left: 3em; padding-left: 4px; }
54div.sourceCode
55 { }
56@media screen {
57pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
58}
59</style>
60
61
62<script src="../site_libs/quarto-nav/quarto-nav.js"></script>
63<script src="../site_libs/quarto-nav/headroom.min.js"></script>
64<script src="../site_libs/clipboard/clipboard.min.js"></script>
65<script src="../site_libs/quarto-search/autocomplete.umd.js"></script>
66<script src="../site_libs/quarto-search/fuse.min.js"></script>
67<script src="../site_libs/quarto-search/quarto-search.js"></script>
68<meta name="quarto:offset" content="../">
69<script src="../site_libs/quarto-html/quarto.js" type="module"></script>
70<script src="../site_libs/quarto-html/tabsets/tabsets.js" type="module"></script>
71<script src="../site_libs/quarto-html/popper.min.js"></script>
72<script src="../site_libs/quarto-html/tippy.umd.min.js"></script>
73<script src="../site_libs/quarto-html/anchor.min.js"></script>
74<link href="../site_libs/quarto-html/tippy.css" rel="stylesheet">
75<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme" id="quarto-text-highlighting-styles">
76<link href="../site_libs/quarto-html/quarto-syntax-highlighting-dark-8dcd8563ea6803ab7cbb3d71ca5772e1.css" rel="stylesheet" class="quarto-color-scheme quarto-color-alternate" id="quarto-text-highlighting-styles">
77<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme-extra" id="quarto-text-highlighting-styles">
78<script src="../site_libs/bootstrap/bootstrap.min.js"></script>
79<link href="../site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
80<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme" id="quarto-bootstrap" data-mode="light">
81<link href="../site_libs/bootstrap/bootstrap-dark-7964ffd8887b0991fe8d71c6c8bc75d6.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme quarto-color-alternate" id="quarto-bootstrap" data-mode="dark">
82<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme-extra" id="quarto-bootstrap" data-mode="light">
83<script id="quarto-search-options" type="application/json">{
84 "location": "navbar",
85 "copy-button": false,
86 "collapse-after": 3,
87 "panel-placement": "end",
88 "type": "overlay",
89 "limit": 50,
90 "keyboard-shortcut": [
91 "f",
92 "/",
93 "s"
94 ],
95 "show-item-context": false,
96 "language": {
97 "search-no-results-text": "No results",
98 "search-matching-documents-text": "matching documents",
99 "search-copy-link-title": "Copy link to search",
100 "search-hide-matches-text": "Hide additional matches",
101 "search-more-match-text": "more match in this document",
102 "search-more-matches-text": "more matches in this document",
103 "search-clear-button-title": "Clear",
104 "search-text-placeholder": "",
105 "search-detached-cancel-button-title": "Cancel",
106 "search-submit-button-title": "Submit",
107 "search-label": "Search"
108 }
109}</script>
110
111
112<link rel="stylesheet" href="../assets/styles.css">
113</head>
114
115<body class="nav-sidebar docked nav-fixed quarto-light"><script id="quarto-html-before-body" type="application/javascript">
116 const toggleBodyColorMode = (bsSheetEl) => {
117 const mode = bsSheetEl.getAttribute("data-mode");
118 const bodyEl = window.document.querySelector("body");
119 if (mode === "dark") {
120 bodyEl.classList.add("quarto-dark");
121 bodyEl.classList.remove("quarto-light");
122 } else {
123 bodyEl.classList.add("quarto-light");
124 bodyEl.classList.remove("quarto-dark");
125 }
126 }
127 const toggleBodyColorPrimary = () => {
128 const bsSheetEl = window.document.querySelector("link#quarto-bootstrap:not([rel=disabled-stylesheet])");
129 if (bsSheetEl) {
130 toggleBodyColorMode(bsSheetEl);
131 }
132 }
133 const setColorSchemeToggle = (alternate) => {
134 const toggles = window.document.querySelectorAll('.quarto-color-scheme-toggle');
135 for (let i=0; i < toggles.length; i++) {
136 const toggle = toggles[i];
137 if (toggle) {
138 if (alternate) {
139 toggle.classList.add("alternate");
140 } else {
141 toggle.classList.remove("alternate");
142 }
143 }
144 }
145 };
146 const toggleColorMode = (alternate) => {
147 // Switch the stylesheets
148 const primaryStylesheets = window.document.querySelectorAll('link.quarto-color-scheme:not(.quarto-color-alternate)');
149 const alternateStylesheets = window.document.querySelectorAll('link.quarto-color-scheme.quarto-color-alternate');
150 manageTransitions('#quarto-margin-sidebar .nav-link', false);
151 if (alternate) {
152 // note: dark is layered on light, we don't disable primary!
153 enableStylesheet(alternateStylesheets);
154 for (const sheetNode of alternateStylesheets) {
155 if (sheetNode.id === "quarto-bootstrap") {
156 toggleBodyColorMode(sheetNode);
157 }
158 }
159 } else {
160 disableStylesheet(alternateStylesheets);
161 enableStylesheet(primaryStylesheets)
162 toggleBodyColorPrimary();
163 }
164 manageTransitions('#quarto-margin-sidebar .nav-link', true);
165 // Switch the toggles
166 setColorSchemeToggle(alternate)
167 // Hack to workaround the fact that safari doesn't
168 // properly recolor the scrollbar when toggling (#1455)
169 if (navigator.userAgent.indexOf('Safari') > 0 && navigator.userAgent.indexOf('Chrome') == -1) {
170 manageTransitions("body", false);
171 window.scrollTo(0, 1);
172 setTimeout(() => {
173 window.scrollTo(0, 0);
174 manageTransitions("body", true);
175 }, 40);
176 }
177 }
178 const disableStylesheet = (stylesheets) => {
179 for (let i=0; i < stylesheets.length; i++) {
180 const stylesheet = stylesheets[i];
181 stylesheet.rel = 'disabled-stylesheet';
182 }
183 }
184 const enableStylesheet = (stylesheets) => {
185 for (let i=0; i < stylesheets.length; i++) {
186 const stylesheet = stylesheets[i];
187 if(stylesheet.rel !== 'stylesheet') { // for Chrome, which will still FOUC without this check
188 stylesheet.rel = 'stylesheet';
189 }
190 }
191 }
192 const manageTransitions = (selector, allowTransitions) => {
193 const els = window.document.querySelectorAll(selector);
194 for (let i=0; i < els.length; i++) {
195 const el = els[i];
196 if (allowTransitions) {
197 el.classList.remove('notransition');
198 } else {
199 el.classList.add('notransition');
200 }
201 }
202 }
203 const isFileUrl = () => {
204 return window.location.protocol === 'file:';
205 }
206 const hasAlternateSentinel = () => {
207 let styleSentinel = getColorSchemeSentinel();
208 if (styleSentinel !== null) {
209 return styleSentinel === "alternate";
210 } else {
211 return false;
212 }
213 }
214 const setStyleSentinel = (alternate) => {
215 const value = alternate ? "alternate" : "default";
216 if (!isFileUrl()) {
217 window.localStorage.setItem("quarto-color-scheme", value);
218 } else {
219 localAlternateSentinel = value;
220 }
221 }
222 const getColorSchemeSentinel = () => {
223 if (!isFileUrl()) {
224 const storageValue = window.localStorage.getItem("quarto-color-scheme");
225 return storageValue != null ? storageValue : localAlternateSentinel;
226 } else {
227 return localAlternateSentinel;
228 }
229 }
230 const toggleGiscusIfUsed = (isAlternate, darkModeDefault) => {
231 const baseTheme = document.querySelector('#giscus-base-theme')?.value ?? 'light';
232 const alternateTheme = document.querySelector('#giscus-alt-theme')?.value ?? 'dark';
233 let newTheme = '';
234 if(authorPrefersDark) {
235 newTheme = isAlternate ? baseTheme : alternateTheme;
236 } else {
237 newTheme = isAlternate ? alternateTheme : baseTheme;
238 }
239 const changeGiscusTheme = () => {
240 // From: https://github.com/giscus/giscus/issues/336
241 const sendMessage = (message) => {
242 const iframe = document.querySelector('iframe.giscus-frame');
243 if (!iframe) return;
244 iframe.contentWindow.postMessage({ giscus: message }, 'https://giscus.app');
245 }
246 sendMessage({
247 setConfig: {
248 theme: newTheme
249 }
250 });
251 }
252 const isGiscussLoaded = window.document.querySelector('iframe.giscus-frame') !== null;
253 if (isGiscussLoaded) {
254 changeGiscusTheme();
255 }
256 };
257 const authorPrefersDark = false;
258 const darkModeDefault = authorPrefersDark;
259 document.querySelector('link#quarto-text-highlighting-styles.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
260 document.querySelector('link#quarto-bootstrap.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
261 let localAlternateSentinel = darkModeDefault ? 'alternate' : 'default';
262 // Dark / light mode switch
263 window.quartoToggleColorScheme = () => {
264 // Read the current dark / light value
265 let toAlternate = !hasAlternateSentinel();
266 toggleColorMode(toAlternate);
267 setStyleSentinel(toAlternate);
268 toggleGiscusIfUsed(toAlternate, darkModeDefault);
269 window.dispatchEvent(new Event('resize'));
270 };
271 // Switch to dark mode if need be
272 if (hasAlternateSentinel()) {
273 toggleColorMode(true);
274 } else {
275 toggleColorMode(false);
276 }
277 </script>
278
279<div id="quarto-search-results"></div>
280 <header id="quarto-header" class="headroom fixed-top">
281 <nav class="navbar navbar-expand-lg " data-bs-theme="dark">
282 <div class="navbar-container container-fluid">
283 <div class="navbar-brand-container mx-auto">
284 <a class="navbar-brand" href="../index.html">
285 <span class="navbar-title">atdata</span>
286 </a>
287 </div>
288 <div id="quarto-search" class="" title="Search"></div>
289 <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarCollapse" aria-controls="navbarCollapse" role="menu" aria-expanded="false" aria-label="Toggle navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
290 <span class="navbar-toggler-icon"></span>
291</button>
292 <div class="collapse navbar-collapse" id="navbarCollapse">
293 <ul class="navbar-nav navbar-nav-scroll me-auto">
294 <li class="nav-item">
295 <a class="nav-link active" href="../index.html" aria-current="page">
296<span class="menu-text">Guide</span></a>
297 </li>
298 <li class="nav-item dropdown ">
299 <a class="nav-link dropdown-toggle" href="#" id="nav-menu-tutorials" role="link" data-bs-toggle="dropdown" aria-expanded="false">
300 <span class="menu-text">Tutorials</span>
301 </a>
302 <ul class="dropdown-menu" aria-labelledby="nav-menu-tutorials">
303 <li>
304 <a class="dropdown-item" href="../tutorials/quickstart.html">
305 <span class="dropdown-text">Quick Start</span></a>
306 </li>
307 <li>
308 <a class="dropdown-item" href="../tutorials/local-workflow.html">
309 <span class="dropdown-text">Local Workflow</span></a>
310 </li>
311 <li>
312 <a class="dropdown-item" href="../tutorials/atmosphere.html">
313 <span class="dropdown-text">Atmosphere Publishing</span></a>
314 </li>
315 <li>
316 <a class="dropdown-item" href="../tutorials/promotion.html">
317 <span class="dropdown-text">Promotion Workflow</span></a>
318 </li>
319 </ul>
320 </li>
321 <li class="nav-item dropdown ">
322 <a class="nav-link dropdown-toggle" href="#" id="nav-menu-reference" role="link" data-bs-toggle="dropdown" aria-expanded="false">
323 <span class="menu-text">Reference</span>
324 </a>
325 <ul class="dropdown-menu" aria-labelledby="nav-menu-reference">
326 <li>
327 <a class="dropdown-item" href="../reference/architecture.html">
328 <span class="dropdown-text">Architecture Overview</span></a>
329 </li>
330 <li>
331 <a class="dropdown-item" href="../reference/packable-samples.html">
332 <span class="dropdown-text">Packable Samples</span></a>
333 </li>
334 <li>
335 <a class="dropdown-item" href="../reference/datasets.html">
336 <span class="dropdown-text">Datasets</span></a>
337 </li>
338 <li>
339 <a class="dropdown-item" href="../reference/lenses.html">
340 <span class="dropdown-text">Lenses</span></a>
341 </li>
342 <li>
343 <a class="dropdown-item" href="../reference/local-storage.html">
344 <span class="dropdown-text">Local Storage</span></a>
345 </li>
346 <li>
347 <a class="dropdown-item" href="../reference/atmosphere.html">
348 <span class="dropdown-text">Atmosphere</span></a>
349 </li>
350 <li>
351 <a class="dropdown-item" href="../reference/promotion.html">
352 <span class="dropdown-text">Promotion</span></a>
353 </li>
354 <li>
355 <a class="dropdown-item" href="../reference/load-dataset.html">
356 <span class="dropdown-text">load_dataset API</span></a>
357 </li>
358 <li>
359 <a class="dropdown-item" href="../reference/protocols.html">
360 <span class="dropdown-text">Protocols</span></a>
361 </li>
362 <li>
363 <a class="dropdown-item" href="../reference/uri-spec.html">
364 <span class="dropdown-text">URI Specification</span></a>
365 </li>
366 <li>
367 <a class="dropdown-item" href="../reference/troubleshooting.html">
368 <span class="dropdown-text">Troubleshooting & FAQ</span></a>
369 </li>
370 <li>
371 <a class="dropdown-item" href="../reference/deployment.html">
372 <span class="dropdown-text">Deployment Guide</span></a>
373 </li>
374 </ul>
375 </li>
376 <li class="nav-item">
377 <a class="nav-link" href="../api/index.html">
378<span class="menu-text">API</span></a>
379 </li>
380</ul>
381 <ul class="navbar-nav navbar-nav-scroll ms-auto">
382 <li class="nav-item compact">
383 <a class="nav-link" href="https://github.com/your-org/atdata"> <i class="bi bi-github" role="img">
384</i>
385<span class="menu-text"></span></a>
386 </li>
387</ul>
388 </div> <!-- /navcollapse -->
389 <div class="quarto-navbar-tools">
390 <a href="" class="quarto-color-scheme-toggle quarto-navigation-tool px-1" onclick="window.quartoToggleColorScheme(); return false;" title="Toggle dark mode"><i class="bi"></i></a>
391</div>
392 </div> <!-- /container-fluid -->
393 </nav>
394 <nav class="quarto-secondary-nav">
395 <div class="container-fluid d-flex">
396 <button type="button" class="quarto-btn-toggle btn" data-bs-toggle="collapse" role="button" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
397 <i class="bi bi-layout-text-sidebar-reverse"></i>
398 </button>
399 <nav class="quarto-page-breadcrumbs" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item"><a href="../tutorials/quickstart.html">Getting Started</a></li><li class="breadcrumb-item"><a href="../tutorials/quickstart.html">Quick Start</a></li></ol></nav>
400 <a class="flex-grow-1" role="navigation" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
401 </a>
402 </div>
403 </nav>
404</header>
405<!-- content -->
406<div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-article page-navbar">
407<!-- sidebar -->
408 <nav id="quarto-sidebar" class="sidebar collapse collapse-horizontal quarto-sidebar-collapse-item sidebar-navigation docked overflow-auto">
409 <div class="sidebar-menu-container">
410 <ul class="list-unstyled mt-1">
411 <li class="sidebar-item">
412 <div class="sidebar-item-container">
413 <a href="../index.html" class="sidebar-item-text sidebar-link">
414 <span class="menu-text">atdata</span></a>
415 </div>
416</li>
417 <li class="sidebar-item sidebar-item-section">
418 <div class="sidebar-item-container">
419 <a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" role="navigation" aria-expanded="true">
420 <span class="menu-text">Getting Started</span></a>
421 <a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" role="navigation" aria-expanded="true" aria-label="Toggle section">
422 <i class="bi bi-chevron-right ms-2"></i>
423 </a>
424 </div>
425 <ul id="quarto-sidebar-section-1" class="collapse list-unstyled sidebar-section depth1 show">
426 <li class="sidebar-item">
427 <div class="sidebar-item-container">
428 <a href="../tutorials/quickstart.html" class="sidebar-item-text sidebar-link active">
429 <span class="menu-text">Quick Start</span></a>
430 </div>
431</li>
432 <li class="sidebar-item">
433 <div class="sidebar-item-container">
434 <a href="../tutorials/local-workflow.html" class="sidebar-item-text sidebar-link">
435 <span class="menu-text">Local Workflow</span></a>
436 </div>
437</li>
438 <li class="sidebar-item">
439 <div class="sidebar-item-container">
440 <a href="../tutorials/atmosphere.html" class="sidebar-item-text sidebar-link">
441 <span class="menu-text">Atmosphere Publishing</span></a>
442 </div>
443</li>
444 <li class="sidebar-item">
445 <div class="sidebar-item-container">
446 <a href="../tutorials/promotion.html" class="sidebar-item-text sidebar-link">
447 <span class="menu-text">Promotion Workflow</span></a>
448 </div>
449</li>
450 </ul>
451 </li>
452 <li class="sidebar-item sidebar-item-section">
453 <div class="sidebar-item-container">
454 <a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" role="navigation" aria-expanded="true">
455 <span class="menu-text">Reference</span></a>
456 <a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" role="navigation" aria-expanded="true" aria-label="Toggle section">
457 <i class="bi bi-chevron-right ms-2"></i>
458 </a>
459 </div>
460 <ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">
461 <li class="sidebar-item">
462 <div class="sidebar-item-container">
463 <a href="../reference/architecture.html" class="sidebar-item-text sidebar-link">
464 <span class="menu-text">Architecture Overview</span></a>
465 </div>
466</li>
467 <li class="sidebar-item">
468 <div class="sidebar-item-container">
469 <a href="../reference/packable-samples.html" class="sidebar-item-text sidebar-link">
470 <span class="menu-text">Packable Samples</span></a>
471 </div>
472</li>
473 <li class="sidebar-item">
474 <div class="sidebar-item-container">
475 <a href="../reference/datasets.html" class="sidebar-item-text sidebar-link">
476 <span class="menu-text">Datasets</span></a>
477 </div>
478</li>
479 <li class="sidebar-item">
480 <div class="sidebar-item-container">
481 <a href="../reference/lenses.html" class="sidebar-item-text sidebar-link">
482 <span class="menu-text">Lenses</span></a>
483 </div>
484</li>
485 <li class="sidebar-item">
486 <div class="sidebar-item-container">
487 <a href="../reference/local-storage.html" class="sidebar-item-text sidebar-link">
488 <span class="menu-text">Local Storage</span></a>
489 </div>
490</li>
491 <li class="sidebar-item">
492 <div class="sidebar-item-container">
493 <a href="../reference/atmosphere.html" class="sidebar-item-text sidebar-link">
494 <span class="menu-text">Atmosphere (ATProto Integration)</span></a>
495 </div>
496</li>
497 <li class="sidebar-item">
498 <div class="sidebar-item-container">
499 <a href="../reference/promotion.html" class="sidebar-item-text sidebar-link">
500 <span class="menu-text">Promotion Workflow</span></a>
501 </div>
502</li>
503 <li class="sidebar-item">
504 <div class="sidebar-item-container">
505 <a href="../reference/load-dataset.html" class="sidebar-item-text sidebar-link">
506 <span class="menu-text">load_dataset API</span></a>
507 </div>
508</li>
509 <li class="sidebar-item">
510 <div class="sidebar-item-container">
511 <a href="../reference/protocols.html" class="sidebar-item-text sidebar-link">
512 <span class="menu-text">Protocols</span></a>
513 </div>
514</li>
515 <li class="sidebar-item">
516 <div class="sidebar-item-container">
517 <a href="../reference/uri-spec.html" class="sidebar-item-text sidebar-link">
518 <span class="menu-text">URI Specification</span></a>
519 </div>
520</li>
521 <li class="sidebar-item">
522 <div class="sidebar-item-container">
523 <a href="../reference/troubleshooting.html" class="sidebar-item-text sidebar-link">
524 <span class="menu-text">Troubleshooting & FAQ</span></a>
525 </div>
526</li>
527 <li class="sidebar-item">
528 <div class="sidebar-item-container">
529 <a href="../reference/deployment.html" class="sidebar-item-text sidebar-link">
530 <span class="menu-text">Deployment Guide</span></a>
531 </div>
532</li>
533 </ul>
534 </li>
535 </ul>
536 </div>
537</nav>
538<div id="quarto-sidebar-glass" class="quarto-sidebar-collapse-item" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item"></div>
539<!-- margin-sidebar -->
540 <div id="quarto-margin-sidebar" class="sidebar margin-sidebar">
541 <nav id="TOC" role="doc-toc" class="toc-active">
542 <h2 id="toc-title">On this page</h2>
543
544 <ul>
545 <li><a href="#where-this-fits" id="toc-where-this-fits" class="nav-link active" data-scroll-target="#where-this-fits">Where This Fits</a></li>
546 <li><a href="#installation" id="toc-installation" class="nav-link" data-scroll-target="#installation">Installation</a></li>
547 <li><a href="#define-a-sample-type" id="toc-define-a-sample-type" class="nav-link" data-scroll-target="#define-a-sample-type">Define a Sample Type</a></li>
548 <li><a href="#create-sample-instances" id="toc-create-sample-instances" class="nav-link" data-scroll-target="#create-sample-instances">Create Sample Instances</a></li>
549 <li><a href="#write-a-dataset" id="toc-write-a-dataset" class="nav-link" data-scroll-target="#write-a-dataset">Write a Dataset</a></li>
550 <li><a href="#load-and-iterate" id="toc-load-and-iterate" class="nav-link" data-scroll-target="#load-and-iterate">Load and Iterate</a></li>
551 <li><a href="#shuffled-iteration" id="toc-shuffled-iteration" class="nav-link" data-scroll-target="#shuffled-iteration">Shuffled Iteration</a></li>
552 <li><a href="#use-lenses-for-type-transformations" id="toc-use-lenses-for-type-transformations" class="nav-link" data-scroll-target="#use-lenses-for-type-transformations">Use Lenses for Type Transformations</a></li>
553 <li><a href="#what-youve-learned" id="toc-what-youve-learned" class="nav-link" data-scroll-target="#what-youve-learned">What You’ve Learned</a></li>
554 <li><a href="#next-steps" id="toc-next-steps" class="nav-link" data-scroll-target="#next-steps">Next Steps</a></li>
555 </ul>
556<div class="toc-actions"><ul><li><a href="https://github.com/your-org/atdata/edit/main/tutorials/quickstart.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></nav>
557 </div>
558<!-- main -->
559<main class="content" id="quarto-document-content">
560
561
562<header id="title-block-header" class="quarto-title-block default"><nav class="quarto-page-breadcrumbs quarto-title-breadcrumbs d-none d-lg-block" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item"><a href="../tutorials/quickstart.html">Getting Started</a></li><li class="breadcrumb-item"><a href="../tutorials/quickstart.html">Quick Start</a></li></ol></nav>
563<div class="quarto-title">
564<h1 class="title">Quick Start</h1>
565</div>
566
567<div>
568 <div class="description">
569 Get up and running with atdata in 5 minutes
570 </div>
571</div>
572
573
574<div class="quarto-title-meta">
575
576
577
578
579 </div>
580
581
582
583</header>
584
585
586<p>This guide walks you through the basics of atdata: defining sample types, writing datasets, and iterating over them. You’ll learn the foundational patterns that enable type-safe, efficient dataset handling—the first layer of atdata’s three-layer architecture.</p>
587<section id="where-this-fits" class="level2">
588<h2 class="anchored" data-anchor-id="where-this-fits">Where This Fits</h2>
589<p>atdata is built around a simple progression:</p>
590<pre><code>Local Development → Team Storage → Federation</code></pre>
591<p>This tutorial covers <strong>local development</strong>—the foundation. Everything you learn here (typed samples, efficient iteration, lens transformations) carries forward as you scale to team storage and federated sharing. The key insight is that your sample types remain the same across all three layers; only the storage backend changes.</p>
592</section>
593<section id="installation" class="level2">
594<h2 class="anchored" data-anchor-id="installation">Installation</h2>
595<div class="sourceCode" id="cb2"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="ex">pip</span> install atdata</span>
596<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a></span>
597<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a><span class="co"># With ATProto support</span></span>
598<span id="cb2-4"><a href="#cb2-4" aria-hidden="true" tabindex="-1"></a><span class="ex">pip</span> install atdata<span class="pp">[</span><span class="ss">atmosphere</span><span class="pp">]</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
599</section>
600<section id="define-a-sample-type" class="level2">
601<h2 class="anchored" data-anchor-id="define-a-sample-type">Define a Sample Type</h2>
602<p>The core abstraction in atdata is the <strong>PackableSample</strong>—a typed, serializable data structure. Unlike raw dictionaries or ad-hoc classes, PackableSamples provide:</p>
603<ul>
604<li><strong>Type safety</strong>: Know your schema at write time, not training time</li>
605<li><strong>Automatic serialization</strong>: msgpack encoding with efficient NDArray handling</li>
606<li><strong>Round-trip fidelity</strong>: Data survives serialization without loss</li>
607</ul>
608<p>Use the <code>@packable</code> decorator to create a typed sample:</p>
609<div id="440626a1" class="cell">
610<div class="sourceCode cell-code" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
611<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> numpy.typing <span class="im">import</span> NDArray</span>
612<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> atdata</span>
613<span id="cb3-4"><a href="#cb3-4" aria-hidden="true" tabindex="-1"></a></span>
614<span id="cb3-5"><a href="#cb3-5" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
615<span id="cb3-6"><a href="#cb3-6" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> ImageSample:</span>
616<span id="cb3-7"><a href="#cb3-7" aria-hidden="true" tabindex="-1"></a> <span class="co">"""A sample containing an image with label and confidence."""</span></span>
617<span id="cb3-8"><a href="#cb3-8" aria-hidden="true" tabindex="-1"></a> image: NDArray</span>
618<span id="cb3-9"><a href="#cb3-9" aria-hidden="true" tabindex="-1"></a> label: <span class="bu">str</span></span>
619<span id="cb3-10"><a href="#cb3-10" aria-hidden="true" tabindex="-1"></a> confidence: <span class="bu">float</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
620</div>
621<p>The <code>@packable</code> decorator:</p>
622<ul>
623<li>Converts your class into a dataclass</li>
624<li>Adds automatic msgpack serialization</li>
625<li>Handles NDArray conversion to/from bytes</li>
626</ul>
627</section>
628<section id="create-sample-instances" class="level2">
629<h2 class="anchored" data-anchor-id="create-sample-instances">Create Sample Instances</h2>
630<div id="c6081379" class="cell">
631<div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Create a single sample</span></span>
632<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a>sample <span class="op">=</span> ImageSample(</span>
633<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a> image<span class="op">=</span>np.random.rand(<span class="dv">224</span>, <span class="dv">224</span>, <span class="dv">3</span>).astype(np.float32),</span>
634<span id="cb4-4"><a href="#cb4-4" aria-hidden="true" tabindex="-1"></a> label<span class="op">=</span><span class="st">"cat"</span>,</span>
635<span id="cb4-5"><a href="#cb4-5" aria-hidden="true" tabindex="-1"></a> confidence<span class="op">=</span><span class="fl">0.95</span>,</span>
636<span id="cb4-6"><a href="#cb4-6" aria-hidden="true" tabindex="-1"></a>)</span>
637<span id="cb4-7"><a href="#cb4-7" aria-hidden="true" tabindex="-1"></a></span>
638<span id="cb4-8"><a href="#cb4-8" aria-hidden="true" tabindex="-1"></a><span class="co"># Check serialization</span></span>
639<span id="cb4-9"><a href="#cb4-9" aria-hidden="true" tabindex="-1"></a>packed_bytes <span class="op">=</span> sample.packed</span>
640<span id="cb4-10"><a href="#cb4-10" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Serialized size: </span><span class="sc">{</span><span class="bu">len</span>(packed_bytes)<span class="sc">:,}</span><span class="ss"> bytes"</span>)</span>
641<span id="cb4-11"><a href="#cb4-11" aria-hidden="true" tabindex="-1"></a></span>
642<span id="cb4-12"><a href="#cb4-12" aria-hidden="true" tabindex="-1"></a><span class="co"># Verify round-trip</span></span>
643<span id="cb4-13"><a href="#cb4-13" aria-hidden="true" tabindex="-1"></a>restored <span class="op">=</span> ImageSample.from_bytes(packed_bytes)</span>
644<span id="cb4-14"><a href="#cb4-14" aria-hidden="true" tabindex="-1"></a><span class="cf">assert</span> np.allclose(sample.image, restored.image)</span>
645<span id="cb4-15"><a href="#cb4-15" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="st">"Round-trip successful!"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
646</div>
647</section>
648<section id="write-a-dataset" class="level2">
649<h2 class="anchored" data-anchor-id="write-a-dataset">Write a Dataset</h2>
650<p>atdata uses <strong>WebDataset’s tar format</strong> for storage. This choice is deliberate:</p>
651<ul>
652<li><strong>Streaming</strong>: Process data without downloading entire datasets</li>
653<li><strong>Sharding</strong>: Split large datasets across multiple files for parallel I/O</li>
654<li><strong>Proven</strong>: Battle-tested at scale by organizations like Google, NVIDIA, and OpenAI</li>
655</ul>
656<p>The <code>as_wds</code> property on your sample provides the dictionary format WebDataset expects:</p>
657<p>Use WebDataset’s <code>TarWriter</code> to create dataset files:</p>
658<div id="f621e87f" class="cell">
659<div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> webdataset <span class="im">as</span> wds</span>
660<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a></span>
661<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a><span class="co"># Create 100 samples</span></span>
662<span id="cb5-4"><a href="#cb5-4" aria-hidden="true" tabindex="-1"></a>samples <span class="op">=</span> [</span>
663<span id="cb5-5"><a href="#cb5-5" aria-hidden="true" tabindex="-1"></a> ImageSample(</span>
664<span id="cb5-6"><a href="#cb5-6" aria-hidden="true" tabindex="-1"></a> image<span class="op">=</span>np.random.rand(<span class="dv">224</span>, <span class="dv">224</span>, <span class="dv">3</span>).astype(np.float32),</span>
665<span id="cb5-7"><a href="#cb5-7" aria-hidden="true" tabindex="-1"></a> label<span class="op">=</span><span class="ss">f"class_</span><span class="sc">{</span>i <span class="op">%</span> <span class="dv">10</span><span class="sc">}</span><span class="ss">"</span>,</span>
666<span id="cb5-8"><a href="#cb5-8" aria-hidden="true" tabindex="-1"></a> confidence<span class="op">=</span>np.random.rand(),</span>
667<span id="cb5-9"><a href="#cb5-9" aria-hidden="true" tabindex="-1"></a> )</span>
668<span id="cb5-10"><a href="#cb5-10" aria-hidden="true" tabindex="-1"></a> <span class="cf">for</span> i <span class="kw">in</span> <span class="bu">range</span>(<span class="dv">100</span>)</span>
669<span id="cb5-11"><a href="#cb5-11" aria-hidden="true" tabindex="-1"></a>]</span>
670<span id="cb5-12"><a href="#cb5-12" aria-hidden="true" tabindex="-1"></a></span>
671<span id="cb5-13"><a href="#cb5-13" aria-hidden="true" tabindex="-1"></a><span class="co"># Write to tar file</span></span>
672<span id="cb5-14"><a href="#cb5-14" aria-hidden="true" tabindex="-1"></a><span class="cf">with</span> wds.writer.TarWriter(<span class="st">"my-dataset-000000.tar"</span>) <span class="im">as</span> sink:</span>
673<span id="cb5-15"><a href="#cb5-15" aria-hidden="true" tabindex="-1"></a> <span class="cf">for</span> i, sample <span class="kw">in</span> <span class="bu">enumerate</span>(samples):</span>
674<span id="cb5-16"><a href="#cb5-16" aria-hidden="true" tabindex="-1"></a> sink.write({<span class="op">**</span>sample.as_wds, <span class="st">"__key__"</span>: <span class="ss">f"sample_</span><span class="sc">{</span>i<span class="sc">:06d}</span><span class="ss">"</span>})</span>
675<span id="cb5-17"><a href="#cb5-17" aria-hidden="true" tabindex="-1"></a></span>
676<span id="cb5-18"><a href="#cb5-18" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="st">"Wrote 100 samples to my-dataset-000000.tar"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
677</div>
678</section>
679<section id="load-and-iterate" class="level2">
680<h2 class="anchored" data-anchor-id="load-and-iterate">Load and Iterate</h2>
681<p>The generic <code>Dataset[T]</code> class connects your sample type to WebDataset’s streaming infrastructure. When you specify <code>Dataset[ImageSample]</code>, atdata knows how to deserialize the msgpack bytes back into fully-typed objects.</p>
682<p><strong>Automatic batch aggregation</strong> is a key feature: when you iterate with <code>batch_size</code>, atdata returns <code>SampleBatch</code> objects that intelligently combine samples:</p>
683<ul>
684<li>NDArray fields are <strong>stacked</strong> into a single array with a batch dimension</li>
685<li>Other fields become <strong>lists</strong> of values</li>
686</ul>
687<p>This eliminates boilerplate collation code and works automatically with any PackableSample type.</p>
688<p>Create a typed <code>Dataset</code> and iterate with batching:</p>
689<div id="e60a7dc5" class="cell">
690<div class="sourceCode cell-code" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Load dataset with type</span></span>
691<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a>dataset <span class="op">=</span> atdata.Dataset[ImageSample](<span class="st">"my-dataset-000000.tar"</span>)</span>
692<span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a></span>
693<span id="cb6-4"><a href="#cb6-4" aria-hidden="true" tabindex="-1"></a><span class="co"># Iterate in order with batching</span></span>
694<span id="cb6-5"><a href="#cb6-5" aria-hidden="true" tabindex="-1"></a><span class="cf">for</span> batch <span class="kw">in</span> dataset.ordered(batch_size<span class="op">=</span><span class="dv">16</span>):</span>
695<span id="cb6-6"><a href="#cb6-6" aria-hidden="true" tabindex="-1"></a> <span class="co"># NDArray fields are stacked</span></span>
696<span id="cb6-7"><a href="#cb6-7" aria-hidden="true" tabindex="-1"></a> images <span class="op">=</span> batch.image <span class="co"># shape: (16, 224, 224, 3)</span></span>
697<span id="cb6-8"><a href="#cb6-8" aria-hidden="true" tabindex="-1"></a></span>
698<span id="cb6-9"><a href="#cb6-9" aria-hidden="true" tabindex="-1"></a> <span class="co"># Other fields become lists</span></span>
699<span id="cb6-10"><a href="#cb6-10" aria-hidden="true" tabindex="-1"></a> labels <span class="op">=</span> batch.label <span class="co"># list of 16 strings</span></span>
700<span id="cb6-11"><a href="#cb6-11" aria-hidden="true" tabindex="-1"></a> confidences <span class="op">=</span> batch.confidence <span class="co"># list of 16 floats</span></span>
701<span id="cb6-12"><a href="#cb6-12" aria-hidden="true" tabindex="-1"></a></span>
702<span id="cb6-13"><a href="#cb6-13" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f"Batch shape: </span><span class="sc">{</span>images<span class="sc">.</span>shape<span class="sc">}</span><span class="ss">"</span>)</span>
703<span id="cb6-14"><a href="#cb6-14" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f"Labels: </span><span class="sc">{</span>labels[:<span class="dv">3</span>]<span class="sc">}</span><span class="ss">..."</span>)</span>
704<span id="cb6-15"><a href="#cb6-15" aria-hidden="true" tabindex="-1"></a> <span class="cf">break</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
705</div>
706</section>
707<section id="shuffled-iteration" class="level2">
708<h2 class="anchored" data-anchor-id="shuffled-iteration">Shuffled Iteration</h2>
709<p>Proper shuffling is critical for training. WebDataset provides <strong>two-level shuffling</strong>:</p>
710<ol type="1">
711<li><strong>Shard shuffling</strong>: Randomize the order of tar files</li>
712<li><strong>Sample shuffling</strong>: Randomize samples within a buffer</li>
713</ol>
714<p>This approach balances randomness with streaming efficiency—you get well-shuffled data without needing random access to the entire dataset.</p>
715<p>For training, use shuffled iteration:</p>
716<div id="7dc74662" class="cell">
717<div class="sourceCode cell-code" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a><span class="cf">for</span> batch <span class="kw">in</span> dataset.shuffled(batch_size<span class="op">=</span><span class="dv">32</span>):</span>
718<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a> <span class="co"># Samples are shuffled at shard and sample level</span></span>
719<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a> images <span class="op">=</span> batch.image</span>
720<span id="cb7-4"><a href="#cb7-4" aria-hidden="true" tabindex="-1"></a> labels <span class="op">=</span> batch.label</span>
721<span id="cb7-5"><a href="#cb7-5" aria-hidden="true" tabindex="-1"></a></span>
722<span id="cb7-6"><a href="#cb7-6" aria-hidden="true" tabindex="-1"></a> <span class="co"># Train your model</span></span>
723<span id="cb7-7"><a href="#cb7-7" aria-hidden="true" tabindex="-1"></a> <span class="co"># model.train(images, labels)</span></span>
724<span id="cb7-8"><a href="#cb7-8" aria-hidden="true" tabindex="-1"></a> <span class="cf">break</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
725</div>
726</section>
727<section id="use-lenses-for-type-transformations" class="level2">
728<h2 class="anchored" data-anchor-id="use-lenses-for-type-transformations">Use Lenses for Type Transformations</h2>
729<p><strong>Lenses</strong> are bidirectional transformations between sample types. They solve a common problem: you have a dataset with a rich schema, but a particular task only needs a subset of fields—or needs derived fields computed on-the-fly.</p>
730<p>Instead of creating separate datasets for each use case (duplicating storage and maintenance burden), lenses let you <strong>view</strong> the same underlying data through different type schemas. This is inspired by functional programming concepts and enables:</p>
731<ul>
732<li><strong>Schema reduction</strong>: Drop fields you don’t need</li>
733<li><strong>Schema migration</strong>: Handle version differences between datasets</li>
734<li><strong>Derived features</strong>: Compute fields on-the-fly during iteration</li>
735</ul>
736<p>View datasets through different schemas:</p>
737<div id="8494cd76" class="cell">
738<div class="sourceCode cell-code" id="cb8"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Define a simplified view type</span></span>
739<span id="cb8-2"><a href="#cb8-2" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
740<span id="cb8-3"><a href="#cb8-3" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> SimplifiedSample:</span>
741<span id="cb8-4"><a href="#cb8-4" aria-hidden="true" tabindex="-1"></a> label: <span class="bu">str</span></span>
742<span id="cb8-5"><a href="#cb8-5" aria-hidden="true" tabindex="-1"></a> confidence: <span class="bu">float</span></span>
743<span id="cb8-6"><a href="#cb8-6" aria-hidden="true" tabindex="-1"></a></span>
744<span id="cb8-7"><a href="#cb8-7" aria-hidden="true" tabindex="-1"></a><span class="co"># Create a lens transformation</span></span>
745<span id="cb8-8"><a href="#cb8-8" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.lens</span></span>
746<span id="cb8-9"><a href="#cb8-9" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> simplify(src: ImageSample) <span class="op">-></span> SimplifiedSample:</span>
747<span id="cb8-10"><a href="#cb8-10" aria-hidden="true" tabindex="-1"></a> <span class="cf">return</span> SimplifiedSample(label<span class="op">=</span>src.label, confidence<span class="op">=</span>src.confidence)</span>
748<span id="cb8-11"><a href="#cb8-11" aria-hidden="true" tabindex="-1"></a></span>
749<span id="cb8-12"><a href="#cb8-12" aria-hidden="true" tabindex="-1"></a><span class="co"># View dataset through lens</span></span>
750<span id="cb8-13"><a href="#cb8-13" aria-hidden="true" tabindex="-1"></a>simple_ds <span class="op">=</span> dataset.as_type(SimplifiedSample)</span>
751<span id="cb8-14"><a href="#cb8-14" aria-hidden="true" tabindex="-1"></a></span>
752<span id="cb8-15"><a href="#cb8-15" aria-hidden="true" tabindex="-1"></a><span class="cf">for</span> batch <span class="kw">in</span> simple_ds.ordered(batch_size<span class="op">=</span><span class="dv">8</span>):</span>
753<span id="cb8-16"><a href="#cb8-16" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f"Labels: </span><span class="sc">{</span>batch<span class="sc">.</span>label<span class="sc">}</span><span class="ss">"</span>)</span>
754<span id="cb8-17"><a href="#cb8-17" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f"Confidences: </span><span class="sc">{</span>batch<span class="sc">.</span>confidence<span class="sc">}</span><span class="ss">"</span>)</span>
755<span id="cb8-18"><a href="#cb8-18" aria-hidden="true" tabindex="-1"></a> <span class="cf">break</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
756</div>
757</section>
758<section id="what-youve-learned" class="level2">
759<h2 class="anchored" data-anchor-id="what-youve-learned">What You’ve Learned</h2>
760<p>You now understand atdata’s foundational concepts:</p>
761<table class="caption-top table">
762<thead>
763<tr class="header">
764<th>Concept</th>
765<th>Purpose</th>
766</tr>
767</thead>
768<tbody>
769<tr class="odd">
770<td><code>@packable</code></td>
771<td>Create typed, serializable sample classes</td>
772</tr>
773<tr class="even">
774<td><code>Dataset[T]</code></td>
775<td>Typed iteration over WebDataset tar files</td>
776</tr>
777<tr class="odd">
778<td><code>SampleBatch[T]</code></td>
779<td>Automatic aggregation with NDArray stacking</td>
780</tr>
781<tr class="even">
782<td><code>@lens</code></td>
783<td>Transform between sample types without data duplication</td>
784</tr>
785</tbody>
786</table>
787<p>These patterns work identically whether your data lives on local disk, in team S3 storage, or published to the ATProto network. The next tutorials show how to scale beyond local files.</p>
788</section>
789<section id="next-steps" class="level2">
790<h2 class="anchored" data-anchor-id="next-steps">Next Steps</h2>
791<div class="callout callout-style-default callout-tip callout-titled">
792<div class="callout-header d-flex align-content-center">
793<div class="callout-icon-container">
794<i class="callout-icon"></i>
795</div>
796<div class="callout-title-container flex-fill">
797Ready to Share with Your Team?
798</div>
799</div>
800<div class="callout-body-container callout-body">
801<p>The <a href="../tutorials/local-workflow.html">Local Workflow</a> tutorial shows how to set up Redis + S3 storage for team-wide dataset discovery and sharing.</p>
802</div>
803</div>
804<ul>
805<li><strong><a href="../tutorials/local-workflow.html">Local Workflow</a></strong> - Store datasets with Redis + S3</li>
806<li><strong><a href="../tutorials/atmosphere.html">Atmosphere Publishing</a></strong> - Publish to ATProto federation</li>
807<li><strong><a href="../reference/packable-samples.html">Packable Samples</a></strong> - Deep dive into sample types</li>
808<li><strong><a href="../reference/datasets.html">Datasets</a></strong> - Advanced dataset operations</li>
809</ul>
810
811
812</section>
813
814</main> <!-- /main -->
815<script id="quarto-html-after-body" type="application/javascript">
816 window.document.addEventListener("DOMContentLoaded", function (event) {
817 // Ensure there is a toggle, if there isn't float one in the top right
818 if (window.document.querySelector('.quarto-color-scheme-toggle') === null) {
819 const a = window.document.createElement('a');
820 a.classList.add('top-right');
821 a.classList.add('quarto-color-scheme-toggle');
822 a.href = "";
823 a.onclick = function() { try { window.quartoToggleColorScheme(); } catch {} return false; };
824 const i = window.document.createElement("i");
825 i.classList.add('bi');
826 a.appendChild(i);
827 window.document.body.appendChild(a);
828 }
829 setColorSchemeToggle(hasAlternateSentinel())
830 const icon = "";
831 const anchorJS = new window.AnchorJS();
832 anchorJS.options = {
833 placement: 'right',
834 icon: icon
835 };
836 anchorJS.add('.anchored');
837 const isCodeAnnotation = (el) => {
838 for (const clz of el.classList) {
839 if (clz.startsWith('code-annotation-')) {
840 return true;
841 }
842 }
843 return false;
844 }
845 const onCopySuccess = function(e) {
846 // button target
847 const button = e.trigger;
848 // don't keep focus
849 button.blur();
850 // flash "checked"
851 button.classList.add('code-copy-button-checked');
852 var currentTitle = button.getAttribute("title");
853 button.setAttribute("title", "Copied!");
854 let tooltip;
855 if (window.bootstrap) {
856 button.setAttribute("data-bs-toggle", "tooltip");
857 button.setAttribute("data-bs-placement", "left");
858 button.setAttribute("data-bs-title", "Copied!");
859 tooltip = new bootstrap.Tooltip(button,
860 { trigger: "manual",
861 customClass: "code-copy-button-tooltip",
862 offset: [0, -8]});
863 tooltip.show();
864 }
865 setTimeout(function() {
866 if (tooltip) {
867 tooltip.hide();
868 button.removeAttribute("data-bs-title");
869 button.removeAttribute("data-bs-toggle");
870 button.removeAttribute("data-bs-placement");
871 }
872 button.setAttribute("title", currentTitle);
873 button.classList.remove('code-copy-button-checked');
874 }, 1000);
875 // clear code selection
876 e.clearSelection();
877 }
878 const getTextToCopy = function(trigger) {
879 const codeEl = trigger.previousElementSibling.cloneNode(true);
880 for (const childEl of codeEl.children) {
881 if (isCodeAnnotation(childEl)) {
882 childEl.remove();
883 }
884 }
885 return codeEl.innerText;
886 }
887 const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
888 text: getTextToCopy
889 });
890 clipboard.on('success', onCopySuccess);
891 if (window.document.getElementById('quarto-embedded-source-code-modal')) {
892 const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', {
893 text: getTextToCopy,
894 container: window.document.getElementById('quarto-embedded-source-code-modal')
895 });
896 clipboardModal.on('success', onCopySuccess);
897 }
898 var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//);
899 var mailtoRegex = new RegExp(/^mailto:/);
900 var filterRegex = new RegExp("https:\/\/github\.com\/your-org\/atdata");
901 var isInternal = (href) => {
902 return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href);
903 }
904 // Inspect non-navigation links and adorn them if external
905 var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)');
906 for (var i=0; i<links.length; i++) {
907 const link = links[i];
908 if (!isInternal(link.href)) {
909 // undo the damage that might have been done by quarto-nav.js in the case of
910 // links that we want to consider external
911 if (link.dataset.originalHref !== undefined) {
912 link.href = link.dataset.originalHref;
913 }
914 }
915 }
916 function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
917 const config = {
918 allowHTML: true,
919 maxWidth: 500,
920 delay: 100,
921 arrow: false,
922 appendTo: function(el) {
923 return el.parentElement;
924 },
925 interactive: true,
926 interactiveBorder: 10,
927 theme: 'quarto',
928 placement: 'bottom-start',
929 };
930 if (contentFn) {
931 config.content = contentFn;
932 }
933 if (onTriggerFn) {
934 config.onTrigger = onTriggerFn;
935 }
936 if (onUntriggerFn) {
937 config.onUntrigger = onUntriggerFn;
938 }
939 window.tippy(el, config);
940 }
941 const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
942 for (var i=0; i<noterefs.length; i++) {
943 const ref = noterefs[i];
944 tippyHover(ref, function() {
945 // use id or data attribute instead here
946 let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
947 try { href = new URL(href).hash; } catch {}
948 const id = href.replace(/^#\/?/, "");
949 const note = window.document.getElementById(id);
950 if (note) {
951 return note.innerHTML;
952 } else {
953 return "";
954 }
955 });
956 }
957 const xrefs = window.document.querySelectorAll('a.quarto-xref');
958 const processXRef = (id, note) => {
959 // Strip column container classes
960 const stripColumnClz = (el) => {
961 el.classList.remove("page-full", "page-columns");
962 if (el.children) {
963 for (const child of el.children) {
964 stripColumnClz(child);
965 }
966 }
967 }
968 stripColumnClz(note)
969 if (id === null || id.startsWith('sec-')) {
970 // Special case sections, only their first couple elements
971 const container = document.createElement("div");
972 if (note.children && note.children.length > 2) {
973 container.appendChild(note.children[0].cloneNode(true));
974 for (let i = 1; i < note.children.length; i++) {
975 const child = note.children[i];
976 if (child.tagName === "P" && child.innerText === "") {
977 continue;
978 } else {
979 container.appendChild(child.cloneNode(true));
980 break;
981 }
982 }
983 if (window.Quarto?.typesetMath) {
984 window.Quarto.typesetMath(container);
985 }
986 return container.innerHTML
987 } else {
988 if (window.Quarto?.typesetMath) {
989 window.Quarto.typesetMath(note);
990 }
991 return note.innerHTML;
992 }
993 } else {
994 // Remove any anchor links if they are present
995 const anchorLink = note.querySelector('a.anchorjs-link');
996 if (anchorLink) {
997 anchorLink.remove();
998 }
999 if (window.Quarto?.typesetMath) {
1000 window.Quarto.typesetMath(note);
1001 }
1002 if (note.classList.contains("callout")) {
1003 return note.outerHTML;
1004 } else {
1005 return note.innerHTML;
1006 }
1007 }
1008 }
1009 for (var i=0; i<xrefs.length; i++) {
1010 const xref = xrefs[i];
1011 tippyHover(xref, undefined, function(instance) {
1012 instance.disable();
1013 let url = xref.getAttribute('href');
1014 let hash = undefined;
1015 if (url.startsWith('#')) {
1016 hash = url;
1017 } else {
1018 try { hash = new URL(url).hash; } catch {}
1019 }
1020 if (hash) {
1021 const id = hash.replace(/^#\/?/, "");
1022 const note = window.document.getElementById(id);
1023 if (note !== null) {
1024 try {
1025 const html = processXRef(id, note.cloneNode(true));
1026 instance.setContent(html);
1027 } finally {
1028 instance.enable();
1029 instance.show();
1030 }
1031 } else {
1032 // See if we can fetch this
1033 fetch(url.split('#')[0])
1034 .then(res => res.text())
1035 .then(html => {
1036 const parser = new DOMParser();
1037 const htmlDoc = parser.parseFromString(html, "text/html");
1038 const note = htmlDoc.getElementById(id);
1039 if (note !== null) {
1040 const html = processXRef(id, note);
1041 instance.setContent(html);
1042 }
1043 }).finally(() => {
1044 instance.enable();
1045 instance.show();
1046 });
1047 }
1048 } else {
1049 // See if we can fetch a full url (with no hash to target)
1050 // This is a special case and we should probably do some content thinning / targeting
1051 fetch(url)
1052 .then(res => res.text())
1053 .then(html => {
1054 const parser = new DOMParser();
1055 const htmlDoc = parser.parseFromString(html, "text/html");
1056 const note = htmlDoc.querySelector('main.content');
1057 if (note !== null) {
1058 // This should only happen for chapter cross references
1059 // (since there is no id in the URL)
1060 // remove the first header
1061 if (note.children.length > 0 && note.children[0].tagName === "HEADER") {
1062 note.children[0].remove();
1063 }
1064 const html = processXRef(null, note);
1065 instance.setContent(html);
1066 }
1067 }).finally(() => {
1068 instance.enable();
1069 instance.show();
1070 });
1071 }
1072 }, function(instance) {
1073 });
1074 }
1075 let selectedAnnoteEl;
1076 const selectorForAnnotation = ( cell, annotation) => {
1077 let cellAttr = 'data-code-cell="' + cell + '"';
1078 let lineAttr = 'data-code-annotation="' + annotation + '"';
1079 const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
1080 return selector;
1081 }
1082 const selectCodeLines = (annoteEl) => {
1083 const doc = window.document;
1084 const targetCell = annoteEl.getAttribute("data-target-cell");
1085 const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
1086 const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
1087 const lines = annoteSpan.getAttribute("data-code-lines").split(",");
1088 const lineIds = lines.map((line) => {
1089 return targetCell + "-" + line;
1090 })
1091 let top = null;
1092 let height = null;
1093 let parent = null;
1094 if (lineIds.length > 0) {
1095 //compute the position of the single el (top and bottom and make a div)
1096 const el = window.document.getElementById(lineIds[0]);
1097 top = el.offsetTop;
1098 height = el.offsetHeight;
1099 parent = el.parentElement.parentElement;
1100 if (lineIds.length > 1) {
1101 const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
1102 const bottom = lastEl.offsetTop + lastEl.offsetHeight;
1103 height = bottom - top;
1104 }
1105 if (top !== null && height !== null && parent !== null) {
1106 // cook up a div (if necessary) and position it
1107 let div = window.document.getElementById("code-annotation-line-highlight");
1108 if (div === null) {
1109 div = window.document.createElement("div");
1110 div.setAttribute("id", "code-annotation-line-highlight");
1111 div.style.position = 'absolute';
1112 parent.appendChild(div);
1113 }
1114 div.style.top = top - 2 + "px";
1115 div.style.height = height + 4 + "px";
1116 div.style.left = 0;
1117 let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
1118 if (gutterDiv === null) {
1119 gutterDiv = window.document.createElement("div");
1120 gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
1121 gutterDiv.style.position = 'absolute';
1122 const codeCell = window.document.getElementById(targetCell);
1123 const gutter = codeCell.querySelector('.code-annotation-gutter');
1124 gutter.appendChild(gutterDiv);
1125 }
1126 gutterDiv.style.top = top - 2 + "px";
1127 gutterDiv.style.height = height + 4 + "px";
1128 }
1129 selectedAnnoteEl = annoteEl;
1130 }
1131 };
1132 const unselectCodeLines = () => {
1133 const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
1134 elementsIds.forEach((elId) => {
1135 const div = window.document.getElementById(elId);
1136 if (div) {
1137 div.remove();
1138 }
1139 });
1140 selectedAnnoteEl = undefined;
1141 };
1142 // Handle positioning of the toggle
1143 window.addEventListener(
1144 "resize",
1145 throttle(() => {
1146 elRect = undefined;
1147 if (selectedAnnoteEl) {
1148 selectCodeLines(selectedAnnoteEl);
1149 }
1150 }, 10)
1151 );
1152 function throttle(fn, ms) {
1153 let throttle = false;
1154 let timer;
1155 return (...args) => {
1156 if(!throttle) { // first call gets through
1157 fn.apply(this, args);
1158 throttle = true;
1159 } else { // all the others get throttled
1160 if(timer) clearTimeout(timer); // cancel #2
1161 timer = setTimeout(() => {
1162 fn.apply(this, args);
1163 timer = throttle = false;
1164 }, ms);
1165 }
1166 };
1167 }
1168 // Attach click handler to the DT
1169 const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
1170 for (const annoteDlNode of annoteDls) {
1171 annoteDlNode.addEventListener('click', (event) => {
1172 const clickedEl = event.target;
1173 if (clickedEl !== selectedAnnoteEl) {
1174 unselectCodeLines();
1175 const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
1176 if (activeEl) {
1177 activeEl.classList.remove('code-annotation-active');
1178 }
1179 selectCodeLines(clickedEl);
1180 clickedEl.classList.add('code-annotation-active');
1181 } else {
1182 // Unselect the line
1183 unselectCodeLines();
1184 clickedEl.classList.remove('code-annotation-active');
1185 }
1186 });
1187 }
1188 const findCites = (el) => {
1189 const parentEl = el.parentElement;
1190 if (parentEl) {
1191 const cites = parentEl.dataset.cites;
1192 if (cites) {
1193 return {
1194 el,
1195 cites: cites.split(' ')
1196 };
1197 } else {
1198 return findCites(el.parentElement)
1199 }
1200 } else {
1201 return undefined;
1202 }
1203 };
1204 var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
1205 for (var i=0; i<bibliorefs.length; i++) {
1206 const ref = bibliorefs[i];
1207 const citeInfo = findCites(ref);
1208 if (citeInfo) {
1209 tippyHover(citeInfo.el, function() {
1210 var popup = window.document.createElement('div');
1211 citeInfo.cites.forEach(function(cite) {
1212 var citeDiv = window.document.createElement('div');
1213 citeDiv.classList.add('hanging-indent');
1214 citeDiv.classList.add('csl-entry');
1215 var biblioDiv = window.document.getElementById('ref-' + cite);
1216 if (biblioDiv) {
1217 citeDiv.innerHTML = biblioDiv.innerHTML;
1218 }
1219 popup.appendChild(citeDiv);
1220 });
1221 return popup.innerHTML;
1222 });
1223 }
1224 }
1225 });
1226 </script>
1227</div> <!-- /content -->
1228<footer class="footer">
1229 <div class="nav-footer">
1230 <div class="nav-footer-left">
1231<p>Built with <a href="https://quarto.org/">Quarto</a></p>
1232</div>
1233 <div class="nav-footer-center">
1234
1235 <div class="toc-actions d-sm-block d-md-none"><ul><li><a href="https://github.com/your-org/atdata/edit/main/tutorials/quickstart.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></div>
1236 <div class="nav-footer-right">
1237<p>MIT License</p>
1238</div>
1239 </div>
1240</footer>
1241
1242
1243
1244
1245</body></html>