A loose federation of distributed, typed datasets
docs(api): add protocol classes to API reference (Packable, AbstractIndex, DataSource)
Generate quartodoc pages for protocol types that define the library's
extension points:
- Packable: structural protocol for packable sample types
- IndexEntry: common interface for index entries
- AbstractIndex: protocol for index operations (LocalIndex, AtmosphereIndex)
- AbstractDataStore: protocol for data storage operations
- DataSource: protocol for shard streaming
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
+5413
-223
+906
docs/api/AbstractDataStore.html
+906
docs/api/AbstractDataStore.html
···
1
+
<!DOCTYPE html>
2
+
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
3
+
4
+
<meta charset="utf-8">
5
+
<meta name="generator" content="quarto-1.7.34">
6
+
7
+
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
8
+
9
+
10
+
<title>abstractdatastore – atdata</title>
11
+
<style>
12
+
code{white-space: pre-wrap;}
13
+
span.smallcaps{font-variant: small-caps;}
14
+
div.columns{display: flex; gap: min(4vw, 1.5em);}
15
+
div.column{flex: auto; overflow-x: auto;}
16
+
div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
17
+
ul.task-list{list-style: none;}
18
+
ul.task-list li input[type="checkbox"] {
19
+
width: 0.8em;
20
+
margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
21
+
vertical-align: middle;
22
+
}
23
+
/* CSS for syntax highlighting */
24
+
html { -webkit-text-size-adjust: 100%; }
25
+
pre > code.sourceCode { white-space: pre; position: relative; }
26
+
pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
27
+
pre > code.sourceCode > span:empty { height: 1.2em; }
28
+
.sourceCode { overflow: visible; }
29
+
code.sourceCode > span { color: inherit; text-decoration: inherit; }
30
+
div.sourceCode { margin: 1em 0; }
31
+
pre.sourceCode { margin: 0; }
32
+
@media screen {
33
+
div.sourceCode { overflow: auto; }
34
+
}
35
+
@media print {
36
+
pre > code.sourceCode { white-space: pre-wrap; }
37
+
pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
38
+
}
39
+
pre.numberSource code
40
+
{ counter-reset: source-line 0; }
41
+
pre.numberSource code > span
42
+
{ position: relative; left: -4em; counter-increment: source-line; }
43
+
pre.numberSource code > span > a:first-child::before
44
+
{ content: counter(source-line);
45
+
position: relative; left: -1em; text-align: right; vertical-align: baseline;
46
+
border: none; display: inline-block;
47
+
-webkit-touch-callout: none; -webkit-user-select: none;
48
+
-khtml-user-select: none; -moz-user-select: none;
49
+
-ms-user-select: none; user-select: none;
50
+
padding: 0 4px; width: 4em;
51
+
}
52
+
pre.numberSource { margin-left: 3em; padding-left: 4px; }
53
+
div.sourceCode
54
+
{ }
55
+
@media screen {
56
+
pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
57
+
}
58
+
</style>
59
+
60
+
61
+
<script src="../site_libs/quarto-nav/quarto-nav.js"></script>
62
+
<script src="../site_libs/quarto-nav/headroom.min.js"></script>
63
+
<script src="../site_libs/clipboard/clipboard.min.js"></script>
64
+
<script src="../site_libs/quarto-search/autocomplete.umd.js"></script>
65
+
<script src="../site_libs/quarto-search/fuse.min.js"></script>
66
+
<script src="../site_libs/quarto-search/quarto-search.js"></script>
67
+
<meta name="quarto:offset" content="../">
68
+
<script src="../site_libs/quarto-html/quarto.js" type="module"></script>
69
+
<script src="../site_libs/quarto-html/tabsets/tabsets.js" type="module"></script>
70
+
<script src="../site_libs/quarto-html/popper.min.js"></script>
71
+
<script src="../site_libs/quarto-html/tippy.umd.min.js"></script>
72
+
<script src="../site_libs/quarto-html/anchor.min.js"></script>
73
+
<link href="../site_libs/quarto-html/tippy.css" rel="stylesheet">
74
+
<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme" id="quarto-text-highlighting-styles">
75
+
<link href="../site_libs/quarto-html/quarto-syntax-highlighting-dark-8dcd8563ea6803ab7cbb3d71ca5772e1.css" rel="stylesheet" class="quarto-color-scheme quarto-color-alternate" id="quarto-text-highlighting-styles">
76
+
<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme-extra" id="quarto-text-highlighting-styles">
77
+
<script src="../site_libs/bootstrap/bootstrap.min.js"></script>
78
+
<link href="../site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
79
+
<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme" id="quarto-bootstrap" data-mode="light">
80
+
<link href="../site_libs/bootstrap/bootstrap-dark-7964ffd8887b0991fe8d71c6c8bc75d6.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme quarto-color-alternate" id="quarto-bootstrap" data-mode="dark">
81
+
<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme-extra" id="quarto-bootstrap" data-mode="light">
82
+
<script id="quarto-search-options" type="application/json">{
83
+
"location": "navbar",
84
+
"copy-button": false,
85
+
"collapse-after": 3,
86
+
"panel-placement": "end",
87
+
"type": "overlay",
88
+
"limit": 50,
89
+
"keyboard-shortcut": [
90
+
"f",
91
+
"/",
92
+
"s"
93
+
],
94
+
"show-item-context": false,
95
+
"language": {
96
+
"search-no-results-text": "No results",
97
+
"search-matching-documents-text": "matching documents",
98
+
"search-copy-link-title": "Copy link to search",
99
+
"search-hide-matches-text": "Hide additional matches",
100
+
"search-more-match-text": "more match in this document",
101
+
"search-more-matches-text": "more matches in this document",
102
+
"search-clear-button-title": "Clear",
103
+
"search-text-placeholder": "",
104
+
"search-detached-cancel-button-title": "Cancel",
105
+
"search-submit-button-title": "Submit",
106
+
"search-label": "Search"
107
+
}
108
+
}</script>
109
+
110
+
111
+
<link rel="stylesheet" href="../assets/styles.css">
112
+
</head>
113
+
114
+
<body class="nav-fixed quarto-light"><script id="quarto-html-before-body" type="application/javascript">
115
+
const toggleBodyColorMode = (bsSheetEl) => {
116
+
const mode = bsSheetEl.getAttribute("data-mode");
117
+
const bodyEl = window.document.querySelector("body");
118
+
if (mode === "dark") {
119
+
bodyEl.classList.add("quarto-dark");
120
+
bodyEl.classList.remove("quarto-light");
121
+
} else {
122
+
bodyEl.classList.add("quarto-light");
123
+
bodyEl.classList.remove("quarto-dark");
124
+
}
125
+
}
126
+
const toggleBodyColorPrimary = () => {
127
+
const bsSheetEl = window.document.querySelector("link#quarto-bootstrap:not([rel=disabled-stylesheet])");
128
+
if (bsSheetEl) {
129
+
toggleBodyColorMode(bsSheetEl);
130
+
}
131
+
}
132
+
const setColorSchemeToggle = (alternate) => {
133
+
const toggles = window.document.querySelectorAll('.quarto-color-scheme-toggle');
134
+
for (let i=0; i < toggles.length; i++) {
135
+
const toggle = toggles[i];
136
+
if (toggle) {
137
+
if (alternate) {
138
+
toggle.classList.add("alternate");
139
+
} else {
140
+
toggle.classList.remove("alternate");
141
+
}
142
+
}
143
+
}
144
+
};
145
+
const toggleColorMode = (alternate) => {
146
+
// Switch the stylesheets
147
+
const primaryStylesheets = window.document.querySelectorAll('link.quarto-color-scheme:not(.quarto-color-alternate)');
148
+
const alternateStylesheets = window.document.querySelectorAll('link.quarto-color-scheme.quarto-color-alternate');
149
+
manageTransitions('#quarto-margin-sidebar .nav-link', false);
150
+
if (alternate) {
151
+
// note: dark is layered on light, we don't disable primary!
152
+
enableStylesheet(alternateStylesheets);
153
+
for (const sheetNode of alternateStylesheets) {
154
+
if (sheetNode.id === "quarto-bootstrap") {
155
+
toggleBodyColorMode(sheetNode);
156
+
}
157
+
}
158
+
} else {
159
+
disableStylesheet(alternateStylesheets);
160
+
enableStylesheet(primaryStylesheets)
161
+
toggleBodyColorPrimary();
162
+
}
163
+
manageTransitions('#quarto-margin-sidebar .nav-link', true);
164
+
// Switch the toggles
165
+
setColorSchemeToggle(alternate)
166
+
// Hack to workaround the fact that safari doesn't
167
+
// properly recolor the scrollbar when toggling (#1455)
168
+
if (navigator.userAgent.indexOf('Safari') > 0 && navigator.userAgent.indexOf('Chrome') == -1) {
169
+
manageTransitions("body", false);
170
+
window.scrollTo(0, 1);
171
+
setTimeout(() => {
172
+
window.scrollTo(0, 0);
173
+
manageTransitions("body", true);
174
+
}, 40);
175
+
}
176
+
}
177
+
const disableStylesheet = (stylesheets) => {
178
+
for (let i=0; i < stylesheets.length; i++) {
179
+
const stylesheet = stylesheets[i];
180
+
stylesheet.rel = 'disabled-stylesheet';
181
+
}
182
+
}
183
+
const enableStylesheet = (stylesheets) => {
184
+
for (let i=0; i < stylesheets.length; i++) {
185
+
const stylesheet = stylesheets[i];
186
+
if(stylesheet.rel !== 'stylesheet') { // for Chrome, which will still FOUC without this check
187
+
stylesheet.rel = 'stylesheet';
188
+
}
189
+
}
190
+
}
191
+
const manageTransitions = (selector, allowTransitions) => {
192
+
const els = window.document.querySelectorAll(selector);
193
+
for (let i=0; i < els.length; i++) {
194
+
const el = els[i];
195
+
if (allowTransitions) {
196
+
el.classList.remove('notransition');
197
+
} else {
198
+
el.classList.add('notransition');
199
+
}
200
+
}
201
+
}
202
+
const isFileUrl = () => {
203
+
return window.location.protocol === 'file:';
204
+
}
205
+
const hasAlternateSentinel = () => {
206
+
let styleSentinel = getColorSchemeSentinel();
207
+
if (styleSentinel !== null) {
208
+
return styleSentinel === "alternate";
209
+
} else {
210
+
return false;
211
+
}
212
+
}
213
+
const setStyleSentinel = (alternate) => {
214
+
const value = alternate ? "alternate" : "default";
215
+
if (!isFileUrl()) {
216
+
window.localStorage.setItem("quarto-color-scheme", value);
217
+
} else {
218
+
localAlternateSentinel = value;
219
+
}
220
+
}
221
+
const getColorSchemeSentinel = () => {
222
+
if (!isFileUrl()) {
223
+
const storageValue = window.localStorage.getItem("quarto-color-scheme");
224
+
return storageValue != null ? storageValue : localAlternateSentinel;
225
+
} else {
226
+
return localAlternateSentinel;
227
+
}
228
+
}
229
+
const toggleGiscusIfUsed = (isAlternate, darkModeDefault) => {
230
+
const baseTheme = document.querySelector('#giscus-base-theme')?.value ?? 'light';
231
+
const alternateTheme = document.querySelector('#giscus-alt-theme')?.value ?? 'dark';
232
+
let newTheme = '';
233
+
if(authorPrefersDark) {
234
+
newTheme = isAlternate ? baseTheme : alternateTheme;
235
+
} else {
236
+
newTheme = isAlternate ? alternateTheme : baseTheme;
237
+
}
238
+
const changeGiscusTheme = () => {
239
+
// From: https://github.com/giscus/giscus/issues/336
240
+
const sendMessage = (message) => {
241
+
const iframe = document.querySelector('iframe.giscus-frame');
242
+
if (!iframe) return;
243
+
iframe.contentWindow.postMessage({ giscus: message }, 'https://giscus.app');
244
+
}
245
+
sendMessage({
246
+
setConfig: {
247
+
theme: newTheme
248
+
}
249
+
});
250
+
}
251
+
const isGiscussLoaded = window.document.querySelector('iframe.giscus-frame') !== null;
252
+
if (isGiscussLoaded) {
253
+
changeGiscusTheme();
254
+
}
255
+
};
256
+
const authorPrefersDark = false;
257
+
const darkModeDefault = authorPrefersDark;
258
+
document.querySelector('link#quarto-text-highlighting-styles.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
259
+
document.querySelector('link#quarto-bootstrap.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
260
+
let localAlternateSentinel = darkModeDefault ? 'alternate' : 'default';
261
+
// Dark / light mode switch
262
+
window.quartoToggleColorScheme = () => {
263
+
// Read the current dark / light value
264
+
let toAlternate = !hasAlternateSentinel();
265
+
toggleColorMode(toAlternate);
266
+
setStyleSentinel(toAlternate);
267
+
toggleGiscusIfUsed(toAlternate, darkModeDefault);
268
+
window.dispatchEvent(new Event('resize'));
269
+
};
270
+
// Switch to dark mode if need be
271
+
if (hasAlternateSentinel()) {
272
+
toggleColorMode(true);
273
+
} else {
274
+
toggleColorMode(false);
275
+
}
276
+
</script>
277
+
278
+
<div id="quarto-search-results"></div>
279
+
<header id="quarto-header" class="headroom fixed-top">
280
+
<nav class="navbar navbar-expand-lg " data-bs-theme="dark">
281
+
<div class="navbar-container container-fluid">
282
+
<div class="navbar-brand-container mx-auto">
283
+
<a class="navbar-brand" href="../index.html">
284
+
<span class="navbar-title">atdata</span>
285
+
</a>
286
+
</div>
287
+
<div id="quarto-search" class="" title="Search"></div>
288
+
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarCollapse" aria-controls="navbarCollapse" role="menu" aria-expanded="false" aria-label="Toggle navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
289
+
<span class="navbar-toggler-icon"></span>
290
+
</button>
291
+
<div class="collapse navbar-collapse" id="navbarCollapse">
292
+
<ul class="navbar-nav navbar-nav-scroll me-auto">
293
+
<li class="nav-item">
294
+
<a class="nav-link" href="../index.html">
295
+
<span class="menu-text">Guide</span></a>
296
+
</li>
297
+
<li class="nav-item dropdown ">
298
+
<a class="nav-link dropdown-toggle" href="#" id="nav-menu-tutorials" role="link" data-bs-toggle="dropdown" aria-expanded="false">
299
+
<span class="menu-text">Tutorials</span>
300
+
</a>
301
+
<ul class="dropdown-menu" aria-labelledby="nav-menu-tutorials">
302
+
<li>
303
+
<a class="dropdown-item" href="../tutorials/quickstart.html">
304
+
<span class="dropdown-text">Quick Start</span></a>
305
+
</li>
306
+
<li>
307
+
<a class="dropdown-item" href="../tutorials/local-workflow.html">
308
+
<span class="dropdown-text">Local Workflow</span></a>
309
+
</li>
310
+
<li>
311
+
<a class="dropdown-item" href="../tutorials/atmosphere.html">
312
+
<span class="dropdown-text">Atmosphere Publishing</span></a>
313
+
</li>
314
+
<li>
315
+
<a class="dropdown-item" href="../tutorials/promotion.html">
316
+
<span class="dropdown-text">Promotion Workflow</span></a>
317
+
</li>
318
+
</ul>
319
+
</li>
320
+
<li class="nav-item dropdown ">
321
+
<a class="nav-link dropdown-toggle" href="#" id="nav-menu-reference" role="link" data-bs-toggle="dropdown" aria-expanded="false">
322
+
<span class="menu-text">Reference</span>
323
+
</a>
324
+
<ul class="dropdown-menu" aria-labelledby="nav-menu-reference">
325
+
<li>
326
+
<a class="dropdown-item" href="../reference/packable-samples.html">
327
+
<span class="dropdown-text">Packable Samples</span></a>
328
+
</li>
329
+
<li>
330
+
<a class="dropdown-item" href="../reference/datasets.html">
331
+
<span class="dropdown-text">Datasets</span></a>
332
+
</li>
333
+
<li>
334
+
<a class="dropdown-item" href="../reference/lenses.html">
335
+
<span class="dropdown-text">Lenses</span></a>
336
+
</li>
337
+
<li>
338
+
<a class="dropdown-item" href="../reference/local-storage.html">
339
+
<span class="dropdown-text">Local Storage</span></a>
340
+
</li>
341
+
<li>
342
+
<a class="dropdown-item" href="../reference/atmosphere.html">
343
+
<span class="dropdown-text">Atmosphere</span></a>
344
+
</li>
345
+
<li>
346
+
<a class="dropdown-item" href="../reference/promotion.html">
347
+
<span class="dropdown-text">Promotion</span></a>
348
+
</li>
349
+
<li>
350
+
<a class="dropdown-item" href="../reference/load-dataset.html">
351
+
<span class="dropdown-text">load_dataset API</span></a>
352
+
</li>
353
+
<li>
354
+
<a class="dropdown-item" href="../reference/protocols.html">
355
+
<span class="dropdown-text">Protocols</span></a>
356
+
</li>
357
+
<li>
358
+
<a class="dropdown-item" href="../reference/uri-spec.html">
359
+
<span class="dropdown-text">URI Specification</span></a>
360
+
</li>
361
+
<li>
362
+
<a class="dropdown-item" href="../reference/troubleshooting.html">
363
+
<span class="dropdown-text">Troubleshooting & FAQ</span></a>
364
+
</li>
365
+
<li>
366
+
<a class="dropdown-item" href="../reference/deployment.html">
367
+
<span class="dropdown-text">Deployment Guide</span></a>
368
+
</li>
369
+
</ul>
370
+
</li>
371
+
<li class="nav-item">
372
+
<a class="nav-link" href="../api/index.html">
373
+
<span class="menu-text">API</span></a>
374
+
</li>
375
+
</ul>
376
+
<ul class="navbar-nav navbar-nav-scroll ms-auto">
377
+
<li class="nav-item compact">
378
+
<a class="nav-link" href="https://github.com/your-org/atdata"> <i class="bi bi-github" role="img">
379
+
</i>
380
+
<span class="menu-text"></span></a>
381
+
</li>
382
+
</ul>
383
+
</div> <!-- /navcollapse -->
384
+
<div class="quarto-navbar-tools">
385
+
<a href="" class="quarto-color-scheme-toggle quarto-navigation-tool px-1" onclick="window.quartoToggleColorScheme(); return false;" title="Toggle dark mode"><i class="bi"></i></a>
386
+
</div>
387
+
</div> <!-- /container-fluid -->
388
+
</nav>
389
+
</header>
390
+
<!-- content -->
391
+
<div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-article page-navbar">
392
+
<!-- sidebar -->
393
+
<!-- margin-sidebar -->
394
+
<div id="quarto-margin-sidebar" class="sidebar margin-sidebar">
395
+
<nav id="TOC" role="doc-toc" class="toc-active">
396
+
<h2 id="toc-title">On this page</h2>
397
+
398
+
<ul>
399
+
<li><a href="#atdata.AbstractDataStore" id="toc-atdata.AbstractDataStore" class="nav-link active" data-scroll-target="#atdata.AbstractDataStore">AbstractDataStore</a>
400
+
<ul class="collapse">
401
+
<li><a href="#methods" id="toc-methods" class="nav-link" data-scroll-target="#methods">Methods</a>
402
+
<ul class="collapse">
403
+
<li><a href="#atdata.AbstractDataStore.read_url" id="toc-atdata.AbstractDataStore.read_url" class="nav-link" data-scroll-target="#atdata.AbstractDataStore.read_url">read_url</a></li>
404
+
<li><a href="#atdata.AbstractDataStore.supports_streaming" id="toc-atdata.AbstractDataStore.supports_streaming" class="nav-link" data-scroll-target="#atdata.AbstractDataStore.supports_streaming">supports_streaming</a></li>
405
+
<li><a href="#atdata.AbstractDataStore.write_shards" id="toc-atdata.AbstractDataStore.write_shards" class="nav-link" data-scroll-target="#atdata.AbstractDataStore.write_shards">write_shards</a></li>
406
+
</ul></li>
407
+
</ul></li>
408
+
</ul>
409
+
<div class="toc-actions"><ul><li><a href="https://github.com/your-org/atdata/edit/main/api/AbstractDataStore.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></nav>
410
+
</div>
411
+
<!-- main -->
412
+
<main class="content" id="quarto-document-content"><header id="title-block-header" class="quarto-title-block"></header>
413
+
414
+
415
+
416
+
417
+
418
+
<section id="atdata.AbstractDataStore" class="level1">
419
+
<h1>AbstractDataStore</h1>
420
+
<div class="sourceCode" id="cb1"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a>AbstractDataStore()</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
421
+
<p>Protocol for data storage operations.</p>
422
+
<p>This protocol abstracts over different storage backends for dataset data: - S3DataStore: S3-compatible object storage - PDSBlobStore: ATProto PDS blob storage (future)</p>
423
+
<p>The separation of index (metadata) from data store (actual files) allows flexible deployment: local index with S3 storage, atmosphere index with S3 storage, or atmosphere index with PDS blobs.</p>
424
+
<p>Example: >>> store = S3DataStore(credentials, bucket=“my-bucket”) >>> urls = store.write_shards(dataset, prefix=“training/v1”) >>> print(urls) [‘s3://my-bucket/training/v1/shard-000000.tar’, …]</p>
425
+
<section id="methods" class="level2">
426
+
<h2 class="anchored" data-anchor-id="methods">Methods</h2>
427
+
<table class="caption-top table">
428
+
<thead>
429
+
<tr class="header">
430
+
<th>Name</th>
431
+
<th>Description</th>
432
+
</tr>
433
+
</thead>
434
+
<tbody>
435
+
<tr class="odd">
436
+
<td><a href="#atdata.AbstractDataStore.read_url">read_url</a></td>
437
+
<td>Resolve a storage URL for reading.</td>
438
+
</tr>
439
+
<tr class="even">
440
+
<td><a href="#atdata.AbstractDataStore.supports_streaming">supports_streaming</a></td>
441
+
<td>Whether this store supports streaming reads.</td>
442
+
</tr>
443
+
<tr class="odd">
444
+
<td><a href="#atdata.AbstractDataStore.write_shards">write_shards</a></td>
445
+
<td>Write dataset shards to storage.</td>
446
+
</tr>
447
+
</tbody>
448
+
</table>
449
+
<section id="atdata.AbstractDataStore.read_url" class="level3">
450
+
<h3 class="anchored" data-anchor-id="atdata.AbstractDataStore.read_url">read_url</h3>
451
+
<div class="sourceCode" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a>AbstractDataStore.read_url(url)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
452
+
<p>Resolve a storage URL for reading.</p>
453
+
<p>Some storage backends may need to transform URLs (e.g., signing S3 URLs or resolving blob references). This method returns a URL that can be used directly with WebDataset.</p>
454
+
<p>Args: url: Storage URL to resolve.</p>
455
+
<p>Returns: WebDataset-compatible URL for reading.</p>
456
+
</section>
457
+
<section id="atdata.AbstractDataStore.supports_streaming" class="level3">
458
+
<h3 class="anchored" data-anchor-id="atdata.AbstractDataStore.supports_streaming">supports_streaming</h3>
459
+
<div class="sourceCode" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a>AbstractDataStore.supports_streaming()</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
460
+
<p>Whether this store supports streaming reads.</p>
461
+
<p>Returns: True if the store supports efficient streaming (like S3), False if data must be fully downloaded first.</p>
462
+
</section>
463
+
<section id="atdata.AbstractDataStore.write_shards" class="level3">
464
+
<h3 class="anchored" data-anchor-id="atdata.AbstractDataStore.write_shards">write_shards</h3>
465
+
<div class="sourceCode" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a>AbstractDataStore.write_shards(ds, <span class="op">*</span>, prefix, <span class="op">**</span>kwargs)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
466
+
<p>Write dataset shards to storage.</p>
467
+
<p>Args: ds: The Dataset to write. prefix: Path prefix for the shards (e.g., ‘datasets/mnist/v1’). **kwargs: Backend-specific options (e.g., maxcount for shard size).</p>
468
+
<p>Returns: List of URLs for the written shards, suitable for use with WebDataset or atdata.Dataset().</p>
469
+
470
+
471
+
</section>
472
+
</section>
473
+
</section>
474
+
475
+
</main> <!-- /main -->
476
+
<script id="quarto-html-after-body" type="application/javascript">
477
+
window.document.addEventListener("DOMContentLoaded", function (event) {
478
+
// Ensure there is a toggle, if there isn't float one in the top right
479
+
if (window.document.querySelector('.quarto-color-scheme-toggle') === null) {
480
+
const a = window.document.createElement('a');
481
+
a.classList.add('top-right');
482
+
a.classList.add('quarto-color-scheme-toggle');
483
+
a.href = "";
484
+
a.onclick = function() { try { window.quartoToggleColorScheme(); } catch {} return false; };
485
+
const i = window.document.createElement("i");
486
+
i.classList.add('bi');
487
+
a.appendChild(i);
488
+
window.document.body.appendChild(a);
489
+
}
490
+
setColorSchemeToggle(hasAlternateSentinel())
491
+
const icon = "";
492
+
const anchorJS = new window.AnchorJS();
493
+
anchorJS.options = {
494
+
placement: 'right',
495
+
icon: icon
496
+
};
497
+
anchorJS.add('.anchored');
498
+
const isCodeAnnotation = (el) => {
499
+
for (const clz of el.classList) {
500
+
if (clz.startsWith('code-annotation-')) {
501
+
return true;
502
+
}
503
+
}
504
+
return false;
505
+
}
506
+
const onCopySuccess = function(e) {
507
+
// button target
508
+
const button = e.trigger;
509
+
// don't keep focus
510
+
button.blur();
511
+
// flash "checked"
512
+
button.classList.add('code-copy-button-checked');
513
+
var currentTitle = button.getAttribute("title");
514
+
button.setAttribute("title", "Copied!");
515
+
let tooltip;
516
+
if (window.bootstrap) {
517
+
button.setAttribute("data-bs-toggle", "tooltip");
518
+
button.setAttribute("data-bs-placement", "left");
519
+
button.setAttribute("data-bs-title", "Copied!");
520
+
tooltip = new bootstrap.Tooltip(button,
521
+
{ trigger: "manual",
522
+
customClass: "code-copy-button-tooltip",
523
+
offset: [0, -8]});
524
+
tooltip.show();
525
+
}
526
+
setTimeout(function() {
527
+
if (tooltip) {
528
+
tooltip.hide();
529
+
button.removeAttribute("data-bs-title");
530
+
button.removeAttribute("data-bs-toggle");
531
+
button.removeAttribute("data-bs-placement");
532
+
}
533
+
button.setAttribute("title", currentTitle);
534
+
button.classList.remove('code-copy-button-checked');
535
+
}, 1000);
536
+
// clear code selection
537
+
e.clearSelection();
538
+
}
539
+
const getTextToCopy = function(trigger) {
540
+
const codeEl = trigger.previousElementSibling.cloneNode(true);
541
+
for (const childEl of codeEl.children) {
542
+
if (isCodeAnnotation(childEl)) {
543
+
childEl.remove();
544
+
}
545
+
}
546
+
return codeEl.innerText;
547
+
}
548
+
const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
549
+
text: getTextToCopy
550
+
});
551
+
clipboard.on('success', onCopySuccess);
552
+
if (window.document.getElementById('quarto-embedded-source-code-modal')) {
553
+
const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', {
554
+
text: getTextToCopy,
555
+
container: window.document.getElementById('quarto-embedded-source-code-modal')
556
+
});
557
+
clipboardModal.on('success', onCopySuccess);
558
+
}
559
+
var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//);
560
+
var mailtoRegex = new RegExp(/^mailto:/);
561
+
var filterRegex = new RegExp("https:\/\/github\.com\/your-org\/atdata");
562
+
var isInternal = (href) => {
563
+
return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href);
564
+
}
565
+
// Inspect non-navigation links and adorn them if external
566
+
var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)');
567
+
for (var i=0; i<links.length; i++) {
568
+
const link = links[i];
569
+
if (!isInternal(link.href)) {
570
+
// undo the damage that might have been done by quarto-nav.js in the case of
571
+
// links that we want to consider external
572
+
if (link.dataset.originalHref !== undefined) {
573
+
link.href = link.dataset.originalHref;
574
+
}
575
+
}
576
+
}
577
+
function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
578
+
const config = {
579
+
allowHTML: true,
580
+
maxWidth: 500,
581
+
delay: 100,
582
+
arrow: false,
583
+
appendTo: function(el) {
584
+
return el.parentElement;
585
+
},
586
+
interactive: true,
587
+
interactiveBorder: 10,
588
+
theme: 'quarto',
589
+
placement: 'bottom-start',
590
+
};
591
+
if (contentFn) {
592
+
config.content = contentFn;
593
+
}
594
+
if (onTriggerFn) {
595
+
config.onTrigger = onTriggerFn;
596
+
}
597
+
if (onUntriggerFn) {
598
+
config.onUntrigger = onUntriggerFn;
599
+
}
600
+
window.tippy(el, config);
601
+
}
602
+
const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
603
+
for (var i=0; i<noterefs.length; i++) {
604
+
const ref = noterefs[i];
605
+
tippyHover(ref, function() {
606
+
// use id or data attribute instead here
607
+
let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
608
+
try { href = new URL(href).hash; } catch {}
609
+
const id = href.replace(/^#\/?/, "");
610
+
const note = window.document.getElementById(id);
611
+
if (note) {
612
+
return note.innerHTML;
613
+
} else {
614
+
return "";
615
+
}
616
+
});
617
+
}
618
+
const xrefs = window.document.querySelectorAll('a.quarto-xref');
619
+
const processXRef = (id, note) => {
620
+
// Strip column container classes
621
+
const stripColumnClz = (el) => {
622
+
el.classList.remove("page-full", "page-columns");
623
+
if (el.children) {
624
+
for (const child of el.children) {
625
+
stripColumnClz(child);
626
+
}
627
+
}
628
+
}
629
+
stripColumnClz(note)
630
+
if (id === null || id.startsWith('sec-')) {
631
+
// Special case sections, only their first couple elements
632
+
const container = document.createElement("div");
633
+
if (note.children && note.children.length > 2) {
634
+
container.appendChild(note.children[0].cloneNode(true));
635
+
for (let i = 1; i < note.children.length; i++) {
636
+
const child = note.children[i];
637
+
if (child.tagName === "P" && child.innerText === "") {
638
+
continue;
639
+
} else {
640
+
container.appendChild(child.cloneNode(true));
641
+
break;
642
+
}
643
+
}
644
+
if (window.Quarto?.typesetMath) {
645
+
window.Quarto.typesetMath(container);
646
+
}
647
+
return container.innerHTML
648
+
} else {
649
+
if (window.Quarto?.typesetMath) {
650
+
window.Quarto.typesetMath(note);
651
+
}
652
+
return note.innerHTML;
653
+
}
654
+
} else {
655
+
// Remove any anchor links if they are present
656
+
const anchorLink = note.querySelector('a.anchorjs-link');
657
+
if (anchorLink) {
658
+
anchorLink.remove();
659
+
}
660
+
if (window.Quarto?.typesetMath) {
661
+
window.Quarto.typesetMath(note);
662
+
}
663
+
if (note.classList.contains("callout")) {
664
+
return note.outerHTML;
665
+
} else {
666
+
return note.innerHTML;
667
+
}
668
+
}
669
+
}
670
+
for (var i=0; i<xrefs.length; i++) {
671
+
const xref = xrefs[i];
672
+
tippyHover(xref, undefined, function(instance) {
673
+
instance.disable();
674
+
let url = xref.getAttribute('href');
675
+
let hash = undefined;
676
+
if (url.startsWith('#')) {
677
+
hash = url;
678
+
} else {
679
+
try { hash = new URL(url).hash; } catch {}
680
+
}
681
+
if (hash) {
682
+
const id = hash.replace(/^#\/?/, "");
683
+
const note = window.document.getElementById(id);
684
+
if (note !== null) {
685
+
try {
686
+
const html = processXRef(id, note.cloneNode(true));
687
+
instance.setContent(html);
688
+
} finally {
689
+
instance.enable();
690
+
instance.show();
691
+
}
692
+
} else {
693
+
// See if we can fetch this
694
+
fetch(url.split('#')[0])
695
+
.then(res => res.text())
696
+
.then(html => {
697
+
const parser = new DOMParser();
698
+
const htmlDoc = parser.parseFromString(html, "text/html");
699
+
const note = htmlDoc.getElementById(id);
700
+
if (note !== null) {
701
+
const html = processXRef(id, note);
702
+
instance.setContent(html);
703
+
}
704
+
}).finally(() => {
705
+
instance.enable();
706
+
instance.show();
707
+
});
708
+
}
709
+
} else {
710
+
// See if we can fetch a full url (with no hash to target)
711
+
// This is a special case and we should probably do some content thinning / targeting
712
+
fetch(url)
713
+
.then(res => res.text())
714
+
.then(html => {
715
+
const parser = new DOMParser();
716
+
const htmlDoc = parser.parseFromString(html, "text/html");
717
+
const note = htmlDoc.querySelector('main.content');
718
+
if (note !== null) {
719
+
// This should only happen for chapter cross references
720
+
// (since there is no id in the URL)
721
+
// remove the first header
722
+
if (note.children.length > 0 && note.children[0].tagName === "HEADER") {
723
+
note.children[0].remove();
724
+
}
725
+
const html = processXRef(null, note);
726
+
instance.setContent(html);
727
+
}
728
+
}).finally(() => {
729
+
instance.enable();
730
+
instance.show();
731
+
});
732
+
}
733
+
}, function(instance) {
734
+
});
735
+
}
736
+
let selectedAnnoteEl;
737
+
const selectorForAnnotation = ( cell, annotation) => {
738
+
let cellAttr = 'data-code-cell="' + cell + '"';
739
+
let lineAttr = 'data-code-annotation="' + annotation + '"';
740
+
const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
741
+
return selector;
742
+
}
743
+
const selectCodeLines = (annoteEl) => {
744
+
const doc = window.document;
745
+
const targetCell = annoteEl.getAttribute("data-target-cell");
746
+
const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
747
+
const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
748
+
const lines = annoteSpan.getAttribute("data-code-lines").split(",");
749
+
const lineIds = lines.map((line) => {
750
+
return targetCell + "-" + line;
751
+
})
752
+
let top = null;
753
+
let height = null;
754
+
let parent = null;
755
+
if (lineIds.length > 0) {
756
+
//compute the position of the single el (top and bottom and make a div)
757
+
const el = window.document.getElementById(lineIds[0]);
758
+
top = el.offsetTop;
759
+
height = el.offsetHeight;
760
+
parent = el.parentElement.parentElement;
761
+
if (lineIds.length > 1) {
762
+
const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
763
+
const bottom = lastEl.offsetTop + lastEl.offsetHeight;
764
+
height = bottom - top;
765
+
}
766
+
if (top !== null && height !== null && parent !== null) {
767
+
// cook up a div (if necessary) and position it
768
+
let div = window.document.getElementById("code-annotation-line-highlight");
769
+
if (div === null) {
770
+
div = window.document.createElement("div");
771
+
div.setAttribute("id", "code-annotation-line-highlight");
772
+
div.style.position = 'absolute';
773
+
parent.appendChild(div);
774
+
}
775
+
div.style.top = top - 2 + "px";
776
+
div.style.height = height + 4 + "px";
777
+
div.style.left = 0;
778
+
let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
779
+
if (gutterDiv === null) {
780
+
gutterDiv = window.document.createElement("div");
781
+
gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
782
+
gutterDiv.style.position = 'absolute';
783
+
const codeCell = window.document.getElementById(targetCell);
784
+
const gutter = codeCell.querySelector('.code-annotation-gutter');
785
+
gutter.appendChild(gutterDiv);
786
+
}
787
+
gutterDiv.style.top = top - 2 + "px";
788
+
gutterDiv.style.height = height + 4 + "px";
789
+
}
790
+
selectedAnnoteEl = annoteEl;
791
+
}
792
+
};
793
+
const unselectCodeLines = () => {
794
+
const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
795
+
elementsIds.forEach((elId) => {
796
+
const div = window.document.getElementById(elId);
797
+
if (div) {
798
+
div.remove();
799
+
}
800
+
});
801
+
selectedAnnoteEl = undefined;
802
+
};
803
+
// Handle positioning of the toggle
804
+
window.addEventListener(
805
+
"resize",
806
+
throttle(() => {
807
+
elRect = undefined;
808
+
if (selectedAnnoteEl) {
809
+
selectCodeLines(selectedAnnoteEl);
810
+
}
811
+
}, 10)
812
+
);
813
+
function throttle(fn, ms) {
814
+
let throttle = false;
815
+
let timer;
816
+
return (...args) => {
817
+
if(!throttle) { // first call gets through
818
+
fn.apply(this, args);
819
+
throttle = true;
820
+
} else { // all the others get throttled
821
+
if(timer) clearTimeout(timer); // cancel #2
822
+
timer = setTimeout(() => {
823
+
fn.apply(this, args);
824
+
timer = throttle = false;
825
+
}, ms);
826
+
}
827
+
};
828
+
}
829
+
// Attach click handler to the DT
830
+
const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
831
+
for (const annoteDlNode of annoteDls) {
832
+
annoteDlNode.addEventListener('click', (event) => {
833
+
const clickedEl = event.target;
834
+
if (clickedEl !== selectedAnnoteEl) {
835
+
unselectCodeLines();
836
+
const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
837
+
if (activeEl) {
838
+
activeEl.classList.remove('code-annotation-active');
839
+
}
840
+
selectCodeLines(clickedEl);
841
+
clickedEl.classList.add('code-annotation-active');
842
+
} else {
843
+
// Unselect the line
844
+
unselectCodeLines();
845
+
clickedEl.classList.remove('code-annotation-active');
846
+
}
847
+
});
848
+
}
849
+
const findCites = (el) => {
850
+
const parentEl = el.parentElement;
851
+
if (parentEl) {
852
+
const cites = parentEl.dataset.cites;
853
+
if (cites) {
854
+
return {
855
+
el,
856
+
cites: cites.split(' ')
857
+
};
858
+
} else {
859
+
return findCites(el.parentElement)
860
+
}
861
+
} else {
862
+
return undefined;
863
+
}
864
+
};
865
+
var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
866
+
for (var i=0; i<bibliorefs.length; i++) {
867
+
const ref = bibliorefs[i];
868
+
const citeInfo = findCites(ref);
869
+
if (citeInfo) {
870
+
tippyHover(citeInfo.el, function() {
871
+
var popup = window.document.createElement('div');
872
+
citeInfo.cites.forEach(function(cite) {
873
+
var citeDiv = window.document.createElement('div');
874
+
citeDiv.classList.add('hanging-indent');
875
+
citeDiv.classList.add('csl-entry');
876
+
var biblioDiv = window.document.getElementById('ref-' + cite);
877
+
if (biblioDiv) {
878
+
citeDiv.innerHTML = biblioDiv.innerHTML;
879
+
}
880
+
popup.appendChild(citeDiv);
881
+
});
882
+
return popup.innerHTML;
883
+
});
884
+
}
885
+
}
886
+
});
887
+
</script>
888
+
</div> <!-- /content -->
889
+
<footer class="footer">
890
+
<div class="nav-footer">
891
+
<div class="nav-footer-left">
892
+
<p>Built with <a href="https://quarto.org/">Quarto</a></p>
893
+
</div>
894
+
<div class="nav-footer-center">
895
+
896
+
<div class="toc-actions d-sm-block d-md-none"><ul><li><a href="https://github.com/your-org/atdata/edit/main/api/AbstractDataStore.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></div>
897
+
<div class="nav-footer-right">
898
+
<p>MIT License</p>
899
+
</div>
900
+
</div>
901
+
</footer>
902
+
903
+
904
+
905
+
906
+
</body></html>
+981
docs/api/AbstractIndex.html
+981
docs/api/AbstractIndex.html
···
1
+
<!DOCTYPE html>
2
+
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
3
+
4
+
<meta charset="utf-8">
5
+
<meta name="generator" content="quarto-1.7.34">
6
+
7
+
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
8
+
9
+
10
+
<title>abstractindex – atdata</title>
11
+
<style>
12
+
code{white-space: pre-wrap;}
13
+
span.smallcaps{font-variant: small-caps;}
14
+
div.columns{display: flex; gap: min(4vw, 1.5em);}
15
+
div.column{flex: auto; overflow-x: auto;}
16
+
div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
17
+
ul.task-list{list-style: none;}
18
+
ul.task-list li input[type="checkbox"] {
19
+
width: 0.8em;
20
+
margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
21
+
vertical-align: middle;
22
+
}
23
+
/* CSS for syntax highlighting */
24
+
html { -webkit-text-size-adjust: 100%; }
25
+
pre > code.sourceCode { white-space: pre; position: relative; }
26
+
pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
27
+
pre > code.sourceCode > span:empty { height: 1.2em; }
28
+
.sourceCode { overflow: visible; }
29
+
code.sourceCode > span { color: inherit; text-decoration: inherit; }
30
+
div.sourceCode { margin: 1em 0; }
31
+
pre.sourceCode { margin: 0; }
32
+
@media screen {
33
+
div.sourceCode { overflow: auto; }
34
+
}
35
+
@media print {
36
+
pre > code.sourceCode { white-space: pre-wrap; }
37
+
pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
38
+
}
39
+
pre.numberSource code
40
+
{ counter-reset: source-line 0; }
41
+
pre.numberSource code > span
42
+
{ position: relative; left: -4em; counter-increment: source-line; }
43
+
pre.numberSource code > span > a:first-child::before
44
+
{ content: counter(source-line);
45
+
position: relative; left: -1em; text-align: right; vertical-align: baseline;
46
+
border: none; display: inline-block;
47
+
-webkit-touch-callout: none; -webkit-user-select: none;
48
+
-khtml-user-select: none; -moz-user-select: none;
49
+
-ms-user-select: none; user-select: none;
50
+
padding: 0 4px; width: 4em;
51
+
}
52
+
pre.numberSource { margin-left: 3em; padding-left: 4px; }
53
+
div.sourceCode
54
+
{ }
55
+
@media screen {
56
+
pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
57
+
}
58
+
</style>
59
+
60
+
61
+
<script src="../site_libs/quarto-nav/quarto-nav.js"></script>
62
+
<script src="../site_libs/quarto-nav/headroom.min.js"></script>
63
+
<script src="../site_libs/clipboard/clipboard.min.js"></script>
64
+
<script src="../site_libs/quarto-search/autocomplete.umd.js"></script>
65
+
<script src="../site_libs/quarto-search/fuse.min.js"></script>
66
+
<script src="../site_libs/quarto-search/quarto-search.js"></script>
67
+
<meta name="quarto:offset" content="../">
68
+
<script src="../site_libs/quarto-html/quarto.js" type="module"></script>
69
+
<script src="../site_libs/quarto-html/tabsets/tabsets.js" type="module"></script>
70
+
<script src="../site_libs/quarto-html/popper.min.js"></script>
71
+
<script src="../site_libs/quarto-html/tippy.umd.min.js"></script>
72
+
<script src="../site_libs/quarto-html/anchor.min.js"></script>
73
+
<link href="../site_libs/quarto-html/tippy.css" rel="stylesheet">
74
+
<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme" id="quarto-text-highlighting-styles">
75
+
<link href="../site_libs/quarto-html/quarto-syntax-highlighting-dark-8dcd8563ea6803ab7cbb3d71ca5772e1.css" rel="stylesheet" class="quarto-color-scheme quarto-color-alternate" id="quarto-text-highlighting-styles">
76
+
<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme-extra" id="quarto-text-highlighting-styles">
77
+
<script src="../site_libs/bootstrap/bootstrap.min.js"></script>
78
+
<link href="../site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
79
+
<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme" id="quarto-bootstrap" data-mode="light">
80
+
<link href="../site_libs/bootstrap/bootstrap-dark-7964ffd8887b0991fe8d71c6c8bc75d6.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme quarto-color-alternate" id="quarto-bootstrap" data-mode="dark">
81
+
<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme-extra" id="quarto-bootstrap" data-mode="light">
82
+
<script id="quarto-search-options" type="application/json">{
83
+
"location": "navbar",
84
+
"copy-button": false,
85
+
"collapse-after": 3,
86
+
"panel-placement": "end",
87
+
"type": "overlay",
88
+
"limit": 50,
89
+
"keyboard-shortcut": [
90
+
"f",
91
+
"/",
92
+
"s"
93
+
],
94
+
"show-item-context": false,
95
+
"language": {
96
+
"search-no-results-text": "No results",
97
+
"search-matching-documents-text": "matching documents",
98
+
"search-copy-link-title": "Copy link to search",
99
+
"search-hide-matches-text": "Hide additional matches",
100
+
"search-more-match-text": "more match in this document",
101
+
"search-more-matches-text": "more matches in this document",
102
+
"search-clear-button-title": "Clear",
103
+
"search-text-placeholder": "",
104
+
"search-detached-cancel-button-title": "Cancel",
105
+
"search-submit-button-title": "Submit",
106
+
"search-label": "Search"
107
+
}
108
+
}</script>
109
+
110
+
111
+
<link rel="stylesheet" href="../assets/styles.css">
112
+
</head>
113
+
114
+
<body class="nav-fixed quarto-light"><script id="quarto-html-before-body" type="application/javascript">
115
+
const toggleBodyColorMode = (bsSheetEl) => {
116
+
const mode = bsSheetEl.getAttribute("data-mode");
117
+
const bodyEl = window.document.querySelector("body");
118
+
if (mode === "dark") {
119
+
bodyEl.classList.add("quarto-dark");
120
+
bodyEl.classList.remove("quarto-light");
121
+
} else {
122
+
bodyEl.classList.add("quarto-light");
123
+
bodyEl.classList.remove("quarto-dark");
124
+
}
125
+
}
126
+
const toggleBodyColorPrimary = () => {
127
+
const bsSheetEl = window.document.querySelector("link#quarto-bootstrap:not([rel=disabled-stylesheet])");
128
+
if (bsSheetEl) {
129
+
toggleBodyColorMode(bsSheetEl);
130
+
}
131
+
}
132
+
const setColorSchemeToggle = (alternate) => {
133
+
const toggles = window.document.querySelectorAll('.quarto-color-scheme-toggle');
134
+
for (let i=0; i < toggles.length; i++) {
135
+
const toggle = toggles[i];
136
+
if (toggle) {
137
+
if (alternate) {
138
+
toggle.classList.add("alternate");
139
+
} else {
140
+
toggle.classList.remove("alternate");
141
+
}
142
+
}
143
+
}
144
+
};
145
+
const toggleColorMode = (alternate) => {
146
+
// Switch the stylesheets
147
+
const primaryStylesheets = window.document.querySelectorAll('link.quarto-color-scheme:not(.quarto-color-alternate)');
148
+
const alternateStylesheets = window.document.querySelectorAll('link.quarto-color-scheme.quarto-color-alternate');
149
+
manageTransitions('#quarto-margin-sidebar .nav-link', false);
150
+
if (alternate) {
151
+
// note: dark is layered on light, we don't disable primary!
152
+
enableStylesheet(alternateStylesheets);
153
+
for (const sheetNode of alternateStylesheets) {
154
+
if (sheetNode.id === "quarto-bootstrap") {
155
+
toggleBodyColorMode(sheetNode);
156
+
}
157
+
}
158
+
} else {
159
+
disableStylesheet(alternateStylesheets);
160
+
enableStylesheet(primaryStylesheets)
161
+
toggleBodyColorPrimary();
162
+
}
163
+
manageTransitions('#quarto-margin-sidebar .nav-link', true);
164
+
// Switch the toggles
165
+
setColorSchemeToggle(alternate)
166
+
// Hack to workaround the fact that safari doesn't
167
+
// properly recolor the scrollbar when toggling (#1455)
168
+
if (navigator.userAgent.indexOf('Safari') > 0 && navigator.userAgent.indexOf('Chrome') == -1) {
169
+
manageTransitions("body", false);
170
+
window.scrollTo(0, 1);
171
+
setTimeout(() => {
172
+
window.scrollTo(0, 0);
173
+
manageTransitions("body", true);
174
+
}, 40);
175
+
}
176
+
}
177
+
const disableStylesheet = (stylesheets) => {
178
+
for (let i=0; i < stylesheets.length; i++) {
179
+
const stylesheet = stylesheets[i];
180
+
stylesheet.rel = 'disabled-stylesheet';
181
+
}
182
+
}
183
+
const enableStylesheet = (stylesheets) => {
184
+
for (let i=0; i < stylesheets.length; i++) {
185
+
const stylesheet = stylesheets[i];
186
+
if(stylesheet.rel !== 'stylesheet') { // for Chrome, which will still FOUC without this check
187
+
stylesheet.rel = 'stylesheet';
188
+
}
189
+
}
190
+
}
191
+
const manageTransitions = (selector, allowTransitions) => {
192
+
const els = window.document.querySelectorAll(selector);
193
+
for (let i=0; i < els.length; i++) {
194
+
const el = els[i];
195
+
if (allowTransitions) {
196
+
el.classList.remove('notransition');
197
+
} else {
198
+
el.classList.add('notransition');
199
+
}
200
+
}
201
+
}
202
+
const isFileUrl = () => {
203
+
return window.location.protocol === 'file:';
204
+
}
205
+
const hasAlternateSentinel = () => {
206
+
let styleSentinel = getColorSchemeSentinel();
207
+
if (styleSentinel !== null) {
208
+
return styleSentinel === "alternate";
209
+
} else {
210
+
return false;
211
+
}
212
+
}
213
+
const setStyleSentinel = (alternate) => {
214
+
const value = alternate ? "alternate" : "default";
215
+
if (!isFileUrl()) {
216
+
window.localStorage.setItem("quarto-color-scheme", value);
217
+
} else {
218
+
localAlternateSentinel = value;
219
+
}
220
+
}
221
+
const getColorSchemeSentinel = () => {
222
+
if (!isFileUrl()) {
223
+
const storageValue = window.localStorage.getItem("quarto-color-scheme");
224
+
return storageValue != null ? storageValue : localAlternateSentinel;
225
+
} else {
226
+
return localAlternateSentinel;
227
+
}
228
+
}
229
+
const toggleGiscusIfUsed = (isAlternate, darkModeDefault) => {
230
+
const baseTheme = document.querySelector('#giscus-base-theme')?.value ?? 'light';
231
+
const alternateTheme = document.querySelector('#giscus-alt-theme')?.value ?? 'dark';
232
+
let newTheme = '';
233
+
if(authorPrefersDark) {
234
+
newTheme = isAlternate ? baseTheme : alternateTheme;
235
+
} else {
236
+
newTheme = isAlternate ? alternateTheme : baseTheme;
237
+
}
238
+
const changeGiscusTheme = () => {
239
+
// From: https://github.com/giscus/giscus/issues/336
240
+
const sendMessage = (message) => {
241
+
const iframe = document.querySelector('iframe.giscus-frame');
242
+
if (!iframe) return;
243
+
iframe.contentWindow.postMessage({ giscus: message }, 'https://giscus.app');
244
+
}
245
+
sendMessage({
246
+
setConfig: {
247
+
theme: newTheme
248
+
}
249
+
});
250
+
}
251
+
const isGiscussLoaded = window.document.querySelector('iframe.giscus-frame') !== null;
252
+
if (isGiscussLoaded) {
253
+
changeGiscusTheme();
254
+
}
255
+
};
256
+
const authorPrefersDark = false;
257
+
const darkModeDefault = authorPrefersDark;
258
+
document.querySelector('link#quarto-text-highlighting-styles.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
259
+
document.querySelector('link#quarto-bootstrap.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
260
+
let localAlternateSentinel = darkModeDefault ? 'alternate' : 'default';
261
+
// Dark / light mode switch
262
+
window.quartoToggleColorScheme = () => {
263
+
// Read the current dark / light value
264
+
let toAlternate = !hasAlternateSentinel();
265
+
toggleColorMode(toAlternate);
266
+
setStyleSentinel(toAlternate);
267
+
toggleGiscusIfUsed(toAlternate, darkModeDefault);
268
+
window.dispatchEvent(new Event('resize'));
269
+
};
270
+
// Switch to dark mode if need be
271
+
if (hasAlternateSentinel()) {
272
+
toggleColorMode(true);
273
+
} else {
274
+
toggleColorMode(false);
275
+
}
276
+
</script>
277
+
278
+
<div id="quarto-search-results"></div>
279
+
<header id="quarto-header" class="headroom fixed-top">
280
+
<nav class="navbar navbar-expand-lg " data-bs-theme="dark">
281
+
<div class="navbar-container container-fluid">
282
+
<div class="navbar-brand-container mx-auto">
283
+
<a class="navbar-brand" href="../index.html">
284
+
<span class="navbar-title">atdata</span>
285
+
</a>
286
+
</div>
287
+
<div id="quarto-search" class="" title="Search"></div>
288
+
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarCollapse" aria-controls="navbarCollapse" role="menu" aria-expanded="false" aria-label="Toggle navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
289
+
<span class="navbar-toggler-icon"></span>
290
+
</button>
291
+
<div class="collapse navbar-collapse" id="navbarCollapse">
292
+
<ul class="navbar-nav navbar-nav-scroll me-auto">
293
+
<li class="nav-item">
294
+
<a class="nav-link" href="../index.html">
295
+
<span class="menu-text">Guide</span></a>
296
+
</li>
297
+
<li class="nav-item dropdown ">
298
+
<a class="nav-link dropdown-toggle" href="#" id="nav-menu-tutorials" role="link" data-bs-toggle="dropdown" aria-expanded="false">
299
+
<span class="menu-text">Tutorials</span>
300
+
</a>
301
+
<ul class="dropdown-menu" aria-labelledby="nav-menu-tutorials">
302
+
<li>
303
+
<a class="dropdown-item" href="../tutorials/quickstart.html">
304
+
<span class="dropdown-text">Quick Start</span></a>
305
+
</li>
306
+
<li>
307
+
<a class="dropdown-item" href="../tutorials/local-workflow.html">
308
+
<span class="dropdown-text">Local Workflow</span></a>
309
+
</li>
310
+
<li>
311
+
<a class="dropdown-item" href="../tutorials/atmosphere.html">
312
+
<span class="dropdown-text">Atmosphere Publishing</span></a>
313
+
</li>
314
+
<li>
315
+
<a class="dropdown-item" href="../tutorials/promotion.html">
316
+
<span class="dropdown-text">Promotion Workflow</span></a>
317
+
</li>
318
+
</ul>
319
+
</li>
320
+
<li class="nav-item dropdown ">
321
+
<a class="nav-link dropdown-toggle" href="#" id="nav-menu-reference" role="link" data-bs-toggle="dropdown" aria-expanded="false">
322
+
<span class="menu-text">Reference</span>
323
+
</a>
324
+
<ul class="dropdown-menu" aria-labelledby="nav-menu-reference">
325
+
<li>
326
+
<a class="dropdown-item" href="../reference/packable-samples.html">
327
+
<span class="dropdown-text">Packable Samples</span></a>
328
+
</li>
329
+
<li>
330
+
<a class="dropdown-item" href="../reference/datasets.html">
331
+
<span class="dropdown-text">Datasets</span></a>
332
+
</li>
333
+
<li>
334
+
<a class="dropdown-item" href="../reference/lenses.html">
335
+
<span class="dropdown-text">Lenses</span></a>
336
+
</li>
337
+
<li>
338
+
<a class="dropdown-item" href="../reference/local-storage.html">
339
+
<span class="dropdown-text">Local Storage</span></a>
340
+
</li>
341
+
<li>
342
+
<a class="dropdown-item" href="../reference/atmosphere.html">
343
+
<span class="dropdown-text">Atmosphere</span></a>
344
+
</li>
345
+
<li>
346
+
<a class="dropdown-item" href="../reference/promotion.html">
347
+
<span class="dropdown-text">Promotion</span></a>
348
+
</li>
349
+
<li>
350
+
<a class="dropdown-item" href="../reference/load-dataset.html">
351
+
<span class="dropdown-text">load_dataset API</span></a>
352
+
</li>
353
+
<li>
354
+
<a class="dropdown-item" href="../reference/protocols.html">
355
+
<span class="dropdown-text">Protocols</span></a>
356
+
</li>
357
+
<li>
358
+
<a class="dropdown-item" href="../reference/uri-spec.html">
359
+
<span class="dropdown-text">URI Specification</span></a>
360
+
</li>
361
+
<li>
362
+
<a class="dropdown-item" href="../reference/troubleshooting.html">
363
+
<span class="dropdown-text">Troubleshooting & FAQ</span></a>
364
+
</li>
365
+
<li>
366
+
<a class="dropdown-item" href="../reference/deployment.html">
367
+
<span class="dropdown-text">Deployment Guide</span></a>
368
+
</li>
369
+
</ul>
370
+
</li>
371
+
<li class="nav-item">
372
+
<a class="nav-link" href="../api/index.html">
373
+
<span class="menu-text">API</span></a>
374
+
</li>
375
+
</ul>
376
+
<ul class="navbar-nav navbar-nav-scroll ms-auto">
377
+
<li class="nav-item compact">
378
+
<a class="nav-link" href="https://github.com/your-org/atdata"> <i class="bi bi-github" role="img">
379
+
</i>
380
+
<span class="menu-text"></span></a>
381
+
</li>
382
+
</ul>
383
+
</div> <!-- /navcollapse -->
384
+
<div class="quarto-navbar-tools">
385
+
<a href="" class="quarto-color-scheme-toggle quarto-navigation-tool px-1" onclick="window.quartoToggleColorScheme(); return false;" title="Toggle dark mode"><i class="bi"></i></a>
386
+
</div>
387
+
</div> <!-- /container-fluid -->
388
+
</nav>
389
+
</header>
390
+
<!-- content -->
391
+
<div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-article page-navbar">
392
+
<!-- sidebar -->
393
+
<!-- margin-sidebar -->
394
+
<div id="quarto-margin-sidebar" class="sidebar margin-sidebar">
395
+
<nav id="TOC" role="doc-toc" class="toc-active">
396
+
<h2 id="toc-title">On this page</h2>
397
+
398
+
<ul>
399
+
<li><a href="#atdata.AbstractIndex" id="toc-atdata.AbstractIndex" class="nav-link active" data-scroll-target="#atdata.AbstractIndex">AbstractIndex</a>
400
+
<ul class="collapse">
401
+
<li><a href="#attributes" id="toc-attributes" class="nav-link" data-scroll-target="#attributes">Attributes</a></li>
402
+
<li><a href="#methods" id="toc-methods" class="nav-link" data-scroll-target="#methods">Methods</a>
403
+
<ul class="collapse">
404
+
<li><a href="#atdata.AbstractIndex.decode_schema" id="toc-atdata.AbstractIndex.decode_schema" class="nav-link" data-scroll-target="#atdata.AbstractIndex.decode_schema">decode_schema</a></li>
405
+
<li><a href="#atdata.AbstractIndex.get_dataset" id="toc-atdata.AbstractIndex.get_dataset" class="nav-link" data-scroll-target="#atdata.AbstractIndex.get_dataset">get_dataset</a></li>
406
+
<li><a href="#atdata.AbstractIndex.get_schema" id="toc-atdata.AbstractIndex.get_schema" class="nav-link" data-scroll-target="#atdata.AbstractIndex.get_schema">get_schema</a></li>
407
+
<li><a href="#atdata.AbstractIndex.insert_dataset" id="toc-atdata.AbstractIndex.insert_dataset" class="nav-link" data-scroll-target="#atdata.AbstractIndex.insert_dataset">insert_dataset</a></li>
408
+
<li><a href="#atdata.AbstractIndex.list_datasets" id="toc-atdata.AbstractIndex.list_datasets" class="nav-link" data-scroll-target="#atdata.AbstractIndex.list_datasets">list_datasets</a></li>
409
+
<li><a href="#atdata.AbstractIndex.list_schemas" id="toc-atdata.AbstractIndex.list_schemas" class="nav-link" data-scroll-target="#atdata.AbstractIndex.list_schemas">list_schemas</a></li>
410
+
<li><a href="#atdata.AbstractIndex.publish_schema" id="toc-atdata.AbstractIndex.publish_schema" class="nav-link" data-scroll-target="#atdata.AbstractIndex.publish_schema">publish_schema</a></li>
411
+
</ul></li>
412
+
</ul></li>
413
+
</ul>
414
+
<div class="toc-actions"><ul><li><a href="https://github.com/your-org/atdata/edit/main/api/AbstractIndex.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></nav>
415
+
</div>
416
+
<!-- main -->
417
+
<main class="content" id="quarto-document-content"><header id="title-block-header" class="quarto-title-block"></header>
418
+
419
+
420
+
421
+
422
+
423
+
<section id="atdata.AbstractIndex" class="level1">
424
+
<h1>AbstractIndex</h1>
425
+
<div class="sourceCode" id="cb1"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a>AbstractIndex()</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
426
+
<p>Protocol for index operations - implemented by LocalIndex and AtmosphereIndex.</p>
427
+
<p>This protocol defines the common interface for managing dataset metadata: - Publishing and retrieving schemas - Inserting and listing datasets - (Future) Publishing and retrieving lenses</p>
428
+
<p>A single index can hold datasets of many different sample types. The sample type is tracked via schema references, not as a generic parameter on the index.</p>
429
+
<p>Optional Extensions: Some index implementations support additional features: - <code>data_store</code>: An AbstractDataStore for reading/writing dataset shards. If present, <code>load_dataset</code> will use it for S3 credential resolution.</p>
430
+
<p>Example: >>> def publish_and_list(index: AbstractIndex) -> None: … # Publish schemas for different types … schema1 = index.publish_schema(ImageSample, version=“1.0.0”) … schema2 = index.publish_schema(TextSample, version=“1.0.0”) … … # Insert datasets of different types … index.insert_dataset(image_ds, name=“images”) … index.insert_dataset(text_ds, name=“texts”) … … # List all datasets (mixed types) … for entry in index.list_datasets(): … print(f”{entry.name} -> {entry.schema_ref}“)</p>
431
+
<section id="attributes" class="level2">
432
+
<h2 class="anchored" data-anchor-id="attributes">Attributes</h2>
433
+
<table class="caption-top table">
434
+
<thead>
435
+
<tr class="header">
436
+
<th>Name</th>
437
+
<th>Description</th>
438
+
</tr>
439
+
</thead>
440
+
<tbody>
441
+
<tr class="odd">
442
+
<td><a href="#atdata.AbstractIndex.datasets">datasets</a></td>
443
+
<td>Lazily iterate over all dataset entries in this index.</td>
444
+
</tr>
445
+
<tr class="even">
446
+
<td><a href="#atdata.AbstractIndex.schemas">schemas</a></td>
447
+
<td>Lazily iterate over all schema records in this index.</td>
448
+
</tr>
449
+
</tbody>
450
+
</table>
451
+
</section>
452
+
<section id="methods" class="level2">
453
+
<h2 class="anchored" data-anchor-id="methods">Methods</h2>
454
+
<table class="caption-top table">
455
+
<thead>
456
+
<tr class="header">
457
+
<th>Name</th>
458
+
<th>Description</th>
459
+
</tr>
460
+
</thead>
461
+
<tbody>
462
+
<tr class="odd">
463
+
<td><a href="#atdata.AbstractIndex.decode_schema">decode_schema</a></td>
464
+
<td>Reconstruct a Python Packable type from a stored schema.</td>
465
+
</tr>
466
+
<tr class="even">
467
+
<td><a href="#atdata.AbstractIndex.get_dataset">get_dataset</a></td>
468
+
<td>Get a dataset entry by name or reference.</td>
469
+
</tr>
470
+
<tr class="odd">
471
+
<td><a href="#atdata.AbstractIndex.get_schema">get_schema</a></td>
472
+
<td>Get a schema record by reference.</td>
473
+
</tr>
474
+
<tr class="even">
475
+
<td><a href="#atdata.AbstractIndex.insert_dataset">insert_dataset</a></td>
476
+
<td>Insert a dataset into the index.</td>
477
+
</tr>
478
+
<tr class="odd">
479
+
<td><a href="#atdata.AbstractIndex.list_datasets">list_datasets</a></td>
480
+
<td>Get all dataset entries as a materialized list.</td>
481
+
</tr>
482
+
<tr class="even">
483
+
<td><a href="#atdata.AbstractIndex.list_schemas">list_schemas</a></td>
484
+
<td>Get all schema records as a materialized list.</td>
485
+
</tr>
486
+
<tr class="odd">
487
+
<td><a href="#atdata.AbstractIndex.publish_schema">publish_schema</a></td>
488
+
<td>Publish a schema for a sample type.</td>
489
+
</tr>
490
+
</tbody>
491
+
</table>
492
+
<section id="atdata.AbstractIndex.decode_schema" class="level3">
493
+
<h3 class="anchored" data-anchor-id="atdata.AbstractIndex.decode_schema">decode_schema</h3>
494
+
<div class="sourceCode" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a>AbstractIndex.decode_schema(ref)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
495
+
<p>Reconstruct a Python Packable type from a stored schema.</p>
496
+
<p>This method enables loading datasets without knowing the sample type ahead of time. The index retrieves the schema record and dynamically generates a Packable class matching the schema definition.</p>
497
+
<p>Args: ref: Schema reference string (local:// or at://).</p>
498
+
<p>Returns: A dynamically generated Packable class with fields matching the schema definition. The class can be used with <code>Dataset[T]</code> to load and iterate over samples.</p>
499
+
<p>Raises: KeyError: If schema not found. ValueError: If schema cannot be decoded (unsupported field types).</p>
500
+
<p>Example: >>> entry = index.get_dataset(“my-dataset”) >>> SampleType = index.decode_schema(entry.schema_ref) >>> ds = Dataset<a href="entry.data_urls[0]">SampleType</a> >>> for sample in ds.ordered(): … print(sample) # sample is instance of SampleType</p>
501
+
</section>
502
+
<section id="atdata.AbstractIndex.get_dataset" class="level3">
503
+
<h3 class="anchored" data-anchor-id="atdata.AbstractIndex.get_dataset">get_dataset</h3>
504
+
<div class="sourceCode" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a>AbstractIndex.get_dataset(ref)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
505
+
<p>Get a dataset entry by name or reference.</p>
506
+
<p>Args: ref: Dataset name, path, or full reference string.</p>
507
+
<p>Returns: IndexEntry for the dataset.</p>
508
+
<p>Raises: KeyError: If dataset not found.</p>
509
+
</section>
510
+
<section id="atdata.AbstractIndex.get_schema" class="level3">
511
+
<h3 class="anchored" data-anchor-id="atdata.AbstractIndex.get_schema">get_schema</h3>
512
+
<div class="sourceCode" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a>AbstractIndex.get_schema(ref)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
513
+
<p>Get a schema record by reference.</p>
514
+
<p>Args: ref: Schema reference string (local:// or at://).</p>
515
+
<p>Returns: Schema record as a dictionary with fields like ‘name’, ‘version’, ‘fields’, etc.</p>
516
+
<p>Raises: KeyError: If schema not found.</p>
517
+
</section>
518
+
<section id="atdata.AbstractIndex.insert_dataset" class="level3">
519
+
<h3 class="anchored" data-anchor-id="atdata.AbstractIndex.insert_dataset">insert_dataset</h3>
520
+
<div class="sourceCode" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a>AbstractIndex.insert_dataset(ds, <span class="op">*</span>, name, schema_ref<span class="op">=</span><span class="va">None</span>, <span class="op">**</span>kwargs)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
521
+
<p>Insert a dataset into the index.</p>
522
+
<p>The sample type is inferred from <code>ds.sample_type</code>. If schema_ref is not provided, the schema may be auto-published based on the sample type.</p>
523
+
<p>Args: ds: The Dataset to register in the index (any sample type). name: Human-readable name for the dataset. schema_ref: Optional explicit schema reference. If not provided, the schema may be auto-published or inferred from ds.sample_type. **kwargs: Additional backend-specific options.</p>
524
+
<p>Returns: IndexEntry for the inserted dataset.</p>
525
+
</section>
526
+
<section id="atdata.AbstractIndex.list_datasets" class="level3">
527
+
<h3 class="anchored" data-anchor-id="atdata.AbstractIndex.list_datasets">list_datasets</h3>
528
+
<div class="sourceCode" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a>AbstractIndex.list_datasets()</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
529
+
<p>Get all dataset entries as a materialized list.</p>
530
+
<p>Returns: List of IndexEntry for each dataset.</p>
531
+
</section>
532
+
<section id="atdata.AbstractIndex.list_schemas" class="level3">
533
+
<h3 class="anchored" data-anchor-id="atdata.AbstractIndex.list_schemas">list_schemas</h3>
534
+
<div class="sourceCode" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a>AbstractIndex.list_schemas()</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
535
+
<p>Get all schema records as a materialized list.</p>
536
+
<p>Returns: List of schema records as dictionaries.</p>
537
+
</section>
538
+
<section id="atdata.AbstractIndex.publish_schema" class="level3">
539
+
<h3 class="anchored" data-anchor-id="atdata.AbstractIndex.publish_schema">publish_schema</h3>
540
+
<div class="sourceCode" id="cb8"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a>AbstractIndex.publish_schema(sample_type, <span class="op">*</span>, version<span class="op">=</span><span class="st">'1.0.0'</span>, <span class="op">**</span>kwargs)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
541
+
<p>Publish a schema for a sample type.</p>
542
+
<p>Args: sample_type: A Packable type (PackableSample subclass or <span class="citation" data-cites="packable-decorated">@packable-decorated</span>). version: Semantic version string for the schema. **kwargs: Additional backend-specific options.</p>
543
+
<p>Returns: Schema reference string: - Local: ‘local://schemas/{module.Class}<span class="citation" data-cites="version">@version</span>’ - Atmosphere: ‘at://did:plc:…/ac.foundation.dataset.sampleSchema/…’</p>
544
+
545
+
546
+
</section>
547
+
</section>
548
+
</section>
549
+
550
+
</main> <!-- /main -->
551
+
<script id="quarto-html-after-body" type="application/javascript">
552
+
window.document.addEventListener("DOMContentLoaded", function (event) {
553
+
// Ensure there is a toggle, if there isn't float one in the top right
554
+
if (window.document.querySelector('.quarto-color-scheme-toggle') === null) {
555
+
const a = window.document.createElement('a');
556
+
a.classList.add('top-right');
557
+
a.classList.add('quarto-color-scheme-toggle');
558
+
a.href = "";
559
+
a.onclick = function() { try { window.quartoToggleColorScheme(); } catch {} return false; };
560
+
const i = window.document.createElement("i");
561
+
i.classList.add('bi');
562
+
a.appendChild(i);
563
+
window.document.body.appendChild(a);
564
+
}
565
+
setColorSchemeToggle(hasAlternateSentinel())
566
+
const icon = "";
567
+
const anchorJS = new window.AnchorJS();
568
+
anchorJS.options = {
569
+
placement: 'right',
570
+
icon: icon
571
+
};
572
+
anchorJS.add('.anchored');
573
+
const isCodeAnnotation = (el) => {
574
+
for (const clz of el.classList) {
575
+
if (clz.startsWith('code-annotation-')) {
576
+
return true;
577
+
}
578
+
}
579
+
return false;
580
+
}
581
+
const onCopySuccess = function(e) {
582
+
// button target
583
+
const button = e.trigger;
584
+
// don't keep focus
585
+
button.blur();
586
+
// flash "checked"
587
+
button.classList.add('code-copy-button-checked');
588
+
var currentTitle = button.getAttribute("title");
589
+
button.setAttribute("title", "Copied!");
590
+
let tooltip;
591
+
if (window.bootstrap) {
592
+
button.setAttribute("data-bs-toggle", "tooltip");
593
+
button.setAttribute("data-bs-placement", "left");
594
+
button.setAttribute("data-bs-title", "Copied!");
595
+
tooltip = new bootstrap.Tooltip(button,
596
+
{ trigger: "manual",
597
+
customClass: "code-copy-button-tooltip",
598
+
offset: [0, -8]});
599
+
tooltip.show();
600
+
}
601
+
setTimeout(function() {
602
+
if (tooltip) {
603
+
tooltip.hide();
604
+
button.removeAttribute("data-bs-title");
605
+
button.removeAttribute("data-bs-toggle");
606
+
button.removeAttribute("data-bs-placement");
607
+
}
608
+
button.setAttribute("title", currentTitle);
609
+
button.classList.remove('code-copy-button-checked');
610
+
}, 1000);
611
+
// clear code selection
612
+
e.clearSelection();
613
+
}
614
+
const getTextToCopy = function(trigger) {
615
+
const codeEl = trigger.previousElementSibling.cloneNode(true);
616
+
for (const childEl of codeEl.children) {
617
+
if (isCodeAnnotation(childEl)) {
618
+
childEl.remove();
619
+
}
620
+
}
621
+
return codeEl.innerText;
622
+
}
623
+
const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
624
+
text: getTextToCopy
625
+
});
626
+
clipboard.on('success', onCopySuccess);
627
+
if (window.document.getElementById('quarto-embedded-source-code-modal')) {
628
+
const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', {
629
+
text: getTextToCopy,
630
+
container: window.document.getElementById('quarto-embedded-source-code-modal')
631
+
});
632
+
clipboardModal.on('success', onCopySuccess);
633
+
}
634
+
var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//);
635
+
var mailtoRegex = new RegExp(/^mailto:/);
636
+
var filterRegex = new RegExp("https:\/\/github\.com\/your-org\/atdata");
637
+
var isInternal = (href) => {
638
+
return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href);
639
+
}
640
+
// Inspect non-navigation links and adorn them if external
641
+
var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)');
642
+
for (var i=0; i<links.length; i++) {
643
+
const link = links[i];
644
+
if (!isInternal(link.href)) {
645
+
// undo the damage that might have been done by quarto-nav.js in the case of
646
+
// links that we want to consider external
647
+
if (link.dataset.originalHref !== undefined) {
648
+
link.href = link.dataset.originalHref;
649
+
}
650
+
}
651
+
}
652
+
function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
653
+
const config = {
654
+
allowHTML: true,
655
+
maxWidth: 500,
656
+
delay: 100,
657
+
arrow: false,
658
+
appendTo: function(el) {
659
+
return el.parentElement;
660
+
},
661
+
interactive: true,
662
+
interactiveBorder: 10,
663
+
theme: 'quarto',
664
+
placement: 'bottom-start',
665
+
};
666
+
if (contentFn) {
667
+
config.content = contentFn;
668
+
}
669
+
if (onTriggerFn) {
670
+
config.onTrigger = onTriggerFn;
671
+
}
672
+
if (onUntriggerFn) {
673
+
config.onUntrigger = onUntriggerFn;
674
+
}
675
+
window.tippy(el, config);
676
+
}
677
+
const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
678
+
for (var i=0; i<noterefs.length; i++) {
679
+
const ref = noterefs[i];
680
+
tippyHover(ref, function() {
681
+
// use id or data attribute instead here
682
+
let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
683
+
try { href = new URL(href).hash; } catch {}
684
+
const id = href.replace(/^#\/?/, "");
685
+
const note = window.document.getElementById(id);
686
+
if (note) {
687
+
return note.innerHTML;
688
+
} else {
689
+
return "";
690
+
}
691
+
});
692
+
}
693
+
const xrefs = window.document.querySelectorAll('a.quarto-xref');
694
+
const processXRef = (id, note) => {
695
+
// Strip column container classes
696
+
const stripColumnClz = (el) => {
697
+
el.classList.remove("page-full", "page-columns");
698
+
if (el.children) {
699
+
for (const child of el.children) {
700
+
stripColumnClz(child);
701
+
}
702
+
}
703
+
}
704
+
stripColumnClz(note)
705
+
if (id === null || id.startsWith('sec-')) {
706
+
// Special case sections, only their first couple elements
707
+
const container = document.createElement("div");
708
+
if (note.children && note.children.length > 2) {
709
+
container.appendChild(note.children[0].cloneNode(true));
710
+
for (let i = 1; i < note.children.length; i++) {
711
+
const child = note.children[i];
712
+
if (child.tagName === "P" && child.innerText === "") {
713
+
continue;
714
+
} else {
715
+
container.appendChild(child.cloneNode(true));
716
+
break;
717
+
}
718
+
}
719
+
if (window.Quarto?.typesetMath) {
720
+
window.Quarto.typesetMath(container);
721
+
}
722
+
return container.innerHTML
723
+
} else {
724
+
if (window.Quarto?.typesetMath) {
725
+
window.Quarto.typesetMath(note);
726
+
}
727
+
return note.innerHTML;
728
+
}
729
+
} else {
730
+
// Remove any anchor links if they are present
731
+
const anchorLink = note.querySelector('a.anchorjs-link');
732
+
if (anchorLink) {
733
+
anchorLink.remove();
734
+
}
735
+
if (window.Quarto?.typesetMath) {
736
+
window.Quarto.typesetMath(note);
737
+
}
738
+
if (note.classList.contains("callout")) {
739
+
return note.outerHTML;
740
+
} else {
741
+
return note.innerHTML;
742
+
}
743
+
}
744
+
}
745
+
for (var i=0; i<xrefs.length; i++) {
746
+
const xref = xrefs[i];
747
+
tippyHover(xref, undefined, function(instance) {
748
+
instance.disable();
749
+
let url = xref.getAttribute('href');
750
+
let hash = undefined;
751
+
if (url.startsWith('#')) {
752
+
hash = url;
753
+
} else {
754
+
try { hash = new URL(url).hash; } catch {}
755
+
}
756
+
if (hash) {
757
+
const id = hash.replace(/^#\/?/, "");
758
+
const note = window.document.getElementById(id);
759
+
if (note !== null) {
760
+
try {
761
+
const html = processXRef(id, note.cloneNode(true));
762
+
instance.setContent(html);
763
+
} finally {
764
+
instance.enable();
765
+
instance.show();
766
+
}
767
+
} else {
768
+
// See if we can fetch this
769
+
fetch(url.split('#')[0])
770
+
.then(res => res.text())
771
+
.then(html => {
772
+
const parser = new DOMParser();
773
+
const htmlDoc = parser.parseFromString(html, "text/html");
774
+
const note = htmlDoc.getElementById(id);
775
+
if (note !== null) {
776
+
const html = processXRef(id, note);
777
+
instance.setContent(html);
778
+
}
779
+
}).finally(() => {
780
+
instance.enable();
781
+
instance.show();
782
+
});
783
+
}
784
+
} else {
785
+
// See if we can fetch a full url (with no hash to target)
786
+
// This is a special case and we should probably do some content thinning / targeting
787
+
fetch(url)
788
+
.then(res => res.text())
789
+
.then(html => {
790
+
const parser = new DOMParser();
791
+
const htmlDoc = parser.parseFromString(html, "text/html");
792
+
const note = htmlDoc.querySelector('main.content');
793
+
if (note !== null) {
794
+
// This should only happen for chapter cross references
795
+
// (since there is no id in the URL)
796
+
// remove the first header
797
+
if (note.children.length > 0 && note.children[0].tagName === "HEADER") {
798
+
note.children[0].remove();
799
+
}
800
+
const html = processXRef(null, note);
801
+
instance.setContent(html);
802
+
}
803
+
}).finally(() => {
804
+
instance.enable();
805
+
instance.show();
806
+
});
807
+
}
808
+
}, function(instance) {
809
+
});
810
+
}
811
+
let selectedAnnoteEl;
812
+
const selectorForAnnotation = ( cell, annotation) => {
813
+
let cellAttr = 'data-code-cell="' + cell + '"';
814
+
let lineAttr = 'data-code-annotation="' + annotation + '"';
815
+
const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
816
+
return selector;
817
+
}
818
+
const selectCodeLines = (annoteEl) => {
819
+
const doc = window.document;
820
+
const targetCell = annoteEl.getAttribute("data-target-cell");
821
+
const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
822
+
const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
823
+
const lines = annoteSpan.getAttribute("data-code-lines").split(",");
824
+
const lineIds = lines.map((line) => {
825
+
return targetCell + "-" + line;
826
+
})
827
+
let top = null;
828
+
let height = null;
829
+
let parent = null;
830
+
if (lineIds.length > 0) {
831
+
//compute the position of the single el (top and bottom and make a div)
832
+
const el = window.document.getElementById(lineIds[0]);
833
+
top = el.offsetTop;
834
+
height = el.offsetHeight;
835
+
parent = el.parentElement.parentElement;
836
+
if (lineIds.length > 1) {
837
+
const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
838
+
const bottom = lastEl.offsetTop + lastEl.offsetHeight;
839
+
height = bottom - top;
840
+
}
841
+
if (top !== null && height !== null && parent !== null) {
842
+
// cook up a div (if necessary) and position it
843
+
let div = window.document.getElementById("code-annotation-line-highlight");
844
+
if (div === null) {
845
+
div = window.document.createElement("div");
846
+
div.setAttribute("id", "code-annotation-line-highlight");
847
+
div.style.position = 'absolute';
848
+
parent.appendChild(div);
849
+
}
850
+
div.style.top = top - 2 + "px";
851
+
div.style.height = height + 4 + "px";
852
+
div.style.left = 0;
853
+
let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
854
+
if (gutterDiv === null) {
855
+
gutterDiv = window.document.createElement("div");
856
+
gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
857
+
gutterDiv.style.position = 'absolute';
858
+
const codeCell = window.document.getElementById(targetCell);
859
+
const gutter = codeCell.querySelector('.code-annotation-gutter');
860
+
gutter.appendChild(gutterDiv);
861
+
}
862
+
gutterDiv.style.top = top - 2 + "px";
863
+
gutterDiv.style.height = height + 4 + "px";
864
+
}
865
+
selectedAnnoteEl = annoteEl;
866
+
}
867
+
};
868
+
const unselectCodeLines = () => {
869
+
const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
870
+
elementsIds.forEach((elId) => {
871
+
const div = window.document.getElementById(elId);
872
+
if (div) {
873
+
div.remove();
874
+
}
875
+
});
876
+
selectedAnnoteEl = undefined;
877
+
};
878
+
// Handle positioning of the toggle
879
+
window.addEventListener(
880
+
"resize",
881
+
throttle(() => {
882
+
elRect = undefined;
883
+
if (selectedAnnoteEl) {
884
+
selectCodeLines(selectedAnnoteEl);
885
+
}
886
+
}, 10)
887
+
);
888
+
function throttle(fn, ms) {
889
+
let throttle = false;
890
+
let timer;
891
+
return (...args) => {
892
+
if(!throttle) { // first call gets through
893
+
fn.apply(this, args);
894
+
throttle = true;
895
+
} else { // all the others get throttled
896
+
if(timer) clearTimeout(timer); // cancel #2
897
+
timer = setTimeout(() => {
898
+
fn.apply(this, args);
899
+
timer = throttle = false;
900
+
}, ms);
901
+
}
902
+
};
903
+
}
904
+
// Attach click handler to the DT
905
+
const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
906
+
for (const annoteDlNode of annoteDls) {
907
+
annoteDlNode.addEventListener('click', (event) => {
908
+
const clickedEl = event.target;
909
+
if (clickedEl !== selectedAnnoteEl) {
910
+
unselectCodeLines();
911
+
const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
912
+
if (activeEl) {
913
+
activeEl.classList.remove('code-annotation-active');
914
+
}
915
+
selectCodeLines(clickedEl);
916
+
clickedEl.classList.add('code-annotation-active');
917
+
} else {
918
+
// Unselect the line
919
+
unselectCodeLines();
920
+
clickedEl.classList.remove('code-annotation-active');
921
+
}
922
+
});
923
+
}
924
+
const findCites = (el) => {
925
+
const parentEl = el.parentElement;
926
+
if (parentEl) {
927
+
const cites = parentEl.dataset.cites;
928
+
if (cites) {
929
+
return {
930
+
el,
931
+
cites: cites.split(' ')
932
+
};
933
+
} else {
934
+
return findCites(el.parentElement)
935
+
}
936
+
} else {
937
+
return undefined;
938
+
}
939
+
};
940
+
var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
941
+
for (var i=0; i<bibliorefs.length; i++) {
942
+
const ref = bibliorefs[i];
943
+
const citeInfo = findCites(ref);
944
+
if (citeInfo) {
945
+
tippyHover(citeInfo.el, function() {
946
+
var popup = window.document.createElement('div');
947
+
citeInfo.cites.forEach(function(cite) {
948
+
var citeDiv = window.document.createElement('div');
949
+
citeDiv.classList.add('hanging-indent');
950
+
citeDiv.classList.add('csl-entry');
951
+
var biblioDiv = window.document.getElementById('ref-' + cite);
952
+
if (biblioDiv) {
953
+
citeDiv.innerHTML = biblioDiv.innerHTML;
954
+
}
955
+
popup.appendChild(citeDiv);
956
+
});
957
+
return popup.innerHTML;
958
+
});
959
+
}
960
+
}
961
+
});
962
+
</script>
963
+
</div> <!-- /content -->
964
+
<footer class="footer">
965
+
<div class="nav-footer">
966
+
<div class="nav-footer-left">
967
+
<p>Built with <a href="https://quarto.org/">Quarto</a></p>
968
+
</div>
969
+
<div class="nav-footer-center">
970
+
971
+
<div class="toc-actions d-sm-block d-md-none"><ul><li><a href="https://github.com/your-org/atdata/edit/main/api/AbstractIndex.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></div>
972
+
<div class="nav-footer-right">
973
+
<p>MIT License</p>
974
+
</div>
975
+
</div>
976
+
</footer>
977
+
978
+
979
+
980
+
981
+
</body></html>
+914
docs/api/DataSource.html
+914
docs/api/DataSource.html
···
1
+
<!DOCTYPE html>
2
+
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
3
+
4
+
<meta charset="utf-8">
5
+
<meta name="generator" content="quarto-1.7.34">
6
+
7
+
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
8
+
9
+
10
+
<title>datasource – atdata</title>
11
+
<style>
12
+
code{white-space: pre-wrap;}
13
+
span.smallcaps{font-variant: small-caps;}
14
+
div.columns{display: flex; gap: min(4vw, 1.5em);}
15
+
div.column{flex: auto; overflow-x: auto;}
16
+
div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
17
+
ul.task-list{list-style: none;}
18
+
ul.task-list li input[type="checkbox"] {
19
+
width: 0.8em;
20
+
margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
21
+
vertical-align: middle;
22
+
}
23
+
/* CSS for syntax highlighting */
24
+
html { -webkit-text-size-adjust: 100%; }
25
+
pre > code.sourceCode { white-space: pre; position: relative; }
26
+
pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
27
+
pre > code.sourceCode > span:empty { height: 1.2em; }
28
+
.sourceCode { overflow: visible; }
29
+
code.sourceCode > span { color: inherit; text-decoration: inherit; }
30
+
div.sourceCode { margin: 1em 0; }
31
+
pre.sourceCode { margin: 0; }
32
+
@media screen {
33
+
div.sourceCode { overflow: auto; }
34
+
}
35
+
@media print {
36
+
pre > code.sourceCode { white-space: pre-wrap; }
37
+
pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
38
+
}
39
+
pre.numberSource code
40
+
{ counter-reset: source-line 0; }
41
+
pre.numberSource code > span
42
+
{ position: relative; left: -4em; counter-increment: source-line; }
43
+
pre.numberSource code > span > a:first-child::before
44
+
{ content: counter(source-line);
45
+
position: relative; left: -1em; text-align: right; vertical-align: baseline;
46
+
border: none; display: inline-block;
47
+
-webkit-touch-callout: none; -webkit-user-select: none;
48
+
-khtml-user-select: none; -moz-user-select: none;
49
+
-ms-user-select: none; user-select: none;
50
+
padding: 0 4px; width: 4em;
51
+
}
52
+
pre.numberSource { margin-left: 3em; padding-left: 4px; }
53
+
div.sourceCode
54
+
{ }
55
+
@media screen {
56
+
pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
57
+
}
58
+
</style>
59
+
60
+
61
+
<script src="../site_libs/quarto-nav/quarto-nav.js"></script>
62
+
<script src="../site_libs/quarto-nav/headroom.min.js"></script>
63
+
<script src="../site_libs/clipboard/clipboard.min.js"></script>
64
+
<script src="../site_libs/quarto-search/autocomplete.umd.js"></script>
65
+
<script src="../site_libs/quarto-search/fuse.min.js"></script>
66
+
<script src="../site_libs/quarto-search/quarto-search.js"></script>
67
+
<meta name="quarto:offset" content="../">
68
+
<script src="../site_libs/quarto-html/quarto.js" type="module"></script>
69
+
<script src="../site_libs/quarto-html/tabsets/tabsets.js" type="module"></script>
70
+
<script src="../site_libs/quarto-html/popper.min.js"></script>
71
+
<script src="../site_libs/quarto-html/tippy.umd.min.js"></script>
72
+
<script src="../site_libs/quarto-html/anchor.min.js"></script>
73
+
<link href="../site_libs/quarto-html/tippy.css" rel="stylesheet">
74
+
<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme" id="quarto-text-highlighting-styles">
75
+
<link href="../site_libs/quarto-html/quarto-syntax-highlighting-dark-8dcd8563ea6803ab7cbb3d71ca5772e1.css" rel="stylesheet" class="quarto-color-scheme quarto-color-alternate" id="quarto-text-highlighting-styles">
76
+
<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme-extra" id="quarto-text-highlighting-styles">
77
+
<script src="../site_libs/bootstrap/bootstrap.min.js"></script>
78
+
<link href="../site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
79
+
<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme" id="quarto-bootstrap" data-mode="light">
80
+
<link href="../site_libs/bootstrap/bootstrap-dark-7964ffd8887b0991fe8d71c6c8bc75d6.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme quarto-color-alternate" id="quarto-bootstrap" data-mode="dark">
81
+
<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme-extra" id="quarto-bootstrap" data-mode="light">
82
+
<script id="quarto-search-options" type="application/json">{
83
+
"location": "navbar",
84
+
"copy-button": false,
85
+
"collapse-after": 3,
86
+
"panel-placement": "end",
87
+
"type": "overlay",
88
+
"limit": 50,
89
+
"keyboard-shortcut": [
90
+
"f",
91
+
"/",
92
+
"s"
93
+
],
94
+
"show-item-context": false,
95
+
"language": {
96
+
"search-no-results-text": "No results",
97
+
"search-matching-documents-text": "matching documents",
98
+
"search-copy-link-title": "Copy link to search",
99
+
"search-hide-matches-text": "Hide additional matches",
100
+
"search-more-match-text": "more match in this document",
101
+
"search-more-matches-text": "more matches in this document",
102
+
"search-clear-button-title": "Clear",
103
+
"search-text-placeholder": "",
104
+
"search-detached-cancel-button-title": "Cancel",
105
+
"search-submit-button-title": "Submit",
106
+
"search-label": "Search"
107
+
}
108
+
}</script>
109
+
110
+
111
+
<link rel="stylesheet" href="../assets/styles.css">
112
+
</head>
113
+
114
+
<body class="nav-fixed quarto-light"><script id="quarto-html-before-body" type="application/javascript">
115
+
const toggleBodyColorMode = (bsSheetEl) => {
116
+
const mode = bsSheetEl.getAttribute("data-mode");
117
+
const bodyEl = window.document.querySelector("body");
118
+
if (mode === "dark") {
119
+
bodyEl.classList.add("quarto-dark");
120
+
bodyEl.classList.remove("quarto-light");
121
+
} else {
122
+
bodyEl.classList.add("quarto-light");
123
+
bodyEl.classList.remove("quarto-dark");
124
+
}
125
+
}
126
+
const toggleBodyColorPrimary = () => {
127
+
const bsSheetEl = window.document.querySelector("link#quarto-bootstrap:not([rel=disabled-stylesheet])");
128
+
if (bsSheetEl) {
129
+
toggleBodyColorMode(bsSheetEl);
130
+
}
131
+
}
132
+
const setColorSchemeToggle = (alternate) => {
133
+
const toggles = window.document.querySelectorAll('.quarto-color-scheme-toggle');
134
+
for (let i=0; i < toggles.length; i++) {
135
+
const toggle = toggles[i];
136
+
if (toggle) {
137
+
if (alternate) {
138
+
toggle.classList.add("alternate");
139
+
} else {
140
+
toggle.classList.remove("alternate");
141
+
}
142
+
}
143
+
}
144
+
};
145
+
const toggleColorMode = (alternate) => {
146
+
// Switch the stylesheets
147
+
const primaryStylesheets = window.document.querySelectorAll('link.quarto-color-scheme:not(.quarto-color-alternate)');
148
+
const alternateStylesheets = window.document.querySelectorAll('link.quarto-color-scheme.quarto-color-alternate');
149
+
manageTransitions('#quarto-margin-sidebar .nav-link', false);
150
+
if (alternate) {
151
+
// note: dark is layered on light, we don't disable primary!
152
+
enableStylesheet(alternateStylesheets);
153
+
for (const sheetNode of alternateStylesheets) {
154
+
if (sheetNode.id === "quarto-bootstrap") {
155
+
toggleBodyColorMode(sheetNode);
156
+
}
157
+
}
158
+
} else {
159
+
disableStylesheet(alternateStylesheets);
160
+
enableStylesheet(primaryStylesheets)
161
+
toggleBodyColorPrimary();
162
+
}
163
+
manageTransitions('#quarto-margin-sidebar .nav-link', true);
164
+
// Switch the toggles
165
+
setColorSchemeToggle(alternate)
166
+
// Hack to workaround the fact that safari doesn't
167
+
// properly recolor the scrollbar when toggling (#1455)
168
+
if (navigator.userAgent.indexOf('Safari') > 0 && navigator.userAgent.indexOf('Chrome') == -1) {
169
+
manageTransitions("body", false);
170
+
window.scrollTo(0, 1);
171
+
setTimeout(() => {
172
+
window.scrollTo(0, 0);
173
+
manageTransitions("body", true);
174
+
}, 40);
175
+
}
176
+
}
177
+
const disableStylesheet = (stylesheets) => {
178
+
for (let i=0; i < stylesheets.length; i++) {
179
+
const stylesheet = stylesheets[i];
180
+
stylesheet.rel = 'disabled-stylesheet';
181
+
}
182
+
}
183
+
const enableStylesheet = (stylesheets) => {
184
+
for (let i=0; i < stylesheets.length; i++) {
185
+
const stylesheet = stylesheets[i];
186
+
if(stylesheet.rel !== 'stylesheet') { // for Chrome, which will still FOUC without this check
187
+
stylesheet.rel = 'stylesheet';
188
+
}
189
+
}
190
+
}
191
+
const manageTransitions = (selector, allowTransitions) => {
192
+
const els = window.document.querySelectorAll(selector);
193
+
for (let i=0; i < els.length; i++) {
194
+
const el = els[i];
195
+
if (allowTransitions) {
196
+
el.classList.remove('notransition');
197
+
} else {
198
+
el.classList.add('notransition');
199
+
}
200
+
}
201
+
}
202
+
const isFileUrl = () => {
203
+
return window.location.protocol === 'file:';
204
+
}
205
+
const hasAlternateSentinel = () => {
206
+
let styleSentinel = getColorSchemeSentinel();
207
+
if (styleSentinel !== null) {
208
+
return styleSentinel === "alternate";
209
+
} else {
210
+
return false;
211
+
}
212
+
}
213
+
const setStyleSentinel = (alternate) => {
214
+
const value = alternate ? "alternate" : "default";
215
+
if (!isFileUrl()) {
216
+
window.localStorage.setItem("quarto-color-scheme", value);
217
+
} else {
218
+
localAlternateSentinel = value;
219
+
}
220
+
}
221
+
const getColorSchemeSentinel = () => {
222
+
if (!isFileUrl()) {
223
+
const storageValue = window.localStorage.getItem("quarto-color-scheme");
224
+
return storageValue != null ? storageValue : localAlternateSentinel;
225
+
} else {
226
+
return localAlternateSentinel;
227
+
}
228
+
}
229
+
const toggleGiscusIfUsed = (isAlternate, darkModeDefault) => {
230
+
const baseTheme = document.querySelector('#giscus-base-theme')?.value ?? 'light';
231
+
const alternateTheme = document.querySelector('#giscus-alt-theme')?.value ?? 'dark';
232
+
let newTheme = '';
233
+
if(authorPrefersDark) {
234
+
newTheme = isAlternate ? baseTheme : alternateTheme;
235
+
} else {
236
+
newTheme = isAlternate ? alternateTheme : baseTheme;
237
+
}
238
+
const changeGiscusTheme = () => {
239
+
// From: https://github.com/giscus/giscus/issues/336
240
+
const sendMessage = (message) => {
241
+
const iframe = document.querySelector('iframe.giscus-frame');
242
+
if (!iframe) return;
243
+
iframe.contentWindow.postMessage({ giscus: message }, 'https://giscus.app');
244
+
}
245
+
sendMessage({
246
+
setConfig: {
247
+
theme: newTheme
248
+
}
249
+
});
250
+
}
251
+
const isGiscussLoaded = window.document.querySelector('iframe.giscus-frame') !== null;
252
+
if (isGiscussLoaded) {
253
+
changeGiscusTheme();
254
+
}
255
+
};
256
+
const authorPrefersDark = false;
257
+
const darkModeDefault = authorPrefersDark;
258
+
document.querySelector('link#quarto-text-highlighting-styles.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
259
+
document.querySelector('link#quarto-bootstrap.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
260
+
let localAlternateSentinel = darkModeDefault ? 'alternate' : 'default';
261
+
// Dark / light mode switch
262
+
window.quartoToggleColorScheme = () => {
263
+
// Read the current dark / light value
264
+
let toAlternate = !hasAlternateSentinel();
265
+
toggleColorMode(toAlternate);
266
+
setStyleSentinel(toAlternate);
267
+
toggleGiscusIfUsed(toAlternate, darkModeDefault);
268
+
window.dispatchEvent(new Event('resize'));
269
+
};
270
+
// Switch to dark mode if need be
271
+
if (hasAlternateSentinel()) {
272
+
toggleColorMode(true);
273
+
} else {
274
+
toggleColorMode(false);
275
+
}
276
+
</script>
277
+
278
+
<div id="quarto-search-results"></div>
279
+
<header id="quarto-header" class="headroom fixed-top">
280
+
<nav class="navbar navbar-expand-lg " data-bs-theme="dark">
281
+
<div class="navbar-container container-fluid">
282
+
<div class="navbar-brand-container mx-auto">
283
+
<a class="navbar-brand" href="../index.html">
284
+
<span class="navbar-title">atdata</span>
285
+
</a>
286
+
</div>
287
+
<div id="quarto-search" class="" title="Search"></div>
288
+
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarCollapse" aria-controls="navbarCollapse" role="menu" aria-expanded="false" aria-label="Toggle navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
289
+
<span class="navbar-toggler-icon"></span>
290
+
</button>
291
+
<div class="collapse navbar-collapse" id="navbarCollapse">
292
+
<ul class="navbar-nav navbar-nav-scroll me-auto">
293
+
<li class="nav-item">
294
+
<a class="nav-link" href="../index.html">
295
+
<span class="menu-text">Guide</span></a>
296
+
</li>
297
+
<li class="nav-item dropdown ">
298
+
<a class="nav-link dropdown-toggle" href="#" id="nav-menu-tutorials" role="link" data-bs-toggle="dropdown" aria-expanded="false">
299
+
<span class="menu-text">Tutorials</span>
300
+
</a>
301
+
<ul class="dropdown-menu" aria-labelledby="nav-menu-tutorials">
302
+
<li>
303
+
<a class="dropdown-item" href="../tutorials/quickstart.html">
304
+
<span class="dropdown-text">Quick Start</span></a>
305
+
</li>
306
+
<li>
307
+
<a class="dropdown-item" href="../tutorials/local-workflow.html">
308
+
<span class="dropdown-text">Local Workflow</span></a>
309
+
</li>
310
+
<li>
311
+
<a class="dropdown-item" href="../tutorials/atmosphere.html">
312
+
<span class="dropdown-text">Atmosphere Publishing</span></a>
313
+
</li>
314
+
<li>
315
+
<a class="dropdown-item" href="../tutorials/promotion.html">
316
+
<span class="dropdown-text">Promotion Workflow</span></a>
317
+
</li>
318
+
</ul>
319
+
</li>
320
+
<li class="nav-item dropdown ">
321
+
<a class="nav-link dropdown-toggle" href="#" id="nav-menu-reference" role="link" data-bs-toggle="dropdown" aria-expanded="false">
322
+
<span class="menu-text">Reference</span>
323
+
</a>
324
+
<ul class="dropdown-menu" aria-labelledby="nav-menu-reference">
325
+
<li>
326
+
<a class="dropdown-item" href="../reference/packable-samples.html">
327
+
<span class="dropdown-text">Packable Samples</span></a>
328
+
</li>
329
+
<li>
330
+
<a class="dropdown-item" href="../reference/datasets.html">
331
+
<span class="dropdown-text">Datasets</span></a>
332
+
</li>
333
+
<li>
334
+
<a class="dropdown-item" href="../reference/lenses.html">
335
+
<span class="dropdown-text">Lenses</span></a>
336
+
</li>
337
+
<li>
338
+
<a class="dropdown-item" href="../reference/local-storage.html">
339
+
<span class="dropdown-text">Local Storage</span></a>
340
+
</li>
341
+
<li>
342
+
<a class="dropdown-item" href="../reference/atmosphere.html">
343
+
<span class="dropdown-text">Atmosphere</span></a>
344
+
</li>
345
+
<li>
346
+
<a class="dropdown-item" href="../reference/promotion.html">
347
+
<span class="dropdown-text">Promotion</span></a>
348
+
</li>
349
+
<li>
350
+
<a class="dropdown-item" href="../reference/load-dataset.html">
351
+
<span class="dropdown-text">load_dataset API</span></a>
352
+
</li>
353
+
<li>
354
+
<a class="dropdown-item" href="../reference/protocols.html">
355
+
<span class="dropdown-text">Protocols</span></a>
356
+
</li>
357
+
<li>
358
+
<a class="dropdown-item" href="../reference/uri-spec.html">
359
+
<span class="dropdown-text">URI Specification</span></a>
360
+
</li>
361
+
<li>
362
+
<a class="dropdown-item" href="../reference/troubleshooting.html">
363
+
<span class="dropdown-text">Troubleshooting & FAQ</span></a>
364
+
</li>
365
+
<li>
366
+
<a class="dropdown-item" href="../reference/deployment.html">
367
+
<span class="dropdown-text">Deployment Guide</span></a>
368
+
</li>
369
+
</ul>
370
+
</li>
371
+
<li class="nav-item">
372
+
<a class="nav-link" href="../api/index.html">
373
+
<span class="menu-text">API</span></a>
374
+
</li>
375
+
</ul>
376
+
<ul class="navbar-nav navbar-nav-scroll ms-auto">
377
+
<li class="nav-item compact">
378
+
<a class="nav-link" href="https://github.com/your-org/atdata"> <i class="bi bi-github" role="img">
379
+
</i>
380
+
<span class="menu-text"></span></a>
381
+
</li>
382
+
</ul>
383
+
</div> <!-- /navcollapse -->
384
+
<div class="quarto-navbar-tools">
385
+
<a href="" class="quarto-color-scheme-toggle quarto-navigation-tool px-1" onclick="window.quartoToggleColorScheme(); return false;" title="Toggle dark mode"><i class="bi"></i></a>
386
+
</div>
387
+
</div> <!-- /container-fluid -->
388
+
</nav>
389
+
</header>
390
+
<!-- content -->
391
+
<div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-article page-navbar">
392
+
<!-- sidebar -->
393
+
<!-- margin-sidebar -->
394
+
<div id="quarto-margin-sidebar" class="sidebar margin-sidebar">
395
+
<nav id="TOC" role="doc-toc" class="toc-active">
396
+
<h2 id="toc-title">On this page</h2>
397
+
398
+
<ul>
399
+
<li><a href="#atdata.DataSource" id="toc-atdata.DataSource" class="nav-link active" data-scroll-target="#atdata.DataSource">DataSource</a>
400
+
<ul class="collapse">
401
+
<li><a href="#attributes" id="toc-attributes" class="nav-link" data-scroll-target="#attributes">Attributes</a></li>
402
+
<li><a href="#methods" id="toc-methods" class="nav-link" data-scroll-target="#methods">Methods</a>
403
+
<ul class="collapse">
404
+
<li><a href="#atdata.DataSource.list_shards" id="toc-atdata.DataSource.list_shards" class="nav-link" data-scroll-target="#atdata.DataSource.list_shards">list_shards</a></li>
405
+
<li><a href="#atdata.DataSource.open_shard" id="toc-atdata.DataSource.open_shard" class="nav-link" data-scroll-target="#atdata.DataSource.open_shard">open_shard</a></li>
406
+
</ul></li>
407
+
</ul></li>
408
+
</ul>
409
+
<div class="toc-actions"><ul><li><a href="https://github.com/your-org/atdata/edit/main/api/DataSource.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></nav>
410
+
</div>
411
+
<!-- main -->
412
+
<main class="content" id="quarto-document-content"><header id="title-block-header" class="quarto-title-block"></header>
413
+
414
+
415
+
416
+
417
+
418
+
<section id="atdata.DataSource" class="level1">
419
+
<h1>DataSource</h1>
420
+
<div class="sourceCode" id="cb1"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a>DataSource()</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
421
+
<p>Protocol for data sources that provide streams to Dataset.</p>
422
+
<p>A DataSource abstracts over different ways of accessing dataset shards: - URLSource: Standard WebDataset-compatible URLs (http, https, pipe, gs, etc.) - S3Source: S3-compatible storage with explicit credentials - BlobSource: ATProto blob references (future)</p>
423
+
<p>The key method is <code>shards()</code>, which yields (identifier, stream) pairs. These are fed directly to WebDataset’s tar_file_expander, bypassing URL resolution entirely. This enables: - Private S3 repos with credentials - Custom endpoints (Cloudflare R2, MinIO) - ATProto blob streaming - Any other source that can provide file-like objects</p>
424
+
<p>Example: >>> source = S3Source( … bucket=“my-bucket”, … keys=[“data-000.tar”, “data-001.tar”], … endpoint=“https://r2.example.com”, … credentials=creds, … ) >>> ds = Dataset<a href="source">MySample</a> >>> for sample in ds.ordered(): … print(sample)</p>
425
+
<section id="attributes" class="level2">
426
+
<h2 class="anchored" data-anchor-id="attributes">Attributes</h2>
427
+
<table class="caption-top table">
428
+
<thead>
429
+
<tr class="header">
430
+
<th>Name</th>
431
+
<th>Description</th>
432
+
</tr>
433
+
</thead>
434
+
<tbody>
435
+
<tr class="odd">
436
+
<td><a href="#atdata.DataSource.shards">shards</a></td>
437
+
<td>Lazily yield (identifier, stream) pairs for each shard.</td>
438
+
</tr>
439
+
</tbody>
440
+
</table>
441
+
</section>
442
+
<section id="methods" class="level2">
443
+
<h2 class="anchored" data-anchor-id="methods">Methods</h2>
444
+
<table class="caption-top table">
445
+
<thead>
446
+
<tr class="header">
447
+
<th>Name</th>
448
+
<th>Description</th>
449
+
</tr>
450
+
</thead>
451
+
<tbody>
452
+
<tr class="odd">
453
+
<td><a href="#atdata.DataSource.list_shards">list_shards</a></td>
454
+
<td>Get list of shard identifiers without opening streams.</td>
455
+
</tr>
456
+
<tr class="even">
457
+
<td><a href="#atdata.DataSource.open_shard">open_shard</a></td>
458
+
<td>Open a single shard by its identifier.</td>
459
+
</tr>
460
+
</tbody>
461
+
</table>
462
+
<section id="atdata.DataSource.list_shards" class="level3">
463
+
<h3 class="anchored" data-anchor-id="atdata.DataSource.list_shards">list_shards</h3>
464
+
<div class="sourceCode" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a>DataSource.list_shards()</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
465
+
<p>Get list of shard identifiers without opening streams.</p>
466
+
<p>Used for metadata queries like counting shards without actually streaming data. Implementations should return identifiers that match what shards would yield.</p>
467
+
<p>Returns: List of shard identifier strings.</p>
468
+
</section>
469
+
<section id="atdata.DataSource.open_shard" class="level3">
470
+
<h3 class="anchored" data-anchor-id="atdata.DataSource.open_shard">open_shard</h3>
471
+
<div class="sourceCode" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a>DataSource.open_shard(shard_id)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
472
+
<p>Open a single shard by its identifier.</p>
473
+
<p>This method enables random access to individual shards, which is required for PyTorch DataLoader worker splitting. Each worker opens only its assigned shards rather than iterating all shards.</p>
474
+
<p>Args: shard_id: Shard identifier from shard_list.</p>
475
+
<p>Returns: File-like stream for reading the shard.</p>
476
+
<p>Raises: KeyError: If shard_id is not in shard_list.</p>
477
+
478
+
479
+
</section>
480
+
</section>
481
+
</section>
482
+
483
+
</main> <!-- /main -->
484
+
<script id="quarto-html-after-body" type="application/javascript">
485
+
window.document.addEventListener("DOMContentLoaded", function (event) {
486
+
// Ensure there is a toggle, if there isn't float one in the top right
487
+
if (window.document.querySelector('.quarto-color-scheme-toggle') === null) {
488
+
const a = window.document.createElement('a');
489
+
a.classList.add('top-right');
490
+
a.classList.add('quarto-color-scheme-toggle');
491
+
a.href = "";
492
+
a.onclick = function() { try { window.quartoToggleColorScheme(); } catch {} return false; };
493
+
const i = window.document.createElement("i");
494
+
i.classList.add('bi');
495
+
a.appendChild(i);
496
+
window.document.body.appendChild(a);
497
+
}
498
+
setColorSchemeToggle(hasAlternateSentinel())
499
+
const icon = "";
500
+
const anchorJS = new window.AnchorJS();
501
+
anchorJS.options = {
502
+
placement: 'right',
503
+
icon: icon
504
+
};
505
+
anchorJS.add('.anchored');
506
+
const isCodeAnnotation = (el) => {
507
+
for (const clz of el.classList) {
508
+
if (clz.startsWith('code-annotation-')) {
509
+
return true;
510
+
}
511
+
}
512
+
return false;
513
+
}
514
+
const onCopySuccess = function(e) {
515
+
// button target
516
+
const button = e.trigger;
517
+
// don't keep focus
518
+
button.blur();
519
+
// flash "checked"
520
+
button.classList.add('code-copy-button-checked');
521
+
var currentTitle = button.getAttribute("title");
522
+
button.setAttribute("title", "Copied!");
523
+
let tooltip;
524
+
if (window.bootstrap) {
525
+
button.setAttribute("data-bs-toggle", "tooltip");
526
+
button.setAttribute("data-bs-placement", "left");
527
+
button.setAttribute("data-bs-title", "Copied!");
528
+
tooltip = new bootstrap.Tooltip(button,
529
+
{ trigger: "manual",
530
+
customClass: "code-copy-button-tooltip",
531
+
offset: [0, -8]});
532
+
tooltip.show();
533
+
}
534
+
setTimeout(function() {
535
+
if (tooltip) {
536
+
tooltip.hide();
537
+
button.removeAttribute("data-bs-title");
538
+
button.removeAttribute("data-bs-toggle");
539
+
button.removeAttribute("data-bs-placement");
540
+
}
541
+
button.setAttribute("title", currentTitle);
542
+
button.classList.remove('code-copy-button-checked');
543
+
}, 1000);
544
+
// clear code selection
545
+
e.clearSelection();
546
+
}
547
+
const getTextToCopy = function(trigger) {
548
+
const codeEl = trigger.previousElementSibling.cloneNode(true);
549
+
for (const childEl of codeEl.children) {
550
+
if (isCodeAnnotation(childEl)) {
551
+
childEl.remove();
552
+
}
553
+
}
554
+
return codeEl.innerText;
555
+
}
556
+
const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
557
+
text: getTextToCopy
558
+
});
559
+
clipboard.on('success', onCopySuccess);
560
+
if (window.document.getElementById('quarto-embedded-source-code-modal')) {
561
+
const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', {
562
+
text: getTextToCopy,
563
+
container: window.document.getElementById('quarto-embedded-source-code-modal')
564
+
});
565
+
clipboardModal.on('success', onCopySuccess);
566
+
}
567
+
var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//);
568
+
var mailtoRegex = new RegExp(/^mailto:/);
569
+
var filterRegex = new RegExp("https:\/\/github\.com\/your-org\/atdata");
570
+
var isInternal = (href) => {
571
+
return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href);
572
+
}
573
+
// Inspect non-navigation links and adorn them if external
574
+
var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)');
575
+
for (var i=0; i<links.length; i++) {
576
+
const link = links[i];
577
+
if (!isInternal(link.href)) {
578
+
// undo the damage that might have been done by quarto-nav.js in the case of
579
+
// links that we want to consider external
580
+
if (link.dataset.originalHref !== undefined) {
581
+
link.href = link.dataset.originalHref;
582
+
}
583
+
}
584
+
}
585
+
function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
586
+
const config = {
587
+
allowHTML: true,
588
+
maxWidth: 500,
589
+
delay: 100,
590
+
arrow: false,
591
+
appendTo: function(el) {
592
+
return el.parentElement;
593
+
},
594
+
interactive: true,
595
+
interactiveBorder: 10,
596
+
theme: 'quarto',
597
+
placement: 'bottom-start',
598
+
};
599
+
if (contentFn) {
600
+
config.content = contentFn;
601
+
}
602
+
if (onTriggerFn) {
603
+
config.onTrigger = onTriggerFn;
604
+
}
605
+
if (onUntriggerFn) {
606
+
config.onUntrigger = onUntriggerFn;
607
+
}
608
+
window.tippy(el, config);
609
+
}
610
+
const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
611
+
for (var i=0; i<noterefs.length; i++) {
612
+
const ref = noterefs[i];
613
+
tippyHover(ref, function() {
614
+
// use id or data attribute instead here
615
+
let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
616
+
try { href = new URL(href).hash; } catch {}
617
+
const id = href.replace(/^#\/?/, "");
618
+
const note = window.document.getElementById(id);
619
+
if (note) {
620
+
return note.innerHTML;
621
+
} else {
622
+
return "";
623
+
}
624
+
});
625
+
}
626
+
const xrefs = window.document.querySelectorAll('a.quarto-xref');
627
+
const processXRef = (id, note) => {
628
+
// Strip column container classes
629
+
const stripColumnClz = (el) => {
630
+
el.classList.remove("page-full", "page-columns");
631
+
if (el.children) {
632
+
for (const child of el.children) {
633
+
stripColumnClz(child);
634
+
}
635
+
}
636
+
}
637
+
stripColumnClz(note)
638
+
if (id === null || id.startsWith('sec-')) {
639
+
// Special case sections, only their first couple elements
640
+
const container = document.createElement("div");
641
+
if (note.children && note.children.length > 2) {
642
+
container.appendChild(note.children[0].cloneNode(true));
643
+
for (let i = 1; i < note.children.length; i++) {
644
+
const child = note.children[i];
645
+
if (child.tagName === "P" && child.innerText === "") {
646
+
continue;
647
+
} else {
648
+
container.appendChild(child.cloneNode(true));
649
+
break;
650
+
}
651
+
}
652
+
if (window.Quarto?.typesetMath) {
653
+
window.Quarto.typesetMath(container);
654
+
}
655
+
return container.innerHTML
656
+
} else {
657
+
if (window.Quarto?.typesetMath) {
658
+
window.Quarto.typesetMath(note);
659
+
}
660
+
return note.innerHTML;
661
+
}
662
+
} else {
663
+
// Remove any anchor links if they are present
664
+
const anchorLink = note.querySelector('a.anchorjs-link');
665
+
if (anchorLink) {
666
+
anchorLink.remove();
667
+
}
668
+
if (window.Quarto?.typesetMath) {
669
+
window.Quarto.typesetMath(note);
670
+
}
671
+
if (note.classList.contains("callout")) {
672
+
return note.outerHTML;
673
+
} else {
674
+
return note.innerHTML;
675
+
}
676
+
}
677
+
}
678
+
for (var i=0; i<xrefs.length; i++) {
679
+
const xref = xrefs[i];
680
+
tippyHover(xref, undefined, function(instance) {
681
+
instance.disable();
682
+
let url = xref.getAttribute('href');
683
+
let hash = undefined;
684
+
if (url.startsWith('#')) {
685
+
hash = url;
686
+
} else {
687
+
try { hash = new URL(url).hash; } catch {}
688
+
}
689
+
if (hash) {
690
+
const id = hash.replace(/^#\/?/, "");
691
+
const note = window.document.getElementById(id);
692
+
if (note !== null) {
693
+
try {
694
+
const html = processXRef(id, note.cloneNode(true));
695
+
instance.setContent(html);
696
+
} finally {
697
+
instance.enable();
698
+
instance.show();
699
+
}
700
+
} else {
701
+
// See if we can fetch this
702
+
fetch(url.split('#')[0])
703
+
.then(res => res.text())
704
+
.then(html => {
705
+
const parser = new DOMParser();
706
+
const htmlDoc = parser.parseFromString(html, "text/html");
707
+
const note = htmlDoc.getElementById(id);
708
+
if (note !== null) {
709
+
const html = processXRef(id, note);
710
+
instance.setContent(html);
711
+
}
712
+
}).finally(() => {
713
+
instance.enable();
714
+
instance.show();
715
+
});
716
+
}
717
+
} else {
718
+
// See if we can fetch a full url (with no hash to target)
719
+
// This is a special case and we should probably do some content thinning / targeting
720
+
fetch(url)
721
+
.then(res => res.text())
722
+
.then(html => {
723
+
const parser = new DOMParser();
724
+
const htmlDoc = parser.parseFromString(html, "text/html");
725
+
const note = htmlDoc.querySelector('main.content');
726
+
if (note !== null) {
727
+
// This should only happen for chapter cross references
728
+
// (since there is no id in the URL)
729
+
// remove the first header
730
+
if (note.children.length > 0 && note.children[0].tagName === "HEADER") {
731
+
note.children[0].remove();
732
+
}
733
+
const html = processXRef(null, note);
734
+
instance.setContent(html);
735
+
}
736
+
}).finally(() => {
737
+
instance.enable();
738
+
instance.show();
739
+
});
740
+
}
741
+
}, function(instance) {
742
+
});
743
+
}
744
+
let selectedAnnoteEl;
745
+
const selectorForAnnotation = ( cell, annotation) => {
746
+
let cellAttr = 'data-code-cell="' + cell + '"';
747
+
let lineAttr = 'data-code-annotation="' + annotation + '"';
748
+
const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
749
+
return selector;
750
+
}
751
+
const selectCodeLines = (annoteEl) => {
752
+
const doc = window.document;
753
+
const targetCell = annoteEl.getAttribute("data-target-cell");
754
+
const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
755
+
const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
756
+
const lines = annoteSpan.getAttribute("data-code-lines").split(",");
757
+
const lineIds = lines.map((line) => {
758
+
return targetCell + "-" + line;
759
+
})
760
+
let top = null;
761
+
let height = null;
762
+
let parent = null;
763
+
if (lineIds.length > 0) {
764
+
//compute the position of the single el (top and bottom and make a div)
765
+
const el = window.document.getElementById(lineIds[0]);
766
+
top = el.offsetTop;
767
+
height = el.offsetHeight;
768
+
parent = el.parentElement.parentElement;
769
+
if (lineIds.length > 1) {
770
+
const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
771
+
const bottom = lastEl.offsetTop + lastEl.offsetHeight;
772
+
height = bottom - top;
773
+
}
774
+
if (top !== null && height !== null && parent !== null) {
775
+
// cook up a div (if necessary) and position it
776
+
let div = window.document.getElementById("code-annotation-line-highlight");
777
+
if (div === null) {
778
+
div = window.document.createElement("div");
779
+
div.setAttribute("id", "code-annotation-line-highlight");
780
+
div.style.position = 'absolute';
781
+
parent.appendChild(div);
782
+
}
783
+
div.style.top = top - 2 + "px";
784
+
div.style.height = height + 4 + "px";
785
+
div.style.left = 0;
786
+
let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
787
+
if (gutterDiv === null) {
788
+
gutterDiv = window.document.createElement("div");
789
+
gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
790
+
gutterDiv.style.position = 'absolute';
791
+
const codeCell = window.document.getElementById(targetCell);
792
+
const gutter = codeCell.querySelector('.code-annotation-gutter');
793
+
gutter.appendChild(gutterDiv);
794
+
}
795
+
gutterDiv.style.top = top - 2 + "px";
796
+
gutterDiv.style.height = height + 4 + "px";
797
+
}
798
+
selectedAnnoteEl = annoteEl;
799
+
}
800
+
};
801
+
const unselectCodeLines = () => {
802
+
const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
803
+
elementsIds.forEach((elId) => {
804
+
const div = window.document.getElementById(elId);
805
+
if (div) {
806
+
div.remove();
807
+
}
808
+
});
809
+
selectedAnnoteEl = undefined;
810
+
};
811
+
// Handle positioning of the toggle
812
+
window.addEventListener(
813
+
"resize",
814
+
throttle(() => {
815
+
elRect = undefined;
816
+
if (selectedAnnoteEl) {
817
+
selectCodeLines(selectedAnnoteEl);
818
+
}
819
+
}, 10)
820
+
);
821
+
function throttle(fn, ms) {
822
+
let throttle = false;
823
+
let timer;
824
+
return (...args) => {
825
+
if(!throttle) { // first call gets through
826
+
fn.apply(this, args);
827
+
throttle = true;
828
+
} else { // all the others get throttled
829
+
if(timer) clearTimeout(timer); // cancel #2
830
+
timer = setTimeout(() => {
831
+
fn.apply(this, args);
832
+
timer = throttle = false;
833
+
}, ms);
834
+
}
835
+
};
836
+
}
837
+
// Attach click handler to the DT
838
+
const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
839
+
for (const annoteDlNode of annoteDls) {
840
+
annoteDlNode.addEventListener('click', (event) => {
841
+
const clickedEl = event.target;
842
+
if (clickedEl !== selectedAnnoteEl) {
843
+
unselectCodeLines();
844
+
const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
845
+
if (activeEl) {
846
+
activeEl.classList.remove('code-annotation-active');
847
+
}
848
+
selectCodeLines(clickedEl);
849
+
clickedEl.classList.add('code-annotation-active');
850
+
} else {
851
+
// Unselect the line
852
+
unselectCodeLines();
853
+
clickedEl.classList.remove('code-annotation-active');
854
+
}
855
+
});
856
+
}
857
+
const findCites = (el) => {
858
+
const parentEl = el.parentElement;
859
+
if (parentEl) {
860
+
const cites = parentEl.dataset.cites;
861
+
if (cites) {
862
+
return {
863
+
el,
864
+
cites: cites.split(' ')
865
+
};
866
+
} else {
867
+
return findCites(el.parentElement)
868
+
}
869
+
} else {
870
+
return undefined;
871
+
}
872
+
};
873
+
var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
874
+
for (var i=0; i<bibliorefs.length; i++) {
875
+
const ref = bibliorefs[i];
876
+
const citeInfo = findCites(ref);
877
+
if (citeInfo) {
878
+
tippyHover(citeInfo.el, function() {
879
+
var popup = window.document.createElement('div');
880
+
citeInfo.cites.forEach(function(cite) {
881
+
var citeDiv = window.document.createElement('div');
882
+
citeDiv.classList.add('hanging-indent');
883
+
citeDiv.classList.add('csl-entry');
884
+
var biblioDiv = window.document.getElementById('ref-' + cite);
885
+
if (biblioDiv) {
886
+
citeDiv.innerHTML = biblioDiv.innerHTML;
887
+
}
888
+
popup.appendChild(citeDiv);
889
+
});
890
+
return popup.innerHTML;
891
+
});
892
+
}
893
+
}
894
+
});
895
+
</script>
896
+
</div> <!-- /content -->
897
+
<footer class="footer">
898
+
<div class="nav-footer">
899
+
<div class="nav-footer-left">
900
+
<p>Built with <a href="https://quarto.org/">Quarto</a></p>
901
+
</div>
902
+
<div class="nav-footer-center">
903
+
904
+
<div class="toc-actions d-sm-block d-md-none"><ul><li><a href="https://github.com/your-org/atdata/edit/main/api/DataSource.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></div>
905
+
<div class="nav-footer-right">
906
+
<p>MIT License</p>
907
+
</div>
908
+
</div>
909
+
</footer>
910
+
911
+
912
+
913
+
914
+
</body></html>
+883
docs/api/IndexEntry.html
+883
docs/api/IndexEntry.html
···
1
+
<!DOCTYPE html>
2
+
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
3
+
4
+
<meta charset="utf-8">
5
+
<meta name="generator" content="quarto-1.7.34">
6
+
7
+
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
8
+
9
+
10
+
<title>indexentry – atdata</title>
11
+
<style>
12
+
code{white-space: pre-wrap;}
13
+
span.smallcaps{font-variant: small-caps;}
14
+
div.columns{display: flex; gap: min(4vw, 1.5em);}
15
+
div.column{flex: auto; overflow-x: auto;}
16
+
div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
17
+
ul.task-list{list-style: none;}
18
+
ul.task-list li input[type="checkbox"] {
19
+
width: 0.8em;
20
+
margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
21
+
vertical-align: middle;
22
+
}
23
+
/* CSS for syntax highlighting */
24
+
html { -webkit-text-size-adjust: 100%; }
25
+
pre > code.sourceCode { white-space: pre; position: relative; }
26
+
pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
27
+
pre > code.sourceCode > span:empty { height: 1.2em; }
28
+
.sourceCode { overflow: visible; }
29
+
code.sourceCode > span { color: inherit; text-decoration: inherit; }
30
+
div.sourceCode { margin: 1em 0; }
31
+
pre.sourceCode { margin: 0; }
32
+
@media screen {
33
+
div.sourceCode { overflow: auto; }
34
+
}
35
+
@media print {
36
+
pre > code.sourceCode { white-space: pre-wrap; }
37
+
pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
38
+
}
39
+
pre.numberSource code
40
+
{ counter-reset: source-line 0; }
41
+
pre.numberSource code > span
42
+
{ position: relative; left: -4em; counter-increment: source-line; }
43
+
pre.numberSource code > span > a:first-child::before
44
+
{ content: counter(source-line);
45
+
position: relative; left: -1em; text-align: right; vertical-align: baseline;
46
+
border: none; display: inline-block;
47
+
-webkit-touch-callout: none; -webkit-user-select: none;
48
+
-khtml-user-select: none; -moz-user-select: none;
49
+
-ms-user-select: none; user-select: none;
50
+
padding: 0 4px; width: 4em;
51
+
}
52
+
pre.numberSource { margin-left: 3em; padding-left: 4px; }
53
+
div.sourceCode
54
+
{ }
55
+
@media screen {
56
+
pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
57
+
}
58
+
</style>
59
+
60
+
61
+
<script src="../site_libs/quarto-nav/quarto-nav.js"></script>
62
+
<script src="../site_libs/quarto-nav/headroom.min.js"></script>
63
+
<script src="../site_libs/clipboard/clipboard.min.js"></script>
64
+
<script src="../site_libs/quarto-search/autocomplete.umd.js"></script>
65
+
<script src="../site_libs/quarto-search/fuse.min.js"></script>
66
+
<script src="../site_libs/quarto-search/quarto-search.js"></script>
67
+
<meta name="quarto:offset" content="../">
68
+
<script src="../site_libs/quarto-html/quarto.js" type="module"></script>
69
+
<script src="../site_libs/quarto-html/tabsets/tabsets.js" type="module"></script>
70
+
<script src="../site_libs/quarto-html/popper.min.js"></script>
71
+
<script src="../site_libs/quarto-html/tippy.umd.min.js"></script>
72
+
<script src="../site_libs/quarto-html/anchor.min.js"></script>
73
+
<link href="../site_libs/quarto-html/tippy.css" rel="stylesheet">
74
+
<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme" id="quarto-text-highlighting-styles">
75
+
<link href="../site_libs/quarto-html/quarto-syntax-highlighting-dark-8dcd8563ea6803ab7cbb3d71ca5772e1.css" rel="stylesheet" class="quarto-color-scheme quarto-color-alternate" id="quarto-text-highlighting-styles">
76
+
<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme-extra" id="quarto-text-highlighting-styles">
77
+
<script src="../site_libs/bootstrap/bootstrap.min.js"></script>
78
+
<link href="../site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
79
+
<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme" id="quarto-bootstrap" data-mode="light">
80
+
<link href="../site_libs/bootstrap/bootstrap-dark-7964ffd8887b0991fe8d71c6c8bc75d6.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme quarto-color-alternate" id="quarto-bootstrap" data-mode="dark">
81
+
<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme-extra" id="quarto-bootstrap" data-mode="light">
82
+
<script id="quarto-search-options" type="application/json">{
83
+
"location": "navbar",
84
+
"copy-button": false,
85
+
"collapse-after": 3,
86
+
"panel-placement": "end",
87
+
"type": "overlay",
88
+
"limit": 50,
89
+
"keyboard-shortcut": [
90
+
"f",
91
+
"/",
92
+
"s"
93
+
],
94
+
"show-item-context": false,
95
+
"language": {
96
+
"search-no-results-text": "No results",
97
+
"search-matching-documents-text": "matching documents",
98
+
"search-copy-link-title": "Copy link to search",
99
+
"search-hide-matches-text": "Hide additional matches",
100
+
"search-more-match-text": "more match in this document",
101
+
"search-more-matches-text": "more matches in this document",
102
+
"search-clear-button-title": "Clear",
103
+
"search-text-placeholder": "",
104
+
"search-detached-cancel-button-title": "Cancel",
105
+
"search-submit-button-title": "Submit",
106
+
"search-label": "Search"
107
+
}
108
+
}</script>
109
+
110
+
111
+
<link rel="stylesheet" href="../assets/styles.css">
112
+
</head>
113
+
114
+
<body class="nav-fixed quarto-light"><script id="quarto-html-before-body" type="application/javascript">
115
+
const toggleBodyColorMode = (bsSheetEl) => {
116
+
const mode = bsSheetEl.getAttribute("data-mode");
117
+
const bodyEl = window.document.querySelector("body");
118
+
if (mode === "dark") {
119
+
bodyEl.classList.add("quarto-dark");
120
+
bodyEl.classList.remove("quarto-light");
121
+
} else {
122
+
bodyEl.classList.add("quarto-light");
123
+
bodyEl.classList.remove("quarto-dark");
124
+
}
125
+
}
126
+
const toggleBodyColorPrimary = () => {
127
+
const bsSheetEl = window.document.querySelector("link#quarto-bootstrap:not([rel=disabled-stylesheet])");
128
+
if (bsSheetEl) {
129
+
toggleBodyColorMode(bsSheetEl);
130
+
}
131
+
}
132
+
const setColorSchemeToggle = (alternate) => {
133
+
const toggles = window.document.querySelectorAll('.quarto-color-scheme-toggle');
134
+
for (let i=0; i < toggles.length; i++) {
135
+
const toggle = toggles[i];
136
+
if (toggle) {
137
+
if (alternate) {
138
+
toggle.classList.add("alternate");
139
+
} else {
140
+
toggle.classList.remove("alternate");
141
+
}
142
+
}
143
+
}
144
+
};
145
+
const toggleColorMode = (alternate) => {
146
+
// Switch the stylesheets
147
+
const primaryStylesheets = window.document.querySelectorAll('link.quarto-color-scheme:not(.quarto-color-alternate)');
148
+
const alternateStylesheets = window.document.querySelectorAll('link.quarto-color-scheme.quarto-color-alternate');
149
+
manageTransitions('#quarto-margin-sidebar .nav-link', false);
150
+
if (alternate) {
151
+
// note: dark is layered on light, we don't disable primary!
152
+
enableStylesheet(alternateStylesheets);
153
+
for (const sheetNode of alternateStylesheets) {
154
+
if (sheetNode.id === "quarto-bootstrap") {
155
+
toggleBodyColorMode(sheetNode);
156
+
}
157
+
}
158
+
} else {
159
+
disableStylesheet(alternateStylesheets);
160
+
enableStylesheet(primaryStylesheets)
161
+
toggleBodyColorPrimary();
162
+
}
163
+
manageTransitions('#quarto-margin-sidebar .nav-link', true);
164
+
// Switch the toggles
165
+
setColorSchemeToggle(alternate)
166
+
// Hack to workaround the fact that safari doesn't
167
+
// properly recolor the scrollbar when toggling (#1455)
168
+
if (navigator.userAgent.indexOf('Safari') > 0 && navigator.userAgent.indexOf('Chrome') == -1) {
169
+
manageTransitions("body", false);
170
+
window.scrollTo(0, 1);
171
+
setTimeout(() => {
172
+
window.scrollTo(0, 0);
173
+
manageTransitions("body", true);
174
+
}, 40);
175
+
}
176
+
}
177
+
const disableStylesheet = (stylesheets) => {
178
+
for (let i=0; i < stylesheets.length; i++) {
179
+
const stylesheet = stylesheets[i];
180
+
stylesheet.rel = 'disabled-stylesheet';
181
+
}
182
+
}
183
+
const enableStylesheet = (stylesheets) => {
184
+
for (let i=0; i < stylesheets.length; i++) {
185
+
const stylesheet = stylesheets[i];
186
+
if(stylesheet.rel !== 'stylesheet') { // for Chrome, which will still FOUC without this check
187
+
stylesheet.rel = 'stylesheet';
188
+
}
189
+
}
190
+
}
191
+
const manageTransitions = (selector, allowTransitions) => {
192
+
const els = window.document.querySelectorAll(selector);
193
+
for (let i=0; i < els.length; i++) {
194
+
const el = els[i];
195
+
if (allowTransitions) {
196
+
el.classList.remove('notransition');
197
+
} else {
198
+
el.classList.add('notransition');
199
+
}
200
+
}
201
+
}
202
+
const isFileUrl = () => {
203
+
return window.location.protocol === 'file:';
204
+
}
205
+
const hasAlternateSentinel = () => {
206
+
let styleSentinel = getColorSchemeSentinel();
207
+
if (styleSentinel !== null) {
208
+
return styleSentinel === "alternate";
209
+
} else {
210
+
return false;
211
+
}
212
+
}
213
+
const setStyleSentinel = (alternate) => {
214
+
const value = alternate ? "alternate" : "default";
215
+
if (!isFileUrl()) {
216
+
window.localStorage.setItem("quarto-color-scheme", value);
217
+
} else {
218
+
localAlternateSentinel = value;
219
+
}
220
+
}
221
+
const getColorSchemeSentinel = () => {
222
+
if (!isFileUrl()) {
223
+
const storageValue = window.localStorage.getItem("quarto-color-scheme");
224
+
return storageValue != null ? storageValue : localAlternateSentinel;
225
+
} else {
226
+
return localAlternateSentinel;
227
+
}
228
+
}
229
+
const toggleGiscusIfUsed = (isAlternate, darkModeDefault) => {
230
+
const baseTheme = document.querySelector('#giscus-base-theme')?.value ?? 'light';
231
+
const alternateTheme = document.querySelector('#giscus-alt-theme')?.value ?? 'dark';
232
+
let newTheme = '';
233
+
if(authorPrefersDark) {
234
+
newTheme = isAlternate ? baseTheme : alternateTheme;
235
+
} else {
236
+
newTheme = isAlternate ? alternateTheme : baseTheme;
237
+
}
238
+
const changeGiscusTheme = () => {
239
+
// From: https://github.com/giscus/giscus/issues/336
240
+
const sendMessage = (message) => {
241
+
const iframe = document.querySelector('iframe.giscus-frame');
242
+
if (!iframe) return;
243
+
iframe.contentWindow.postMessage({ giscus: message }, 'https://giscus.app');
244
+
}
245
+
sendMessage({
246
+
setConfig: {
247
+
theme: newTheme
248
+
}
249
+
});
250
+
}
251
+
const isGiscussLoaded = window.document.querySelector('iframe.giscus-frame') !== null;
252
+
if (isGiscussLoaded) {
253
+
changeGiscusTheme();
254
+
}
255
+
};
256
+
const authorPrefersDark = false;
257
+
const darkModeDefault = authorPrefersDark;
258
+
document.querySelector('link#quarto-text-highlighting-styles.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
259
+
document.querySelector('link#quarto-bootstrap.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
260
+
let localAlternateSentinel = darkModeDefault ? 'alternate' : 'default';
261
+
// Dark / light mode switch
262
+
window.quartoToggleColorScheme = () => {
263
+
// Read the current dark / light value
264
+
let toAlternate = !hasAlternateSentinel();
265
+
toggleColorMode(toAlternate);
266
+
setStyleSentinel(toAlternate);
267
+
toggleGiscusIfUsed(toAlternate, darkModeDefault);
268
+
window.dispatchEvent(new Event('resize'));
269
+
};
270
+
// Switch to dark mode if need be
271
+
if (hasAlternateSentinel()) {
272
+
toggleColorMode(true);
273
+
} else {
274
+
toggleColorMode(false);
275
+
}
276
+
</script>
277
+
278
+
<div id="quarto-search-results"></div>
279
+
<header id="quarto-header" class="headroom fixed-top">
280
+
<nav class="navbar navbar-expand-lg " data-bs-theme="dark">
281
+
<div class="navbar-container container-fluid">
282
+
<div class="navbar-brand-container mx-auto">
283
+
<a class="navbar-brand" href="../index.html">
284
+
<span class="navbar-title">atdata</span>
285
+
</a>
286
+
</div>
287
+
<div id="quarto-search" class="" title="Search"></div>
288
+
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarCollapse" aria-controls="navbarCollapse" role="menu" aria-expanded="false" aria-label="Toggle navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
289
+
<span class="navbar-toggler-icon"></span>
290
+
</button>
291
+
<div class="collapse navbar-collapse" id="navbarCollapse">
292
+
<ul class="navbar-nav navbar-nav-scroll me-auto">
293
+
<li class="nav-item">
294
+
<a class="nav-link" href="../index.html">
295
+
<span class="menu-text">Guide</span></a>
296
+
</li>
297
+
<li class="nav-item dropdown ">
298
+
<a class="nav-link dropdown-toggle" href="#" id="nav-menu-tutorials" role="link" data-bs-toggle="dropdown" aria-expanded="false">
299
+
<span class="menu-text">Tutorials</span>
300
+
</a>
301
+
<ul class="dropdown-menu" aria-labelledby="nav-menu-tutorials">
302
+
<li>
303
+
<a class="dropdown-item" href="../tutorials/quickstart.html">
304
+
<span class="dropdown-text">Quick Start</span></a>
305
+
</li>
306
+
<li>
307
+
<a class="dropdown-item" href="../tutorials/local-workflow.html">
308
+
<span class="dropdown-text">Local Workflow</span></a>
309
+
</li>
310
+
<li>
311
+
<a class="dropdown-item" href="../tutorials/atmosphere.html">
312
+
<span class="dropdown-text">Atmosphere Publishing</span></a>
313
+
</li>
314
+
<li>
315
+
<a class="dropdown-item" href="../tutorials/promotion.html">
316
+
<span class="dropdown-text">Promotion Workflow</span></a>
317
+
</li>
318
+
</ul>
319
+
</li>
320
+
<li class="nav-item dropdown ">
321
+
<a class="nav-link dropdown-toggle" href="#" id="nav-menu-reference" role="link" data-bs-toggle="dropdown" aria-expanded="false">
322
+
<span class="menu-text">Reference</span>
323
+
</a>
324
+
<ul class="dropdown-menu" aria-labelledby="nav-menu-reference">
325
+
<li>
326
+
<a class="dropdown-item" href="../reference/packable-samples.html">
327
+
<span class="dropdown-text">Packable Samples</span></a>
328
+
</li>
329
+
<li>
330
+
<a class="dropdown-item" href="../reference/datasets.html">
331
+
<span class="dropdown-text">Datasets</span></a>
332
+
</li>
333
+
<li>
334
+
<a class="dropdown-item" href="../reference/lenses.html">
335
+
<span class="dropdown-text">Lenses</span></a>
336
+
</li>
337
+
<li>
338
+
<a class="dropdown-item" href="../reference/local-storage.html">
339
+
<span class="dropdown-text">Local Storage</span></a>
340
+
</li>
341
+
<li>
342
+
<a class="dropdown-item" href="../reference/atmosphere.html">
343
+
<span class="dropdown-text">Atmosphere</span></a>
344
+
</li>
345
+
<li>
346
+
<a class="dropdown-item" href="../reference/promotion.html">
347
+
<span class="dropdown-text">Promotion</span></a>
348
+
</li>
349
+
<li>
350
+
<a class="dropdown-item" href="../reference/load-dataset.html">
351
+
<span class="dropdown-text">load_dataset API</span></a>
352
+
</li>
353
+
<li>
354
+
<a class="dropdown-item" href="../reference/protocols.html">
355
+
<span class="dropdown-text">Protocols</span></a>
356
+
</li>
357
+
<li>
358
+
<a class="dropdown-item" href="../reference/uri-spec.html">
359
+
<span class="dropdown-text">URI Specification</span></a>
360
+
</li>
361
+
<li>
362
+
<a class="dropdown-item" href="../reference/troubleshooting.html">
363
+
<span class="dropdown-text">Troubleshooting & FAQ</span></a>
364
+
</li>
365
+
<li>
366
+
<a class="dropdown-item" href="../reference/deployment.html">
367
+
<span class="dropdown-text">Deployment Guide</span></a>
368
+
</li>
369
+
</ul>
370
+
</li>
371
+
<li class="nav-item">
372
+
<a class="nav-link" href="../api/index.html">
373
+
<span class="menu-text">API</span></a>
374
+
</li>
375
+
</ul>
376
+
<ul class="navbar-nav navbar-nav-scroll ms-auto">
377
+
<li class="nav-item compact">
378
+
<a class="nav-link" href="https://github.com/your-org/atdata"> <i class="bi bi-github" role="img">
379
+
</i>
380
+
<span class="menu-text"></span></a>
381
+
</li>
382
+
</ul>
383
+
</div> <!-- /navcollapse -->
384
+
<div class="quarto-navbar-tools">
385
+
<a href="" class="quarto-color-scheme-toggle quarto-navigation-tool px-1" onclick="window.quartoToggleColorScheme(); return false;" title="Toggle dark mode"><i class="bi"></i></a>
386
+
</div>
387
+
</div> <!-- /container-fluid -->
388
+
</nav>
389
+
</header>
390
+
<!-- content -->
391
+
<div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-article page-navbar">
392
+
<!-- sidebar -->
393
+
<!-- margin-sidebar -->
394
+
<div id="quarto-margin-sidebar" class="sidebar margin-sidebar">
395
+
<nav id="TOC" role="doc-toc" class="toc-active">
396
+
<h2 id="toc-title">On this page</h2>
397
+
398
+
<ul>
399
+
<li><a href="#atdata.IndexEntry" id="toc-atdata.IndexEntry" class="nav-link active" data-scroll-target="#atdata.IndexEntry">IndexEntry</a>
400
+
<ul class="collapse">
401
+
<li><a href="#attributes" id="toc-attributes" class="nav-link" data-scroll-target="#attributes">Attributes</a></li>
402
+
</ul></li>
403
+
</ul>
404
+
<div class="toc-actions"><ul><li><a href="https://github.com/your-org/atdata/edit/main/api/IndexEntry.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></nav>
405
+
</div>
406
+
<!-- main -->
407
+
<main class="content" id="quarto-document-content"><header id="title-block-header" class="quarto-title-block"></header>
408
+
409
+
410
+
411
+
412
+
413
+
<section id="atdata.IndexEntry" class="level1">
414
+
<h1>IndexEntry</h1>
415
+
<div class="sourceCode" id="cb1"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a>IndexEntry()</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
416
+
<p>Common interface for index entries (local or atmosphere).</p>
417
+
<p>Both LocalDatasetEntry and atmosphere DatasetRecord-based entries should satisfy this protocol, enabling code that works with either.</p>
418
+
<p>Properties: name: Human-readable dataset name schema_ref: Reference to schema (local:// path or AT URI) data_urls: WebDataset URLs for the data metadata: Arbitrary metadata dict, or None</p>
419
+
<section id="attributes" class="level2">
420
+
<h2 class="anchored" data-anchor-id="attributes">Attributes</h2>
421
+
<table class="caption-top table">
422
+
<thead>
423
+
<tr class="header">
424
+
<th>Name</th>
425
+
<th>Description</th>
426
+
</tr>
427
+
</thead>
428
+
<tbody>
429
+
<tr class="odd">
430
+
<td><a href="#atdata.IndexEntry.data_urls">data_urls</a></td>
431
+
<td>WebDataset URLs for the data.</td>
432
+
</tr>
433
+
<tr class="even">
434
+
<td><a href="#atdata.IndexEntry.metadata">metadata</a></td>
435
+
<td>Arbitrary metadata dictionary, or None if not set.</td>
436
+
</tr>
437
+
<tr class="odd">
438
+
<td><a href="#atdata.IndexEntry.name">name</a></td>
439
+
<td>Human-readable dataset name.</td>
440
+
</tr>
441
+
<tr class="even">
442
+
<td><a href="#atdata.IndexEntry.schema_ref">schema_ref</a></td>
443
+
<td>Reference to the schema for this dataset.</td>
444
+
</tr>
445
+
</tbody>
446
+
</table>
447
+
448
+
449
+
</section>
450
+
</section>
451
+
452
+
</main> <!-- /main -->
453
+
<script id="quarto-html-after-body" type="application/javascript">
454
+
window.document.addEventListener("DOMContentLoaded", function (event) {
455
+
// Ensure there is a toggle, if there isn't float one in the top right
456
+
if (window.document.querySelector('.quarto-color-scheme-toggle') === null) {
457
+
const a = window.document.createElement('a');
458
+
a.classList.add('top-right');
459
+
a.classList.add('quarto-color-scheme-toggle');
460
+
a.href = "";
461
+
a.onclick = function() { try { window.quartoToggleColorScheme(); } catch {} return false; };
462
+
const i = window.document.createElement("i");
463
+
i.classList.add('bi');
464
+
a.appendChild(i);
465
+
window.document.body.appendChild(a);
466
+
}
467
+
setColorSchemeToggle(hasAlternateSentinel())
468
+
const icon = "";
469
+
const anchorJS = new window.AnchorJS();
470
+
anchorJS.options = {
471
+
placement: 'right',
472
+
icon: icon
473
+
};
474
+
anchorJS.add('.anchored');
475
+
const isCodeAnnotation = (el) => {
476
+
for (const clz of el.classList) {
477
+
if (clz.startsWith('code-annotation-')) {
478
+
return true;
479
+
}
480
+
}
481
+
return false;
482
+
}
483
+
const onCopySuccess = function(e) {
484
+
// button target
485
+
const button = e.trigger;
486
+
// don't keep focus
487
+
button.blur();
488
+
// flash "checked"
489
+
button.classList.add('code-copy-button-checked');
490
+
var currentTitle = button.getAttribute("title");
491
+
button.setAttribute("title", "Copied!");
492
+
let tooltip;
493
+
if (window.bootstrap) {
494
+
button.setAttribute("data-bs-toggle", "tooltip");
495
+
button.setAttribute("data-bs-placement", "left");
496
+
button.setAttribute("data-bs-title", "Copied!");
497
+
tooltip = new bootstrap.Tooltip(button,
498
+
{ trigger: "manual",
499
+
customClass: "code-copy-button-tooltip",
500
+
offset: [0, -8]});
501
+
tooltip.show();
502
+
}
503
+
setTimeout(function() {
504
+
if (tooltip) {
505
+
tooltip.hide();
506
+
button.removeAttribute("data-bs-title");
507
+
button.removeAttribute("data-bs-toggle");
508
+
button.removeAttribute("data-bs-placement");
509
+
}
510
+
button.setAttribute("title", currentTitle);
511
+
button.classList.remove('code-copy-button-checked');
512
+
}, 1000);
513
+
// clear code selection
514
+
e.clearSelection();
515
+
}
516
+
const getTextToCopy = function(trigger) {
517
+
const codeEl = trigger.previousElementSibling.cloneNode(true);
518
+
for (const childEl of codeEl.children) {
519
+
if (isCodeAnnotation(childEl)) {
520
+
childEl.remove();
521
+
}
522
+
}
523
+
return codeEl.innerText;
524
+
}
525
+
const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
526
+
text: getTextToCopy
527
+
});
528
+
clipboard.on('success', onCopySuccess);
529
+
if (window.document.getElementById('quarto-embedded-source-code-modal')) {
530
+
const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', {
531
+
text: getTextToCopy,
532
+
container: window.document.getElementById('quarto-embedded-source-code-modal')
533
+
});
534
+
clipboardModal.on('success', onCopySuccess);
535
+
}
536
+
var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//);
537
+
var mailtoRegex = new RegExp(/^mailto:/);
538
+
var filterRegex = new RegExp("https:\/\/github\.com\/your-org\/atdata");
539
+
var isInternal = (href) => {
540
+
return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href);
541
+
}
542
+
// Inspect non-navigation links and adorn them if external
543
+
var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)');
544
+
for (var i=0; i<links.length; i++) {
545
+
const link = links[i];
546
+
if (!isInternal(link.href)) {
547
+
// undo the damage that might have been done by quarto-nav.js in the case of
548
+
// links that we want to consider external
549
+
if (link.dataset.originalHref !== undefined) {
550
+
link.href = link.dataset.originalHref;
551
+
}
552
+
}
553
+
}
554
+
function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
555
+
const config = {
556
+
allowHTML: true,
557
+
maxWidth: 500,
558
+
delay: 100,
559
+
arrow: false,
560
+
appendTo: function(el) {
561
+
return el.parentElement;
562
+
},
563
+
interactive: true,
564
+
interactiveBorder: 10,
565
+
theme: 'quarto',
566
+
placement: 'bottom-start',
567
+
};
568
+
if (contentFn) {
569
+
config.content = contentFn;
570
+
}
571
+
if (onTriggerFn) {
572
+
config.onTrigger = onTriggerFn;
573
+
}
574
+
if (onUntriggerFn) {
575
+
config.onUntrigger = onUntriggerFn;
576
+
}
577
+
window.tippy(el, config);
578
+
}
579
+
const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
580
+
for (var i=0; i<noterefs.length; i++) {
581
+
const ref = noterefs[i];
582
+
tippyHover(ref, function() {
583
+
// use id or data attribute instead here
584
+
let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
585
+
try { href = new URL(href).hash; } catch {}
586
+
const id = href.replace(/^#\/?/, "");
587
+
const note = window.document.getElementById(id);
588
+
if (note) {
589
+
return note.innerHTML;
590
+
} else {
591
+
return "";
592
+
}
593
+
});
594
+
}
595
+
const xrefs = window.document.querySelectorAll('a.quarto-xref');
596
+
const processXRef = (id, note) => {
597
+
// Strip column container classes
598
+
const stripColumnClz = (el) => {
599
+
el.classList.remove("page-full", "page-columns");
600
+
if (el.children) {
601
+
for (const child of el.children) {
602
+
stripColumnClz(child);
603
+
}
604
+
}
605
+
}
606
+
stripColumnClz(note)
607
+
if (id === null || id.startsWith('sec-')) {
608
+
// Special case sections, only their first couple elements
609
+
const container = document.createElement("div");
610
+
if (note.children && note.children.length > 2) {
611
+
container.appendChild(note.children[0].cloneNode(true));
612
+
for (let i = 1; i < note.children.length; i++) {
613
+
const child = note.children[i];
614
+
if (child.tagName === "P" && child.innerText === "") {
615
+
continue;
616
+
} else {
617
+
container.appendChild(child.cloneNode(true));
618
+
break;
619
+
}
620
+
}
621
+
if (window.Quarto?.typesetMath) {
622
+
window.Quarto.typesetMath(container);
623
+
}
624
+
return container.innerHTML
625
+
} else {
626
+
if (window.Quarto?.typesetMath) {
627
+
window.Quarto.typesetMath(note);
628
+
}
629
+
return note.innerHTML;
630
+
}
631
+
} else {
632
+
// Remove any anchor links if they are present
633
+
const anchorLink = note.querySelector('a.anchorjs-link');
634
+
if (anchorLink) {
635
+
anchorLink.remove();
636
+
}
637
+
if (window.Quarto?.typesetMath) {
638
+
window.Quarto.typesetMath(note);
639
+
}
640
+
if (note.classList.contains("callout")) {
641
+
return note.outerHTML;
642
+
} else {
643
+
return note.innerHTML;
644
+
}
645
+
}
646
+
}
647
+
for (var i=0; i<xrefs.length; i++) {
648
+
const xref = xrefs[i];
649
+
tippyHover(xref, undefined, function(instance) {
650
+
instance.disable();
651
+
let url = xref.getAttribute('href');
652
+
let hash = undefined;
653
+
if (url.startsWith('#')) {
654
+
hash = url;
655
+
} else {
656
+
try { hash = new URL(url).hash; } catch {}
657
+
}
658
+
if (hash) {
659
+
const id = hash.replace(/^#\/?/, "");
660
+
const note = window.document.getElementById(id);
661
+
if (note !== null) {
662
+
try {
663
+
const html = processXRef(id, note.cloneNode(true));
664
+
instance.setContent(html);
665
+
} finally {
666
+
instance.enable();
667
+
instance.show();
668
+
}
669
+
} else {
670
+
// See if we can fetch this
671
+
fetch(url.split('#')[0])
672
+
.then(res => res.text())
673
+
.then(html => {
674
+
const parser = new DOMParser();
675
+
const htmlDoc = parser.parseFromString(html, "text/html");
676
+
const note = htmlDoc.getElementById(id);
677
+
if (note !== null) {
678
+
const html = processXRef(id, note);
679
+
instance.setContent(html);
680
+
}
681
+
}).finally(() => {
682
+
instance.enable();
683
+
instance.show();
684
+
});
685
+
}
686
+
} else {
687
+
// See if we can fetch a full url (with no hash to target)
688
+
// This is a special case and we should probably do some content thinning / targeting
689
+
fetch(url)
690
+
.then(res => res.text())
691
+
.then(html => {
692
+
const parser = new DOMParser();
693
+
const htmlDoc = parser.parseFromString(html, "text/html");
694
+
const note = htmlDoc.querySelector('main.content');
695
+
if (note !== null) {
696
+
// This should only happen for chapter cross references
697
+
// (since there is no id in the URL)
698
+
// remove the first header
699
+
if (note.children.length > 0 && note.children[0].tagName === "HEADER") {
700
+
note.children[0].remove();
701
+
}
702
+
const html = processXRef(null, note);
703
+
instance.setContent(html);
704
+
}
705
+
}).finally(() => {
706
+
instance.enable();
707
+
instance.show();
708
+
});
709
+
}
710
+
}, function(instance) {
711
+
});
712
+
}
713
+
let selectedAnnoteEl;
714
+
const selectorForAnnotation = ( cell, annotation) => {
715
+
let cellAttr = 'data-code-cell="' + cell + '"';
716
+
let lineAttr = 'data-code-annotation="' + annotation + '"';
717
+
const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
718
+
return selector;
719
+
}
720
+
const selectCodeLines = (annoteEl) => {
721
+
const doc = window.document;
722
+
const targetCell = annoteEl.getAttribute("data-target-cell");
723
+
const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
724
+
const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
725
+
const lines = annoteSpan.getAttribute("data-code-lines").split(",");
726
+
const lineIds = lines.map((line) => {
727
+
return targetCell + "-" + line;
728
+
})
729
+
let top = null;
730
+
let height = null;
731
+
let parent = null;
732
+
if (lineIds.length > 0) {
733
+
//compute the position of the single el (top and bottom and make a div)
734
+
const el = window.document.getElementById(lineIds[0]);
735
+
top = el.offsetTop;
736
+
height = el.offsetHeight;
737
+
parent = el.parentElement.parentElement;
738
+
if (lineIds.length > 1) {
739
+
const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
740
+
const bottom = lastEl.offsetTop + lastEl.offsetHeight;
741
+
height = bottom - top;
742
+
}
743
+
if (top !== null && height !== null && parent !== null) {
744
+
// cook up a div (if necessary) and position it
745
+
let div = window.document.getElementById("code-annotation-line-highlight");
746
+
if (div === null) {
747
+
div = window.document.createElement("div");
748
+
div.setAttribute("id", "code-annotation-line-highlight");
749
+
div.style.position = 'absolute';
750
+
parent.appendChild(div);
751
+
}
752
+
div.style.top = top - 2 + "px";
753
+
div.style.height = height + 4 + "px";
754
+
div.style.left = 0;
755
+
let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
756
+
if (gutterDiv === null) {
757
+
gutterDiv = window.document.createElement("div");
758
+
gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
759
+
gutterDiv.style.position = 'absolute';
760
+
const codeCell = window.document.getElementById(targetCell);
761
+
const gutter = codeCell.querySelector('.code-annotation-gutter');
762
+
gutter.appendChild(gutterDiv);
763
+
}
764
+
gutterDiv.style.top = top - 2 + "px";
765
+
gutterDiv.style.height = height + 4 + "px";
766
+
}
767
+
selectedAnnoteEl = annoteEl;
768
+
}
769
+
};
770
+
const unselectCodeLines = () => {
771
+
const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
772
+
elementsIds.forEach((elId) => {
773
+
const div = window.document.getElementById(elId);
774
+
if (div) {
775
+
div.remove();
776
+
}
777
+
});
778
+
selectedAnnoteEl = undefined;
779
+
};
780
+
// Handle positioning of the toggle
781
+
window.addEventListener(
782
+
"resize",
783
+
throttle(() => {
784
+
elRect = undefined;
785
+
if (selectedAnnoteEl) {
786
+
selectCodeLines(selectedAnnoteEl);
787
+
}
788
+
}, 10)
789
+
);
790
+
function throttle(fn, ms) {
791
+
let throttle = false;
792
+
let timer;
793
+
return (...args) => {
794
+
if(!throttle) { // first call gets through
795
+
fn.apply(this, args);
796
+
throttle = true;
797
+
} else { // all the others get throttled
798
+
if(timer) clearTimeout(timer); // cancel #2
799
+
timer = setTimeout(() => {
800
+
fn.apply(this, args);
801
+
timer = throttle = false;
802
+
}, ms);
803
+
}
804
+
};
805
+
}
806
+
// Attach click handler to the DT
807
+
const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
808
+
for (const annoteDlNode of annoteDls) {
809
+
annoteDlNode.addEventListener('click', (event) => {
810
+
const clickedEl = event.target;
811
+
if (clickedEl !== selectedAnnoteEl) {
812
+
unselectCodeLines();
813
+
const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
814
+
if (activeEl) {
815
+
activeEl.classList.remove('code-annotation-active');
816
+
}
817
+
selectCodeLines(clickedEl);
818
+
clickedEl.classList.add('code-annotation-active');
819
+
} else {
820
+
// Unselect the line
821
+
unselectCodeLines();
822
+
clickedEl.classList.remove('code-annotation-active');
823
+
}
824
+
});
825
+
}
826
+
const findCites = (el) => {
827
+
const parentEl = el.parentElement;
828
+
if (parentEl) {
829
+
const cites = parentEl.dataset.cites;
830
+
if (cites) {
831
+
return {
832
+
el,
833
+
cites: cites.split(' ')
834
+
};
835
+
} else {
836
+
return findCites(el.parentElement)
837
+
}
838
+
} else {
839
+
return undefined;
840
+
}
841
+
};
842
+
var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
843
+
for (var i=0; i<bibliorefs.length; i++) {
844
+
const ref = bibliorefs[i];
845
+
const citeInfo = findCites(ref);
846
+
if (citeInfo) {
847
+
tippyHover(citeInfo.el, function() {
848
+
var popup = window.document.createElement('div');
849
+
citeInfo.cites.forEach(function(cite) {
850
+
var citeDiv = window.document.createElement('div');
851
+
citeDiv.classList.add('hanging-indent');
852
+
citeDiv.classList.add('csl-entry');
853
+
var biblioDiv = window.document.getElementById('ref-' + cite);
854
+
if (biblioDiv) {
855
+
citeDiv.innerHTML = biblioDiv.innerHTML;
856
+
}
857
+
popup.appendChild(citeDiv);
858
+
});
859
+
return popup.innerHTML;
860
+
});
861
+
}
862
+
}
863
+
});
864
+
</script>
865
+
</div> <!-- /content -->
866
+
<footer class="footer">
867
+
<div class="nav-footer">
868
+
<div class="nav-footer-left">
869
+
<p>Built with <a href="https://quarto.org/">Quarto</a></p>
870
+
</div>
871
+
<div class="nav-footer-center">
872
+
873
+
<div class="toc-actions d-sm-block d-md-none"><ul><li><a href="https://github.com/your-org/atdata/edit/main/api/IndexEntry.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></div>
874
+
<div class="nav-footer-right">
875
+
<p>MIT License</p>
876
+
</div>
877
+
</div>
878
+
</footer>
879
+
880
+
881
+
882
+
883
+
</body></html>
+913
docs/api/Packable-protocol.html
+913
docs/api/Packable-protocol.html
···
1
+
<!DOCTYPE html>
2
+
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
3
+
4
+
<meta charset="utf-8">
5
+
<meta name="generator" content="quarto-1.7.34">
6
+
7
+
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
8
+
9
+
10
+
<title>packable-protocol – atdata</title>
11
+
<style>
12
+
code{white-space: pre-wrap;}
13
+
span.smallcaps{font-variant: small-caps;}
14
+
div.columns{display: flex; gap: min(4vw, 1.5em);}
15
+
div.column{flex: auto; overflow-x: auto;}
16
+
div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
17
+
ul.task-list{list-style: none;}
18
+
ul.task-list li input[type="checkbox"] {
19
+
width: 0.8em;
20
+
margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
21
+
vertical-align: middle;
22
+
}
23
+
/* CSS for syntax highlighting */
24
+
html { -webkit-text-size-adjust: 100%; }
25
+
pre > code.sourceCode { white-space: pre; position: relative; }
26
+
pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
27
+
pre > code.sourceCode > span:empty { height: 1.2em; }
28
+
.sourceCode { overflow: visible; }
29
+
code.sourceCode > span { color: inherit; text-decoration: inherit; }
30
+
div.sourceCode { margin: 1em 0; }
31
+
pre.sourceCode { margin: 0; }
32
+
@media screen {
33
+
div.sourceCode { overflow: auto; }
34
+
}
35
+
@media print {
36
+
pre > code.sourceCode { white-space: pre-wrap; }
37
+
pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
38
+
}
39
+
pre.numberSource code
40
+
{ counter-reset: source-line 0; }
41
+
pre.numberSource code > span
42
+
{ position: relative; left: -4em; counter-increment: source-line; }
43
+
pre.numberSource code > span > a:first-child::before
44
+
{ content: counter(source-line);
45
+
position: relative; left: -1em; text-align: right; vertical-align: baseline;
46
+
border: none; display: inline-block;
47
+
-webkit-touch-callout: none; -webkit-user-select: none;
48
+
-khtml-user-select: none; -moz-user-select: none;
49
+
-ms-user-select: none; user-select: none;
50
+
padding: 0 4px; width: 4em;
51
+
}
52
+
pre.numberSource { margin-left: 3em; padding-left: 4px; }
53
+
div.sourceCode
54
+
{ }
55
+
@media screen {
56
+
pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
57
+
}
58
+
</style>
59
+
60
+
61
+
<script src="../site_libs/quarto-nav/quarto-nav.js"></script>
62
+
<script src="../site_libs/quarto-nav/headroom.min.js"></script>
63
+
<script src="../site_libs/clipboard/clipboard.min.js"></script>
64
+
<script src="../site_libs/quarto-search/autocomplete.umd.js"></script>
65
+
<script src="../site_libs/quarto-search/fuse.min.js"></script>
66
+
<script src="../site_libs/quarto-search/quarto-search.js"></script>
67
+
<meta name="quarto:offset" content="../">
68
+
<script src="../site_libs/quarto-html/quarto.js" type="module"></script>
69
+
<script src="../site_libs/quarto-html/tabsets/tabsets.js" type="module"></script>
70
+
<script src="../site_libs/quarto-html/popper.min.js"></script>
71
+
<script src="../site_libs/quarto-html/tippy.umd.min.js"></script>
72
+
<script src="../site_libs/quarto-html/anchor.min.js"></script>
73
+
<link href="../site_libs/quarto-html/tippy.css" rel="stylesheet">
74
+
<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme" id="quarto-text-highlighting-styles">
75
+
<link href="../site_libs/quarto-html/quarto-syntax-highlighting-dark-8dcd8563ea6803ab7cbb3d71ca5772e1.css" rel="stylesheet" class="quarto-color-scheme quarto-color-alternate" id="quarto-text-highlighting-styles">
76
+
<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme-extra" id="quarto-text-highlighting-styles">
77
+
<script src="../site_libs/bootstrap/bootstrap.min.js"></script>
78
+
<link href="../site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
79
+
<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme" id="quarto-bootstrap" data-mode="light">
80
+
<link href="../site_libs/bootstrap/bootstrap-dark-7964ffd8887b0991fe8d71c6c8bc75d6.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme quarto-color-alternate" id="quarto-bootstrap" data-mode="dark">
81
+
<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme-extra" id="quarto-bootstrap" data-mode="light">
82
+
<script id="quarto-search-options" type="application/json">{
83
+
"location": "navbar",
84
+
"copy-button": false,
85
+
"collapse-after": 3,
86
+
"panel-placement": "end",
87
+
"type": "overlay",
88
+
"limit": 50,
89
+
"keyboard-shortcut": [
90
+
"f",
91
+
"/",
92
+
"s"
93
+
],
94
+
"show-item-context": false,
95
+
"language": {
96
+
"search-no-results-text": "No results",
97
+
"search-matching-documents-text": "matching documents",
98
+
"search-copy-link-title": "Copy link to search",
99
+
"search-hide-matches-text": "Hide additional matches",
100
+
"search-more-match-text": "more match in this document",
101
+
"search-more-matches-text": "more matches in this document",
102
+
"search-clear-button-title": "Clear",
103
+
"search-text-placeholder": "",
104
+
"search-detached-cancel-button-title": "Cancel",
105
+
"search-submit-button-title": "Submit",
106
+
"search-label": "Search"
107
+
}
108
+
}</script>
109
+
110
+
111
+
<link rel="stylesheet" href="../assets/styles.css">
112
+
</head>
113
+
114
+
<body class="nav-fixed quarto-light"><script id="quarto-html-before-body" type="application/javascript">
115
+
const toggleBodyColorMode = (bsSheetEl) => {
116
+
const mode = bsSheetEl.getAttribute("data-mode");
117
+
const bodyEl = window.document.querySelector("body");
118
+
if (mode === "dark") {
119
+
bodyEl.classList.add("quarto-dark");
120
+
bodyEl.classList.remove("quarto-light");
121
+
} else {
122
+
bodyEl.classList.add("quarto-light");
123
+
bodyEl.classList.remove("quarto-dark");
124
+
}
125
+
}
126
+
const toggleBodyColorPrimary = () => {
127
+
const bsSheetEl = window.document.querySelector("link#quarto-bootstrap:not([rel=disabled-stylesheet])");
128
+
if (bsSheetEl) {
129
+
toggleBodyColorMode(bsSheetEl);
130
+
}
131
+
}
132
+
const setColorSchemeToggle = (alternate) => {
133
+
const toggles = window.document.querySelectorAll('.quarto-color-scheme-toggle');
134
+
for (let i=0; i < toggles.length; i++) {
135
+
const toggle = toggles[i];
136
+
if (toggle) {
137
+
if (alternate) {
138
+
toggle.classList.add("alternate");
139
+
} else {
140
+
toggle.classList.remove("alternate");
141
+
}
142
+
}
143
+
}
144
+
};
145
+
const toggleColorMode = (alternate) => {
146
+
// Switch the stylesheets
147
+
const primaryStylesheets = window.document.querySelectorAll('link.quarto-color-scheme:not(.quarto-color-alternate)');
148
+
const alternateStylesheets = window.document.querySelectorAll('link.quarto-color-scheme.quarto-color-alternate');
149
+
manageTransitions('#quarto-margin-sidebar .nav-link', false);
150
+
if (alternate) {
151
+
// note: dark is layered on light, we don't disable primary!
152
+
enableStylesheet(alternateStylesheets);
153
+
for (const sheetNode of alternateStylesheets) {
154
+
if (sheetNode.id === "quarto-bootstrap") {
155
+
toggleBodyColorMode(sheetNode);
156
+
}
157
+
}
158
+
} else {
159
+
disableStylesheet(alternateStylesheets);
160
+
enableStylesheet(primaryStylesheets)
161
+
toggleBodyColorPrimary();
162
+
}
163
+
manageTransitions('#quarto-margin-sidebar .nav-link', true);
164
+
// Switch the toggles
165
+
setColorSchemeToggle(alternate)
166
+
// Hack to workaround the fact that safari doesn't
167
+
// properly recolor the scrollbar when toggling (#1455)
168
+
if (navigator.userAgent.indexOf('Safari') > 0 && navigator.userAgent.indexOf('Chrome') == -1) {
169
+
manageTransitions("body", false);
170
+
window.scrollTo(0, 1);
171
+
setTimeout(() => {
172
+
window.scrollTo(0, 0);
173
+
manageTransitions("body", true);
174
+
}, 40);
175
+
}
176
+
}
177
+
const disableStylesheet = (stylesheets) => {
178
+
for (let i=0; i < stylesheets.length; i++) {
179
+
const stylesheet = stylesheets[i];
180
+
stylesheet.rel = 'disabled-stylesheet';
181
+
}
182
+
}
183
+
const enableStylesheet = (stylesheets) => {
184
+
for (let i=0; i < stylesheets.length; i++) {
185
+
const stylesheet = stylesheets[i];
186
+
if(stylesheet.rel !== 'stylesheet') { // for Chrome, which will still FOUC without this check
187
+
stylesheet.rel = 'stylesheet';
188
+
}
189
+
}
190
+
}
191
+
const manageTransitions = (selector, allowTransitions) => {
192
+
const els = window.document.querySelectorAll(selector);
193
+
for (let i=0; i < els.length; i++) {
194
+
const el = els[i];
195
+
if (allowTransitions) {
196
+
el.classList.remove('notransition');
197
+
} else {
198
+
el.classList.add('notransition');
199
+
}
200
+
}
201
+
}
202
+
const isFileUrl = () => {
203
+
return window.location.protocol === 'file:';
204
+
}
205
+
const hasAlternateSentinel = () => {
206
+
let styleSentinel = getColorSchemeSentinel();
207
+
if (styleSentinel !== null) {
208
+
return styleSentinel === "alternate";
209
+
} else {
210
+
return false;
211
+
}
212
+
}
213
+
const setStyleSentinel = (alternate) => {
214
+
const value = alternate ? "alternate" : "default";
215
+
if (!isFileUrl()) {
216
+
window.localStorage.setItem("quarto-color-scheme", value);
217
+
} else {
218
+
localAlternateSentinel = value;
219
+
}
220
+
}
221
+
const getColorSchemeSentinel = () => {
222
+
if (!isFileUrl()) {
223
+
const storageValue = window.localStorage.getItem("quarto-color-scheme");
224
+
return storageValue != null ? storageValue : localAlternateSentinel;
225
+
} else {
226
+
return localAlternateSentinel;
227
+
}
228
+
}
229
+
const toggleGiscusIfUsed = (isAlternate, darkModeDefault) => {
230
+
const baseTheme = document.querySelector('#giscus-base-theme')?.value ?? 'light';
231
+
const alternateTheme = document.querySelector('#giscus-alt-theme')?.value ?? 'dark';
232
+
let newTheme = '';
233
+
if(authorPrefersDark) {
234
+
newTheme = isAlternate ? baseTheme : alternateTheme;
235
+
} else {
236
+
newTheme = isAlternate ? alternateTheme : baseTheme;
237
+
}
238
+
const changeGiscusTheme = () => {
239
+
// From: https://github.com/giscus/giscus/issues/336
240
+
const sendMessage = (message) => {
241
+
const iframe = document.querySelector('iframe.giscus-frame');
242
+
if (!iframe) return;
243
+
iframe.contentWindow.postMessage({ giscus: message }, 'https://giscus.app');
244
+
}
245
+
sendMessage({
246
+
setConfig: {
247
+
theme: newTheme
248
+
}
249
+
});
250
+
}
251
+
const isGiscussLoaded = window.document.querySelector('iframe.giscus-frame') !== null;
252
+
if (isGiscussLoaded) {
253
+
changeGiscusTheme();
254
+
}
255
+
};
256
+
const authorPrefersDark = false;
257
+
const darkModeDefault = authorPrefersDark;
258
+
document.querySelector('link#quarto-text-highlighting-styles.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
259
+
document.querySelector('link#quarto-bootstrap.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
260
+
let localAlternateSentinel = darkModeDefault ? 'alternate' : 'default';
261
+
// Dark / light mode switch
262
+
window.quartoToggleColorScheme = () => {
263
+
// Read the current dark / light value
264
+
let toAlternate = !hasAlternateSentinel();
265
+
toggleColorMode(toAlternate);
266
+
setStyleSentinel(toAlternate);
267
+
toggleGiscusIfUsed(toAlternate, darkModeDefault);
268
+
window.dispatchEvent(new Event('resize'));
269
+
};
270
+
// Switch to dark mode if need be
271
+
if (hasAlternateSentinel()) {
272
+
toggleColorMode(true);
273
+
} else {
274
+
toggleColorMode(false);
275
+
}
276
+
</script>
277
+
278
+
<div id="quarto-search-results"></div>
279
+
<header id="quarto-header" class="headroom fixed-top">
280
+
<nav class="navbar navbar-expand-lg " data-bs-theme="dark">
281
+
<div class="navbar-container container-fluid">
282
+
<div class="navbar-brand-container mx-auto">
283
+
<a class="navbar-brand" href="../index.html">
284
+
<span class="navbar-title">atdata</span>
285
+
</a>
286
+
</div>
287
+
<div id="quarto-search" class="" title="Search"></div>
288
+
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarCollapse" aria-controls="navbarCollapse" role="menu" aria-expanded="false" aria-label="Toggle navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
289
+
<span class="navbar-toggler-icon"></span>
290
+
</button>
291
+
<div class="collapse navbar-collapse" id="navbarCollapse">
292
+
<ul class="navbar-nav navbar-nav-scroll me-auto">
293
+
<li class="nav-item">
294
+
<a class="nav-link" href="../index.html">
295
+
<span class="menu-text">Guide</span></a>
296
+
</li>
297
+
<li class="nav-item dropdown ">
298
+
<a class="nav-link dropdown-toggle" href="#" id="nav-menu-tutorials" role="link" data-bs-toggle="dropdown" aria-expanded="false">
299
+
<span class="menu-text">Tutorials</span>
300
+
</a>
301
+
<ul class="dropdown-menu" aria-labelledby="nav-menu-tutorials">
302
+
<li>
303
+
<a class="dropdown-item" href="../tutorials/quickstart.html">
304
+
<span class="dropdown-text">Quick Start</span></a>
305
+
</li>
306
+
<li>
307
+
<a class="dropdown-item" href="../tutorials/local-workflow.html">
308
+
<span class="dropdown-text">Local Workflow</span></a>
309
+
</li>
310
+
<li>
311
+
<a class="dropdown-item" href="../tutorials/atmosphere.html">
312
+
<span class="dropdown-text">Atmosphere Publishing</span></a>
313
+
</li>
314
+
<li>
315
+
<a class="dropdown-item" href="../tutorials/promotion.html">
316
+
<span class="dropdown-text">Promotion Workflow</span></a>
317
+
</li>
318
+
</ul>
319
+
</li>
320
+
<li class="nav-item dropdown ">
321
+
<a class="nav-link dropdown-toggle" href="#" id="nav-menu-reference" role="link" data-bs-toggle="dropdown" aria-expanded="false">
322
+
<span class="menu-text">Reference</span>
323
+
</a>
324
+
<ul class="dropdown-menu" aria-labelledby="nav-menu-reference">
325
+
<li>
326
+
<a class="dropdown-item" href="../reference/packable-samples.html">
327
+
<span class="dropdown-text">Packable Samples</span></a>
328
+
</li>
329
+
<li>
330
+
<a class="dropdown-item" href="../reference/datasets.html">
331
+
<span class="dropdown-text">Datasets</span></a>
332
+
</li>
333
+
<li>
334
+
<a class="dropdown-item" href="../reference/lenses.html">
335
+
<span class="dropdown-text">Lenses</span></a>
336
+
</li>
337
+
<li>
338
+
<a class="dropdown-item" href="../reference/local-storage.html">
339
+
<span class="dropdown-text">Local Storage</span></a>
340
+
</li>
341
+
<li>
342
+
<a class="dropdown-item" href="../reference/atmosphere.html">
343
+
<span class="dropdown-text">Atmosphere</span></a>
344
+
</li>
345
+
<li>
346
+
<a class="dropdown-item" href="../reference/promotion.html">
347
+
<span class="dropdown-text">Promotion</span></a>
348
+
</li>
349
+
<li>
350
+
<a class="dropdown-item" href="../reference/load-dataset.html">
351
+
<span class="dropdown-text">load_dataset API</span></a>
352
+
</li>
353
+
<li>
354
+
<a class="dropdown-item" href="../reference/protocols.html">
355
+
<span class="dropdown-text">Protocols</span></a>
356
+
</li>
357
+
<li>
358
+
<a class="dropdown-item" href="../reference/uri-spec.html">
359
+
<span class="dropdown-text">URI Specification</span></a>
360
+
</li>
361
+
<li>
362
+
<a class="dropdown-item" href="../reference/troubleshooting.html">
363
+
<span class="dropdown-text">Troubleshooting & FAQ</span></a>
364
+
</li>
365
+
<li>
366
+
<a class="dropdown-item" href="../reference/deployment.html">
367
+
<span class="dropdown-text">Deployment Guide</span></a>
368
+
</li>
369
+
</ul>
370
+
</li>
371
+
<li class="nav-item">
372
+
<a class="nav-link" href="../api/index.html">
373
+
<span class="menu-text">API</span></a>
374
+
</li>
375
+
</ul>
376
+
<ul class="navbar-nav navbar-nav-scroll ms-auto">
377
+
<li class="nav-item compact">
378
+
<a class="nav-link" href="https://github.com/your-org/atdata"> <i class="bi bi-github" role="img">
379
+
</i>
380
+
<span class="menu-text"></span></a>
381
+
</li>
382
+
</ul>
383
+
</div> <!-- /navcollapse -->
384
+
<div class="quarto-navbar-tools">
385
+
<a href="" class="quarto-color-scheme-toggle quarto-navigation-tool px-1" onclick="window.quartoToggleColorScheme(); return false;" title="Toggle dark mode"><i class="bi"></i></a>
386
+
</div>
387
+
</div> <!-- /container-fluid -->
388
+
</nav>
389
+
</header>
390
+
<!-- content -->
391
+
<div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-article page-navbar">
392
+
<!-- sidebar -->
393
+
<!-- margin-sidebar -->
394
+
<div id="quarto-margin-sidebar" class="sidebar margin-sidebar">
395
+
<nav id="TOC" role="doc-toc" class="toc-active">
396
+
<h2 id="toc-title">On this page</h2>
397
+
398
+
<ul>
399
+
<li><a href="#atdata.Packable" id="toc-atdata.Packable" class="nav-link active" data-scroll-target="#atdata.Packable">Packable</a>
400
+
<ul class="collapse">
401
+
<li><a href="#attributes" id="toc-attributes" class="nav-link" data-scroll-target="#attributes">Attributes</a></li>
402
+
<li><a href="#methods" id="toc-methods" class="nav-link" data-scroll-target="#methods">Methods</a>
403
+
<ul class="collapse">
404
+
<li><a href="#atdata.Packable.from_bytes" id="toc-atdata.Packable.from_bytes" class="nav-link" data-scroll-target="#atdata.Packable.from_bytes">from_bytes</a></li>
405
+
<li><a href="#atdata.Packable.from_data" id="toc-atdata.Packable.from_data" class="nav-link" data-scroll-target="#atdata.Packable.from_data">from_data</a></li>
406
+
</ul></li>
407
+
</ul></li>
408
+
</ul>
409
+
<div class="toc-actions"><ul><li><a href="https://github.com/your-org/atdata/edit/main/api/Packable-protocol.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></nav>
410
+
</div>
411
+
<!-- main -->
412
+
<main class="content" id="quarto-document-content"><header id="title-block-header" class="quarto-title-block"></header>
413
+
414
+
415
+
416
+
417
+
418
+
<section id="atdata.Packable" class="level1">
419
+
<h1>Packable</h1>
420
+
<div class="sourceCode" id="cb1"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a>Packable()</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
421
+
<p>Structural protocol for packable sample types.</p>
422
+
<p>This protocol allows classes decorated with <code>@packable</code> to be recognized as valid types for lens transformations and schema operations, even though the decorator doesn’t change the class’s nominal type at static analysis time.</p>
423
+
<p>Both <code>PackableSample</code> subclasses and <code>@packable</code>-decorated classes satisfy this protocol structurally.</p>
424
+
<p>The protocol captures the full interface needed for: - Lens type transformations (as_wds, from_data) - Schema publishing (class introspection via dataclass fields) - Serialization/deserialization (packed, from_bytes)</p>
425
+
<p>Example: >>> <span class="citation" data-cites="packable">@packable</span> … class MySample: … name: str … value: int … >>> def process(sample_type: Type<a href="#atdata.Packable">Packable</a>) -> None: … # Type checker knows sample_type has from_bytes, packed, etc. … instance = sample_type.from_bytes(data) … print(instance.packed)</p>
426
+
<section id="attributes" class="level2">
427
+
<h2 class="anchored" data-anchor-id="attributes">Attributes</h2>
428
+
<table class="caption-top table">
429
+
<thead>
430
+
<tr class="header">
431
+
<th>Name</th>
432
+
<th>Description</th>
433
+
</tr>
434
+
</thead>
435
+
<tbody>
436
+
<tr class="odd">
437
+
<td><a href="#atdata.Packable.as_wds">as_wds</a></td>
438
+
<td>WebDataset-compatible representation with <strong>key</strong> and msgpack.</td>
439
+
</tr>
440
+
<tr class="even">
441
+
<td><a href="#atdata.Packable.packed">packed</a></td>
442
+
<td>Pack this sample’s data into msgpack bytes.</td>
443
+
</tr>
444
+
</tbody>
445
+
</table>
446
+
</section>
447
+
<section id="methods" class="level2">
448
+
<h2 class="anchored" data-anchor-id="methods">Methods</h2>
449
+
<table class="caption-top table">
450
+
<thead>
451
+
<tr class="header">
452
+
<th>Name</th>
453
+
<th>Description</th>
454
+
</tr>
455
+
</thead>
456
+
<tbody>
457
+
<tr class="odd">
458
+
<td><a href="#atdata.Packable.from_bytes">from_bytes</a></td>
459
+
<td>Create instance from raw msgpack bytes.</td>
460
+
</tr>
461
+
<tr class="even">
462
+
<td><a href="#atdata.Packable.from_data">from_data</a></td>
463
+
<td>Create instance from unpacked msgpack data dictionary.</td>
464
+
</tr>
465
+
</tbody>
466
+
</table>
467
+
<section id="atdata.Packable.from_bytes" class="level3">
468
+
<h3 class="anchored" data-anchor-id="atdata.Packable.from_bytes">from_bytes</h3>
469
+
<div class="sourceCode" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a>Packable.from_bytes(bs)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
470
+
<p>Create instance from raw msgpack bytes.</p>
471
+
</section>
472
+
<section id="atdata.Packable.from_data" class="level3">
473
+
<h3 class="anchored" data-anchor-id="atdata.Packable.from_data">from_data</h3>
474
+
<div class="sourceCode" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a>Packable.from_data(data)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
475
+
<p>Create instance from unpacked msgpack data dictionary.</p>
476
+
477
+
478
+
</section>
479
+
</section>
480
+
</section>
481
+
482
+
</main> <!-- /main -->
483
+
<script id="quarto-html-after-body" type="application/javascript">
484
+
window.document.addEventListener("DOMContentLoaded", function (event) {
485
+
// Ensure there is a toggle, if there isn't float one in the top right
486
+
if (window.document.querySelector('.quarto-color-scheme-toggle') === null) {
487
+
const a = window.document.createElement('a');
488
+
a.classList.add('top-right');
489
+
a.classList.add('quarto-color-scheme-toggle');
490
+
a.href = "";
491
+
a.onclick = function() { try { window.quartoToggleColorScheme(); } catch {} return false; };
492
+
const i = window.document.createElement("i");
493
+
i.classList.add('bi');
494
+
a.appendChild(i);
495
+
window.document.body.appendChild(a);
496
+
}
497
+
setColorSchemeToggle(hasAlternateSentinel())
498
+
const icon = "";
499
+
const anchorJS = new window.AnchorJS();
500
+
anchorJS.options = {
501
+
placement: 'right',
502
+
icon: icon
503
+
};
504
+
anchorJS.add('.anchored');
505
+
const isCodeAnnotation = (el) => {
506
+
for (const clz of el.classList) {
507
+
if (clz.startsWith('code-annotation-')) {
508
+
return true;
509
+
}
510
+
}
511
+
return false;
512
+
}
513
+
const onCopySuccess = function(e) {
514
+
// button target
515
+
const button = e.trigger;
516
+
// don't keep focus
517
+
button.blur();
518
+
// flash "checked"
519
+
button.classList.add('code-copy-button-checked');
520
+
var currentTitle = button.getAttribute("title");
521
+
button.setAttribute("title", "Copied!");
522
+
let tooltip;
523
+
if (window.bootstrap) {
524
+
button.setAttribute("data-bs-toggle", "tooltip");
525
+
button.setAttribute("data-bs-placement", "left");
526
+
button.setAttribute("data-bs-title", "Copied!");
527
+
tooltip = new bootstrap.Tooltip(button,
528
+
{ trigger: "manual",
529
+
customClass: "code-copy-button-tooltip",
530
+
offset: [0, -8]});
531
+
tooltip.show();
532
+
}
533
+
setTimeout(function() {
534
+
if (tooltip) {
535
+
tooltip.hide();
536
+
button.removeAttribute("data-bs-title");
537
+
button.removeAttribute("data-bs-toggle");
538
+
button.removeAttribute("data-bs-placement");
539
+
}
540
+
button.setAttribute("title", currentTitle);
541
+
button.classList.remove('code-copy-button-checked');
542
+
}, 1000);
543
+
// clear code selection
544
+
e.clearSelection();
545
+
}
546
+
const getTextToCopy = function(trigger) {
547
+
const codeEl = trigger.previousElementSibling.cloneNode(true);
548
+
for (const childEl of codeEl.children) {
549
+
if (isCodeAnnotation(childEl)) {
550
+
childEl.remove();
551
+
}
552
+
}
553
+
return codeEl.innerText;
554
+
}
555
+
const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
556
+
text: getTextToCopy
557
+
});
558
+
clipboard.on('success', onCopySuccess);
559
+
if (window.document.getElementById('quarto-embedded-source-code-modal')) {
560
+
const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', {
561
+
text: getTextToCopy,
562
+
container: window.document.getElementById('quarto-embedded-source-code-modal')
563
+
});
564
+
clipboardModal.on('success', onCopySuccess);
565
+
}
566
+
var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//);
567
+
var mailtoRegex = new RegExp(/^mailto:/);
568
+
var filterRegex = new RegExp("https:\/\/github\.com\/your-org\/atdata");
569
+
var isInternal = (href) => {
570
+
return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href);
571
+
}
572
+
// Inspect non-navigation links and adorn them if external
573
+
var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)');
574
+
for (var i=0; i<links.length; i++) {
575
+
const link = links[i];
576
+
if (!isInternal(link.href)) {
577
+
// undo the damage that might have been done by quarto-nav.js in the case of
578
+
// links that we want to consider external
579
+
if (link.dataset.originalHref !== undefined) {
580
+
link.href = link.dataset.originalHref;
581
+
}
582
+
}
583
+
}
584
+
function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
585
+
const config = {
586
+
allowHTML: true,
587
+
maxWidth: 500,
588
+
delay: 100,
589
+
arrow: false,
590
+
appendTo: function(el) {
591
+
return el.parentElement;
592
+
},
593
+
interactive: true,
594
+
interactiveBorder: 10,
595
+
theme: 'quarto',
596
+
placement: 'bottom-start',
597
+
};
598
+
if (contentFn) {
599
+
config.content = contentFn;
600
+
}
601
+
if (onTriggerFn) {
602
+
config.onTrigger = onTriggerFn;
603
+
}
604
+
if (onUntriggerFn) {
605
+
config.onUntrigger = onUntriggerFn;
606
+
}
607
+
window.tippy(el, config);
608
+
}
609
+
const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
610
+
for (var i=0; i<noterefs.length; i++) {
611
+
const ref = noterefs[i];
612
+
tippyHover(ref, function() {
613
+
// use id or data attribute instead here
614
+
let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
615
+
try { href = new URL(href).hash; } catch {}
616
+
const id = href.replace(/^#\/?/, "");
617
+
const note = window.document.getElementById(id);
618
+
if (note) {
619
+
return note.innerHTML;
620
+
} else {
621
+
return "";
622
+
}
623
+
});
624
+
}
625
+
const xrefs = window.document.querySelectorAll('a.quarto-xref');
626
+
const processXRef = (id, note) => {
627
+
// Strip column container classes
628
+
const stripColumnClz = (el) => {
629
+
el.classList.remove("page-full", "page-columns");
630
+
if (el.children) {
631
+
for (const child of el.children) {
632
+
stripColumnClz(child);
633
+
}
634
+
}
635
+
}
636
+
stripColumnClz(note)
637
+
if (id === null || id.startsWith('sec-')) {
638
+
// Special case sections, only their first couple elements
639
+
const container = document.createElement("div");
640
+
if (note.children && note.children.length > 2) {
641
+
container.appendChild(note.children[0].cloneNode(true));
642
+
for (let i = 1; i < note.children.length; i++) {
643
+
const child = note.children[i];
644
+
if (child.tagName === "P" && child.innerText === "") {
645
+
continue;
646
+
} else {
647
+
container.appendChild(child.cloneNode(true));
648
+
break;
649
+
}
650
+
}
651
+
if (window.Quarto?.typesetMath) {
652
+
window.Quarto.typesetMath(container);
653
+
}
654
+
return container.innerHTML
655
+
} else {
656
+
if (window.Quarto?.typesetMath) {
657
+
window.Quarto.typesetMath(note);
658
+
}
659
+
return note.innerHTML;
660
+
}
661
+
} else {
662
+
// Remove any anchor links if they are present
663
+
const anchorLink = note.querySelector('a.anchorjs-link');
664
+
if (anchorLink) {
665
+
anchorLink.remove();
666
+
}
667
+
if (window.Quarto?.typesetMath) {
668
+
window.Quarto.typesetMath(note);
669
+
}
670
+
if (note.classList.contains("callout")) {
671
+
return note.outerHTML;
672
+
} else {
673
+
return note.innerHTML;
674
+
}
675
+
}
676
+
}
677
+
for (var i=0; i<xrefs.length; i++) {
678
+
const xref = xrefs[i];
679
+
tippyHover(xref, undefined, function(instance) {
680
+
instance.disable();
681
+
let url = xref.getAttribute('href');
682
+
let hash = undefined;
683
+
if (url.startsWith('#')) {
684
+
hash = url;
685
+
} else {
686
+
try { hash = new URL(url).hash; } catch {}
687
+
}
688
+
if (hash) {
689
+
const id = hash.replace(/^#\/?/, "");
690
+
const note = window.document.getElementById(id);
691
+
if (note !== null) {
692
+
try {
693
+
const html = processXRef(id, note.cloneNode(true));
694
+
instance.setContent(html);
695
+
} finally {
696
+
instance.enable();
697
+
instance.show();
698
+
}
699
+
} else {
700
+
// See if we can fetch this
701
+
fetch(url.split('#')[0])
702
+
.then(res => res.text())
703
+
.then(html => {
704
+
const parser = new DOMParser();
705
+
const htmlDoc = parser.parseFromString(html, "text/html");
706
+
const note = htmlDoc.getElementById(id);
707
+
if (note !== null) {
708
+
const html = processXRef(id, note);
709
+
instance.setContent(html);
710
+
}
711
+
}).finally(() => {
712
+
instance.enable();
713
+
instance.show();
714
+
});
715
+
}
716
+
} else {
717
+
// See if we can fetch a full url (with no hash to target)
718
+
// This is a special case and we should probably do some content thinning / targeting
719
+
fetch(url)
720
+
.then(res => res.text())
721
+
.then(html => {
722
+
const parser = new DOMParser();
723
+
const htmlDoc = parser.parseFromString(html, "text/html");
724
+
const note = htmlDoc.querySelector('main.content');
725
+
if (note !== null) {
726
+
// This should only happen for chapter cross references
727
+
// (since there is no id in the URL)
728
+
// remove the first header
729
+
if (note.children.length > 0 && note.children[0].tagName === "HEADER") {
730
+
note.children[0].remove();
731
+
}
732
+
const html = processXRef(null, note);
733
+
instance.setContent(html);
734
+
}
735
+
}).finally(() => {
736
+
instance.enable();
737
+
instance.show();
738
+
});
739
+
}
740
+
}, function(instance) {
741
+
});
742
+
}
743
+
let selectedAnnoteEl;
744
+
const selectorForAnnotation = ( cell, annotation) => {
745
+
let cellAttr = 'data-code-cell="' + cell + '"';
746
+
let lineAttr = 'data-code-annotation="' + annotation + '"';
747
+
const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
748
+
return selector;
749
+
}
750
+
const selectCodeLines = (annoteEl) => {
751
+
const doc = window.document;
752
+
const targetCell = annoteEl.getAttribute("data-target-cell");
753
+
const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
754
+
const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
755
+
const lines = annoteSpan.getAttribute("data-code-lines").split(",");
756
+
const lineIds = lines.map((line) => {
757
+
return targetCell + "-" + line;
758
+
})
759
+
let top = null;
760
+
let height = null;
761
+
let parent = null;
762
+
if (lineIds.length > 0) {
763
+
//compute the position of the single el (top and bottom and make a div)
764
+
const el = window.document.getElementById(lineIds[0]);
765
+
top = el.offsetTop;
766
+
height = el.offsetHeight;
767
+
parent = el.parentElement.parentElement;
768
+
if (lineIds.length > 1) {
769
+
const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
770
+
const bottom = lastEl.offsetTop + lastEl.offsetHeight;
771
+
height = bottom - top;
772
+
}
773
+
if (top !== null && height !== null && parent !== null) {
774
+
// cook up a div (if necessary) and position it
775
+
let div = window.document.getElementById("code-annotation-line-highlight");
776
+
if (div === null) {
777
+
div = window.document.createElement("div");
778
+
div.setAttribute("id", "code-annotation-line-highlight");
779
+
div.style.position = 'absolute';
780
+
parent.appendChild(div);
781
+
}
782
+
div.style.top = top - 2 + "px";
783
+
div.style.height = height + 4 + "px";
784
+
div.style.left = 0;
785
+
let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
786
+
if (gutterDiv === null) {
787
+
gutterDiv = window.document.createElement("div");
788
+
gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
789
+
gutterDiv.style.position = 'absolute';
790
+
const codeCell = window.document.getElementById(targetCell);
791
+
const gutter = codeCell.querySelector('.code-annotation-gutter');
792
+
gutter.appendChild(gutterDiv);
793
+
}
794
+
gutterDiv.style.top = top - 2 + "px";
795
+
gutterDiv.style.height = height + 4 + "px";
796
+
}
797
+
selectedAnnoteEl = annoteEl;
798
+
}
799
+
};
800
+
const unselectCodeLines = () => {
801
+
const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
802
+
elementsIds.forEach((elId) => {
803
+
const div = window.document.getElementById(elId);
804
+
if (div) {
805
+
div.remove();
806
+
}
807
+
});
808
+
selectedAnnoteEl = undefined;
809
+
};
810
+
// Handle positioning of the toggle
811
+
window.addEventListener(
812
+
"resize",
813
+
throttle(() => {
814
+
elRect = undefined;
815
+
if (selectedAnnoteEl) {
816
+
selectCodeLines(selectedAnnoteEl);
817
+
}
818
+
}, 10)
819
+
);
820
+
function throttle(fn, ms) {
821
+
let throttle = false;
822
+
let timer;
823
+
return (...args) => {
824
+
if(!throttle) { // first call gets through
825
+
fn.apply(this, args);
826
+
throttle = true;
827
+
} else { // all the others get throttled
828
+
if(timer) clearTimeout(timer); // cancel #2
829
+
timer = setTimeout(() => {
830
+
fn.apply(this, args);
831
+
timer = throttle = false;
832
+
}, ms);
833
+
}
834
+
};
835
+
}
836
+
// Attach click handler to the DT
837
+
const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
838
+
for (const annoteDlNode of annoteDls) {
839
+
annoteDlNode.addEventListener('click', (event) => {
840
+
const clickedEl = event.target;
841
+
if (clickedEl !== selectedAnnoteEl) {
842
+
unselectCodeLines();
843
+
const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
844
+
if (activeEl) {
845
+
activeEl.classList.remove('code-annotation-active');
846
+
}
847
+
selectCodeLines(clickedEl);
848
+
clickedEl.classList.add('code-annotation-active');
849
+
} else {
850
+
// Unselect the line
851
+
unselectCodeLines();
852
+
clickedEl.classList.remove('code-annotation-active');
853
+
}
854
+
});
855
+
}
856
+
const findCites = (el) => {
857
+
const parentEl = el.parentElement;
858
+
if (parentEl) {
859
+
const cites = parentEl.dataset.cites;
860
+
if (cites) {
861
+
return {
862
+
el,
863
+
cites: cites.split(' ')
864
+
};
865
+
} else {
866
+
return findCites(el.parentElement)
867
+
}
868
+
} else {
869
+
return undefined;
870
+
}
871
+
};
872
+
var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
873
+
for (var i=0; i<bibliorefs.length; i++) {
874
+
const ref = bibliorefs[i];
875
+
const citeInfo = findCites(ref);
876
+
if (citeInfo) {
877
+
tippyHover(citeInfo.el, function() {
878
+
var popup = window.document.createElement('div');
879
+
citeInfo.cites.forEach(function(cite) {
880
+
var citeDiv = window.document.createElement('div');
881
+
citeDiv.classList.add('hanging-indent');
882
+
citeDiv.classList.add('csl-entry');
883
+
var biblioDiv = window.document.getElementById('ref-' + cite);
884
+
if (biblioDiv) {
885
+
citeDiv.innerHTML = biblioDiv.innerHTML;
886
+
}
887
+
popup.appendChild(citeDiv);
888
+
});
889
+
return popup.innerHTML;
890
+
});
891
+
}
892
+
}
893
+
});
894
+
</script>
895
+
</div> <!-- /content -->
896
+
<footer class="footer">
897
+
<div class="nav-footer">
898
+
<div class="nav-footer-left">
899
+
<p>Built with <a href="https://quarto.org/">Quarto</a></p>
900
+
</div>
901
+
<div class="nav-footer-center">
902
+
903
+
<div class="toc-actions d-sm-block d-md-none"><ul><li><a href="https://github.com/your-org/atdata/edit/main/api/Packable-protocol.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></div>
904
+
<div class="nav-footer-right">
905
+
<p>MIT License</p>
906
+
</div>
907
+
</div>
908
+
</footer>
909
+
910
+
911
+
912
+
913
+
</body></html>
+29
docs/api/index.html
+29
docs/api/index.html
···
364
364
<li><a href="#api-reference" id="toc-api-reference" class="nav-link active" data-scroll-target="#api-reference">API Reference</a>
365
365
<ul class="collapse">
366
366
<li><a href="#core" id="toc-core" class="nav-link" data-scroll-target="#core">Core</a></li>
367
+
<li><a href="#protocols" id="toc-protocols" class="nav-link" data-scroll-target="#protocols">Protocols</a></li>
367
368
<li><a href="#data-sources" id="toc-data-sources" class="nav-link" data-scroll-target="#data-sources">Data Sources</a></li>
368
369
<li><a href="#local-storage" id="toc-local-storage" class="nav-link" data-scroll-target="#local-storage">Local Storage</a></li>
369
370
<li><a href="#atmosphere" id="toc-atmosphere" class="nav-link" data-scroll-target="#atmosphere">Atmosphere</a></li>
···
421
422
<tr class="odd">
422
423
<td><a href="../api/DatasetDict.html#atdata.DatasetDict">DatasetDict</a></td>
423
424
<td>A dictionary of split names to Dataset instances.</td>
425
+
</tr>
426
+
</tbody>
427
+
</table>
428
+
</section>
429
+
<section id="protocols" class="level2">
430
+
<h2 class="anchored" data-anchor-id="protocols">Protocols</h2>
431
+
<p>Abstract protocols for storage backends</p>
432
+
<table class="caption-top table">
433
+
<tbody>
434
+
<tr class="odd">
435
+
<td><a href="../api/Packable-protocol.html#atdata.Packable">Packable</a></td>
436
+
<td>Structural protocol for packable sample types.</td>
437
+
</tr>
438
+
<tr class="even">
439
+
<td><a href="../api/IndexEntry.html#atdata.IndexEntry">IndexEntry</a></td>
440
+
<td>Common interface for index entries (local or atmosphere).</td>
441
+
</tr>
442
+
<tr class="odd">
443
+
<td><a href="../api/AbstractIndex.html#atdata.AbstractIndex">AbstractIndex</a></td>
444
+
<td>Protocol for index operations - implemented by LocalIndex and AtmosphereIndex.</td>
445
+
</tr>
446
+
<tr class="even">
447
+
<td><a href="../api/AbstractDataStore.html#atdata.AbstractDataStore">AbstractDataStore</a></td>
448
+
<td>Protocol for data storage operations.</td>
449
+
</tr>
450
+
<tr class="odd">
451
+
<td><a href="../api/DataSource.html#atdata.DataSource">DataSource</a></td>
452
+
<td>Protocol for data sources that provide streams to Dataset.</td>
424
453
</tr>
425
454
</tbody>
426
455
</table>
+6
-6
docs/index.html
+6
-6
docs/index.html
···
619
619
<h2 class="anchored" data-anchor-id="quick-example">Quick Example</h2>
620
620
<section id="define-a-sample-type" class="level3">
621
621
<h3 class="anchored" data-anchor-id="define-a-sample-type">Define a Sample Type</h3>
622
-
<div id="516ceb12" class="cell">
622
+
<div id="79809729" class="cell">
623
623
<div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
624
624
<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> numpy.typing <span class="im">import</span> NDArray</span>
625
625
<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> atdata</span>
···
633
633
</section>
634
634
<section id="create-and-write-samples" class="level3">
635
635
<h3 class="anchored" data-anchor-id="create-and-write-samples">Create and Write Samples</h3>
636
-
<div id="54c4ced8" class="cell">
636
+
<div id="e00777c6" class="cell">
637
637
<div class="sourceCode cell-code" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> webdataset <span class="im">as</span> wds</span>
638
638
<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a></span>
639
639
<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a>samples <span class="op">=</span> [</span>
···
652
652
</section>
653
653
<section id="load-and-iterate" class="level3">
654
654
<h3 class="anchored" data-anchor-id="load-and-iterate">Load and Iterate</h3>
655
-
<div id="6057ca8b" class="cell">
655
+
<div id="c980da75" class="cell">
656
656
<div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a>dataset <span class="op">=</span> atdata.Dataset[ImageSample](<span class="st">"data-000000.tar"</span>)</span>
657
657
<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a></span>
658
658
<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a><span class="co"># Iterate with batching</span></span>
···
665
665
</section>
666
666
<section id="huggingface-style-loading" class="level2">
667
667
<h2 class="anchored" data-anchor-id="huggingface-style-loading">HuggingFace-Style Loading</h2>
668
-
<div id="dca3b66c" class="cell">
668
+
<div id="1146ac5b" class="cell">
669
669
<div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Load from local path</span></span>
670
670
<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a>ds <span class="op">=</span> atdata.load_dataset(<span class="st">"path/to/data-{000000..000009}.tar"</span>, split<span class="op">=</span><span class="st">"train"</span>)</span>
671
671
<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a></span>
···
677
677
</section>
678
678
<section id="local-storage-with-redis-s3" class="level2">
679
679
<h2 class="anchored" data-anchor-id="local-storage-with-redis-s3">Local Storage with Redis + S3</h2>
680
-
<div id="31d64339" class="cell">
680
+
<div id="8667ef7f" class="cell">
681
681
<div class="sourceCode cell-code" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.local <span class="im">import</span> LocalIndex, S3DataStore</span>
682
682
<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> webdataset <span class="im">as</span> wds</span>
683
683
<span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a></span>
···
701
701
</section>
702
702
<section id="publish-to-atproto-federation" class="level2">
703
703
<h2 class="anchored" data-anchor-id="publish-to-atproto-federation">Publish to ATProto Federation</h2>
704
-
<div id="9604f1b8" class="cell">
704
+
<div id="cca46e22" class="cell">
705
705
<div class="sourceCode cell-code" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.atmosphere <span class="im">import</span> AtmosphereClient</span>
706
706
<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.promote <span class="im">import</span> promote_to_atmosphere</span>
707
707
<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a></span>
+17
-17
docs/reference/atmosphere.html
+17
-17
docs/reference/atmosphere.html
···
611
611
<section id="atmosphereclient" class="level2">
612
612
<h2 class="anchored" data-anchor-id="atmosphereclient">AtmosphereClient</h2>
613
613
<p>The client handles authentication and record operations:</p>
614
-
<div id="31b83b69" class="cell">
614
+
<div id="989b75fb" class="cell">
615
615
<div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.atmosphere <span class="im">import</span> AtmosphereClient</span>
616
616
<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a></span>
617
617
<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a>client <span class="op">=</span> AtmosphereClient()</span>
···
638
638
<section id="session-management" class="level3">
639
639
<h3 class="anchored" data-anchor-id="session-management">Session Management</h3>
640
640
<p>Save and restore sessions to avoid re-authentication:</p>
641
-
<div id="d6ef1975" class="cell">
641
+
<div id="646c68b2" class="cell">
642
642
<div class="sourceCode cell-code" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Export session for later</span></span>
643
643
<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a>session_string <span class="op">=</span> client.export_session()</span>
644
644
<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a></span>
···
650
650
<section id="custom-pds" class="level3">
651
651
<h3 class="anchored" data-anchor-id="custom-pds">Custom PDS</h3>
652
652
<p>Connect to a custom PDS instead of bsky.social:</p>
653
-
<div id="91f36df0" class="cell">
653
+
<div id="6ef0e288" class="cell">
654
654
<div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a>client <span class="op">=</span> AtmosphereClient(base_url<span class="op">=</span><span class="st">"https://pds.example.com"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
655
655
</div>
656
656
</section>
···
658
658
<section id="atmosphereindex" class="level2">
659
659
<h2 class="anchored" data-anchor-id="atmosphereindex">AtmosphereIndex</h2>
660
660
<p>The unified interface for ATProto operations, implementing the AbstractIndex protocol:</p>
661
-
<div id="93984c15" class="cell">
661
+
<div id="96f270b3" class="cell">
662
662
<div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.atmosphere <span class="im">import</span> AtmosphereClient, AtmosphereIndex</span>
663
663
<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a></span>
664
664
<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a>client <span class="op">=</span> AtmosphereClient()</span>
···
668
668
</div>
669
669
<section id="publishing-schemas" class="level3">
670
670
<h3 class="anchored" data-anchor-id="publishing-schemas">Publishing Schemas</h3>
671
-
<div id="a492d55b" class="cell">
671
+
<div id="eb6b576f" class="cell">
672
672
<div class="sourceCode cell-code" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> atdata</span>
673
673
<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> numpy.typing <span class="im">import</span> NDArray</span>
674
674
<span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a></span>
···
689
689
</section>
690
690
<section id="publishing-datasets" class="level3">
691
691
<h3 class="anchored" data-anchor-id="publishing-datasets">Publishing Datasets</h3>
692
-
<div id="aaa124a4" class="cell">
692
+
<div id="cc9f6d3f" class="cell">
693
693
<div class="sourceCode cell-code" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a>dataset <span class="op">=</span> atdata.Dataset[ImageSample](<span class="st">"data-{000000..000009}.tar"</span>)</span>
694
694
<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a></span>
695
695
<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a>entry <span class="op">=</span> index.insert_dataset(</span>
···
707
707
</section>
708
708
<section id="listing-and-retrieving" class="level3">
709
709
<h3 class="anchored" data-anchor-id="listing-and-retrieving">Listing and Retrieving</h3>
710
-
<div id="7cc0b3ba" class="cell">
710
+
<div id="06b6f91b" class="cell">
711
711
<div class="sourceCode cell-code" id="cb8"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a><span class="co"># List your datasets</span></span>
712
712
<span id="cb8-2"><a href="#cb8-2" aria-hidden="true" tabindex="-1"></a><span class="cf">for</span> entry <span class="kw">in</span> index.list_datasets():</span>
713
713
<span id="cb8-3"><a href="#cb8-3" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f"</span><span class="sc">{</span>entry<span class="sc">.</span>name<span class="sc">}</span><span class="ss">: </span><span class="sc">{</span>entry<span class="sc">.</span>schema_ref<span class="sc">}</span><span class="ss">"</span>)</span>
···
733
733
<p>For more control, use the individual publisher classes:</p>
734
734
<section id="schemapublisher" class="level3">
735
735
<h3 class="anchored" data-anchor-id="schemapublisher">SchemaPublisher</h3>
736
-
<div id="38427cdd" class="cell">
736
+
<div id="2bc57530" class="cell">
737
737
<div class="sourceCode cell-code" id="cb9"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb9-1"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.atmosphere <span class="im">import</span> SchemaPublisher</span>
738
738
<span id="cb9-2"><a href="#cb9-2" aria-hidden="true" tabindex="-1"></a></span>
739
739
<span id="cb9-3"><a href="#cb9-3" aria-hidden="true" tabindex="-1"></a>publisher <span class="op">=</span> SchemaPublisher(client)</span>
···
749
749
</section>
750
750
<section id="datasetpublisher" class="level3">
751
751
<h3 class="anchored" data-anchor-id="datasetpublisher">DatasetPublisher</h3>
752
-
<div id="73f9f633" class="cell">
752
+
<div id="ce2208c7" class="cell">
753
753
<div class="sourceCode cell-code" id="cb10"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.atmosphere <span class="im">import</span> DatasetPublisher</span>
754
754
<span id="cb10-2"><a href="#cb10-2" aria-hidden="true" tabindex="-1"></a></span>
755
755
<span id="cb10-3"><a href="#cb10-3" aria-hidden="true" tabindex="-1"></a>publisher <span class="op">=</span> DatasetPublisher(client)</span>
···
767
767
<section id="blob-storage" class="level4">
768
768
<h4 class="anchored" data-anchor-id="blob-storage">Blob Storage</h4>
769
769
<p>For smaller datasets (up to ~50MB per shard), you can store data directly in ATProto blobs instead of external URLs:</p>
770
-
<div id="38ea0ba2" class="cell">
770
+
<div id="b9ea7102" class="cell">
771
771
<div class="sourceCode cell-code" id="cb11"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb11-1"><a href="#cb11-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> io</span>
772
772
<span id="cb11-2"><a href="#cb11-2" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> webdataset <span class="im">as</span> wds</span>
773
773
<span id="cb11-3"><a href="#cb11-3" aria-hidden="true" tabindex="-1"></a></span>
···
787
787
<span id="cb11-17"><a href="#cb11-17" aria-hidden="true" tabindex="-1"></a>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
788
788
</div>
789
789
<p>To load datasets with blob storage:</p>
790
-
<div id="8ea67230" class="cell">
790
+
<div id="d420ed30" class="cell">
791
791
<div class="sourceCode cell-code" id="cb12"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb12-1"><a href="#cb12-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.atmosphere <span class="im">import</span> DatasetLoader</span>
792
792
<span id="cb12-2"><a href="#cb12-2" aria-hidden="true" tabindex="-1"></a></span>
793
793
<span id="cb12-3"><a href="#cb12-3" aria-hidden="true" tabindex="-1"></a>loader <span class="op">=</span> DatasetLoader(client)</span>
···
808
808
</section>
809
809
<section id="lenspublisher" class="level3">
810
810
<h3 class="anchored" data-anchor-id="lenspublisher">LensPublisher</h3>
811
-
<div id="52e22d84" class="cell">
811
+
<div id="af52b23d" class="cell">
812
812
<div class="sourceCode cell-code" id="cb13"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb13-1"><a href="#cb13-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.atmosphere <span class="im">import</span> LensPublisher</span>
813
813
<span id="cb13-2"><a href="#cb13-2" aria-hidden="true" tabindex="-1"></a></span>
814
814
<span id="cb13-3"><a href="#cb13-3" aria-hidden="true" tabindex="-1"></a>publisher <span class="op">=</span> LensPublisher(client)</span>
···
851
851
<p>For direct access to records, use the loader classes:</p>
852
852
<section id="schemaloader" class="level3">
853
853
<h3 class="anchored" data-anchor-id="schemaloader">SchemaLoader</h3>
854
-
<div id="dc0d0924" class="cell">
854
+
<div id="3b15532c" class="cell">
855
855
<div class="sourceCode cell-code" id="cb14"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb14-1"><a href="#cb14-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.atmosphere <span class="im">import</span> SchemaLoader</span>
856
856
<span id="cb14-2"><a href="#cb14-2" aria-hidden="true" tabindex="-1"></a></span>
857
857
<span id="cb14-3"><a href="#cb14-3" aria-hidden="true" tabindex="-1"></a>loader <span class="op">=</span> SchemaLoader(client)</span>
···
867
867
</section>
868
868
<section id="datasetloader" class="level3">
869
869
<h3 class="anchored" data-anchor-id="datasetloader">DatasetLoader</h3>
870
-
<div id="5b85a753" class="cell">
870
+
<div id="4cb996c8" class="cell">
871
871
<div class="sourceCode cell-code" id="cb15"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb15-1"><a href="#cb15-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.atmosphere <span class="im">import</span> DatasetLoader</span>
872
872
<span id="cb15-2"><a href="#cb15-2" aria-hidden="true" tabindex="-1"></a></span>
873
873
<span id="cb15-3"><a href="#cb15-3" aria-hidden="true" tabindex="-1"></a>loader <span class="op">=</span> DatasetLoader(client)</span>
···
895
895
</section>
896
896
<section id="lensloader" class="level3">
897
897
<h3 class="anchored" data-anchor-id="lensloader">LensLoader</h3>
898
-
<div id="3b83ae6b" class="cell">
898
+
<div id="89d9f79a" class="cell">
899
899
<div class="sourceCode cell-code" id="cb16"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb16-1"><a href="#cb16-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.atmosphere <span class="im">import</span> LensLoader</span>
900
900
<span id="cb16-2"><a href="#cb16-2" aria-hidden="true" tabindex="-1"></a></span>
901
901
<span id="cb16-3"><a href="#cb16-3" aria-hidden="true" tabindex="-1"></a>loader <span class="op">=</span> LensLoader(client)</span>
···
920
920
<section id="at-uris" class="level2">
921
921
<h2 class="anchored" data-anchor-id="at-uris">AT URIs</h2>
922
922
<p>ATProto records are identified by AT URIs:</p>
923
-
<div id="20bc6657" class="cell">
923
+
<div id="c79de45e" class="cell">
924
924
<div class="sourceCode cell-code" id="cb17"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb17-1"><a href="#cb17-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.atmosphere <span class="im">import</span> AtUri</span>
925
925
<span id="cb17-2"><a href="#cb17-2" aria-hidden="true" tabindex="-1"></a></span>
926
926
<span id="cb17-3"><a href="#cb17-3" aria-hidden="true" tabindex="-1"></a><span class="co"># Parse an AT URI</span></span>
···
986
986
</section>
987
987
<section id="complete-example" class="level2">
988
988
<h2 class="anchored" data-anchor-id="complete-example">Complete Example</h2>
989
-
<div id="efb60b1d" class="cell">
989
+
<div id="46b43b4f" class="cell">
990
990
<div class="sourceCode cell-code" id="cb18"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb18-1"><a href="#cb18-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
991
991
<span id="cb18-2"><a href="#cb18-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> numpy.typing <span class="im">import</span> NDArray</span>
992
992
<span id="cb18-3"><a href="#cb18-3" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> atdata</span>
+13
-13
docs/reference/datasets.html
+13
-13
docs/reference/datasets.html
···
593
593
<p>The <code>Dataset</code> class provides typed iteration over WebDataset tar files with automatic batching and lens transformations.</p>
594
594
<section id="creating-a-dataset" class="level2">
595
595
<h2 class="anchored" data-anchor-id="creating-a-dataset">Creating a Dataset</h2>
596
-
<div id="4118909e" class="cell">
596
+
<div id="51fa5997" class="cell">
597
597
<div class="sourceCode cell-code" id="cb1"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> atdata</span>
598
598
<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> numpy.typing <span class="im">import</span> NDArray</span>
599
599
<span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a></span>
···
616
616
<section id="url-source-default" class="level3">
617
617
<h3 class="anchored" data-anchor-id="url-source-default">URL Source (default)</h3>
618
618
<p>When you pass a string to <code>Dataset</code>, it automatically wraps it in a <code>URLSource</code>:</p>
619
-
<div id="2d14e057" class="cell">
619
+
<div id="0263f709" class="cell">
620
620
<div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="co"># These are equivalent:</span></span>
621
621
<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a>dataset <span class="op">=</span> atdata.Dataset[ImageSample](<span class="st">"data-{000000..000009}.tar"</span>)</span>
622
622
<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a>dataset <span class="op">=</span> atdata.Dataset[ImageSample](atdata.URLSource(<span class="st">"data-{000000..000009}.tar"</span>))</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
···
625
625
<section id="s3-source" class="level3">
626
626
<h3 class="anchored" data-anchor-id="s3-source">S3 Source</h3>
627
627
<p>For private S3 buckets or S3-compatible storage (Cloudflare R2, MinIO), use <code>S3Source</code>:</p>
628
-
<div id="71d49898" class="cell">
628
+
<div id="0d5b4a74" class="cell">
629
629
<div class="sourceCode cell-code" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="co"># From explicit credentials</span></span>
630
630
<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a>source <span class="op">=</span> atdata.S3Source(</span>
631
631
<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a> bucket<span class="op">=</span><span class="st">"my-bucket"</span>,</span>
···
663
663
<section id="ordered-iteration" class="level3">
664
664
<h3 class="anchored" data-anchor-id="ordered-iteration">Ordered Iteration</h3>
665
665
<p>Iterate through samples in their original order:</p>
666
-
<div id="2b6712c8" class="cell">
666
+
<div id="0b54f5da" class="cell">
667
667
<div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="co"># With batching (default batch_size=1)</span></span>
668
668
<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a><span class="cf">for</span> batch <span class="kw">in</span> dataset.ordered(batch_size<span class="op">=</span><span class="dv">32</span>):</span>
669
669
<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a> images <span class="op">=</span> batch.image <span class="co"># numpy array (32, H, W, C)</span></span>
···
677
677
<section id="shuffled-iteration" class="level3">
678
678
<h3 class="anchored" data-anchor-id="shuffled-iteration">Shuffled Iteration</h3>
679
679
<p>Iterate with randomized order at both shard and sample levels:</p>
680
-
<div id="5e5984cd" class="cell">
680
+
<div id="f15460c3" class="cell">
681
681
<div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="cf">for</span> batch <span class="kw">in</span> dataset.shuffled(batch_size<span class="op">=</span><span class="dv">32</span>):</span>
682
682
<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a> <span class="co"># Samples are shuffled</span></span>
683
683
<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a> process(batch)</span>
···
708
708
<section id="samplebatch" class="level2">
709
709
<h2 class="anchored" data-anchor-id="samplebatch">SampleBatch</h2>
710
710
<p>When iterating with a <code>batch_size</code>, each iteration yields a <code>SampleBatch</code> with automatic attribute aggregation.</p>
711
-
<div id="e7493a86" class="cell">
711
+
<div id="04f9af45" class="cell">
712
712
<div class="sourceCode cell-code" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
713
713
<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> Sample:</span>
714
714
<span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a> features: NDArray <span class="co"># shape (256,)</span></span>
···
728
728
<section id="type-transformations-with-lenses" class="level2">
729
729
<h2 class="anchored" data-anchor-id="type-transformations-with-lenses">Type Transformations with Lenses</h2>
730
730
<p>View a dataset through a different sample type using registered lenses:</p>
731
-
<div id="9656b5d0" class="cell">
731
+
<div id="e178d059" class="cell">
732
732
<div class="sourceCode cell-code" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
733
733
<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> SimplifiedSample:</span>
734
734
<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a> label: <span class="bu">str</span></span>
···
750
750
<section id="shard-list" class="level3">
751
751
<h3 class="anchored" data-anchor-id="shard-list">Shard List</h3>
752
752
<p>Get the list of individual tar files:</p>
753
-
<div id="368b3865" class="cell">
753
+
<div id="a83b495c" class="cell">
754
754
<div class="sourceCode cell-code" id="cb8"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a>dataset <span class="op">=</span> atdata.Dataset[Sample](<span class="st">"data-{000000..000009}.tar"</span>)</span>
755
755
<span id="cb8-2"><a href="#cb8-2" aria-hidden="true" tabindex="-1"></a>shards <span class="op">=</span> dataset.shard_list</span>
756
756
<span id="cb8-3"><a href="#cb8-3" aria-hidden="true" tabindex="-1"></a><span class="co"># ['data-000000.tar', 'data-000001.tar', ..., 'data-000009.tar']</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
···
759
759
<section id="metadata" class="level3">
760
760
<h3 class="anchored" data-anchor-id="metadata">Metadata</h3>
761
761
<p>Datasets can have associated metadata from a URL:</p>
762
-
<div id="2c7b7a9d" class="cell">
762
+
<div id="b93a6baf" class="cell">
763
763
<div class="sourceCode cell-code" id="cb9"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb9-1"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a>dataset <span class="op">=</span> atdata.Dataset[Sample](</span>
764
764
<span id="cb9-2"><a href="#cb9-2" aria-hidden="true" tabindex="-1"></a> <span class="st">"data-{000000..000009}.tar"</span>,</span>
765
765
<span id="cb9-3"><a href="#cb9-3" aria-hidden="true" tabindex="-1"></a> metadata_url<span class="op">=</span><span class="st">"https://example.com/metadata.msgpack"</span></span>
···
773
773
<section id="writing-datasets" class="level2">
774
774
<h2 class="anchored" data-anchor-id="writing-datasets">Writing Datasets</h2>
775
775
<p>Use WebDataset’s <code>TarWriter</code> or <code>ShardWriter</code> to create datasets:</p>
776
-
<div id="94773ad6" class="cell">
776
+
<div id="6915fff3" class="cell">
777
777
<div class="sourceCode cell-code" id="cb10"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> webdataset <span class="im">as</span> wds</span>
778
778
<span id="cb10-2"><a href="#cb10-2" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
779
779
<span id="cb10-3"><a href="#cb10-3" aria-hidden="true" tabindex="-1"></a></span>
···
796
796
<section id="parquet-export" class="level2">
797
797
<h2 class="anchored" data-anchor-id="parquet-export">Parquet Export</h2>
798
798
<p>Export dataset contents to parquet format:</p>
799
-
<div id="3ed69191" class="cell">
799
+
<div id="293e4903" class="cell">
800
800
<div class="sourceCode cell-code" id="cb11"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb11-1"><a href="#cb11-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Export entire dataset</span></span>
801
801
<span id="cb11-2"><a href="#cb11-2" aria-hidden="true" tabindex="-1"></a>dataset.to_parquet(<span class="st">"output.parquet"</span>)</span>
802
802
<span id="cb11-3"><a href="#cb11-3" aria-hidden="true" tabindex="-1"></a></span>
···
847
847
<section id="source" class="level3">
848
848
<h3 class="anchored" data-anchor-id="source">Source</h3>
849
849
<p>Access the underlying <code>DataSource</code>:</p>
850
-
<div id="f6b00d04" class="cell">
850
+
<div id="2f5c0957" class="cell">
851
851
<div class="sourceCode cell-code" id="cb12"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb12-1"><a href="#cb12-1" aria-hidden="true" tabindex="-1"></a>dataset <span class="op">=</span> atdata.Dataset[Sample](<span class="st">"data.tar"</span>)</span>
852
852
<span id="cb12-2"><a href="#cb12-2" aria-hidden="true" tabindex="-1"></a>source <span class="op">=</span> dataset.source <span class="co"># URLSource instance</span></span>
853
853
<span id="cb12-3"><a href="#cb12-3" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(source.shard_list) <span class="co"># ['data.tar']</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
···
856
856
<section id="sample-type" class="level3">
857
857
<h3 class="anchored" data-anchor-id="sample-type">Sample Type</h3>
858
858
<p>Get the type parameter used to create the dataset:</p>
859
-
<div id="4266807d" class="cell">
859
+
<div id="92f39647" class="cell">
860
860
<div class="sourceCode cell-code" id="cb13"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb13-1"><a href="#cb13-1" aria-hidden="true" tabindex="-1"></a>dataset <span class="op">=</span> atdata.Dataset[ImageSample](<span class="st">"data.tar"</span>)</span>
861
861
<span id="cb13-2"><a href="#cb13-2" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(dataset.sample_type) <span class="co"># <class 'ImageSample'></span></span>
862
862
<span id="cb13-3"><a href="#cb13-3" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(dataset.batch_type) <span class="co"># SampleBatch[ImageSample]</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
+10
-10
docs/reference/lenses.html
+10
-10
docs/reference/lenses.html
···
585
585
<section id="creating-a-lens" class="level2">
586
586
<h2 class="anchored" data-anchor-id="creating-a-lens">Creating a Lens</h2>
587
587
<p>Use the <code>@lens</code> decorator to define a getter:</p>
588
-
<div id="3a946ec6" class="cell">
588
+
<div id="33ea6879" class="cell">
589
589
<div class="sourceCode cell-code" id="cb1"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> atdata</span>
590
590
<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> numpy.typing <span class="im">import</span> NDArray</span>
591
591
<span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a></span>
···
615
615
<section id="adding-a-putter" class="level2">
616
616
<h2 class="anchored" data-anchor-id="adding-a-putter">Adding a Putter</h2>
617
617
<p>To enable bidirectional updates, add a putter:</p>
618
-
<div id="d4481020" class="cell">
618
+
<div id="7bdbe3d5" class="cell">
619
619
<div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="at">@simplify.putter</span></span>
620
620
<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> simplify_put(view: SimpleSample, source: FullSample) <span class="op">-></span> FullSample:</span>
621
621
<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a> <span class="cf">return</span> FullSample(</span>
···
635
635
<section id="using-lenses-with-datasets" class="level2">
636
636
<h2 class="anchored" data-anchor-id="using-lenses-with-datasets">Using Lenses with Datasets</h2>
637
637
<p>Lenses integrate with <code>Dataset.as_type()</code>:</p>
638
-
<div id="ec867516" class="cell">
638
+
<div id="e87c44ae" class="cell">
639
639
<div class="sourceCode cell-code" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a>dataset <span class="op">=</span> atdata.Dataset[FullSample](<span class="st">"data-{000000..000009}.tar"</span>)</span>
640
640
<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a></span>
641
641
<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a><span class="co"># View through a different type</span></span>
···
650
650
<section id="direct-lens-usage" class="level2">
651
651
<h2 class="anchored" data-anchor-id="direct-lens-usage">Direct Lens Usage</h2>
652
652
<p>Lenses can also be called directly:</p>
653
-
<div id="d0795fd9" class="cell">
653
+
<div id="c0864fed" class="cell">
654
654
<div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
655
655
<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a></span>
656
656
<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a>full <span class="op">=</span> FullSample(</span>
···
679
679
<div class="tab-content">
680
680
<div id="tabset-1-1" class="tab-pane active" role="tabpanel" aria-labelledby="tabset-1-1-tab">
681
681
<p>If you get a view and immediately put it back, the source is unchanged:</p>
682
-
<div id="39a52956" class="cell">
682
+
<div id="cb89e22c" class="cell">
683
683
<div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a>view <span class="op">=</span> lens.get(source)</span>
684
684
<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a><span class="cf">assert</span> lens.put(view, source) <span class="op">==</span> source</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
685
685
</div>
686
686
</div>
687
687
<div id="tabset-1-2" class="tab-pane" role="tabpanel" aria-labelledby="tabset-1-2-tab">
688
688
<p>If you put a view, getting it back yields that view:</p>
689
-
<div id="b27726d9" class="cell">
689
+
<div id="4982d795" class="cell">
690
690
<div class="sourceCode cell-code" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a>updated <span class="op">=</span> lens.put(view, source)</span>
691
691
<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a><span class="cf">assert</span> lens.get(updated) <span class="op">==</span> view</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
692
692
</div>
693
693
</div>
694
694
<div id="tabset-1-3" class="tab-pane" role="tabpanel" aria-labelledby="tabset-1-3-tab">
695
695
<p>Putting twice is equivalent to putting once with the final value:</p>
696
-
<div id="7f0d416d" class="cell">
696
+
<div id="9eb6fcc6" class="cell">
697
697
<div class="sourceCode cell-code" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a>result1 <span class="op">=</span> lens.put(v2, lens.put(v1, source))</span>
698
698
<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a>result2 <span class="op">=</span> lens.put(v2, source)</span>
699
699
<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a><span class="cf">assert</span> result1 <span class="op">==</span> result2</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
···
705
705
<section id="trivial-putter" class="level2">
706
706
<h2 class="anchored" data-anchor-id="trivial-putter">Trivial Putter</h2>
707
707
<p>If no putter is defined, a trivial putter is used that ignores view updates:</p>
708
-
<div id="f96f4352" class="cell">
708
+
<div id="437ffe9e" class="cell">
709
709
<div class="sourceCode cell-code" id="cb8"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.lens</span></span>
710
710
<span id="cb8-2"><a href="#cb8-2" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> extract_label(src: FullSample) <span class="op">-></span> SimpleSample:</span>
711
711
<span id="cb8-3"><a href="#cb8-3" aria-hidden="true" tabindex="-1"></a> <span class="cf">return</span> SimpleSample(label<span class="op">=</span>src.label, confidence<span class="op">=</span>src.confidence)</span>
···
719
719
<section id="lensnetwork-registry" class="level2">
720
720
<h2 class="anchored" data-anchor-id="lensnetwork-registry">LensNetwork Registry</h2>
721
721
<p>The <code>LensNetwork</code> is a singleton that stores all registered lenses:</p>
722
-
<div id="7f6e63d0" class="cell">
722
+
<div id="25019e3a" class="cell">
723
723
<div class="sourceCode cell-code" id="cb9"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb9-1"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.lens <span class="im">import</span> LensNetwork</span>
724
724
<span id="cb9-2"><a href="#cb9-2" aria-hidden="true" tabindex="-1"></a></span>
725
725
<span id="cb9-3"><a href="#cb9-3" aria-hidden="true" tabindex="-1"></a>network <span class="op">=</span> LensNetwork()</span>
···
736
736
</section>
737
737
<section id="example-feature-extraction" class="level2">
738
738
<h2 class="anchored" data-anchor-id="example-feature-extraction">Example: Feature Extraction</h2>
739
-
<div id="46d3d4e2" class="cell">
739
+
<div id="9f113564" class="cell">
740
740
<div class="sourceCode cell-code" id="cb10"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
741
741
<span id="cb10-2"><a href="#cb10-2" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> RawSample:</span>
742
742
<span id="cb10-3"><a href="#cb10-3" aria-hidden="true" tabindex="-1"></a> audio: NDArray</span>
+12
-12
docs/reference/load-dataset.html
+12
-12
docs/reference/load-dataset.html
···
594
594
</section>
595
595
<section id="basic-usage" class="level2">
596
596
<h2 class="anchored" data-anchor-id="basic-usage">Basic Usage</h2>
597
-
<div id="6694c30d" class="cell">
597
+
<div id="f1266ba6" class="cell">
598
598
<div class="sourceCode cell-code" id="cb1"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> atdata</span>
599
599
<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata <span class="im">import</span> load_dataset</span>
600
600
<span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> numpy.typing <span class="im">import</span> NDArray</span>
···
617
617
<h2 class="anchored" data-anchor-id="path-formats">Path Formats</h2>
618
618
<section id="webdataset-brace-notation" class="level3">
619
619
<h3 class="anchored" data-anchor-id="webdataset-brace-notation">WebDataset Brace Notation</h3>
620
-
<div id="07af51f6" class="cell">
620
+
<div id="27909507" class="cell">
621
621
<div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Range notation</span></span>
622
622
<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a>ds <span class="op">=</span> load_dataset(<span class="st">"data-{000000..000099}.tar"</span>, MySample, split<span class="op">=</span><span class="st">"train"</span>)</span>
623
623
<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a></span>
···
627
627
</section>
628
628
<section id="glob-patterns" class="level3">
629
629
<h3 class="anchored" data-anchor-id="glob-patterns">Glob Patterns</h3>
630
-
<div id="e2e9a735" class="cell">
630
+
<div id="d5dbbe76" class="cell">
631
631
<div class="sourceCode cell-code" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Match all tar files</span></span>
632
632
<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a>ds <span class="op">=</span> load_dataset(<span class="st">"path/to/*.tar"</span>, MySample)</span>
633
633
<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a></span>
···
637
637
</section>
638
638
<section id="local-directory" class="level3">
639
639
<h3 class="anchored" data-anchor-id="local-directory">Local Directory</h3>
640
-
<div id="eee1c155" class="cell">
640
+
<div id="944279f4" class="cell">
641
641
<div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Scans for .tar files</span></span>
642
642
<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a>ds <span class="op">=</span> load_dataset(<span class="st">"./my-dataset/"</span>, MySample)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
643
643
</div>
644
644
</section>
645
645
<section id="remote-urls" class="level3">
646
646
<h3 class="anchored" data-anchor-id="remote-urls">Remote URLs</h3>
647
-
<div id="4ced3f6a" class="cell">
647
+
<div id="7a335878" class="cell">
648
648
<div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="co"># S3 (public buckets)</span></span>
649
649
<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a>ds <span class="op">=</span> load_dataset(<span class="st">"s3://bucket/data-{000..099}.tar"</span>, MySample, split<span class="op">=</span><span class="st">"train"</span>)</span>
650
650
<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a></span>
···
670
670
</section>
671
671
<section id="index-lookup" class="level3">
672
672
<h3 class="anchored" data-anchor-id="index-lookup">Index Lookup</h3>
673
-
<div id="5eb99c2d" class="cell">
673
+
<div id="4af74f7c" class="cell">
674
674
<div class="sourceCode cell-code" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.local <span class="im">import</span> LocalIndex</span>
675
675
<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a></span>
676
676
<span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a>index <span class="op">=</span> LocalIndex()</span>
···
737
737
<section id="datasetdict" class="level2">
738
738
<h2 class="anchored" data-anchor-id="datasetdict">DatasetDict</h2>
739
739
<p>When loading without <code>split=</code>, returns a <code>DatasetDict</code>:</p>
740
-
<div id="88518d38" class="cell">
740
+
<div id="fb9261b7" class="cell">
741
741
<div class="sourceCode cell-code" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a>ds_dict <span class="op">=</span> load_dataset(<span class="st">"path/to/data/"</span>, MySample)</span>
742
742
<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a></span>
743
743
<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a><span class="co"># Access splits</span></span>
···
757
757
<section id="explicit-data-files" class="level2">
758
758
<h2 class="anchored" data-anchor-id="explicit-data-files">Explicit Data Files</h2>
759
759
<p>Override automatic detection with <code>data_files</code>:</p>
760
-
<div id="34fc1207" class="cell">
760
+
<div id="36a08da1" class="cell">
761
761
<div class="sourceCode cell-code" id="cb8"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Single pattern</span></span>
762
762
<span id="cb8-2"><a href="#cb8-2" aria-hidden="true" tabindex="-1"></a>ds <span class="op">=</span> load_dataset(</span>
763
763
<span id="cb8-3"><a href="#cb8-3" aria-hidden="true" tabindex="-1"></a> <span class="st">"path/to/"</span>,</span>
···
786
786
<section id="streaming-mode" class="level2">
787
787
<h2 class="anchored" data-anchor-id="streaming-mode">Streaming Mode</h2>
788
788
<p>The <code>streaming</code> parameter signals intent for streaming mode:</p>
789
-
<div id="b0f8dba2" class="cell">
789
+
<div id="f2d4367e" class="cell">
790
790
<div class="sourceCode cell-code" id="cb9"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb9-1"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Mark as streaming</span></span>
791
791
<span id="cb9-2"><a href="#cb9-2" aria-hidden="true" tabindex="-1"></a>ds_dict <span class="op">=</span> load_dataset(<span class="st">"path/to/data.tar"</span>, MySample, streaming<span class="op">=</span><span class="va">True</span>)</span>
792
792
<span id="cb9-3"><a href="#cb9-3" aria-hidden="true" tabindex="-1"></a></span>
···
811
811
<section id="auto-type-resolution" class="level2">
812
812
<h2 class="anchored" data-anchor-id="auto-type-resolution">Auto Type Resolution</h2>
813
813
<p>When using index lookup, the sample type can be resolved automatically:</p>
814
-
<div id="452c4d40" class="cell">
814
+
<div id="a0ce944d" class="cell">
815
815
<div class="sourceCode cell-code" id="cb10"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.local <span class="im">import</span> LocalIndex</span>
816
816
<span id="cb10-2"><a href="#cb10-2" aria-hidden="true" tabindex="-1"></a></span>
817
817
<span id="cb10-3"><a href="#cb10-3" aria-hidden="true" tabindex="-1"></a>index <span class="op">=</span> LocalIndex()</span>
···
825
825
</section>
826
826
<section id="error-handling" class="level2">
827
827
<h2 class="anchored" data-anchor-id="error-handling">Error Handling</h2>
828
-
<div id="661afea1" class="cell">
828
+
<div id="e19c3b8b" class="cell">
829
829
<div class="sourceCode cell-code" id="cb11"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb11-1"><a href="#cb11-1" aria-hidden="true" tabindex="-1"></a><span class="cf">try</span>:</span>
830
830
<span id="cb11-2"><a href="#cb11-2" aria-hidden="true" tabindex="-1"></a> ds <span class="op">=</span> load_dataset(<span class="st">"path/to/data.tar"</span>, MySample, split<span class="op">=</span><span class="st">"train"</span>)</span>
831
831
<span id="cb11-3"><a href="#cb11-3" aria-hidden="true" tabindex="-1"></a><span class="cf">except</span> <span class="pp">FileNotFoundError</span>:</span>
···
841
841
</section>
842
842
<section id="complete-example" class="level2">
843
843
<h2 class="anchored" data-anchor-id="complete-example">Complete Example</h2>
844
-
<div id="ece4a85d" class="cell">
844
+
<div id="86a0c23d" class="cell">
845
845
<div class="sourceCode cell-code" id="cb12"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb12-1"><a href="#cb12-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
846
846
<span id="cb12-2"><a href="#cb12-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> numpy.typing <span class="im">import</span> NDArray</span>
847
847
<span id="cb12-3"><a href="#cb12-3" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> atdata</span>
+11
-11
docs/reference/local-storage.html
+11
-11
docs/reference/local-storage.html
···
593
593
<section id="localindex" class="level2">
594
594
<h2 class="anchored" data-anchor-id="localindex">LocalIndex</h2>
595
595
<p>The index tracks datasets in Redis:</p>
596
-
<div id="4bfce399" class="cell">
596
+
<div id="e4892e76" class="cell">
597
597
<div class="sourceCode cell-code" id="cb1"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.local <span class="im">import</span> LocalIndex</span>
598
598
<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a></span>
599
599
<span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a><span class="co"># Default connection (localhost:6379)</span></span>
···
609
609
</div>
610
610
<section id="adding-entries" class="level3">
611
611
<h3 class="anchored" data-anchor-id="adding-entries">Adding Entries</h3>
612
-
<div id="6bd56062" class="cell">
612
+
<div id="32b365cd" class="cell">
613
613
<div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> atdata</span>
614
614
<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> numpy.typing <span class="im">import</span> NDArray</span>
615
615
<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a></span>
···
634
634
</section>
635
635
<section id="listing-and-retrieving" class="level3">
636
636
<h3 class="anchored" data-anchor-id="listing-and-retrieving">Listing and Retrieving</h3>
637
-
<div id="d04a5d6e" class="cell">
637
+
<div id="ee6440d2" class="cell">
638
638
<div class="sourceCode cell-code" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Iterate all entries</span></span>
639
639
<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a><span class="cf">for</span> entry <span class="kw">in</span> index.entries:</span>
640
640
<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="ss">f"</span><span class="sc">{</span>entry<span class="sc">.</span>name<span class="sc">}</span><span class="ss">: </span><span class="sc">{</span>entry<span class="sc">.</span>cid<span class="sc">}</span><span class="ss">"</span>)</span>
···
666
666
</div>
667
667
</div>
668
668
<p>The Repo class combines S3 storage with Redis indexing:</p>
669
-
<div id="d6d52fcb" class="cell">
669
+
<div id="7a621962" class="cell">
670
670
<div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.local <span class="im">import</span> Repo</span>
671
671
<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a></span>
672
672
<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a><span class="co"># From credentials file</span></span>
···
686
686
<span id="cb4-17"><a href="#cb4-17" aria-hidden="true" tabindex="-1"></a>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
687
687
</div>
688
688
<p><strong>Preferred approach</strong> - Use <code>LocalIndex</code> with <code>S3DataStore</code>:</p>
689
-
<div id="6622b354" class="cell">
689
+
<div id="9d875541" class="cell">
690
690
<div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.local <span class="im">import</span> LocalIndex, S3DataStore</span>
691
691
<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a></span>
692
692
<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a>store <span class="op">=</span> S3DataStore(</span>
···
724
724
</section>
725
725
<section id="inserting-datasets" class="level3">
726
726
<h3 class="anchored" data-anchor-id="inserting-datasets">Inserting Datasets</h3>
727
-
<div id="e6d7b89a" class="cell">
727
+
<div id="903bddd9" class="cell">
728
728
<div class="sourceCode cell-code" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> webdataset <span class="im">as</span> wds</span>
729
729
<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
730
730
<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a></span>
···
754
754
</section>
755
755
<section id="insert-options" class="level3">
756
756
<h3 class="anchored" data-anchor-id="insert-options">Insert Options</h3>
757
-
<div id="814629fc" class="cell">
757
+
<div id="19bb1cb2" class="cell">
758
758
<div class="sourceCode cell-code" id="cb8"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a>entry, ds <span class="op">=</span> repo.insert(</span>
759
759
<span id="cb8-2"><a href="#cb8-2" aria-hidden="true" tabindex="-1"></a> dataset,</span>
760
760
<span id="cb8-3"><a href="#cb8-3" aria-hidden="true" tabindex="-1"></a> name<span class="op">=</span><span class="st">"my-dataset"</span>,</span>
···
768
768
<section id="localdatasetentry" class="level2">
769
769
<h2 class="anchored" data-anchor-id="localdatasetentry">LocalDatasetEntry</h2>
770
770
<p>Index entries provide content-addressable identification:</p>
771
-
<div id="b293b1be" class="cell">
771
+
<div id="44c581ea" class="cell">
772
772
<div class="sourceCode cell-code" id="cb9"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb9-1"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a>entry <span class="op">=</span> index.get_entry_by_name(<span class="st">"my-dataset"</span>)</span>
773
773
<span id="cb9-2"><a href="#cb9-2" aria-hidden="true" tabindex="-1"></a></span>
774
774
<span id="cb9-3"><a href="#cb9-3" aria-hidden="true" tabindex="-1"></a><span class="co"># Core properties (IndexEntry protocol)</span></span>
···
801
801
<section id="schema-storage" class="level2">
802
802
<h2 class="anchored" data-anchor-id="schema-storage">Schema Storage</h2>
803
803
<p>Schemas can be stored and retrieved from the index:</p>
804
-
<div id="38f50c79" class="cell">
804
+
<div id="d7acfc3b" class="cell">
805
805
<div class="sourceCode cell-code" id="cb10"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Publish a schema</span></span>
806
806
<span id="cb10-2"><a href="#cb10-2" aria-hidden="true" tabindex="-1"></a>schema_ref <span class="op">=</span> index.publish_schema(</span>
807
807
<span id="cb10-3"><a href="#cb10-3" aria-hidden="true" tabindex="-1"></a> ImageSample,</span>
···
832
832
<section id="s3datastore" class="level2">
833
833
<h2 class="anchored" data-anchor-id="s3datastore">S3DataStore</h2>
834
834
<p>For direct S3 operations without Redis indexing:</p>
835
-
<div id="3330c365" class="cell">
835
+
<div id="ca987136" class="cell">
836
836
<div class="sourceCode cell-code" id="cb11"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb11-1"><a href="#cb11-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.local <span class="im">import</span> S3DataStore</span>
837
837
<span id="cb11-2"><a href="#cb11-2" aria-hidden="true" tabindex="-1"></a></span>
838
838
<span id="cb11-3"><a href="#cb11-3" aria-hidden="true" tabindex="-1"></a>store <span class="op">=</span> S3DataStore(</span>
···
854
854
</section>
855
855
<section id="complete-workflow-example" class="level2">
856
856
<h2 class="anchored" data-anchor-id="complete-workflow-example">Complete Workflow Example</h2>
857
-
<div id="bd605c97" class="cell">
857
+
<div id="405d0248" class="cell">
858
858
<div class="sourceCode cell-code" id="cb12"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb12-1"><a href="#cb12-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
859
859
<span id="cb12-2"><a href="#cb12-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> numpy.typing <span class="im">import</span> NDArray</span>
860
860
<span id="cb12-3"><a href="#cb12-3" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> atdata</span>
+12
-12
docs/reference/packable-samples.html
+12
-12
docs/reference/packable-samples.html
···
588
588
<section id="the-packable-decorator" class="level2">
589
589
<h2 class="anchored" data-anchor-id="the-packable-decorator">The <code>@packable</code> Decorator</h2>
590
590
<p>The recommended way to define a sample type is with the <code>@packable</code> decorator:</p>
591
-
<div id="65794c7b" class="cell">
591
+
<div id="72c5b921" class="cell">
592
592
<div class="sourceCode cell-code" id="cb1"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
593
593
<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> numpy.typing <span class="im">import</span> NDArray</span>
594
594
<span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> atdata</span>
···
610
610
<h2 class="anchored" data-anchor-id="supported-field-types">Supported Field Types</h2>
611
611
<section id="primitives" class="level3">
612
612
<h3 class="anchored" data-anchor-id="primitives">Primitives</h3>
613
-
<div id="b369087d" class="cell">
613
+
<div id="54f1d59c" class="cell">
614
614
<div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
615
615
<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> PrimitiveSample:</span>
616
616
<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a> name: <span class="bu">str</span></span>
···
623
623
<section id="numpy-arrays" class="level3">
624
624
<h3 class="anchored" data-anchor-id="numpy-arrays">NumPy Arrays</h3>
625
625
<p>Fields annotated as <code>NDArray</code> are automatically converted:</p>
626
-
<div id="893ab4da" class="cell">
626
+
<div id="c4830c19" class="cell">
627
627
<div class="sourceCode cell-code" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
628
628
<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> ArraySample:</span>
629
629
<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a> features: NDArray <span class="co"># Required array</span></span>
···
645
645
</section>
646
646
<section id="lists" class="level3">
647
647
<h3 class="anchored" data-anchor-id="lists">Lists</h3>
648
-
<div id="08382e89" class="cell">
648
+
<div id="6e0f1b5a" class="cell">
649
649
<div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
650
650
<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> ListSample:</span>
651
651
<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a> tags: <span class="bu">list</span>[<span class="bu">str</span>]</span>
···
657
657
<h2 class="anchored" data-anchor-id="serialization">Serialization</h2>
658
658
<section id="packing-to-bytes" class="level3">
659
659
<h3 class="anchored" data-anchor-id="packing-to-bytes">Packing to Bytes</h3>
660
-
<div id="08b8a4e6" class="cell">
660
+
<div id="79959f17" class="cell">
661
661
<div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a>sample <span class="op">=</span> ImageSample(</span>
662
662
<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a> image<span class="op">=</span>np.random.rand(<span class="dv">224</span>, <span class="dv">224</span>, <span class="dv">3</span>).astype(np.float32),</span>
663
663
<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a> label<span class="op">=</span><span class="st">"cat"</span>,</span>
···
671
671
</section>
672
672
<section id="unpacking-from-bytes" class="level3">
673
673
<h3 class="anchored" data-anchor-id="unpacking-from-bytes">Unpacking from Bytes</h3>
674
-
<div id="ce8b264b" class="cell">
674
+
<div id="8f5716b7" class="cell">
675
675
<div class="sourceCode cell-code" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Deserialize from bytes</span></span>
676
676
<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a>restored <span class="op">=</span> ImageSample.from_bytes(packed_bytes)</span>
677
677
<span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a></span>
···
683
683
<section id="webdataset-format" class="level3">
684
684
<h3 class="anchored" data-anchor-id="webdataset-format">WebDataset Format</h3>
685
685
<p>The <code>as_wds</code> property returns a dict ready for WebDataset:</p>
686
-
<div id="92a3b706" class="cell">
686
+
<div id="7204f700" class="cell">
687
687
<div class="sourceCode cell-code" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a>wds_dict <span class="op">=</span> sample.as_wds</span>
688
688
<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a><span class="co"># {'__key__': '1234...', 'msgpack': b'...'}</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
689
689
</div>
690
690
<p>Write samples to a tar file:</p>
691
-
<div id="da4f3a43" class="cell">
691
+
<div id="4d6fce32" class="cell">
692
692
<div class="sourceCode cell-code" id="cb8"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> webdataset <span class="im">as</span> wds</span>
693
693
<span id="cb8-2"><a href="#cb8-2" aria-hidden="true" tabindex="-1"></a></span>
694
694
<span id="cb8-3"><a href="#cb8-3" aria-hidden="true" tabindex="-1"></a><span class="cf">with</span> wds.writer.TarWriter(<span class="st">"data-000000.tar"</span>) <span class="im">as</span> sink:</span>
···
701
701
<section id="direct-inheritance-alternative" class="level2">
702
702
<h2 class="anchored" data-anchor-id="direct-inheritance-alternative">Direct Inheritance (Alternative)</h2>
703
703
<p>You can also inherit directly from <code>PackableSample</code>:</p>
704
-
<div id="01fbbe82" class="cell">
704
+
<div id="292b25b4" class="cell">
705
705
<div class="sourceCode cell-code" id="cb9"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb9-1"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> dataclasses <span class="im">import</span> dataclass</span>
706
706
<span id="cb9-2"><a href="#cb9-2" aria-hidden="true" tabindex="-1"></a></span>
707
707
<span id="cb9-3"><a href="#cb9-3" aria-hidden="true" tabindex="-1"></a><span class="at">@dataclass</span></span>
···
739
739
<section id="the-_ensure_good-method" class="level3">
740
740
<h3 class="anchored" data-anchor-id="the-_ensure_good-method">The <code>_ensure_good()</code> Method</h3>
741
741
<p>This method runs automatically after construction and handles NDArray conversion:</p>
742
-
<div id="00e6fad7" class="cell">
742
+
<div id="cb200293" class="cell">
743
743
<div class="sourceCode cell-code" id="cb10"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> _ensure_good(<span class="va">self</span>):</span>
744
744
<span id="cb10-2"><a href="#cb10-2" aria-hidden="true" tabindex="-1"></a> <span class="cf">for</span> field <span class="kw">in</span> dataclasses.fields(<span class="va">self</span>):</span>
745
745
<span id="cb10-3"><a href="#cb10-3" aria-hidden="true" tabindex="-1"></a> <span class="cf">if</span> _is_possibly_ndarray_type(field.<span class="bu">type</span>):</span>
···
755
755
<ul class="nav nav-tabs" role="tablist"><li class="nav-item" role="presentation"><a class="nav-link active" id="tabset-2-1-tab" data-bs-toggle="tab" data-bs-target="#tabset-2-1" role="tab" aria-controls="tabset-2-1" aria-selected="true">Do</a></li><li class="nav-item" role="presentation"><a class="nav-link" id="tabset-2-2-tab" data-bs-toggle="tab" data-bs-target="#tabset-2-2" role="tab" aria-controls="tabset-2-2" aria-selected="false">Don’t</a></li></ul>
756
756
<div class="tab-content">
757
757
<div id="tabset-2-1" class="tab-pane active" role="tabpanel" aria-labelledby="tabset-2-1-tab">
758
-
<div id="78ffbdd5" class="cell">
758
+
<div id="d6fc0558" class="cell">
759
759
<div class="sourceCode cell-code" id="cb11"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb11-1"><a href="#cb11-1" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
760
760
<span id="cb11-2"><a href="#cb11-2" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> GoodSample:</span>
761
761
<span id="cb11-3"><a href="#cb11-3" aria-hidden="true" tabindex="-1"></a> features: NDArray <span class="co"># Clear type annotation</span></span>
···
765
765
</div>
766
766
</div>
767
767
<div id="tabset-2-2" class="tab-pane" role="tabpanel" aria-labelledby="tabset-2-2-tab">
768
-
<div id="81e6bde1" class="cell">
768
+
<div id="c2e9f475" class="cell">
769
769
<div class="sourceCode cell-code" id="cb12"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb12-1"><a href="#cb12-1" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
770
770
<span id="cb12-2"><a href="#cb12-2" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> BadSample:</span>
771
771
<span id="cb12-3"><a href="#cb12-3" aria-hidden="true" tabindex="-1"></a> <span class="co"># DON'T: Nested dataclasses not supported</span></span>
+7
-7
docs/reference/promotion.html
+7
-7
docs/reference/promotion.html
···
584
584
</section>
585
585
<section id="basic-usage" class="level2">
586
586
<h2 class="anchored" data-anchor-id="basic-usage">Basic Usage</h2>
587
-
<div id="515920bd" class="cell">
587
+
<div id="7d14b429" class="cell">
588
588
<div class="sourceCode cell-code" id="cb1"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.local <span class="im">import</span> LocalIndex</span>
589
589
<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.atmosphere <span class="im">import</span> AtmosphereClient</span>
590
590
<span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.promote <span class="im">import</span> promote_to_atmosphere</span>
···
604
604
</section>
605
605
<section id="with-metadata" class="level2">
606
606
<h2 class="anchored" data-anchor-id="with-metadata">With Metadata</h2>
607
-
<div id="d98cd383" class="cell">
607
+
<div id="65df81fd" class="cell">
608
608
<div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a>at_uri <span class="op">=</span> promote_to_atmosphere(</span>
609
609
<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a> entry,</span>
610
610
<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a> local_index,</span>
···
619
619
<section id="schema-deduplication" class="level2">
620
620
<h2 class="anchored" data-anchor-id="schema-deduplication">Schema Deduplication</h2>
621
621
<p>The promotion workflow automatically checks for existing schemas:</p>
622
-
<div id="0f520d9a" class="cell">
622
+
<div id="ee6fb5c5" class="cell">
623
623
<div class="sourceCode cell-code" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="co"># First promotion: publishes schema</span></span>
624
624
<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a>uri1 <span class="op">=</span> promote_to_atmosphere(entry1, local_index, client)</span>
625
625
<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a></span>
···
639
639
<div class="tab-content">
640
640
<div id="tabset-1-1" class="tab-pane active" role="tabpanel" aria-labelledby="tabset-1-1-tab">
641
641
<p>By default, promotion keeps the original data URLs:</p>
642
-
<div id="cc8777d5" class="cell">
642
+
<div id="c90baa61" class="cell">
643
643
<div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Data stays in original S3 location</span></span>
644
644
<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a>at_uri <span class="op">=</span> promote_to_atmosphere(entry, local_index, client)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
645
645
</div>
···
652
652
</div>
653
653
<div id="tabset-1-2" class="tab-pane" role="tabpanel" aria-labelledby="tabset-1-2-tab">
654
654
<p>To copy data to a different storage location:</p>
655
-
<div id="a2f195b4" class="cell">
655
+
<div id="f038936e" class="cell">
656
656
<div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.local <span class="im">import</span> S3DataStore</span>
657
657
<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a></span>
658
658
<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a><span class="co"># Create new data store</span></span>
···
680
680
</section>
681
681
<section id="complete-workflow-example" class="level2">
682
682
<h2 class="anchored" data-anchor-id="complete-workflow-example">Complete Workflow Example</h2>
683
-
<div id="9859dbf6" class="cell">
683
+
<div id="50096160" class="cell">
684
684
<div class="sourceCode cell-code" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
685
685
<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> numpy.typing <span class="im">import</span> NDArray</span>
686
686
<span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> atdata</span>
···
751
751
</section>
752
752
<section id="error-handling" class="level2">
753
753
<h2 class="anchored" data-anchor-id="error-handling">Error Handling</h2>
754
-
<div id="f0fa474f" class="cell">
754
+
<div id="9ab9d034" class="cell">
755
755
<div class="sourceCode cell-code" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a><span class="cf">try</span>:</span>
756
756
<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a> at_uri <span class="op">=</span> promote_to_atmosphere(entry, local_index, client)</span>
757
757
<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a><span class="cf">except</span> <span class="pp">KeyError</span> <span class="im">as</span> e:</span>
+12
-12
docs/reference/protocols.html
+12
-12
docs/reference/protocols.html
···
605
605
<section id="indexentry-protocol" class="level2">
606
606
<h2 class="anchored" data-anchor-id="indexentry-protocol">IndexEntry Protocol</h2>
607
607
<p>Represents a dataset entry in any index:</p>
608
-
<div id="19fe2cd6" class="cell">
608
+
<div id="9cab6a83" class="cell">
609
609
<div class="sourceCode cell-code" id="cb1"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata._protocols <span class="im">import</span> IndexEntry</span>
610
610
<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a></span>
611
611
<span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> process_entry(entry: IndexEntry) <span class="op">-></span> <span class="va">None</span>:</span>
···
659
659
<section id="abstractindex-protocol" class="level2">
660
660
<h2 class="anchored" data-anchor-id="abstractindex-protocol">AbstractIndex Protocol</h2>
661
661
<p>Defines operations for managing schemas and datasets:</p>
662
-
<div id="a75470ed" class="cell">
662
+
<div id="3e25c48c" class="cell">
663
663
<div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata._protocols <span class="im">import</span> AbstractIndex</span>
664
664
<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a></span>
665
665
<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> list_all_datasets(index: AbstractIndex) <span class="op">-></span> <span class="va">None</span>:</span>
···
669
669
</div>
670
670
<section id="dataset-operations" class="level3">
671
671
<h3 class="anchored" data-anchor-id="dataset-operations">Dataset Operations</h3>
672
-
<div id="ff782e98" class="cell">
672
+
<div id="791a6bd5" class="cell">
673
673
<div class="sourceCode cell-code" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Insert a dataset</span></span>
674
674
<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a>entry <span class="op">=</span> index.insert_dataset(</span>
675
675
<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a> dataset,</span>
···
687
687
</section>
688
688
<section id="schema-operations" class="level3">
689
689
<h3 class="anchored" data-anchor-id="schema-operations">Schema Operations</h3>
690
-
<div id="57c9d3e6" class="cell">
690
+
<div id="9054849a" class="cell">
691
691
<div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Publish a schema</span></span>
692
692
<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a>schema_ref <span class="op">=</span> index.publish_schema(</span>
693
693
<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a> MySample,</span>
···
718
718
<section id="abstractdatastore-protocol" class="level2">
719
719
<h2 class="anchored" data-anchor-id="abstractdatastore-protocol">AbstractDataStore Protocol</h2>
720
720
<p>Abstracts over different storage backends:</p>
721
-
<div id="8b36ad14" class="cell">
721
+
<div id="fabf7644" class="cell">
722
722
<div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata._protocols <span class="im">import</span> AbstractDataStore</span>
723
723
<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a></span>
724
724
<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> write_dataset(store: AbstractDataStore, dataset) <span class="op">-></span> <span class="bu">list</span>[<span class="bu">str</span>]:</span>
···
728
728
</div>
729
729
<section id="methods" class="level3">
730
730
<h3 class="anchored" data-anchor-id="methods">Methods</h3>
731
-
<div id="5043521e" class="cell">
731
+
<div id="4a9f329b" class="cell">
732
732
<div class="sourceCode cell-code" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Write dataset shards</span></span>
733
733
<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a>urls <span class="op">=</span> store.write_shards(</span>
734
734
<span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a> dataset,</span>
···
755
755
<section id="datasource-protocol" class="level2">
756
756
<h2 class="anchored" data-anchor-id="datasource-protocol">DataSource Protocol</h2>
757
757
<p>Abstracts over different data source backends for streaming dataset shards:</p>
758
-
<div id="7a4b4c2d" class="cell">
758
+
<div id="cbc7c002" class="cell">
759
759
<div class="sourceCode cell-code" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata._protocols <span class="im">import</span> DataSource</span>
760
760
<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a></span>
761
761
<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a><span class="kw">def</span> load_from_source(source: DataSource) <span class="op">-></span> <span class="va">None</span>:</span>
···
768
768
</div>
769
769
<section id="methods-1" class="level3">
770
770
<h3 class="anchored" data-anchor-id="methods-1">Methods</h3>
771
-
<div id="515e17c1" class="cell">
771
+
<div id="9316c6c5" class="cell">
772
772
<div class="sourceCode cell-code" id="cb8"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Get list of shard identifiers</span></span>
773
773
<span id="cb8-2"><a href="#cb8-2" aria-hidden="true" tabindex="-1"></a>shard_ids <span class="op">=</span> source.shard_list <span class="co"># ['data-000000.tar', 'data-000001.tar', ...]</span></span>
774
774
<span id="cb8-3"><a href="#cb8-3" aria-hidden="true" tabindex="-1"></a></span>
···
791
791
<section id="creating-custom-data-sources" class="level3">
792
792
<h3 class="anchored" data-anchor-id="creating-custom-data-sources">Creating Custom Data Sources</h3>
793
793
<p>Implement the <code>DataSource</code> protocol for custom backends:</p>
794
-
<div id="a4acbb90" class="cell">
794
+
<div id="d217fc30" class="cell">
795
795
<div class="sourceCode cell-code" id="cb9"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb9-1"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> typing <span class="im">import</span> Iterator, IO</span>
796
796
<span id="cb9-2"><a href="#cb9-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata._protocols <span class="im">import</span> DataSource</span>
797
797
<span id="cb9-3"><a href="#cb9-3" aria-hidden="true" tabindex="-1"></a></span>
···
829
829
<section id="using-protocols-for-polymorphism" class="level2">
830
830
<h2 class="anchored" data-anchor-id="using-protocols-for-polymorphism">Using Protocols for Polymorphism</h2>
831
831
<p>Write code that works with any backend:</p>
832
-
<div id="c67263a4" class="cell">
832
+
<div id="ad7b8caa" class="cell">
833
833
<div class="sourceCode cell-code" id="cb10"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata._protocols <span class="im">import</span> AbstractIndex, IndexEntry</span>
834
834
<span id="cb10-2"><a href="#cb10-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata <span class="im">import</span> Dataset</span>
835
835
<span id="cb10-3"><a href="#cb10-3" aria-hidden="true" tabindex="-1"></a></span>
···
900
900
<section id="type-checking" class="level2">
901
901
<h2 class="anchored" data-anchor-id="type-checking">Type Checking</h2>
902
902
<p>Protocols are runtime-checkable:</p>
903
-
<div id="9f6cc88a" class="cell">
903
+
<div id="a1a0abd3" class="cell">
904
904
<div class="sourceCode cell-code" id="cb11"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb11-1"><a href="#cb11-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata._protocols <span class="im">import</span> IndexEntry, AbstractIndex</span>
905
905
<span id="cb11-2"><a href="#cb11-2" aria-hidden="true" tabindex="-1"></a></span>
906
906
<span id="cb11-3"><a href="#cb11-3" aria-hidden="true" tabindex="-1"></a><span class="co"># Check if object implements protocol</span></span>
···
914
914
</section>
915
915
<section id="complete-example" class="level2">
916
916
<h2 class="anchored" data-anchor-id="complete-example">Complete Example</h2>
917
-
<div id="498dd50e" class="cell">
917
+
<div id="3dc654c8" class="cell">
918
918
<div class="sourceCode cell-code" id="cb12"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb12-1"><a href="#cb12-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> atdata</span>
919
919
<span id="cb12-2"><a href="#cb12-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.local <span class="im">import</span> LocalIndex, S3DataStore</span>
920
920
<span id="cb12-3"><a href="#cb12-3" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.atmosphere <span class="im">import</span> AtmosphereClient, AtmosphereIndex</span>
+2
-2
docs/reference/uri-spec.html
+2
-2
docs/reference/uri-spec.html
···
675
675
<h2 class="anchored" data-anchor-id="examples">Examples</h2>
676
676
<section id="local-development" class="level3">
677
677
<h3 class="anchored" data-anchor-id="local-development">Local Development</h3>
678
-
<div id="46154f2f" class="cell">
678
+
<div id="b57d1bc3" class="cell">
679
679
<div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.local <span class="im">import</span> Index</span>
680
680
<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a></span>
681
681
<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a>index <span class="op">=</span> Index()</span>
···
694
694
</section>
695
695
<section id="atmosphere-atproto-federation" class="level3">
696
696
<h3 class="anchored" data-anchor-id="atmosphere-atproto-federation">Atmosphere (ATProto Federation)</h3>
697
-
<div id="98230d1d" class="cell">
697
+
<div id="0af8bcbf" class="cell">
698
698
<div class="sourceCode cell-code" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.atmosphere <span class="im">import</span> Client</span>
699
699
<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a></span>
700
700
<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a>client <span class="op">=</span> Client()</span>
+169
-71
docs/search.json
+169
-71
docs/search.json
···
1135
1135
"text": "Name\nDescription\n\n\n\n\ndecode_schema\nReconstruct a Python type from a schema record.\n\n\nget_dataset\nGet a dataset by AT URI.\n\n\nget_schema\nGet a schema record by AT URI.\n\n\ninsert_dataset\nInsert a dataset into ATProto.\n\n\nlist_datasets\nGet all dataset entries as a materialized list (AbstractIndex protocol).\n\n\nlist_schemas\nGet all schema records as a materialized list (AbstractIndex protocol).\n\n\npublish_schema\nPublish a schema to ATProto.\n\n\n\n\n\natmosphere.AtmosphereIndex.decode_schema(ref)\nReconstruct a Python type from a schema record.\nArgs: ref: AT URI of the schema record.\nReturns: Dynamically generated Packable type.\nRaises: ValueError: If schema cannot be decoded.\n\n\n\natmosphere.AtmosphereIndex.get_dataset(ref)\nGet a dataset by AT URI.\nArgs: ref: AT URI of the dataset record.\nReturns: AtmosphereIndexEntry for the dataset.\nRaises: ValueError: If record is not a dataset.\n\n\n\natmosphere.AtmosphereIndex.get_schema(ref)\nGet a schema record by AT URI.\nArgs: ref: AT URI of the schema record.\nReturns: Schema record dictionary.\nRaises: ValueError: If record is not a schema.\n\n\n\natmosphere.AtmosphereIndex.insert_dataset(\n ds,\n *,\n name,\n schema_ref=None,\n **kwargs,\n)\nInsert a dataset into ATProto.\nArgs: ds: The Dataset to publish. name: Human-readable name. schema_ref: Optional schema AT URI. If None, auto-publishes schema. **kwargs: Additional options (description, tags, license).\nReturns: AtmosphereIndexEntry for the inserted dataset.\n\n\n\natmosphere.AtmosphereIndex.list_datasets(repo=None)\nGet all dataset entries as a materialized list (AbstractIndex protocol).\nArgs: repo: DID of repository. Defaults to authenticated user.\nReturns: List of AtmosphereIndexEntry for each dataset.\n\n\n\natmosphere.AtmosphereIndex.list_schemas(repo=None)\nGet all schema records as a materialized list (AbstractIndex protocol).\nArgs: repo: DID of repository. Defaults to authenticated user.\nReturns: List of schema records as dictionaries.\n\n\n\natmosphere.AtmosphereIndex.publish_schema(\n sample_type,\n *,\n version='1.0.0',\n **kwargs,\n)\nPublish a schema to ATProto.\nArgs: sample_type: A Packable type (PackableSample subclass or @packable-decorated). version: Semantic version string. **kwargs: Additional options (description, metadata).\nReturns: AT URI of the schema record."
1136
1136
},
1137
1137
{
1138
-
"objectID": "api/URLSource.html",
1139
-
"href": "api/URLSource.html",
1140
-
"title": "URLSource",
1138
+
"objectID": "api/DataSource.html",
1139
+
"href": "api/DataSource.html",
1140
+
"title": "DataSource",
1141
1141
"section": "",
1142
-
"text": "URLSource(url)\nData source for WebDataset-compatible URLs.\nWraps WebDataset’s gopen to open URLs using built-in handlers for http, https, pipe, gs, hf, sftp, etc. Supports brace expansion for shard patterns like “data-{000..099}.tar”.\nThis is the default source type when a string URL is passed to Dataset.\nAttributes: url: URL or brace pattern for the shards.\nExample: >>> source = URLSource(“https://example.com/train-{000..009}.tar”) >>> for shard_id, stream in source.shards: … print(f”Streaming {shard_id}“)\n\n\n\n\n\nName\nDescription\n\n\n\n\nshard_list\nExpand brace pattern and return list of shard URLs (deprecated, use list_shards()).\n\n\nshards\nLazily yield (url, stream) pairs for each shard.\n\n\n\n\n\n\n\n\n\nName\nDescription\n\n\n\n\nlist_shards\nExpand brace pattern and return list of shard URLs.\n\n\nopen_shard\nOpen a single shard by URL.\n\n\n\n\n\nURLSource.list_shards()\nExpand brace pattern and return list of shard URLs.\n\n\n\nURLSource.open_shard(shard_id)\nOpen a single shard by URL.\nArgs: shard_id: URL of the shard to open.\nReturns: File-like stream from gopen.\nRaises: KeyError: If shard_id is not in list_shards()."
1143
-
},
1144
-
{
1145
-
"objectID": "api/URLSource.html#attributes",
1146
-
"href": "api/URLSource.html#attributes",
1147
-
"title": "URLSource",
1148
-
"section": "",
1149
-
"text": "Name\nDescription\n\n\n\n\nshard_list\nExpand brace pattern and return list of shard URLs (deprecated, use list_shards()).\n\n\nshards\nLazily yield (url, stream) pairs for each shard."
1142
+
"text": "DataSource()\nProtocol for data sources that provide streams to Dataset.\nA DataSource abstracts over different ways of accessing dataset shards: - URLSource: Standard WebDataset-compatible URLs (http, https, pipe, gs, etc.) - S3Source: S3-compatible storage with explicit credentials - BlobSource: ATProto blob references (future)\nThe key method is shards(), which yields (identifier, stream) pairs. These are fed directly to WebDataset’s tar_file_expander, bypassing URL resolution entirely. This enables: - Private S3 repos with credentials - Custom endpoints (Cloudflare R2, MinIO) - ATProto blob streaming - Any other source that can provide file-like objects\nExample: >>> source = S3Source( … bucket=“my-bucket”, … keys=[“data-000.tar”, “data-001.tar”], … endpoint=“https://r2.example.com”, … credentials=creds, … ) >>> ds = DatasetMySample >>> for sample in ds.ordered(): … print(sample)\n\n\n\n\n\nName\nDescription\n\n\n\n\nshards\nLazily yield (identifier, stream) pairs for each shard.\n\n\n\n\n\n\n\n\n\nName\nDescription\n\n\n\n\nlist_shards\nGet list of shard identifiers without opening streams.\n\n\nopen_shard\nOpen a single shard by its identifier.\n\n\n\n\n\nDataSource.list_shards()\nGet list of shard identifiers without opening streams.\nUsed for metadata queries like counting shards without actually streaming data. Implementations should return identifiers that match what shards would yield.\nReturns: List of shard identifier strings.\n\n\n\nDataSource.open_shard(shard_id)\nOpen a single shard by its identifier.\nThis method enables random access to individual shards, which is required for PyTorch DataLoader worker splitting. Each worker opens only its assigned shards rather than iterating all shards.\nArgs: shard_id: Shard identifier from shard_list.\nReturns: File-like stream for reading the shard.\nRaises: KeyError: If shard_id is not in shard_list."
1150
1143
},
1151
1144
{
1152
-
"objectID": "api/URLSource.html#methods",
1153
-
"href": "api/URLSource.html#methods",
1154
-
"title": "URLSource",
1145
+
"objectID": "api/DataSource.html#attributes",
1146
+
"href": "api/DataSource.html#attributes",
1147
+
"title": "DataSource",
1155
1148
"section": "",
1156
-
"text": "Name\nDescription\n\n\n\n\nlist_shards\nExpand brace pattern and return list of shard URLs.\n\n\nopen_shard\nOpen a single shard by URL.\n\n\n\n\n\nURLSource.list_shards()\nExpand brace pattern and return list of shard URLs.\n\n\n\nURLSource.open_shard(shard_id)\nOpen a single shard by URL.\nArgs: shard_id: URL of the shard to open.\nReturns: File-like stream from gopen.\nRaises: KeyError: If shard_id is not in list_shards()."
1149
+
"text": "Name\nDescription\n\n\n\n\nshards\nLazily yield (identifier, stream) pairs for each shard."
1157
1150
},
1158
1151
{
1159
-
"objectID": "api/index.html",
1160
-
"href": "api/index.html",
1161
-
"title": "API Reference",
1152
+
"objectID": "api/DataSource.html#methods",
1153
+
"href": "api/DataSource.html#methods",
1154
+
"title": "DataSource",
1162
1155
"section": "",
1163
-
"text": "Core types, decorators, and dataset classes\n\n\n\npackable\nDecorator to convert a regular class into a PackableSample.\n\n\nPackableSample\nBase class for samples that can be serialized with msgpack.\n\n\nDictSample\nDynamic sample type providing dict-like access to raw msgpack data.\n\n\nDataset\nA typed dataset built on WebDataset with lens transformations.\n\n\nSampleBatch\nA batch of samples with automatic attribute aggregation.\n\n\nLens\nA bidirectional transformation between two sample types.\n\n\nlens\nLens-based type transformations for datasets.\n\n\nload_dataset\nLoad a dataset from local files, remote URLs, or an index.\n\n\nDatasetDict\nA dictionary of split names to Dataset instances.\n\n\n\n\n\n\nData source implementations for streaming\n\n\n\nURLSource\nData source for WebDataset-compatible URLs.\n\n\nS3Source\nData source for S3-compatible storage with explicit credentials.\n\n\n\n\n\n\nLocal Redis/S3 storage backend\n\n\n\nlocal.Index\nRedis-backed index for tracking datasets in a repository.\n\n\nlocal.LocalDatasetEntry\nIndex entry for a dataset stored in the local repository.\n\n\nlocal.S3DataStore\nS3-compatible data store implementing AbstractDataStore protocol.\n\n\n\n\n\n\nATProto federation\n\n\n\nAtmosphereClient\nATProto client wrapper for atdata operations.\n\n\nAtmosphereIndex\nATProto index implementing AbstractIndex protocol.\n\n\nAtmosphereIndexEntry\nEntry wrapper for ATProto dataset records implementing IndexEntry protocol.\n\n\nSchemaPublisher\nPublishes PackableSample schemas to ATProto.\n\n\nSchemaLoader\nLoads PackableSample schemas from ATProto.\n\n\nDatasetPublisher\nPublishes dataset index records to ATProto.\n\n\nDatasetLoader\nLoads dataset records from ATProto.\n\n\nLensPublisher\nPublishes Lens transformation records to ATProto.\n\n\nLensLoader\nLoads lens records from ATProto.\n\n\nAtUri\nParsed AT Protocol URI.\n\n\n\n\n\n\nLocal to atmosphere migration\n\n\n\npromote_to_atmosphere\nPromote a local dataset to the atmosphere network."
1156
+
"text": "Name\nDescription\n\n\n\n\nlist_shards\nGet list of shard identifiers without opening streams.\n\n\nopen_shard\nOpen a single shard by its identifier.\n\n\n\n\n\nDataSource.list_shards()\nGet list of shard identifiers without opening streams.\nUsed for metadata queries like counting shards without actually streaming data. Implementations should return identifiers that match what shards would yield.\nReturns: List of shard identifier strings.\n\n\n\nDataSource.open_shard(shard_id)\nOpen a single shard by its identifier.\nThis method enables random access to individual shards, which is required for PyTorch DataLoader worker splitting. Each worker opens only its assigned shards rather than iterating all shards.\nArgs: shard_id: Shard identifier from shard_list.\nReturns: File-like stream for reading the shard.\nRaises: KeyError: If shard_id is not in shard_list."
1164
1157
},
1165
1158
{
1166
-
"objectID": "api/index.html#core",
1167
-
"href": "api/index.html#core",
1168
-
"title": "API Reference",
1159
+
"objectID": "api/DatasetLoader.html",
1160
+
"href": "api/DatasetLoader.html",
1161
+
"title": "DatasetLoader",
1169
1162
"section": "",
1170
-
"text": "Core types, decorators, and dataset classes\n\n\n\npackable\nDecorator to convert a regular class into a PackableSample.\n\n\nPackableSample\nBase class for samples that can be serialized with msgpack.\n\n\nDictSample\nDynamic sample type providing dict-like access to raw msgpack data.\n\n\nDataset\nA typed dataset built on WebDataset with lens transformations.\n\n\nSampleBatch\nA batch of samples with automatic attribute aggregation.\n\n\nLens\nA bidirectional transformation between two sample types.\n\n\nlens\nLens-based type transformations for datasets.\n\n\nload_dataset\nLoad a dataset from local files, remote URLs, or an index.\n\n\nDatasetDict\nA dictionary of split names to Dataset instances."
1163
+
"text": "atmosphere.DatasetLoader(client)\nLoads dataset records from ATProto.\nThis class fetches dataset index records and can create Dataset objects from them. Note that loading a dataset requires having the corresponding Python class for the sample type.\nExample: >>> client = AtmosphereClient() >>> loader = DatasetLoader(client) >>> >>> # List available datasets >>> datasets = loader.list() >>> for ds in datasets: … print(ds[“name”], ds[“schemaRef”]) >>> >>> # Get a specific dataset record >>> record = loader.get(“at://did:plc:abc/ac.foundation.dataset.record/xyz”)\n\n\n\n\n\nName\nDescription\n\n\n\n\nget\nFetch a dataset record by AT URI.\n\n\nget_blob_urls\nGet fetchable URLs for blob-stored dataset shards.\n\n\nget_blobs\nGet the blob references from a dataset record.\n\n\nget_metadata\nGet the metadata from a dataset record.\n\n\nget_storage_type\nGet the storage type of a dataset record.\n\n\nget_urls\nGet the WebDataset URLs from a dataset record.\n\n\nlist_all\nList dataset records from a repository.\n\n\nto_dataset\nCreate a Dataset object from an ATProto record.\n\n\n\n\n\natmosphere.DatasetLoader.get(uri)\nFetch a dataset record by AT URI.\nArgs: uri: The AT URI of the dataset record.\nReturns: The dataset record as a dictionary.\nRaises: ValueError: If the record is not a dataset record.\n\n\n\natmosphere.DatasetLoader.get_blob_urls(uri)\nGet fetchable URLs for blob-stored dataset shards.\nThis resolves the PDS endpoint and constructs URLs that can be used to fetch the blob data directly.\nArgs: uri: The AT URI of the dataset record.\nReturns: List of URLs for fetching the blob data.\nRaises: ValueError: If storage type is not blobs or PDS cannot be resolved.\n\n\n\natmosphere.DatasetLoader.get_blobs(uri)\nGet the blob references from a dataset record.\nArgs: uri: The AT URI of the dataset record.\nReturns: List of blob reference dicts with keys: $type, ref, mimeType, size.\nRaises: ValueError: If the storage type is not blobs.\n\n\n\natmosphere.DatasetLoader.get_metadata(uri)\nGet the metadata from a dataset record.\nArgs: uri: The AT URI of the dataset record.\nReturns: The metadata dictionary, or None if no metadata.\n\n\n\natmosphere.DatasetLoader.get_storage_type(uri)\nGet the storage type of a dataset record.\nArgs: uri: The AT URI of the dataset record.\nReturns: Either “external” or “blobs”.\nRaises: ValueError: If storage type is unknown.\n\n\n\natmosphere.DatasetLoader.get_urls(uri)\nGet the WebDataset URLs from a dataset record.\nArgs: uri: The AT URI of the dataset record.\nReturns: List of WebDataset URLs.\nRaises: ValueError: If the storage type is not external URLs.\n\n\n\natmosphere.DatasetLoader.list_all(repo=None, limit=100)\nList dataset records from a repository.\nArgs: repo: The DID of the repository. Defaults to authenticated user. limit: Maximum number of records to return.\nReturns: List of dataset records.\n\n\n\natmosphere.DatasetLoader.to_dataset(uri, sample_type)\nCreate a Dataset object from an ATProto record.\nThis method creates a Dataset instance from a published record. You must provide the sample type class, which should match the schema referenced by the record.\nSupports both external URL storage and ATProto blob storage.\nArgs: uri: The AT URI of the dataset record. sample_type: The Python class for the sample type.\nReturns: A Dataset instance configured from the record.\nRaises: ValueError: If no storage URLs can be resolved.\nExample: >>> loader = DatasetLoader(client) >>> dataset = loader.to_dataset(uri, MySampleType) >>> for batch in dataset.shuffled(batch_size=32): … process(batch)"
1171
1164
},
1172
1165
{
1173
-
"objectID": "api/index.html#data-sources",
1174
-
"href": "api/index.html#data-sources",
1175
-
"title": "API Reference",
1166
+
"objectID": "api/DatasetLoader.html#methods",
1167
+
"href": "api/DatasetLoader.html#methods",
1168
+
"title": "DatasetLoader",
1176
1169
"section": "",
1177
-
"text": "Data source implementations for streaming\n\n\n\nURLSource\nData source for WebDataset-compatible URLs.\n\n\nS3Source\nData source for S3-compatible storage with explicit credentials."
1170
+
"text": "Name\nDescription\n\n\n\n\nget\nFetch a dataset record by AT URI.\n\n\nget_blob_urls\nGet fetchable URLs for blob-stored dataset shards.\n\n\nget_blobs\nGet the blob references from a dataset record.\n\n\nget_metadata\nGet the metadata from a dataset record.\n\n\nget_storage_type\nGet the storage type of a dataset record.\n\n\nget_urls\nGet the WebDataset URLs from a dataset record.\n\n\nlist_all\nList dataset records from a repository.\n\n\nto_dataset\nCreate a Dataset object from an ATProto record.\n\n\n\n\n\natmosphere.DatasetLoader.get(uri)\nFetch a dataset record by AT URI.\nArgs: uri: The AT URI of the dataset record.\nReturns: The dataset record as a dictionary.\nRaises: ValueError: If the record is not a dataset record.\n\n\n\natmosphere.DatasetLoader.get_blob_urls(uri)\nGet fetchable URLs for blob-stored dataset shards.\nThis resolves the PDS endpoint and constructs URLs that can be used to fetch the blob data directly.\nArgs: uri: The AT URI of the dataset record.\nReturns: List of URLs for fetching the blob data.\nRaises: ValueError: If storage type is not blobs or PDS cannot be resolved.\n\n\n\natmosphere.DatasetLoader.get_blobs(uri)\nGet the blob references from a dataset record.\nArgs: uri: The AT URI of the dataset record.\nReturns: List of blob reference dicts with keys: $type, ref, mimeType, size.\nRaises: ValueError: If the storage type is not blobs.\n\n\n\natmosphere.DatasetLoader.get_metadata(uri)\nGet the metadata from a dataset record.\nArgs: uri: The AT URI of the dataset record.\nReturns: The metadata dictionary, or None if no metadata.\n\n\n\natmosphere.DatasetLoader.get_storage_type(uri)\nGet the storage type of a dataset record.\nArgs: uri: The AT URI of the dataset record.\nReturns: Either “external” or “blobs”.\nRaises: ValueError: If storage type is unknown.\n\n\n\natmosphere.DatasetLoader.get_urls(uri)\nGet the WebDataset URLs from a dataset record.\nArgs: uri: The AT URI of the dataset record.\nReturns: List of WebDataset URLs.\nRaises: ValueError: If the storage type is not external URLs.\n\n\n\natmosphere.DatasetLoader.list_all(repo=None, limit=100)\nList dataset records from a repository.\nArgs: repo: The DID of the repository. Defaults to authenticated user. limit: Maximum number of records to return.\nReturns: List of dataset records.\n\n\n\natmosphere.DatasetLoader.to_dataset(uri, sample_type)\nCreate a Dataset object from an ATProto record.\nThis method creates a Dataset instance from a published record. You must provide the sample type class, which should match the schema referenced by the record.\nSupports both external URL storage and ATProto blob storage.\nArgs: uri: The AT URI of the dataset record. sample_type: The Python class for the sample type.\nReturns: A Dataset instance configured from the record.\nRaises: ValueError: If no storage URLs can be resolved.\nExample: >>> loader = DatasetLoader(client) >>> dataset = loader.to_dataset(uri, MySampleType) >>> for batch in dataset.shuffled(batch_size=32): … process(batch)"
1178
1171
},
1179
1172
{
1180
-
"objectID": "api/index.html#local-storage",
1181
-
"href": "api/index.html#local-storage",
1182
-
"title": "API Reference",
1173
+
"objectID": "api/Lens.html",
1174
+
"href": "api/Lens.html",
1175
+
"title": "lens",
1183
1176
"section": "",
1184
-
"text": "Local Redis/S3 storage backend\n\n\n\nlocal.Index\nRedis-backed index for tracking datasets in a repository.\n\n\nlocal.LocalDatasetEntry\nIndex entry for a dataset stored in the local repository.\n\n\nlocal.S3DataStore\nS3-compatible data store implementing AbstractDataStore protocol."
1177
+
"text": "lens\nLens-based type transformations for datasets.\nThis module implements a lens system for bidirectional transformations between different sample types. Lenses enable viewing a dataset through different type schemas without duplicating the underlying data.\nKey components:\n\nLens: Bidirectional transformation with getter (S -> V) and optional putter (V, S -> S)\nLensNetwork: Global singleton registry for lens transformations\n@lens: Decorator to create and register lens transformations\n\nLenses support the functional programming concept of composable, well-behaved transformations that satisfy lens laws (GetPut and PutGet).\nExample: >>> @packable … class FullData: … name: str … age: int … embedding: NDArray … >>> @packable … class NameOnly: … name: str … >>> @lens … def name_view(full: FullData) -> NameOnly: … return NameOnly(name=full.name) … >>> @name_view.putter … def name_view_put(view: NameOnly, source: FullData) -> FullData: … return FullData(name=view.name, age=source.age, … embedding=source.embedding) … >>> ds = DatasetFullData >>> ds_names = ds.as_type(NameOnly) # Uses registered lens\n\n\n\n\n\nName\nDescription\n\n\n\n\nLens\nA bidirectional transformation between two sample types.\n\n\nLensNetwork\nGlobal registry for lens transformations between sample types.\n\n\n\n\n\nlens.Lens(get, put=None)\nA bidirectional transformation between two sample types.\nA lens provides a way to view and update data of type S (source) as if it were type V (view). It consists of a getter that transforms S -> V and an optional putter that transforms (V, S) -> S, enabling updates to the view to be reflected back in the source.\nType Parameters: S: The source type, must derive from PackableSample. V: The view type, must derive from PackableSample.\nExample: >>> @lens … def name_lens(full: FullData) -> NameOnly: … return NameOnly(name=full.name) … >>> @name_lens.putter … def name_lens_put(view: NameOnly, source: FullData) -> FullData: … return FullData(name=view.name, age=source.age)\n\n\n\n\n\nName\nDescription\n\n\n\n\nget\nTransform the source into the view type.\n\n\nput\nUpdate the source based on a modified view.\n\n\nputter\nDecorator to register a putter function for this lens.\n\n\n\n\n\nlens.Lens.get(s)\nTransform the source into the view type.\nArgs: s: The source sample of type S.\nReturns: A view of the source as type V.\n\n\n\nlens.Lens.put(v, s)\nUpdate the source based on a modified view.\nArgs: v: The modified view of type V. s: The original source of type S.\nReturns: An updated source of type S that reflects changes from the view.\n\n\n\nlens.Lens.putter(put)\nDecorator to register a putter function for this lens.\nArgs: put: A function that takes a view of type V and source of type S, and returns an updated source of type S.\nReturns: The putter function, allowing this to be used as a decorator.\nExample: >>> @my_lens.putter … def my_lens_put(view: ViewType, source: SourceType) -> SourceType: … return SourceType(…)\n\n\n\n\n\nlens.LensNetwork()\nGlobal registry for lens transformations between sample types.\nThis class implements a singleton pattern to maintain a global registry of all lenses decorated with @lens. It enables looking up transformations between different PackableSample types.\nAttributes: _instance: The singleton instance of this class. _registry: Dictionary mapping (source_type, view_type) tuples to their corresponding Lens objects.\n\n\n\n\n\nName\nDescription\n\n\n\n\nregister\nRegister a lens as the canonical transformation between two types.\n\n\ntransform\nLook up the lens transformation between two sample types.\n\n\n\n\n\nlens.LensNetwork.register(_lens)\nRegister a lens as the canonical transformation between two types.\nArgs: _lens: The lens to register. Will be stored in the registry under the key (_lens.source_type, _lens.view_type).\nNote: If a lens already exists for the same type pair, it will be overwritten.\n\n\n\nlens.LensNetwork.transform(source, view)\nLook up the lens transformation between two sample types.\nArgs: source: The source sample type (must derive from PackableSample). view: The target view type (must derive from PackableSample).\nReturns: The registered Lens that transforms from source to view.\nRaises: ValueError: If no lens has been registered for the given type pair.\nNote: Currently only supports direct transformations. Compositional transformations (chaining multiple lenses) are not yet implemented.\n\n\n\n\n\n\n\n\n\nName\nDescription\n\n\n\n\nlens\nDecorator to create and register a lens transformation.\n\n\n\n\n\nlens.lens(f)\nDecorator to create and register a lens transformation.\nThis decorator converts a getter function into a Lens object and automatically registers it in the global LensNetwork registry.\nArgs: f: A getter function that transforms from source type S to view type V. Must have exactly one parameter with a type annotation.\nReturns: A Lens[S, V] object that can be called to apply the transformation or decorated with @lens_name.putter to add a putter function.\nExample: >>> @lens … def extract_name(full: FullData) -> NameOnly: … return NameOnly(name=full.name) … >>> @extract_name.putter … def extract_name_put(view: NameOnly, source: FullData) -> FullData: … return FullData(name=view.name, age=source.age)"
1185
1178
},
1186
1179
{
1187
-
"objectID": "api/index.html#atmosphere",
1188
-
"href": "api/index.html#atmosphere",
1189
-
"title": "API Reference",
1180
+
"objectID": "api/Lens.html#classes",
1181
+
"href": "api/Lens.html#classes",
1182
+
"title": "lens",
1190
1183
"section": "",
1191
-
"text": "ATProto federation\n\n\n\nAtmosphereClient\nATProto client wrapper for atdata operations.\n\n\nAtmosphereIndex\nATProto index implementing AbstractIndex protocol.\n\n\nAtmosphereIndexEntry\nEntry wrapper for ATProto dataset records implementing IndexEntry protocol.\n\n\nSchemaPublisher\nPublishes PackableSample schemas to ATProto.\n\n\nSchemaLoader\nLoads PackableSample schemas from ATProto.\n\n\nDatasetPublisher\nPublishes dataset index records to ATProto.\n\n\nDatasetLoader\nLoads dataset records from ATProto.\n\n\nLensPublisher\nPublishes Lens transformation records to ATProto.\n\n\nLensLoader\nLoads lens records from ATProto.\n\n\nAtUri\nParsed AT Protocol URI."
1184
+
"text": "Name\nDescription\n\n\n\n\nLens\nA bidirectional transformation between two sample types.\n\n\nLensNetwork\nGlobal registry for lens transformations between sample types.\n\n\n\n\n\nlens.Lens(get, put=None)\nA bidirectional transformation between two sample types.\nA lens provides a way to view and update data of type S (source) as if it were type V (view). It consists of a getter that transforms S -> V and an optional putter that transforms (V, S) -> S, enabling updates to the view to be reflected back in the source.\nType Parameters: S: The source type, must derive from PackableSample. V: The view type, must derive from PackableSample.\nExample: >>> @lens … def name_lens(full: FullData) -> NameOnly: … return NameOnly(name=full.name) … >>> @name_lens.putter … def name_lens_put(view: NameOnly, source: FullData) -> FullData: … return FullData(name=view.name, age=source.age)\n\n\n\n\n\nName\nDescription\n\n\n\n\nget\nTransform the source into the view type.\n\n\nput\nUpdate the source based on a modified view.\n\n\nputter\nDecorator to register a putter function for this lens.\n\n\n\n\n\nlens.Lens.get(s)\nTransform the source into the view type.\nArgs: s: The source sample of type S.\nReturns: A view of the source as type V.\n\n\n\nlens.Lens.put(v, s)\nUpdate the source based on a modified view.\nArgs: v: The modified view of type V. s: The original source of type S.\nReturns: An updated source of type S that reflects changes from the view.\n\n\n\nlens.Lens.putter(put)\nDecorator to register a putter function for this lens.\nArgs: put: A function that takes a view of type V and source of type S, and returns an updated source of type S.\nReturns: The putter function, allowing this to be used as a decorator.\nExample: >>> @my_lens.putter … def my_lens_put(view: ViewType, source: SourceType) -> SourceType: … return SourceType(…)\n\n\n\n\n\nlens.LensNetwork()\nGlobal registry for lens transformations between sample types.\nThis class implements a singleton pattern to maintain a global registry of all lenses decorated with @lens. It enables looking up transformations between different PackableSample types.\nAttributes: _instance: The singleton instance of this class. _registry: Dictionary mapping (source_type, view_type) tuples to their corresponding Lens objects.\n\n\n\n\n\nName\nDescription\n\n\n\n\nregister\nRegister a lens as the canonical transformation between two types.\n\n\ntransform\nLook up the lens transformation between two sample types.\n\n\n\n\n\nlens.LensNetwork.register(_lens)\nRegister a lens as the canonical transformation between two types.\nArgs: _lens: The lens to register. Will be stored in the registry under the key (_lens.source_type, _lens.view_type).\nNote: If a lens already exists for the same type pair, it will be overwritten.\n\n\n\nlens.LensNetwork.transform(source, view)\nLook up the lens transformation between two sample types.\nArgs: source: The source sample type (must derive from PackableSample). view: The target view type (must derive from PackableSample).\nReturns: The registered Lens that transforms from source to view.\nRaises: ValueError: If no lens has been registered for the given type pair.\nNote: Currently only supports direct transformations. Compositional transformations (chaining multiple lenses) are not yet implemented."
1192
1185
},
1193
1186
{
1194
-
"objectID": "api/index.html#promotion",
1195
-
"href": "api/index.html#promotion",
1196
-
"title": "API Reference",
1187
+
"objectID": "api/Lens.html#functions",
1188
+
"href": "api/Lens.html#functions",
1189
+
"title": "lens",
1197
1190
"section": "",
1198
-
"text": "Local to atmosphere migration\n\n\n\npromote_to_atmosphere\nPromote a local dataset to the atmosphere network."
1191
+
"text": "Name\nDescription\n\n\n\n\nlens\nDecorator to create and register a lens transformation.\n\n\n\n\n\nlens.lens(f)\nDecorator to create and register a lens transformation.\nThis decorator converts a getter function into a Lens object and automatically registers it in the global LensNetwork registry.\nArgs: f: A getter function that transforms from source type S to view type V. Must have exactly one parameter with a type annotation.\nReturns: A Lens[S, V] object that can be called to apply the transformation or decorated with @lens_name.putter to add a putter function.\nExample: >>> @lens … def extract_name(full: FullData) -> NameOnly: … return NameOnly(name=full.name) … >>> @extract_name.putter … def extract_name_put(view: NameOnly, source: FullData) -> FullData: … return FullData(name=view.name, age=source.age)"
1199
1192
},
1200
1193
{
1201
1194
"objectID": "api/local.Index.html",
···
1240
1233
"text": "Name\nDescription\n\n\n\n\nas_type\nView this dataset through a different sample type using a registered lens.\n\n\nlist_shards\nGet list of individual dataset shards.\n\n\nordered\nIterate over the dataset in order\n\n\nshuffled\nIterate over the dataset in random order.\n\n\nto_parquet\nExport dataset contents to parquet format.\n\n\nwrap\nWrap a raw msgpack sample into the appropriate dataset-specific type.\n\n\nwrap_batch\nWrap a batch of raw msgpack samples into a typed SampleBatch.\n\n\n\n\n\nDataset.as_type(other)\nView this dataset through a different sample type using a registered lens.\nArgs: other: The target sample type to transform into. Must be a type derived from PackableSample.\nReturns: A new Dataset instance that yields samples of type other by applying the appropriate lens transformation from the global LensNetwork registry.\nRaises: ValueError: If no registered lens exists between the current sample type and the target type.\n\n\n\nDataset.list_shards()\nGet list of individual dataset shards.\nReturns: A full (non-lazy) list of the individual tar files within the source WebDataset.\n\n\n\nDataset.ordered(batch_size=None)\nIterate over the dataset in order\nArgs: batch_size (:obj:int, optional): The size of iterated batches. Default: None (unbatched). If None, iterates over one sample at a time with no batch dimension.\nReturns: :obj:webdataset.DataPipeline A data pipeline that iterates over the dataset in its original sample order\n\n\n\nDataset.shuffled(buffer_shards=100, buffer_samples=10000, batch_size=None)\nIterate over the dataset in random order.\nArgs: buffer_shards: Number of shards to buffer for shuffling at the shard level. Larger values increase randomness but use more memory. Default: 100. buffer_samples: Number of samples to buffer for shuffling within shards. Larger values increase randomness but use more memory. Default: 10,000. batch_size: The size of iterated batches. Default: None (unbatched). If None, iterates over one sample at a time with no batch dimension.\nReturns: A WebDataset data pipeline that iterates over the dataset in randomized order. If batch_size is not None, yields SampleBatch[ST] instances; otherwise yields individual ST samples.\n\n\n\nDataset.to_parquet(path, sample_map=None, maxcount=None, **kwargs)\nExport dataset contents to parquet format.\nConverts all samples to a pandas DataFrame and saves to parquet file(s). Useful for interoperability with data analysis tools.\nArgs: path: Output path for the parquet file. If maxcount is specified, files are named {stem}-{segment:06d}.parquet. sample_map: Optional function to convert samples to dictionaries. Defaults to dataclasses.asdict. maxcount: If specified, split output into multiple files with at most this many samples each. Recommended for large datasets. **kwargs: Additional arguments passed to pandas.DataFrame.to_parquet(). Common options include compression, index, engine.\nWarning: Memory Usage: When maxcount=None (default), this method loads the entire dataset into memory as a pandas DataFrame before writing. For large datasets, this can cause memory exhaustion.\nFor datasets larger than available RAM, always specify ``maxcount``::\n\n # Safe for large datasets - processes in chunks\n ds.to_parquet(\"output.parquet\", maxcount=10000)\n\nThis creates multiple parquet files: ``output-000000.parquet``,\n``output-000001.parquet``, etc.\nExample: >>> ds = DatasetMySample >>> # Small dataset - load all at once >>> ds.to_parquet(“output.parquet”) >>> >>> # Large dataset - process in chunks >>> ds.to_parquet(“output.parquet”, maxcount=50000)\n\n\n\nDataset.wrap(sample)\nWrap a raw msgpack sample into the appropriate dataset-specific type.\nArgs: sample: A dictionary containing at minimum a 'msgpack' key with serialized sample bytes.\nReturns: A deserialized sample of type ST, optionally transformed through a lens if as_type() was called.\n\n\n\nDataset.wrap_batch(batch)\nWrap a batch of raw msgpack samples into a typed SampleBatch.\nArgs: batch: A dictionary containing a 'msgpack' key with a list of serialized sample bytes.\nReturns: A SampleBatch[ST] containing deserialized samples, optionally transformed through a lens if as_type() was called.\nNote: This implementation deserializes samples one at a time, then aggregates them into a batch."
1241
1234
},
1242
1235
{
1236
+
"objectID": "api/AbstractDataStore.html",
1237
+
"href": "api/AbstractDataStore.html",
1238
+
"title": "AbstractDataStore",
1239
+
"section": "",
1240
+
"text": "AbstractDataStore()\nProtocol for data storage operations.\nThis protocol abstracts over different storage backends for dataset data: - S3DataStore: S3-compatible object storage - PDSBlobStore: ATProto PDS blob storage (future)\nThe separation of index (metadata) from data store (actual files) allows flexible deployment: local index with S3 storage, atmosphere index with S3 storage, or atmosphere index with PDS blobs.\nExample: >>> store = S3DataStore(credentials, bucket=“my-bucket”) >>> urls = store.write_shards(dataset, prefix=“training/v1”) >>> print(urls) [‘s3://my-bucket/training/v1/shard-000000.tar’, …]\n\n\n\n\n\nName\nDescription\n\n\n\n\nread_url\nResolve a storage URL for reading.\n\n\nsupports_streaming\nWhether this store supports streaming reads.\n\n\nwrite_shards\nWrite dataset shards to storage.\n\n\n\n\n\nAbstractDataStore.read_url(url)\nResolve a storage URL for reading.\nSome storage backends may need to transform URLs (e.g., signing S3 URLs or resolving blob references). This method returns a URL that can be used directly with WebDataset.\nArgs: url: Storage URL to resolve.\nReturns: WebDataset-compatible URL for reading.\n\n\n\nAbstractDataStore.supports_streaming()\nWhether this store supports streaming reads.\nReturns: True if the store supports efficient streaming (like S3), False if data must be fully downloaded first.\n\n\n\nAbstractDataStore.write_shards(ds, *, prefix, **kwargs)\nWrite dataset shards to storage.\nArgs: ds: The Dataset to write. prefix: Path prefix for the shards (e.g., ‘datasets/mnist/v1’). **kwargs: Backend-specific options (e.g., maxcount for shard size).\nReturns: List of URLs for the written shards, suitable for use with WebDataset or atdata.Dataset()."
1241
+
},
1242
+
{
1243
+
"objectID": "api/AbstractDataStore.html#methods",
1244
+
"href": "api/AbstractDataStore.html#methods",
1245
+
"title": "AbstractDataStore",
1246
+
"section": "",
1247
+
"text": "Name\nDescription\n\n\n\n\nread_url\nResolve a storage URL for reading.\n\n\nsupports_streaming\nWhether this store supports streaming reads.\n\n\nwrite_shards\nWrite dataset shards to storage.\n\n\n\n\n\nAbstractDataStore.read_url(url)\nResolve a storage URL for reading.\nSome storage backends may need to transform URLs (e.g., signing S3 URLs or resolving blob references). This method returns a URL that can be used directly with WebDataset.\nArgs: url: Storage URL to resolve.\nReturns: WebDataset-compatible URL for reading.\n\n\n\nAbstractDataStore.supports_streaming()\nWhether this store supports streaming reads.\nReturns: True if the store supports efficient streaming (like S3), False if data must be fully downloaded first.\n\n\n\nAbstractDataStore.write_shards(ds, *, prefix, **kwargs)\nWrite dataset shards to storage.\nArgs: ds: The Dataset to write. prefix: Path prefix for the shards (e.g., ‘datasets/mnist/v1’). **kwargs: Backend-specific options (e.g., maxcount for shard size).\nReturns: List of URLs for the written shards, suitable for use with WebDataset or atdata.Dataset()."
1248
+
},
1249
+
{
1243
1250
"objectID": "api/local.S3DataStore.html",
1244
1251
"href": "api/local.S3DataStore.html",
1245
1252
"title": "local.S3DataStore",
···
1275
1282
"text": "Name\nDescription\n\n\n\n\nparse\nParse an AT URI string into components.\n\n\n\n\n\natmosphere.AtUri.parse(uri)\nParse an AT URI string into components.\nArgs: uri: AT URI string in format at://<authority>/<collection>/<rkey>\nReturns: Parsed AtUri instance.\nRaises: ValueError: If the URI format is invalid."
1276
1283
},
1277
1284
{
1278
-
"objectID": "api/SampleBatch.html",
1279
-
"href": "api/SampleBatch.html",
1280
-
"title": "SampleBatch",
1285
+
"objectID": "api/Packable-protocol.html",
1286
+
"href": "api/Packable-protocol.html",
1287
+
"title": "Packable",
1281
1288
"section": "",
1282
-
"text": "SampleBatch(samples)\nA batch of samples with automatic attribute aggregation.\nThis class wraps a sequence of samples and provides magic __getattr__ access to aggregate sample attributes. When you access an attribute that exists on the sample type, it automatically aggregates values across all samples in the batch.\nNDArray fields are stacked into a numpy array with a batch dimension. Other fields are aggregated into a list.\nType Parameters: DT: The sample type, must derive from PackableSample.\nAttributes: samples: The list of sample instances in this batch.\nExample: >>> batch = SampleBatchMyData >>> batch.embeddings # Returns stacked numpy array of shape (3, …) >>> batch.names # Returns list of names\nNote: This class uses Python’s __orig_class__ mechanism to extract the type parameter at runtime. Instances must be created using the subscripted syntax SampleBatch[MyType](samples) rather than calling the constructor directly with an unsubscripted class.\n\n\n\n\n\nName\nDescription\n\n\n\n\nsample_type\nThe type of each sample in this batch."
1289
+
"text": "Packable()\nStructural protocol for packable sample types.\nThis protocol allows classes decorated with @packable to be recognized as valid types for lens transformations and schema operations, even though the decorator doesn’t change the class’s nominal type at static analysis time.\nBoth PackableSample subclasses and @packable-decorated classes satisfy this protocol structurally.\nThe protocol captures the full interface needed for: - Lens type transformations (as_wds, from_data) - Schema publishing (class introspection via dataclass fields) - Serialization/deserialization (packed, from_bytes)\nExample: >>> @packable … class MySample: … name: str … value: int … >>> def process(sample_type: TypePackable) -> None: … # Type checker knows sample_type has from_bytes, packed, etc. … instance = sample_type.from_bytes(data) … print(instance.packed)\n\n\n\n\n\nName\nDescription\n\n\n\n\nas_wds\nWebDataset-compatible representation with key and msgpack.\n\n\npacked\nPack this sample’s data into msgpack bytes.\n\n\n\n\n\n\n\n\n\nName\nDescription\n\n\n\n\nfrom_bytes\nCreate instance from raw msgpack bytes.\n\n\nfrom_data\nCreate instance from unpacked msgpack data dictionary.\n\n\n\n\n\nPackable.from_bytes(bs)\nCreate instance from raw msgpack bytes.\n\n\n\nPackable.from_data(data)\nCreate instance from unpacked msgpack data dictionary."
1290
+
},
1291
+
{
1292
+
"objectID": "api/Packable-protocol.html#attributes",
1293
+
"href": "api/Packable-protocol.html#attributes",
1294
+
"title": "Packable",
1295
+
"section": "",
1296
+
"text": "Name\nDescription\n\n\n\n\nas_wds\nWebDataset-compatible representation with key and msgpack.\n\n\npacked\nPack this sample’s data into msgpack bytes."
1297
+
},
1298
+
{
1299
+
"objectID": "api/Packable-protocol.html#methods",
1300
+
"href": "api/Packable-protocol.html#methods",
1301
+
"title": "Packable",
1302
+
"section": "",
1303
+
"text": "Name\nDescription\n\n\n\n\nfrom_bytes\nCreate instance from raw msgpack bytes.\n\n\nfrom_data\nCreate instance from unpacked msgpack data dictionary.\n\n\n\n\n\nPackable.from_bytes(bs)\nCreate instance from raw msgpack bytes.\n\n\n\nPackable.from_data(data)\nCreate instance from unpacked msgpack data dictionary."
1283
1304
},
1284
1305
{
1285
-
"objectID": "api/SampleBatch.html#attributes",
1286
-
"href": "api/SampleBatch.html#attributes",
1287
-
"title": "SampleBatch",
1306
+
"objectID": "api/packable.html",
1307
+
"href": "api/packable.html",
1308
+
"title": "packable",
1288
1309
"section": "",
1289
-
"text": "Name\nDescription\n\n\n\n\nsample_type\nThe type of each sample in this batch."
1310
+
"text": "packable\npackable(cls)\nDecorator to convert a regular class into a PackableSample.\nThis decorator transforms a class into a dataclass that inherits from PackableSample, enabling automatic msgpack serialization/deserialization with special handling for NDArray fields.\nThe resulting class satisfies the Packable protocol, making it compatible with all atdata APIs that accept packable types (e.g., publish_schema, lens transformations, etc.).\nArgs: cls: The class to convert. Should have type annotations for its fields.\nReturns: A new dataclass that inherits from PackableSample with the same name and annotations as the original class. The class satisfies the Packable protocol and can be used with Type[Packable] signatures.\nExample: >>> @packable … class MyData: … name: str … values: NDArray … >>> sample = MyData(name=“test”, values=np.array([1, 2, 3])) >>> bytes_data = sample.packed >>> restored = MyData.from_bytes(bytes_data) >>> >>> # Works with Packable-typed APIs >>> index.publish_schema(MyData, version=“1.0.0”) # Type-safe"
1290
1311
},
1291
1312
{
1292
1313
"objectID": "index.html",
···
1377
1398
]
1378
1399
},
1379
1400
{
1380
-
"objectID": "api/packable.html",
1381
-
"href": "api/packable.html",
1382
-
"title": "packable",
1401
+
"objectID": "api/SampleBatch.html",
1402
+
"href": "api/SampleBatch.html",
1403
+
"title": "SampleBatch",
1383
1404
"section": "",
1384
-
"text": "packable\npackable(cls)\nDecorator to convert a regular class into a PackableSample.\nThis decorator transforms a class into a dataclass that inherits from PackableSample, enabling automatic msgpack serialization/deserialization with special handling for NDArray fields.\nThe resulting class satisfies the Packable protocol, making it compatible with all atdata APIs that accept packable types (e.g., publish_schema, lens transformations, etc.).\nArgs: cls: The class to convert. Should have type annotations for its fields.\nReturns: A new dataclass that inherits from PackableSample with the same name and annotations as the original class. The class satisfies the Packable protocol and can be used with Type[Packable] signatures.\nExample: >>> @packable … class MyData: … name: str … values: NDArray … >>> sample = MyData(name=“test”, values=np.array([1, 2, 3])) >>> bytes_data = sample.packed >>> restored = MyData.from_bytes(bytes_data) >>> >>> # Works with Packable-typed APIs >>> index.publish_schema(MyData, version=“1.0.0”) # Type-safe"
1405
+
"text": "SampleBatch(samples)\nA batch of samples with automatic attribute aggregation.\nThis class wraps a sequence of samples and provides magic __getattr__ access to aggregate sample attributes. When you access an attribute that exists on the sample type, it automatically aggregates values across all samples in the batch.\nNDArray fields are stacked into a numpy array with a batch dimension. Other fields are aggregated into a list.\nType Parameters: DT: The sample type, must derive from PackableSample.\nAttributes: samples: The list of sample instances in this batch.\nExample: >>> batch = SampleBatchMyData >>> batch.embeddings # Returns stacked numpy array of shape (3, …) >>> batch.names # Returns list of names\nNote: This class uses Python’s __orig_class__ mechanism to extract the type parameter at runtime. Instances must be created using the subscripted syntax SampleBatch[MyType](samples) rather than calling the constructor directly with an unsubscripted class.\n\n\n\n\n\nName\nDescription\n\n\n\n\nsample_type\nThe type of each sample in this batch."
1406
+
},
1407
+
{
1408
+
"objectID": "api/SampleBatch.html#attributes",
1409
+
"href": "api/SampleBatch.html#attributes",
1410
+
"title": "SampleBatch",
1411
+
"section": "",
1412
+
"text": "Name\nDescription\n\n\n\n\nsample_type\nThe type of each sample in this batch."
1385
1413
},
1386
1414
{
1387
1415
"objectID": "api/LensPublisher.html",
···
1412
1440
"text": "Name\nDescription\n\n\n\n\ndata_urls\nWebDataset URLs from external storage.\n\n\nmetadata\nMetadata from the record, if any.\n\n\nname\nHuman-readable dataset name.\n\n\nschema_ref\nAT URI of the schema record.\n\n\nuri\nAT URI of this record."
1413
1441
},
1414
1442
{
1443
+
"objectID": "api/AbstractIndex.html",
1444
+
"href": "api/AbstractIndex.html",
1445
+
"title": "AbstractIndex",
1446
+
"section": "",
1447
+
"text": "AbstractIndex()\nProtocol for index operations - implemented by LocalIndex and AtmosphereIndex.\nThis protocol defines the common interface for managing dataset metadata: - Publishing and retrieving schemas - Inserting and listing datasets - (Future) Publishing and retrieving lenses\nA single index can hold datasets of many different sample types. The sample type is tracked via schema references, not as a generic parameter on the index.\nOptional Extensions: Some index implementations support additional features: - data_store: An AbstractDataStore for reading/writing dataset shards. If present, load_dataset will use it for S3 credential resolution.\nExample: >>> def publish_and_list(index: AbstractIndex) -> None: … # Publish schemas for different types … schema1 = index.publish_schema(ImageSample, version=“1.0.0”) … schema2 = index.publish_schema(TextSample, version=“1.0.0”) … … # Insert datasets of different types … index.insert_dataset(image_ds, name=“images”) … index.insert_dataset(text_ds, name=“texts”) … … # List all datasets (mixed types) … for entry in index.list_datasets(): … print(f”{entry.name} -> {entry.schema_ref}“)\n\n\n\n\n\nName\nDescription\n\n\n\n\ndatasets\nLazily iterate over all dataset entries in this index.\n\n\nschemas\nLazily iterate over all schema records in this index.\n\n\n\n\n\n\n\n\n\nName\nDescription\n\n\n\n\ndecode_schema\nReconstruct a Python Packable type from a stored schema.\n\n\nget_dataset\nGet a dataset entry by name or reference.\n\n\nget_schema\nGet a schema record by reference.\n\n\ninsert_dataset\nInsert a dataset into the index.\n\n\nlist_datasets\nGet all dataset entries as a materialized list.\n\n\nlist_schemas\nGet all schema records as a materialized list.\n\n\npublish_schema\nPublish a schema for a sample type.\n\n\n\n\n\nAbstractIndex.decode_schema(ref)\nReconstruct a Python Packable type from a stored schema.\nThis method enables loading datasets without knowing the sample type ahead of time. The index retrieves the schema record and dynamically generates a Packable class matching the schema definition.\nArgs: ref: Schema reference string (local:// or at://).\nReturns: A dynamically generated Packable class with fields matching the schema definition. The class can be used with Dataset[T] to load and iterate over samples.\nRaises: KeyError: If schema not found. ValueError: If schema cannot be decoded (unsupported field types).\nExample: >>> entry = index.get_dataset(“my-dataset”) >>> SampleType = index.decode_schema(entry.schema_ref) >>> ds = DatasetSampleType >>> for sample in ds.ordered(): … print(sample) # sample is instance of SampleType\n\n\n\nAbstractIndex.get_dataset(ref)\nGet a dataset entry by name or reference.\nArgs: ref: Dataset name, path, or full reference string.\nReturns: IndexEntry for the dataset.\nRaises: KeyError: If dataset not found.\n\n\n\nAbstractIndex.get_schema(ref)\nGet a schema record by reference.\nArgs: ref: Schema reference string (local:// or at://).\nReturns: Schema record as a dictionary with fields like ‘name’, ‘version’, ‘fields’, etc.\nRaises: KeyError: If schema not found.\n\n\n\nAbstractIndex.insert_dataset(ds, *, name, schema_ref=None, **kwargs)\nInsert a dataset into the index.\nThe sample type is inferred from ds.sample_type. If schema_ref is not provided, the schema may be auto-published based on the sample type.\nArgs: ds: The Dataset to register in the index (any sample type). name: Human-readable name for the dataset. schema_ref: Optional explicit schema reference. If not provided, the schema may be auto-published or inferred from ds.sample_type. **kwargs: Additional backend-specific options.\nReturns: IndexEntry for the inserted dataset.\n\n\n\nAbstractIndex.list_datasets()\nGet all dataset entries as a materialized list.\nReturns: List of IndexEntry for each dataset.\n\n\n\nAbstractIndex.list_schemas()\nGet all schema records as a materialized list.\nReturns: List of schema records as dictionaries.\n\n\n\nAbstractIndex.publish_schema(sample_type, *, version='1.0.0', **kwargs)\nPublish a schema for a sample type.\nArgs: sample_type: A Packable type (PackableSample subclass or @packable-decorated). version: Semantic version string for the schema. **kwargs: Additional backend-specific options.\nReturns: Schema reference string: - Local: ‘local://schemas/{module.Class}@version’ - Atmosphere: ‘at://did:plc:…/ac.foundation.dataset.sampleSchema/…’"
1448
+
},
1449
+
{
1450
+
"objectID": "api/AbstractIndex.html#attributes",
1451
+
"href": "api/AbstractIndex.html#attributes",
1452
+
"title": "AbstractIndex",
1453
+
"section": "",
1454
+
"text": "Name\nDescription\n\n\n\n\ndatasets\nLazily iterate over all dataset entries in this index.\n\n\nschemas\nLazily iterate over all schema records in this index."
1455
+
},
1456
+
{
1457
+
"objectID": "api/AbstractIndex.html#methods",
1458
+
"href": "api/AbstractIndex.html#methods",
1459
+
"title": "AbstractIndex",
1460
+
"section": "",
1461
+
"text": "Name\nDescription\n\n\n\n\ndecode_schema\nReconstruct a Python Packable type from a stored schema.\n\n\nget_dataset\nGet a dataset entry by name or reference.\n\n\nget_schema\nGet a schema record by reference.\n\n\ninsert_dataset\nInsert a dataset into the index.\n\n\nlist_datasets\nGet all dataset entries as a materialized list.\n\n\nlist_schemas\nGet all schema records as a materialized list.\n\n\npublish_schema\nPublish a schema for a sample type.\n\n\n\n\n\nAbstractIndex.decode_schema(ref)\nReconstruct a Python Packable type from a stored schema.\nThis method enables loading datasets without knowing the sample type ahead of time. The index retrieves the schema record and dynamically generates a Packable class matching the schema definition.\nArgs: ref: Schema reference string (local:// or at://).\nReturns: A dynamically generated Packable class with fields matching the schema definition. The class can be used with Dataset[T] to load and iterate over samples.\nRaises: KeyError: If schema not found. ValueError: If schema cannot be decoded (unsupported field types).\nExample: >>> entry = index.get_dataset(“my-dataset”) >>> SampleType = index.decode_schema(entry.schema_ref) >>> ds = DatasetSampleType >>> for sample in ds.ordered(): … print(sample) # sample is instance of SampleType\n\n\n\nAbstractIndex.get_dataset(ref)\nGet a dataset entry by name or reference.\nArgs: ref: Dataset name, path, or full reference string.\nReturns: IndexEntry for the dataset.\nRaises: KeyError: If dataset not found.\n\n\n\nAbstractIndex.get_schema(ref)\nGet a schema record by reference.\nArgs: ref: Schema reference string (local:// or at://).\nReturns: Schema record as a dictionary with fields like ‘name’, ‘version’, ‘fields’, etc.\nRaises: KeyError: If schema not found.\n\n\n\nAbstractIndex.insert_dataset(ds, *, name, schema_ref=None, **kwargs)\nInsert a dataset into the index.\nThe sample type is inferred from ds.sample_type. If schema_ref is not provided, the schema may be auto-published based on the sample type.\nArgs: ds: The Dataset to register in the index (any sample type). name: Human-readable name for the dataset. schema_ref: Optional explicit schema reference. If not provided, the schema may be auto-published or inferred from ds.sample_type. **kwargs: Additional backend-specific options.\nReturns: IndexEntry for the inserted dataset.\n\n\n\nAbstractIndex.list_datasets()\nGet all dataset entries as a materialized list.\nReturns: List of IndexEntry for each dataset.\n\n\n\nAbstractIndex.list_schemas()\nGet all schema records as a materialized list.\nReturns: List of schema records as dictionaries.\n\n\n\nAbstractIndex.publish_schema(sample_type, *, version='1.0.0', **kwargs)\nPublish a schema for a sample type.\nArgs: sample_type: A Packable type (PackableSample subclass or @packable-decorated). version: Semantic version string for the schema. **kwargs: Additional backend-specific options.\nReturns: Schema reference string: - Local: ‘local://schemas/{module.Class}@version’ - Atmosphere: ‘at://did:plc:…/ac.foundation.dataset.sampleSchema/…’"
1462
+
},
1463
+
{
1415
1464
"objectID": "api/local.LocalDatasetEntry.html",
1416
1465
"href": "api/local.LocalDatasetEntry.html",
1417
1466
"title": "local.LocalDatasetEntry",
···
1454
1503
"text": "Name\nDescription\n\n\n\n\nfrom_credentials\nCreate S3Source from a credentials dictionary.\n\n\nfrom_urls\nCreate S3Source from s3:// URLs.\n\n\nlist_shards\nReturn list of S3 URIs for the shards.\n\n\nopen_shard\nOpen a single shard by S3 URI.\n\n\n\n\n\nS3Source.from_credentials(credentials, bucket, keys)\nCreate S3Source from a credentials dictionary.\nAccepts the same credential format used by S3DataStore.\nArgs: credentials: Dict with AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, and optionally AWS_ENDPOINT. bucket: S3 bucket name. keys: List of object keys.\nReturns: Configured S3Source.\nExample: >>> creds = { … “AWS_ACCESS_KEY_ID”: “…”, … “AWS_SECRET_ACCESS_KEY”: “…”, … “AWS_ENDPOINT”: “https://r2.example.com”, … } >>> source = S3Source.from_credentials(creds, “my-bucket”, [“data.tar”])\n\n\n\nS3Source.from_urls(\n urls,\n *,\n endpoint=None,\n access_key=None,\n secret_key=None,\n region=None,\n)\nCreate S3Source from s3:// URLs.\nParses s3://bucket/key URLs and extracts bucket and keys. All URLs must be in the same bucket.\nArgs: urls: List of s3:// URLs. endpoint: Optional custom endpoint. access_key: Optional access key. secret_key: Optional secret key. region: Optional region.\nReturns: S3Source configured for the given URLs.\nRaises: ValueError: If URLs are not valid s3:// URLs or span multiple buckets.\nExample: >>> source = S3Source.from_urls( … [“s3://my-bucket/train-000.tar”, “s3://my-bucket/train-001.tar”], … endpoint=“https://r2.example.com”, … )\n\n\n\nS3Source.list_shards()\nReturn list of S3 URIs for the shards.\n\n\n\nS3Source.open_shard(shard_id)\nOpen a single shard by S3 URI.\nArgs: shard_id: S3 URI of the shard (s3://bucket/key).\nReturns: StreamingBody for reading the object.\nRaises: KeyError: If shard_id is not in list_shards()."
1455
1504
},
1456
1505
{
1457
-
"objectID": "api/Lens.html",
1458
-
"href": "api/Lens.html",
1459
-
"title": "lens",
1506
+
"objectID": "api/IndexEntry.html",
1507
+
"href": "api/IndexEntry.html",
1508
+
"title": "IndexEntry",
1460
1509
"section": "",
1461
-
"text": "lens\nLens-based type transformations for datasets.\nThis module implements a lens system for bidirectional transformations between different sample types. Lenses enable viewing a dataset through different type schemas without duplicating the underlying data.\nKey components:\n\nLens: Bidirectional transformation with getter (S -> V) and optional putter (V, S -> S)\nLensNetwork: Global singleton registry for lens transformations\n@lens: Decorator to create and register lens transformations\n\nLenses support the functional programming concept of composable, well-behaved transformations that satisfy lens laws (GetPut and PutGet).\nExample: >>> @packable … class FullData: … name: str … age: int … embedding: NDArray … >>> @packable … class NameOnly: … name: str … >>> @lens … def name_view(full: FullData) -> NameOnly: … return NameOnly(name=full.name) … >>> @name_view.putter … def name_view_put(view: NameOnly, source: FullData) -> FullData: … return FullData(name=view.name, age=source.age, … embedding=source.embedding) … >>> ds = DatasetFullData >>> ds_names = ds.as_type(NameOnly) # Uses registered lens\n\n\n\n\n\nName\nDescription\n\n\n\n\nLens\nA bidirectional transformation between two sample types.\n\n\nLensNetwork\nGlobal registry for lens transformations between sample types.\n\n\n\n\n\nlens.Lens(get, put=None)\nA bidirectional transformation between two sample types.\nA lens provides a way to view and update data of type S (source) as if it were type V (view). It consists of a getter that transforms S -> V and an optional putter that transforms (V, S) -> S, enabling updates to the view to be reflected back in the source.\nType Parameters: S: The source type, must derive from PackableSample. V: The view type, must derive from PackableSample.\nExample: >>> @lens … def name_lens(full: FullData) -> NameOnly: … return NameOnly(name=full.name) … >>> @name_lens.putter … def name_lens_put(view: NameOnly, source: FullData) -> FullData: … return FullData(name=view.name, age=source.age)\n\n\n\n\n\nName\nDescription\n\n\n\n\nget\nTransform the source into the view type.\n\n\nput\nUpdate the source based on a modified view.\n\n\nputter\nDecorator to register a putter function for this lens.\n\n\n\n\n\nlens.Lens.get(s)\nTransform the source into the view type.\nArgs: s: The source sample of type S.\nReturns: A view of the source as type V.\n\n\n\nlens.Lens.put(v, s)\nUpdate the source based on a modified view.\nArgs: v: The modified view of type V. s: The original source of type S.\nReturns: An updated source of type S that reflects changes from the view.\n\n\n\nlens.Lens.putter(put)\nDecorator to register a putter function for this lens.\nArgs: put: A function that takes a view of type V and source of type S, and returns an updated source of type S.\nReturns: The putter function, allowing this to be used as a decorator.\nExample: >>> @my_lens.putter … def my_lens_put(view: ViewType, source: SourceType) -> SourceType: … return SourceType(…)\n\n\n\n\n\nlens.LensNetwork()\nGlobal registry for lens transformations between sample types.\nThis class implements a singleton pattern to maintain a global registry of all lenses decorated with @lens. It enables looking up transformations between different PackableSample types.\nAttributes: _instance: The singleton instance of this class. _registry: Dictionary mapping (source_type, view_type) tuples to their corresponding Lens objects.\n\n\n\n\n\nName\nDescription\n\n\n\n\nregister\nRegister a lens as the canonical transformation between two types.\n\n\ntransform\nLook up the lens transformation between two sample types.\n\n\n\n\n\nlens.LensNetwork.register(_lens)\nRegister a lens as the canonical transformation between two types.\nArgs: _lens: The lens to register. Will be stored in the registry under the key (_lens.source_type, _lens.view_type).\nNote: If a lens already exists for the same type pair, it will be overwritten.\n\n\n\nlens.LensNetwork.transform(source, view)\nLook up the lens transformation between two sample types.\nArgs: source: The source sample type (must derive from PackableSample). view: The target view type (must derive from PackableSample).\nReturns: The registered Lens that transforms from source to view.\nRaises: ValueError: If no lens has been registered for the given type pair.\nNote: Currently only supports direct transformations. Compositional transformations (chaining multiple lenses) are not yet implemented.\n\n\n\n\n\n\n\n\n\nName\nDescription\n\n\n\n\nlens\nDecorator to create and register a lens transformation.\n\n\n\n\n\nlens.lens(f)\nDecorator to create and register a lens transformation.\nThis decorator converts a getter function into a Lens object and automatically registers it in the global LensNetwork registry.\nArgs: f: A getter function that transforms from source type S to view type V. Must have exactly one parameter with a type annotation.\nReturns: A Lens[S, V] object that can be called to apply the transformation or decorated with @lens_name.putter to add a putter function.\nExample: >>> @lens … def extract_name(full: FullData) -> NameOnly: … return NameOnly(name=full.name) … >>> @extract_name.putter … def extract_name_put(view: NameOnly, source: FullData) -> FullData: … return FullData(name=view.name, age=source.age)"
1510
+
"text": "IndexEntry()\nCommon interface for index entries (local or atmosphere).\nBoth LocalDatasetEntry and atmosphere DatasetRecord-based entries should satisfy this protocol, enabling code that works with either.\nProperties: name: Human-readable dataset name schema_ref: Reference to schema (local:// path or AT URI) data_urls: WebDataset URLs for the data metadata: Arbitrary metadata dict, or None\n\n\n\n\n\nName\nDescription\n\n\n\n\ndata_urls\nWebDataset URLs for the data.\n\n\nmetadata\nArbitrary metadata dictionary, or None if not set.\n\n\nname\nHuman-readable dataset name.\n\n\nschema_ref\nReference to the schema for this dataset."
1462
1511
},
1463
1512
{
1464
-
"objectID": "api/Lens.html#classes",
1465
-
"href": "api/Lens.html#classes",
1466
-
"title": "lens",
1513
+
"objectID": "api/IndexEntry.html#attributes",
1514
+
"href": "api/IndexEntry.html#attributes",
1515
+
"title": "IndexEntry",
1467
1516
"section": "",
1468
-
"text": "Name\nDescription\n\n\n\n\nLens\nA bidirectional transformation between two sample types.\n\n\nLensNetwork\nGlobal registry for lens transformations between sample types.\n\n\n\n\n\nlens.Lens(get, put=None)\nA bidirectional transformation between two sample types.\nA lens provides a way to view and update data of type S (source) as if it were type V (view). It consists of a getter that transforms S -> V and an optional putter that transforms (V, S) -> S, enabling updates to the view to be reflected back in the source.\nType Parameters: S: The source type, must derive from PackableSample. V: The view type, must derive from PackableSample.\nExample: >>> @lens … def name_lens(full: FullData) -> NameOnly: … return NameOnly(name=full.name) … >>> @name_lens.putter … def name_lens_put(view: NameOnly, source: FullData) -> FullData: … return FullData(name=view.name, age=source.age)\n\n\n\n\n\nName\nDescription\n\n\n\n\nget\nTransform the source into the view type.\n\n\nput\nUpdate the source based on a modified view.\n\n\nputter\nDecorator to register a putter function for this lens.\n\n\n\n\n\nlens.Lens.get(s)\nTransform the source into the view type.\nArgs: s: The source sample of type S.\nReturns: A view of the source as type V.\n\n\n\nlens.Lens.put(v, s)\nUpdate the source based on a modified view.\nArgs: v: The modified view of type V. s: The original source of type S.\nReturns: An updated source of type S that reflects changes from the view.\n\n\n\nlens.Lens.putter(put)\nDecorator to register a putter function for this lens.\nArgs: put: A function that takes a view of type V and source of type S, and returns an updated source of type S.\nReturns: The putter function, allowing this to be used as a decorator.\nExample: >>> @my_lens.putter … def my_lens_put(view: ViewType, source: SourceType) -> SourceType: … return SourceType(…)\n\n\n\n\n\nlens.LensNetwork()\nGlobal registry for lens transformations between sample types.\nThis class implements a singleton pattern to maintain a global registry of all lenses decorated with @lens. It enables looking up transformations between different PackableSample types.\nAttributes: _instance: The singleton instance of this class. _registry: Dictionary mapping (source_type, view_type) tuples to their corresponding Lens objects.\n\n\n\n\n\nName\nDescription\n\n\n\n\nregister\nRegister a lens as the canonical transformation between two types.\n\n\ntransform\nLook up the lens transformation between two sample types.\n\n\n\n\n\nlens.LensNetwork.register(_lens)\nRegister a lens as the canonical transformation between two types.\nArgs: _lens: The lens to register. Will be stored in the registry under the key (_lens.source_type, _lens.view_type).\nNote: If a lens already exists for the same type pair, it will be overwritten.\n\n\n\nlens.LensNetwork.transform(source, view)\nLook up the lens transformation between two sample types.\nArgs: source: The source sample type (must derive from PackableSample). view: The target view type (must derive from PackableSample).\nReturns: The registered Lens that transforms from source to view.\nRaises: ValueError: If no lens has been registered for the given type pair.\nNote: Currently only supports direct transformations. Compositional transformations (chaining multiple lenses) are not yet implemented."
1517
+
"text": "Name\nDescription\n\n\n\n\ndata_urls\nWebDataset URLs for the data.\n\n\nmetadata\nArbitrary metadata dictionary, or None if not set.\n\n\nname\nHuman-readable dataset name.\n\n\nschema_ref\nReference to the schema for this dataset."
1469
1518
},
1470
1519
{
1471
-
"objectID": "api/Lens.html#functions",
1472
-
"href": "api/Lens.html#functions",
1473
-
"title": "lens",
1520
+
"objectID": "api/index.html",
1521
+
"href": "api/index.html",
1522
+
"title": "API Reference",
1474
1523
"section": "",
1475
-
"text": "Name\nDescription\n\n\n\n\nlens\nDecorator to create and register a lens transformation.\n\n\n\n\n\nlens.lens(f)\nDecorator to create and register a lens transformation.\nThis decorator converts a getter function into a Lens object and automatically registers it in the global LensNetwork registry.\nArgs: f: A getter function that transforms from source type S to view type V. Must have exactly one parameter with a type annotation.\nReturns: A Lens[S, V] object that can be called to apply the transformation or decorated with @lens_name.putter to add a putter function.\nExample: >>> @lens … def extract_name(full: FullData) -> NameOnly: … return NameOnly(name=full.name) … >>> @extract_name.putter … def extract_name_put(view: NameOnly, source: FullData) -> FullData: … return FullData(name=view.name, age=source.age)"
1524
+
"text": "Core types, decorators, and dataset classes\n\n\n\npackable\nDecorator to convert a regular class into a PackableSample.\n\n\nPackableSample\nBase class for samples that can be serialized with msgpack.\n\n\nDictSample\nDynamic sample type providing dict-like access to raw msgpack data.\n\n\nDataset\nA typed dataset built on WebDataset with lens transformations.\n\n\nSampleBatch\nA batch of samples with automatic attribute aggregation.\n\n\nLens\nA bidirectional transformation between two sample types.\n\n\nlens\nLens-based type transformations for datasets.\n\n\nload_dataset\nLoad a dataset from local files, remote URLs, or an index.\n\n\nDatasetDict\nA dictionary of split names to Dataset instances.\n\n\n\n\n\n\nAbstract protocols for storage backends\n\n\n\nPackable\nStructural protocol for packable sample types.\n\n\nIndexEntry\nCommon interface for index entries (local or atmosphere).\n\n\nAbstractIndex\nProtocol for index operations - implemented by LocalIndex and AtmosphereIndex.\n\n\nAbstractDataStore\nProtocol for data storage operations.\n\n\nDataSource\nProtocol for data sources that provide streams to Dataset.\n\n\n\n\n\n\nData source implementations for streaming\n\n\n\nURLSource\nData source for WebDataset-compatible URLs.\n\n\nS3Source\nData source for S3-compatible storage with explicit credentials.\n\n\n\n\n\n\nLocal Redis/S3 storage backend\n\n\n\nlocal.Index\nRedis-backed index for tracking datasets in a repository.\n\n\nlocal.LocalDatasetEntry\nIndex entry for a dataset stored in the local repository.\n\n\nlocal.S3DataStore\nS3-compatible data store implementing AbstractDataStore protocol.\n\n\n\n\n\n\nATProto federation\n\n\n\nAtmosphereClient\nATProto client wrapper for atdata operations.\n\n\nAtmosphereIndex\nATProto index implementing AbstractIndex protocol.\n\n\nAtmosphereIndexEntry\nEntry wrapper for ATProto dataset records implementing IndexEntry protocol.\n\n\nSchemaPublisher\nPublishes PackableSample schemas to ATProto.\n\n\nSchemaLoader\nLoads PackableSample schemas from ATProto.\n\n\nDatasetPublisher\nPublishes dataset index records to ATProto.\n\n\nDatasetLoader\nLoads dataset records from ATProto.\n\n\nLensPublisher\nPublishes Lens transformation records to ATProto.\n\n\nLensLoader\nLoads lens records from ATProto.\n\n\nAtUri\nParsed AT Protocol URI.\n\n\n\n\n\n\nLocal to atmosphere migration\n\n\n\npromote_to_atmosphere\nPromote a local dataset to the atmosphere network."
1476
1525
},
1477
1526
{
1478
-
"objectID": "api/DatasetLoader.html",
1479
-
"href": "api/DatasetLoader.html",
1480
-
"title": "DatasetLoader",
1527
+
"objectID": "api/index.html#core",
1528
+
"href": "api/index.html#core",
1529
+
"title": "API Reference",
1481
1530
"section": "",
1482
-
"text": "atmosphere.DatasetLoader(client)\nLoads dataset records from ATProto.\nThis class fetches dataset index records and can create Dataset objects from them. Note that loading a dataset requires having the corresponding Python class for the sample type.\nExample: >>> client = AtmosphereClient() >>> loader = DatasetLoader(client) >>> >>> # List available datasets >>> datasets = loader.list() >>> for ds in datasets: … print(ds[“name”], ds[“schemaRef”]) >>> >>> # Get a specific dataset record >>> record = loader.get(“at://did:plc:abc/ac.foundation.dataset.record/xyz”)\n\n\n\n\n\nName\nDescription\n\n\n\n\nget\nFetch a dataset record by AT URI.\n\n\nget_blob_urls\nGet fetchable URLs for blob-stored dataset shards.\n\n\nget_blobs\nGet the blob references from a dataset record.\n\n\nget_metadata\nGet the metadata from a dataset record.\n\n\nget_storage_type\nGet the storage type of a dataset record.\n\n\nget_urls\nGet the WebDataset URLs from a dataset record.\n\n\nlist_all\nList dataset records from a repository.\n\n\nto_dataset\nCreate a Dataset object from an ATProto record.\n\n\n\n\n\natmosphere.DatasetLoader.get(uri)\nFetch a dataset record by AT URI.\nArgs: uri: The AT URI of the dataset record.\nReturns: The dataset record as a dictionary.\nRaises: ValueError: If the record is not a dataset record.\n\n\n\natmosphere.DatasetLoader.get_blob_urls(uri)\nGet fetchable URLs for blob-stored dataset shards.\nThis resolves the PDS endpoint and constructs URLs that can be used to fetch the blob data directly.\nArgs: uri: The AT URI of the dataset record.\nReturns: List of URLs for fetching the blob data.\nRaises: ValueError: If storage type is not blobs or PDS cannot be resolved.\n\n\n\natmosphere.DatasetLoader.get_blobs(uri)\nGet the blob references from a dataset record.\nArgs: uri: The AT URI of the dataset record.\nReturns: List of blob reference dicts with keys: $type, ref, mimeType, size.\nRaises: ValueError: If the storage type is not blobs.\n\n\n\natmosphere.DatasetLoader.get_metadata(uri)\nGet the metadata from a dataset record.\nArgs: uri: The AT URI of the dataset record.\nReturns: The metadata dictionary, or None if no metadata.\n\n\n\natmosphere.DatasetLoader.get_storage_type(uri)\nGet the storage type of a dataset record.\nArgs: uri: The AT URI of the dataset record.\nReturns: Either “external” or “blobs”.\nRaises: ValueError: If storage type is unknown.\n\n\n\natmosphere.DatasetLoader.get_urls(uri)\nGet the WebDataset URLs from a dataset record.\nArgs: uri: The AT URI of the dataset record.\nReturns: List of WebDataset URLs.\nRaises: ValueError: If the storage type is not external URLs.\n\n\n\natmosphere.DatasetLoader.list_all(repo=None, limit=100)\nList dataset records from a repository.\nArgs: repo: The DID of the repository. Defaults to authenticated user. limit: Maximum number of records to return.\nReturns: List of dataset records.\n\n\n\natmosphere.DatasetLoader.to_dataset(uri, sample_type)\nCreate a Dataset object from an ATProto record.\nThis method creates a Dataset instance from a published record. You must provide the sample type class, which should match the schema referenced by the record.\nSupports both external URL storage and ATProto blob storage.\nArgs: uri: The AT URI of the dataset record. sample_type: The Python class for the sample type.\nReturns: A Dataset instance configured from the record.\nRaises: ValueError: If no storage URLs can be resolved.\nExample: >>> loader = DatasetLoader(client) >>> dataset = loader.to_dataset(uri, MySampleType) >>> for batch in dataset.shuffled(batch_size=32): … process(batch)"
1531
+
"text": "Core types, decorators, and dataset classes\n\n\n\npackable\nDecorator to convert a regular class into a PackableSample.\n\n\nPackableSample\nBase class for samples that can be serialized with msgpack.\n\n\nDictSample\nDynamic sample type providing dict-like access to raw msgpack data.\n\n\nDataset\nA typed dataset built on WebDataset with lens transformations.\n\n\nSampleBatch\nA batch of samples with automatic attribute aggregation.\n\n\nLens\nA bidirectional transformation between two sample types.\n\n\nlens\nLens-based type transformations for datasets.\n\n\nload_dataset\nLoad a dataset from local files, remote URLs, or an index.\n\n\nDatasetDict\nA dictionary of split names to Dataset instances."
1483
1532
},
1484
1533
{
1485
-
"objectID": "api/DatasetLoader.html#methods",
1486
-
"href": "api/DatasetLoader.html#methods",
1487
-
"title": "DatasetLoader",
1534
+
"objectID": "api/index.html#protocols",
1535
+
"href": "api/index.html#protocols",
1536
+
"title": "API Reference",
1488
1537
"section": "",
1489
-
"text": "Name\nDescription\n\n\n\n\nget\nFetch a dataset record by AT URI.\n\n\nget_blob_urls\nGet fetchable URLs for blob-stored dataset shards.\n\n\nget_blobs\nGet the blob references from a dataset record.\n\n\nget_metadata\nGet the metadata from a dataset record.\n\n\nget_storage_type\nGet the storage type of a dataset record.\n\n\nget_urls\nGet the WebDataset URLs from a dataset record.\n\n\nlist_all\nList dataset records from a repository.\n\n\nto_dataset\nCreate a Dataset object from an ATProto record.\n\n\n\n\n\natmosphere.DatasetLoader.get(uri)\nFetch a dataset record by AT URI.\nArgs: uri: The AT URI of the dataset record.\nReturns: The dataset record as a dictionary.\nRaises: ValueError: If the record is not a dataset record.\n\n\n\natmosphere.DatasetLoader.get_blob_urls(uri)\nGet fetchable URLs for blob-stored dataset shards.\nThis resolves the PDS endpoint and constructs URLs that can be used to fetch the blob data directly.\nArgs: uri: The AT URI of the dataset record.\nReturns: List of URLs for fetching the blob data.\nRaises: ValueError: If storage type is not blobs or PDS cannot be resolved.\n\n\n\natmosphere.DatasetLoader.get_blobs(uri)\nGet the blob references from a dataset record.\nArgs: uri: The AT URI of the dataset record.\nReturns: List of blob reference dicts with keys: $type, ref, mimeType, size.\nRaises: ValueError: If the storage type is not blobs.\n\n\n\natmosphere.DatasetLoader.get_metadata(uri)\nGet the metadata from a dataset record.\nArgs: uri: The AT URI of the dataset record.\nReturns: The metadata dictionary, or None if no metadata.\n\n\n\natmosphere.DatasetLoader.get_storage_type(uri)\nGet the storage type of a dataset record.\nArgs: uri: The AT URI of the dataset record.\nReturns: Either “external” or “blobs”.\nRaises: ValueError: If storage type is unknown.\n\n\n\natmosphere.DatasetLoader.get_urls(uri)\nGet the WebDataset URLs from a dataset record.\nArgs: uri: The AT URI of the dataset record.\nReturns: List of WebDataset URLs.\nRaises: ValueError: If the storage type is not external URLs.\n\n\n\natmosphere.DatasetLoader.list_all(repo=None, limit=100)\nList dataset records from a repository.\nArgs: repo: The DID of the repository. Defaults to authenticated user. limit: Maximum number of records to return.\nReturns: List of dataset records.\n\n\n\natmosphere.DatasetLoader.to_dataset(uri, sample_type)\nCreate a Dataset object from an ATProto record.\nThis method creates a Dataset instance from a published record. You must provide the sample type class, which should match the schema referenced by the record.\nSupports both external URL storage and ATProto blob storage.\nArgs: uri: The AT URI of the dataset record. sample_type: The Python class for the sample type.\nReturns: A Dataset instance configured from the record.\nRaises: ValueError: If no storage URLs can be resolved.\nExample: >>> loader = DatasetLoader(client) >>> dataset = loader.to_dataset(uri, MySampleType) >>> for batch in dataset.shuffled(batch_size=32): … process(batch)"
1538
+
"text": "Abstract protocols for storage backends\n\n\n\nPackable\nStructural protocol for packable sample types.\n\n\nIndexEntry\nCommon interface for index entries (local or atmosphere).\n\n\nAbstractIndex\nProtocol for index operations - implemented by LocalIndex and AtmosphereIndex.\n\n\nAbstractDataStore\nProtocol for data storage operations.\n\n\nDataSource\nProtocol for data sources that provide streams to Dataset."
1539
+
},
1540
+
{
1541
+
"objectID": "api/index.html#data-sources",
1542
+
"href": "api/index.html#data-sources",
1543
+
"title": "API Reference",
1544
+
"section": "",
1545
+
"text": "Data source implementations for streaming\n\n\n\nURLSource\nData source for WebDataset-compatible URLs.\n\n\nS3Source\nData source for S3-compatible storage with explicit credentials."
1546
+
},
1547
+
{
1548
+
"objectID": "api/index.html#local-storage",
1549
+
"href": "api/index.html#local-storage",
1550
+
"title": "API Reference",
1551
+
"section": "",
1552
+
"text": "Local Redis/S3 storage backend\n\n\n\nlocal.Index\nRedis-backed index for tracking datasets in a repository.\n\n\nlocal.LocalDatasetEntry\nIndex entry for a dataset stored in the local repository.\n\n\nlocal.S3DataStore\nS3-compatible data store implementing AbstractDataStore protocol."
1553
+
},
1554
+
{
1555
+
"objectID": "api/index.html#atmosphere",
1556
+
"href": "api/index.html#atmosphere",
1557
+
"title": "API Reference",
1558
+
"section": "",
1559
+
"text": "ATProto federation\n\n\n\nAtmosphereClient\nATProto client wrapper for atdata operations.\n\n\nAtmosphereIndex\nATProto index implementing AbstractIndex protocol.\n\n\nAtmosphereIndexEntry\nEntry wrapper for ATProto dataset records implementing IndexEntry protocol.\n\n\nSchemaPublisher\nPublishes PackableSample schemas to ATProto.\n\n\nSchemaLoader\nLoads PackableSample schemas from ATProto.\n\n\nDatasetPublisher\nPublishes dataset index records to ATProto.\n\n\nDatasetLoader\nLoads dataset records from ATProto.\n\n\nLensPublisher\nPublishes Lens transformation records to ATProto.\n\n\nLensLoader\nLoads lens records from ATProto.\n\n\nAtUri\nParsed AT Protocol URI."
1560
+
},
1561
+
{
1562
+
"objectID": "api/index.html#promotion",
1563
+
"href": "api/index.html#promotion",
1564
+
"title": "API Reference",
1565
+
"section": "",
1566
+
"text": "Local to atmosphere migration\n\n\n\npromote_to_atmosphere\nPromote a local dataset to the atmosphere network."
1567
+
},
1568
+
{
1569
+
"objectID": "api/URLSource.html",
1570
+
"href": "api/URLSource.html",
1571
+
"title": "URLSource",
1572
+
"section": "",
1573
+
"text": "URLSource(url)\nData source for WebDataset-compatible URLs.\nWraps WebDataset’s gopen to open URLs using built-in handlers for http, https, pipe, gs, hf, sftp, etc. Supports brace expansion for shard patterns like “data-{000..099}.tar”.\nThis is the default source type when a string URL is passed to Dataset.\nAttributes: url: URL or brace pattern for the shards.\nExample: >>> source = URLSource(“https://example.com/train-{000..009}.tar”) >>> for shard_id, stream in source.shards: … print(f”Streaming {shard_id}“)\n\n\n\n\n\nName\nDescription\n\n\n\n\nshard_list\nExpand brace pattern and return list of shard URLs (deprecated, use list_shards()).\n\n\nshards\nLazily yield (url, stream) pairs for each shard.\n\n\n\n\n\n\n\n\n\nName\nDescription\n\n\n\n\nlist_shards\nExpand brace pattern and return list of shard URLs.\n\n\nopen_shard\nOpen a single shard by URL.\n\n\n\n\n\nURLSource.list_shards()\nExpand brace pattern and return list of shard URLs.\n\n\n\nURLSource.open_shard(shard_id)\nOpen a single shard by URL.\nArgs: shard_id: URL of the shard to open.\nReturns: File-like stream from gopen.\nRaises: KeyError: If shard_id is not in list_shards()."
1574
+
},
1575
+
{
1576
+
"objectID": "api/URLSource.html#attributes",
1577
+
"href": "api/URLSource.html#attributes",
1578
+
"title": "URLSource",
1579
+
"section": "",
1580
+
"text": "Name\nDescription\n\n\n\n\nshard_list\nExpand brace pattern and return list of shard URLs (deprecated, use list_shards()).\n\n\nshards\nLazily yield (url, stream) pairs for each shard."
1581
+
},
1582
+
{
1583
+
"objectID": "api/URLSource.html#methods",
1584
+
"href": "api/URLSource.html#methods",
1585
+
"title": "URLSource",
1586
+
"section": "",
1587
+
"text": "Name\nDescription\n\n\n\n\nlist_shards\nExpand brace pattern and return list of shard URLs.\n\n\nopen_shard\nOpen a single shard by URL.\n\n\n\n\n\nURLSource.list_shards()\nExpand brace pattern and return list of shard URLs.\n\n\n\nURLSource.open_shard(shard_id)\nOpen a single shard by URL.\nArgs: shard_id: URL of the shard to open.\nReturns: File-like stream from gopen.\nRaises: KeyError: If shard_id is not in list_shards()."
1490
1588
},
1491
1589
{
1492
1590
"objectID": "api/DatasetPublisher.html",
+32
-12
docs/sitemap.xml
+32
-12
docs/sitemap.xml
···
53
53
<lastmod>2026-01-23T03:31:26.320Z</lastmod>
54
54
</url>
55
55
<url>
56
-
<loc>https://github.com/your-org/atdata/api/URLSource.html</loc>
57
-
<lastmod>2026-01-23T03:40:16.518Z</lastmod>
56
+
<loc>https://github.com/your-org/atdata/api/DataSource.html</loc>
57
+
<lastmod>2026-01-23T22:28:15.705Z</lastmod>
58
58
</url>
59
59
<url>
60
-
<loc>https://github.com/your-org/atdata/api/index.html</loc>
61
-
<lastmod>2026-01-23T03:40:16.476Z</lastmod>
60
+
<loc>https://github.com/your-org/atdata/api/DatasetLoader.html</loc>
61
+
<lastmod>2026-01-23T03:31:26.340Z</lastmod>
62
+
</url>
63
+
<url>
64
+
<loc>https://github.com/your-org/atdata/api/Lens.html</loc>
65
+
<lastmod>2026-01-23T22:28:15.684Z</lastmod>
62
66
</url>
63
67
<url>
64
68
<loc>https://github.com/your-org/atdata/api/local.Index.html</loc>
···
69
73
<lastmod>2026-01-23T03:31:26.234Z</lastmod>
70
74
</url>
71
75
<url>
76
+
<loc>https://github.com/your-org/atdata/api/AbstractDataStore.html</loc>
77
+
<lastmod>2026-01-23T22:28:15.703Z</lastmod>
78
+
</url>
79
+
<url>
72
80
<loc>https://github.com/your-org/atdata/api/local.S3DataStore.html</loc>
73
81
<lastmod>2026-01-23T03:31:26.298Z</lastmod>
74
82
</url>
···
77
85
<lastmod>2026-01-23T03:31:26.350Z</lastmod>
78
86
</url>
79
87
<url>
80
-
<loc>https://github.com/your-org/atdata/api/SampleBatch.html</loc>
81
-
<lastmod>2026-01-23T03:31:26.235Z</lastmod>
88
+
<loc>https://github.com/your-org/atdata/api/Packable-protocol.html</loc>
89
+
<lastmod>2026-01-23T22:28:15.690Z</lastmod>
90
+
</url>
91
+
<url>
92
+
<loc>https://github.com/your-org/atdata/api/packable.html</loc>
93
+
<lastmod>2026-01-23T22:28:15.652Z</lastmod>
82
94
</url>
83
95
<url>
84
96
<loc>https://github.com/your-org/atdata/index.html</loc>
85
97
<lastmod>2026-01-22T19:31:03.722Z</lastmod>
86
98
</url>
87
99
<url>
88
-
<loc>https://github.com/your-org/atdata/api/packable.html</loc>
89
-
<lastmod>2026-01-23T03:38:22.881Z</lastmod>
100
+
<loc>https://github.com/your-org/atdata/api/SampleBatch.html</loc>
101
+
<lastmod>2026-01-23T03:31:26.235Z</lastmod>
90
102
</url>
91
103
<url>
92
104
<loc>https://github.com/your-org/atdata/api/LensPublisher.html</loc>
···
97
109
<lastmod>2026-01-23T03:31:26.322Z</lastmod>
98
110
</url>
99
111
<url>
112
+
<loc>https://github.com/your-org/atdata/api/AbstractIndex.html</loc>
113
+
<lastmod>2026-01-23T22:28:15.699Z</lastmod>
114
+
</url>
115
+
<url>
100
116
<loc>https://github.com/your-org/atdata/api/local.LocalDatasetEntry.html</loc>
101
117
<lastmod>2026-01-23T03:31:26.295Z</lastmod>
102
118
</url>
···
105
121
<lastmod>2026-01-23T03:40:16.524Z</lastmod>
106
122
</url>
107
123
<url>
108
-
<loc>https://github.com/your-org/atdata/api/Lens.html</loc>
109
-
<lastmod>2026-01-23T03:40:16.513Z</lastmod>
124
+
<loc>https://github.com/your-org/atdata/api/IndexEntry.html</loc>
125
+
<lastmod>2026-01-23T22:28:15.692Z</lastmod>
126
+
</url>
127
+
<url>
128
+
<loc>https://github.com/your-org/atdata/api/index.html</loc>
129
+
<lastmod>2026-01-23T22:28:15.643Z</lastmod>
110
130
</url>
111
131
<url>
112
-
<loc>https://github.com/your-org/atdata/api/DatasetLoader.html</loc>
113
-
<lastmod>2026-01-23T03:31:26.340Z</lastmod>
132
+
<loc>https://github.com/your-org/atdata/api/URLSource.html</loc>
133
+
<lastmod>2026-01-23T03:40:16.518Z</lastmod>
114
134
</url>
115
135
<url>
116
136
<loc>https://github.com/your-org/atdata/api/DatasetPublisher.html</loc>
+12
-12
docs/tutorials/atmosphere.html
+12
-12
docs/tutorials/atmosphere.html
···
603
603
</section>
604
604
<section id="setup" class="level2">
605
605
<h2 class="anchored" data-anchor-id="setup">Setup</h2>
606
-
<div id="4d1fc71c" class="cell">
606
+
<div id="42091cc7" class="cell">
607
607
<div class="sourceCode cell-code" id="cb1"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
608
608
<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> numpy.typing <span class="im">import</span> NDArray</span>
609
609
<span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> atdata</span>
···
620
620
</section>
621
621
<section id="define-sample-types" class="level2">
622
622
<h2 class="anchored" data-anchor-id="define-sample-types">Define Sample Types</h2>
623
-
<div id="0ba8f069" class="cell">
623
+
<div id="ff718eaf" class="cell">
624
624
<div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
625
625
<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> ImageSample:</span>
626
626
<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a> <span class="co">"""A sample containing image data with metadata."""</span></span>
···
639
639
<section id="type-introspection" class="level2">
640
640
<h2 class="anchored" data-anchor-id="type-introspection">Type Introspection</h2>
641
641
<p>See what information is available from a PackableSample type:</p>
642
-
<div id="50ce0702" class="cell">
642
+
<div id="5cf4ce28" class="cell">
643
643
<div class="sourceCode cell-code" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> dataclasses <span class="im">import</span> fields, is_dataclass</span>
644
644
<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a></span>
645
645
<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Sample type: </span><span class="sc">{</span>ImageSample<span class="sc">.</span><span class="va">__name__</span><span class="sc">}</span><span class="ss">"</span>)</span>
···
667
667
<section id="at-uri-parsing" class="level2">
668
668
<h2 class="anchored" data-anchor-id="at-uri-parsing">AT URI Parsing</h2>
669
669
<p>ATProto records are identified by AT URIs:</p>
670
-
<div id="d7811962" class="cell">
670
+
<div id="3d758573" class="cell">
671
671
<div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a>uris <span class="op">=</span> [</span>
672
672
<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a> <span class="st">"at://did:plc:abc123/ac.foundation.dataset.sampleSchema/xyz789"</span>,</span>
673
673
<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a> <span class="st">"at://alice.bsky.social/ac.foundation.dataset.record/my-dataset"</span>,</span>
···
684
684
<section id="authentication" class="level2">
685
685
<h2 class="anchored" data-anchor-id="authentication">Authentication</h2>
686
686
<p>Connect to ATProto:</p>
687
-
<div id="f66e9fa9" class="cell">
687
+
<div id="f298476b" class="cell">
688
688
<div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a>client <span class="op">=</span> AtmosphereClient()</span>
689
689
<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a>client.login(<span class="st">"your.handle.social"</span>, <span class="st">"your-app-password"</span>)</span>
690
690
<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a></span>
···
694
694
</section>
695
695
<section id="publish-a-schema" class="level2">
696
696
<h2 class="anchored" data-anchor-id="publish-a-schema">Publish a Schema</h2>
697
-
<div id="ee3db495" class="cell">
697
+
<div id="6bd6e999" class="cell">
698
698
<div class="sourceCode cell-code" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a>schema_publisher <span class="op">=</span> SchemaPublisher(client)</span>
699
699
<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a>schema_uri <span class="op">=</span> schema_publisher.publish(</span>
700
700
<span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a> ImageSample,</span>
···
707
707
</section>
708
708
<section id="list-your-schemas" class="level2">
709
709
<h2 class="anchored" data-anchor-id="list-your-schemas">List Your Schemas</h2>
710
-
<div id="0299655d" class="cell">
710
+
<div id="8dd66d61" class="cell">
711
711
<div class="sourceCode cell-code" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a>schema_loader <span class="op">=</span> SchemaLoader(client)</span>
712
712
<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a>schemas <span class="op">=</span> schema_loader.list_all(limit<span class="op">=</span><span class="dv">10</span>)</span>
713
713
<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Found </span><span class="sc">{</span><span class="bu">len</span>(schemas)<span class="sc">}</span><span class="ss"> schema(s)"</span>)</span>
···
720
720
<h2 class="anchored" data-anchor-id="publish-a-dataset">Publish a Dataset</h2>
721
721
<section id="with-external-urls" class="level3">
722
722
<h3 class="anchored" data-anchor-id="with-external-urls">With External URLs</h3>
723
-
<div id="a1e63a6d" class="cell">
723
+
<div id="01e06939" class="cell">
724
724
<div class="sourceCode cell-code" id="cb8"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a>dataset_publisher <span class="op">=</span> DatasetPublisher(client)</span>
725
725
<span id="cb8-2"><a href="#cb8-2" aria-hidden="true" tabindex="-1"></a>dataset_uri <span class="op">=</span> dataset_publisher.publish_with_urls(</span>
726
726
<span id="cb8-3"><a href="#cb8-3" aria-hidden="true" tabindex="-1"></a> urls<span class="op">=</span>[<span class="st">"s3://example-bucket/demo-data-{000000..000009}.tar"</span>],</span>
···
736
736
<section id="with-blob-storage" class="level3">
737
737
<h3 class="anchored" data-anchor-id="with-blob-storage">With Blob Storage</h3>
738
738
<p>For smaller datasets, store data directly in ATProto blobs:</p>
739
-
<div id="c5b1646e" class="cell">
739
+
<div id="7f52243b" class="cell">
740
740
<div class="sourceCode cell-code" id="cb9"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb9-1"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> io</span>
741
741
<span id="cb9-2"><a href="#cb9-2" aria-hidden="true" tabindex="-1"></a></span>
742
742
<span id="cb9-3"><a href="#cb9-3" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
···
777
777
</section>
778
778
<section id="list-and-load-datasets" class="level2">
779
779
<h2 class="anchored" data-anchor-id="list-and-load-datasets">List and Load Datasets</h2>
780
-
<div id="9a0c581f" class="cell">
780
+
<div id="a1ca081c" class="cell">
781
781
<div class="sourceCode cell-code" id="cb10"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a>dataset_loader <span class="op">=</span> DatasetLoader(client)</span>
782
782
<span id="cb10-2"><a href="#cb10-2" aria-hidden="true" tabindex="-1"></a>datasets <span class="op">=</span> dataset_loader.list_all(limit<span class="op">=</span><span class="dv">10</span>)</span>
783
783
<span id="cb10-3"><a href="#cb10-3" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Found </span><span class="sc">{</span><span class="bu">len</span>(datasets)<span class="sc">}</span><span class="ss"> dataset(s)"</span>)</span>
···
792
792
</section>
793
793
<section id="load-a-dataset" class="level2">
794
794
<h2 class="anchored" data-anchor-id="load-a-dataset">Load a Dataset</h2>
795
-
<div id="3be3ed0a" class="cell">
795
+
<div id="68c36a8c" class="cell">
796
796
<div class="sourceCode cell-code" id="cb11"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb11-1"><a href="#cb11-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Check storage type</span></span>
797
797
<span id="cb11-2"><a href="#cb11-2" aria-hidden="true" tabindex="-1"></a>storage_type <span class="op">=</span> dataset_loader.get_storage_type(<span class="bu">str</span>(blob_dataset_uri))</span>
798
798
<span id="cb11-3"><a href="#cb11-3" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Storage type: </span><span class="sc">{</span>storage_type<span class="sc">}</span><span class="ss">"</span>)</span>
···
809
809
</section>
810
810
<section id="complete-publishing-workflow" class="level2">
811
811
<h2 class="anchored" data-anchor-id="complete-publishing-workflow">Complete Publishing Workflow</h2>
812
-
<div id="9b9a784f" class="cell">
812
+
<div id="a7412936" class="cell">
813
813
<div class="sourceCode cell-code" id="cb12"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb12-1"><a href="#cb12-1" aria-hidden="true" tabindex="-1"></a><span class="co"># 1. Define and create samples</span></span>
814
814
<span id="cb12-2"><a href="#cb12-2" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
815
815
<span id="cb12-3"><a href="#cb12-3" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> FeatureSample:</span>
+8
-8
docs/tutorials/local-workflow.html
+8
-8
docs/tutorials/local-workflow.html
···
599
599
</section>
600
600
<section id="setup" class="level2">
601
601
<h2 class="anchored" data-anchor-id="setup">Setup</h2>
602
-
<div id="b12058ff" class="cell">
602
+
<div id="26c13c51" class="cell">
603
603
<div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
604
604
<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> numpy.typing <span class="im">import</span> NDArray</span>
605
605
<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> atdata</span>
···
609
609
</section>
610
610
<section id="define-sample-types" class="level2">
611
611
<h2 class="anchored" data-anchor-id="define-sample-types">Define Sample Types</h2>
612
-
<div id="c645069f" class="cell">
612
+
<div id="ff37eb62" class="cell">
613
613
<div class="sourceCode cell-code" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
614
614
<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> TrainingSample:</span>
615
615
<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a> <span class="co">"""A sample containing features and label for training."""</span></span>
···
626
626
<section id="localdatasetentry" class="level2">
627
627
<h2 class="anchored" data-anchor-id="localdatasetentry">LocalDatasetEntry</h2>
628
628
<p>Create entries with content-addressable CIDs:</p>
629
-
<div id="85a6f459" class="cell">
629
+
<div id="556be281" class="cell">
630
630
<div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Create an entry manually</span></span>
631
631
<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a>entry <span class="op">=</span> LocalDatasetEntry(</span>
632
632
<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a> _name<span class="op">=</span><span class="st">"my-dataset"</span>,</span>
···
658
658
<section id="localindex" class="level2">
659
659
<h2 class="anchored" data-anchor-id="localindex">LocalIndex</h2>
660
660
<p>The index tracks datasets in Redis:</p>
661
-
<div id="35c5503d" class="cell">
661
+
<div id="236e1cae" class="cell">
662
662
<div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> redis <span class="im">import</span> Redis</span>
663
663
<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a></span>
664
664
<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a><span class="co"># Connect to Redis</span></span>
···
669
669
</div>
670
670
<section id="schema-management" class="level3">
671
671
<h3 class="anchored" data-anchor-id="schema-management">Schema Management</h3>
672
-
<div id="c3a01346" class="cell">
672
+
<div id="09cfff1f" class="cell">
673
673
<div class="sourceCode cell-code" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Publish a schema</span></span>
674
674
<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a>schema_ref <span class="op">=</span> index.publish_schema(TrainingSample, version<span class="op">=</span><span class="st">"1.0.0"</span>)</span>
675
675
<span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a><span class="bu">print</span>(<span class="ss">f"Published schema: </span><span class="sc">{</span>schema_ref<span class="sc">}</span><span class="ss">"</span>)</span>
···
691
691
<section id="s3datastore" class="level2">
692
692
<h2 class="anchored" data-anchor-id="s3datastore">S3DataStore</h2>
693
693
<p>For direct S3 operations:</p>
694
-
<div id="71427c87" class="cell">
694
+
<div id="335e8653" class="cell">
695
695
<div class="sourceCode cell-code" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a>creds <span class="op">=</span> {</span>
696
696
<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a> <span class="st">"AWS_ENDPOINT"</span>: <span class="st">"http://localhost:9000"</span>,</span>
697
697
<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a> <span class="st">"AWS_ACCESS_KEY_ID"</span>: <span class="st">"minioadmin"</span>,</span>
···
707
707
<section id="complete-index-workflow" class="level2">
708
708
<h2 class="anchored" data-anchor-id="complete-index-workflow">Complete Index Workflow</h2>
709
709
<p>Use <code>LocalIndex</code> with <code>S3DataStore</code> to store datasets with S3 storage and Redis indexing:</p>
710
-
<div id="97fbf0e7" class="cell">
710
+
<div id="db2d9cf4" class="cell">
711
711
<div class="sourceCode cell-code" id="cb8"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a><span class="co"># 1. Create sample data</span></span>
712
712
<span id="cb8-2"><a href="#cb8-2" aria-hidden="true" tabindex="-1"></a>samples <span class="op">=</span> [</span>
713
713
<span id="cb8-3"><a href="#cb8-3" aria-hidden="true" tabindex="-1"></a> TrainingSample(</span>
···
756
756
<section id="using-load_dataset-with-index" class="level2">
757
757
<h2 class="anchored" data-anchor-id="using-load_dataset-with-index">Using load_dataset with Index</h2>
758
758
<p>The <code>load_dataset()</code> function supports index lookup:</p>
759
-
<div id="d1c14e5d" class="cell">
759
+
<div id="c15c4f5e" class="cell">
760
760
<div class="sourceCode cell-code" id="cb9"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb9-1"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata <span class="im">import</span> load_dataset</span>
761
761
<span id="cb9-2"><a href="#cb9-2" aria-hidden="true" tabindex="-1"></a></span>
762
762
<span id="cb9-3"><a href="#cb9-3" aria-hidden="true" tabindex="-1"></a><span class="co"># Load from local index</span></span>
+11
-11
docs/tutorials/promotion.html
+11
-11
docs/tutorials/promotion.html
···
593
593
</section>
594
594
<section id="setup" class="level2">
595
595
<h2 class="anchored" data-anchor-id="setup">Setup</h2>
596
-
<div id="fd89a23a" class="cell">
596
+
<div id="c53852d7" class="cell">
597
597
<div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
598
598
<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> numpy.typing <span class="im">import</span> NDArray</span>
599
599
<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> atdata</span>
···
606
606
<section id="prepare-a-local-dataset" class="level2">
607
607
<h2 class="anchored" data-anchor-id="prepare-a-local-dataset">Prepare a Local Dataset</h2>
608
608
<p>First, set up a dataset in local storage:</p>
609
-
<div id="870f5e40" class="cell">
609
+
<div id="4c1ed06c" class="cell">
610
610
<div class="sourceCode cell-code" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="co"># 1. Define sample type</span></span>
611
611
<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
612
612
<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> ExperimentSample:</span>
···
656
656
<section id="basic-promotion" class="level2">
657
657
<h2 class="anchored" data-anchor-id="basic-promotion">Basic Promotion</h2>
658
658
<p>Promote the dataset to ATProto:</p>
659
-
<div id="4d8936ed" class="cell">
659
+
<div id="0aaa5713" class="cell">
660
660
<div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Connect to atmosphere</span></span>
661
661
<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a>client <span class="op">=</span> AtmosphereClient()</span>
662
662
<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a>client.login(<span class="st">"myhandle.bsky.social"</span>, <span class="st">"app-password"</span>)</span>
···
669
669
<section id="promotion-with-metadata" class="level2">
670
670
<h2 class="anchored" data-anchor-id="promotion-with-metadata">Promotion with Metadata</h2>
671
671
<p>Add description, tags, and license:</p>
672
-
<div id="27f45de0" class="cell">
672
+
<div id="2722dded" class="cell">
673
673
<div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a>at_uri <span class="op">=</span> promote_to_atmosphere(</span>
674
674
<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a> local_entry,</span>
675
675
<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a> local_index,</span>
···
685
685
<section id="schema-deduplication" class="level2">
686
686
<h2 class="anchored" data-anchor-id="schema-deduplication">Schema Deduplication</h2>
687
687
<p>The promotion workflow automatically checks for existing schemas:</p>
688
-
<div id="74a5b6e6" class="cell">
688
+
<div id="f726c241" class="cell">
689
689
<div class="sourceCode cell-code" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.promote <span class="im">import</span> _find_existing_schema</span>
690
690
<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a></span>
691
691
<span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a><span class="co"># Check if schema already exists</span></span>
···
697
697
<span id="cb6-9"><a href="#cb6-9" aria-hidden="true" tabindex="-1"></a> <span class="bu">print</span>(<span class="st">"No existing schema found, will publish new one"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
698
698
</div>
699
699
<p>When you promote multiple datasets with the same sample type:</p>
700
-
<div id="ee0aaf6d" class="cell">
700
+
<div id="6c840cfa" class="cell">
701
701
<div class="sourceCode cell-code" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a><span class="co"># First promotion: publishes schema</span></span>
702
702
<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a>uri1 <span class="op">=</span> promote_to_atmosphere(entry1, local_index, client)</span>
703
703
<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a></span>
···
712
712
<div class="tab-content">
713
713
<div id="tabset-1-1" class="tab-pane active" role="tabpanel" aria-labelledby="tabset-1-1-tab">
714
714
<p>By default, promotion keeps the original data URLs:</p>
715
-
<div id="756d2e85" class="cell">
715
+
<div id="19fb0aeb" class="cell">
716
716
<div class="sourceCode cell-code" id="cb8"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Data stays in original S3 location</span></span>
717
717
<span id="cb8-2"><a href="#cb8-2" aria-hidden="true" tabindex="-1"></a>at_uri <span class="op">=</span> promote_to_atmosphere(local_entry, local_index, client)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
718
718
</div>
···
725
725
</div>
726
726
<div id="tabset-1-2" class="tab-pane" role="tabpanel" aria-labelledby="tabset-1-2-tab">
727
727
<p>To copy data to a different storage location:</p>
728
-
<div id="b9a51abe" class="cell">
728
+
<div id="a0c75fc7" class="cell">
729
729
<div class="sourceCode cell-code" id="cb9"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb9-1"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.local <span class="im">import</span> S3DataStore</span>
730
730
<span id="cb9-2"><a href="#cb9-2" aria-hidden="true" tabindex="-1"></a></span>
731
731
<span id="cb9-3"><a href="#cb9-3" aria-hidden="true" tabindex="-1"></a><span class="co"># Create new data store</span></span>
···
755
755
<section id="verify-on-atmosphere" class="level2">
756
756
<h2 class="anchored" data-anchor-id="verify-on-atmosphere">Verify on Atmosphere</h2>
757
757
<p>After promotion, verify the dataset is accessible:</p>
758
-
<div id="d63c9542" class="cell">
758
+
<div id="55e29732" class="cell">
759
759
<div class="sourceCode cell-code" id="cb10"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> atdata.atmosphere <span class="im">import</span> AtmosphereIndex</span>
760
760
<span id="cb10-2"><a href="#cb10-2" aria-hidden="true" tabindex="-1"></a></span>
761
761
<span id="cb10-3"><a href="#cb10-3" aria-hidden="true" tabindex="-1"></a>atm_index <span class="op">=</span> AtmosphereIndex(client)</span>
···
776
776
</section>
777
777
<section id="error-handling" class="level2">
778
778
<h2 class="anchored" data-anchor-id="error-handling">Error Handling</h2>
779
-
<div id="7d482404" class="cell">
779
+
<div id="fb917dd1" class="cell">
780
780
<div class="sourceCode cell-code" id="cb11"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb11-1"><a href="#cb11-1" aria-hidden="true" tabindex="-1"></a><span class="cf">try</span>:</span>
781
781
<span id="cb11-2"><a href="#cb11-2" aria-hidden="true" tabindex="-1"></a> at_uri <span class="op">=</span> promote_to_atmosphere(local_entry, local_index, client)</span>
782
782
<span id="cb11-3"><a href="#cb11-3" aria-hidden="true" tabindex="-1"></a><span class="cf">except</span> <span class="pp">KeyError</span> <span class="im">as</span> e:</span>
···
800
800
</section>
801
801
<section id="complete-workflow" class="level2">
802
802
<h2 class="anchored" data-anchor-id="complete-workflow">Complete Workflow</h2>
803
-
<div id="928b4c17" class="cell">
803
+
<div id="9910ed97" class="cell">
804
804
<div class="sourceCode cell-code" id="cb12"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb12-1"><a href="#cb12-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Complete local-to-atmosphere workflow</span></span>
805
805
<span id="cb12-2"><a href="#cb12-2" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
806
806
<span id="cb12-3"><a href="#cb12-3" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> numpy.typing <span class="im">import</span> NDArray</span>
+6
-6
docs/tutorials/quickstart.html
+6
-6
docs/tutorials/quickstart.html
···
582
582
<section id="define-a-sample-type" class="level2">
583
583
<h2 class="anchored" data-anchor-id="define-a-sample-type">Define a Sample Type</h2>
584
584
<p>Use the <code>@packable</code> decorator to create a typed sample:</p>
585
-
<div id="206b96f0" class="cell">
585
+
<div id="63c15979" class="cell">
586
586
<div class="sourceCode cell-code" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> numpy <span class="im">as</span> np</span>
587
587
<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a><span class="im">from</span> numpy.typing <span class="im">import</span> NDArray</span>
588
588
<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> atdata</span>
···
603
603
</section>
604
604
<section id="create-sample-instances" class="level2">
605
605
<h2 class="anchored" data-anchor-id="create-sample-instances">Create Sample Instances</h2>
606
-
<div id="799f1261" class="cell">
606
+
<div id="bf1cb844" class="cell">
607
607
<div class="sourceCode cell-code" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Create a single sample</span></span>
608
608
<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a>sample <span class="op">=</span> ImageSample(</span>
609
609
<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a> image<span class="op">=</span>np.random.rand(<span class="dv">224</span>, <span class="dv">224</span>, <span class="dv">3</span>).astype(np.float32),</span>
···
624
624
<section id="write-a-dataset" class="level2">
625
625
<h2 class="anchored" data-anchor-id="write-a-dataset">Write a Dataset</h2>
626
626
<p>Use WebDataset’s <code>TarWriter</code> to create dataset files:</p>
627
-
<div id="4c01914a" class="cell">
627
+
<div id="d2143ab5" class="cell">
628
628
<div class="sourceCode cell-code" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="im">import</span> webdataset <span class="im">as</span> wds</span>
629
629
<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a></span>
630
630
<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a><span class="co"># Create 100 samples</span></span>
···
648
648
<section id="load-and-iterate" class="level2">
649
649
<h2 class="anchored" data-anchor-id="load-and-iterate">Load and Iterate</h2>
650
650
<p>Create a typed <code>Dataset</code> and iterate with batching:</p>
651
-
<div id="d0e4bf5b" class="cell">
651
+
<div id="49ef5046" class="cell">
652
652
<div class="sourceCode cell-code" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Load dataset with type</span></span>
653
653
<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a>dataset <span class="op">=</span> atdata.Dataset[ImageSample](<span class="st">"my-dataset-000000.tar"</span>)</span>
654
654
<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a></span>
···
669
669
<section id="shuffled-iteration" class="level2">
670
670
<h2 class="anchored" data-anchor-id="shuffled-iteration">Shuffled Iteration</h2>
671
671
<p>For training, use shuffled iteration:</p>
672
-
<div id="60ad7bce" class="cell">
672
+
<div id="3eaaf99d" class="cell">
673
673
<div class="sourceCode cell-code" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="cf">for</span> batch <span class="kw">in</span> dataset.shuffled(batch_size<span class="op">=</span><span class="dv">32</span>):</span>
674
674
<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a> <span class="co"># Samples are shuffled at shard and sample level</span></span>
675
675
<span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a> images <span class="op">=</span> batch.image</span>
···
683
683
<section id="use-lenses-for-type-transformations" class="level2">
684
684
<h2 class="anchored" data-anchor-id="use-lenses-for-type-transformations">Use Lenses for Type Transformations</h2>
685
685
<p>View datasets through different schemas:</p>
686
-
<div id="e1813a01" class="cell">
686
+
<div id="e660fe92" class="cell">
687
687
<div class="sourceCode cell-code" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Define a simplified view type</span></span>
688
688
<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a><span class="at">@atdata.packable</span></span>
689
689
<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a><span class="kw">class</span> SimplifiedSample:</span>
+13
docs_src/_quarto.yml
+13
docs_src/_quarto.yml
···
30
30
- name: DatasetDict
31
31
children: embedded
32
32
33
+
# Protocols
34
+
- title: Protocols
35
+
desc: Abstract protocols for storage backends
36
+
contents:
37
+
- kind: page
38
+
path: Packable-protocol
39
+
contents:
40
+
- name: Packable
41
+
- name: IndexEntry
42
+
- name: AbstractIndex
43
+
- name: AbstractDataStore
44
+
- name: DataSource
45
+
33
46
# Data Sources
34
47
- title: Data Sources
35
48
desc: Data source implementations for streaming
+76
docs_src/api/AbstractDataStore.qmd
+76
docs_src/api/AbstractDataStore.qmd
···
1
+
# AbstractDataStore { #atdata.AbstractDataStore }
2
+
3
+
```python
4
+
AbstractDataStore()
5
+
```
6
+
7
+
Protocol for data storage operations.
8
+
9
+
This protocol abstracts over different storage backends for dataset data:
10
+
- S3DataStore: S3-compatible object storage
11
+
- PDSBlobStore: ATProto PDS blob storage (future)
12
+
13
+
The separation of index (metadata) from data store (actual files) allows
14
+
flexible deployment: local index with S3 storage, atmosphere index with
15
+
S3 storage, or atmosphere index with PDS blobs.
16
+
17
+
Example:
18
+
>>> store = S3DataStore(credentials, bucket="my-bucket")
19
+
>>> urls = store.write_shards(dataset, prefix="training/v1")
20
+
>>> print(urls)
21
+
['s3://my-bucket/training/v1/shard-000000.tar', ...]
22
+
23
+
## Methods
24
+
25
+
| Name | Description |
26
+
| --- | --- |
27
+
| [read_url](#atdata.AbstractDataStore.read_url) | Resolve a storage URL for reading. |
28
+
| [supports_streaming](#atdata.AbstractDataStore.supports_streaming) | Whether this store supports streaming reads. |
29
+
| [write_shards](#atdata.AbstractDataStore.write_shards) | Write dataset shards to storage. |
30
+
31
+
### read_url { #atdata.AbstractDataStore.read_url }
32
+
33
+
```python
34
+
AbstractDataStore.read_url(url)
35
+
```
36
+
37
+
Resolve a storage URL for reading.
38
+
39
+
Some storage backends may need to transform URLs (e.g., signing S3 URLs
40
+
or resolving blob references). This method returns a URL that can be
41
+
used directly with WebDataset.
42
+
43
+
Args:
44
+
url: Storage URL to resolve.
45
+
46
+
Returns:
47
+
WebDataset-compatible URL for reading.
48
+
49
+
### supports_streaming { #atdata.AbstractDataStore.supports_streaming }
50
+
51
+
```python
52
+
AbstractDataStore.supports_streaming()
53
+
```
54
+
55
+
Whether this store supports streaming reads.
56
+
57
+
Returns:
58
+
True if the store supports efficient streaming (like S3),
59
+
False if data must be fully downloaded first.
60
+
61
+
### write_shards { #atdata.AbstractDataStore.write_shards }
62
+
63
+
```python
64
+
AbstractDataStore.write_shards(ds, *, prefix, **kwargs)
65
+
```
66
+
67
+
Write dataset shards to storage.
68
+
69
+
Args:
70
+
ds: The Dataset to write.
71
+
prefix: Path prefix for the shards (e.g., 'datasets/mnist/v1').
72
+
**kwargs: Backend-specific options (e.g., maxcount for shard size).
73
+
74
+
Returns:
75
+
List of URLs for the written shards, suitable for use with
76
+
WebDataset or atdata.Dataset().
+180
docs_src/api/AbstractIndex.qmd
+180
docs_src/api/AbstractIndex.qmd
···
1
+
# AbstractIndex { #atdata.AbstractIndex }
2
+
3
+
```python
4
+
AbstractIndex()
5
+
```
6
+
7
+
Protocol for index operations - implemented by LocalIndex and AtmosphereIndex.
8
+
9
+
This protocol defines the common interface for managing dataset metadata:
10
+
- Publishing and retrieving schemas
11
+
- Inserting and listing datasets
12
+
- (Future) Publishing and retrieving lenses
13
+
14
+
A single index can hold datasets of many different sample types. The sample
15
+
type is tracked via schema references, not as a generic parameter on the index.
16
+
17
+
Optional Extensions:
18
+
Some index implementations support additional features:
19
+
- ``data_store``: An AbstractDataStore for reading/writing dataset shards.
20
+
If present, ``load_dataset`` will use it for S3 credential resolution.
21
+
22
+
Example:
23
+
>>> def publish_and_list(index: AbstractIndex) -> None:
24
+
... # Publish schemas for different types
25
+
... schema1 = index.publish_schema(ImageSample, version="1.0.0")
26
+
... schema2 = index.publish_schema(TextSample, version="1.0.0")
27
+
...
28
+
... # Insert datasets of different types
29
+
... index.insert_dataset(image_ds, name="images")
30
+
... index.insert_dataset(text_ds, name="texts")
31
+
...
32
+
... # List all datasets (mixed types)
33
+
... for entry in index.list_datasets():
34
+
... print(f"{entry.name} -> {entry.schema_ref}")
35
+
36
+
## Attributes
37
+
38
+
| Name | Description |
39
+
| --- | --- |
40
+
| [datasets](#atdata.AbstractIndex.datasets) | Lazily iterate over all dataset entries in this index. |
41
+
| [schemas](#atdata.AbstractIndex.schemas) | Lazily iterate over all schema records in this index. |
42
+
43
+
## Methods
44
+
45
+
| Name | Description |
46
+
| --- | --- |
47
+
| [decode_schema](#atdata.AbstractIndex.decode_schema) | Reconstruct a Python Packable type from a stored schema. |
48
+
| [get_dataset](#atdata.AbstractIndex.get_dataset) | Get a dataset entry by name or reference. |
49
+
| [get_schema](#atdata.AbstractIndex.get_schema) | Get a schema record by reference. |
50
+
| [insert_dataset](#atdata.AbstractIndex.insert_dataset) | Insert a dataset into the index. |
51
+
| [list_datasets](#atdata.AbstractIndex.list_datasets) | Get all dataset entries as a materialized list. |
52
+
| [list_schemas](#atdata.AbstractIndex.list_schemas) | Get all schema records as a materialized list. |
53
+
| [publish_schema](#atdata.AbstractIndex.publish_schema) | Publish a schema for a sample type. |
54
+
55
+
### decode_schema { #atdata.AbstractIndex.decode_schema }
56
+
57
+
```python
58
+
AbstractIndex.decode_schema(ref)
59
+
```
60
+
61
+
Reconstruct a Python Packable type from a stored schema.
62
+
63
+
This method enables loading datasets without knowing the sample type
64
+
ahead of time. The index retrieves the schema record and dynamically
65
+
generates a Packable class matching the schema definition.
66
+
67
+
Args:
68
+
ref: Schema reference string (local:// or at://).
69
+
70
+
Returns:
71
+
A dynamically generated Packable class with fields matching
72
+
the schema definition. The class can be used with
73
+
``Dataset[T]`` to load and iterate over samples.
74
+
75
+
Raises:
76
+
KeyError: If schema not found.
77
+
ValueError: If schema cannot be decoded (unsupported field types).
78
+
79
+
Example:
80
+
>>> entry = index.get_dataset("my-dataset")
81
+
>>> SampleType = index.decode_schema(entry.schema_ref)
82
+
>>> ds = Dataset[SampleType](entry.data_urls[0])
83
+
>>> for sample in ds.ordered():
84
+
... print(sample) # sample is instance of SampleType
85
+
86
+
### get_dataset { #atdata.AbstractIndex.get_dataset }
87
+
88
+
```python
89
+
AbstractIndex.get_dataset(ref)
90
+
```
91
+
92
+
Get a dataset entry by name or reference.
93
+
94
+
Args:
95
+
ref: Dataset name, path, or full reference string.
96
+
97
+
Returns:
98
+
IndexEntry for the dataset.
99
+
100
+
Raises:
101
+
KeyError: If dataset not found.
102
+
103
+
### get_schema { #atdata.AbstractIndex.get_schema }
104
+
105
+
```python
106
+
AbstractIndex.get_schema(ref)
107
+
```
108
+
109
+
Get a schema record by reference.
110
+
111
+
Args:
112
+
ref: Schema reference string (local:// or at://).
113
+
114
+
Returns:
115
+
Schema record as a dictionary with fields like 'name', 'version',
116
+
'fields', etc.
117
+
118
+
Raises:
119
+
KeyError: If schema not found.
120
+
121
+
### insert_dataset { #atdata.AbstractIndex.insert_dataset }
122
+
123
+
```python
124
+
AbstractIndex.insert_dataset(ds, *, name, schema_ref=None, **kwargs)
125
+
```
126
+
127
+
Insert a dataset into the index.
128
+
129
+
The sample type is inferred from ``ds.sample_type``. If schema_ref is not
130
+
provided, the schema may be auto-published based on the sample type.
131
+
132
+
Args:
133
+
ds: The Dataset to register in the index (any sample type).
134
+
name: Human-readable name for the dataset.
135
+
schema_ref: Optional explicit schema reference. If not provided,
136
+
the schema may be auto-published or inferred from ds.sample_type.
137
+
**kwargs: Additional backend-specific options.
138
+
139
+
Returns:
140
+
IndexEntry for the inserted dataset.
141
+
142
+
### list_datasets { #atdata.AbstractIndex.list_datasets }
143
+
144
+
```python
145
+
AbstractIndex.list_datasets()
146
+
```
147
+
148
+
Get all dataset entries as a materialized list.
149
+
150
+
Returns:
151
+
List of IndexEntry for each dataset.
152
+
153
+
### list_schemas { #atdata.AbstractIndex.list_schemas }
154
+
155
+
```python
156
+
AbstractIndex.list_schemas()
157
+
```
158
+
159
+
Get all schema records as a materialized list.
160
+
161
+
Returns:
162
+
List of schema records as dictionaries.
163
+
164
+
### publish_schema { #atdata.AbstractIndex.publish_schema }
165
+
166
+
```python
167
+
AbstractIndex.publish_schema(sample_type, *, version='1.0.0', **kwargs)
168
+
```
169
+
170
+
Publish a schema for a sample type.
171
+
172
+
Args:
173
+
sample_type: A Packable type (PackableSample subclass or @packable-decorated).
174
+
version: Semantic version string for the schema.
175
+
**kwargs: Additional backend-specific options.
176
+
177
+
Returns:
178
+
Schema reference string:
179
+
- Local: 'local://schemas/{module.Class}@{version}'
180
+
- Atmosphere: 'at://did:plc:.../ac.foundation.dataset.sampleSchema/...'
+80
docs_src/api/DataSource.qmd
+80
docs_src/api/DataSource.qmd
···
1
+
# DataSource { #atdata.DataSource }
2
+
3
+
```python
4
+
DataSource()
5
+
```
6
+
7
+
Protocol for data sources that provide streams to Dataset.
8
+
9
+
A DataSource abstracts over different ways of accessing dataset shards:
10
+
- URLSource: Standard WebDataset-compatible URLs (http, https, pipe, gs, etc.)
11
+
- S3Source: S3-compatible storage with explicit credentials
12
+
- BlobSource: ATProto blob references (future)
13
+
14
+
The key method is ``shards()``, which yields (identifier, stream) pairs.
15
+
These are fed directly to WebDataset's tar_file_expander, bypassing URL
16
+
resolution entirely. This enables:
17
+
- Private S3 repos with credentials
18
+
- Custom endpoints (Cloudflare R2, MinIO)
19
+
- ATProto blob streaming
20
+
- Any other source that can provide file-like objects
21
+
22
+
Example:
23
+
>>> source = S3Source(
24
+
... bucket="my-bucket",
25
+
... keys=["data-000.tar", "data-001.tar"],
26
+
... endpoint="https://r2.example.com",
27
+
... credentials=creds,
28
+
... )
29
+
>>> ds = Dataset[MySample](source)
30
+
>>> for sample in ds.ordered():
31
+
... print(sample)
32
+
33
+
## Attributes
34
+
35
+
| Name | Description |
36
+
| --- | --- |
37
+
| [shards](#atdata.DataSource.shards) | Lazily yield (identifier, stream) pairs for each shard. |
38
+
39
+
## Methods
40
+
41
+
| Name | Description |
42
+
| --- | --- |
43
+
| [list_shards](#atdata.DataSource.list_shards) | Get list of shard identifiers without opening streams. |
44
+
| [open_shard](#atdata.DataSource.open_shard) | Open a single shard by its identifier. |
45
+
46
+
### list_shards { #atdata.DataSource.list_shards }
47
+
48
+
```python
49
+
DataSource.list_shards()
50
+
```
51
+
52
+
Get list of shard identifiers without opening streams.
53
+
54
+
Used for metadata queries like counting shards without actually
55
+
streaming data. Implementations should return identifiers that
56
+
match what shards would yield.
57
+
58
+
Returns:
59
+
List of shard identifier strings.
60
+
61
+
### open_shard { #atdata.DataSource.open_shard }
62
+
63
+
```python
64
+
DataSource.open_shard(shard_id)
65
+
```
66
+
67
+
Open a single shard by its identifier.
68
+
69
+
This method enables random access to individual shards, which is
70
+
required for PyTorch DataLoader worker splitting. Each worker opens
71
+
only its assigned shards rather than iterating all shards.
72
+
73
+
Args:
74
+
shard_id: Shard identifier from shard_list.
75
+
76
+
Returns:
77
+
File-like stream for reading the shard.
78
+
79
+
Raises:
80
+
KeyError: If shard_id is not in shard_list.
+25
docs_src/api/IndexEntry.qmd
+25
docs_src/api/IndexEntry.qmd
···
1
+
# IndexEntry { #atdata.IndexEntry }
2
+
3
+
```python
4
+
IndexEntry()
5
+
```
6
+
7
+
Common interface for index entries (local or atmosphere).
8
+
9
+
Both LocalDatasetEntry and atmosphere DatasetRecord-based entries
10
+
should satisfy this protocol, enabling code that works with either.
11
+
12
+
Properties:
13
+
name: Human-readable dataset name
14
+
schema_ref: Reference to schema (local:// path or AT URI)
15
+
data_urls: WebDataset URLs for the data
16
+
metadata: Arbitrary metadata dict, or None
17
+
18
+
## Attributes
19
+
20
+
| Name | Description |
21
+
| --- | --- |
22
+
| [data_urls](#atdata.IndexEntry.data_urls) | WebDataset URLs for the data. |
23
+
| [metadata](#atdata.IndexEntry.metadata) | Arbitrary metadata dictionary, or None if not set. |
24
+
| [name](#atdata.IndexEntry.name) | Human-readable dataset name. |
25
+
| [schema_ref](#atdata.IndexEntry.schema_ref) | Reference to the schema for this dataset. |
+60
docs_src/api/Packable-protocol.qmd
+60
docs_src/api/Packable-protocol.qmd
···
1
+
# Packable { #atdata.Packable }
2
+
3
+
```python
4
+
Packable()
5
+
```
6
+
7
+
Structural protocol for packable sample types.
8
+
9
+
This protocol allows classes decorated with ``@packable`` to be recognized
10
+
as valid types for lens transformations and schema operations, even though
11
+
the decorator doesn't change the class's nominal type at static analysis time.
12
+
13
+
Both ``PackableSample`` subclasses and ``@packable``-decorated classes
14
+
satisfy this protocol structurally.
15
+
16
+
The protocol captures the full interface needed for:
17
+
- Lens type transformations (as_wds, from_data)
18
+
- Schema publishing (class introspection via dataclass fields)
19
+
- Serialization/deserialization (packed, from_bytes)
20
+
21
+
Example:
22
+
>>> @packable
23
+
... class MySample:
24
+
... name: str
25
+
... value: int
26
+
...
27
+
>>> def process(sample_type: Type[Packable]) -> None:
28
+
... # Type checker knows sample_type has from_bytes, packed, etc.
29
+
... instance = sample_type.from_bytes(data)
30
+
... print(instance.packed)
31
+
32
+
## Attributes
33
+
34
+
| Name | Description |
35
+
| --- | --- |
36
+
| [as_wds](#atdata.Packable.as_wds) | WebDataset-compatible representation with __key__ and msgpack. |
37
+
| [packed](#atdata.Packable.packed) | Pack this sample's data into msgpack bytes. |
38
+
39
+
## Methods
40
+
41
+
| Name | Description |
42
+
| --- | --- |
43
+
| [from_bytes](#atdata.Packable.from_bytes) | Create instance from raw msgpack bytes. |
44
+
| [from_data](#atdata.Packable.from_data) | Create instance from unpacked msgpack data dictionary. |
45
+
46
+
### from_bytes { #atdata.Packable.from_bytes }
47
+
48
+
```python
49
+
Packable.from_bytes(bs)
50
+
```
51
+
52
+
Create instance from raw msgpack bytes.
53
+
54
+
### from_data { #atdata.Packable.from_data }
55
+
56
+
```python
57
+
Packable.from_data(data)
58
+
```
59
+
60
+
Create instance from unpacked msgpack data dictionary.
+12
docs_src/api/index.qmd
+12
docs_src/api/index.qmd
···
16
16
| [load_dataset](load_dataset.qmd#atdata.load_dataset) | Load a dataset from local files, remote URLs, or an index. |
17
17
| [DatasetDict](DatasetDict.qmd#atdata.DatasetDict) | A dictionary of split names to Dataset instances. |
18
18
19
+
## Protocols
20
+
21
+
Abstract protocols for storage backends
22
+
23
+
| | |
24
+
| --- | --- |
25
+
| [Packable](Packable-protocol.qmd#atdata.Packable) | Structural protocol for packable sample types. |
26
+
| [IndexEntry](IndexEntry.qmd#atdata.IndexEntry) | Common interface for index entries (local or atmosphere). |
27
+
| [AbstractIndex](AbstractIndex.qmd#atdata.AbstractIndex) | Protocol for index operations - implemented by LocalIndex and AtmosphereIndex. |
28
+
| [AbstractDataStore](AbstractDataStore.qmd#atdata.AbstractDataStore) | Protocol for data storage operations. |
29
+
| [DataSource](DataSource.qmd#atdata.DataSource) | Protocol for data sources that provide streams to Dataset. |
30
+
19
31
## Data Sources
20
32
21
33
Data source implementations for streaming
+1
-1
docs_src/objects.json
+1
-1
docs_src/objects.json
···
1
-
{"project": "atdata", "version": "0.0.9999", "count": 266, "items": [{"name": "atdata.packable", "domain": "py", "role": "function", "priority": "1", "uri": "api/packable.html#atdata.packable", "dispname": "-"}, {"name": "atdata.dataset.packable", "domain": "py", "role": "function", "priority": "1", "uri": "api/packable.html#atdata.packable", "dispname": "atdata.packable"}, {"name": "atdata.PackableSample.as_wds", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/PackableSample.html#atdata.PackableSample.as_wds", "dispname": "-"}, {"name": "atdata.dataset.PackableSample.as_wds", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/PackableSample.html#atdata.PackableSample.as_wds", "dispname": "atdata.PackableSample.as_wds"}, {"name": "atdata.PackableSample.from_bytes", "domain": "py", "role": "function", "priority": "1", "uri": "api/PackableSample.html#atdata.PackableSample.from_bytes", "dispname": "-"}, {"name": "atdata.dataset.PackableSample.from_bytes", "domain": "py", "role": "function", "priority": "1", "uri": "api/PackableSample.html#atdata.PackableSample.from_bytes", "dispname": "atdata.PackableSample.from_bytes"}, {"name": "atdata.PackableSample.from_data", "domain": "py", "role": "function", "priority": "1", "uri": "api/PackableSample.html#atdata.PackableSample.from_data", "dispname": "-"}, {"name": "atdata.dataset.PackableSample.from_data", "domain": "py", "role": "function", "priority": "1", "uri": "api/PackableSample.html#atdata.PackableSample.from_data", "dispname": "atdata.PackableSample.from_data"}, {"name": "atdata.PackableSample.packed", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/PackableSample.html#atdata.PackableSample.packed", "dispname": "-"}, {"name": "atdata.dataset.PackableSample.packed", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/PackableSample.html#atdata.PackableSample.packed", "dispname": "atdata.PackableSample.packed"}, {"name": "atdata.PackableSample", "domain": "py", "role": "class", "priority": "1", "uri": "api/PackableSample.html#atdata.PackableSample", "dispname": "-"}, {"name": "atdata.dataset.PackableSample", "domain": "py", "role": "class", "priority": "1", "uri": "api/PackableSample.html#atdata.PackableSample", "dispname": "atdata.PackableSample"}, {"name": "atdata.DictSample.as_wds", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.as_wds", "dispname": "-"}, {"name": "atdata.dataset.DictSample.as_wds", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.as_wds", "dispname": "atdata.DictSample.as_wds"}, {"name": "atdata.DictSample.from_bytes", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.from_bytes", "dispname": "-"}, {"name": "atdata.dataset.DictSample.from_bytes", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.from_bytes", "dispname": "atdata.DictSample.from_bytes"}, {"name": "atdata.DictSample.from_data", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.from_data", "dispname": "-"}, {"name": "atdata.dataset.DictSample.from_data", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.from_data", "dispname": "atdata.DictSample.from_data"}, {"name": "atdata.DictSample.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.get", "dispname": "-"}, {"name": "atdata.dataset.DictSample.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.get", "dispname": "atdata.DictSample.get"}, {"name": "atdata.DictSample.items", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.items", "dispname": "-"}, {"name": "atdata.dataset.DictSample.items", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.items", "dispname": "atdata.DictSample.items"}, {"name": "atdata.DictSample.keys", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.keys", "dispname": "-"}, {"name": "atdata.dataset.DictSample.keys", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.keys", "dispname": "atdata.DictSample.keys"}, {"name": "atdata.DictSample.packed", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.packed", "dispname": "-"}, {"name": "atdata.dataset.DictSample.packed", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.packed", "dispname": "atdata.DictSample.packed"}, {"name": "atdata.DictSample.to_dict", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.to_dict", "dispname": "-"}, {"name": "atdata.dataset.DictSample.to_dict", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.to_dict", "dispname": "atdata.DictSample.to_dict"}, {"name": "atdata.DictSample.values", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.values", "dispname": "-"}, {"name": "atdata.dataset.DictSample.values", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.values", "dispname": "atdata.DictSample.values"}, {"name": "atdata.DictSample", "domain": "py", "role": "class", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample", "dispname": "-"}, {"name": "atdata.dataset.DictSample", "domain": "py", "role": "class", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample", "dispname": "atdata.DictSample"}, {"name": "atdata.Dataset.as_type", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.as_type", "dispname": "-"}, {"name": "atdata.dataset.Dataset.as_type", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.as_type", "dispname": "atdata.Dataset.as_type"}, {"name": "atdata.Dataset.batch_type", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.batch_type", "dispname": "-"}, {"name": "atdata.dataset.Dataset.batch_type", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.batch_type", "dispname": "atdata.Dataset.batch_type"}, {"name": "atdata.Dataset.list_shards", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.list_shards", "dispname": "-"}, {"name": "atdata.dataset.Dataset.list_shards", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.list_shards", "dispname": "atdata.Dataset.list_shards"}, {"name": "atdata.Dataset.metadata", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.metadata", "dispname": "-"}, {"name": "atdata.dataset.Dataset.metadata", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.metadata", "dispname": "atdata.Dataset.metadata"}, {"name": "atdata.Dataset.metadata_url", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.metadata_url", "dispname": "-"}, {"name": "atdata.dataset.Dataset.metadata_url", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.metadata_url", "dispname": "atdata.Dataset.metadata_url"}, {"name": "atdata.Dataset.ordered", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.ordered", "dispname": "-"}, {"name": "atdata.dataset.Dataset.ordered", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.ordered", "dispname": "atdata.Dataset.ordered"}, {"name": "atdata.Dataset.sample_type", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.sample_type", "dispname": "-"}, {"name": "atdata.dataset.Dataset.sample_type", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.sample_type", "dispname": "atdata.Dataset.sample_type"}, {"name": "atdata.Dataset.shard_list", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.shard_list", "dispname": "-"}, {"name": "atdata.dataset.Dataset.shard_list", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.shard_list", "dispname": "atdata.Dataset.shard_list"}, {"name": "atdata.Dataset.shuffled", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.shuffled", "dispname": "-"}, {"name": "atdata.dataset.Dataset.shuffled", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.shuffled", "dispname": "atdata.Dataset.shuffled"}, {"name": "atdata.Dataset.source", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.source", "dispname": "-"}, {"name": "atdata.dataset.Dataset.source", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.source", "dispname": "atdata.Dataset.source"}, {"name": "atdata.Dataset.to_parquet", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.to_parquet", "dispname": "-"}, {"name": "atdata.dataset.Dataset.to_parquet", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.to_parquet", "dispname": "atdata.Dataset.to_parquet"}, {"name": "atdata.Dataset.wrap", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.wrap", "dispname": "-"}, {"name": "atdata.dataset.Dataset.wrap", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.wrap", "dispname": "atdata.Dataset.wrap"}, {"name": "atdata.Dataset.wrap_batch", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.wrap_batch", "dispname": "-"}, {"name": "atdata.dataset.Dataset.wrap_batch", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.wrap_batch", "dispname": "atdata.Dataset.wrap_batch"}, {"name": "atdata.Dataset", "domain": "py", "role": "class", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset", "dispname": "-"}, {"name": "atdata.dataset.Dataset", "domain": "py", "role": "class", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset", "dispname": "atdata.Dataset"}, {"name": "atdata.SampleBatch.sample_type", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/SampleBatch.html#atdata.SampleBatch.sample_type", "dispname": "-"}, {"name": "atdata.dataset.SampleBatch.sample_type", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/SampleBatch.html#atdata.SampleBatch.sample_type", "dispname": "atdata.SampleBatch.sample_type"}, {"name": "atdata.SampleBatch", "domain": "py", "role": "class", "priority": "1", "uri": "api/SampleBatch.html#atdata.SampleBatch", "dispname": "-"}, {"name": "atdata.dataset.SampleBatch", "domain": "py", "role": "class", "priority": "1", "uri": "api/SampleBatch.html#atdata.SampleBatch", "dispname": "atdata.SampleBatch"}, {"name": "atdata.Lens.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/Lens.html#atdata.Lens.get", "dispname": "-"}, {"name": "atdata.lens.Lens.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/Lens.html#atdata.Lens.get", "dispname": "atdata.Lens.get"}, {"name": "atdata.Lens.put", "domain": "py", "role": "function", "priority": "1", "uri": "api/Lens.html#atdata.Lens.put", "dispname": "-"}, {"name": "atdata.lens.Lens.put", "domain": "py", "role": "function", "priority": "1", "uri": "api/Lens.html#atdata.Lens.put", "dispname": "atdata.Lens.put"}, {"name": "atdata.Lens.putter", "domain": "py", "role": "function", "priority": "1", "uri": "api/Lens.html#atdata.Lens.putter", "dispname": "-"}, {"name": "atdata.lens.Lens.putter", "domain": "py", "role": "function", "priority": "1", "uri": "api/Lens.html#atdata.Lens.putter", "dispname": "atdata.Lens.putter"}, {"name": "atdata.Lens", "domain": "py", "role": "class", "priority": "1", "uri": "api/Lens.html#atdata.Lens", "dispname": "-"}, {"name": "atdata.lens.Lens", "domain": "py", "role": "class", "priority": "1", "uri": "api/Lens.html#atdata.Lens", "dispname": "atdata.Lens"}, {"name": "atdata.lens.Lens.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/lens.html#atdata.lens.Lens.get", "dispname": "-"}, {"name": "atdata.lens.Lens.put", "domain": "py", "role": "function", "priority": "1", "uri": "api/lens.html#atdata.lens.Lens.put", "dispname": "-"}, {"name": "atdata.lens.Lens.putter", "domain": "py", "role": "function", "priority": "1", "uri": "api/lens.html#atdata.lens.Lens.putter", "dispname": "-"}, {"name": "atdata.lens.Lens", "domain": "py", "role": "class", "priority": "1", "uri": "api/lens.html#atdata.lens.Lens", "dispname": "-"}, {"name": "atdata.lens.LensNetwork.register", "domain": "py", "role": "function", "priority": "1", "uri": "api/lens.html#atdata.lens.LensNetwork.register", "dispname": "-"}, {"name": "atdata.lens.LensNetwork.transform", "domain": "py", "role": "function", "priority": "1", "uri": "api/lens.html#atdata.lens.LensNetwork.transform", "dispname": "-"}, {"name": "atdata.lens.LensNetwork", "domain": "py", "role": "class", "priority": "1", "uri": "api/lens.html#atdata.lens.LensNetwork", "dispname": "-"}, {"name": "atdata.lens.lens", "domain": "py", "role": "function", "priority": "1", "uri": "api/lens.html#atdata.lens.lens", "dispname": "-"}, {"name": "atdata.lens", "domain": "py", "role": "module", "priority": "1", "uri": "api/lens.html#atdata.lens", "dispname": "-"}, {"name": "atdata.load_dataset", "domain": "py", "role": "function", "priority": "1", "uri": "api/load_dataset.html#atdata.load_dataset", "dispname": "-"}, {"name": "atdata._hf_api.load_dataset", "domain": "py", "role": "function", "priority": "1", "uri": "api/load_dataset.html#atdata.load_dataset", "dispname": "atdata.load_dataset"}, {"name": "atdata.DatasetDict.num_shards", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DatasetDict.html#atdata.DatasetDict.num_shards", "dispname": "-"}, {"name": "atdata._hf_api.DatasetDict.num_shards", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DatasetDict.html#atdata.DatasetDict.num_shards", "dispname": "atdata.DatasetDict.num_shards"}, {"name": "atdata.DatasetDict.sample_type", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DatasetDict.html#atdata.DatasetDict.sample_type", "dispname": "-"}, {"name": "atdata._hf_api.DatasetDict.sample_type", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DatasetDict.html#atdata.DatasetDict.sample_type", "dispname": "atdata.DatasetDict.sample_type"}, {"name": "atdata.DatasetDict.streaming", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DatasetDict.html#atdata.DatasetDict.streaming", "dispname": "-"}, {"name": "atdata._hf_api.DatasetDict.streaming", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DatasetDict.html#atdata.DatasetDict.streaming", "dispname": "atdata.DatasetDict.streaming"}, {"name": "atdata.DatasetDict", "domain": "py", "role": "class", "priority": "1", "uri": "api/DatasetDict.html#atdata.DatasetDict", "dispname": "-"}, {"name": "atdata._hf_api.DatasetDict", "domain": "py", "role": "class", "priority": "1", "uri": "api/DatasetDict.html#atdata.DatasetDict", "dispname": "atdata.DatasetDict"}, {"name": "atdata.URLSource.list_shards", "domain": "py", "role": "function", "priority": "1", "uri": "api/URLSource.html#atdata.URLSource.list_shards", "dispname": "-"}, {"name": "atdata._sources.URLSource.list_shards", "domain": "py", "role": "function", "priority": "1", "uri": "api/URLSource.html#atdata.URLSource.list_shards", "dispname": "atdata.URLSource.list_shards"}, {"name": "atdata.URLSource.open_shard", "domain": "py", "role": "function", "priority": "1", "uri": "api/URLSource.html#atdata.URLSource.open_shard", "dispname": "-"}, {"name": "atdata._sources.URLSource.open_shard", "domain": "py", "role": "function", "priority": "1", "uri": "api/URLSource.html#atdata.URLSource.open_shard", "dispname": "atdata.URLSource.open_shard"}, {"name": "atdata.URLSource.shard_list", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/URLSource.html#atdata.URLSource.shard_list", "dispname": "-"}, {"name": "atdata._sources.URLSource.shard_list", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/URLSource.html#atdata.URLSource.shard_list", "dispname": "atdata.URLSource.shard_list"}, {"name": "atdata.URLSource.shards", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/URLSource.html#atdata.URLSource.shards", "dispname": "-"}, {"name": "atdata._sources.URLSource.shards", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/URLSource.html#atdata.URLSource.shards", "dispname": "atdata.URLSource.shards"}, {"name": "atdata.URLSource", "domain": "py", "role": "class", "priority": "1", "uri": "api/URLSource.html#atdata.URLSource", "dispname": "-"}, {"name": "atdata._sources.URLSource", "domain": "py", "role": "class", "priority": "1", "uri": "api/URLSource.html#atdata.URLSource", "dispname": "atdata.URLSource"}, {"name": "atdata.S3Source.from_credentials", "domain": "py", "role": "function", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.from_credentials", "dispname": "-"}, {"name": "atdata._sources.S3Source.from_credentials", "domain": "py", "role": "function", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.from_credentials", "dispname": "atdata.S3Source.from_credentials"}, {"name": "atdata.S3Source.from_urls", "domain": "py", "role": "function", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.from_urls", "dispname": "-"}, {"name": "atdata._sources.S3Source.from_urls", "domain": "py", "role": "function", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.from_urls", "dispname": "atdata.S3Source.from_urls"}, {"name": "atdata.S3Source.list_shards", "domain": "py", "role": "function", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.list_shards", "dispname": "-"}, {"name": "atdata._sources.S3Source.list_shards", "domain": "py", "role": "function", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.list_shards", "dispname": "atdata.S3Source.list_shards"}, {"name": "atdata.S3Source.open_shard", "domain": "py", "role": "function", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.open_shard", "dispname": "-"}, {"name": "atdata._sources.S3Source.open_shard", "domain": "py", "role": "function", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.open_shard", "dispname": "atdata.S3Source.open_shard"}, {"name": "atdata.S3Source.shard_list", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.shard_list", "dispname": "-"}, {"name": "atdata._sources.S3Source.shard_list", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.shard_list", "dispname": "atdata.S3Source.shard_list"}, {"name": "atdata.S3Source.shards", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.shards", "dispname": "-"}, {"name": "atdata._sources.S3Source.shards", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.shards", "dispname": "atdata.S3Source.shards"}, {"name": "atdata.S3Source", "domain": "py", "role": "class", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source", "dispname": "-"}, {"name": "atdata._sources.S3Source", "domain": "py", "role": "class", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source", "dispname": "atdata.S3Source"}, {"name": "atdata.local.Index.add_entry", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.add_entry", "dispname": "-"}, {"name": "atdata.local.Index.all_entries", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.all_entries", "dispname": "-"}, {"name": "atdata.local.Index.clear_stubs", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.clear_stubs", "dispname": "-"}, {"name": "atdata.local.Index.data_store", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.data_store", "dispname": "-"}, {"name": "atdata.local.Index.datasets", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.datasets", "dispname": "-"}, {"name": "atdata.local.Index.decode_schema", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.decode_schema", "dispname": "-"}, {"name": "atdata.local.Index.decode_schema_as", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.decode_schema_as", "dispname": "-"}, {"name": "atdata.local.Index.entries", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.entries", "dispname": "-"}, {"name": "atdata.local.Index.get_dataset", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.get_dataset", "dispname": "-"}, {"name": "atdata.local.Index.get_entry", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.get_entry", "dispname": "-"}, {"name": "atdata.local.Index.get_entry_by_name", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.get_entry_by_name", "dispname": "-"}, {"name": "atdata.local.Index.get_import_path", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.get_import_path", "dispname": "-"}, {"name": "atdata.local.Index.get_schema", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.get_schema", "dispname": "-"}, {"name": "atdata.local.Index.get_schema_record", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.get_schema_record", "dispname": "-"}, {"name": "atdata.local.Index.insert_dataset", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.insert_dataset", "dispname": "-"}, {"name": "atdata.local.Index.list_datasets", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.list_datasets", "dispname": "-"}, {"name": "atdata.local.Index.list_entries", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.list_entries", "dispname": "-"}, {"name": "atdata.local.Index.list_schemas", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.list_schemas", "dispname": "-"}, {"name": "atdata.local.Index.load_schema", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.load_schema", "dispname": "-"}, {"name": "atdata.local.Index.publish_schema", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.publish_schema", "dispname": "-"}, {"name": "atdata.local.Index.schemas", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.schemas", "dispname": "-"}, {"name": "atdata.local.Index.stub_dir", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.stub_dir", "dispname": "-"}, {"name": "atdata.local.Index.types", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.types", "dispname": "-"}, {"name": "atdata.local.Index", "domain": "py", "role": "class", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index", "dispname": "-"}, {"name": "atdata.local.LocalDatasetEntry.cid", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.LocalDatasetEntry.html#atdata.local.LocalDatasetEntry.cid", "dispname": "-"}, {"name": "atdata.local.LocalDatasetEntry.data_urls", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.LocalDatasetEntry.html#atdata.local.LocalDatasetEntry.data_urls", "dispname": "-"}, {"name": "atdata.local.LocalDatasetEntry.from_redis", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.LocalDatasetEntry.html#atdata.local.LocalDatasetEntry.from_redis", "dispname": "-"}, {"name": "atdata.local.LocalDatasetEntry.metadata", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.LocalDatasetEntry.html#atdata.local.LocalDatasetEntry.metadata", "dispname": "-"}, {"name": "atdata.local.LocalDatasetEntry.name", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.LocalDatasetEntry.html#atdata.local.LocalDatasetEntry.name", "dispname": "-"}, {"name": "atdata.local.LocalDatasetEntry.sample_kind", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.LocalDatasetEntry.html#atdata.local.LocalDatasetEntry.sample_kind", "dispname": "-"}, {"name": "atdata.local.LocalDatasetEntry.schema_ref", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.LocalDatasetEntry.html#atdata.local.LocalDatasetEntry.schema_ref", "dispname": "-"}, {"name": "atdata.local.LocalDatasetEntry.wds_url", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.LocalDatasetEntry.html#atdata.local.LocalDatasetEntry.wds_url", "dispname": "-"}, {"name": "atdata.local.LocalDatasetEntry.write_to", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.LocalDatasetEntry.html#atdata.local.LocalDatasetEntry.write_to", "dispname": "-"}, {"name": "atdata.local.LocalDatasetEntry", "domain": "py", "role": "class", "priority": "1", "uri": "api/local.LocalDatasetEntry.html#atdata.local.LocalDatasetEntry", "dispname": "-"}, {"name": "atdata.local.S3DataStore.read_url", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.S3DataStore.html#atdata.local.S3DataStore.read_url", "dispname": "-"}, {"name": "atdata.local.S3DataStore.supports_streaming", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.S3DataStore.html#atdata.local.S3DataStore.supports_streaming", "dispname": "-"}, {"name": "atdata.local.S3DataStore.write_shards", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.S3DataStore.html#atdata.local.S3DataStore.write_shards", "dispname": "-"}, {"name": "atdata.local.S3DataStore", "domain": "py", "role": "class", "priority": "1", "uri": "api/local.S3DataStore.html#atdata.local.S3DataStore", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereClient.create_record", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.create_record", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.create_record", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.create_record", "dispname": "atdata.atmosphere.AtmosphereClient.create_record"}, {"name": "atdata.atmosphere.AtmosphereClient.delete_record", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.delete_record", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.delete_record", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.delete_record", "dispname": "atdata.atmosphere.AtmosphereClient.delete_record"}, {"name": "atdata.atmosphere.AtmosphereClient.did", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.did", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.did", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.did", "dispname": "atdata.atmosphere.AtmosphereClient.did"}, {"name": "atdata.atmosphere.AtmosphereClient.export_session", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.export_session", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.export_session", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.export_session", "dispname": "atdata.atmosphere.AtmosphereClient.export_session"}, {"name": "atdata.atmosphere.AtmosphereClient.get_blob", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.get_blob", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.get_blob", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.get_blob", "dispname": "atdata.atmosphere.AtmosphereClient.get_blob"}, {"name": "atdata.atmosphere.AtmosphereClient.get_blob_url", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.get_blob_url", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.get_blob_url", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.get_blob_url", "dispname": "atdata.atmosphere.AtmosphereClient.get_blob_url"}, {"name": "atdata.atmosphere.AtmosphereClient.get_record", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.get_record", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.get_record", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.get_record", "dispname": "atdata.atmosphere.AtmosphereClient.get_record"}, {"name": "atdata.atmosphere.AtmosphereClient.handle", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.handle", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.handle", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.handle", "dispname": "atdata.atmosphere.AtmosphereClient.handle"}, {"name": "atdata.atmosphere.AtmosphereClient.is_authenticated", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.is_authenticated", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.is_authenticated", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.is_authenticated", "dispname": "atdata.atmosphere.AtmosphereClient.is_authenticated"}, {"name": "atdata.atmosphere.AtmosphereClient.list_datasets", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.list_datasets", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.list_datasets", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.list_datasets", "dispname": "atdata.atmosphere.AtmosphereClient.list_datasets"}, {"name": "atdata.atmosphere.AtmosphereClient.list_lenses", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.list_lenses", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.list_lenses", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.list_lenses", "dispname": "atdata.atmosphere.AtmosphereClient.list_lenses"}, {"name": "atdata.atmosphere.AtmosphereClient.list_records", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.list_records", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.list_records", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.list_records", "dispname": "atdata.atmosphere.AtmosphereClient.list_records"}, {"name": "atdata.atmosphere.AtmosphereClient.list_schemas", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.list_schemas", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.list_schemas", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.list_schemas", "dispname": "atdata.atmosphere.AtmosphereClient.list_schemas"}, {"name": "atdata.atmosphere.AtmosphereClient.login", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.login", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.login", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.login", "dispname": "atdata.atmosphere.AtmosphereClient.login"}, {"name": "atdata.atmosphere.AtmosphereClient.login_with_session", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.login_with_session", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.login_with_session", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.login_with_session", "dispname": "atdata.atmosphere.AtmosphereClient.login_with_session"}, {"name": "atdata.atmosphere.AtmosphereClient.put_record", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.put_record", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.put_record", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.put_record", "dispname": "atdata.atmosphere.AtmosphereClient.put_record"}, {"name": "atdata.atmosphere.AtmosphereClient.upload_blob", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.upload_blob", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.upload_blob", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.upload_blob", "dispname": "atdata.atmosphere.AtmosphereClient.upload_blob"}, {"name": "atdata.atmosphere.AtmosphereClient", "domain": "py", "role": "class", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient", "domain": "py", "role": "class", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient", "dispname": "atdata.atmosphere.AtmosphereClient"}, {"name": "atdata.atmosphere.AtmosphereIndex.datasets", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereIndex.html#atdata.atmosphere.AtmosphereIndex.datasets", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndex.decode_schema", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereIndex.html#atdata.atmosphere.AtmosphereIndex.decode_schema", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndex.get_dataset", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereIndex.html#atdata.atmosphere.AtmosphereIndex.get_dataset", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndex.get_schema", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereIndex.html#atdata.atmosphere.AtmosphereIndex.get_schema", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndex.insert_dataset", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereIndex.html#atdata.atmosphere.AtmosphereIndex.insert_dataset", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndex.list_datasets", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereIndex.html#atdata.atmosphere.AtmosphereIndex.list_datasets", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndex.list_schemas", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereIndex.html#atdata.atmosphere.AtmosphereIndex.list_schemas", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndex.publish_schema", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereIndex.html#atdata.atmosphere.AtmosphereIndex.publish_schema", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndex.schemas", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereIndex.html#atdata.atmosphere.AtmosphereIndex.schemas", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndex", "domain": "py", "role": "class", "priority": "1", "uri": "api/AtmosphereIndex.html#atdata.atmosphere.AtmosphereIndex", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndexEntry.data_urls", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereIndexEntry.html#atdata.atmosphere.AtmosphereIndexEntry.data_urls", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndexEntry.metadata", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereIndexEntry.html#atdata.atmosphere.AtmosphereIndexEntry.metadata", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndexEntry.name", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereIndexEntry.html#atdata.atmosphere.AtmosphereIndexEntry.name", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndexEntry.schema_ref", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereIndexEntry.html#atdata.atmosphere.AtmosphereIndexEntry.schema_ref", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndexEntry.uri", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereIndexEntry.html#atdata.atmosphere.AtmosphereIndexEntry.uri", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndexEntry", "domain": "py", "role": "class", "priority": "1", "uri": "api/AtmosphereIndexEntry.html#atdata.atmosphere.AtmosphereIndexEntry", "dispname": "-"}, {"name": "atdata.atmosphere.SchemaPublisher.publish", "domain": "py", "role": "function", "priority": "1", "uri": "api/SchemaPublisher.html#atdata.atmosphere.SchemaPublisher.publish", "dispname": "-"}, {"name": "atdata.atmosphere.schema.SchemaPublisher.publish", "domain": "py", "role": "function", "priority": "1", "uri": "api/SchemaPublisher.html#atdata.atmosphere.SchemaPublisher.publish", "dispname": "atdata.atmosphere.SchemaPublisher.publish"}, {"name": "atdata.atmosphere.SchemaPublisher", "domain": "py", "role": "class", "priority": "1", "uri": "api/SchemaPublisher.html#atdata.atmosphere.SchemaPublisher", "dispname": "-"}, {"name": "atdata.atmosphere.schema.SchemaPublisher", "domain": "py", "role": "class", "priority": "1", "uri": "api/SchemaPublisher.html#atdata.atmosphere.SchemaPublisher", "dispname": "atdata.atmosphere.SchemaPublisher"}, {"name": "atdata.atmosphere.SchemaLoader.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/SchemaLoader.html#atdata.atmosphere.SchemaLoader.get", "dispname": "-"}, {"name": "atdata.atmosphere.schema.SchemaLoader.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/SchemaLoader.html#atdata.atmosphere.SchemaLoader.get", "dispname": "atdata.atmosphere.SchemaLoader.get"}, {"name": "atdata.atmosphere.SchemaLoader.list_all", "domain": "py", "role": "function", "priority": "1", "uri": "api/SchemaLoader.html#atdata.atmosphere.SchemaLoader.list_all", "dispname": "-"}, {"name": "atdata.atmosphere.schema.SchemaLoader.list_all", "domain": "py", "role": "function", "priority": "1", "uri": "api/SchemaLoader.html#atdata.atmosphere.SchemaLoader.list_all", "dispname": "atdata.atmosphere.SchemaLoader.list_all"}, {"name": "atdata.atmosphere.SchemaLoader", "domain": "py", "role": "class", "priority": "1", "uri": "api/SchemaLoader.html#atdata.atmosphere.SchemaLoader", "dispname": "-"}, {"name": "atdata.atmosphere.schema.SchemaLoader", "domain": "py", "role": "class", "priority": "1", "uri": "api/SchemaLoader.html#atdata.atmosphere.SchemaLoader", "dispname": "atdata.atmosphere.SchemaLoader"}, {"name": "atdata.atmosphere.DatasetPublisher.publish", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetPublisher.html#atdata.atmosphere.DatasetPublisher.publish", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetPublisher.publish", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetPublisher.html#atdata.atmosphere.DatasetPublisher.publish", "dispname": "atdata.atmosphere.DatasetPublisher.publish"}, {"name": "atdata.atmosphere.DatasetPublisher.publish_with_blobs", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetPublisher.html#atdata.atmosphere.DatasetPublisher.publish_with_blobs", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetPublisher.publish_with_blobs", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetPublisher.html#atdata.atmosphere.DatasetPublisher.publish_with_blobs", "dispname": "atdata.atmosphere.DatasetPublisher.publish_with_blobs"}, {"name": "atdata.atmosphere.DatasetPublisher.publish_with_urls", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetPublisher.html#atdata.atmosphere.DatasetPublisher.publish_with_urls", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetPublisher.publish_with_urls", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetPublisher.html#atdata.atmosphere.DatasetPublisher.publish_with_urls", "dispname": "atdata.atmosphere.DatasetPublisher.publish_with_urls"}, {"name": "atdata.atmosphere.DatasetPublisher", "domain": "py", "role": "class", "priority": "1", "uri": "api/DatasetPublisher.html#atdata.atmosphere.DatasetPublisher", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetPublisher", "domain": "py", "role": "class", "priority": "1", "uri": "api/DatasetPublisher.html#atdata.atmosphere.DatasetPublisher", "dispname": "atdata.atmosphere.DatasetPublisher"}, {"name": "atdata.atmosphere.DatasetLoader.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetLoader.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get", "dispname": "atdata.atmosphere.DatasetLoader.get"}, {"name": "atdata.atmosphere.DatasetLoader.get_blob_urls", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get_blob_urls", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetLoader.get_blob_urls", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get_blob_urls", "dispname": "atdata.atmosphere.DatasetLoader.get_blob_urls"}, {"name": "atdata.atmosphere.DatasetLoader.get_blobs", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get_blobs", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetLoader.get_blobs", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get_blobs", "dispname": "atdata.atmosphere.DatasetLoader.get_blobs"}, {"name": "atdata.atmosphere.DatasetLoader.get_metadata", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get_metadata", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetLoader.get_metadata", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get_metadata", "dispname": "atdata.atmosphere.DatasetLoader.get_metadata"}, {"name": "atdata.atmosphere.DatasetLoader.get_storage_type", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get_storage_type", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetLoader.get_storage_type", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get_storage_type", "dispname": "atdata.atmosphere.DatasetLoader.get_storage_type"}, {"name": "atdata.atmosphere.DatasetLoader.get_urls", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get_urls", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetLoader.get_urls", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get_urls", "dispname": "atdata.atmosphere.DatasetLoader.get_urls"}, {"name": "atdata.atmosphere.DatasetLoader.list_all", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.list_all", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetLoader.list_all", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.list_all", "dispname": "atdata.atmosphere.DatasetLoader.list_all"}, {"name": "atdata.atmosphere.DatasetLoader.to_dataset", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.to_dataset", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetLoader.to_dataset", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.to_dataset", "dispname": "atdata.atmosphere.DatasetLoader.to_dataset"}, {"name": "atdata.atmosphere.DatasetLoader", "domain": "py", "role": "class", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetLoader", "domain": "py", "role": "class", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader", "dispname": "atdata.atmosphere.DatasetLoader"}, {"name": "atdata.atmosphere.LensPublisher.publish", "domain": "py", "role": "function", "priority": "1", "uri": "api/LensPublisher.html#atdata.atmosphere.LensPublisher.publish", "dispname": "-"}, {"name": "atdata.atmosphere.lens.LensPublisher.publish", "domain": "py", "role": "function", "priority": "1", "uri": "api/LensPublisher.html#atdata.atmosphere.LensPublisher.publish", "dispname": "atdata.atmosphere.LensPublisher.publish"}, {"name": "atdata.atmosphere.LensPublisher.publish_from_lens", "domain": "py", "role": "function", "priority": "1", "uri": "api/LensPublisher.html#atdata.atmosphere.LensPublisher.publish_from_lens", "dispname": "-"}, {"name": "atdata.atmosphere.lens.LensPublisher.publish_from_lens", "domain": "py", "role": "function", "priority": "1", "uri": "api/LensPublisher.html#atdata.atmosphere.LensPublisher.publish_from_lens", "dispname": "atdata.atmosphere.LensPublisher.publish_from_lens"}, {"name": "atdata.atmosphere.LensPublisher", "domain": "py", "role": "class", "priority": "1", "uri": "api/LensPublisher.html#atdata.atmosphere.LensPublisher", "dispname": "-"}, {"name": "atdata.atmosphere.lens.LensPublisher", "domain": "py", "role": "class", "priority": "1", "uri": "api/LensPublisher.html#atdata.atmosphere.LensPublisher", "dispname": "atdata.atmosphere.LensPublisher"}, {"name": "atdata.atmosphere.LensLoader.find_by_schemas", "domain": "py", "role": "function", "priority": "1", "uri": "api/LensLoader.html#atdata.atmosphere.LensLoader.find_by_schemas", "dispname": "-"}, {"name": "atdata.atmosphere.lens.LensLoader.find_by_schemas", "domain": "py", "role": "function", "priority": "1", "uri": "api/LensLoader.html#atdata.atmosphere.LensLoader.find_by_schemas", "dispname": "atdata.atmosphere.LensLoader.find_by_schemas"}, {"name": "atdata.atmosphere.LensLoader.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/LensLoader.html#atdata.atmosphere.LensLoader.get", "dispname": "-"}, {"name": "atdata.atmosphere.lens.LensLoader.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/LensLoader.html#atdata.atmosphere.LensLoader.get", "dispname": "atdata.atmosphere.LensLoader.get"}, {"name": "atdata.atmosphere.LensLoader.list_all", "domain": "py", "role": "function", "priority": "1", "uri": "api/LensLoader.html#atdata.atmosphere.LensLoader.list_all", "dispname": "-"}, {"name": "atdata.atmosphere.lens.LensLoader.list_all", "domain": "py", "role": "function", "priority": "1", "uri": "api/LensLoader.html#atdata.atmosphere.LensLoader.list_all", "dispname": "atdata.atmosphere.LensLoader.list_all"}, {"name": "atdata.atmosphere.LensLoader", "domain": "py", "role": "class", "priority": "1", "uri": "api/LensLoader.html#atdata.atmosphere.LensLoader", "dispname": "-"}, {"name": "atdata.atmosphere.lens.LensLoader", "domain": "py", "role": "class", "priority": "1", "uri": "api/LensLoader.html#atdata.atmosphere.LensLoader", "dispname": "atdata.atmosphere.LensLoader"}, {"name": "atdata.atmosphere.AtUri.authority", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtUri.html#atdata.atmosphere.AtUri.authority", "dispname": "-"}, {"name": "atdata.atmosphere._types.AtUri.authority", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtUri.html#atdata.atmosphere.AtUri.authority", "dispname": "atdata.atmosphere.AtUri.authority"}, {"name": "atdata.atmosphere.AtUri.collection", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtUri.html#atdata.atmosphere.AtUri.collection", "dispname": "-"}, {"name": "atdata.atmosphere._types.AtUri.collection", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtUri.html#atdata.atmosphere.AtUri.collection", "dispname": "atdata.atmosphere.AtUri.collection"}, {"name": "atdata.atmosphere.AtUri.parse", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtUri.html#atdata.atmosphere.AtUri.parse", "dispname": "-"}, {"name": "atdata.atmosphere._types.AtUri.parse", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtUri.html#atdata.atmosphere.AtUri.parse", "dispname": "atdata.atmosphere.AtUri.parse"}, {"name": "atdata.atmosphere.AtUri.rkey", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtUri.html#atdata.atmosphere.AtUri.rkey", "dispname": "-"}, {"name": "atdata.atmosphere._types.AtUri.rkey", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtUri.html#atdata.atmosphere.AtUri.rkey", "dispname": "atdata.atmosphere.AtUri.rkey"}, {"name": "atdata.atmosphere.AtUri", "domain": "py", "role": "class", "priority": "1", "uri": "api/AtUri.html#atdata.atmosphere.AtUri", "dispname": "-"}, {"name": "atdata.atmosphere._types.AtUri", "domain": "py", "role": "class", "priority": "1", "uri": "api/AtUri.html#atdata.atmosphere.AtUri", "dispname": "atdata.atmosphere.AtUri"}, {"name": "atdata.promote.promote_to_atmosphere", "domain": "py", "role": "function", "priority": "1", "uri": "api/promote_to_atmosphere.html#atdata.promote.promote_to_atmosphere", "dispname": "-"}]}
1
+
{"project": "atdata", "version": "0.0.9999", "count": 322, "items": [{"name": "atdata.packable", "domain": "py", "role": "function", "priority": "1", "uri": "api/packable.html#atdata.packable", "dispname": "-"}, {"name": "atdata.dataset.packable", "domain": "py", "role": "function", "priority": "1", "uri": "api/packable.html#atdata.packable", "dispname": "atdata.packable"}, {"name": "atdata.PackableSample.as_wds", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/PackableSample.html#atdata.PackableSample.as_wds", "dispname": "-"}, {"name": "atdata.dataset.PackableSample.as_wds", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/PackableSample.html#atdata.PackableSample.as_wds", "dispname": "atdata.PackableSample.as_wds"}, {"name": "atdata.PackableSample.from_bytes", "domain": "py", "role": "function", "priority": "1", "uri": "api/PackableSample.html#atdata.PackableSample.from_bytes", "dispname": "-"}, {"name": "atdata.dataset.PackableSample.from_bytes", "domain": "py", "role": "function", "priority": "1", "uri": "api/PackableSample.html#atdata.PackableSample.from_bytes", "dispname": "atdata.PackableSample.from_bytes"}, {"name": "atdata.PackableSample.from_data", "domain": "py", "role": "function", "priority": "1", "uri": "api/PackableSample.html#atdata.PackableSample.from_data", "dispname": "-"}, {"name": "atdata.dataset.PackableSample.from_data", "domain": "py", "role": "function", "priority": "1", "uri": "api/PackableSample.html#atdata.PackableSample.from_data", "dispname": "atdata.PackableSample.from_data"}, {"name": "atdata.PackableSample.packed", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/PackableSample.html#atdata.PackableSample.packed", "dispname": "-"}, {"name": "atdata.dataset.PackableSample.packed", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/PackableSample.html#atdata.PackableSample.packed", "dispname": "atdata.PackableSample.packed"}, {"name": "atdata.PackableSample", "domain": "py", "role": "class", "priority": "1", "uri": "api/PackableSample.html#atdata.PackableSample", "dispname": "-"}, {"name": "atdata.dataset.PackableSample", "domain": "py", "role": "class", "priority": "1", "uri": "api/PackableSample.html#atdata.PackableSample", "dispname": "atdata.PackableSample"}, {"name": "atdata.DictSample.as_wds", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.as_wds", "dispname": "-"}, {"name": "atdata.dataset.DictSample.as_wds", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.as_wds", "dispname": "atdata.DictSample.as_wds"}, {"name": "atdata.DictSample.from_bytes", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.from_bytes", "dispname": "-"}, {"name": "atdata.dataset.DictSample.from_bytes", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.from_bytes", "dispname": "atdata.DictSample.from_bytes"}, {"name": "atdata.DictSample.from_data", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.from_data", "dispname": "-"}, {"name": "atdata.dataset.DictSample.from_data", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.from_data", "dispname": "atdata.DictSample.from_data"}, {"name": "atdata.DictSample.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.get", "dispname": "-"}, {"name": "atdata.dataset.DictSample.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.get", "dispname": "atdata.DictSample.get"}, {"name": "atdata.DictSample.items", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.items", "dispname": "-"}, {"name": "atdata.dataset.DictSample.items", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.items", "dispname": "atdata.DictSample.items"}, {"name": "atdata.DictSample.keys", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.keys", "dispname": "-"}, {"name": "atdata.dataset.DictSample.keys", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.keys", "dispname": "atdata.DictSample.keys"}, {"name": "atdata.DictSample.packed", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.packed", "dispname": "-"}, {"name": "atdata.dataset.DictSample.packed", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.packed", "dispname": "atdata.DictSample.packed"}, {"name": "atdata.DictSample.to_dict", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.to_dict", "dispname": "-"}, {"name": "atdata.dataset.DictSample.to_dict", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.to_dict", "dispname": "atdata.DictSample.to_dict"}, {"name": "atdata.DictSample.values", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.values", "dispname": "-"}, {"name": "atdata.dataset.DictSample.values", "domain": "py", "role": "function", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample.values", "dispname": "atdata.DictSample.values"}, {"name": "atdata.DictSample", "domain": "py", "role": "class", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample", "dispname": "-"}, {"name": "atdata.dataset.DictSample", "domain": "py", "role": "class", "priority": "1", "uri": "api/DictSample.html#atdata.DictSample", "dispname": "atdata.DictSample"}, {"name": "atdata.Dataset.as_type", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.as_type", "dispname": "-"}, {"name": "atdata.dataset.Dataset.as_type", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.as_type", "dispname": "atdata.Dataset.as_type"}, {"name": "atdata.Dataset.batch_type", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.batch_type", "dispname": "-"}, {"name": "atdata.dataset.Dataset.batch_type", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.batch_type", "dispname": "atdata.Dataset.batch_type"}, {"name": "atdata.Dataset.list_shards", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.list_shards", "dispname": "-"}, {"name": "atdata.dataset.Dataset.list_shards", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.list_shards", "dispname": "atdata.Dataset.list_shards"}, {"name": "atdata.Dataset.metadata", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.metadata", "dispname": "-"}, {"name": "atdata.dataset.Dataset.metadata", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.metadata", "dispname": "atdata.Dataset.metadata"}, {"name": "atdata.Dataset.metadata_url", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.metadata_url", "dispname": "-"}, {"name": "atdata.dataset.Dataset.metadata_url", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.metadata_url", "dispname": "atdata.Dataset.metadata_url"}, {"name": "atdata.Dataset.ordered", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.ordered", "dispname": "-"}, {"name": "atdata.dataset.Dataset.ordered", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.ordered", "dispname": "atdata.Dataset.ordered"}, {"name": "atdata.Dataset.sample_type", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.sample_type", "dispname": "-"}, {"name": "atdata.dataset.Dataset.sample_type", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.sample_type", "dispname": "atdata.Dataset.sample_type"}, {"name": "atdata.Dataset.shard_list", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.shard_list", "dispname": "-"}, {"name": "atdata.dataset.Dataset.shard_list", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.shard_list", "dispname": "atdata.Dataset.shard_list"}, {"name": "atdata.Dataset.shuffled", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.shuffled", "dispname": "-"}, {"name": "atdata.dataset.Dataset.shuffled", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.shuffled", "dispname": "atdata.Dataset.shuffled"}, {"name": "atdata.Dataset.source", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.source", "dispname": "-"}, {"name": "atdata.dataset.Dataset.source", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.source", "dispname": "atdata.Dataset.source"}, {"name": "atdata.Dataset.to_parquet", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.to_parquet", "dispname": "-"}, {"name": "atdata.dataset.Dataset.to_parquet", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.to_parquet", "dispname": "atdata.Dataset.to_parquet"}, {"name": "atdata.Dataset.wrap", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.wrap", "dispname": "-"}, {"name": "atdata.dataset.Dataset.wrap", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.wrap", "dispname": "atdata.Dataset.wrap"}, {"name": "atdata.Dataset.wrap_batch", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.wrap_batch", "dispname": "-"}, {"name": "atdata.dataset.Dataset.wrap_batch", "domain": "py", "role": "function", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset.wrap_batch", "dispname": "atdata.Dataset.wrap_batch"}, {"name": "atdata.Dataset", "domain": "py", "role": "class", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset", "dispname": "-"}, {"name": "atdata.dataset.Dataset", "domain": "py", "role": "class", "priority": "1", "uri": "api/Dataset.html#atdata.Dataset", "dispname": "atdata.Dataset"}, {"name": "atdata.SampleBatch.sample_type", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/SampleBatch.html#atdata.SampleBatch.sample_type", "dispname": "-"}, {"name": "atdata.dataset.SampleBatch.sample_type", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/SampleBatch.html#atdata.SampleBatch.sample_type", "dispname": "atdata.SampleBatch.sample_type"}, {"name": "atdata.SampleBatch", "domain": "py", "role": "class", "priority": "1", "uri": "api/SampleBatch.html#atdata.SampleBatch", "dispname": "-"}, {"name": "atdata.dataset.SampleBatch", "domain": "py", "role": "class", "priority": "1", "uri": "api/SampleBatch.html#atdata.SampleBatch", "dispname": "atdata.SampleBatch"}, {"name": "atdata.Lens.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/Lens.html#atdata.Lens.get", "dispname": "-"}, {"name": "atdata.lens.Lens.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/Lens.html#atdata.Lens.get", "dispname": "atdata.Lens.get"}, {"name": "atdata.Lens.put", "domain": "py", "role": "function", "priority": "1", "uri": "api/Lens.html#atdata.Lens.put", "dispname": "-"}, {"name": "atdata.lens.Lens.put", "domain": "py", "role": "function", "priority": "1", "uri": "api/Lens.html#atdata.Lens.put", "dispname": "atdata.Lens.put"}, {"name": "atdata.Lens.putter", "domain": "py", "role": "function", "priority": "1", "uri": "api/Lens.html#atdata.Lens.putter", "dispname": "-"}, {"name": "atdata.lens.Lens.putter", "domain": "py", "role": "function", "priority": "1", "uri": "api/Lens.html#atdata.Lens.putter", "dispname": "atdata.Lens.putter"}, {"name": "atdata.Lens", "domain": "py", "role": "class", "priority": "1", "uri": "api/Lens.html#atdata.Lens", "dispname": "-"}, {"name": "atdata.lens.Lens", "domain": "py", "role": "class", "priority": "1", "uri": "api/Lens.html#atdata.Lens", "dispname": "atdata.Lens"}, {"name": "atdata.lens.Lens.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/lens.html#atdata.lens.Lens.get", "dispname": "-"}, {"name": "atdata.lens.Lens.put", "domain": "py", "role": "function", "priority": "1", "uri": "api/lens.html#atdata.lens.Lens.put", "dispname": "-"}, {"name": "atdata.lens.Lens.putter", "domain": "py", "role": "function", "priority": "1", "uri": "api/lens.html#atdata.lens.Lens.putter", "dispname": "-"}, {"name": "atdata.lens.Lens", "domain": "py", "role": "class", "priority": "1", "uri": "api/lens.html#atdata.lens.Lens", "dispname": "-"}, {"name": "atdata.lens.LensNetwork.register", "domain": "py", "role": "function", "priority": "1", "uri": "api/lens.html#atdata.lens.LensNetwork.register", "dispname": "-"}, {"name": "atdata.lens.LensNetwork.transform", "domain": "py", "role": "function", "priority": "1", "uri": "api/lens.html#atdata.lens.LensNetwork.transform", "dispname": "-"}, {"name": "atdata.lens.LensNetwork", "domain": "py", "role": "class", "priority": "1", "uri": "api/lens.html#atdata.lens.LensNetwork", "dispname": "-"}, {"name": "atdata.lens.lens", "domain": "py", "role": "function", "priority": "1", "uri": "api/lens.html#atdata.lens.lens", "dispname": "-"}, {"name": "atdata.lens", "domain": "py", "role": "module", "priority": "1", "uri": "api/lens.html#atdata.lens", "dispname": "-"}, {"name": "atdata.load_dataset", "domain": "py", "role": "function", "priority": "1", "uri": "api/load_dataset.html#atdata.load_dataset", "dispname": "-"}, {"name": "atdata._hf_api.load_dataset", "domain": "py", "role": "function", "priority": "1", "uri": "api/load_dataset.html#atdata.load_dataset", "dispname": "atdata.load_dataset"}, {"name": "atdata.DatasetDict.num_shards", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DatasetDict.html#atdata.DatasetDict.num_shards", "dispname": "-"}, {"name": "atdata._hf_api.DatasetDict.num_shards", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DatasetDict.html#atdata.DatasetDict.num_shards", "dispname": "atdata.DatasetDict.num_shards"}, {"name": "atdata.DatasetDict.sample_type", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DatasetDict.html#atdata.DatasetDict.sample_type", "dispname": "-"}, {"name": "atdata._hf_api.DatasetDict.sample_type", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DatasetDict.html#atdata.DatasetDict.sample_type", "dispname": "atdata.DatasetDict.sample_type"}, {"name": "atdata.DatasetDict.streaming", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DatasetDict.html#atdata.DatasetDict.streaming", "dispname": "-"}, {"name": "atdata._hf_api.DatasetDict.streaming", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DatasetDict.html#atdata.DatasetDict.streaming", "dispname": "atdata.DatasetDict.streaming"}, {"name": "atdata.DatasetDict", "domain": "py", "role": "class", "priority": "1", "uri": "api/DatasetDict.html#atdata.DatasetDict", "dispname": "-"}, {"name": "atdata._hf_api.DatasetDict", "domain": "py", "role": "class", "priority": "1", "uri": "api/DatasetDict.html#atdata.DatasetDict", "dispname": "atdata.DatasetDict"}, {"name": "atdata.Packable.as_wds", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Packable-protocol.html#atdata.Packable.as_wds", "dispname": "-"}, {"name": "atdata._protocols.Packable.as_wds", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Packable-protocol.html#atdata.Packable.as_wds", "dispname": "atdata.Packable.as_wds"}, {"name": "atdata.Packable.from_bytes", "domain": "py", "role": "function", "priority": "1", "uri": "api/Packable-protocol.html#atdata.Packable.from_bytes", "dispname": "-"}, {"name": "atdata._protocols.Packable.from_bytes", "domain": "py", "role": "function", "priority": "1", "uri": "api/Packable-protocol.html#atdata.Packable.from_bytes", "dispname": "atdata.Packable.from_bytes"}, {"name": "atdata.Packable.from_data", "domain": "py", "role": "function", "priority": "1", "uri": "api/Packable-protocol.html#atdata.Packable.from_data", "dispname": "-"}, {"name": "atdata._protocols.Packable.from_data", "domain": "py", "role": "function", "priority": "1", "uri": "api/Packable-protocol.html#atdata.Packable.from_data", "dispname": "atdata.Packable.from_data"}, {"name": "atdata.Packable.packed", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Packable-protocol.html#atdata.Packable.packed", "dispname": "-"}, {"name": "atdata._protocols.Packable.packed", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/Packable-protocol.html#atdata.Packable.packed", "dispname": "atdata.Packable.packed"}, {"name": "atdata.Packable", "domain": "py", "role": "class", "priority": "1", "uri": "api/Packable-protocol.html#atdata.Packable", "dispname": "-"}, {"name": "atdata._protocols.Packable", "domain": "py", "role": "class", "priority": "1", "uri": "api/Packable-protocol.html#atdata.Packable", "dispname": "atdata.Packable"}, {"name": "atdata.IndexEntry.data_urls", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/IndexEntry.html#atdata.IndexEntry.data_urls", "dispname": "-"}, {"name": "atdata._protocols.IndexEntry.data_urls", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/IndexEntry.html#atdata.IndexEntry.data_urls", "dispname": "atdata.IndexEntry.data_urls"}, {"name": "atdata.IndexEntry.metadata", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/IndexEntry.html#atdata.IndexEntry.metadata", "dispname": "-"}, {"name": "atdata._protocols.IndexEntry.metadata", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/IndexEntry.html#atdata.IndexEntry.metadata", "dispname": "atdata.IndexEntry.metadata"}, {"name": "atdata.IndexEntry.name", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/IndexEntry.html#atdata.IndexEntry.name", "dispname": "-"}, {"name": "atdata._protocols.IndexEntry.name", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/IndexEntry.html#atdata.IndexEntry.name", "dispname": "atdata.IndexEntry.name"}, {"name": "atdata.IndexEntry.schema_ref", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/IndexEntry.html#atdata.IndexEntry.schema_ref", "dispname": "-"}, {"name": "atdata._protocols.IndexEntry.schema_ref", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/IndexEntry.html#atdata.IndexEntry.schema_ref", "dispname": "atdata.IndexEntry.schema_ref"}, {"name": "atdata.IndexEntry", "domain": "py", "role": "class", "priority": "1", "uri": "api/IndexEntry.html#atdata.IndexEntry", "dispname": "-"}, {"name": "atdata._protocols.IndexEntry", "domain": "py", "role": "class", "priority": "1", "uri": "api/IndexEntry.html#atdata.IndexEntry", "dispname": "atdata.IndexEntry"}, {"name": "atdata.AbstractIndex.datasets", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AbstractIndex.html#atdata.AbstractIndex.datasets", "dispname": "-"}, {"name": "atdata._protocols.AbstractIndex.datasets", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AbstractIndex.html#atdata.AbstractIndex.datasets", "dispname": "atdata.AbstractIndex.datasets"}, {"name": "atdata.AbstractIndex.decode_schema", "domain": "py", "role": "function", "priority": "1", "uri": "api/AbstractIndex.html#atdata.AbstractIndex.decode_schema", "dispname": "-"}, {"name": "atdata._protocols.AbstractIndex.decode_schema", "domain": "py", "role": "function", "priority": "1", "uri": "api/AbstractIndex.html#atdata.AbstractIndex.decode_schema", "dispname": "atdata.AbstractIndex.decode_schema"}, {"name": "atdata.AbstractIndex.get_dataset", "domain": "py", "role": "function", "priority": "1", "uri": "api/AbstractIndex.html#atdata.AbstractIndex.get_dataset", "dispname": "-"}, {"name": "atdata._protocols.AbstractIndex.get_dataset", "domain": "py", "role": "function", "priority": "1", "uri": "api/AbstractIndex.html#atdata.AbstractIndex.get_dataset", "dispname": "atdata.AbstractIndex.get_dataset"}, {"name": "atdata.AbstractIndex.get_schema", "domain": "py", "role": "function", "priority": "1", "uri": "api/AbstractIndex.html#atdata.AbstractIndex.get_schema", "dispname": "-"}, {"name": "atdata._protocols.AbstractIndex.get_schema", "domain": "py", "role": "function", "priority": "1", "uri": "api/AbstractIndex.html#atdata.AbstractIndex.get_schema", "dispname": "atdata.AbstractIndex.get_schema"}, {"name": "atdata.AbstractIndex.insert_dataset", "domain": "py", "role": "function", "priority": "1", "uri": "api/AbstractIndex.html#atdata.AbstractIndex.insert_dataset", "dispname": "-"}, {"name": "atdata._protocols.AbstractIndex.insert_dataset", "domain": "py", "role": "function", "priority": "1", "uri": "api/AbstractIndex.html#atdata.AbstractIndex.insert_dataset", "dispname": "atdata.AbstractIndex.insert_dataset"}, {"name": "atdata.AbstractIndex.list_datasets", "domain": "py", "role": "function", "priority": "1", "uri": "api/AbstractIndex.html#atdata.AbstractIndex.list_datasets", "dispname": "-"}, {"name": "atdata._protocols.AbstractIndex.list_datasets", "domain": "py", "role": "function", "priority": "1", "uri": "api/AbstractIndex.html#atdata.AbstractIndex.list_datasets", "dispname": "atdata.AbstractIndex.list_datasets"}, {"name": "atdata.AbstractIndex.list_schemas", "domain": "py", "role": "function", "priority": "1", "uri": "api/AbstractIndex.html#atdata.AbstractIndex.list_schemas", "dispname": "-"}, {"name": "atdata._protocols.AbstractIndex.list_schemas", "domain": "py", "role": "function", "priority": "1", "uri": "api/AbstractIndex.html#atdata.AbstractIndex.list_schemas", "dispname": "atdata.AbstractIndex.list_schemas"}, {"name": "atdata.AbstractIndex.publish_schema", "domain": "py", "role": "function", "priority": "1", "uri": "api/AbstractIndex.html#atdata.AbstractIndex.publish_schema", "dispname": "-"}, {"name": "atdata._protocols.AbstractIndex.publish_schema", "domain": "py", "role": "function", "priority": "1", "uri": "api/AbstractIndex.html#atdata.AbstractIndex.publish_schema", "dispname": "atdata.AbstractIndex.publish_schema"}, {"name": "atdata.AbstractIndex.schemas", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AbstractIndex.html#atdata.AbstractIndex.schemas", "dispname": "-"}, {"name": "atdata._protocols.AbstractIndex.schemas", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AbstractIndex.html#atdata.AbstractIndex.schemas", "dispname": "atdata.AbstractIndex.schemas"}, {"name": "atdata.AbstractIndex", "domain": "py", "role": "class", "priority": "1", "uri": "api/AbstractIndex.html#atdata.AbstractIndex", "dispname": "-"}, {"name": "atdata._protocols.AbstractIndex", "domain": "py", "role": "class", "priority": "1", "uri": "api/AbstractIndex.html#atdata.AbstractIndex", "dispname": "atdata.AbstractIndex"}, {"name": "atdata.AbstractDataStore.read_url", "domain": "py", "role": "function", "priority": "1", "uri": "api/AbstractDataStore.html#atdata.AbstractDataStore.read_url", "dispname": "-"}, {"name": "atdata._protocols.AbstractDataStore.read_url", "domain": "py", "role": "function", "priority": "1", "uri": "api/AbstractDataStore.html#atdata.AbstractDataStore.read_url", "dispname": "atdata.AbstractDataStore.read_url"}, {"name": "atdata.AbstractDataStore.supports_streaming", "domain": "py", "role": "function", "priority": "1", "uri": "api/AbstractDataStore.html#atdata.AbstractDataStore.supports_streaming", "dispname": "-"}, {"name": "atdata._protocols.AbstractDataStore.supports_streaming", "domain": "py", "role": "function", "priority": "1", "uri": "api/AbstractDataStore.html#atdata.AbstractDataStore.supports_streaming", "dispname": "atdata.AbstractDataStore.supports_streaming"}, {"name": "atdata.AbstractDataStore.write_shards", "domain": "py", "role": "function", "priority": "1", "uri": "api/AbstractDataStore.html#atdata.AbstractDataStore.write_shards", "dispname": "-"}, {"name": "atdata._protocols.AbstractDataStore.write_shards", "domain": "py", "role": "function", "priority": "1", "uri": "api/AbstractDataStore.html#atdata.AbstractDataStore.write_shards", "dispname": "atdata.AbstractDataStore.write_shards"}, {"name": "atdata.AbstractDataStore", "domain": "py", "role": "class", "priority": "1", "uri": "api/AbstractDataStore.html#atdata.AbstractDataStore", "dispname": "-"}, {"name": "atdata._protocols.AbstractDataStore", "domain": "py", "role": "class", "priority": "1", "uri": "api/AbstractDataStore.html#atdata.AbstractDataStore", "dispname": "atdata.AbstractDataStore"}, {"name": "atdata.DataSource.list_shards", "domain": "py", "role": "function", "priority": "1", "uri": "api/DataSource.html#atdata.DataSource.list_shards", "dispname": "-"}, {"name": "atdata._protocols.DataSource.list_shards", "domain": "py", "role": "function", "priority": "1", "uri": "api/DataSource.html#atdata.DataSource.list_shards", "dispname": "atdata.DataSource.list_shards"}, {"name": "atdata.DataSource.open_shard", "domain": "py", "role": "function", "priority": "1", "uri": "api/DataSource.html#atdata.DataSource.open_shard", "dispname": "-"}, {"name": "atdata._protocols.DataSource.open_shard", "domain": "py", "role": "function", "priority": "1", "uri": "api/DataSource.html#atdata.DataSource.open_shard", "dispname": "atdata.DataSource.open_shard"}, {"name": "atdata.DataSource.shards", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DataSource.html#atdata.DataSource.shards", "dispname": "-"}, {"name": "atdata._protocols.DataSource.shards", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/DataSource.html#atdata.DataSource.shards", "dispname": "atdata.DataSource.shards"}, {"name": "atdata.DataSource", "domain": "py", "role": "class", "priority": "1", "uri": "api/DataSource.html#atdata.DataSource", "dispname": "-"}, {"name": "atdata._protocols.DataSource", "domain": "py", "role": "class", "priority": "1", "uri": "api/DataSource.html#atdata.DataSource", "dispname": "atdata.DataSource"}, {"name": "atdata.URLSource.list_shards", "domain": "py", "role": "function", "priority": "1", "uri": "api/URLSource.html#atdata.URLSource.list_shards", "dispname": "-"}, {"name": "atdata._sources.URLSource.list_shards", "domain": "py", "role": "function", "priority": "1", "uri": "api/URLSource.html#atdata.URLSource.list_shards", "dispname": "atdata.URLSource.list_shards"}, {"name": "atdata.URLSource.open_shard", "domain": "py", "role": "function", "priority": "1", "uri": "api/URLSource.html#atdata.URLSource.open_shard", "dispname": "-"}, {"name": "atdata._sources.URLSource.open_shard", "domain": "py", "role": "function", "priority": "1", "uri": "api/URLSource.html#atdata.URLSource.open_shard", "dispname": "atdata.URLSource.open_shard"}, {"name": "atdata.URLSource.shard_list", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/URLSource.html#atdata.URLSource.shard_list", "dispname": "-"}, {"name": "atdata._sources.URLSource.shard_list", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/URLSource.html#atdata.URLSource.shard_list", "dispname": "atdata.URLSource.shard_list"}, {"name": "atdata.URLSource.shards", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/URLSource.html#atdata.URLSource.shards", "dispname": "-"}, {"name": "atdata._sources.URLSource.shards", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/URLSource.html#atdata.URLSource.shards", "dispname": "atdata.URLSource.shards"}, {"name": "atdata.URLSource", "domain": "py", "role": "class", "priority": "1", "uri": "api/URLSource.html#atdata.URLSource", "dispname": "-"}, {"name": "atdata._sources.URLSource", "domain": "py", "role": "class", "priority": "1", "uri": "api/URLSource.html#atdata.URLSource", "dispname": "atdata.URLSource"}, {"name": "atdata.S3Source.from_credentials", "domain": "py", "role": "function", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.from_credentials", "dispname": "-"}, {"name": "atdata._sources.S3Source.from_credentials", "domain": "py", "role": "function", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.from_credentials", "dispname": "atdata.S3Source.from_credentials"}, {"name": "atdata.S3Source.from_urls", "domain": "py", "role": "function", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.from_urls", "dispname": "-"}, {"name": "atdata._sources.S3Source.from_urls", "domain": "py", "role": "function", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.from_urls", "dispname": "atdata.S3Source.from_urls"}, {"name": "atdata.S3Source.list_shards", "domain": "py", "role": "function", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.list_shards", "dispname": "-"}, {"name": "atdata._sources.S3Source.list_shards", "domain": "py", "role": "function", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.list_shards", "dispname": "atdata.S3Source.list_shards"}, {"name": "atdata.S3Source.open_shard", "domain": "py", "role": "function", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.open_shard", "dispname": "-"}, {"name": "atdata._sources.S3Source.open_shard", "domain": "py", "role": "function", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.open_shard", "dispname": "atdata.S3Source.open_shard"}, {"name": "atdata.S3Source.shard_list", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.shard_list", "dispname": "-"}, {"name": "atdata._sources.S3Source.shard_list", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.shard_list", "dispname": "atdata.S3Source.shard_list"}, {"name": "atdata.S3Source.shards", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.shards", "dispname": "-"}, {"name": "atdata._sources.S3Source.shards", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source.shards", "dispname": "atdata.S3Source.shards"}, {"name": "atdata.S3Source", "domain": "py", "role": "class", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source", "dispname": "-"}, {"name": "atdata._sources.S3Source", "domain": "py", "role": "class", "priority": "1", "uri": "api/S3Source.html#atdata.S3Source", "dispname": "atdata.S3Source"}, {"name": "atdata.local.Index.add_entry", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.add_entry", "dispname": "-"}, {"name": "atdata.local.Index.all_entries", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.all_entries", "dispname": "-"}, {"name": "atdata.local.Index.clear_stubs", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.clear_stubs", "dispname": "-"}, {"name": "atdata.local.Index.data_store", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.data_store", "dispname": "-"}, {"name": "atdata.local.Index.datasets", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.datasets", "dispname": "-"}, {"name": "atdata.local.Index.decode_schema", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.decode_schema", "dispname": "-"}, {"name": "atdata.local.Index.decode_schema_as", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.decode_schema_as", "dispname": "-"}, {"name": "atdata.local.Index.entries", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.entries", "dispname": "-"}, {"name": "atdata.local.Index.get_dataset", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.get_dataset", "dispname": "-"}, {"name": "atdata.local.Index.get_entry", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.get_entry", "dispname": "-"}, {"name": "atdata.local.Index.get_entry_by_name", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.get_entry_by_name", "dispname": "-"}, {"name": "atdata.local.Index.get_import_path", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.get_import_path", "dispname": "-"}, {"name": "atdata.local.Index.get_schema", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.get_schema", "dispname": "-"}, {"name": "atdata.local.Index.get_schema_record", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.get_schema_record", "dispname": "-"}, {"name": "atdata.local.Index.insert_dataset", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.insert_dataset", "dispname": "-"}, {"name": "atdata.local.Index.list_datasets", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.list_datasets", "dispname": "-"}, {"name": "atdata.local.Index.list_entries", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.list_entries", "dispname": "-"}, {"name": "atdata.local.Index.list_schemas", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.list_schemas", "dispname": "-"}, {"name": "atdata.local.Index.load_schema", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.load_schema", "dispname": "-"}, {"name": "atdata.local.Index.publish_schema", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.publish_schema", "dispname": "-"}, {"name": "atdata.local.Index.schemas", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.schemas", "dispname": "-"}, {"name": "atdata.local.Index.stub_dir", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.stub_dir", "dispname": "-"}, {"name": "atdata.local.Index.types", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index.types", "dispname": "-"}, {"name": "atdata.local.Index", "domain": "py", "role": "class", "priority": "1", "uri": "api/local.Index.html#atdata.local.Index", "dispname": "-"}, {"name": "atdata.local.LocalDatasetEntry.cid", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.LocalDatasetEntry.html#atdata.local.LocalDatasetEntry.cid", "dispname": "-"}, {"name": "atdata.local.LocalDatasetEntry.data_urls", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.LocalDatasetEntry.html#atdata.local.LocalDatasetEntry.data_urls", "dispname": "-"}, {"name": "atdata.local.LocalDatasetEntry.from_redis", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.LocalDatasetEntry.html#atdata.local.LocalDatasetEntry.from_redis", "dispname": "-"}, {"name": "atdata.local.LocalDatasetEntry.metadata", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.LocalDatasetEntry.html#atdata.local.LocalDatasetEntry.metadata", "dispname": "-"}, {"name": "atdata.local.LocalDatasetEntry.name", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.LocalDatasetEntry.html#atdata.local.LocalDatasetEntry.name", "dispname": "-"}, {"name": "atdata.local.LocalDatasetEntry.sample_kind", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.LocalDatasetEntry.html#atdata.local.LocalDatasetEntry.sample_kind", "dispname": "-"}, {"name": "atdata.local.LocalDatasetEntry.schema_ref", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.LocalDatasetEntry.html#atdata.local.LocalDatasetEntry.schema_ref", "dispname": "-"}, {"name": "atdata.local.LocalDatasetEntry.wds_url", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/local.LocalDatasetEntry.html#atdata.local.LocalDatasetEntry.wds_url", "dispname": "-"}, {"name": "atdata.local.LocalDatasetEntry.write_to", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.LocalDatasetEntry.html#atdata.local.LocalDatasetEntry.write_to", "dispname": "-"}, {"name": "atdata.local.LocalDatasetEntry", "domain": "py", "role": "class", "priority": "1", "uri": "api/local.LocalDatasetEntry.html#atdata.local.LocalDatasetEntry", "dispname": "-"}, {"name": "atdata.local.S3DataStore.read_url", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.S3DataStore.html#atdata.local.S3DataStore.read_url", "dispname": "-"}, {"name": "atdata.local.S3DataStore.supports_streaming", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.S3DataStore.html#atdata.local.S3DataStore.supports_streaming", "dispname": "-"}, {"name": "atdata.local.S3DataStore.write_shards", "domain": "py", "role": "function", "priority": "1", "uri": "api/local.S3DataStore.html#atdata.local.S3DataStore.write_shards", "dispname": "-"}, {"name": "atdata.local.S3DataStore", "domain": "py", "role": "class", "priority": "1", "uri": "api/local.S3DataStore.html#atdata.local.S3DataStore", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereClient.create_record", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.create_record", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.create_record", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.create_record", "dispname": "atdata.atmosphere.AtmosphereClient.create_record"}, {"name": "atdata.atmosphere.AtmosphereClient.delete_record", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.delete_record", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.delete_record", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.delete_record", "dispname": "atdata.atmosphere.AtmosphereClient.delete_record"}, {"name": "atdata.atmosphere.AtmosphereClient.did", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.did", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.did", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.did", "dispname": "atdata.atmosphere.AtmosphereClient.did"}, {"name": "atdata.atmosphere.AtmosphereClient.export_session", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.export_session", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.export_session", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.export_session", "dispname": "atdata.atmosphere.AtmosphereClient.export_session"}, {"name": "atdata.atmosphere.AtmosphereClient.get_blob", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.get_blob", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.get_blob", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.get_blob", "dispname": "atdata.atmosphere.AtmosphereClient.get_blob"}, {"name": "atdata.atmosphere.AtmosphereClient.get_blob_url", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.get_blob_url", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.get_blob_url", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.get_blob_url", "dispname": "atdata.atmosphere.AtmosphereClient.get_blob_url"}, {"name": "atdata.atmosphere.AtmosphereClient.get_record", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.get_record", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.get_record", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.get_record", "dispname": "atdata.atmosphere.AtmosphereClient.get_record"}, {"name": "atdata.atmosphere.AtmosphereClient.handle", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.handle", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.handle", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.handle", "dispname": "atdata.atmosphere.AtmosphereClient.handle"}, {"name": "atdata.atmosphere.AtmosphereClient.is_authenticated", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.is_authenticated", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.is_authenticated", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.is_authenticated", "dispname": "atdata.atmosphere.AtmosphereClient.is_authenticated"}, {"name": "atdata.atmosphere.AtmosphereClient.list_datasets", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.list_datasets", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.list_datasets", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.list_datasets", "dispname": "atdata.atmosphere.AtmosphereClient.list_datasets"}, {"name": "atdata.atmosphere.AtmosphereClient.list_lenses", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.list_lenses", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.list_lenses", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.list_lenses", "dispname": "atdata.atmosphere.AtmosphereClient.list_lenses"}, {"name": "atdata.atmosphere.AtmosphereClient.list_records", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.list_records", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.list_records", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.list_records", "dispname": "atdata.atmosphere.AtmosphereClient.list_records"}, {"name": "atdata.atmosphere.AtmosphereClient.list_schemas", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.list_schemas", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.list_schemas", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.list_schemas", "dispname": "atdata.atmosphere.AtmosphereClient.list_schemas"}, {"name": "atdata.atmosphere.AtmosphereClient.login", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.login", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.login", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.login", "dispname": "atdata.atmosphere.AtmosphereClient.login"}, {"name": "atdata.atmosphere.AtmosphereClient.login_with_session", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.login_with_session", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.login_with_session", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.login_with_session", "dispname": "atdata.atmosphere.AtmosphereClient.login_with_session"}, {"name": "atdata.atmosphere.AtmosphereClient.put_record", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.put_record", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.put_record", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.put_record", "dispname": "atdata.atmosphere.AtmosphereClient.put_record"}, {"name": "atdata.atmosphere.AtmosphereClient.upload_blob", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.upload_blob", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient.upload_blob", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient.upload_blob", "dispname": "atdata.atmosphere.AtmosphereClient.upload_blob"}, {"name": "atdata.atmosphere.AtmosphereClient", "domain": "py", "role": "class", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient", "dispname": "-"}, {"name": "atdata.atmosphere.client.AtmosphereClient", "domain": "py", "role": "class", "priority": "1", "uri": "api/AtmosphereClient.html#atdata.atmosphere.AtmosphereClient", "dispname": "atdata.atmosphere.AtmosphereClient"}, {"name": "atdata.atmosphere.AtmosphereIndex.datasets", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereIndex.html#atdata.atmosphere.AtmosphereIndex.datasets", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndex.decode_schema", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereIndex.html#atdata.atmosphere.AtmosphereIndex.decode_schema", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndex.get_dataset", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereIndex.html#atdata.atmosphere.AtmosphereIndex.get_dataset", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndex.get_schema", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereIndex.html#atdata.atmosphere.AtmosphereIndex.get_schema", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndex.insert_dataset", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereIndex.html#atdata.atmosphere.AtmosphereIndex.insert_dataset", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndex.list_datasets", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereIndex.html#atdata.atmosphere.AtmosphereIndex.list_datasets", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndex.list_schemas", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereIndex.html#atdata.atmosphere.AtmosphereIndex.list_schemas", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndex.publish_schema", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtmosphereIndex.html#atdata.atmosphere.AtmosphereIndex.publish_schema", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndex.schemas", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereIndex.html#atdata.atmosphere.AtmosphereIndex.schemas", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndex", "domain": "py", "role": "class", "priority": "1", "uri": "api/AtmosphereIndex.html#atdata.atmosphere.AtmosphereIndex", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndexEntry.data_urls", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereIndexEntry.html#atdata.atmosphere.AtmosphereIndexEntry.data_urls", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndexEntry.metadata", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereIndexEntry.html#atdata.atmosphere.AtmosphereIndexEntry.metadata", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndexEntry.name", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereIndexEntry.html#atdata.atmosphere.AtmosphereIndexEntry.name", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndexEntry.schema_ref", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereIndexEntry.html#atdata.atmosphere.AtmosphereIndexEntry.schema_ref", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndexEntry.uri", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtmosphereIndexEntry.html#atdata.atmosphere.AtmosphereIndexEntry.uri", "dispname": "-"}, {"name": "atdata.atmosphere.AtmosphereIndexEntry", "domain": "py", "role": "class", "priority": "1", "uri": "api/AtmosphereIndexEntry.html#atdata.atmosphere.AtmosphereIndexEntry", "dispname": "-"}, {"name": "atdata.atmosphere.SchemaPublisher.publish", "domain": "py", "role": "function", "priority": "1", "uri": "api/SchemaPublisher.html#atdata.atmosphere.SchemaPublisher.publish", "dispname": "-"}, {"name": "atdata.atmosphere.schema.SchemaPublisher.publish", "domain": "py", "role": "function", "priority": "1", "uri": "api/SchemaPublisher.html#atdata.atmosphere.SchemaPublisher.publish", "dispname": "atdata.atmosphere.SchemaPublisher.publish"}, {"name": "atdata.atmosphere.SchemaPublisher", "domain": "py", "role": "class", "priority": "1", "uri": "api/SchemaPublisher.html#atdata.atmosphere.SchemaPublisher", "dispname": "-"}, {"name": "atdata.atmosphere.schema.SchemaPublisher", "domain": "py", "role": "class", "priority": "1", "uri": "api/SchemaPublisher.html#atdata.atmosphere.SchemaPublisher", "dispname": "atdata.atmosphere.SchemaPublisher"}, {"name": "atdata.atmosphere.SchemaLoader.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/SchemaLoader.html#atdata.atmosphere.SchemaLoader.get", "dispname": "-"}, {"name": "atdata.atmosphere.schema.SchemaLoader.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/SchemaLoader.html#atdata.atmosphere.SchemaLoader.get", "dispname": "atdata.atmosphere.SchemaLoader.get"}, {"name": "atdata.atmosphere.SchemaLoader.list_all", "domain": "py", "role": "function", "priority": "1", "uri": "api/SchemaLoader.html#atdata.atmosphere.SchemaLoader.list_all", "dispname": "-"}, {"name": "atdata.atmosphere.schema.SchemaLoader.list_all", "domain": "py", "role": "function", "priority": "1", "uri": "api/SchemaLoader.html#atdata.atmosphere.SchemaLoader.list_all", "dispname": "atdata.atmosphere.SchemaLoader.list_all"}, {"name": "atdata.atmosphere.SchemaLoader", "domain": "py", "role": "class", "priority": "1", "uri": "api/SchemaLoader.html#atdata.atmosphere.SchemaLoader", "dispname": "-"}, {"name": "atdata.atmosphere.schema.SchemaLoader", "domain": "py", "role": "class", "priority": "1", "uri": "api/SchemaLoader.html#atdata.atmosphere.SchemaLoader", "dispname": "atdata.atmosphere.SchemaLoader"}, {"name": "atdata.atmosphere.DatasetPublisher.publish", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetPublisher.html#atdata.atmosphere.DatasetPublisher.publish", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetPublisher.publish", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetPublisher.html#atdata.atmosphere.DatasetPublisher.publish", "dispname": "atdata.atmosphere.DatasetPublisher.publish"}, {"name": "atdata.atmosphere.DatasetPublisher.publish_with_blobs", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetPublisher.html#atdata.atmosphere.DatasetPublisher.publish_with_blobs", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetPublisher.publish_with_blobs", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetPublisher.html#atdata.atmosphere.DatasetPublisher.publish_with_blobs", "dispname": "atdata.atmosphere.DatasetPublisher.publish_with_blobs"}, {"name": "atdata.atmosphere.DatasetPublisher.publish_with_urls", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetPublisher.html#atdata.atmosphere.DatasetPublisher.publish_with_urls", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetPublisher.publish_with_urls", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetPublisher.html#atdata.atmosphere.DatasetPublisher.publish_with_urls", "dispname": "atdata.atmosphere.DatasetPublisher.publish_with_urls"}, {"name": "atdata.atmosphere.DatasetPublisher", "domain": "py", "role": "class", "priority": "1", "uri": "api/DatasetPublisher.html#atdata.atmosphere.DatasetPublisher", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetPublisher", "domain": "py", "role": "class", "priority": "1", "uri": "api/DatasetPublisher.html#atdata.atmosphere.DatasetPublisher", "dispname": "atdata.atmosphere.DatasetPublisher"}, {"name": "atdata.atmosphere.DatasetLoader.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetLoader.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get", "dispname": "atdata.atmosphere.DatasetLoader.get"}, {"name": "atdata.atmosphere.DatasetLoader.get_blob_urls", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get_blob_urls", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetLoader.get_blob_urls", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get_blob_urls", "dispname": "atdata.atmosphere.DatasetLoader.get_blob_urls"}, {"name": "atdata.atmosphere.DatasetLoader.get_blobs", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get_blobs", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetLoader.get_blobs", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get_blobs", "dispname": "atdata.atmosphere.DatasetLoader.get_blobs"}, {"name": "atdata.atmosphere.DatasetLoader.get_metadata", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get_metadata", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetLoader.get_metadata", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get_metadata", "dispname": "atdata.atmosphere.DatasetLoader.get_metadata"}, {"name": "atdata.atmosphere.DatasetLoader.get_storage_type", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get_storage_type", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetLoader.get_storage_type", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get_storage_type", "dispname": "atdata.atmosphere.DatasetLoader.get_storage_type"}, {"name": "atdata.atmosphere.DatasetLoader.get_urls", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get_urls", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetLoader.get_urls", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.get_urls", "dispname": "atdata.atmosphere.DatasetLoader.get_urls"}, {"name": "atdata.atmosphere.DatasetLoader.list_all", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.list_all", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetLoader.list_all", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.list_all", "dispname": "atdata.atmosphere.DatasetLoader.list_all"}, {"name": "atdata.atmosphere.DatasetLoader.to_dataset", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.to_dataset", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetLoader.to_dataset", "domain": "py", "role": "function", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader.to_dataset", "dispname": "atdata.atmosphere.DatasetLoader.to_dataset"}, {"name": "atdata.atmosphere.DatasetLoader", "domain": "py", "role": "class", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader", "dispname": "-"}, {"name": "atdata.atmosphere.records.DatasetLoader", "domain": "py", "role": "class", "priority": "1", "uri": "api/DatasetLoader.html#atdata.atmosphere.DatasetLoader", "dispname": "atdata.atmosphere.DatasetLoader"}, {"name": "atdata.atmosphere.LensPublisher.publish", "domain": "py", "role": "function", "priority": "1", "uri": "api/LensPublisher.html#atdata.atmosphere.LensPublisher.publish", "dispname": "-"}, {"name": "atdata.atmosphere.lens.LensPublisher.publish", "domain": "py", "role": "function", "priority": "1", "uri": "api/LensPublisher.html#atdata.atmosphere.LensPublisher.publish", "dispname": "atdata.atmosphere.LensPublisher.publish"}, {"name": "atdata.atmosphere.LensPublisher.publish_from_lens", "domain": "py", "role": "function", "priority": "1", "uri": "api/LensPublisher.html#atdata.atmosphere.LensPublisher.publish_from_lens", "dispname": "-"}, {"name": "atdata.atmosphere.lens.LensPublisher.publish_from_lens", "domain": "py", "role": "function", "priority": "1", "uri": "api/LensPublisher.html#atdata.atmosphere.LensPublisher.publish_from_lens", "dispname": "atdata.atmosphere.LensPublisher.publish_from_lens"}, {"name": "atdata.atmosphere.LensPublisher", "domain": "py", "role": "class", "priority": "1", "uri": "api/LensPublisher.html#atdata.atmosphere.LensPublisher", "dispname": "-"}, {"name": "atdata.atmosphere.lens.LensPublisher", "domain": "py", "role": "class", "priority": "1", "uri": "api/LensPublisher.html#atdata.atmosphere.LensPublisher", "dispname": "atdata.atmosphere.LensPublisher"}, {"name": "atdata.atmosphere.LensLoader.find_by_schemas", "domain": "py", "role": "function", "priority": "1", "uri": "api/LensLoader.html#atdata.atmosphere.LensLoader.find_by_schemas", "dispname": "-"}, {"name": "atdata.atmosphere.lens.LensLoader.find_by_schemas", "domain": "py", "role": "function", "priority": "1", "uri": "api/LensLoader.html#atdata.atmosphere.LensLoader.find_by_schemas", "dispname": "atdata.atmosphere.LensLoader.find_by_schemas"}, {"name": "atdata.atmosphere.LensLoader.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/LensLoader.html#atdata.atmosphere.LensLoader.get", "dispname": "-"}, {"name": "atdata.atmosphere.lens.LensLoader.get", "domain": "py", "role": "function", "priority": "1", "uri": "api/LensLoader.html#atdata.atmosphere.LensLoader.get", "dispname": "atdata.atmosphere.LensLoader.get"}, {"name": "atdata.atmosphere.LensLoader.list_all", "domain": "py", "role": "function", "priority": "1", "uri": "api/LensLoader.html#atdata.atmosphere.LensLoader.list_all", "dispname": "-"}, {"name": "atdata.atmosphere.lens.LensLoader.list_all", "domain": "py", "role": "function", "priority": "1", "uri": "api/LensLoader.html#atdata.atmosphere.LensLoader.list_all", "dispname": "atdata.atmosphere.LensLoader.list_all"}, {"name": "atdata.atmosphere.LensLoader", "domain": "py", "role": "class", "priority": "1", "uri": "api/LensLoader.html#atdata.atmosphere.LensLoader", "dispname": "-"}, {"name": "atdata.atmosphere.lens.LensLoader", "domain": "py", "role": "class", "priority": "1", "uri": "api/LensLoader.html#atdata.atmosphere.LensLoader", "dispname": "atdata.atmosphere.LensLoader"}, {"name": "atdata.atmosphere.AtUri.authority", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtUri.html#atdata.atmosphere.AtUri.authority", "dispname": "-"}, {"name": "atdata.atmosphere._types.AtUri.authority", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtUri.html#atdata.atmosphere.AtUri.authority", "dispname": "atdata.atmosphere.AtUri.authority"}, {"name": "atdata.atmosphere.AtUri.collection", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtUri.html#atdata.atmosphere.AtUri.collection", "dispname": "-"}, {"name": "atdata.atmosphere._types.AtUri.collection", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtUri.html#atdata.atmosphere.AtUri.collection", "dispname": "atdata.atmosphere.AtUri.collection"}, {"name": "atdata.atmosphere.AtUri.parse", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtUri.html#atdata.atmosphere.AtUri.parse", "dispname": "-"}, {"name": "atdata.atmosphere._types.AtUri.parse", "domain": "py", "role": "function", "priority": "1", "uri": "api/AtUri.html#atdata.atmosphere.AtUri.parse", "dispname": "atdata.atmosphere.AtUri.parse"}, {"name": "atdata.atmosphere.AtUri.rkey", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtUri.html#atdata.atmosphere.AtUri.rkey", "dispname": "-"}, {"name": "atdata.atmosphere._types.AtUri.rkey", "domain": "py", "role": "attribute", "priority": "1", "uri": "api/AtUri.html#atdata.atmosphere.AtUri.rkey", "dispname": "atdata.atmosphere.AtUri.rkey"}, {"name": "atdata.atmosphere.AtUri", "domain": "py", "role": "class", "priority": "1", "uri": "api/AtUri.html#atdata.atmosphere.AtUri", "dispname": "-"}, {"name": "atdata.atmosphere._types.AtUri", "domain": "py", "role": "class", "priority": "1", "uri": "api/AtUri.html#atdata.atmosphere.AtUri", "dispname": "atdata.atmosphere.AtUri"}, {"name": "atdata.promote.promote_to_atmosphere", "domain": "py", "role": "function", "priority": "1", "uri": "api/promote_to_atmosphere.html#atdata.promote.promote_to_atmosphere", "dispname": "-"}]}