A loose federation of distributed, typed datasets
1<!DOCTYPE html>
2<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
3
4<meta charset="utf-8">
5<meta name="generator" content="quarto-1.7.34">
6
7<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
8
9
10<title>datasetloader – atdata</title>
11<style>
12code{white-space: pre-wrap;}
13span.smallcaps{font-variant: small-caps;}
14div.columns{display: flex; gap: min(4vw, 1.5em);}
15div.column{flex: auto; overflow-x: auto;}
16div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
17ul.task-list{list-style: none;}
18ul.task-list li input[type="checkbox"] {
19 width: 0.8em;
20 margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
21 vertical-align: middle;
22}
23/* CSS for syntax highlighting */
24html { -webkit-text-size-adjust: 100%; }
25pre > code.sourceCode { white-space: pre; position: relative; }
26pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
27pre > code.sourceCode > span:empty { height: 1.2em; }
28.sourceCode { overflow: visible; }
29code.sourceCode > span { color: inherit; text-decoration: inherit; }
30div.sourceCode { margin: 1em 0; }
31pre.sourceCode { margin: 0; }
32@media screen {
33div.sourceCode { overflow: auto; }
34}
35@media print {
36pre > code.sourceCode { white-space: pre-wrap; }
37pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
38}
39pre.numberSource code
40 { counter-reset: source-line 0; }
41pre.numberSource code > span
42 { position: relative; left: -4em; counter-increment: source-line; }
43pre.numberSource code > span > a:first-child::before
44 { content: counter(source-line);
45 position: relative; left: -1em; text-align: right; vertical-align: baseline;
46 border: none; display: inline-block;
47 -webkit-touch-callout: none; -webkit-user-select: none;
48 -khtml-user-select: none; -moz-user-select: none;
49 -ms-user-select: none; user-select: none;
50 padding: 0 4px; width: 4em;
51 }
52pre.numberSource { margin-left: 3em; padding-left: 4px; }
53div.sourceCode
54 { }
55@media screen {
56pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
57}
58</style>
59
60
61<script src="../site_libs/quarto-nav/quarto-nav.js"></script>
62<script src="../site_libs/quarto-nav/headroom.min.js"></script>
63<script src="../site_libs/clipboard/clipboard.min.js"></script>
64<script src="../site_libs/quarto-search/autocomplete.umd.js"></script>
65<script src="../site_libs/quarto-search/fuse.min.js"></script>
66<script src="../site_libs/quarto-search/quarto-search.js"></script>
67<meta name="quarto:offset" content="../">
68<script src="../site_libs/quarto-html/quarto.js" type="module"></script>
69<script src="../site_libs/quarto-html/tabsets/tabsets.js" type="module"></script>
70<script src="../site_libs/quarto-html/popper.min.js"></script>
71<script src="../site_libs/quarto-html/tippy.umd.min.js"></script>
72<script src="../site_libs/quarto-html/anchor.min.js"></script>
73<link href="../site_libs/quarto-html/tippy.css" rel="stylesheet">
74<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme" id="quarto-text-highlighting-styles">
75<link href="../site_libs/quarto-html/quarto-syntax-highlighting-dark-8dcd8563ea6803ab7cbb3d71ca5772e1.css" rel="stylesheet" class="quarto-color-scheme quarto-color-alternate" id="quarto-text-highlighting-styles">
76<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme-extra" id="quarto-text-highlighting-styles">
77<script src="../site_libs/bootstrap/bootstrap.min.js"></script>
78<link href="../site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
79<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme" id="quarto-bootstrap" data-mode="light">
80<link href="../site_libs/bootstrap/bootstrap-dark-7964ffd8887b0991fe8d71c6c8bc75d6.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme quarto-color-alternate" id="quarto-bootstrap" data-mode="dark">
81<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme-extra" id="quarto-bootstrap" data-mode="light">
82<script id="quarto-search-options" type="application/json">{
83 "location": "navbar",
84 "copy-button": false,
85 "collapse-after": 3,
86 "panel-placement": "end",
87 "type": "overlay",
88 "limit": 50,
89 "keyboard-shortcut": [
90 "f",
91 "/",
92 "s"
93 ],
94 "show-item-context": false,
95 "language": {
96 "search-no-results-text": "No results",
97 "search-matching-documents-text": "matching documents",
98 "search-copy-link-title": "Copy link to search",
99 "search-hide-matches-text": "Hide additional matches",
100 "search-more-match-text": "more match in this document",
101 "search-more-matches-text": "more matches in this document",
102 "search-clear-button-title": "Clear",
103 "search-text-placeholder": "",
104 "search-detached-cancel-button-title": "Cancel",
105 "search-submit-button-title": "Submit",
106 "search-label": "Search"
107 }
108}</script>
109
110
111<link rel="stylesheet" href="../assets/styles.css">
112</head>
113
114<body class="nav-fixed quarto-light"><script id="quarto-html-before-body" type="application/javascript">
115 const toggleBodyColorMode = (bsSheetEl) => {
116 const mode = bsSheetEl.getAttribute("data-mode");
117 const bodyEl = window.document.querySelector("body");
118 if (mode === "dark") {
119 bodyEl.classList.add("quarto-dark");
120 bodyEl.classList.remove("quarto-light");
121 } else {
122 bodyEl.classList.add("quarto-light");
123 bodyEl.classList.remove("quarto-dark");
124 }
125 }
126 const toggleBodyColorPrimary = () => {
127 const bsSheetEl = window.document.querySelector("link#quarto-bootstrap:not([rel=disabled-stylesheet])");
128 if (bsSheetEl) {
129 toggleBodyColorMode(bsSheetEl);
130 }
131 }
132 const setColorSchemeToggle = (alternate) => {
133 const toggles = window.document.querySelectorAll('.quarto-color-scheme-toggle');
134 for (let i=0; i < toggles.length; i++) {
135 const toggle = toggles[i];
136 if (toggle) {
137 if (alternate) {
138 toggle.classList.add("alternate");
139 } else {
140 toggle.classList.remove("alternate");
141 }
142 }
143 }
144 };
145 const toggleColorMode = (alternate) => {
146 // Switch the stylesheets
147 const primaryStylesheets = window.document.querySelectorAll('link.quarto-color-scheme:not(.quarto-color-alternate)');
148 const alternateStylesheets = window.document.querySelectorAll('link.quarto-color-scheme.quarto-color-alternate');
149 manageTransitions('#quarto-margin-sidebar .nav-link', false);
150 if (alternate) {
151 // note: dark is layered on light, we don't disable primary!
152 enableStylesheet(alternateStylesheets);
153 for (const sheetNode of alternateStylesheets) {
154 if (sheetNode.id === "quarto-bootstrap") {
155 toggleBodyColorMode(sheetNode);
156 }
157 }
158 } else {
159 disableStylesheet(alternateStylesheets);
160 enableStylesheet(primaryStylesheets)
161 toggleBodyColorPrimary();
162 }
163 manageTransitions('#quarto-margin-sidebar .nav-link', true);
164 // Switch the toggles
165 setColorSchemeToggle(alternate)
166 // Hack to workaround the fact that safari doesn't
167 // properly recolor the scrollbar when toggling (#1455)
168 if (navigator.userAgent.indexOf('Safari') > 0 && navigator.userAgent.indexOf('Chrome') == -1) {
169 manageTransitions("body", false);
170 window.scrollTo(0, 1);
171 setTimeout(() => {
172 window.scrollTo(0, 0);
173 manageTransitions("body", true);
174 }, 40);
175 }
176 }
177 const disableStylesheet = (stylesheets) => {
178 for (let i=0; i < stylesheets.length; i++) {
179 const stylesheet = stylesheets[i];
180 stylesheet.rel = 'disabled-stylesheet';
181 }
182 }
183 const enableStylesheet = (stylesheets) => {
184 for (let i=0; i < stylesheets.length; i++) {
185 const stylesheet = stylesheets[i];
186 if(stylesheet.rel !== 'stylesheet') { // for Chrome, which will still FOUC without this check
187 stylesheet.rel = 'stylesheet';
188 }
189 }
190 }
191 const manageTransitions = (selector, allowTransitions) => {
192 const els = window.document.querySelectorAll(selector);
193 for (let i=0; i < els.length; i++) {
194 const el = els[i];
195 if (allowTransitions) {
196 el.classList.remove('notransition');
197 } else {
198 el.classList.add('notransition');
199 }
200 }
201 }
202 const isFileUrl = () => {
203 return window.location.protocol === 'file:';
204 }
205 const hasAlternateSentinel = () => {
206 let styleSentinel = getColorSchemeSentinel();
207 if (styleSentinel !== null) {
208 return styleSentinel === "alternate";
209 } else {
210 return false;
211 }
212 }
213 const setStyleSentinel = (alternate) => {
214 const value = alternate ? "alternate" : "default";
215 if (!isFileUrl()) {
216 window.localStorage.setItem("quarto-color-scheme", value);
217 } else {
218 localAlternateSentinel = value;
219 }
220 }
221 const getColorSchemeSentinel = () => {
222 if (!isFileUrl()) {
223 const storageValue = window.localStorage.getItem("quarto-color-scheme");
224 return storageValue != null ? storageValue : localAlternateSentinel;
225 } else {
226 return localAlternateSentinel;
227 }
228 }
229 const toggleGiscusIfUsed = (isAlternate, darkModeDefault) => {
230 const baseTheme = document.querySelector('#giscus-base-theme')?.value ?? 'light';
231 const alternateTheme = document.querySelector('#giscus-alt-theme')?.value ?? 'dark';
232 let newTheme = '';
233 if(authorPrefersDark) {
234 newTheme = isAlternate ? baseTheme : alternateTheme;
235 } else {
236 newTheme = isAlternate ? alternateTheme : baseTheme;
237 }
238 const changeGiscusTheme = () => {
239 // From: https://github.com/giscus/giscus/issues/336
240 const sendMessage = (message) => {
241 const iframe = document.querySelector('iframe.giscus-frame');
242 if (!iframe) return;
243 iframe.contentWindow.postMessage({ giscus: message }, 'https://giscus.app');
244 }
245 sendMessage({
246 setConfig: {
247 theme: newTheme
248 }
249 });
250 }
251 const isGiscussLoaded = window.document.querySelector('iframe.giscus-frame') !== null;
252 if (isGiscussLoaded) {
253 changeGiscusTheme();
254 }
255 };
256 const authorPrefersDark = false;
257 const darkModeDefault = authorPrefersDark;
258 document.querySelector('link#quarto-text-highlighting-styles.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
259 document.querySelector('link#quarto-bootstrap.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
260 let localAlternateSentinel = darkModeDefault ? 'alternate' : 'default';
261 // Dark / light mode switch
262 window.quartoToggleColorScheme = () => {
263 // Read the current dark / light value
264 let toAlternate = !hasAlternateSentinel();
265 toggleColorMode(toAlternate);
266 setStyleSentinel(toAlternate);
267 toggleGiscusIfUsed(toAlternate, darkModeDefault);
268 window.dispatchEvent(new Event('resize'));
269 };
270 // Switch to dark mode if need be
271 if (hasAlternateSentinel()) {
272 toggleColorMode(true);
273 } else {
274 toggleColorMode(false);
275 }
276 </script>
277
278<div id="quarto-search-results"></div>
279 <header id="quarto-header" class="headroom fixed-top">
280 <nav class="navbar navbar-expand-lg " data-bs-theme="dark">
281 <div class="navbar-container container-fluid">
282 <div class="navbar-brand-container mx-auto">
283 <a class="navbar-brand" href="../index.html">
284 <span class="navbar-title">atdata</span>
285 </a>
286 </div>
287 <div id="quarto-search" class="" title="Search"></div>
288 <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarCollapse" aria-controls="navbarCollapse" role="menu" aria-expanded="false" aria-label="Toggle navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
289 <span class="navbar-toggler-icon"></span>
290</button>
291 <div class="collapse navbar-collapse" id="navbarCollapse">
292 <ul class="navbar-nav navbar-nav-scroll me-auto">
293 <li class="nav-item">
294 <a class="nav-link" href="../index.html">
295<span class="menu-text">Guide</span></a>
296 </li>
297 <li class="nav-item dropdown ">
298 <a class="nav-link dropdown-toggle" href="#" id="nav-menu-tutorials" role="link" data-bs-toggle="dropdown" aria-expanded="false">
299 <span class="menu-text">Tutorials</span>
300 </a>
301 <ul class="dropdown-menu" aria-labelledby="nav-menu-tutorials">
302 <li>
303 <a class="dropdown-item" href="../tutorials/quickstart.html">
304 <span class="dropdown-text">Quick Start</span></a>
305 </li>
306 <li>
307 <a class="dropdown-item" href="../tutorials/local-workflow.html">
308 <span class="dropdown-text">Local Workflow</span></a>
309 </li>
310 <li>
311 <a class="dropdown-item" href="../tutorials/atmosphere.html">
312 <span class="dropdown-text">Atmosphere Publishing</span></a>
313 </li>
314 <li>
315 <a class="dropdown-item" href="../tutorials/promotion.html">
316 <span class="dropdown-text">Promotion Workflow</span></a>
317 </li>
318 </ul>
319 </li>
320 <li class="nav-item dropdown ">
321 <a class="nav-link dropdown-toggle" href="#" id="nav-menu-reference" role="link" data-bs-toggle="dropdown" aria-expanded="false">
322 <span class="menu-text">Reference</span>
323 </a>
324 <ul class="dropdown-menu" aria-labelledby="nav-menu-reference">
325 <li>
326 <a class="dropdown-item" href="../reference/architecture.html">
327 <span class="dropdown-text">Architecture Overview</span></a>
328 </li>
329 <li>
330 <a class="dropdown-item" href="../reference/packable-samples.html">
331 <span class="dropdown-text">Packable Samples</span></a>
332 </li>
333 <li>
334 <a class="dropdown-item" href="../reference/datasets.html">
335 <span class="dropdown-text">Datasets</span></a>
336 </li>
337 <li>
338 <a class="dropdown-item" href="../reference/lenses.html">
339 <span class="dropdown-text">Lenses</span></a>
340 </li>
341 <li>
342 <a class="dropdown-item" href="../reference/local-storage.html">
343 <span class="dropdown-text">Local Storage</span></a>
344 </li>
345 <li>
346 <a class="dropdown-item" href="../reference/atmosphere.html">
347 <span class="dropdown-text">Atmosphere</span></a>
348 </li>
349 <li>
350 <a class="dropdown-item" href="../reference/promotion.html">
351 <span class="dropdown-text">Promotion</span></a>
352 </li>
353 <li>
354 <a class="dropdown-item" href="../reference/load-dataset.html">
355 <span class="dropdown-text">load_dataset API</span></a>
356 </li>
357 <li>
358 <a class="dropdown-item" href="../reference/protocols.html">
359 <span class="dropdown-text">Protocols</span></a>
360 </li>
361 <li>
362 <a class="dropdown-item" href="../reference/uri-spec.html">
363 <span class="dropdown-text">URI Specification</span></a>
364 </li>
365 <li>
366 <a class="dropdown-item" href="../reference/troubleshooting.html">
367 <span class="dropdown-text">Troubleshooting & FAQ</span></a>
368 </li>
369 <li>
370 <a class="dropdown-item" href="../reference/deployment.html">
371 <span class="dropdown-text">Deployment Guide</span></a>
372 </li>
373 </ul>
374 </li>
375 <li class="nav-item">
376 <a class="nav-link" href="../api/index.html">
377<span class="menu-text">API</span></a>
378 </li>
379</ul>
380 <ul class="navbar-nav navbar-nav-scroll ms-auto">
381 <li class="nav-item compact">
382 <a class="nav-link" href="https://github.com/your-org/atdata"> <i class="bi bi-github" role="img">
383</i>
384<span class="menu-text"></span></a>
385 </li>
386</ul>
387 </div> <!-- /navcollapse -->
388 <div class="quarto-navbar-tools">
389 <a href="" class="quarto-color-scheme-toggle quarto-navigation-tool px-1" onclick="window.quartoToggleColorScheme(); return false;" title="Toggle dark mode"><i class="bi"></i></a>
390</div>
391 </div> <!-- /container-fluid -->
392 </nav>
393</header>
394<!-- content -->
395<div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-article page-navbar">
396<!-- sidebar -->
397<!-- margin-sidebar -->
398 <div id="quarto-margin-sidebar" class="sidebar margin-sidebar">
399 <nav id="TOC" role="doc-toc" class="toc-active">
400 <h2 id="toc-title">On this page</h2>
401
402 <ul>
403 <li><a href="#atdata.atmosphere.DatasetLoader" id="toc-atdata.atmosphere.DatasetLoader" class="nav-link active" data-scroll-target="#atdata.atmosphere.DatasetLoader">DatasetLoader</a>
404 <ul class="collapse">
405 <li><a href="#examples" id="toc-examples" class="nav-link" data-scroll-target="#examples">Examples</a></li>
406 <li><a href="#methods" id="toc-methods" class="nav-link" data-scroll-target="#methods">Methods</a>
407 <ul class="collapse">
408 <li><a href="#atdata.atmosphere.DatasetLoader.get" id="toc-atdata.atmosphere.DatasetLoader.get" class="nav-link" data-scroll-target="#atdata.atmosphere.DatasetLoader.get">get</a></li>
409 <li><a href="#atdata.atmosphere.DatasetLoader.get_blob_urls" id="toc-atdata.atmosphere.DatasetLoader.get_blob_urls" class="nav-link" data-scroll-target="#atdata.atmosphere.DatasetLoader.get_blob_urls">get_blob_urls</a></li>
410 <li><a href="#atdata.atmosphere.DatasetLoader.get_blobs" id="toc-atdata.atmosphere.DatasetLoader.get_blobs" class="nav-link" data-scroll-target="#atdata.atmosphere.DatasetLoader.get_blobs">get_blobs</a></li>
411 <li><a href="#atdata.atmosphere.DatasetLoader.get_metadata" id="toc-atdata.atmosphere.DatasetLoader.get_metadata" class="nav-link" data-scroll-target="#atdata.atmosphere.DatasetLoader.get_metadata">get_metadata</a></li>
412 <li><a href="#atdata.atmosphere.DatasetLoader.get_storage_type" id="toc-atdata.atmosphere.DatasetLoader.get_storage_type" class="nav-link" data-scroll-target="#atdata.atmosphere.DatasetLoader.get_storage_type">get_storage_type</a></li>
413 <li><a href="#atdata.atmosphere.DatasetLoader.get_urls" id="toc-atdata.atmosphere.DatasetLoader.get_urls" class="nav-link" data-scroll-target="#atdata.atmosphere.DatasetLoader.get_urls">get_urls</a></li>
414 <li><a href="#atdata.atmosphere.DatasetLoader.list_all" id="toc-atdata.atmosphere.DatasetLoader.list_all" class="nav-link" data-scroll-target="#atdata.atmosphere.DatasetLoader.list_all">list_all</a></li>
415 <li><a href="#atdata.atmosphere.DatasetLoader.to_dataset" id="toc-atdata.atmosphere.DatasetLoader.to_dataset" class="nav-link" data-scroll-target="#atdata.atmosphere.DatasetLoader.to_dataset">to_dataset</a></li>
416 </ul></li>
417 </ul></li>
418 </ul>
419<div class="toc-actions"><ul><li><a href="https://github.com/your-org/atdata/edit/main/api/DatasetLoader.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></nav>
420 </div>
421<!-- main -->
422<main class="content" id="quarto-document-content"><header id="title-block-header" class="quarto-title-block"></header>
423
424
425
426
427
428<section id="atdata.atmosphere.DatasetLoader" class="level1">
429<h1>DatasetLoader</h1>
430<div class="sourceCode" id="cb1"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a>atmosphere.DatasetLoader(client)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
431<p>Loads dataset records from ATProto.</p>
432<p>This class fetches dataset index records and can create Dataset objects from them. Note that loading a dataset requires having the corresponding Python class for the sample type.</p>
433<section id="examples" class="level2 doc-section doc-section-examples">
434<h2 class="doc-section doc-section-examples anchored" data-anchor-id="examples">Examples</h2>
435<div class="sourceCode" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="op">>>></span> client <span class="op">=</span> AtmosphereClient()</span>
436<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a><span class="op">>>></span> loader <span class="op">=</span> DatasetLoader(client)</span>
437<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a><span class="op">>>></span></span>
438<span id="cb2-4"><a href="#cb2-4" aria-hidden="true" tabindex="-1"></a><span class="op">>>></span> <span class="co"># List available datasets</span></span>
439<span id="cb2-5"><a href="#cb2-5" aria-hidden="true" tabindex="-1"></a><span class="op">>>></span> datasets <span class="op">=</span> loader.<span class="bu">list</span>()</span>
440<span id="cb2-6"><a href="#cb2-6" aria-hidden="true" tabindex="-1"></a><span class="op">>>></span> <span class="cf">for</span> ds <span class="kw">in</span> datasets:</span>
441<span id="cb2-7"><a href="#cb2-7" aria-hidden="true" tabindex="-1"></a>... <span class="bu">print</span>(ds[<span class="st">"name"</span>], ds[<span class="st">"schemaRef"</span>])</span>
442<span id="cb2-8"><a href="#cb2-8" aria-hidden="true" tabindex="-1"></a><span class="op">>>></span></span>
443<span id="cb2-9"><a href="#cb2-9" aria-hidden="true" tabindex="-1"></a><span class="op">>>></span> <span class="co"># Get a specific dataset record</span></span>
444<span id="cb2-10"><a href="#cb2-10" aria-hidden="true" tabindex="-1"></a><span class="op">>>></span> record <span class="op">=</span> loader.get(<span class="st">"at://did:plc:abc/ac.foundation.dataset.record/xyz"</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
445</section>
446<section id="methods" class="level2">
447<h2 class="anchored" data-anchor-id="methods">Methods</h2>
448<table class="caption-top table">
449<thead>
450<tr class="header">
451<th>Name</th>
452<th>Description</th>
453</tr>
454</thead>
455<tbody>
456<tr class="odd">
457<td><a href="#atdata.atmosphere.DatasetLoader.get">get</a></td>
458<td>Fetch a dataset record by AT URI.</td>
459</tr>
460<tr class="even">
461<td><a href="#atdata.atmosphere.DatasetLoader.get_blob_urls">get_blob_urls</a></td>
462<td>Get fetchable URLs for blob-stored dataset shards.</td>
463</tr>
464<tr class="odd">
465<td><a href="#atdata.atmosphere.DatasetLoader.get_blobs">get_blobs</a></td>
466<td>Get the blob references from a dataset record.</td>
467</tr>
468<tr class="even">
469<td><a href="#atdata.atmosphere.DatasetLoader.get_metadata">get_metadata</a></td>
470<td>Get the metadata from a dataset record.</td>
471</tr>
472<tr class="odd">
473<td><a href="#atdata.atmosphere.DatasetLoader.get_storage_type">get_storage_type</a></td>
474<td>Get the storage type of a dataset record.</td>
475</tr>
476<tr class="even">
477<td><a href="#atdata.atmosphere.DatasetLoader.get_urls">get_urls</a></td>
478<td>Get the WebDataset URLs from a dataset record.</td>
479</tr>
480<tr class="odd">
481<td><a href="#atdata.atmosphere.DatasetLoader.list_all">list_all</a></td>
482<td>List dataset records from a repository.</td>
483</tr>
484<tr class="even">
485<td><a href="#atdata.atmosphere.DatasetLoader.to_dataset">to_dataset</a></td>
486<td>Create a Dataset object from an ATProto record.</td>
487</tr>
488</tbody>
489</table>
490<section id="atdata.atmosphere.DatasetLoader.get" class="level3">
491<h3 class="anchored" data-anchor-id="atdata.atmosphere.DatasetLoader.get">get</h3>
492<div class="sourceCode" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a>atmosphere.DatasetLoader.get(uri)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
493<p>Fetch a dataset record by AT URI.</p>
494<section id="parameters" class="level4 doc-section doc-section-parameters">
495<h4 class="doc-section doc-section-parameters anchored" data-anchor-id="parameters">Parameters</h4>
496<table class="caption-top table">
497<thead>
498<tr class="header">
499<th>Name</th>
500<th>Type</th>
501<th>Description</th>
502<th>Default</th>
503</tr>
504</thead>
505<tbody>
506<tr class="odd">
507<td>uri</td>
508<td><a href="`str`">str</a> | <a href="`atdata.atmosphere._types.AtUri`">AtUri</a></td>
509<td>The AT URI of the dataset record.</td>
510<td><em>required</em></td>
511</tr>
512</tbody>
513</table>
514</section>
515<section id="returns" class="level4 doc-section doc-section-returns">
516<h4 class="doc-section doc-section-returns anchored" data-anchor-id="returns">Returns</h4>
517<table class="caption-top table">
518<thead>
519<tr class="header">
520<th>Name</th>
521<th>Type</th>
522<th>Description</th>
523</tr>
524</thead>
525<tbody>
526<tr class="odd">
527<td></td>
528<td><a href="`dict`">dict</a></td>
529<td>The dataset record as a dictionary.</td>
530</tr>
531</tbody>
532</table>
533</section>
534<section id="raises" class="level4 doc-section doc-section-raises">
535<h4 class="doc-section doc-section-raises anchored" data-anchor-id="raises">Raises</h4>
536<table class="caption-top table">
537<thead>
538<tr class="header">
539<th>Name</th>
540<th>Type</th>
541<th>Description</th>
542</tr>
543</thead>
544<tbody>
545<tr class="odd">
546<td></td>
547<td><a href="`ValueError`">ValueError</a></td>
548<td>If the record is not a dataset record.</td>
549</tr>
550</tbody>
551</table>
552</section>
553</section>
554<section id="atdata.atmosphere.DatasetLoader.get_blob_urls" class="level3">
555<h3 class="anchored" data-anchor-id="atdata.atmosphere.DatasetLoader.get_blob_urls">get_blob_urls</h3>
556<div class="sourceCode" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a>atmosphere.DatasetLoader.get_blob_urls(uri)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
557<p>Get fetchable URLs for blob-stored dataset shards.</p>
558<p>This resolves the PDS endpoint and constructs URLs that can be used to fetch the blob data directly.</p>
559<section id="parameters-1" class="level4 doc-section doc-section-parameters">
560<h4 class="doc-section doc-section-parameters anchored" data-anchor-id="parameters-1">Parameters</h4>
561<table class="caption-top table">
562<thead>
563<tr class="header">
564<th>Name</th>
565<th>Type</th>
566<th>Description</th>
567<th>Default</th>
568</tr>
569</thead>
570<tbody>
571<tr class="odd">
572<td>uri</td>
573<td><a href="`str`">str</a> | <a href="`atdata.atmosphere._types.AtUri`">AtUri</a></td>
574<td>The AT URI of the dataset record.</td>
575<td><em>required</em></td>
576</tr>
577</tbody>
578</table>
579</section>
580<section id="returns-1" class="level4 doc-section doc-section-returns">
581<h4 class="doc-section doc-section-returns anchored" data-anchor-id="returns-1">Returns</h4>
582<table class="caption-top table">
583<thead>
584<tr class="header">
585<th>Name</th>
586<th>Type</th>
587<th>Description</th>
588</tr>
589</thead>
590<tbody>
591<tr class="odd">
592<td></td>
593<td><a href="`list`">list</a>[<a href="`str`">str</a>]</td>
594<td>List of URLs for fetching the blob data.</td>
595</tr>
596</tbody>
597</table>
598</section>
599<section id="raises-1" class="level4 doc-section doc-section-raises">
600<h4 class="doc-section doc-section-raises anchored" data-anchor-id="raises-1">Raises</h4>
601<table class="caption-top table">
602<thead>
603<tr class="header">
604<th>Name</th>
605<th>Type</th>
606<th>Description</th>
607</tr>
608</thead>
609<tbody>
610<tr class="odd">
611<td></td>
612<td><a href="`ValueError`">ValueError</a></td>
613<td>If storage type is not blobs or PDS cannot be resolved.</td>
614</tr>
615</tbody>
616</table>
617</section>
618</section>
619<section id="atdata.atmosphere.DatasetLoader.get_blobs" class="level3">
620<h3 class="anchored" data-anchor-id="atdata.atmosphere.DatasetLoader.get_blobs">get_blobs</h3>
621<div class="sourceCode" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a>atmosphere.DatasetLoader.get_blobs(uri)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
622<p>Get the blob references from a dataset record.</p>
623<section id="parameters-2" class="level4 doc-section doc-section-parameters">
624<h4 class="doc-section doc-section-parameters anchored" data-anchor-id="parameters-2">Parameters</h4>
625<table class="caption-top table">
626<thead>
627<tr class="header">
628<th>Name</th>
629<th>Type</th>
630<th>Description</th>
631<th>Default</th>
632</tr>
633</thead>
634<tbody>
635<tr class="odd">
636<td>uri</td>
637<td><a href="`str`">str</a> | <a href="`atdata.atmosphere._types.AtUri`">AtUri</a></td>
638<td>The AT URI of the dataset record.</td>
639<td><em>required</em></td>
640</tr>
641</tbody>
642</table>
643</section>
644<section id="returns-2" class="level4 doc-section doc-section-returns">
645<h4 class="doc-section doc-section-returns anchored" data-anchor-id="returns-2">Returns</h4>
646<table class="caption-top table">
647<thead>
648<tr class="header">
649<th>Name</th>
650<th>Type</th>
651<th>Description</th>
652</tr>
653</thead>
654<tbody>
655<tr class="odd">
656<td></td>
657<td><a href="`list`">list</a>[<a href="`dict`">dict</a>]</td>
658<td>List of blob reference dicts with keys: $type, ref, mimeType, size.</td>
659</tr>
660</tbody>
661</table>
662</section>
663<section id="raises-2" class="level4 doc-section doc-section-raises">
664<h4 class="doc-section doc-section-raises anchored" data-anchor-id="raises-2">Raises</h4>
665<table class="caption-top table">
666<thead>
667<tr class="header">
668<th>Name</th>
669<th>Type</th>
670<th>Description</th>
671</tr>
672</thead>
673<tbody>
674<tr class="odd">
675<td></td>
676<td><a href="`ValueError`">ValueError</a></td>
677<td>If the storage type is not blobs.</td>
678</tr>
679</tbody>
680</table>
681</section>
682</section>
683<section id="atdata.atmosphere.DatasetLoader.get_metadata" class="level3">
684<h3 class="anchored" data-anchor-id="atdata.atmosphere.DatasetLoader.get_metadata">get_metadata</h3>
685<div class="sourceCode" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a>atmosphere.DatasetLoader.get_metadata(uri)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
686<p>Get the metadata from a dataset record.</p>
687<section id="parameters-3" class="level4 doc-section doc-section-parameters">
688<h4 class="doc-section doc-section-parameters anchored" data-anchor-id="parameters-3">Parameters</h4>
689<table class="caption-top table">
690<thead>
691<tr class="header">
692<th>Name</th>
693<th>Type</th>
694<th>Description</th>
695<th>Default</th>
696</tr>
697</thead>
698<tbody>
699<tr class="odd">
700<td>uri</td>
701<td><a href="`str`">str</a> | <a href="`atdata.atmosphere._types.AtUri`">AtUri</a></td>
702<td>The AT URI of the dataset record.</td>
703<td><em>required</em></td>
704</tr>
705</tbody>
706</table>
707</section>
708<section id="returns-3" class="level4 doc-section doc-section-returns">
709<h4 class="doc-section doc-section-returns anchored" data-anchor-id="returns-3">Returns</h4>
710<table class="caption-top table">
711<thead>
712<tr class="header">
713<th>Name</th>
714<th>Type</th>
715<th>Description</th>
716</tr>
717</thead>
718<tbody>
719<tr class="odd">
720<td></td>
721<td><a href="`typing.Optional`">Optional</a>[<a href="`dict`">dict</a>]</td>
722<td>The metadata dictionary, or None if no metadata.</td>
723</tr>
724</tbody>
725</table>
726</section>
727</section>
728<section id="atdata.atmosphere.DatasetLoader.get_storage_type" class="level3">
729<h3 class="anchored" data-anchor-id="atdata.atmosphere.DatasetLoader.get_storage_type">get_storage_type</h3>
730<div class="sourceCode" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a>atmosphere.DatasetLoader.get_storage_type(uri)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
731<p>Get the storage type of a dataset record.</p>
732<section id="parameters-4" class="level4 doc-section doc-section-parameters">
733<h4 class="doc-section doc-section-parameters anchored" data-anchor-id="parameters-4">Parameters</h4>
734<table class="caption-top table">
735<thead>
736<tr class="header">
737<th>Name</th>
738<th>Type</th>
739<th>Description</th>
740<th>Default</th>
741</tr>
742</thead>
743<tbody>
744<tr class="odd">
745<td>uri</td>
746<td><a href="`str`">str</a> | <a href="`atdata.atmosphere._types.AtUri`">AtUri</a></td>
747<td>The AT URI of the dataset record.</td>
748<td><em>required</em></td>
749</tr>
750</tbody>
751</table>
752</section>
753<section id="returns-4" class="level4 doc-section doc-section-returns">
754<h4 class="doc-section doc-section-returns anchored" data-anchor-id="returns-4">Returns</h4>
755<table class="caption-top table">
756<thead>
757<tr class="header">
758<th>Name</th>
759<th>Type</th>
760<th>Description</th>
761</tr>
762</thead>
763<tbody>
764<tr class="odd">
765<td></td>
766<td><a href="`str`">str</a></td>
767<td>Either “external” or “blobs”.</td>
768</tr>
769</tbody>
770</table>
771</section>
772<section id="raises-3" class="level4 doc-section doc-section-raises">
773<h4 class="doc-section doc-section-raises anchored" data-anchor-id="raises-3">Raises</h4>
774<table class="caption-top table">
775<thead>
776<tr class="header">
777<th>Name</th>
778<th>Type</th>
779<th>Description</th>
780</tr>
781</thead>
782<tbody>
783<tr class="odd">
784<td></td>
785<td><a href="`ValueError`">ValueError</a></td>
786<td>If storage type is unknown.</td>
787</tr>
788</tbody>
789</table>
790</section>
791</section>
792<section id="atdata.atmosphere.DatasetLoader.get_urls" class="level3">
793<h3 class="anchored" data-anchor-id="atdata.atmosphere.DatasetLoader.get_urls">get_urls</h3>
794<div class="sourceCode" id="cb8"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a>atmosphere.DatasetLoader.get_urls(uri)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
795<p>Get the WebDataset URLs from a dataset record.</p>
796<section id="parameters-5" class="level4 doc-section doc-section-parameters">
797<h4 class="doc-section doc-section-parameters anchored" data-anchor-id="parameters-5">Parameters</h4>
798<table class="caption-top table">
799<thead>
800<tr class="header">
801<th>Name</th>
802<th>Type</th>
803<th>Description</th>
804<th>Default</th>
805</tr>
806</thead>
807<tbody>
808<tr class="odd">
809<td>uri</td>
810<td><a href="`str`">str</a> | <a href="`atdata.atmosphere._types.AtUri`">AtUri</a></td>
811<td>The AT URI of the dataset record.</td>
812<td><em>required</em></td>
813</tr>
814</tbody>
815</table>
816</section>
817<section id="returns-5" class="level4 doc-section doc-section-returns">
818<h4 class="doc-section doc-section-returns anchored" data-anchor-id="returns-5">Returns</h4>
819<table class="caption-top table">
820<thead>
821<tr class="header">
822<th>Name</th>
823<th>Type</th>
824<th>Description</th>
825</tr>
826</thead>
827<tbody>
828<tr class="odd">
829<td></td>
830<td><a href="`list`">list</a>[<a href="`str`">str</a>]</td>
831<td>List of WebDataset URLs.</td>
832</tr>
833</tbody>
834</table>
835</section>
836<section id="raises-4" class="level4 doc-section doc-section-raises">
837<h4 class="doc-section doc-section-raises anchored" data-anchor-id="raises-4">Raises</h4>
838<table class="caption-top table">
839<thead>
840<tr class="header">
841<th>Name</th>
842<th>Type</th>
843<th>Description</th>
844</tr>
845</thead>
846<tbody>
847<tr class="odd">
848<td></td>
849<td><a href="`ValueError`">ValueError</a></td>
850<td>If the storage type is not external URLs.</td>
851</tr>
852</tbody>
853</table>
854</section>
855</section>
856<section id="atdata.atmosphere.DatasetLoader.list_all" class="level3">
857<h3 class="anchored" data-anchor-id="atdata.atmosphere.DatasetLoader.list_all">list_all</h3>
858<div class="sourceCode" id="cb9"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb9-1"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a>atmosphere.DatasetLoader.list_all(repo<span class="op">=</span><span class="va">None</span>, limit<span class="op">=</span><span class="dv">100</span>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
859<p>List dataset records from a repository.</p>
860<section id="parameters-6" class="level4 doc-section doc-section-parameters">
861<h4 class="doc-section doc-section-parameters anchored" data-anchor-id="parameters-6">Parameters</h4>
862<table class="caption-top table">
863<thead>
864<tr class="header">
865<th>Name</th>
866<th>Type</th>
867<th>Description</th>
868<th>Default</th>
869</tr>
870</thead>
871<tbody>
872<tr class="odd">
873<td>repo</td>
874<td><a href="`typing.Optional`">Optional</a>[<a href="`str`">str</a>]</td>
875<td>The DID of the repository. Defaults to authenticated user.</td>
876<td><code>None</code></td>
877</tr>
878<tr class="even">
879<td>limit</td>
880<td><a href="`int`">int</a></td>
881<td>Maximum number of records to return.</td>
882<td><code>100</code></td>
883</tr>
884</tbody>
885</table>
886</section>
887<section id="returns-6" class="level4 doc-section doc-section-returns">
888<h4 class="doc-section doc-section-returns anchored" data-anchor-id="returns-6">Returns</h4>
889<table class="caption-top table">
890<thead>
891<tr class="header">
892<th>Name</th>
893<th>Type</th>
894<th>Description</th>
895</tr>
896</thead>
897<tbody>
898<tr class="odd">
899<td></td>
900<td><a href="`list`">list</a>[<a href="`dict`">dict</a>]</td>
901<td>List of dataset records.</td>
902</tr>
903</tbody>
904</table>
905</section>
906</section>
907<section id="atdata.atmosphere.DatasetLoader.to_dataset" class="level3">
908<h3 class="anchored" data-anchor-id="atdata.atmosphere.DatasetLoader.to_dataset">to_dataset</h3>
909<div class="sourceCode" id="cb10"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a>atmosphere.DatasetLoader.to_dataset(uri, sample_type)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
910<p>Create a Dataset object from an ATProto record.</p>
911<p>This method creates a Dataset instance from a published record. You must provide the sample type class, which should match the schema referenced by the record.</p>
912<p>Supports both external URL storage and ATProto blob storage.</p>
913<section id="parameters-7" class="level4 doc-section doc-section-parameters">
914<h4 class="doc-section doc-section-parameters anchored" data-anchor-id="parameters-7">Parameters</h4>
915<table class="caption-top table">
916<thead>
917<tr class="header">
918<th>Name</th>
919<th>Type</th>
920<th>Description</th>
921<th>Default</th>
922</tr>
923</thead>
924<tbody>
925<tr class="odd">
926<td>uri</td>
927<td><a href="`str`">str</a> | <a href="`atdata.atmosphere._types.AtUri`">AtUri</a></td>
928<td>The AT URI of the dataset record.</td>
929<td><em>required</em></td>
930</tr>
931<tr class="even">
932<td>sample_type</td>
933<td><a href="`typing.Type`">Type</a>[<a href="`atdata.atmosphere.records.ST`">ST</a>]</td>
934<td>The Python class for the sample type.</td>
935<td><em>required</em></td>
936</tr>
937</tbody>
938</table>
939</section>
940<section id="returns-7" class="level4 doc-section doc-section-returns">
941<h4 class="doc-section doc-section-returns anchored" data-anchor-id="returns-7">Returns</h4>
942<table class="caption-top table">
943<thead>
944<tr class="header">
945<th>Name</th>
946<th>Type</th>
947<th>Description</th>
948</tr>
949</thead>
950<tbody>
951<tr class="odd">
952<td></td>
953<td><a href="`atdata.dataset.Dataset`">Dataset</a>[<a href="`atdata.atmosphere.records.ST`">ST</a>]</td>
954<td>A Dataset instance configured from the record.</td>
955</tr>
956</tbody>
957</table>
958</section>
959<section id="raises-5" class="level4 doc-section doc-section-raises">
960<h4 class="doc-section doc-section-raises anchored" data-anchor-id="raises-5">Raises</h4>
961<table class="caption-top table">
962<thead>
963<tr class="header">
964<th>Name</th>
965<th>Type</th>
966<th>Description</th>
967</tr>
968</thead>
969<tbody>
970<tr class="odd">
971<td></td>
972<td><a href="`ValueError`">ValueError</a></td>
973<td>If no storage URLs can be resolved.</td>
974</tr>
975</tbody>
976</table>
977</section>
978<section id="examples-1" class="level4 doc-section doc-section-examples">
979<h4 class="doc-section doc-section-examples anchored" data-anchor-id="examples-1">Examples</h4>
980<div class="sourceCode" id="cb11"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb11-1"><a href="#cb11-1" aria-hidden="true" tabindex="-1"></a><span class="op">>>></span> loader <span class="op">=</span> DatasetLoader(client)</span>
981<span id="cb11-2"><a href="#cb11-2" aria-hidden="true" tabindex="-1"></a><span class="op">>>></span> dataset <span class="op">=</span> loader.to_dataset(uri, MySampleType)</span>
982<span id="cb11-3"><a href="#cb11-3" aria-hidden="true" tabindex="-1"></a><span class="op">>>></span> <span class="cf">for</span> batch <span class="kw">in</span> dataset.shuffled(batch_size<span class="op">=</span><span class="dv">32</span>):</span>
983<span id="cb11-4"><a href="#cb11-4" aria-hidden="true" tabindex="-1"></a>... process(batch)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
984
985
986</section>
987</section>
988</section>
989</section>
990
991</main> <!-- /main -->
992<script id="quarto-html-after-body" type="application/javascript">
993 window.document.addEventListener("DOMContentLoaded", function (event) {
994 // Ensure there is a toggle, if there isn't float one in the top right
995 if (window.document.querySelector('.quarto-color-scheme-toggle') === null) {
996 const a = window.document.createElement('a');
997 a.classList.add('top-right');
998 a.classList.add('quarto-color-scheme-toggle');
999 a.href = "";
1000 a.onclick = function() { try { window.quartoToggleColorScheme(); } catch {} return false; };
1001 const i = window.document.createElement("i");
1002 i.classList.add('bi');
1003 a.appendChild(i);
1004 window.document.body.appendChild(a);
1005 }
1006 setColorSchemeToggle(hasAlternateSentinel())
1007 const icon = "";
1008 const anchorJS = new window.AnchorJS();
1009 anchorJS.options = {
1010 placement: 'right',
1011 icon: icon
1012 };
1013 anchorJS.add('.anchored');
1014 const isCodeAnnotation = (el) => {
1015 for (const clz of el.classList) {
1016 if (clz.startsWith('code-annotation-')) {
1017 return true;
1018 }
1019 }
1020 return false;
1021 }
1022 const onCopySuccess = function(e) {
1023 // button target
1024 const button = e.trigger;
1025 // don't keep focus
1026 button.blur();
1027 // flash "checked"
1028 button.classList.add('code-copy-button-checked');
1029 var currentTitle = button.getAttribute("title");
1030 button.setAttribute("title", "Copied!");
1031 let tooltip;
1032 if (window.bootstrap) {
1033 button.setAttribute("data-bs-toggle", "tooltip");
1034 button.setAttribute("data-bs-placement", "left");
1035 button.setAttribute("data-bs-title", "Copied!");
1036 tooltip = new bootstrap.Tooltip(button,
1037 { trigger: "manual",
1038 customClass: "code-copy-button-tooltip",
1039 offset: [0, -8]});
1040 tooltip.show();
1041 }
1042 setTimeout(function() {
1043 if (tooltip) {
1044 tooltip.hide();
1045 button.removeAttribute("data-bs-title");
1046 button.removeAttribute("data-bs-toggle");
1047 button.removeAttribute("data-bs-placement");
1048 }
1049 button.setAttribute("title", currentTitle);
1050 button.classList.remove('code-copy-button-checked');
1051 }, 1000);
1052 // clear code selection
1053 e.clearSelection();
1054 }
1055 const getTextToCopy = function(trigger) {
1056 const codeEl = trigger.previousElementSibling.cloneNode(true);
1057 for (const childEl of codeEl.children) {
1058 if (isCodeAnnotation(childEl)) {
1059 childEl.remove();
1060 }
1061 }
1062 return codeEl.innerText;
1063 }
1064 const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
1065 text: getTextToCopy
1066 });
1067 clipboard.on('success', onCopySuccess);
1068 if (window.document.getElementById('quarto-embedded-source-code-modal')) {
1069 const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', {
1070 text: getTextToCopy,
1071 container: window.document.getElementById('quarto-embedded-source-code-modal')
1072 });
1073 clipboardModal.on('success', onCopySuccess);
1074 }
1075 var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//);
1076 var mailtoRegex = new RegExp(/^mailto:/);
1077 var filterRegex = new RegExp("https:\/\/github\.com\/your-org\/atdata");
1078 var isInternal = (href) => {
1079 return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href);
1080 }
1081 // Inspect non-navigation links and adorn them if external
1082 var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)');
1083 for (var i=0; i<links.length; i++) {
1084 const link = links[i];
1085 if (!isInternal(link.href)) {
1086 // undo the damage that might have been done by quarto-nav.js in the case of
1087 // links that we want to consider external
1088 if (link.dataset.originalHref !== undefined) {
1089 link.href = link.dataset.originalHref;
1090 }
1091 }
1092 }
1093 function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
1094 const config = {
1095 allowHTML: true,
1096 maxWidth: 500,
1097 delay: 100,
1098 arrow: false,
1099 appendTo: function(el) {
1100 return el.parentElement;
1101 },
1102 interactive: true,
1103 interactiveBorder: 10,
1104 theme: 'quarto',
1105 placement: 'bottom-start',
1106 };
1107 if (contentFn) {
1108 config.content = contentFn;
1109 }
1110 if (onTriggerFn) {
1111 config.onTrigger = onTriggerFn;
1112 }
1113 if (onUntriggerFn) {
1114 config.onUntrigger = onUntriggerFn;
1115 }
1116 window.tippy(el, config);
1117 }
1118 const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
1119 for (var i=0; i<noterefs.length; i++) {
1120 const ref = noterefs[i];
1121 tippyHover(ref, function() {
1122 // use id or data attribute instead here
1123 let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
1124 try { href = new URL(href).hash; } catch {}
1125 const id = href.replace(/^#\/?/, "");
1126 const note = window.document.getElementById(id);
1127 if (note) {
1128 return note.innerHTML;
1129 } else {
1130 return "";
1131 }
1132 });
1133 }
1134 const xrefs = window.document.querySelectorAll('a.quarto-xref');
1135 const processXRef = (id, note) => {
1136 // Strip column container classes
1137 const stripColumnClz = (el) => {
1138 el.classList.remove("page-full", "page-columns");
1139 if (el.children) {
1140 for (const child of el.children) {
1141 stripColumnClz(child);
1142 }
1143 }
1144 }
1145 stripColumnClz(note)
1146 if (id === null || id.startsWith('sec-')) {
1147 // Special case sections, only their first couple elements
1148 const container = document.createElement("div");
1149 if (note.children && note.children.length > 2) {
1150 container.appendChild(note.children[0].cloneNode(true));
1151 for (let i = 1; i < note.children.length; i++) {
1152 const child = note.children[i];
1153 if (child.tagName === "P" && child.innerText === "") {
1154 continue;
1155 } else {
1156 container.appendChild(child.cloneNode(true));
1157 break;
1158 }
1159 }
1160 if (window.Quarto?.typesetMath) {
1161 window.Quarto.typesetMath(container);
1162 }
1163 return container.innerHTML
1164 } else {
1165 if (window.Quarto?.typesetMath) {
1166 window.Quarto.typesetMath(note);
1167 }
1168 return note.innerHTML;
1169 }
1170 } else {
1171 // Remove any anchor links if they are present
1172 const anchorLink = note.querySelector('a.anchorjs-link');
1173 if (anchorLink) {
1174 anchorLink.remove();
1175 }
1176 if (window.Quarto?.typesetMath) {
1177 window.Quarto.typesetMath(note);
1178 }
1179 if (note.classList.contains("callout")) {
1180 return note.outerHTML;
1181 } else {
1182 return note.innerHTML;
1183 }
1184 }
1185 }
1186 for (var i=0; i<xrefs.length; i++) {
1187 const xref = xrefs[i];
1188 tippyHover(xref, undefined, function(instance) {
1189 instance.disable();
1190 let url = xref.getAttribute('href');
1191 let hash = undefined;
1192 if (url.startsWith('#')) {
1193 hash = url;
1194 } else {
1195 try { hash = new URL(url).hash; } catch {}
1196 }
1197 if (hash) {
1198 const id = hash.replace(/^#\/?/, "");
1199 const note = window.document.getElementById(id);
1200 if (note !== null) {
1201 try {
1202 const html = processXRef(id, note.cloneNode(true));
1203 instance.setContent(html);
1204 } finally {
1205 instance.enable();
1206 instance.show();
1207 }
1208 } else {
1209 // See if we can fetch this
1210 fetch(url.split('#')[0])
1211 .then(res => res.text())
1212 .then(html => {
1213 const parser = new DOMParser();
1214 const htmlDoc = parser.parseFromString(html, "text/html");
1215 const note = htmlDoc.getElementById(id);
1216 if (note !== null) {
1217 const html = processXRef(id, note);
1218 instance.setContent(html);
1219 }
1220 }).finally(() => {
1221 instance.enable();
1222 instance.show();
1223 });
1224 }
1225 } else {
1226 // See if we can fetch a full url (with no hash to target)
1227 // This is a special case and we should probably do some content thinning / targeting
1228 fetch(url)
1229 .then(res => res.text())
1230 .then(html => {
1231 const parser = new DOMParser();
1232 const htmlDoc = parser.parseFromString(html, "text/html");
1233 const note = htmlDoc.querySelector('main.content');
1234 if (note !== null) {
1235 // This should only happen for chapter cross references
1236 // (since there is no id in the URL)
1237 // remove the first header
1238 if (note.children.length > 0 && note.children[0].tagName === "HEADER") {
1239 note.children[0].remove();
1240 }
1241 const html = processXRef(null, note);
1242 instance.setContent(html);
1243 }
1244 }).finally(() => {
1245 instance.enable();
1246 instance.show();
1247 });
1248 }
1249 }, function(instance) {
1250 });
1251 }
1252 let selectedAnnoteEl;
1253 const selectorForAnnotation = ( cell, annotation) => {
1254 let cellAttr = 'data-code-cell="' + cell + '"';
1255 let lineAttr = 'data-code-annotation="' + annotation + '"';
1256 const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
1257 return selector;
1258 }
1259 const selectCodeLines = (annoteEl) => {
1260 const doc = window.document;
1261 const targetCell = annoteEl.getAttribute("data-target-cell");
1262 const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
1263 const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
1264 const lines = annoteSpan.getAttribute("data-code-lines").split(",");
1265 const lineIds = lines.map((line) => {
1266 return targetCell + "-" + line;
1267 })
1268 let top = null;
1269 let height = null;
1270 let parent = null;
1271 if (lineIds.length > 0) {
1272 //compute the position of the single el (top and bottom and make a div)
1273 const el = window.document.getElementById(lineIds[0]);
1274 top = el.offsetTop;
1275 height = el.offsetHeight;
1276 parent = el.parentElement.parentElement;
1277 if (lineIds.length > 1) {
1278 const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
1279 const bottom = lastEl.offsetTop + lastEl.offsetHeight;
1280 height = bottom - top;
1281 }
1282 if (top !== null && height !== null && parent !== null) {
1283 // cook up a div (if necessary) and position it
1284 let div = window.document.getElementById("code-annotation-line-highlight");
1285 if (div === null) {
1286 div = window.document.createElement("div");
1287 div.setAttribute("id", "code-annotation-line-highlight");
1288 div.style.position = 'absolute';
1289 parent.appendChild(div);
1290 }
1291 div.style.top = top - 2 + "px";
1292 div.style.height = height + 4 + "px";
1293 div.style.left = 0;
1294 let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
1295 if (gutterDiv === null) {
1296 gutterDiv = window.document.createElement("div");
1297 gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
1298 gutterDiv.style.position = 'absolute';
1299 const codeCell = window.document.getElementById(targetCell);
1300 const gutter = codeCell.querySelector('.code-annotation-gutter');
1301 gutter.appendChild(gutterDiv);
1302 }
1303 gutterDiv.style.top = top - 2 + "px";
1304 gutterDiv.style.height = height + 4 + "px";
1305 }
1306 selectedAnnoteEl = annoteEl;
1307 }
1308 };
1309 const unselectCodeLines = () => {
1310 const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
1311 elementsIds.forEach((elId) => {
1312 const div = window.document.getElementById(elId);
1313 if (div) {
1314 div.remove();
1315 }
1316 });
1317 selectedAnnoteEl = undefined;
1318 };
1319 // Handle positioning of the toggle
1320 window.addEventListener(
1321 "resize",
1322 throttle(() => {
1323 elRect = undefined;
1324 if (selectedAnnoteEl) {
1325 selectCodeLines(selectedAnnoteEl);
1326 }
1327 }, 10)
1328 );
1329 function throttle(fn, ms) {
1330 let throttle = false;
1331 let timer;
1332 return (...args) => {
1333 if(!throttle) { // first call gets through
1334 fn.apply(this, args);
1335 throttle = true;
1336 } else { // all the others get throttled
1337 if(timer) clearTimeout(timer); // cancel #2
1338 timer = setTimeout(() => {
1339 fn.apply(this, args);
1340 timer = throttle = false;
1341 }, ms);
1342 }
1343 };
1344 }
1345 // Attach click handler to the DT
1346 const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
1347 for (const annoteDlNode of annoteDls) {
1348 annoteDlNode.addEventListener('click', (event) => {
1349 const clickedEl = event.target;
1350 if (clickedEl !== selectedAnnoteEl) {
1351 unselectCodeLines();
1352 const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
1353 if (activeEl) {
1354 activeEl.classList.remove('code-annotation-active');
1355 }
1356 selectCodeLines(clickedEl);
1357 clickedEl.classList.add('code-annotation-active');
1358 } else {
1359 // Unselect the line
1360 unselectCodeLines();
1361 clickedEl.classList.remove('code-annotation-active');
1362 }
1363 });
1364 }
1365 const findCites = (el) => {
1366 const parentEl = el.parentElement;
1367 if (parentEl) {
1368 const cites = parentEl.dataset.cites;
1369 if (cites) {
1370 return {
1371 el,
1372 cites: cites.split(' ')
1373 };
1374 } else {
1375 return findCites(el.parentElement)
1376 }
1377 } else {
1378 return undefined;
1379 }
1380 };
1381 var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
1382 for (var i=0; i<bibliorefs.length; i++) {
1383 const ref = bibliorefs[i];
1384 const citeInfo = findCites(ref);
1385 if (citeInfo) {
1386 tippyHover(citeInfo.el, function() {
1387 var popup = window.document.createElement('div');
1388 citeInfo.cites.forEach(function(cite) {
1389 var citeDiv = window.document.createElement('div');
1390 citeDiv.classList.add('hanging-indent');
1391 citeDiv.classList.add('csl-entry');
1392 var biblioDiv = window.document.getElementById('ref-' + cite);
1393 if (biblioDiv) {
1394 citeDiv.innerHTML = biblioDiv.innerHTML;
1395 }
1396 popup.appendChild(citeDiv);
1397 });
1398 return popup.innerHTML;
1399 });
1400 }
1401 }
1402 });
1403 </script>
1404</div> <!-- /content -->
1405<footer class="footer">
1406 <div class="nav-footer">
1407 <div class="nav-footer-left">
1408<p>Built with <a href="https://quarto.org/">Quarto</a></p>
1409</div>
1410 <div class="nav-footer-center">
1411
1412 <div class="toc-actions d-sm-block d-md-none"><ul><li><a href="https://github.com/your-org/atdata/edit/main/api/DatasetLoader.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></div>
1413 <div class="nav-footer-right">
1414<p>MIT License</p>
1415</div>
1416 </div>
1417</footer>
1418
1419
1420
1421
1422</body></html>