A loose federation of distributed, typed datasets
1<!DOCTYPE html>
2<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
3
4<meta charset="utf-8">
5<meta name="generator" content="quarto-1.7.34">
6
7<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
8
9
10<title>s3source – atdata</title>
11<style>
12code{white-space: pre-wrap;}
13span.smallcaps{font-variant: small-caps;}
14div.columns{display: flex; gap: min(4vw, 1.5em);}
15div.column{flex: auto; overflow-x: auto;}
16div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
17ul.task-list{list-style: none;}
18ul.task-list li input[type="checkbox"] {
19 width: 0.8em;
20 margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
21 vertical-align: middle;
22}
23/* CSS for syntax highlighting */
24html { -webkit-text-size-adjust: 100%; }
25pre > code.sourceCode { white-space: pre; position: relative; }
26pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
27pre > code.sourceCode > span:empty { height: 1.2em; }
28.sourceCode { overflow: visible; }
29code.sourceCode > span { color: inherit; text-decoration: inherit; }
30div.sourceCode { margin: 1em 0; }
31pre.sourceCode { margin: 0; }
32@media screen {
33div.sourceCode { overflow: auto; }
34}
35@media print {
36pre > code.sourceCode { white-space: pre-wrap; }
37pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
38}
39pre.numberSource code
40 { counter-reset: source-line 0; }
41pre.numberSource code > span
42 { position: relative; left: -4em; counter-increment: source-line; }
43pre.numberSource code > span > a:first-child::before
44 { content: counter(source-line);
45 position: relative; left: -1em; text-align: right; vertical-align: baseline;
46 border: none; display: inline-block;
47 -webkit-touch-callout: none; -webkit-user-select: none;
48 -khtml-user-select: none; -moz-user-select: none;
49 -ms-user-select: none; user-select: none;
50 padding: 0 4px; width: 4em;
51 }
52pre.numberSource { margin-left: 3em; padding-left: 4px; }
53div.sourceCode
54 { }
55@media screen {
56pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
57}
58</style>
59
60
61<script src="../site_libs/quarto-nav/quarto-nav.js"></script>
62<script src="../site_libs/quarto-nav/headroom.min.js"></script>
63<script src="../site_libs/clipboard/clipboard.min.js"></script>
64<script src="../site_libs/quarto-search/autocomplete.umd.js"></script>
65<script src="../site_libs/quarto-search/fuse.min.js"></script>
66<script src="../site_libs/quarto-search/quarto-search.js"></script>
67<meta name="quarto:offset" content="../">
68<script src="../site_libs/quarto-html/quarto.js" type="module"></script>
69<script src="../site_libs/quarto-html/tabsets/tabsets.js" type="module"></script>
70<script src="../site_libs/quarto-html/popper.min.js"></script>
71<script src="../site_libs/quarto-html/tippy.umd.min.js"></script>
72<script src="../site_libs/quarto-html/anchor.min.js"></script>
73<link href="../site_libs/quarto-html/tippy.css" rel="stylesheet">
74<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme" id="quarto-text-highlighting-styles">
75<link href="../site_libs/quarto-html/quarto-syntax-highlighting-dark-8dcd8563ea6803ab7cbb3d71ca5772e1.css" rel="stylesheet" class="quarto-color-scheme quarto-color-alternate" id="quarto-text-highlighting-styles">
76<link href="../site_libs/quarto-html/quarto-syntax-highlighting-9582434199d49cc9e91654cdeeb4866b.css" rel="stylesheet" class="quarto-color-scheme-extra" id="quarto-text-highlighting-styles">
77<script src="../site_libs/bootstrap/bootstrap.min.js"></script>
78<link href="../site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
79<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme" id="quarto-bootstrap" data-mode="light">
80<link href="../site_libs/bootstrap/bootstrap-dark-7964ffd8887b0991fe8d71c6c8bc75d6.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme quarto-color-alternate" id="quarto-bootstrap" data-mode="dark">
81<link href="../site_libs/bootstrap/bootstrap-62bce24ca844314e7bb1a34dbdfe05cc.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme-extra" id="quarto-bootstrap" data-mode="light">
82<script id="quarto-search-options" type="application/json">{
83 "location": "navbar",
84 "copy-button": false,
85 "collapse-after": 3,
86 "panel-placement": "end",
87 "type": "overlay",
88 "limit": 50,
89 "keyboard-shortcut": [
90 "f",
91 "/",
92 "s"
93 ],
94 "show-item-context": false,
95 "language": {
96 "search-no-results-text": "No results",
97 "search-matching-documents-text": "matching documents",
98 "search-copy-link-title": "Copy link to search",
99 "search-hide-matches-text": "Hide additional matches",
100 "search-more-match-text": "more match in this document",
101 "search-more-matches-text": "more matches in this document",
102 "search-clear-button-title": "Clear",
103 "search-text-placeholder": "",
104 "search-detached-cancel-button-title": "Cancel",
105 "search-submit-button-title": "Submit",
106 "search-label": "Search"
107 }
108}</script>
109
110
111<link rel="stylesheet" href="../assets/styles.css">
112</head>
113
114<body class="nav-fixed quarto-light"><script id="quarto-html-before-body" type="application/javascript">
115 const toggleBodyColorMode = (bsSheetEl) => {
116 const mode = bsSheetEl.getAttribute("data-mode");
117 const bodyEl = window.document.querySelector("body");
118 if (mode === "dark") {
119 bodyEl.classList.add("quarto-dark");
120 bodyEl.classList.remove("quarto-light");
121 } else {
122 bodyEl.classList.add("quarto-light");
123 bodyEl.classList.remove("quarto-dark");
124 }
125 }
126 const toggleBodyColorPrimary = () => {
127 const bsSheetEl = window.document.querySelector("link#quarto-bootstrap:not([rel=disabled-stylesheet])");
128 if (bsSheetEl) {
129 toggleBodyColorMode(bsSheetEl);
130 }
131 }
132 const setColorSchemeToggle = (alternate) => {
133 const toggles = window.document.querySelectorAll('.quarto-color-scheme-toggle');
134 for (let i=0; i < toggles.length; i++) {
135 const toggle = toggles[i];
136 if (toggle) {
137 if (alternate) {
138 toggle.classList.add("alternate");
139 } else {
140 toggle.classList.remove("alternate");
141 }
142 }
143 }
144 };
145 const toggleColorMode = (alternate) => {
146 // Switch the stylesheets
147 const primaryStylesheets = window.document.querySelectorAll('link.quarto-color-scheme:not(.quarto-color-alternate)');
148 const alternateStylesheets = window.document.querySelectorAll('link.quarto-color-scheme.quarto-color-alternate');
149 manageTransitions('#quarto-margin-sidebar .nav-link', false);
150 if (alternate) {
151 // note: dark is layered on light, we don't disable primary!
152 enableStylesheet(alternateStylesheets);
153 for (const sheetNode of alternateStylesheets) {
154 if (sheetNode.id === "quarto-bootstrap") {
155 toggleBodyColorMode(sheetNode);
156 }
157 }
158 } else {
159 disableStylesheet(alternateStylesheets);
160 enableStylesheet(primaryStylesheets)
161 toggleBodyColorPrimary();
162 }
163 manageTransitions('#quarto-margin-sidebar .nav-link', true);
164 // Switch the toggles
165 setColorSchemeToggle(alternate)
166 // Hack to workaround the fact that safari doesn't
167 // properly recolor the scrollbar when toggling (#1455)
168 if (navigator.userAgent.indexOf('Safari') > 0 && navigator.userAgent.indexOf('Chrome') == -1) {
169 manageTransitions("body", false);
170 window.scrollTo(0, 1);
171 setTimeout(() => {
172 window.scrollTo(0, 0);
173 manageTransitions("body", true);
174 }, 40);
175 }
176 }
177 const disableStylesheet = (stylesheets) => {
178 for (let i=0; i < stylesheets.length; i++) {
179 const stylesheet = stylesheets[i];
180 stylesheet.rel = 'disabled-stylesheet';
181 }
182 }
183 const enableStylesheet = (stylesheets) => {
184 for (let i=0; i < stylesheets.length; i++) {
185 const stylesheet = stylesheets[i];
186 if(stylesheet.rel !== 'stylesheet') { // for Chrome, which will still FOUC without this check
187 stylesheet.rel = 'stylesheet';
188 }
189 }
190 }
191 const manageTransitions = (selector, allowTransitions) => {
192 const els = window.document.querySelectorAll(selector);
193 for (let i=0; i < els.length; i++) {
194 const el = els[i];
195 if (allowTransitions) {
196 el.classList.remove('notransition');
197 } else {
198 el.classList.add('notransition');
199 }
200 }
201 }
202 const isFileUrl = () => {
203 return window.location.protocol === 'file:';
204 }
205 const hasAlternateSentinel = () => {
206 let styleSentinel = getColorSchemeSentinel();
207 if (styleSentinel !== null) {
208 return styleSentinel === "alternate";
209 } else {
210 return false;
211 }
212 }
213 const setStyleSentinel = (alternate) => {
214 const value = alternate ? "alternate" : "default";
215 if (!isFileUrl()) {
216 window.localStorage.setItem("quarto-color-scheme", value);
217 } else {
218 localAlternateSentinel = value;
219 }
220 }
221 const getColorSchemeSentinel = () => {
222 if (!isFileUrl()) {
223 const storageValue = window.localStorage.getItem("quarto-color-scheme");
224 return storageValue != null ? storageValue : localAlternateSentinel;
225 } else {
226 return localAlternateSentinel;
227 }
228 }
229 const toggleGiscusIfUsed = (isAlternate, darkModeDefault) => {
230 const baseTheme = document.querySelector('#giscus-base-theme')?.value ?? 'light';
231 const alternateTheme = document.querySelector('#giscus-alt-theme')?.value ?? 'dark';
232 let newTheme = '';
233 if(authorPrefersDark) {
234 newTheme = isAlternate ? baseTheme : alternateTheme;
235 } else {
236 newTheme = isAlternate ? alternateTheme : baseTheme;
237 }
238 const changeGiscusTheme = () => {
239 // From: https://github.com/giscus/giscus/issues/336
240 const sendMessage = (message) => {
241 const iframe = document.querySelector('iframe.giscus-frame');
242 if (!iframe) return;
243 iframe.contentWindow.postMessage({ giscus: message }, 'https://giscus.app');
244 }
245 sendMessage({
246 setConfig: {
247 theme: newTheme
248 }
249 });
250 }
251 const isGiscussLoaded = window.document.querySelector('iframe.giscus-frame') !== null;
252 if (isGiscussLoaded) {
253 changeGiscusTheme();
254 }
255 };
256 const authorPrefersDark = false;
257 const darkModeDefault = authorPrefersDark;
258 document.querySelector('link#quarto-text-highlighting-styles.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
259 document.querySelector('link#quarto-bootstrap.quarto-color-scheme-extra').rel = 'disabled-stylesheet';
260 let localAlternateSentinel = darkModeDefault ? 'alternate' : 'default';
261 // Dark / light mode switch
262 window.quartoToggleColorScheme = () => {
263 // Read the current dark / light value
264 let toAlternate = !hasAlternateSentinel();
265 toggleColorMode(toAlternate);
266 setStyleSentinel(toAlternate);
267 toggleGiscusIfUsed(toAlternate, darkModeDefault);
268 window.dispatchEvent(new Event('resize'));
269 };
270 // Switch to dark mode if need be
271 if (hasAlternateSentinel()) {
272 toggleColorMode(true);
273 } else {
274 toggleColorMode(false);
275 }
276 </script>
277
278<div id="quarto-search-results"></div>
279 <header id="quarto-header" class="headroom fixed-top">
280 <nav class="navbar navbar-expand-lg " data-bs-theme="dark">
281 <div class="navbar-container container-fluid">
282 <div class="navbar-brand-container mx-auto">
283 <a class="navbar-brand" href="../index.html">
284 <span class="navbar-title">atdata</span>
285 </a>
286 </div>
287 <div id="quarto-search" class="" title="Search"></div>
288 <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarCollapse" aria-controls="navbarCollapse" role="menu" aria-expanded="false" aria-label="Toggle navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
289 <span class="navbar-toggler-icon"></span>
290</button>
291 <div class="collapse navbar-collapse" id="navbarCollapse">
292 <ul class="navbar-nav navbar-nav-scroll me-auto">
293 <li class="nav-item">
294 <a class="nav-link" href="../index.html">
295<span class="menu-text">Guide</span></a>
296 </li>
297 <li class="nav-item dropdown ">
298 <a class="nav-link dropdown-toggle" href="#" id="nav-menu-tutorials" role="link" data-bs-toggle="dropdown" aria-expanded="false">
299 <span class="menu-text">Tutorials</span>
300 </a>
301 <ul class="dropdown-menu" aria-labelledby="nav-menu-tutorials">
302 <li>
303 <a class="dropdown-item" href="../tutorials/quickstart.html">
304 <span class="dropdown-text">Quick Start</span></a>
305 </li>
306 <li>
307 <a class="dropdown-item" href="../tutorials/local-workflow.html">
308 <span class="dropdown-text">Local Workflow</span></a>
309 </li>
310 <li>
311 <a class="dropdown-item" href="../tutorials/atmosphere.html">
312 <span class="dropdown-text">Atmosphere Publishing</span></a>
313 </li>
314 <li>
315 <a class="dropdown-item" href="../tutorials/promotion.html">
316 <span class="dropdown-text">Promotion Workflow</span></a>
317 </li>
318 </ul>
319 </li>
320 <li class="nav-item dropdown ">
321 <a class="nav-link dropdown-toggle" href="#" id="nav-menu-reference" role="link" data-bs-toggle="dropdown" aria-expanded="false">
322 <span class="menu-text">Reference</span>
323 </a>
324 <ul class="dropdown-menu" aria-labelledby="nav-menu-reference">
325 <li>
326 <a class="dropdown-item" href="../reference/architecture.html">
327 <span class="dropdown-text">Architecture Overview</span></a>
328 </li>
329 <li>
330 <a class="dropdown-item" href="../reference/packable-samples.html">
331 <span class="dropdown-text">Packable Samples</span></a>
332 </li>
333 <li>
334 <a class="dropdown-item" href="../reference/datasets.html">
335 <span class="dropdown-text">Datasets</span></a>
336 </li>
337 <li>
338 <a class="dropdown-item" href="../reference/lenses.html">
339 <span class="dropdown-text">Lenses</span></a>
340 </li>
341 <li>
342 <a class="dropdown-item" href="../reference/local-storage.html">
343 <span class="dropdown-text">Local Storage</span></a>
344 </li>
345 <li>
346 <a class="dropdown-item" href="../reference/atmosphere.html">
347 <span class="dropdown-text">Atmosphere</span></a>
348 </li>
349 <li>
350 <a class="dropdown-item" href="../reference/promotion.html">
351 <span class="dropdown-text">Promotion</span></a>
352 </li>
353 <li>
354 <a class="dropdown-item" href="../reference/load-dataset.html">
355 <span class="dropdown-text">load_dataset API</span></a>
356 </li>
357 <li>
358 <a class="dropdown-item" href="../reference/protocols.html">
359 <span class="dropdown-text">Protocols</span></a>
360 </li>
361 <li>
362 <a class="dropdown-item" href="../reference/uri-spec.html">
363 <span class="dropdown-text">URI Specification</span></a>
364 </li>
365 <li>
366 <a class="dropdown-item" href="../reference/troubleshooting.html">
367 <span class="dropdown-text">Troubleshooting & FAQ</span></a>
368 </li>
369 <li>
370 <a class="dropdown-item" href="../reference/deployment.html">
371 <span class="dropdown-text">Deployment Guide</span></a>
372 </li>
373 </ul>
374 </li>
375 <li class="nav-item">
376 <a class="nav-link" href="../api/index.html">
377<span class="menu-text">API</span></a>
378 </li>
379</ul>
380 <ul class="navbar-nav navbar-nav-scroll ms-auto">
381 <li class="nav-item compact">
382 <a class="nav-link" href="https://github.com/your-org/atdata"> <i class="bi bi-github" role="img">
383</i>
384<span class="menu-text"></span></a>
385 </li>
386</ul>
387 </div> <!-- /navcollapse -->
388 <div class="quarto-navbar-tools">
389 <a href="" class="quarto-color-scheme-toggle quarto-navigation-tool px-1" onclick="window.quartoToggleColorScheme(); return false;" title="Toggle dark mode"><i class="bi"></i></a>
390</div>
391 </div> <!-- /container-fluid -->
392 </nav>
393</header>
394<!-- content -->
395<div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-article page-navbar">
396<!-- sidebar -->
397<!-- margin-sidebar -->
398 <div id="quarto-margin-sidebar" class="sidebar margin-sidebar">
399 <nav id="TOC" role="doc-toc" class="toc-active">
400 <h2 id="toc-title">On this page</h2>
401
402 <ul>
403 <li><a href="#atdata.S3Source" id="toc-atdata.S3Source" class="nav-link active" data-scroll-target="#atdata.S3Source">S3Source</a>
404 <ul class="collapse">
405 <li><a href="#attributes" id="toc-attributes" class="nav-link" data-scroll-target="#attributes">Attributes</a></li>
406 <li><a href="#examples" id="toc-examples" class="nav-link" data-scroll-target="#examples">Examples</a></li>
407 <li><a href="#methods" id="toc-methods" class="nav-link" data-scroll-target="#methods">Methods</a>
408 <ul class="collapse">
409 <li><a href="#atdata.S3Source.from_credentials" id="toc-atdata.S3Source.from_credentials" class="nav-link" data-scroll-target="#atdata.S3Source.from_credentials">from_credentials</a></li>
410 <li><a href="#atdata.S3Source.from_urls" id="toc-atdata.S3Source.from_urls" class="nav-link" data-scroll-target="#atdata.S3Source.from_urls">from_urls</a></li>
411 <li><a href="#atdata.S3Source.list_shards" id="toc-atdata.S3Source.list_shards" class="nav-link" data-scroll-target="#atdata.S3Source.list_shards">list_shards</a></li>
412 <li><a href="#atdata.S3Source.open_shard" id="toc-atdata.S3Source.open_shard" class="nav-link" data-scroll-target="#atdata.S3Source.open_shard">open_shard</a></li>
413 </ul></li>
414 </ul></li>
415 </ul>
416<div class="toc-actions"><ul><li><a href="https://github.com/your-org/atdata/edit/main/api/S3Source.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></nav>
417 </div>
418<!-- main -->
419<main class="content" id="quarto-document-content"><header id="title-block-header" class="quarto-title-block"></header>
420
421
422
423
424
425<section id="atdata.S3Source" class="level1">
426<h1>S3Source</h1>
427<div class="sourceCode" id="cb1"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a>S3Source(</span>
428<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a> bucket,</span>
429<span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a> keys,</span>
430<span id="cb1-4"><a href="#cb1-4" aria-hidden="true" tabindex="-1"></a> endpoint<span class="op">=</span><span class="va">None</span>,</span>
431<span id="cb1-5"><a href="#cb1-5" aria-hidden="true" tabindex="-1"></a> access_key<span class="op">=</span><span class="va">None</span>,</span>
432<span id="cb1-6"><a href="#cb1-6" aria-hidden="true" tabindex="-1"></a> secret_key<span class="op">=</span><span class="va">None</span>,</span>
433<span id="cb1-7"><a href="#cb1-7" aria-hidden="true" tabindex="-1"></a> region<span class="op">=</span><span class="va">None</span>,</span>
434<span id="cb1-8"><a href="#cb1-8" aria-hidden="true" tabindex="-1"></a> _client<span class="op">=</span><span class="va">None</span>,</span>
435<span id="cb1-9"><a href="#cb1-9" aria-hidden="true" tabindex="-1"></a>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
436<p>Data source for S3-compatible storage with explicit credentials.</p>
437<p>Uses boto3 to stream directly from S3, supporting: - Standard AWS S3 - S3-compatible endpoints (Cloudflare R2, MinIO, etc.) - Private buckets with credentials - IAM role authentication (when keys not provided)</p>
438<p>Unlike URL-based approaches, this doesn’t require URL transformation or global gopen_schemes registration. Credentials are scoped to the source instance.</p>
439<section id="attributes" class="level2 doc-section doc-section-attributes">
440<h2 class="doc-section doc-section-attributes anchored" data-anchor-id="attributes">Attributes</h2>
441<table class="caption-top table">
442<thead>
443<tr class="header">
444<th>Name</th>
445<th>Type</th>
446<th>Description</th>
447</tr>
448</thead>
449<tbody>
450<tr class="odd">
451<td>bucket</td>
452<td><a href="`str`">str</a></td>
453<td>S3 bucket name.</td>
454</tr>
455<tr class="even">
456<td>keys</td>
457<td><a href="`list`">list</a>[<a href="`str`">str</a>]</td>
458<td>List of object keys (paths within bucket).</td>
459</tr>
460<tr class="odd">
461<td>endpoint</td>
462<td><a href="`str`">str</a> | None</td>
463<td>Optional custom endpoint URL for S3-compatible services.</td>
464</tr>
465<tr class="even">
466<td>access_key</td>
467<td><a href="`str`">str</a> | None</td>
468<td>Optional AWS access key ID.</td>
469</tr>
470<tr class="odd">
471<td>secret_key</td>
472<td><a href="`str`">str</a> | None</td>
473<td>Optional AWS secret access key.</td>
474</tr>
475<tr class="even">
476<td>region</td>
477<td><a href="`str`">str</a> | None</td>
478<td>Optional AWS region (defaults to us-east-1).</td>
479</tr>
480</tbody>
481</table>
482</section>
483<section id="examples" class="level2 doc-section doc-section-examples">
484<h2 class="doc-section doc-section-examples anchored" data-anchor-id="examples">Examples</h2>
485<div class="sourceCode" id="cb2"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="op">>>></span> source <span class="op">=</span> S3Source(</span>
486<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a>... bucket<span class="op">=</span><span class="st">"my-datasets"</span>,</span>
487<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a>... keys<span class="op">=</span>[<span class="st">"train/shard-000.tar"</span>, <span class="st">"train/shard-001.tar"</span>],</span>
488<span id="cb2-4"><a href="#cb2-4" aria-hidden="true" tabindex="-1"></a>... endpoint<span class="op">=</span><span class="st">"https://abc123.r2.cloudflarestorage.com"</span>,</span>
489<span id="cb2-5"><a href="#cb2-5" aria-hidden="true" tabindex="-1"></a>... access_key<span class="op">=</span><span class="st">"AKIAIOSFODNN7EXAMPLE"</span>,</span>
490<span id="cb2-6"><a href="#cb2-6" aria-hidden="true" tabindex="-1"></a>... secret_key<span class="op">=</span><span class="st">"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"</span>,</span>
491<span id="cb2-7"><a href="#cb2-7" aria-hidden="true" tabindex="-1"></a>... )</span>
492<span id="cb2-8"><a href="#cb2-8" aria-hidden="true" tabindex="-1"></a><span class="op">>>></span> <span class="cf">for</span> shard_id, stream <span class="kw">in</span> source.shards:</span>
493<span id="cb2-9"><a href="#cb2-9" aria-hidden="true" tabindex="-1"></a>... process(stream)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
494</section>
495<section id="methods" class="level2">
496<h2 class="anchored" data-anchor-id="methods">Methods</h2>
497<table class="caption-top table">
498<thead>
499<tr class="header">
500<th>Name</th>
501<th>Description</th>
502</tr>
503</thead>
504<tbody>
505<tr class="odd">
506<td><a href="#atdata.S3Source.from_credentials">from_credentials</a></td>
507<td>Create S3Source from a credentials dictionary.</td>
508</tr>
509<tr class="even">
510<td><a href="#atdata.S3Source.from_urls">from_urls</a></td>
511<td>Create S3Source from s3:// URLs.</td>
512</tr>
513<tr class="odd">
514<td><a href="#atdata.S3Source.list_shards">list_shards</a></td>
515<td>Return list of S3 URIs for the shards.</td>
516</tr>
517<tr class="even">
518<td><a href="#atdata.S3Source.open_shard">open_shard</a></td>
519<td>Open a single shard by S3 URI.</td>
520</tr>
521</tbody>
522</table>
523<section id="atdata.S3Source.from_credentials" class="level3">
524<h3 class="anchored" data-anchor-id="atdata.S3Source.from_credentials">from_credentials</h3>
525<div class="sourceCode" id="cb3"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a>S3Source.from_credentials(credentials, bucket, keys)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
526<p>Create S3Source from a credentials dictionary.</p>
527<p>Accepts the same credential format used by S3DataStore.</p>
528<section id="parameters" class="level4 doc-section doc-section-parameters">
529<h4 class="doc-section doc-section-parameters anchored" data-anchor-id="parameters">Parameters</h4>
530<table class="caption-top table">
531<thead>
532<tr class="header">
533<th>Name</th>
534<th>Type</th>
535<th>Description</th>
536<th>Default</th>
537</tr>
538</thead>
539<tbody>
540<tr class="odd">
541<td>credentials</td>
542<td><a href="`dict`">dict</a>[<a href="`str`">str</a>, <a href="`str`">str</a>]</td>
543<td>Dict with AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, and optionally AWS_ENDPOINT.</td>
544<td><em>required</em></td>
545</tr>
546<tr class="even">
547<td>bucket</td>
548<td><a href="`str`">str</a></td>
549<td>S3 bucket name.</td>
550<td><em>required</em></td>
551</tr>
552<tr class="odd">
553<td>keys</td>
554<td><a href="`list`">list</a>[<a href="`str`">str</a>]</td>
555<td>List of object keys.</td>
556<td><em>required</em></td>
557</tr>
558</tbody>
559</table>
560</section>
561<section id="returns" class="level4 doc-section doc-section-returns">
562<h4 class="doc-section doc-section-returns anchored" data-anchor-id="returns">Returns</h4>
563<table class="caption-top table">
564<thead>
565<tr class="header">
566<th>Name</th>
567<th>Type</th>
568<th>Description</th>
569</tr>
570</thead>
571<tbody>
572<tr class="odd">
573<td></td>
574<td>'S3Source'</td>
575<td>Configured S3Source.</td>
576</tr>
577</tbody>
578</table>
579</section>
580<section id="examples-1" class="level4 doc-section doc-section-examples">
581<h4 class="doc-section doc-section-examples anchored" data-anchor-id="examples-1">Examples</h4>
582<div class="sourceCode" id="cb4"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="op">>>></span> creds <span class="op">=</span> {</span>
583<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a>... <span class="st">"AWS_ACCESS_KEY_ID"</span>: <span class="st">"..."</span>,</span>
584<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a>... <span class="st">"AWS_SECRET_ACCESS_KEY"</span>: <span class="st">"..."</span>,</span>
585<span id="cb4-4"><a href="#cb4-4" aria-hidden="true" tabindex="-1"></a>... <span class="st">"AWS_ENDPOINT"</span>: <span class="st">"https://r2.example.com"</span>,</span>
586<span id="cb4-5"><a href="#cb4-5" aria-hidden="true" tabindex="-1"></a>... }</span>
587<span id="cb4-6"><a href="#cb4-6" aria-hidden="true" tabindex="-1"></a><span class="op">>>></span> source <span class="op">=</span> S3Source.from_credentials(creds, <span class="st">"my-bucket"</span>, [<span class="st">"data.tar"</span>])</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
588</section>
589</section>
590<section id="atdata.S3Source.from_urls" class="level3">
591<h3 class="anchored" data-anchor-id="atdata.S3Source.from_urls">from_urls</h3>
592<div class="sourceCode" id="cb5"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a>S3Source.from_urls(</span>
593<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a> urls,</span>
594<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a> <span class="op">*</span>,</span>
595<span id="cb5-4"><a href="#cb5-4" aria-hidden="true" tabindex="-1"></a> endpoint<span class="op">=</span><span class="va">None</span>,</span>
596<span id="cb5-5"><a href="#cb5-5" aria-hidden="true" tabindex="-1"></a> access_key<span class="op">=</span><span class="va">None</span>,</span>
597<span id="cb5-6"><a href="#cb5-6" aria-hidden="true" tabindex="-1"></a> secret_key<span class="op">=</span><span class="va">None</span>,</span>
598<span id="cb5-7"><a href="#cb5-7" aria-hidden="true" tabindex="-1"></a> region<span class="op">=</span><span class="va">None</span>,</span>
599<span id="cb5-8"><a href="#cb5-8" aria-hidden="true" tabindex="-1"></a>)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
600<p>Create S3Source from s3:// URLs.</p>
601<p>Parses s3://bucket/key URLs and extracts bucket and keys. All URLs must be in the same bucket.</p>
602<section id="parameters-1" class="level4 doc-section doc-section-parameters">
603<h4 class="doc-section doc-section-parameters anchored" data-anchor-id="parameters-1">Parameters</h4>
604<table class="caption-top table">
605<thead>
606<tr class="header">
607<th>Name</th>
608<th>Type</th>
609<th>Description</th>
610<th>Default</th>
611</tr>
612</thead>
613<tbody>
614<tr class="odd">
615<td>urls</td>
616<td><a href="`list`">list</a>[<a href="`str`">str</a>]</td>
617<td>List of s3:// URLs.</td>
618<td><em>required</em></td>
619</tr>
620<tr class="even">
621<td>endpoint</td>
622<td><a href="`str`">str</a> | None</td>
623<td>Optional custom endpoint.</td>
624<td><code>None</code></td>
625</tr>
626<tr class="odd">
627<td>access_key</td>
628<td><a href="`str`">str</a> | None</td>
629<td>Optional access key.</td>
630<td><code>None</code></td>
631</tr>
632<tr class="even">
633<td>secret_key</td>
634<td><a href="`str`">str</a> | None</td>
635<td>Optional secret key.</td>
636<td><code>None</code></td>
637</tr>
638<tr class="odd">
639<td>region</td>
640<td><a href="`str`">str</a> | None</td>
641<td>Optional region.</td>
642<td><code>None</code></td>
643</tr>
644</tbody>
645</table>
646</section>
647<section id="returns-1" class="level4 doc-section doc-section-returns">
648<h4 class="doc-section doc-section-returns anchored" data-anchor-id="returns-1">Returns</h4>
649<table class="caption-top table">
650<thead>
651<tr class="header">
652<th>Name</th>
653<th>Type</th>
654<th>Description</th>
655</tr>
656</thead>
657<tbody>
658<tr class="odd">
659<td></td>
660<td>'S3Source'</td>
661<td>S3Source configured for the given URLs.</td>
662</tr>
663</tbody>
664</table>
665</section>
666<section id="raises" class="level4 doc-section doc-section-raises">
667<h4 class="doc-section doc-section-raises anchored" data-anchor-id="raises">Raises</h4>
668<table class="caption-top table">
669<thead>
670<tr class="header">
671<th>Name</th>
672<th>Type</th>
673<th>Description</th>
674</tr>
675</thead>
676<tbody>
677<tr class="odd">
678<td></td>
679<td><a href="`ValueError`">ValueError</a></td>
680<td>If URLs are not valid s3:// URLs or span multiple buckets.</td>
681</tr>
682</tbody>
683</table>
684</section>
685<section id="examples-2" class="level4 doc-section doc-section-examples">
686<h4 class="doc-section doc-section-examples anchored" data-anchor-id="examples-2">Examples</h4>
687<div class="sourceCode" id="cb6"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="op">>>></span> source <span class="op">=</span> S3Source.from_urls(</span>
688<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a>... [<span class="st">"s3://my-bucket/train-000.tar"</span>, <span class="st">"s3://my-bucket/train-001.tar"</span>],</span>
689<span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a>... endpoint<span class="op">=</span><span class="st">"https://r2.example.com"</span>,</span>
690<span id="cb6-4"><a href="#cb6-4" aria-hidden="true" tabindex="-1"></a>... )</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
691</section>
692</section>
693<section id="atdata.S3Source.list_shards" class="level3">
694<h3 class="anchored" data-anchor-id="atdata.S3Source.list_shards">list_shards</h3>
695<div class="sourceCode" id="cb7"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a>S3Source.list_shards()</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
696<p>Return list of S3 URIs for the shards.</p>
697</section>
698<section id="atdata.S3Source.open_shard" class="level3">
699<h3 class="anchored" data-anchor-id="atdata.S3Source.open_shard">open_shard</h3>
700<div class="sourceCode" id="cb8"><pre class="sourceCode python code-with-copy"><code class="sourceCode python"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a>S3Source.open_shard(shard_id)</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
701<p>Open a single shard by S3 URI.</p>
702<section id="parameters-2" class="level4 doc-section doc-section-parameters">
703<h4 class="doc-section doc-section-parameters anchored" data-anchor-id="parameters-2">Parameters</h4>
704<table class="caption-top table">
705<thead>
706<tr class="header">
707<th>Name</th>
708<th>Type</th>
709<th>Description</th>
710<th>Default</th>
711</tr>
712</thead>
713<tbody>
714<tr class="odd">
715<td>shard_id</td>
716<td><a href="`str`">str</a></td>
717<td>S3 URI of the shard (s3://bucket/key).</td>
718<td><em>required</em></td>
719</tr>
720</tbody>
721</table>
722</section>
723<section id="returns-2" class="level4 doc-section doc-section-returns">
724<h4 class="doc-section doc-section-returns anchored" data-anchor-id="returns-2">Returns</h4>
725<table class="caption-top table">
726<thead>
727<tr class="header">
728<th>Name</th>
729<th>Type</th>
730<th>Description</th>
731</tr>
732</thead>
733<tbody>
734<tr class="odd">
735<td></td>
736<td><a href="`typing.IO`">IO</a>[<a href="`bytes`">bytes</a>]</td>
737<td>StreamingBody for reading the object.</td>
738</tr>
739</tbody>
740</table>
741</section>
742<section id="raises-1" class="level4 doc-section doc-section-raises">
743<h4 class="doc-section doc-section-raises anchored" data-anchor-id="raises-1">Raises</h4>
744<table class="caption-top table">
745<thead>
746<tr class="header">
747<th>Name</th>
748<th>Type</th>
749<th>Description</th>
750</tr>
751</thead>
752<tbody>
753<tr class="odd">
754<td></td>
755<td><a href="`KeyError`">KeyError</a></td>
756<td>If shard_id is not in list_shards().</td>
757</tr>
758</tbody>
759</table>
760
761
762</section>
763</section>
764</section>
765</section>
766
767</main> <!-- /main -->
768<script id="quarto-html-after-body" type="application/javascript">
769 window.document.addEventListener("DOMContentLoaded", function (event) {
770 // Ensure there is a toggle, if there isn't float one in the top right
771 if (window.document.querySelector('.quarto-color-scheme-toggle') === null) {
772 const a = window.document.createElement('a');
773 a.classList.add('top-right');
774 a.classList.add('quarto-color-scheme-toggle');
775 a.href = "";
776 a.onclick = function() { try { window.quartoToggleColorScheme(); } catch {} return false; };
777 const i = window.document.createElement("i");
778 i.classList.add('bi');
779 a.appendChild(i);
780 window.document.body.appendChild(a);
781 }
782 setColorSchemeToggle(hasAlternateSentinel())
783 const icon = "";
784 const anchorJS = new window.AnchorJS();
785 anchorJS.options = {
786 placement: 'right',
787 icon: icon
788 };
789 anchorJS.add('.anchored');
790 const isCodeAnnotation = (el) => {
791 for (const clz of el.classList) {
792 if (clz.startsWith('code-annotation-')) {
793 return true;
794 }
795 }
796 return false;
797 }
798 const onCopySuccess = function(e) {
799 // button target
800 const button = e.trigger;
801 // don't keep focus
802 button.blur();
803 // flash "checked"
804 button.classList.add('code-copy-button-checked');
805 var currentTitle = button.getAttribute("title");
806 button.setAttribute("title", "Copied!");
807 let tooltip;
808 if (window.bootstrap) {
809 button.setAttribute("data-bs-toggle", "tooltip");
810 button.setAttribute("data-bs-placement", "left");
811 button.setAttribute("data-bs-title", "Copied!");
812 tooltip = new bootstrap.Tooltip(button,
813 { trigger: "manual",
814 customClass: "code-copy-button-tooltip",
815 offset: [0, -8]});
816 tooltip.show();
817 }
818 setTimeout(function() {
819 if (tooltip) {
820 tooltip.hide();
821 button.removeAttribute("data-bs-title");
822 button.removeAttribute("data-bs-toggle");
823 button.removeAttribute("data-bs-placement");
824 }
825 button.setAttribute("title", currentTitle);
826 button.classList.remove('code-copy-button-checked');
827 }, 1000);
828 // clear code selection
829 e.clearSelection();
830 }
831 const getTextToCopy = function(trigger) {
832 const codeEl = trigger.previousElementSibling.cloneNode(true);
833 for (const childEl of codeEl.children) {
834 if (isCodeAnnotation(childEl)) {
835 childEl.remove();
836 }
837 }
838 return codeEl.innerText;
839 }
840 const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
841 text: getTextToCopy
842 });
843 clipboard.on('success', onCopySuccess);
844 if (window.document.getElementById('quarto-embedded-source-code-modal')) {
845 const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', {
846 text: getTextToCopy,
847 container: window.document.getElementById('quarto-embedded-source-code-modal')
848 });
849 clipboardModal.on('success', onCopySuccess);
850 }
851 var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//);
852 var mailtoRegex = new RegExp(/^mailto:/);
853 var filterRegex = new RegExp("https:\/\/github\.com\/your-org\/atdata");
854 var isInternal = (href) => {
855 return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href);
856 }
857 // Inspect non-navigation links and adorn them if external
858 var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)');
859 for (var i=0; i<links.length; i++) {
860 const link = links[i];
861 if (!isInternal(link.href)) {
862 // undo the damage that might have been done by quarto-nav.js in the case of
863 // links that we want to consider external
864 if (link.dataset.originalHref !== undefined) {
865 link.href = link.dataset.originalHref;
866 }
867 }
868 }
869 function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
870 const config = {
871 allowHTML: true,
872 maxWidth: 500,
873 delay: 100,
874 arrow: false,
875 appendTo: function(el) {
876 return el.parentElement;
877 },
878 interactive: true,
879 interactiveBorder: 10,
880 theme: 'quarto',
881 placement: 'bottom-start',
882 };
883 if (contentFn) {
884 config.content = contentFn;
885 }
886 if (onTriggerFn) {
887 config.onTrigger = onTriggerFn;
888 }
889 if (onUntriggerFn) {
890 config.onUntrigger = onUntriggerFn;
891 }
892 window.tippy(el, config);
893 }
894 const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
895 for (var i=0; i<noterefs.length; i++) {
896 const ref = noterefs[i];
897 tippyHover(ref, function() {
898 // use id or data attribute instead here
899 let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
900 try { href = new URL(href).hash; } catch {}
901 const id = href.replace(/^#\/?/, "");
902 const note = window.document.getElementById(id);
903 if (note) {
904 return note.innerHTML;
905 } else {
906 return "";
907 }
908 });
909 }
910 const xrefs = window.document.querySelectorAll('a.quarto-xref');
911 const processXRef = (id, note) => {
912 // Strip column container classes
913 const stripColumnClz = (el) => {
914 el.classList.remove("page-full", "page-columns");
915 if (el.children) {
916 for (const child of el.children) {
917 stripColumnClz(child);
918 }
919 }
920 }
921 stripColumnClz(note)
922 if (id === null || id.startsWith('sec-')) {
923 // Special case sections, only their first couple elements
924 const container = document.createElement("div");
925 if (note.children && note.children.length > 2) {
926 container.appendChild(note.children[0].cloneNode(true));
927 for (let i = 1; i < note.children.length; i++) {
928 const child = note.children[i];
929 if (child.tagName === "P" && child.innerText === "") {
930 continue;
931 } else {
932 container.appendChild(child.cloneNode(true));
933 break;
934 }
935 }
936 if (window.Quarto?.typesetMath) {
937 window.Quarto.typesetMath(container);
938 }
939 return container.innerHTML
940 } else {
941 if (window.Quarto?.typesetMath) {
942 window.Quarto.typesetMath(note);
943 }
944 return note.innerHTML;
945 }
946 } else {
947 // Remove any anchor links if they are present
948 const anchorLink = note.querySelector('a.anchorjs-link');
949 if (anchorLink) {
950 anchorLink.remove();
951 }
952 if (window.Quarto?.typesetMath) {
953 window.Quarto.typesetMath(note);
954 }
955 if (note.classList.contains("callout")) {
956 return note.outerHTML;
957 } else {
958 return note.innerHTML;
959 }
960 }
961 }
962 for (var i=0; i<xrefs.length; i++) {
963 const xref = xrefs[i];
964 tippyHover(xref, undefined, function(instance) {
965 instance.disable();
966 let url = xref.getAttribute('href');
967 let hash = undefined;
968 if (url.startsWith('#')) {
969 hash = url;
970 } else {
971 try { hash = new URL(url).hash; } catch {}
972 }
973 if (hash) {
974 const id = hash.replace(/^#\/?/, "");
975 const note = window.document.getElementById(id);
976 if (note !== null) {
977 try {
978 const html = processXRef(id, note.cloneNode(true));
979 instance.setContent(html);
980 } finally {
981 instance.enable();
982 instance.show();
983 }
984 } else {
985 // See if we can fetch this
986 fetch(url.split('#')[0])
987 .then(res => res.text())
988 .then(html => {
989 const parser = new DOMParser();
990 const htmlDoc = parser.parseFromString(html, "text/html");
991 const note = htmlDoc.getElementById(id);
992 if (note !== null) {
993 const html = processXRef(id, note);
994 instance.setContent(html);
995 }
996 }).finally(() => {
997 instance.enable();
998 instance.show();
999 });
1000 }
1001 } else {
1002 // See if we can fetch a full url (with no hash to target)
1003 // This is a special case and we should probably do some content thinning / targeting
1004 fetch(url)
1005 .then(res => res.text())
1006 .then(html => {
1007 const parser = new DOMParser();
1008 const htmlDoc = parser.parseFromString(html, "text/html");
1009 const note = htmlDoc.querySelector('main.content');
1010 if (note !== null) {
1011 // This should only happen for chapter cross references
1012 // (since there is no id in the URL)
1013 // remove the first header
1014 if (note.children.length > 0 && note.children[0].tagName === "HEADER") {
1015 note.children[0].remove();
1016 }
1017 const html = processXRef(null, note);
1018 instance.setContent(html);
1019 }
1020 }).finally(() => {
1021 instance.enable();
1022 instance.show();
1023 });
1024 }
1025 }, function(instance) {
1026 });
1027 }
1028 let selectedAnnoteEl;
1029 const selectorForAnnotation = ( cell, annotation) => {
1030 let cellAttr = 'data-code-cell="' + cell + '"';
1031 let lineAttr = 'data-code-annotation="' + annotation + '"';
1032 const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
1033 return selector;
1034 }
1035 const selectCodeLines = (annoteEl) => {
1036 const doc = window.document;
1037 const targetCell = annoteEl.getAttribute("data-target-cell");
1038 const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
1039 const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
1040 const lines = annoteSpan.getAttribute("data-code-lines").split(",");
1041 const lineIds = lines.map((line) => {
1042 return targetCell + "-" + line;
1043 })
1044 let top = null;
1045 let height = null;
1046 let parent = null;
1047 if (lineIds.length > 0) {
1048 //compute the position of the single el (top and bottom and make a div)
1049 const el = window.document.getElementById(lineIds[0]);
1050 top = el.offsetTop;
1051 height = el.offsetHeight;
1052 parent = el.parentElement.parentElement;
1053 if (lineIds.length > 1) {
1054 const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
1055 const bottom = lastEl.offsetTop + lastEl.offsetHeight;
1056 height = bottom - top;
1057 }
1058 if (top !== null && height !== null && parent !== null) {
1059 // cook up a div (if necessary) and position it
1060 let div = window.document.getElementById("code-annotation-line-highlight");
1061 if (div === null) {
1062 div = window.document.createElement("div");
1063 div.setAttribute("id", "code-annotation-line-highlight");
1064 div.style.position = 'absolute';
1065 parent.appendChild(div);
1066 }
1067 div.style.top = top - 2 + "px";
1068 div.style.height = height + 4 + "px";
1069 div.style.left = 0;
1070 let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
1071 if (gutterDiv === null) {
1072 gutterDiv = window.document.createElement("div");
1073 gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
1074 gutterDiv.style.position = 'absolute';
1075 const codeCell = window.document.getElementById(targetCell);
1076 const gutter = codeCell.querySelector('.code-annotation-gutter');
1077 gutter.appendChild(gutterDiv);
1078 }
1079 gutterDiv.style.top = top - 2 + "px";
1080 gutterDiv.style.height = height + 4 + "px";
1081 }
1082 selectedAnnoteEl = annoteEl;
1083 }
1084 };
1085 const unselectCodeLines = () => {
1086 const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
1087 elementsIds.forEach((elId) => {
1088 const div = window.document.getElementById(elId);
1089 if (div) {
1090 div.remove();
1091 }
1092 });
1093 selectedAnnoteEl = undefined;
1094 };
1095 // Handle positioning of the toggle
1096 window.addEventListener(
1097 "resize",
1098 throttle(() => {
1099 elRect = undefined;
1100 if (selectedAnnoteEl) {
1101 selectCodeLines(selectedAnnoteEl);
1102 }
1103 }, 10)
1104 );
1105 function throttle(fn, ms) {
1106 let throttle = false;
1107 let timer;
1108 return (...args) => {
1109 if(!throttle) { // first call gets through
1110 fn.apply(this, args);
1111 throttle = true;
1112 } else { // all the others get throttled
1113 if(timer) clearTimeout(timer); // cancel #2
1114 timer = setTimeout(() => {
1115 fn.apply(this, args);
1116 timer = throttle = false;
1117 }, ms);
1118 }
1119 };
1120 }
1121 // Attach click handler to the DT
1122 const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
1123 for (const annoteDlNode of annoteDls) {
1124 annoteDlNode.addEventListener('click', (event) => {
1125 const clickedEl = event.target;
1126 if (clickedEl !== selectedAnnoteEl) {
1127 unselectCodeLines();
1128 const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
1129 if (activeEl) {
1130 activeEl.classList.remove('code-annotation-active');
1131 }
1132 selectCodeLines(clickedEl);
1133 clickedEl.classList.add('code-annotation-active');
1134 } else {
1135 // Unselect the line
1136 unselectCodeLines();
1137 clickedEl.classList.remove('code-annotation-active');
1138 }
1139 });
1140 }
1141 const findCites = (el) => {
1142 const parentEl = el.parentElement;
1143 if (parentEl) {
1144 const cites = parentEl.dataset.cites;
1145 if (cites) {
1146 return {
1147 el,
1148 cites: cites.split(' ')
1149 };
1150 } else {
1151 return findCites(el.parentElement)
1152 }
1153 } else {
1154 return undefined;
1155 }
1156 };
1157 var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
1158 for (var i=0; i<bibliorefs.length; i++) {
1159 const ref = bibliorefs[i];
1160 const citeInfo = findCites(ref);
1161 if (citeInfo) {
1162 tippyHover(citeInfo.el, function() {
1163 var popup = window.document.createElement('div');
1164 citeInfo.cites.forEach(function(cite) {
1165 var citeDiv = window.document.createElement('div');
1166 citeDiv.classList.add('hanging-indent');
1167 citeDiv.classList.add('csl-entry');
1168 var biblioDiv = window.document.getElementById('ref-' + cite);
1169 if (biblioDiv) {
1170 citeDiv.innerHTML = biblioDiv.innerHTML;
1171 }
1172 popup.appendChild(citeDiv);
1173 });
1174 return popup.innerHTML;
1175 });
1176 }
1177 }
1178 });
1179 </script>
1180</div> <!-- /content -->
1181<footer class="footer">
1182 <div class="nav-footer">
1183 <div class="nav-footer-left">
1184<p>Built with <a href="https://quarto.org/">Quarto</a></p>
1185</div>
1186 <div class="nav-footer-center">
1187
1188 <div class="toc-actions d-sm-block d-md-none"><ul><li><a href="https://github.com/your-org/atdata/edit/main/api/S3Source.qmd" class="toc-action"><i class="bi bi-github"></i>Edit this page</a></li><li><a href="https://github.com/your-org/atdata/issues/new" class="toc-action"><i class="bi empty"></i>Report an issue</a></li></ul></div></div>
1189 <div class="nav-footer-right">
1190<p>MIT License</p>
1191</div>
1192 </div>
1193</footer>
1194
1195
1196
1197
1198</body></html>