-390

app/utils/pyodide/pyodide.js

reviewed

··· 1 1 - /** 2 2 - * The main bootstrap script for loading pyodide. 3 3 - */ 4 4 - const port = process.env.PORT || 1212; 5 5 - 6 6 - export const languagePluginLoader = new Promise((resolve, reject) => { 7 7 - let baseURL = `http://localhost:${port}/src/`; 8 8 - // var baseURL = self.languagePluginUrl || 'https://iodide.io/pyodide-demo/'; 9 9 - baseURL = `${baseURL.substr(0, baseURL.lastIndexOf('/'))}/`; 10 10 - 11 11 - /// ///////////////////////////////////////////////////////// 12 12 - // Package loading 13 13 - const loadedPackages = []; 14 14 - let loadPackagePromise = new Promise((resolve) => resolve()); 15 15 - // Regexp for validating package name and URI 16 16 - var package_name_regexp = '[a-z0-9_][a-z0-9_-]*'; 17 17 - const package_uri_regexp = new RegExp( 18 18 - `^https?://.*?(${package_name_regexp}).js$`, 19 19 - 'i' 20 20 - ); 21 21 - var package_name_regexp = new RegExp(`^${package_name_regexp}$`, 'i'); 22 22 - 23 23 - const _uri_to_package_name = (package_uri) => { 24 24 - // Generate a unique package name from URI 25 25 - 26 26 - if (package_name_regexp.test(package_uri)) { 27 27 - return package_uri; 28 28 - } 29 29 - if (package_uri_regexp.test(package_uri)) { 30 30 - const match = package_uri_regexp.exec(package_uri); 31 31 - // Get the regexp group corresponding to the package name 32 32 - return match[1]; 33 33 - } 34 34 - return null; 35 35 - }; 36 36 - 37 37 - // clang-format off 38 38 - const preloadWasm = () => { 39 39 - // On Chrome, we have to instantiate wasm asynchronously. Since that 40 40 - // can't be done synchronously within the call to dlopen, we instantiate 41 41 - // every .so that comes our way up front, caching it in the 42 42 - // `preloadedWasm` dictionary. 43 43 - 44 44 - let promise = new Promise((resolve) => resolve()); 45 45 - const { FS } = pyodide._module; 46 46 - 47 47 - function recurseDir(rootpath) { 48 48 - let dirs; 49 49 - try { 50 50 - dirs = FS.readdir(rootpath); 51 51 - } catch (err) { 52 52 - return; 53 53 - } 54 54 - for (const entry of dirs) { 55 55 - if (entry.startsWith('.')) { 56 56 - continue; 57 57 - } 58 58 - const path = rootpath + entry; 59 59 - if (entry.endsWith('.so')) { 60 60 - if (Module.preloadedWasm[path] === undefined) { 61 61 - promise = promise 62 62 - .then(() => 63 63 - Module.loadWebAssemblyModule(FS.readFile(path), { 64 64 - loadAsync: true, 65 65 - }) 66 66 - ) 67 67 - .then((module) => { 68 68 - Module.preloadedWasm[path] = module; 69 69 - }); 70 70 - } 71 71 - } else if (FS.isDir(FS.lookupPath(path).node.mode)) { 72 72 - recurseDir(`${path}/`); 73 73 - } 74 74 - } 75 75 - } 76 76 - 77 77 - recurseDir('/'); 78 78 - 79 79 - return promise; 80 80 - }; 81 81 - // clang-format on 82 82 - 83 83 - function loadScript(url, onload, onerror) { 84 84 - if (self.document) { 85 85 - // browser 86 86 - const script = self.document.createElement('script'); 87 87 - script.src = url; 88 88 - script.onload = (e) => { 89 89 - onload(); 90 90 - }; 91 91 - script.onerror = (e) => { 92 92 - onerror(); 93 93 - }; 94 94 - self.document.head.appendChild(script); 95 95 - } else if (self.importScripts) { 96 96 - // webworker 97 97 - try { 98 98 - self.importScripts(url); 99 99 - onload(); 100 100 - } catch (err) { 101 101 - onerror(); 102 102 - } 103 103 - } 104 104 - } 105 105 - 106 106 - const _loadPackage = (names, messageCallback) => { 107 107 - // DFS to find all dependencies of the requested packages 108 108 - const packages = self.pyodide._module.packages.dependencies; 109 109 - const { loadedPackages } = self.pyodide; 110 110 - const queue = [].concat(names || []); 111 111 - const toLoad = []; 112 112 - while (queue.length) { 113 113 - let package_uri = queue.pop(); 114 114 - 115 115 - const packageName = _uri_to_package_name(package_uri); 116 116 - 117 117 - if (packageName == null) { 118 118 - console.error(`Invalid package name or URI '${package_uri}'`); 119 119 - return; 120 120 - } 121 121 - if (packageName == package_uri) { 122 122 - package_uri = 'default channel'; 123 123 - } 124 124 - 125 125 - if (packageName in loadedPackages) { 126 126 - if (package_uri != loadedPackages[packageName]) { 127 127 - console.error( 128 128 - `URI mismatch, attempting to load package ` + 129 129 - `${packageName} from ${package_uri} while it is already ` + 130 130 - `loaded from ${loadedPackages[packageName]}!` 131 131 - ); 132 132 - return; 133 133 - } 134 134 - } else if (packageName in toLoad) { 135 135 - if (package_uri != toLoad[packageName]) { 136 136 - console.error( 137 137 - `URI mismatch, attempting to load package ` + 138 138 - `${packageName} from ${package_uri} while it is already ` + 139 139 - `being loaded from ${toLoad[packageName]}!` 140 140 - ); 141 141 - return; 142 142 - } 143 143 - } else { 144 144 - console.log(`Loading ${packageName} from ${package_uri}`); 145 145 - 146 146 - toLoad[packageName] = package_uri; 147 147 - if (packages.hasOwnProperty(packageName)) { 148 148 - packages[packageName].forEach((subpackage) => { 149 149 - if (!(subpackage in loadedPackages) && !(subpackage in toLoad)) { 150 150 - queue.push(subpackage); 151 151 - } 152 152 - }); 153 153 - } else { 154 154 - console.error(`Unknown package '${packageName}'`); 155 155 - } 156 156 - } 157 157 - } 158 158 - 159 159 - self.pyodide._module.locateFile = (path) => { 160 160 - // handle packages loaded from custom URLs 161 161 - const packageName = path.replace(/\.data$/, ''); 162 162 - if (packageName in toLoad) { 163 163 - const package_uri = toLoad[packageName]; 164 164 - if (package_uri != 'default channel') { 165 165 - return package_uri.replace(/\.js$/, '.data'); 166 166 - } 167 167 - } 168 168 - return baseURL + path; 169 169 - }; 170 170 - 171 171 - const promise = new Promise((resolve, reject) => { 172 172 - if (Object.keys(toLoad).length === 0) { 173 173 - resolve('No new packages to load'); 174 174 - return; 175 175 - } 176 176 - 177 177 - const packageList = Array.from(Object.keys(toLoad)).join(', '); 178 178 - if (messageCallback !== undefined) { 179 179 - messageCallback(`Loading ${packageList}`); 180 180 - } 181 181 - 182 182 - // monitorRunDependencies is called at the beginning and the end of each 183 183 - // package being loaded. We know we are done when it has been called 184 184 - // exactly "toLoad * 2" times. 185 185 - let packageCounter = Object.keys(toLoad).length * 2; 186 186 - 187 187 - self.pyodide._module.monitorRunDependencies = () => { 188 188 - packageCounter--; 189 189 - if (packageCounter === 0) { 190 190 - for (const packageName in toLoad) { 191 191 - self.pyodide.loadedPackages[packageName] = toLoad[packageName]; 192 192 - } 193 193 - delete self.pyodide._module.monitorRunDependencies; 194 194 - self.removeEventListener('error', windowErrorHandler); 195 195 - if (!isFirefox) { 196 196 - preloadWasm().then(() => { 197 197 - resolve(`Loaded ${packageList}`); 198 198 - }); 199 199 - } else { 200 200 - resolve(`Loaded ${packageList}`); 201 201 - } 202 202 - } 203 203 - }; 204 204 - 205 205 - // Add a handler for any exceptions that are thrown in the process of 206 206 - // loading a package 207 207 - var windowErrorHandler = (err) => { 208 208 - delete self.pyodide._module.monitorRunDependencies; 209 209 - self.removeEventListener('error', windowErrorHandler); 210 210 - // Set up a new Promise chain, since this one failed 211 211 - loadPackagePromise = new Promise((resolve) => resolve()); 212 212 - reject(err.message); 213 213 - }; 214 214 - self.addEventListener('error', windowErrorHandler); 215 215 - 216 216 - for (const packageName in toLoad) { 217 217 - let scriptSrc; 218 218 - const package_uri = toLoad[packageName]; 219 219 - if (package_uri == 'default channel') { 220 220 - scriptSrc = `${baseURL}${packageName}.js`; 221 221 - } else { 222 222 - scriptSrc = `${package_uri}`; 223 223 - } 224 224 - loadScript( 225 225 - scriptSrc, 226 226 - () => {}, 227 227 - () => { 228 228 - // If the package_uri fails to load, call monitorRunDependencies twice 229 229 - // (so packageCounter will still hit 0 and finish loading), and remove 230 230 - // the package from toLoad so we don't mark it as loaded. 231 231 - console.error(`Couldn't load package from URL ${scriptSrc}`); 232 232 - const index = toLoad.indexOf(packageName); 233 233 - if (index !== -1) { 234 234 - toLoad.splice(index, 1); 235 235 - } 236 236 - for (let i = 0; i < 2; i++) { 237 237 - self.pyodide._module.monitorRunDependencies(); 238 238 - } 239 239 - } 240 240 - ); 241 241 - } 242 242 - 243 243 - // We have to invalidate Python's import caches, or it won't 244 244 - // see the new files. This is done here so it happens in parallel 245 245 - // with the fetching over the network. 246 246 - self.pyodide.runPython( 247 247 - 'import importlib as _importlib\n' + '_importlib.invalidate_caches()\n' 248 248 - ); 249 249 - }); 250 250 - 251 251 - return promise; 252 252 - }; 253 253 - 254 254 - const loadPackage = (names, messageCallback) => { 255 255 - /* We want to make sure that only one loadPackage invocation runs at any 256 256 - * given time, so this creates a "chain" of promises. */ 257 257 - loadPackagePromise = loadPackagePromise.then(() => 258 258 - _loadPackage(names, messageCallback) 259 259 - ); 260 260 - return loadPackagePromise; 261 261 - }; 262 262 - 263 263 - /// ///////////////////////////////////////////////////////// 264 264 - // Fix Python recursion limit 265 265 - function fixRecursionLimit(pyodide) { 266 266 - // The Javascript/Wasm call stack may be too small to handle the default 267 267 - // Python call stack limit of 1000 frames. This is generally the case on 268 268 - // Chrom(ium), but not on Firefox. Here, we determine the Javascript call 269 269 - // stack depth available, and then divide by 50 (determined heuristically) 270 270 - // to set the maximum Python call stack depth. 271 271 - 272 272 - let depth = 0; 273 273 - function recurse() { 274 274 - depth += 1; 275 275 - recurse(); 276 276 - } 277 277 - try { 278 278 - recurse(); 279 279 - } catch (err) {} 280 280 - 281 281 - let recursionLimit = depth / 50; 282 282 - if (recursionLimit > 1000) { 283 283 - recursionLimit = 1000; 284 284 - } 285 285 - pyodide.runPython( 286 286 - `import sys; sys.setrecursionlimit(int(${recursionLimit}))` 287 287 - ); 288 288 - } 289 289 - 290 290 - /// ///////////////////////////////////////////////////////// 291 291 - // Rearrange namespace for public API 292 292 - const PUBLIC_API = [ 293 293 - 'globals', 294 294 - 'loadPackage', 295 295 - 'loadedPackages', 296 296 - 'pyimport', 297 297 - 'repr', 298 298 - 'runPython', 299 299 - 'runPythonAsync', 300 300 - 'checkABI', 301 301 - 'version', 302 302 - ]; 303 303 - 304 304 - function makePublicAPI(module, public_api) { 305 305 - const namespace = { _module: module }; 306 306 - for (const name of public_api) { 307 307 - namespace[name] = module[name]; 308 308 - } 309 309 - return namespace; 310 310 - } 311 311 - 312 312 - /// ///////////////////////////////////////////////////////// 313 313 - // Loading Pyodide 314 314 - const wasmURL = `${baseURL}pyodide.asm.wasm`; 315 315 - let Module = {}; 316 316 - self.Module = Module; 317 317 - 318 318 - Module.noImageDecoding = true; 319 319 - Module.noAudioDecoding = true; 320 320 - Module.noWasmDecoding = true; 321 321 - Module.preloadedWasm = {}; 322 322 - let isFirefox = navigator.userAgent.toLowerCase().indexOf('firefox') > -1; 323 323 - 324 324 - const wasm_promise = WebAssembly.compileStreaming(fetch(wasmURL)); 325 325 - Module.instantiateWasm = (info, receiveInstance) => { 326 326 - wasm_promise 327 327 - .then((module) => WebAssembly.instantiate(module, info)) 328 328 - .then((instance) => receiveInstance(instance)); 329 329 - return {}; 330 330 - }; 331 331 - 332 332 - Module.checkABI = function (ABI_number) { 333 333 - if (ABI_number !== parseInt('1')) { 334 334 - const ABI_mismatch_exception = `ABI numbers differ. Expected 1, got ${ABI_number}`; 335 335 - console.error(ABI_mismatch_exception); 336 336 - throw ABI_mismatch_exception; 337 337 - } 338 338 - return true; 339 339 - }; 340 340 - 341 341 - Module.locateFile = (path) => baseURL + path; 342 342 - const postRunPromise = new Promise((resolve, reject) => { 343 343 - Module.postRun = () => { 344 344 - delete self.Module; 345 345 - fetch(`${baseURL}packages.json`) 346 346 - .then((response) => response.json()) 347 347 - .then((json) => { 348 348 - fixRecursionLimit(self.pyodide); 349 349 - self.pyodide.globals = self.pyodide.runPython( 350 350 - 'import sys\nsys.modules["__main__"]' 351 351 - ); 352 352 - self.pyodide = makePublicAPI(self.pyodide, PUBLIC_API); 353 353 - self.pyodide._module.packages = json; 354 354 - resolve(); 355 355 - }); 356 356 - }; 357 357 - }); 358 358 - 359 359 - const dataLoadPromise = new Promise((resolve, reject) => { 360 360 - Module.monitorRunDependencies = (n) => { 361 361 - if (n === 0) { 362 362 - delete Module.monitorRunDependencies; 363 363 - resolve(); 364 364 - } 365 365 - }; 366 366 - }); 367 367 - 368 368 - Promise.all([postRunPromise, dataLoadPromise]).then(() => resolve()); 369 369 - 370 370 - const data_script_src = `${baseURL}pyodide.asm.data.js`; 371 371 - loadScript( 372 372 - data_script_src, 373 373 - () => { 374 374 - const scriptSrc = `${baseURL}pyodide.asm.js`; 375 375 - loadScript( 376 376 - scriptSrc, 377 377 - () => { 378 378 - // The emscripten module needs to be at this location for the core 379 379 - // filesystem to install itself. Once that's complete, it will be replaced 380 380 - // by the call to `makePublicAPI` with a more limited public API. 381 381 - self.pyodide = pyodide(Module); 382 382 - self.pyodide.loadedPackages = []; 383 383 - self.pyodide.loadPackage = loadPackage; 384 384 - }, 385 385 - () => {} 386 386 - ); 387 387 - }, 388 388 - () => {} 389 389 - ); 390 390 - });

+23 -28

app/utils/pyodide/webworker.js

reviewed

··· 3 3 * pyodide to be used in a web worker within this 4 4 */ 5 5 6 6 - self.languagePluginUrl = './src'; 7 7 - importScripts('./pyodide.js'); 6 6 + // self.languagePluginUrl = './src'; 7 7 + importScripts('./src/pyodide/pyodide.js'); 8 8 9 9 - const onmessage = function (e) { 10 10 - // eslint-disable-line no-unused-vars 11 11 - languagePluginLoader.then(() => { 12 12 - // Preloaded packages 13 13 - self.pyodide.loadPackage(['matplotlib', 'mne', 'pandas']).then(() => { 14 14 - const { data } = e; 15 15 - const keys = Object.keys(data); 16 16 - for (const key of keys) { 17 17 - if (key !== 'python') { 18 18 - // Keys other than python must be arguments for the python script. 19 19 - // Set them on self, so that `from js import key` works. 20 20 - self[key] = data[key]; 21 21 - } 22 22 - } 9 9 + async function loadPyodideAndPackages() { 10 10 + await loadPyodide({ indexURL: './src/pyodide/' }); 11 11 + await self.pyodide.loadPackage(['matplotlib', 'mne', 'pandas']); 12 12 + } 13 13 + let pyodideReadyPromise = loadPyodideAndPackages(); 23 14 24 24 - self.pyodide 25 25 - .runPythonAsync(data.python, () => {}) 26 26 - .then((results) => { 27 27 - self.postMessage({ results }); 28 28 - }) 29 29 - .catch((err) => { 30 30 - // if you prefer messages with the error 31 31 - self.postMessage({ error: err.message }); 32 32 - // if you prefer onerror events 33 33 - // setTimeout(() => { throw err; }); 34 34 - }); 15 15 + self.onmessage = async (event) => { 16 16 + // make sure loading is done 17 17 + await pyodideReadyPromise; 18 18 + // Don't bother yet with this line, suppose our API is built in such a way: 19 19 + const { python, ...context } = event.data; 20 20 + // The worker copies the context in its own "memory" (an object mapping name to values) 21 21 + for (const key of Object.keys(context)) { 22 22 + self[key] = context[key]; 23 23 + } 24 24 + // Now is the easy part, the one that is similar to working in the main thread: 25 25 + try { 26 26 + self.postMessage({ 27 27 + results: await self.pyodide.runPythonAsync(python), 35 28 }); 36 36 - }); 29 29 + } catch (error) { 30 30 + self.postMessage({ error: error.message }); 31 31 + } 37 32 };