massive overhaul of pyodide in the app · teonbrooks.com/BrainWaves@d021015

+19 -41

app/epics/pyodideEpics.js

··· 5 5 mergeMap, 6 6 tap, 7 7 pluck, 8 - ignoreElements, 9 8 filter 10 9 } from "rxjs/operators"; 11 10 import { getWorkspaceDir } from "../utils/filesystem/storage"; 12 - import { languagePluginLoader } from "../utils/pyodide/pyodide"; 11 + import { parseSingleQuoteJSON } from "../utils/pyodide/functions" 12 + import { readFiles } from "../utils/filesystem/read"; 13 13 import { 14 14 LAUNCH, 15 15 LOAD_EPOCHS, ··· 22 22 loadERP 23 23 } from "../actions/pyodideActions"; 24 24 import { 25 - test, 26 - imports, 25 + loadPackages, 26 + utils, 27 27 loadCSV, 28 28 loadCleanedEpochs, 29 29 filterIIR, ··· 34 34 plotPSD, 35 35 plotERP, 36 36 plotTopoMap, 37 - saveEpochs 37 + saveEpochs, 38 38 } from "../utils/pyodide/commands"; 39 39 import { 40 40 EMOTIV_CHANNELS, ··· 94 94 const setPyodideStatus = payload => ({ 95 95 payload, 96 96 type: SET_PYODIDE_STATUS 97 - }); 98 - 99 - const receiveExecuteReply = payload => ({ 100 - payload, 101 - type: RECEIVE_EXECUTE_REPLY, 102 - }); 103 - 104 - const receiveExecuteResult = (payload) => ({ 105 - payload, 106 - type: RECEIVE_EXECUTE_RESULT, 107 - }); 108 - 109 - const receiveDisplayData = (payload) => ({ 110 - payload, 111 - type: RECEIVE_DISPLAY_DATA, 112 - }); 113 - 114 - const receiveStream = (payload) => ({ 115 - payload, 116 - type: RECEIVE_STREAM, 117 97 }); 118 98 119 99 // ------------------------------------------------------------------------- ··· 122 102 const launchEpic = action$ => 123 103 action$.ofType(LAUNCH).pipe( 124 104 tap(() => console.log("launching")), 125 - mergeMap(async () => { 126 - await languagePluginLoader; 127 - console.log("loaded language plugin"); 128 - // using window.pyodide instead of pyodide to get linter to stop yelling ;) 129 - await window.pyodide.loadPackage(["mne"]); 130 - console.log("loaded mne package"); 131 - }), 105 + mergeMap(loadPackages), 106 + mergeMap(utils), 132 107 map(() => setPyodideStatus(PYODIDE_STATUS.LOADED)) 133 108 ); 134 109 ··· 136 111 action$.ofType(LOAD_EPOCHS).pipe( 137 112 pluck("payload"), 138 113 filter(filePathsArray => filePathsArray.length >= 1), 139 - map(filePathsArray => loadCSV(filePathsArray)), 140 - map(() => filterIIR(1, 30)), 141 - map(() => 114 + tap(files => console.log('files:', files)), 115 + map((filePathsArray => readFiles(filePathsArray))), 116 + tap(csvArray => console.log('csvs:', csvArray)), 117 + mergeMap(csvArray => loadCSV(csvArray)), 118 + mergeMap(() => filterIIR(1, 30)), 119 + mergeMap(() => 142 120 epochEvents( 143 121 { 144 122 [state$.value.experiment.params.stimulus1.title]: EVENTS.STIMULUS_1, ··· 150 128 0.8 151 129 ) 152 130 ), 153 - map(epochEventsCommand => epochEventsCommand), 154 131 map(() => getEpochsInfo(PYODIDE_VARIABLE_NAMES.RAW_EPOCHS)) 155 132 ); 156 133 157 - const loadCleanedEpochsEpic = (action$, state$) => 134 + const loadCleanedEpochsEpic = (action$) => 158 135 action$.ofType(LOAD_CLEANED_EPOCHS).pipe( 159 136 pluck("payload"), 160 137 filter(filePathsArray => filePathsArray.length >= 1), ··· 180 157 map(() => getEpochsInfo(PYODIDE_VARIABLE_NAMES.RAW_EPOCHS)) 181 158 ); 182 159 183 - const getEpochsInfoEpic = (action$, state$) => 160 + const getEpochsInfoEpic = (action$) => 184 161 action$.ofType(GET_EPOCHS_INFO).pipe( 185 162 pluck("payload"), 186 - map(variableName => requestEpochsInfo(variableName)), 187 - map(epochInfoString => 188 - parseSingleQuoteJSON(epochInfoString).map(infoObj => ({ 163 + tap(payload => console.log('payload: ', payload)), 164 + mergeMap(requestEpochsInfo), 165 + map(epochInfoArray => 166 + epochInfoArray.map(infoObj => ({ 189 167 name: Object.keys(infoObj)[0], 190 168 value: infoObj[Object.keys(infoObj)[0]], 191 169 }))

+18

app/utils/filesystem/read.js

··· 1 + const fs = require("fs"); 2 + 3 + export const readFiles = (filePathsArray) => { 4 + return filePathsArray.map(path => { 5 + console.log('about to read file') 6 + const file = fs.readFileSync(path, 'utf8') 7 + console.log('read file') 8 + return file 9 + }) 10 + } 11 + 12 + 13 + 14 + // ------------------------------------------- 15 + // Helper methods 16 + 17 + const formatFilePath = (filePath: string) => 18 + `"${filePath.replace(/\\/g, "/")}"`;

+37 -53

app/utils/pyodide/commands.js

··· 1 1 import * as path from "path"; 2 2 import { readFileSync } from "fs"; 3 + import { languagePluginLoader } from "./pyodide"; 4 + 3 5 4 6 let pyodide; 5 7 // ----------------------------- 6 8 // Imports and Utility functions 7 9 8 - export const test = async () => { 9 - await window.pyodide.loadPackage(["mne"]); 10 + export const loadPackages = async () => { 11 + await languagePluginLoader; 12 + console.log("loaded language plugin"); 13 + // using window.pyodide instead of pyodide to get linter to stop yelling ;) 14 + await window.pyodide.loadPackage(["matplotlib", "mne", "pandas"]); 15 + await window.pyodide.runPython("import js"); 16 + console.log("loaded mne package"); 10 17 11 - const mneCommands = [ 12 - `import numpy as np`, 13 - `import mne`, 14 - `data = np.repeat(np.atleast_2d(np.arange(1000)), 8, axis=0)`, 15 - `info = mne.create_info(8, 250)`, 16 - `raw = mne.io.RawArray(data=data, info=info)`, 17 - `raw.save("test_brainwaves.fif")` 18 - ]; 19 - await window.pyodide.runPython(mneCommands.join("; ")); 20 - }; 18 + } 21 19 22 - export const loadPackages = async () => window.pyodide.loadPackage(["mne"]); 23 - 24 - export const imports = () => 25 - pyodide.runPython( 26 - readFileSync(path.join(__dirname, "/utils/pyodide/pyimport.py"), "utf8") 27 - ); 28 - 29 - export const utils = () => 30 - pyodide.runPython( 20 + export const utils = async () => 21 + window.pyodide.runPython( 31 22 readFileSync(path.join(__dirname, "/utils/pyodide/utils.py"), "utf8") 32 23 ); 33 24 34 - export const loadCSV = (filePathArray: Array<string>) => 35 - [ 36 - `files = [${filePathArray.map(filePath => formatFilePath(filePath))}]`, 37 - `replace_ch_names = None`, 38 - `raw = load_data(files, replace_ch_names)` 39 - ].join("\n"); 25 + export const loadCSV = async (csvArray: Array<any>) => { 26 + window.csvArray = csvArray; 27 + // TODO: Pass attached variable name as parameter to load_data 28 + await window.pyodide.runPython( 29 + `raw = load_data()`) 30 + } 40 31 41 32 // --------------------------- 42 33 // MNE-Related Data Processing 43 - export const loadCleanedEpochs = (filePathArray: Array<string>) => 34 + export const loadCleanedEpochs = (epocsArray: Array<any>) => 44 35 [ 45 - `files = [${filePathArray.map(filePath => formatFilePath(filePath))}]`, 46 36 `clean_epochs = concatenate_epochs([read_epochs(file) for file in files])`, 47 37 `conditions = OrderedDict({key: [value] for (key, value) in clean_epochs.event_id.items()})` 48 38 ].join("\n"); 49 39 50 40 // NOTE: this command includes a ';' to prevent returning data 51 - export const filterIIR = (lowCutoff: number, highCutoff: number) => 52 - `raw.filter(${lowCutoff}, ${highCutoff}, method='iir');`; 41 + export const filterIIR = async (lowCutoff: number, highCutoff: number) => 42 + window.pyodide.runPython(`raw.filter(${lowCutoff}, ${highCutoff}, method='iir');`) 53 43 54 - export const epochEvents = ( 44 + export const epochEvents = async ( 55 45 eventIDs: { [string]: number }, 56 46 tmin: number, 57 47 tmax: number, 58 48 reject?: Array<string> | string = "None" 59 - ) => { 60 - const command = [ 61 - `event_id = ${JSON.stringify(eventIDs)}`, 62 - `tmin=${tmin}`, 63 - `tmax=${tmax}`, 64 - `baseline= (tmin, tmax)`, 65 - `picks = None`, 66 - `reject = ${reject}`, 67 - "events = find_events(raw)", 68 - `raw_epochs = Epochs(raw, events=events, event_id=event_id, 49 + ) => window.pyodide.runPython([ 50 + `event_id = ${JSON.stringify(eventIDs)}`, 51 + `tmin=${tmin}`, 52 + `tmax=${tmax}`, 53 + `baseline= (tmin, tmax)`, 54 + `picks = None`, 55 + `reject = ${reject}`, 56 + "events = find_events(raw)", 57 + `raw_epochs = Epochs(raw, events=events, event_id=event_id, 69 58 tmin=tmin, tmax=tmax, baseline=baseline, reject=reject, preload=True, 70 59 verbose=False, picks=picks)`, 71 - `conditions = OrderedDict({key: [value] for (key, value) in raw_epochs.event_id.items()})` 72 - ].join("\n"); 73 - return command; 74 - }; 60 + `conditions = OrderedDict({key: [value] for (key, value) in raw_epochs.event_id.items()})` 61 + ].join("\n")) 75 62 76 - export const requestEpochsInfo = (variableName: string) => 77 - `get_epochs_info(${variableName})`; 63 + export const requestEpochsInfo = async (variableName: string) => { 64 + const pyodideReturn = await window.pyodide.runPython(`get_epochs_info(${variableName})`); 65 + return pyodideReturn 66 + } 78 67 79 68 export const requestChannelInfo = () => 80 69 `[ch for ch in clean_epochs.ch_names if ch != 'Marker']`; ··· 104 93 ) 105 94 )})`; 106 95 107 - // ------------------------------------------- 108 - // Helper methods 109 - 110 - const formatFilePath = (filePath: string) => 111 - `"${filePath.replace(/\\/g, "/")}"`;

-15

app/utils/pyodide/pyimport.py

··· 1 - from time import time, strftime, gmtime 2 - import os 3 - from collections import OrderedDict 4 - from glob import glob 5 1 6 - import numpy as np 7 - import pandas as pd # maybe we can remove this dependency 8 - import seaborn as sns 9 - from matplotlib import pyplot as plt 10 - 11 - from mne import (Epochs, RawArray, concatenate_raws, concatenate_epochs, 12 - create_info, find_events, read_epochs, set_eeg_reference) 13 - from mne.channels import read_montage 14 - 15 - 16 - plt.style.use(fivethirtyeight)

+28 -15

app/utils/pyodide/utils.py

··· 1 1 from glob import glob 2 2 import os 3 + from time import time, strftime, gmtime 3 4 from collections import OrderedDict 4 - from mne import create_info, concatenate_raws, viz 5 + 6 + import numpy as np 7 + from matplotlib import pyplot as plt 8 + import pandas as pd # maybe we can remove this dependency 9 + # import seaborn as sns 10 + 11 + from mne import (Epochs, concatenate_raws, concatenate_epochs, create_info, 12 + find_events, read_epochs, set_eeg_reference, viz) 5 13 from mne.io import RawArray 14 + from io import StringIO 6 15 from mne.channels import read_montage 7 - import pandas as pd 8 - import numpy as np 9 - import seaborn as sns 10 - from matplotlib import pyplot as plt 11 16 12 - sns.set_context('talk') 13 - sns.set_style('white') 14 17 18 + # plt.style.use(fivethirtyeight) 15 19 16 - def load_data(fnames, sfreq=128., replace_ch_names=None): 20 + # sns.set_context('talk') 21 + # sns.set_style('white') 22 + 23 + 24 + def load_data(sfreq=128., replace_ch_names=None): 17 25 """Load CSV files from the /data directory into a RawArray object. 18 26 19 27 Parameters 20 28 ---------- 21 - fnames : list 22 - CSV filepaths from which to load data 29 + 23 30 sfreq : float 24 31 EEG sampling frequency 25 32 replace_ch_names : dict | None ··· 31 38 raw : an instance of mne.io.RawArray 32 39 The loaded data. 33 40 """ 34 - 41 + ## js is loaded in loadPackages 42 + ## TODO: Received attached variable name 35 43 raw = [] 36 - print(fnames) 37 - for fname in fnames: 44 + for csv in js.csvArray: 45 + string_io = StringIO(csv) 38 46 # read the file 39 - data = pd.read_csv(fname, index_col=0) 47 + data = pd.read_csv(string_io, index_col=0) 40 48 41 49 data = data.dropna() 42 50 ··· 77 85 78 86 79 87 def plot_topo(epochs, conditions=OrderedDict()): 80 - palette = sns.color_palette("hls", len(conditions) + 1) 88 + # palette = sns.color_palette("hls", len(conditions) + 1) 89 + # temp hack, just pull in the color palette from seaborn 90 + palette = [(0.85999999999999999, 0.37119999999999997, 0.33999999999999997), 91 + (0.33999999999999997, 0.85999999999999999, 0.37119999999999997), 92 + (0.37119999999999997, 0.33999999999999997, 0.85999999999999999)] 81 93 evokeds = [epochs[name].average() for name in (conditions)] 82 94 83 95 evoked_topo = viz.plot_evoked_topo( ··· 189 201 return fig, ax 190 202 191 203 def get_epochs_info(epochs): 204 + print('Get Epochs Info:') 192 205 return [*[{x: len(epochs[x])} for x in epochs.event_id], 193 206 {"Drop Percentage": round((1 - len(epochs.events) / 194 207 len(epochs.drop_log)) * 100, 2)},

Configure Feed

Configure Feed