From b917a674849cf55f7a16a7e82dca40b11e162b2f Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Sat, 3 Aug 2019 18:17:41 -0400 Subject: [PATCH] massive overhaul of pyodide in the app Co-authored-by: Dano Morrison --- app/epics/pyodideEpics.js | 60 ++++++++--------------- app/utils/filesystem/read.js | 18 +++++++ app/utils/pyodide/commands.js | 92 +++++++++++++++-------------------- app/utils/pyodide/pyimport.py | 15 ------ app/utils/pyodide/utils.py | 43 ++++++++++------ 5 files changed, 103 insertions(+), 125 deletions(-) create mode 100644 app/utils/filesystem/read.js diff --git a/app/epics/pyodideEpics.js b/app/epics/pyodideEpics.js index de7e61cc..6ceeb8f0 100644 --- a/app/epics/pyodideEpics.js +++ b/app/epics/pyodideEpics.js @@ -5,11 +5,11 @@ import { mergeMap, tap, pluck, - ignoreElements, filter } from "rxjs/operators"; import { getWorkspaceDir } from "../utils/filesystem/storage"; -import { languagePluginLoader } from "../utils/pyodide/pyodide"; +import { parseSingleQuoteJSON } from "../utils/pyodide/functions" +import { readFiles } from "../utils/filesystem/read"; import { LAUNCH, LOAD_EPOCHS, @@ -22,8 +22,8 @@ import { loadERP } from "../actions/pyodideActions"; import { - test, - imports, + loadPackages, + utils, loadCSV, loadCleanedEpochs, filterIIR, @@ -34,7 +34,7 @@ import { plotPSD, plotERP, plotTopoMap, - saveEpochs + saveEpochs, } from "../utils/pyodide/commands"; import { EMOTIV_CHANNELS, @@ -96,39 +96,14 @@ const setPyodideStatus = payload => ({ type: SET_PYODIDE_STATUS }); -const receiveExecuteReply = payload => ({ - payload, - type: RECEIVE_EXECUTE_REPLY -}); - -const receiveExecuteResult = payload => ({ - payload, - type: RECEIVE_EXECUTE_RESULT -}); - -const receiveDisplayData = payload => ({ - payload, - type: RECEIVE_DISPLAY_DATA -}); - -const receiveStream = payload => ({ - payload, - type: RECEIVE_STREAM -}); - // ------------------------------------------------------------------------- // Epics const launchEpic = action$ => action$.ofType(LAUNCH).pipe( tap(() => console.log("launching")), - mergeMap(async () => { - await languagePluginLoader; - console.log("loaded language plugin"); - // using window.pyodide instead of pyodide to get linter to stop yelling ;) - await window.pyodide.loadPackage(["mne"]); - console.log("loaded mne package"); - }), + mergeMap(loadPackages), + mergeMap(utils), map(() => setPyodideStatus(PYODIDE_STATUS.LOADED)) ); @@ -136,9 +111,12 @@ const loadEpochsEpic = (action$, state$) => action$.ofType(LOAD_EPOCHS).pipe( pluck("payload"), filter(filePathsArray => filePathsArray.length >= 1), - map(filePathsArray => loadCSV(filePathsArray)), - map(() => filterIIR(1, 30)), - map(() => + tap(files => console.log('files:', files)), + map((filePathsArray => readFiles(filePathsArray))), + tap(csvArray => console.log('csvs:', csvArray)), + mergeMap(csvArray => loadCSV(csvArray)), + mergeMap(() => filterIIR(1, 30)), + mergeMap(() => epochEvents( { [state$.value.experiment.params.stimulus1.title]: EVENTS.STIMULUS_1, @@ -148,11 +126,10 @@ const loadEpochsEpic = (action$, state$) => 0.8 ) ), - map(epochEventsCommand => epochEventsCommand), map(() => getEpochsInfo(PYODIDE_VARIABLE_NAMES.RAW_EPOCHS)) ); -const loadCleanedEpochsEpic = (action$, state$) => +const loadCleanedEpochsEpic = (action$) => action$.ofType(LOAD_CLEANED_EPOCHS).pipe( pluck("payload"), filter(filePathsArray => filePathsArray.length >= 1), @@ -178,12 +155,13 @@ const cleanEpochsEpic = (action$, state$) => map(() => getEpochsInfo(PYODIDE_VARIABLE_NAMES.RAW_EPOCHS)) ); -const getEpochsInfoEpic = (action$, state$) => +const getEpochsInfoEpic = (action$) => action$.ofType(GET_EPOCHS_INFO).pipe( pluck("payload"), - map(variableName => requestEpochsInfo(variableName)), - map(epochInfoString => - parseSingleQuoteJSON(epochInfoString).map(infoObj => ({ + tap(payload => console.log('payload: ', payload)), + mergeMap(requestEpochsInfo), + map(epochInfoArray => + epochInfoArray.map(infoObj => ({ name: Object.keys(infoObj)[0], value: infoObj[Object.keys(infoObj)[0]] })) diff --git a/app/utils/filesystem/read.js b/app/utils/filesystem/read.js new file mode 100644 index 00000000..23bc63c4 --- /dev/null +++ b/app/utils/filesystem/read.js @@ -0,0 +1,18 @@ +const fs = require("fs"); + +export const readFiles = (filePathsArray) => { + return filePathsArray.map(path => { + console.log('about to read file') + const file = fs.readFileSync(path, 'utf8') + console.log('read file') + return file + }) +} + + + +// ------------------------------------------- +// Helper methods + +const formatFilePath = (filePath: string) => + `"${filePath.replace(/\\/g, "/")}"`; diff --git a/app/utils/pyodide/commands.js b/app/utils/pyodide/commands.js index b4c71a65..4b718834 100644 --- a/app/utils/pyodide/commands.js +++ b/app/utils/pyodide/commands.js @@ -1,80 +1,69 @@ import * as path from "path"; import { readFileSync } from "fs"; +import { languagePluginLoader } from "./pyodide"; + let pyodide; // ----------------------------- // Imports and Utility functions -export const test = async () => { - await window.pyodide.loadPackage(["mne"]); - - const mneCommands = [ - `import numpy as np`, - `import mne`, - `data = np.repeat(np.atleast_2d(np.arange(1000)), 8, axis=0)`, - `info = mne.create_info(8, 250)`, - `raw = mne.io.RawArray(data=data, info=info)`, - `raw.save("test_brainwaves.fif")` - ]; - await window.pyodide.runPython(mneCommands.join("; ")); -}; - -export const loadPackages = async () => window.pyodide.loadPackage(["mne"]); - -export const imports = () => - pyodide.runPython( - readFileSync(path.join(__dirname, "/utils/pyodide/pyimport.py"), "utf8") - ); +export const loadPackages = async () => { + await languagePluginLoader; + console.log("loaded language plugin"); + // using window.pyodide instead of pyodide to get linter to stop yelling ;) + await window.pyodide.loadPackage(["matplotlib", "mne", "pandas"]); + await window.pyodide.runPython("import js"); + console.log("loaded mne package"); -export const utils = () => - pyodide.runPython( +} + +export const utils = async () => + window.pyodide.runPython( readFileSync(path.join(__dirname, "/utils/pyodide/utils.py"), "utf8") ); -export const loadCSV = (filePathArray: Array) => - [ - `files = [${filePathArray.map(filePath => formatFilePath(filePath))}]`, - `replace_ch_names = None`, - `raw = load_data(files, replace_ch_names)` - ].join("\n"); +export const loadCSV = async (csvArray: Array) => { + window.csvArray = csvArray; + // TODO: Pass attached variable name as parameter to load_data + await window.pyodide.runPython( + `raw = load_data()`) +} // --------------------------- // MNE-Related Data Processing -export const loadCleanedEpochs = (filePathArray: Array) => +export const loadCleanedEpochs = (epocsArray: Array) => [ - `files = [${filePathArray.map(filePath => formatFilePath(filePath))}]`, `clean_epochs = concatenate_epochs([read_epochs(file) for file in files])`, `conditions = OrderedDict({key: [value] for (key, value) in clean_epochs.event_id.items()})` ].join("\n"); // NOTE: this command includes a ';' to prevent returning data -export const filterIIR = (lowCutoff: number, highCutoff: number) => - `raw.filter(${lowCutoff}, ${highCutoff}, method='iir');`; +export const filterIIR = async (lowCutoff: number, highCutoff: number) => + window.pyodide.runPython(`raw.filter(${lowCutoff}, ${highCutoff}, method='iir');`) -export const epochEvents = ( +export const epochEvents = async ( eventIDs: { [string]: number }, tmin: number, tmax: number, reject?: Array | string = "None" -) => { - const command = [ - `event_id = ${JSON.stringify(eventIDs)}`, - `tmin=${tmin}`, - `tmax=${tmax}`, - `baseline= (tmin, tmax)`, - `picks = None`, - `reject = ${reject}`, - "events = find_events(raw)", - `raw_epochs = Epochs(raw, events=events, event_id=event_id, +) => window.pyodide.runPython([ + `event_id = ${JSON.stringify(eventIDs)}`, + `tmin=${tmin}`, + `tmax=${tmax}`, + `baseline= (tmin, tmax)`, + `picks = None`, + `reject = ${reject}`, + "events = find_events(raw)", + `raw_epochs = Epochs(raw, events=events, event_id=event_id, tmin=tmin, tmax=tmax, baseline=baseline, reject=reject, preload=True, verbose=False, picks=picks)`, - `conditions = OrderedDict({key: [value] for (key, value) in raw_epochs.event_id.items()})` - ].join("\n"); - return command; -}; + `conditions = OrderedDict({key: [value] for (key, value) in raw_epochs.event_id.items()})` +].join("\n")) -export const requestEpochsInfo = (variableName: string) => - `get_epochs_info(${variableName})`; +export const requestEpochsInfo = async (variableName: string) => { + const pyodideReturn = await window.pyodide.runPython(`get_epochs_info(${variableName})`); + return pyodideReturn +} export const requestChannelInfo = () => `[ch for ch in clean_epochs.ch_names if ch != 'Marker']`; @@ -104,8 +93,3 @@ export const saveEpochs = (workspaceDir: string, subject: string) => ) )})`; -// ------------------------------------------- -// Helper methods - -const formatFilePath = (filePath: string) => - `"${filePath.replace(/\\/g, "/")}"`; diff --git a/app/utils/pyodide/pyimport.py b/app/utils/pyodide/pyimport.py index 2149f2c7..8b137891 100644 --- a/app/utils/pyodide/pyimport.py +++ b/app/utils/pyodide/pyimport.py @@ -1,16 +1 @@ -from time import time, strftime, gmtime -import os -from collections import OrderedDict -from glob import glob -import numpy as np -import pandas as pd # maybe we can remove this dependency -import seaborn as sns -from matplotlib import pyplot as plt - -from mne import (Epochs, RawArray, concatenate_raws, concatenate_epochs, - create_info, find_events, read_epochs, set_eeg_reference) -from mne.channels import read_montage - - -plt.style.use(fivethirtyeight) diff --git a/app/utils/pyodide/utils.py b/app/utils/pyodide/utils.py index 52a5eae0..ce736458 100644 --- a/app/utils/pyodide/utils.py +++ b/app/utils/pyodide/utils.py @@ -1,25 +1,32 @@ from glob import glob import os +from time import time, strftime, gmtime from collections import OrderedDict -from mne import create_info, concatenate_raws, viz -from mne.io import RawArray -from mne.channels import read_montage -import pandas as pd + import numpy as np -import seaborn as sns from matplotlib import pyplot as plt +import pandas as pd # maybe we can remove this dependency +# import seaborn as sns + +from mne import (Epochs, concatenate_raws, concatenate_epochs, create_info, + find_events, read_epochs, set_eeg_reference, viz) +from mne.io import RawArray +from io import StringIO +from mne.channels import read_montage + + +# plt.style.use(fivethirtyeight) -sns.set_context('talk') -sns.set_style('white') +# sns.set_context('talk') +# sns.set_style('white') -def load_data(fnames, sfreq=128., replace_ch_names=None): +def load_data(sfreq=128., replace_ch_names=None): """Load CSV files from the /data directory into a RawArray object. Parameters ---------- - fnames : list - CSV filepaths from which to load data + sfreq : float EEG sampling frequency replace_ch_names : dict | None @@ -31,12 +38,13 @@ def load_data(fnames, sfreq=128., replace_ch_names=None): raw : an instance of mne.io.RawArray The loaded data. """ - + ## js is loaded in loadPackages + ## TODO: Received attached variable name raw = [] - print(fnames) - for fname in fnames: + for csv in js.csvArray: + string_io = StringIO(csv) # read the file - data = pd.read_csv(fname, index_col=0) + data = pd.read_csv(string_io, index_col=0) data = data.dropna() @@ -77,7 +85,11 @@ def load_data(fnames, sfreq=128., replace_ch_names=None): def plot_topo(epochs, conditions=OrderedDict()): - palette = sns.color_palette("hls", len(conditions) + 1) + # palette = sns.color_palette("hls", len(conditions) + 1) + # temp hack, just pull in the color palette from seaborn + palette = [(0.85999999999999999, 0.37119999999999997, 0.33999999999999997), + (0.33999999999999997, 0.85999999999999999, 0.37119999999999997), + (0.37119999999999997, 0.33999999999999997, 0.85999999999999999)] evokeds = [epochs[name].average() for name in (conditions)] evoked_topo = viz.plot_evoked_topo( @@ -189,6 +201,7 @@ def plot_conditions(epochs, ch_ind=0, conditions=OrderedDict(), ci=97.5, return fig, ax def get_epochs_info(epochs): + print('Get Epochs Info:') return [*[{x: len(epochs[x])} for x in epochs.event_id], {"Drop Percentage": round((1 - len(epochs.events) / len(epochs.drop_log)) * 100, 2)},