From d7babda1a723d2cd11e90aa53b6f687689d3074b Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Mon, 27 Nov 2023 22:44:07 -0500 Subject: [PATCH 01/28] add generation function, api, and test --- pyflask/apis/tutorial.py | 30 ++++++ pyflask/manageNeuroconv/__init__.py | 1 + pyflask/manageNeuroconv/manage_neuroconv.py | 105 ++++++++++++++++++- pyflask/tests/test_generate_tutorial_data.py | 8 ++ 4 files changed, 142 insertions(+), 2 deletions(-) create mode 100644 pyflask/apis/tutorial.py create mode 100644 pyflask/tests/test_generate_tutorial_data.py diff --git a/pyflask/apis/tutorial.py b/pyflask/apis/tutorial.py new file mode 100644 index 000000000..c60adb45e --- /dev/null +++ b/pyflask/apis/tutorial.py @@ -0,0 +1,30 @@ +"""API endpoint definitions for interacting with NeuroConv.""" +import traceback + +from flask_restx import Namespace, Resource, reqparse + +from manageNeuroconv import generate_tutorial_data +from errorHandlers import notBadRequestException + +tutorial_api = Namespace("tutorial", description="API route for tutorial operations in the NWB GUIDE.") + +parser = reqparse.RequestParser() +parser.add_argument("interfaces", type=str, action="split", help="Interfaces cannot be converted") + + +@tutorial_api.errorhandler(Exception) +def exception_handler(error): + exceptiondata = traceback.format_exception(type(error), error, error.__traceback__) + return {"message": exceptiondata[-1], "traceback": "".join(exceptiondata)} + + +@tutorial_api.route("/generate/") +class GenerateTutorialData(Resource): + @tutorial_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) + def post(self, base_path: str): + try: + generate_tutorial_data(base_path=base_path) + except Exception as exception: + if notBadRequestException(exception): + tutorial_api.abort(500, str(exception)) + raise exception diff --git a/pyflask/manageNeuroconv/__init__.py b/pyflask/manageNeuroconv/__init__.py index d040232f3..241e8c316 100644 --- a/pyflask/manageNeuroconv/__init__.py +++ b/pyflask/manageNeuroconv/__init__.py @@ -14,6 +14,7 @@ inspect_nwb_file, inspect_nwb_folder, inspect_multiple_filesystem_objects, + generate_tutorial_data, ) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index ea3accbb4..42bcd5467 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -4,9 +4,10 @@ import math import copy import re - +import hashlib +from pathlib import Path from datetime import datetime -from typing import Dict, Optional # , List, Union # TODO: figure way to add these back in without importing other class +from typing import Dict, Optional from shutil import rmtree, copytree from pathlib import Path @@ -648,3 +649,103 @@ def inspect_multiple_filesystem_objects(paths): result = inspect_nwb_folder({"path": tmp_folder_path}) rmtree(tmp_folder_path) return result + + +def _format_spikeglx_meta_file(bin_file_path: str) -> str: + bin_file_path = Path(bin_file_path) + + with open(file=bin_file_path, mode="rb") as io: + file_sha1 = hashlib.sha1(string=io.read()).hexdigest().upper() + file_size = bin_file_path.stat().st_size + file_time_seconds = file_size / (384 * 2) # 384 channels with int16 itemsize + + meta_structure = f"""acqApLfSy=384,384,1 +appVersion=20190327 +fileCreateTime={datetime.now().isoformat(timespec='seconds')} +fileName={bin_file_path} +fileSHA1={file_sha1} +fileSizeBytes={file_size} +fileTimeSecs={file_time_seconds} +firstSample=0 +gateMode=Immediate +imAiRangeMax=0.6 +imAiRangeMin=-0.6 +imCalibrated=true +imDatApi=1.15 +imDatBs_fw=1.1.128 +imDatBsc_fw=1.0.151 +imDatBsc_hw=2.1 +imDatBsc_pn=NP2_QBSC_00 +imDatBsc_sn=434 +imDatFx_hw=1.2 +imDatFx_pn=NP2_FLEX_0 +imDatHs_fw=5.1 +imDatHs_pn=NP2_HS_30 +imDatHs_sn=1116 +imDatPrb_pn=PRB_1_4_0480_1 +imDatPrb_port=2 +imDatPrb_slot=2 +imDatPrb_sn=18194809281 +imDatPrb_type=0 +imLEDEnable=false +imRoFile=C:/fake_windows_path/Gain_ap500_lfp125.imro +imSampRate=30000 +imStdby= +imTrgRising=true +imTrgSource=0 +nSavedChans=385 +snsApLfSy=384,0,1 +snsSaveChanSubset=0:383,768 +syncImInputSlot=2 +syncSourceIdx=0 +syncSourcePeriod=1 +trigMode=Immediate +typeImEnabled=1 +typeNiEnabled=1 +typeThis=imec +userNotes= +~imroTbl=(0,384)(0 0 0 500 125 1)(1 0 0 500 125 1)(2 0 0 500 125 1)(3 0 0 500 125 1)(4 0 0 500 125 1)(5 0 0 500 125 1)(6 0 0 500 125 1)(7 0 0 500 125 1)(8 0 0 500 125 1)(9 0 0 500 125 1)(10 0 0 500 125 1)(11 0 0 500 125 1)(12 0 0 500 125 1)(13 0 0 500 125 1)(14 0 0 500 125 1)(15 0 0 500 125 1)(16 0 0 500 125 1)(17 0 0 500 125 1)(18 0 0 500 125 1)(19 0 0 500 125 1)(20 0 0 500 125 1)(21 0 0 500 125 1)(22 0 0 500 125 1)(23 0 0 500 125 1)(24 0 0 500 125 1)(25 0 0 500 125 1)(26 0 0 500 125 1)(27 0 0 500 125 1)(28 0 0 500 125 1)(29 0 0 500 125 1)(30 0 0 500 125 1)(31 0 0 500 125 1)(32 0 0 500 125 1)(33 0 0 500 125 1)(34 0 0 500 125 1)(35 0 0 500 125 1)(36 0 0 500 125 1)(37 0 0 500 125 1)(38 0 0 500 125 1)(39 0 0 500 125 1)(40 0 0 500 125 1)(41 0 0 500 125 1)(42 0 0 500 125 1)(43 0 0 500 125 1)(44 0 0 500 125 1)(45 0 0 500 125 1)(46 0 0 500 125 1)(47 0 0 500 125 1)(48 0 0 500 125 1)(49 0 0 500 125 1)(50 0 0 500 125 1)(51 0 0 500 125 1)(52 0 0 500 125 1)(53 0 0 500 125 1)(54 0 0 500 125 1)(55 0 0 500 125 1)(56 0 0 500 125 1)(57 0 0 500 125 1)(58 0 0 500 125 1)(59 0 0 500 125 1)(60 0 0 500 125 1)(61 0 0 500 125 1)(62 0 0 500 125 1)(63 0 0 500 125 1)(64 0 0 500 125 1)(65 0 0 500 125 1)(66 0 0 500 125 1)(67 0 0 500 125 1)(68 0 0 500 125 1)(69 0 0 500 125 1)(70 0 0 500 125 1)(71 0 0 500 125 1)(72 0 0 500 125 1)(73 0 0 500 125 1)(74 0 0 500 125 1)(75 0 0 500 125 1)(76 0 0 500 125 1)(77 0 0 500 125 1)(78 0 0 500 125 1)(79 0 0 500 125 1)(80 0 0 500 125 1)(81 0 0 500 125 1)(82 0 0 500 125 1)(83 0 0 500 125 1)(84 0 0 500 125 1)(85 0 0 500 125 1)(86 0 0 500 125 1)(87 0 0 500 125 1)(88 0 0 500 125 1)(89 0 0 500 125 1)(90 0 0 500 125 1)(91 0 0 500 125 1)(92 0 0 500 125 1)(93 0 0 500 125 1)(94 0 0 500 125 1)(95 0 0 500 125 1)(96 0 0 500 125 1)(97 0 0 500 125 1)(98 0 0 500 125 1)(99 0 0 500 125 1)(100 0 0 500 125 1)(101 0 0 500 125 1)(102 0 0 500 125 1)(103 0 0 500 125 1)(104 0 0 500 125 1)(105 0 0 500 125 1)(106 0 0 500 125 1)(107 0 0 500 125 1)(108 0 0 500 125 1)(109 0 0 500 125 1)(110 0 0 500 125 1)(111 0 0 500 125 1)(112 0 0 500 125 1)(113 0 0 500 125 1)(114 0 0 500 125 1)(115 0 0 500 125 1)(116 0 0 500 125 1)(117 0 0 500 125 1)(118 0 0 500 125 1)(119 0 0 500 125 1)(120 0 0 500 125 1)(121 0 0 500 125 1)(122 0 0 500 125 1)(123 0 0 500 125 1)(124 0 0 500 125 1)(125 0 0 500 125 1)(126 0 0 500 125 1)(127 0 0 500 125 1)(128 0 0 500 125 1)(129 0 0 500 125 1)(130 0 0 500 125 1)(131 0 0 500 125 1)(132 0 0 500 125 1)(133 0 0 500 125 1)(134 0 0 500 125 1)(135 0 0 500 125 1)(136 0 0 500 125 1)(137 0 0 500 125 1)(138 0 0 500 125 1)(139 0 0 500 125 1)(140 0 0 500 125 1)(141 0 0 500 125 1)(142 0 0 500 125 1)(143 0 0 500 125 1)(144 0 0 500 125 1)(145 0 0 500 125 1)(146 0 0 500 125 1)(147 0 0 500 125 1)(148 0 0 500 125 1)(149 0 0 500 125 1)(150 0 0 500 125 1)(151 0 0 500 125 1)(152 0 0 500 125 1)(153 0 0 500 125 1)(154 0 0 500 125 1)(155 0 0 500 125 1)(156 0 0 500 125 1)(157 0 0 500 125 1)(158 0 0 500 125 1)(159 0 0 500 125 1)(160 0 0 500 125 1)(161 0 0 500 125 1)(162 0 0 500 125 1)(163 0 0 500 125 1)(164 0 0 500 125 1)(165 0 0 500 125 1)(166 0 0 500 125 1)(167 0 0 500 125 1)(168 0 0 500 125 1)(169 0 0 500 125 1)(170 0 0 500 125 1)(171 0 0 500 125 1)(172 0 0 500 125 1)(173 0 0 500 125 1)(174 0 0 500 125 1)(175 0 0 500 125 1)(176 0 0 500 125 1)(177 0 0 500 125 1)(178 0 0 500 125 1)(179 0 0 500 125 1)(180 0 0 500 125 1)(181 0 0 500 125 1)(182 0 0 500 125 1)(183 0 0 500 125 1)(184 0 0 500 125 1)(185 0 0 500 125 1)(186 0 0 500 125 1)(187 0 0 500 125 1)(188 0 0 500 125 1)(189 0 0 500 125 1)(190 0 0 500 125 1)(191 0 0 500 125 1)(192 0 0 500 125 1)(193 0 0 500 125 1)(194 0 0 500 125 1)(195 0 0 500 125 1)(196 0 0 500 125 1)(197 0 0 500 125 1)(198 0 0 500 125 1)(199 0 0 500 125 1)(200 0 0 500 125 1)(201 0 0 500 125 1)(202 0 0 500 125 1)(203 0 0 500 125 1)(204 0 0 500 125 1)(205 0 0 500 125 1)(206 0 0 500 125 1)(207 0 0 500 125 1)(208 0 0 500 125 1)(209 0 0 500 125 1)(210 0 0 500 125 1)(211 0 0 500 125 1)(212 0 0 500 125 1)(213 0 0 500 125 1)(214 0 0 500 125 1)(215 0 0 500 125 1)(216 0 0 500 125 1)(217 0 0 500 125 1)(218 0 0 500 125 1)(219 0 0 500 125 1)(220 0 0 500 125 1)(221 0 0 500 125 1)(222 0 0 500 125 1)(223 0 0 500 125 1)(224 0 0 500 125 1)(225 0 0 500 125 1)(226 0 0 500 125 1)(227 0 0 500 125 1)(228 0 0 500 125 1)(229 0 0 500 125 1)(230 0 0 500 125 1)(231 0 0 500 125 1)(232 0 0 500 125 1)(233 0 0 500 125 1)(234 0 0 500 125 1)(235 0 0 500 125 1)(236 0 0 500 125 1)(237 0 0 500 125 1)(238 0 0 500 125 1)(239 0 0 500 125 1)(240 0 0 500 125 1)(241 0 0 500 125 1)(242 0 0 500 125 1)(243 0 0 500 125 1)(244 0 0 500 125 1)(245 0 0 500 125 1)(246 0 0 500 125 1)(247 0 0 500 125 1)(248 0 0 500 125 1)(249 0 0 500 125 1)(250 0 0 500 125 1)(251 0 0 500 125 1)(252 0 0 500 125 1)(253 0 0 500 125 1)(254 0 0 500 125 1)(255 0 0 500 125 1)(256 0 0 500 125 1)(257 0 0 500 125 1)(258 0 0 500 125 1)(259 0 0 500 125 1)(260 0 0 500 125 1)(261 0 0 500 125 1)(262 0 0 500 125 1)(263 0 0 500 125 1)(264 0 0 500 125 1)(265 0 0 500 125 1)(266 0 0 500 125 1)(267 0 0 500 125 1)(268 0 0 500 125 1)(269 0 0 500 125 1)(270 0 0 500 125 1)(271 0 0 500 125 1)(272 0 0 500 125 1)(273 0 0 500 125 1)(274 0 0 500 125 1)(275 0 0 500 125 1)(276 0 0 500 125 1)(277 0 0 500 125 1)(278 0 0 500 125 1)(279 0 0 500 125 1)(280 0 0 500 125 1)(281 0 0 500 125 1)(282 0 0 500 125 1)(283 0 0 500 125 1)(284 0 0 500 125 1)(285 0 0 500 125 1)(286 0 0 500 125 1)(287 0 0 500 125 1)(288 0 0 500 125 1)(289 0 0 500 125 1)(290 0 0 500 125 1)(291 0 0 500 125 1)(292 0 0 500 125 1)(293 0 0 500 125 1)(294 0 0 500 125 1)(295 0 0 500 125 1)(296 0 0 500 125 1)(297 0 0 500 125 1)(298 0 0 500 125 1)(299 0 0 500 125 1)(300 0 0 500 125 1)(301 0 0 500 125 1)(302 0 0 500 125 1)(303 0 0 500 125 1)(304 0 0 500 125 1)(305 0 0 500 125 1)(306 0 0 500 125 1)(307 0 0 500 125 1)(308 0 0 500 125 1)(309 0 0 500 125 1)(310 0 0 500 125 1)(311 0 0 500 125 1)(312 0 0 500 125 1)(313 0 0 500 125 1)(314 0 0 500 125 1)(315 0 0 500 125 1)(316 0 0 500 125 1)(317 0 0 500 125 1)(318 0 0 500 125 1)(319 0 0 500 125 1)(320 0 0 500 125 1)(321 0 0 500 125 1)(322 0 0 500 125 1)(323 0 0 500 125 1)(324 0 0 500 125 1)(325 0 0 500 125 1)(326 0 0 500 125 1)(327 0 0 500 125 1)(328 0 0 500 125 1)(329 0 0 500 125 1)(330 0 0 500 125 1)(331 0 0 500 125 1)(332 0 0 500 125 1)(333 0 0 500 125 1)(334 0 0 500 125 1)(335 0 0 500 125 1)(336 0 0 500 125 1)(337 0 0 500 125 1)(338 0 0 500 125 1)(339 0 0 500 125 1)(340 0 0 500 125 1)(341 0 0 500 125 1)(342 0 0 500 125 1)(343 0 0 500 125 1)(344 0 0 500 125 1)(345 0 0 500 125 1)(346 0 0 500 125 1)(347 0 0 500 125 1)(348 0 0 500 125 1)(349 0 0 500 125 1)(350 0 0 500 125 1)(351 0 0 500 125 1)(352 0 0 500 125 1)(353 0 0 500 125 1)(354 0 0 500 125 1)(355 0 0 500 125 1)(356 0 0 500 125 1)(357 0 0 500 125 1)(358 0 0 500 125 1)(359 0 0 500 125 1)(360 0 0 500 125 1)(361 0 0 500 125 1)(362 0 0 500 125 1)(363 0 0 500 125 1)(364 0 0 500 125 1)(365 0 0 500 125 1)(366 0 0 500 125 1)(367 0 0 500 125 1)(368 0 0 500 125 1)(369 0 0 500 125 1)(370 0 0 500 125 1)(371 0 0 500 125 1)(372 0 0 500 125 1)(373 0 0 500 125 1)(374 0 0 500 125 1)(375 0 0 500 125 1)(376 0 0 500 125 1)(377 0 0 500 125 1)(378 0 0 500 125 1)(379 0 0 500 125 1)(380 0 0 500 125 1)(381 0 0 500 125 1)(382 0 0 500 125 1)(383 0 0 500 125 1) +~snsChanMap=(384,384,1)(AP0;0:0)(AP1;1:1)(AP2;2:2)(AP3;3:3)(AP4;4:4)(AP5;5:5)(AP6;6:6)(AP7;7:7)(AP8;8:8)(AP9;9:9)(AP10;10:10)(AP11;11:11)(AP12;12:12)(AP13;13:13)(AP14;14:14)(AP15;15:15)(AP16;16:16)(AP17;17:17)(AP18;18:18)(AP19;19:19)(AP20;20:20)(AP21;21:21)(AP22;22:22)(AP23;23:23)(AP24;24:24)(AP25;25:25)(AP26;26:26)(AP27;27:27)(AP28;28:28)(AP29;29:29)(AP30;30:30)(AP31;31:31)(AP32;32:32)(AP33;33:33)(AP34;34:34)(AP35;35:35)(AP36;36:36)(AP37;37:37)(AP38;38:38)(AP39;39:39)(AP40;40:40)(AP41;41:41)(AP42;42:42)(AP43;43:43)(AP44;44:44)(AP45;45:45)(AP46;46:46)(AP47;47:47)(AP48;48:48)(AP49;49:49)(AP50;50:50)(AP51;51:51)(AP52;52:52)(AP53;53:53)(AP54;54:54)(AP55;55:55)(AP56;56:56)(AP57;57:57)(AP58;58:58)(AP59;59:59)(AP60;60:60)(AP61;61:61)(AP62;62:62)(AP63;63:63)(AP64;64:64)(AP65;65:65)(AP66;66:66)(AP67;67:67)(AP68;68:68)(AP69;69:69)(AP70;70:70)(AP71;71:71)(AP72;72:72)(AP73;73:73)(AP74;74:74)(AP75;75:75)(AP76;76:76)(AP77;77:77)(AP78;78:78)(AP79;79:79)(AP80;80:80)(AP81;81:81)(AP82;82:82)(AP83;83:83)(AP84;84:84)(AP85;85:85)(AP86;86:86)(AP87;87:87)(AP88;88:88)(AP89;89:89)(AP90;90:90)(AP91;91:91)(AP92;92:92)(AP93;93:93)(AP94;94:94)(AP95;95:95)(AP96;96:96)(AP97;97:97)(AP98;98:98)(AP99;99:99)(AP100;100:100)(AP101;101:101)(AP102;102:102)(AP103;103:103)(AP104;104:104)(AP105;105:105)(AP106;106:106)(AP107;107:107)(AP108;108:108)(AP109;109:109)(AP110;110:110)(AP111;111:111)(AP112;112:112)(AP113;113:113)(AP114;114:114)(AP115;115:115)(AP116;116:116)(AP117;117:117)(AP118;118:118)(AP119;119:119)(AP120;120:120)(AP121;121:121)(AP122;122:122)(AP123;123:123)(AP124;124:124)(AP125;125:125)(AP126;126:126)(AP127;127:127)(AP128;128:128)(AP129;129:129)(AP130;130:130)(AP131;131:131)(AP132;132:132)(AP133;133:133)(AP134;134:134)(AP135;135:135)(AP136;136:136)(AP137;137:137)(AP138;138:138)(AP139;139:139)(AP140;140:140)(AP141;141:141)(AP142;142:142)(AP143;143:143)(AP144;144:144)(AP145;145:145)(AP146;146:146)(AP147;147:147)(AP148;148:148)(AP149;149:149)(AP150;150:150)(AP151;151:151)(AP152;152:152)(AP153;153:153)(AP154;154:154)(AP155;155:155)(AP156;156:156)(AP157;157:157)(AP158;158:158)(AP159;159:159)(AP160;160:160)(AP161;161:161)(AP162;162:162)(AP163;163:163)(AP164;164:164)(AP165;165:165)(AP166;166:166)(AP167;167:167)(AP168;168:168)(AP169;169:169)(AP170;170:170)(AP171;171:171)(AP172;172:172)(AP173;173:173)(AP174;174:174)(AP175;175:175)(AP176;176:176)(AP177;177:177)(AP178;178:178)(AP179;179:179)(AP180;180:180)(AP181;181:181)(AP182;182:182)(AP183;183:183)(AP184;184:184)(AP185;185:185)(AP186;186:186)(AP187;187:187)(AP188;188:188)(AP189;189:189)(AP190;190:190)(AP191;191:191)(AP192;192:192)(AP193;193:193)(AP194;194:194)(AP195;195:195)(AP196;196:196)(AP197;197:197)(AP198;198:198)(AP199;199:199)(AP200;200:200)(AP201;201:201)(AP202;202:202)(AP203;203:203)(AP204;204:204)(AP205;205:205)(AP206;206:206)(AP207;207:207)(AP208;208:208)(AP209;209:209)(AP210;210:210)(AP211;211:211)(AP212;212:212)(AP213;213:213)(AP214;214:214)(AP215;215:215)(AP216;216:216)(AP217;217:217)(AP218;218:218)(AP219;219:219)(AP220;220:220)(AP221;221:221)(AP222;222:222)(AP223;223:223)(AP224;224:224)(AP225;225:225)(AP226;226:226)(AP227;227:227)(AP228;228:228)(AP229;229:229)(AP230;230:230)(AP231;231:231)(AP232;232:232)(AP233;233:233)(AP234;234:234)(AP235;235:235)(AP236;236:236)(AP237;237:237)(AP238;238:238)(AP239;239:239)(AP240;240:240)(AP241;241:241)(AP242;242:242)(AP243;243:243)(AP244;244:244)(AP245;245:245)(AP246;246:246)(AP247;247:247)(AP248;248:248)(AP249;249:249)(AP250;250:250)(AP251;251:251)(AP252;252:252)(AP253;253:253)(AP254;254:254)(AP255;255:255)(AP256;256:256)(AP257;257:257)(AP258;258:258)(AP259;259:259)(AP260;260:260)(AP261;261:261)(AP262;262:262)(AP263;263:263)(AP264;264:264)(AP265;265:265)(AP266;266:266)(AP267;267:267)(AP268;268:268)(AP269;269:269)(AP270;270:270)(AP271;271:271)(AP272;272:272)(AP273;273:273)(AP274;274:274)(AP275;275:275)(AP276;276:276)(AP277;277:277)(AP278;278:278)(AP279;279:279)(AP280;280:280)(AP281;281:281)(AP282;282:282)(AP283;283:283)(AP284;284:284)(AP285;285:285)(AP286;286:286)(AP287;287:287)(AP288;288:288)(AP289;289:289)(AP290;290:290)(AP291;291:291)(AP292;292:292)(AP293;293:293)(AP294;294:294)(AP295;295:295)(AP296;296:296)(AP297;297:297)(AP298;298:298)(AP299;299:299)(AP300;300:300)(AP301;301:301)(AP302;302:302)(AP303;303:303)(AP304;304:304)(AP305;305:305)(AP306;306:306)(AP307;307:307)(AP308;308:308)(AP309;309:309)(AP310;310:310)(AP311;311:311)(AP312;312:312)(AP313;313:313)(AP314;314:314)(AP315;315:315)(AP316;316:316)(AP317;317:317)(AP318;318:318)(AP319;319:319)(AP320;320:320)(AP321;321:321)(AP322;322:322)(AP323;323:323)(AP324;324:324)(AP325;325:325)(AP326;326:326)(AP327;327:327)(AP328;328:328)(AP329;329:329)(AP330;330:330)(AP331;331:331)(AP332;332:332)(AP333;333:333)(AP334;334:334)(AP335;335:335)(AP336;336:336)(AP337;337:337)(AP338;338:338)(AP339;339:339)(AP340;340:340)(AP341;341:341)(AP342;342:342)(AP343;343:343)(AP344;344:344)(AP345;345:345)(AP346;346:346)(AP347;347:347)(AP348;348:348)(AP349;349:349)(AP350;350:350)(AP351;351:351)(AP352;352:352)(AP353;353:353)(AP354;354:354)(AP355;355:355)(AP356;356:356)(AP357;357:357)(AP358;358:358)(AP359;359:359)(AP360;360:360)(AP361;361:361)(AP362;362:362)(AP363;363:363)(AP364;364:364)(AP365;365:365)(AP366;366:366)(AP367;367:367)(AP368;368:368)(AP369;369:369)(AP370;370:370)(AP371;371:371)(AP372;372:372)(AP373;373:373)(AP374;374:374)(AP375;375:375)(AP376;376:376)(AP377;377:377)(AP378;378:378)(AP379;379:379)(AP380;380:380)(AP381;381:381)(AP382;382:382)(AP383;383:383)(SY0;768:768) +~snsShankMap=(1,2,480)(0:0:0:1)(0:1:0:1)(0:0:1:1)(0:1:1:1)(0:0:2:1)(0:1:2:1)(0:0:3:1)(0:1:3:1)(0:0:4:1)(0:1:4:1)(0:0:5:1)(0:1:5:1)(0:0:6:1)(0:1:6:1)(0:0:7:1)(0:1:7:1)(0:0:8:1)(0:1:8:1)(0:0:9:1)(0:1:9:1)(0:0:10:1)(0:1:10:1)(0:0:11:1)(0:1:11:1)(0:0:12:1)(0:1:12:1)(0:0:13:1)(0:1:13:1)(0:0:14:1)(0:1:14:1)(0:0:15:1)(0:1:15:1)(0:0:16:1)(0:1:16:1)(0:0:17:1)(0:1:17:1)(0:0:18:1)(0:1:18:1)(0:0:19:1)(0:1:19:1)(0:0:20:1)(0:1:20:1)(0:0:21:1)(0:1:21:1)(0:0:22:1)(0:1:22:1)(0:0:23:1)(0:1:23:1)(0:0:24:1)(0:1:24:1)(0:0:25:1)(0:1:25:1)(0:0:26:1)(0:1:26:1)(0:0:27:1)(0:1:27:1)(0:0:28:1)(0:1:28:1)(0:0:29:1)(0:1:29:1)(0:0:30:1)(0:1:30:1)(0:0:31:1)(0:1:31:1)(0:0:32:1)(0:1:32:1)(0:0:33:1)(0:1:33:1)(0:0:34:1)(0:1:34:1)(0:0:35:1)(0:1:35:1)(0:0:36:1)(0:1:36:1)(0:0:37:1)(0:1:37:1)(0:0:38:1)(0:1:38:1)(0:0:39:1)(0:1:39:1)(0:0:40:1)(0:1:40:1)(0:0:41:1)(0:1:41:1)(0:0:42:1)(0:1:42:1)(0:0:43:1)(0:1:43:1)(0:0:44:1)(0:1:44:1)(0:0:45:1)(0:1:45:1)(0:0:46:1)(0:1:46:1)(0:0:47:1)(0:1:47:1)(0:0:48:1)(0:1:48:1)(0:0:49:1)(0:1:49:1)(0:0:50:1)(0:1:50:1)(0:0:51:1)(0:1:51:1)(0:0:52:1)(0:1:52:1)(0:0:53:1)(0:1:53:1)(0:0:54:1)(0:1:54:1)(0:0:55:1)(0:1:55:1)(0:0:56:1)(0:1:56:1)(0:0:57:1)(0:1:57:1)(0:0:58:1)(0:1:58:1)(0:0:59:1)(0:1:59:1)(0:0:60:1)(0:1:60:1)(0:0:61:1)(0:1:61:1)(0:0:62:1)(0:1:62:1)(0:0:63:1)(0:1:63:1)(0:0:64:1)(0:1:64:1)(0:0:65:1)(0:1:65:1)(0:0:66:1)(0:1:66:1)(0:0:67:1)(0:1:67:1)(0:0:68:1)(0:1:68:1)(0:0:69:1)(0:1:69:1)(0:0:70:1)(0:1:70:1)(0:0:71:1)(0:1:71:1)(0:0:72:1)(0:1:72:1)(0:0:73:1)(0:1:73:1)(0:0:74:1)(0:1:74:1)(0:0:75:1)(0:1:75:1)(0:0:76:1)(0:1:76:1)(0:0:77:1)(0:1:77:1)(0:0:78:1)(0:1:78:1)(0:0:79:1)(0:1:79:1)(0:0:80:1)(0:1:80:1)(0:0:81:1)(0:1:81:1)(0:0:82:1)(0:1:82:1)(0:0:83:1)(0:1:83:1)(0:0:84:1)(0:1:84:1)(0:0:85:1)(0:1:85:1)(0:0:86:1)(0:1:86:1)(0:0:87:1)(0:1:87:1)(0:0:88:1)(0:1:88:1)(0:0:89:1)(0:1:89:1)(0:0:90:1)(0:1:90:1)(0:0:91:1)(0:1:91:1)(0:0:92:1)(0:1:92:1)(0:0:93:1)(0:1:93:1)(0:0:94:1)(0:1:94:1)(0:0:95:1)(0:1:95:0)(0:0:96:1)(0:1:96:1)(0:0:97:1)(0:1:97:1)(0:0:98:1)(0:1:98:1)(0:0:99:1)(0:1:99:1)(0:0:100:1)(0:1:100:1)(0:0:101:1)(0:1:101:1)(0:0:102:1)(0:1:102:1)(0:0:103:1)(0:1:103:1)(0:0:104:1)(0:1:104:1)(0:0:105:1)(0:1:105:1)(0:0:106:1)(0:1:106:1)(0:0:107:1)(0:1:107:1)(0:0:108:1)(0:1:108:1)(0:0:109:1)(0:1:109:1)(0:0:110:1)(0:1:110:1)(0:0:111:1)(0:1:111:1)(0:0:112:1)(0:1:112:1)(0:0:113:1)(0:1:113:1)(0:0:114:1)(0:1:114:1)(0:0:115:1)(0:1:115:1)(0:0:116:1)(0:1:116:1)(0:0:117:1)(0:1:117:1)(0:0:118:1)(0:1:118:1)(0:0:119:1)(0:1:119:1)(0:0:120:1)(0:1:120:1)(0:0:121:1)(0:1:121:1)(0:0:122:1)(0:1:122:1)(0:0:123:1)(0:1:123:1)(0:0:124:1)(0:1:124:1)(0:0:125:1)(0:1:125:1)(0:0:126:1)(0:1:126:1)(0:0:127:1)(0:1:127:1)(0:0:128:1)(0:1:128:1)(0:0:129:1)(0:1:129:1)(0:0:130:1)(0:1:130:1)(0:0:131:1)(0:1:131:1)(0:0:132:1)(0:1:132:1)(0:0:133:1)(0:1:133:1)(0:0:134:1)(0:1:134:1)(0:0:135:1)(0:1:135:1)(0:0:136:1)(0:1:136:1)(0:0:137:1)(0:1:137:1)(0:0:138:1)(0:1:138:1)(0:0:139:1)(0:1:139:1)(0:0:140:1)(0:1:140:1)(0:0:141:1)(0:1:141:1)(0:0:142:1)(0:1:142:1)(0:0:143:1)(0:1:143:1)(0:0:144:1)(0:1:144:1)(0:0:145:1)(0:1:145:1)(0:0:146:1)(0:1:146:1)(0:0:147:1)(0:1:147:1)(0:0:148:1)(0:1:148:1)(0:0:149:1)(0:1:149:1)(0:0:150:1)(0:1:150:1)(0:0:151:1)(0:1:151:1)(0:0:152:1)(0:1:152:1)(0:0:153:1)(0:1:153:1)(0:0:154:1)(0:1:154:1)(0:0:155:1)(0:1:155:1)(0:0:156:1)(0:1:156:1)(0:0:157:1)(0:1:157:1)(0:0:158:1)(0:1:158:1)(0:0:159:1)(0:1:159:1)(0:0:160:1)(0:1:160:1)(0:0:161:1)(0:1:161:1)(0:0:162:1)(0:1:162:1)(0:0:163:1)(0:1:163:1)(0:0:164:1)(0:1:164:1)(0:0:165:1)(0:1:165:1)(0:0:166:1)(0:1:166:1)(0:0:167:1)(0:1:167:1)(0:0:168:1)(0:1:168:1)(0:0:169:1)(0:1:169:1)(0:0:170:1)(0:1:170:1)(0:0:171:1)(0:1:171:1)(0:0:172:1)(0:1:172:1)(0:0:173:1)(0:1:173:1)(0:0:174:1)(0:1:174:1)(0:0:175:1)(0:1:175:1)(0:0:176:1)(0:1:176:1)(0:0:177:1)(0:1:177:1)(0:0:178:1)(0:1:178:1)(0:0:179:1)(0:1:179:1)(0:0:180:1)(0:1:180:1)(0:0:181:1)(0:1:181:1)(0:0:182:1)(0:1:182:1)(0:0:183:1)(0:1:183:1)(0:0:184:1)(0:1:184:1)(0:0:185:1)(0:1:185:1)(0:0:186:1)(0:1:186:1)(0:0:187:1)(0:1:187:1)(0:0:188:1)(0:1:188:1)(0:0:189:1)(0:1:189:1)(0:0:190:1)(0:1:190:1)(0:0:191:1)(0:1:191:1) + """ + return meta_structure + + +def generate_tutorial_data(base_path: str): + from spikeinterface import generate_ground_truth_recording, write_binary_recording, extract_waveforms + from spikeinterface.preprocessing import bandpass_filter + from spikeinterface.exporters import export_to_phy + + base_path = Path(base_path) + + base_output_folder = Path("/Users/codybaker/Downloads/test_generation/") + spikeglx_output_folder = base_output_folder / "spikeglx" + phy_output_folder = base_output_folder / "phy" + + recording, sorting = generate_ground_truth_recording( + sampling_frequency=30_000.0, num_channels=384, num_units=200, seed=0 + ) + artificial_lf_band = bandpass_filter(recording=recording, freq_min=300, freq_max=6000) + waveform_extractor = extract_waveforms(recording=recording, sorting=sorting, mode="memory") + + ap_file_path = spikeglx_output_folder / "Session1_g0" / "Session1_g0_imec0" / "Session1_g0_t0.imec0.ap.bin" + ap_meta_file_path = spikeglx_output_folder / "Session1_g0" / "Session1_g0_imec0" / "Session1_g0_t0.imec0.ap.meta" + lf_file_path = spikeglx_output_folder / "Session1_g0" / "Session1_g0_imec0" / "Session1_g0_t0.imec0.lf.bin" + lf_meta_file_path = spikeglx_output_folder / "Session1_g0" / "Session1_g0_imec0" / "Session1_g0_t0.imec0.lf.meta" + + # Make .bin files + ap_file_path.parent.mkdir(parents=True, exist_ok=True) + write_binary_recording(recording=recording, file_paths=[ap_file_path]) + write_binary_recording(recording=artificial_lf_band, file_paths=[lf_file_path]) + + # Make .meta files + ap_meta_content = _format_spikeglx_meta_file(bin_file_path=ap_file_path) + with open(file=ap_meta_file_path, mode="w") as io: + io.write(ap_meta_content) + + lf_meta_content = _format_spikeglx_meta_file(bin_file_path=lf_file_path) + with open(file=lf_meta_file_path, mode="w") as io: + io.write(lf_meta_content) + + # Make Phy folder + export_to_phy(waveform_extractor=waveform_extractor, output_folder=phy_output_folder, remove_if_exists=True) diff --git a/pyflask/tests/test_generate_tutorial_data.py b/pyflask/tests/test_generate_tutorial_data.py new file mode 100644 index 000000000..5d472898b --- /dev/null +++ b/pyflask/tests/test_generate_tutorial_data.py @@ -0,0 +1,8 @@ +from utils import post +from pathlib import Path + + +def test_generate_tutorial_data(tmp_path, client): + post(f"tutorial/generate/{tmp_path}", tmp_path, client) + assert (Path(tmp_path) / "spikeglx").exists() + assert (Path(tmp_path) / "phy").exists() From f2902ccd51a0ebce19582896bbb2563e26e1d1c2 Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Mon, 27 Nov 2023 22:55:58 -0500 Subject: [PATCH 02/28] fix --- pyflask/tests/test_generate_tutorial_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyflask/tests/test_generate_tutorial_data.py b/pyflask/tests/test_generate_tutorial_data.py index 5d472898b..cf5ea3cbd 100644 --- a/pyflask/tests/test_generate_tutorial_data.py +++ b/pyflask/tests/test_generate_tutorial_data.py @@ -3,6 +3,6 @@ def test_generate_tutorial_data(tmp_path, client): - post(f"tutorial/generate/{tmp_path}", tmp_path, client) + post(f"tutorial/generate/{tmp_path}", client) assert (Path(tmp_path) / "spikeglx").exists() assert (Path(tmp_path) / "phy").exists() From db493a2615b318332b676b5a6870b767a2ccf7f8 Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Tue, 28 Nov 2023 00:01:17 -0500 Subject: [PATCH 03/28] various exposures and fixes --- package.json | 2 +- pyflask/apis/__init__.py | 1 + pyflask/apis/tutorial.py | 25 +++++++++++++++----- pyflask/app.py | 3 ++- pyflask/manageNeuroconv/manage_neuroconv.py | 11 +++++---- pyflask/tests/test_generate_tutorial_data.py | 13 ++++++++-- 6 files changed, 41 insertions(+), 14 deletions(-) diff --git a/package.json b/package.json index 9b6ff91f6..ba0c5d414 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "build:electron:linux": "electron-builder build --linux --publish never", "test": "npm run test:app && npm run test:server", "test:app": "vitest run", - "test:server": "pytest pyflask/tests/ -s", + "test:server": "pytest pyflask/tests/ -s -vv", "wait3s": "node -e \"setTimeout(() => process.exit(0),3000)\"", "test:executable": "concurrently -n EXE,TEST --kill-others --success first \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"npm run wait3s && pytest pyflask/tests/ -s --target http://localhost:3434\"", "test:coverage": "npm run coverage:app && npm run coverage:server", diff --git a/pyflask/apis/__init__.py b/pyflask/apis/__init__.py index 1aed2c307..e786b7865 100644 --- a/pyflask/apis/__init__.py +++ b/pyflask/apis/__init__.py @@ -1,2 +1,3 @@ from .startup import startup_api from .neuroconv import neuroconv_api +from .tutorial import tutorial_api diff --git a/pyflask/apis/tutorial.py b/pyflask/apis/tutorial.py index c60adb45e..c9f7dfa94 100644 --- a/pyflask/apis/tutorial.py +++ b/pyflask/apis/tutorial.py @@ -8,9 +8,6 @@ tutorial_api = Namespace("tutorial", description="API route for tutorial operations in the NWB GUIDE.") -parser = reqparse.RequestParser() -parser.add_argument("interfaces", type=str, action="split", help="Interfaces cannot be converted") - @tutorial_api.errorhandler(Exception) def exception_handler(error): @@ -18,12 +15,28 @@ def exception_handler(error): return {"message": exceptiondata[-1], "traceback": "".join(exceptiondata)} -@tutorial_api.route("/generate/") +# @tutorial_api.route("/generate/") +# class GenerateTutorialData(Resource): +# @tutorial_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) +# def post(self, base_path: str): +# try: +# generate_tutorial_data(base_path=base_path) +# except Exception as exception: +# if notBadRequestException(exception): +# tutorial_api.abort(500, str(exception)) +# raise exception + +generate_tutorial_data_parser = reqparse.RequestParser() +generate_tutorial_data_parser.add_argument("base_path", type=str) + + +@tutorial_api.route("/generate") +@tutorial_api.expect(generate_tutorial_data_parser) class GenerateTutorialData(Resource): @tutorial_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self, base_path: str): + def post(self): try: - generate_tutorial_data(base_path=base_path) + generate_tutorial_data(**tutorial_api.payload) except Exception as exception: if notBadRequestException(exception): tutorial_api.abort(500, str(exception)) diff --git a/pyflask/app.py b/pyflask/app.py index ea666c0ce..0c057ef56 100644 --- a/pyflask/app.py +++ b/pyflask/app.py @@ -20,7 +20,7 @@ from flask_cors import CORS from flask_restx import Api, Resource -from apis import startup_api, neuroconv_api +from apis import startup_api, neuroconv_api, tutorial_api from manageNeuroconv.info import resource_path, STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH app = Flask(__name__) @@ -56,6 +56,7 @@ ) api.add_namespace(startup_api) api.add_namespace(neuroconv_api) +api.add_namespace(tutorial_api) api.init_app(app) registered = {} diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 42bcd5467..c1ebd6843 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -712,15 +712,18 @@ def _format_spikeglx_meta_file(bin_file_path: str) -> str: def generate_tutorial_data(base_path: str): + """ + Autogenerate the data formats needed for the tutorial pipeline. + + Consists of a single-probe single-segment SpikeGLX recording (both AP and LF bands) as well as Phy spiking data. + """ from spikeinterface import generate_ground_truth_recording, write_binary_recording, extract_waveforms from spikeinterface.preprocessing import bandpass_filter from spikeinterface.exporters import export_to_phy base_path = Path(base_path) - - base_output_folder = Path("/Users/codybaker/Downloads/test_generation/") - spikeglx_output_folder = base_output_folder / "spikeglx" - phy_output_folder = base_output_folder / "phy" + spikeglx_output_folder = base_path / "spikeglx" + phy_output_folder = base_path / "phy" recording, sorting = generate_ground_truth_recording( sampling_frequency=30_000.0, num_channels=384, num_units=200, seed=0 diff --git a/pyflask/tests/test_generate_tutorial_data.py b/pyflask/tests/test_generate_tutorial_data.py index cf5ea3cbd..c458ad3fe 100644 --- a/pyflask/tests/test_generate_tutorial_data.py +++ b/pyflask/tests/test_generate_tutorial_data.py @@ -2,7 +2,16 @@ from pathlib import Path -def test_generate_tutorial_data(tmp_path, client): - post(f"tutorial/generate/{tmp_path}", client) +def test_generate_tutorial_data(client, tmp_path: Path): + # assert client is None + result = client.post( + path="/tutorial/generate", + # path=f"/tutorial/generate/base_path={tmp_path}", + # json=dict(), + json=dict(base_path=str(tmp_path)), + # client=client, + ) + assert result is None + assert len(list(Path(tmp_path).iterdir())) != 0 assert (Path(tmp_path) / "spikeglx").exists() assert (Path(tmp_path) / "phy").exists() From 67e2ad878794275435dfd632f60fd190c3571137 Mon Sep 17 00:00:00 2001 From: CodyCBakerPhD Date: Tue, 28 Nov 2023 00:02:34 -0500 Subject: [PATCH 04/28] remove comments --- pyflask/tests/test_generate_tutorial_data.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyflask/tests/test_generate_tutorial_data.py b/pyflask/tests/test_generate_tutorial_data.py index c458ad3fe..482c72c3e 100644 --- a/pyflask/tests/test_generate_tutorial_data.py +++ b/pyflask/tests/test_generate_tutorial_data.py @@ -6,10 +6,7 @@ def test_generate_tutorial_data(client, tmp_path: Path): # assert client is None result = client.post( path="/tutorial/generate", - # path=f"/tutorial/generate/base_path={tmp_path}", - # json=dict(), json=dict(base_path=str(tmp_path)), - # client=client, ) assert result is None assert len(list(Path(tmp_path).iterdir())) != 0 From 4ab16540de1375033d32a8bc34fa81ed3225c03b Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Wed, 29 Nov 2023 12:15:46 -0500 Subject: [PATCH 05/28] fix endpoint --- pyflask/apis/tutorial.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/pyflask/apis/tutorial.py b/pyflask/apis/tutorial.py index c9f7dfa94..cf469e2ad 100644 --- a/pyflask/apis/tutorial.py +++ b/pyflask/apis/tutorial.py @@ -15,19 +15,8 @@ def exception_handler(error): return {"message": exceptiondata[-1], "traceback": "".join(exceptiondata)} -# @tutorial_api.route("/generate/") -# class GenerateTutorialData(Resource): -# @tutorial_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) -# def post(self, base_path: str): -# try: -# generate_tutorial_data(base_path=base_path) -# except Exception as exception: -# if notBadRequestException(exception): -# tutorial_api.abort(500, str(exception)) -# raise exception - generate_tutorial_data_parser = reqparse.RequestParser() -generate_tutorial_data_parser.add_argument("base_path", type=str) +generate_tutorial_data_parser.add_argument("base_path", type=str, required=True) @tutorial_api.route("/generate") @@ -36,7 +25,8 @@ class GenerateTutorialData(Resource): @tutorial_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): try: - generate_tutorial_data(**tutorial_api.payload) + arguments = generate_tutorial_data_parser.parse_args() + generate_tutorial_data(base_path=arguments["base_path"]) except Exception as exception: if notBadRequestException(exception): tutorial_api.abort(500, str(exception)) From 8968a127532ca620272be2af30ff07dadec3d25b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 29 Jan 2024 19:54:42 +0000 Subject: [PATCH 06/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/apis/tutorial.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyflask/apis/tutorial.py b/pyflask/apis/tutorial.py index cf469e2ad..027dca2c0 100644 --- a/pyflask/apis/tutorial.py +++ b/pyflask/apis/tutorial.py @@ -1,4 +1,5 @@ """API endpoint definitions for interacting with NeuroConv.""" + import traceback from flask_restx import Namespace, Resource, reqparse From 79c48a7561ac2c84c722900186bedce537928dd5 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Thu, 1 Feb 2024 14:02:27 -0800 Subject: [PATCH 07/28] Replace tutorial with data(set) generation --- paths.config.json | 2 +- pyflask/apis/__init__.py | 2 +- pyflask/apis/data.py | 54 +++++ pyflask/apis/neuroconv.py | 13 -- pyflask/apis/tutorial.py | 34 ---- pyflask/app.py | 4 +- pyflask/manageNeuroconv/__init__.py | 2 +- pyflask/manageNeuroconv/info/__init__.py | 1 - pyflask/manageNeuroconv/info/urls.py | 2 - pyflask/manageNeuroconv/manage_neuroconv.py | 25 +-- pyflask/tests/test_generate_tutorial_data.py | 9 +- src/renderer/src/dependencies/simple.js | 5 + src/renderer/src/pages.js | 7 - .../pages/documentation/Documentation.js | 6 +- .../pages/guided-mode/options/utils.js | 12 +- .../stories/pages/settings/SettingsPage.js | 62 +++++- .../src/stories/pages/tutorial/Tutorial.js | 184 ------------------ 17 files changed, 154 insertions(+), 270 deletions(-) create mode 100644 pyflask/apis/data.py delete mode 100644 pyflask/apis/tutorial.py delete mode 100644 src/renderer/src/stories/pages/tutorial/Tutorial.js diff --git a/paths.config.json b/paths.config.json index 86631532e..01347166a 100644 --- a/paths.config.json +++ b/paths.config.json @@ -4,6 +4,6 @@ "progress": ["pipelines"], "conversions": ["conversions"], "preview": ["preview"], - "tutorial": ["tutorial"] + "testdata": ["test-data"] } } diff --git a/pyflask/apis/__init__.py b/pyflask/apis/__init__.py index e786b7865..9368a0aa8 100644 --- a/pyflask/apis/__init__.py +++ b/pyflask/apis/__init__.py @@ -1,3 +1,3 @@ from .startup import startup_api from .neuroconv import neuroconv_api -from .tutorial import tutorial_api +from .data import data_api diff --git a/pyflask/apis/data.py b/pyflask/apis/data.py new file mode 100644 index 000000000..fe5315db8 --- /dev/null +++ b/pyflask/apis/data.py @@ -0,0 +1,54 @@ +"""API endpoint definitions for interacting with NeuroConv.""" + +import traceback + +from flask_restx import Namespace, Resource, reqparse + +from manageNeuroconv import generate_test_data, generate_dataset +from errorHandlers import notBadRequestException + +data_api = Namespace("data", description="API route for dataset generation in the NWB GUIDE.") + + +@data_api.errorhandler(Exception) +def exception_handler(error): + exceptiondata = traceback.format_exception(type(error), error, error.__traceback__) + return {"message": exceptiondata[-1], "traceback": "".join(exceptiondata)} + + +generate_test_data_parser = reqparse.RequestParser() +generate_test_data_parser.add_argument("output_path", type=str, required=True) + + +@data_api.route("/generate") +@data_api.expect(generate_test_data_parser) +class GeneratetestData(Resource): + @data_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) + def post(self): + try: + arguments = generate_test_data_parser.parse_args() + generate_test_data(output_path=arguments["output_path"]) + except Exception as exception: + if notBadRequestException(exception): + data_api.abort(500, str(exception)) + raise exception + + +generate_test_dataset_parser = reqparse.RequestParser() +generate_test_dataset_parser.add_argument("output_path", type=str, required=True) +generate_test_dataset_parser.add_argument("input_path", type=str, required=True) + + +@data_api.route("/generate/dataset") +@data_api.expect(generate_test_data_parser) +class GenerateDataset(Resource): + @data_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) + def post(self): + try: + arguments = generate_test_dataset_parser.parse_args() + return generate_dataset(input_path=arguments['input_path'], output_path=arguments['output_path']) + + except Exception as exception: + if notBadRequestException(exception): + data_api.abort(500, str(exception)) + diff --git a/pyflask/apis/neuroconv.py b/pyflask/apis/neuroconv.py index 00bf11e00..7a92f5032 100644 --- a/pyflask/apis/neuroconv.py +++ b/pyflask/apis/neuroconv.py @@ -263,19 +263,6 @@ def post(self): if notBadRequestException(exception): neuroconv_api.abort(500, str(exception)) - -@neuroconv_api.route("/generate_dataset") -class GenerateDataset(Resource): - @neuroconv_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - try: - return generate_dataset(**neuroconv_api.payload) - - except Exception as exception: - if notBadRequestException(exception): - neuroconv_api.abort(500, str(exception)) - - # Create an events endpoint # announcer.announce('test', 'publish') @neuroconv_api.route("/events", methods=["GET"]) diff --git a/pyflask/apis/tutorial.py b/pyflask/apis/tutorial.py deleted file mode 100644 index 027dca2c0..000000000 --- a/pyflask/apis/tutorial.py +++ /dev/null @@ -1,34 +0,0 @@ -"""API endpoint definitions for interacting with NeuroConv.""" - -import traceback - -from flask_restx import Namespace, Resource, reqparse - -from manageNeuroconv import generate_tutorial_data -from errorHandlers import notBadRequestException - -tutorial_api = Namespace("tutorial", description="API route for tutorial operations in the NWB GUIDE.") - - -@tutorial_api.errorhandler(Exception) -def exception_handler(error): - exceptiondata = traceback.format_exception(type(error), error, error.__traceback__) - return {"message": exceptiondata[-1], "traceback": "".join(exceptiondata)} - - -generate_tutorial_data_parser = reqparse.RequestParser() -generate_tutorial_data_parser.add_argument("base_path", type=str, required=True) - - -@tutorial_api.route("/generate") -@tutorial_api.expect(generate_tutorial_data_parser) -class GenerateTutorialData(Resource): - @tutorial_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - try: - arguments = generate_tutorial_data_parser.parse_args() - generate_tutorial_data(base_path=arguments["base_path"]) - except Exception as exception: - if notBadRequestException(exception): - tutorial_api.abort(500, str(exception)) - raise exception diff --git a/pyflask/app.py b/pyflask/app.py index 5724a5ffe..924e9454a 100644 --- a/pyflask/app.py +++ b/pyflask/app.py @@ -21,7 +21,7 @@ from flask_cors import CORS from flask_restx import Api, Resource -from apis import startup_api, neuroconv_api, tutorial_api +from apis import startup_api, neuroconv_api, data_api from manageNeuroconv.info import resource_path, STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH app = Flask(__name__) @@ -57,7 +57,7 @@ ) api.add_namespace(startup_api) api.add_namespace(neuroconv_api) -api.add_namespace(tutorial_api) +api.add_namespace(data_api) api.init_app(app) registered = {} diff --git a/pyflask/manageNeuroconv/__init__.py b/pyflask/manageNeuroconv/__init__.py index 99b7c5dbc..aca8f4125 100644 --- a/pyflask/manageNeuroconv/__init__.py +++ b/pyflask/manageNeuroconv/__init__.py @@ -15,7 +15,7 @@ inspect_nwb_folder, inspect_multiple_filesystem_objects, get_interface_alignment, - generate_tutorial_data, + generate_test_data, ) diff --git a/pyflask/manageNeuroconv/info/__init__.py b/pyflask/manageNeuroconv/info/__init__.py index f1fa1a023..8758c6f86 100644 --- a/pyflask/manageNeuroconv/info/__init__.py +++ b/pyflask/manageNeuroconv/info/__init__.py @@ -3,5 +3,4 @@ GUIDE_ROOT_FOLDER, STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH, - TUTORIAL_SAVE_FOLDER_PATH, ) diff --git a/pyflask/manageNeuroconv/info/urls.py b/pyflask/manageNeuroconv/info/urls.py index e3b3d78d4..82f1dbeaa 100644 --- a/pyflask/manageNeuroconv/info/urls.py +++ b/pyflask/manageNeuroconv/info/urls.py @@ -23,11 +23,9 @@ def resource_path(relative_path): GUIDE_ROOT_FOLDER = Path(Path.home(), data["root"]) STUB_SAVE_FOLDER_PATH = Path(Path.home(), data["root"], *data["subfolders"]["preview"]) CONVERSION_SAVE_FOLDER_PATH = Path(Path.home(), data["root"], *data["subfolders"]["conversions"]) -TUTORIAL_SAVE_FOLDER_PATH = Path(Path.home(), data["root"], *data["subfolders"]["tutorial"]) f.close() # Create all nested home folders STUB_SAVE_FOLDER_PATH.mkdir(exist_ok=True, parents=True) CONVERSION_SAVE_FOLDER_PATH.mkdir(exist_ok=True, parents=True) -TUTORIAL_SAVE_FOLDER_PATH.mkdir(exist_ok=True, parents=True) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 09c43a223..fff210dfe 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -13,7 +13,7 @@ from pathlib import Path from sse import MessageAnnouncer -from .info import GUIDE_ROOT_FOLDER, STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH, TUTORIAL_SAVE_FOLDER_PATH +from .info import GUIDE_ROOT_FOLDER, STUB_SAVE_FOLDER_PATH, CONVERSION_SAVE_FOLDER_PATH announcer = MessageAnnouncer() @@ -572,23 +572,24 @@ def listen_to_neuroconv_events(): yield msg -def generate_dataset(test_data_directory_path: str): - base_path = Path(test_data_directory_path) - output_directory = TUTORIAL_SAVE_FOLDER_PATH / "Dataset" +def generate_dataset(input_path: str, output_path: str): + + base_path = Path(input_path) + output_path = Path(output_path) - if TUTORIAL_SAVE_FOLDER_PATH.exists(): - rmtree(TUTORIAL_SAVE_FOLDER_PATH) + if output_path.exists(): + rmtree(output_path) subjects = ["mouse1", "mouse2"] - sessions = ["070623", "060623"] + sessions = ["Session1", "Session2"] - base_id = "Noise4Sam" + base_id = "Session1" for subject in subjects: for session in sessions: full_id = f"{subject}_{session}" - session_output_directory = output_directory / subject / full_id + session_output_directory = output_path / subject / full_id spikeglx_base_directory = base_path / "spikeglx" / f"{base_id}_g0" phy_base_directory = base_path / "phy" / "phy_example_0" @@ -613,7 +614,7 @@ def generate_dataset(test_data_directory_path: str): phy_output_dir.symlink_to(phy_base_directory, True) - return {"output_directory": str(output_directory)} + return { "output_path": str(output_path) } def inspect_nwb_file(payload): @@ -762,7 +763,7 @@ def _format_spikeglx_meta_file(bin_file_path: str) -> str: return meta_structure -def generate_tutorial_data(base_path: str): +def generate_test_data(output_path: str): """ Autogenerate the data formats needed for the tutorial pipeline. @@ -772,7 +773,7 @@ def generate_tutorial_data(base_path: str): from spikeinterface.preprocessing import bandpass_filter from spikeinterface.exporters import export_to_phy - base_path = Path(base_path) + base_path = Path(output_path) spikeglx_output_folder = base_path / "spikeglx" phy_output_folder = base_path / "phy" diff --git a/pyflask/tests/test_generate_tutorial_data.py b/pyflask/tests/test_generate_tutorial_data.py index 482c72c3e..fbc6fd400 100644 --- a/pyflask/tests/test_generate_tutorial_data.py +++ b/pyflask/tests/test_generate_tutorial_data.py @@ -2,11 +2,12 @@ from pathlib import Path -def test_generate_tutorial_data(client, tmp_path: Path): +def test_generate_test_data(client, tmp_path: Path): # assert client is None - result = client.post( - path="/tutorial/generate", - json=dict(base_path=str(tmp_path)), + result = post( + path="/data/generate", + json=dict(output_path=str(tmp_path)), + client=client ) assert result is None assert len(list(Path(tmp_path).iterdir())) != 0 diff --git a/src/renderer/src/dependencies/simple.js b/src/renderer/src/dependencies/simple.js index 7951eca4c..95d8163f1 100644 --- a/src/renderer/src/dependencies/simple.js +++ b/src/renderer/src/dependencies/simple.js @@ -20,6 +20,11 @@ export const conversionSaveFolderPath = homeDirectory ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.conversions) : ""; +export const testDataFolderPath = homeDirectory + ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.testdata) + : ""; + + // Encryption const IV_LENGTH = 16; const KEY_LENGTH = 32; diff --git a/src/renderer/src/pages.js b/src/renderer/src/pages.js index 4fcd2d7ec..edf1c3ac2 100644 --- a/src/renderer/src/pages.js +++ b/src/renderer/src/pages.js @@ -18,8 +18,6 @@ import { GuidedInspectorPage } from "./stories/pages/guided-mode/options/GuidedI import logo from "../assets/img/logo-guide-draft-transparent-tight.png"; import { GuidedPathExpansionPage } from "./stories/pages/guided-mode/data/GuidedPathExpansion"; -import { TutorialPage } from "./stories/pages/tutorial/Tutorial"; -import tutorialIcon from "./stories/assets/exploration.svg?raw"; import uploadIcon from "./stories/assets/dandi.svg?raw"; import inspectIcon from "./stories/assets/inspect.svg?raw"; import neurosiftIcon from "./stories/assets/neurosift-logo.svg?raw"; @@ -166,11 +164,6 @@ const pages = { label: "Upload", icon: uploadIcon, }), - tutorial: new TutorialPage({ - label: "Tutorial", - icon: tutorialIcon, - group: resourcesGroup, - }), docs: new DocumentationPage({ label: "Documentation", icon: documentationIcon, diff --git a/src/renderer/src/stories/pages/documentation/Documentation.js b/src/renderer/src/stories/pages/documentation/Documentation.js index 312270113..5b913e219 100644 --- a/src/renderer/src/stories/pages/documentation/Documentation.js +++ b/src/renderer/src/stories/pages/documentation/Documentation.js @@ -51,10 +51,10 @@ export class DocumentationPage extends Page {

Getting Started

Converting your data

-

Most users will want to start with the { +

Most users will want to start by generating a { ev.preventDefault(); - this.to("tutorial"); - }}>Tutorial to learn how to use the NWB GUIDE.

+ this.to("settings"); + }}>test dataset to learn how to use the NWB GUIDE.

If you'd like to jump right in, head to the { ev.preventDefault(); this.to("/"); diff --git a/src/renderer/src/stories/pages/guided-mode/options/utils.js b/src/renderer/src/stories/pages/guided-mode/options/utils.js index 1b5b9c7fd..5cb722a8f 100644 --- a/src/renderer/src/stories/pages/guided-mode/options/utils.js +++ b/src/renderer/src/stories/pages/guided-mode/options/utils.js @@ -45,12 +45,21 @@ export const run = async (url, payload, options = {}) => { })); const element = popup.getHtmlContainer(); + const actions = popup.getActions(); const loader = actions.querySelector(".swal2-loader"); const container = document.createElement("div"); container.append(loader); - element.innerText = ""; + + const notDisplayed = element.style.display === 'none' + + Object.assign(element.style, { + marginTop: notDisplayed ? '' : '0', + display: 'unset' + }) + Object.assign(container.style, { + marginTop: notDisplayed ? '' : '25px', display: "flex", flexDirection: "column", alignItems: "center", @@ -64,6 +73,7 @@ export const run = async (url, payload, options = {}) => { } if (!("base" in options)) options.base = "/neuroconv"; + if (options.base[0] !== '/') options.base = `/${options.base}` // Clear private keys from being passed payload = sanitize(structuredClone(payload)); diff --git a/src/renderer/src/stories/pages/settings/SettingsPage.js b/src/renderer/src/stories/pages/settings/SettingsPage.js index 4bc7354c9..a16ed08e4 100644 --- a/src/renderer/src/stories/pages/settings/SettingsPage.js +++ b/src/renderer/src/stories/pages/settings/SettingsPage.js @@ -12,14 +12,19 @@ import { Button } from "../../Button.js"; import { global, remove, save } from "../../../progress/index.js"; import { merge, setUndefinedIfNotDeclared } from "../utils.js"; -import { notyf } from "../../../dependencies/globals.js"; -import { SERVER_FILE_PATH, fs, path, port } from "../../../electron/index.js"; +import { appDirectory, notyf, testDataFolderPath } from "../../../dependencies/globals.js"; +import { SERVER_FILE_PATH, electron, path, port, fs } from "../../../electron/index.js"; + +const { shell } = electron; + import saveSVG from "../../assets/save.svg?raw"; import { header } from "../../forms/utils"; import testingSuiteYaml from "../../../../../../guide_testing_suite.yml"; +import { run } from "../guided-mode/options/utils.js"; +import { joinPath } from "../../../globals.js"; const propertiesToTransform = ["folder_path", "file_path"]; @@ -188,9 +193,58 @@ export class SettingsPage extends Page { testFolderInput.after(generatePipelineButton); }, 100); + const dataOutputPath = joinPath(testDataFolderPath, 'data') + const datasetOutputPath = joinPath(testDataFolderPath, 'dataset') + return html` -

Server Port: ${port}

-

Server File Location: ${SERVER_FILE_PATH}

+
+
+

Server Port: ${port}

+

Server File Location: ${SERVER_FILE_PATH}

+
+
+ ${fs.existsSync(datasetOutputPath) ? new Button({ + label: 'Delete Test Dataset', + onClick: async () => { + fs.rmSync(datasetOutputPath, { recursive: true }) + this.notify(`Test dataset successfully deleted from your system.`) + this.requestUpdate() + } + }) : new Button({ + label: 'Generate Test Dataset', + onClick: async () => { + + await run('generate', { + output_path: dataOutputPath + }, { + title: 'Generating test data', + html: 'This will take ~1min to complete.', + base: 'data' + }).catch((error) => { + this.notify(error.message, "error"); + throw error; + }); + + const { output_path } = await run('generate/dataset', { + input_path: dataOutputPath, + output_path: datasetOutputPath + }, { + title: 'Generating test dataset', + base: 'data' + }).catch((error) => { + this.notify(error.message, "error"); + throw error; + }); + + this.notify(`Test dataset successfully generated at ${output_path}!`); + if (shell) shell.showItemInFolder(output_path); + + this.requestUpdate() + } + }) + } +
+

${this.form} diff --git a/src/renderer/src/stories/pages/tutorial/Tutorial.js b/src/renderer/src/stories/pages/tutorial/Tutorial.js deleted file mode 100644 index 08e1bf8a9..000000000 --- a/src/renderer/src/stories/pages/tutorial/Tutorial.js +++ /dev/null @@ -1,184 +0,0 @@ -import { html } from "lit"; -import { unsafeSVG } from "lit/directives/unsafe-svg.js"; - -import { JSONSchemaForm } from "../../JSONSchemaForm.js"; -import { Page } from "../Page.js"; -import tutorialSchema from "../../../../../../schemas/json/tutorial.schema.json" assert { type: "json" }; - -import { run } from "../guided-mode/options/utils.js"; -import "../../Button.js"; -import { InfoBox } from "../../InfoBox.js"; -import { hasEntry, get, save, remove, resume, global } from "../../../progress/index.js"; - -import { electron } from "../../../electron/index.js"; - -import { InspectorListItem } from "../../preview/inspector/InspectorList"; - -const { shell } = electron; - -const tutorialPipelineName = "NWB GUIDE Tutorial Data"; - -export class TutorialPage extends Page { - header = { - title: "Tutorial Data Generation", - subtitle: - "This page allows you to generate a dataset with multiple subjects and sessions so you can practice using NWB GUIDE before converting your own datasets.", - }; - - constructor(...args) { - super(...args); - } - - render() { - const state = (global.data.tutorial = global.data.tutorial ?? {}); - - const form = new JSONSchemaForm({ - schema: tutorialSchema, - dialogOptions: { - properties: ["createDirectory"], - }, - results: state, - onUpdate: () => global.save(), - }); - - form.style.width = "100%"; - - const entryExists = hasEntry(tutorialPipelineName); - const entry = entryExists ? get(tutorialPipelineName) : {}; - - return html` - ${hasEntry(tutorialPipelineName) - ? html`
-

- A dataset has been generated for you at - shell?.showItemInFolder(entry.project.initialized)} - >${entry.project.initialized} - and preloaded into the ${tutorialPipelineName} conversion pipeline. -

- -

Try to fill out as much metadata as you can while going through this tutorial. Be creative!

- -

- Don't worry about providing incorrect information. We'll let you know if something will break - the conversion. -

- -

- And remember, - the more you provide Global Metadata, the less you'll have to specify later. -

- -
- ${new InspectorListItem({ - message: html`

A Final Note on DANDI Upload

- When you reach the Upload page, make sure to provide a Dandiset ID on the - DANDI Staging server — just so we don't overrun the main server with test - data.`, - type: "warning", - })} -
- -

Let's get started with your first conversion on the GUIDE!

- - { - resume.call(this, tutorialPipelineName); - }} - >${"page-before-exit" in entry ? "Resume" : "Begin"} Conversion - - { - const hasBeenDeleted = await remove(tutorialPipelineName); - if (hasBeenDeleted) this.requestUpdate(); - }} - >Delete Pipeline -
` - : html` - ${new InfoBox({ - header: "How to download test data", - content: `Please refer to the - example data documentation - on the NeuroConv documentation site to learn how to download this dataset using DataLad.`, - })} - -

- - ${form} - - { - await form.validate(); // Will throw an error in the callback - - const { output_directory } = await run("generate_dataset", state, { - title: "Generating tutorial data", - }).catch((error) => { - this.notify(error.message, "error"); - throw error; - }); - - this.notify("Tutorial data successfully generated!"); - if (shell) shell.showItemInFolder(output_directory); - - // Limit the data structures included in the tutorial - const dataStructureResults = { - PhySorting: { - base_directory: output_directory, - format_string_path: - "{subject_id}/{subject_id}_{session_id}/{subject_id}_{session_id}_phy", - }, - SpikeGLXRecording: { - base_directory: output_directory, - format_string_path: - "{subject_id}/{subject_id}_{session_id}/{subject_id}_{session_id}_g0/{subject_id}_{session_id}_g0_imec0/{subject_id}_{session_id}_g0_t0.imec0.ap.bin", - }, - }; - - save({ - info: { - globalState: { - project: { - name: tutorialPipelineName, - initialized: output_directory, // Declare where all the data is here - }, - - // provide data for all supported interfaces - interfaces: Object.keys(dataStructureResults).reduce((acc, key) => { - acc[key] = `${key}Interface`; - return acc; - }, {}), - - // Manually fill out the structure of supported data interfaces - structure: { - results: dataStructureResults, - state: true, - }, - }, - }, - }); - - this.requestUpdate(); // Re-render - }} - >Generate Dataset - `} - `; - } -} - -customElements.get("nwb-tutorial-page") || customElements.define("nwb-tutorial-page", TutorialPage); From 20e578814e1f25a581fc2d92a561ee0af28580f2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 22:02:46 +0000 Subject: [PATCH 08/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/apis/data.py | 3 +- pyflask/apis/neuroconv.py | 1 + pyflask/manageNeuroconv/manage_neuroconv.py | 4 +- pyflask/tests/test_generate_tutorial_data.py | 6 +- src/renderer/src/dependencies/simple.js | 1 - .../pages/guided-mode/options/utils.js | 14 +-- .../stories/pages/settings/SettingsPage.js | 93 ++++++++++--------- 7 files changed, 62 insertions(+), 60 deletions(-) diff --git a/pyflask/apis/data.py b/pyflask/apis/data.py index fe5315db8..122559150 100644 --- a/pyflask/apis/data.py +++ b/pyflask/apis/data.py @@ -46,9 +46,8 @@ class GenerateDataset(Resource): def post(self): try: arguments = generate_test_dataset_parser.parse_args() - return generate_dataset(input_path=arguments['input_path'], output_path=arguments['output_path']) + return generate_dataset(input_path=arguments["input_path"], output_path=arguments["output_path"]) except Exception as exception: if notBadRequestException(exception): data_api.abort(500, str(exception)) - diff --git a/pyflask/apis/neuroconv.py b/pyflask/apis/neuroconv.py index 7a92f5032..8b780c11e 100644 --- a/pyflask/apis/neuroconv.py +++ b/pyflask/apis/neuroconv.py @@ -263,6 +263,7 @@ def post(self): if notBadRequestException(exception): neuroconv_api.abort(500, str(exception)) + # Create an events endpoint # announcer.announce('test', 'publish') @neuroconv_api.route("/events", methods=["GET"]) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index fff210dfe..c4d7cce06 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -573,7 +573,7 @@ def listen_to_neuroconv_events(): def generate_dataset(input_path: str, output_path: str): - + base_path = Path(input_path) output_path = Path(output_path) @@ -614,7 +614,7 @@ def generate_dataset(input_path: str, output_path: str): phy_output_dir.symlink_to(phy_base_directory, True) - return { "output_path": str(output_path) } + return {"output_path": str(output_path)} def inspect_nwb_file(payload): diff --git a/pyflask/tests/test_generate_tutorial_data.py b/pyflask/tests/test_generate_tutorial_data.py index fbc6fd400..80a5f8c53 100644 --- a/pyflask/tests/test_generate_tutorial_data.py +++ b/pyflask/tests/test_generate_tutorial_data.py @@ -4,11 +4,7 @@ def test_generate_test_data(client, tmp_path: Path): # assert client is None - result = post( - path="/data/generate", - json=dict(output_path=str(tmp_path)), - client=client - ) + result = post(path="/data/generate", json=dict(output_path=str(tmp_path)), client=client) assert result is None assert len(list(Path(tmp_path).iterdir())) != 0 assert (Path(tmp_path) / "spikeglx").exists() diff --git a/src/renderer/src/dependencies/simple.js b/src/renderer/src/dependencies/simple.js index 95d8163f1..f9de872a8 100644 --- a/src/renderer/src/dependencies/simple.js +++ b/src/renderer/src/dependencies/simple.js @@ -24,7 +24,6 @@ export const testDataFolderPath = homeDirectory ? joinPath(homeDirectory, paths["root"], ...paths.subfolders.testdata) : ""; - // Encryption const IV_LENGTH = 16; const KEY_LENGTH = 32; diff --git a/src/renderer/src/stories/pages/guided-mode/options/utils.js b/src/renderer/src/stories/pages/guided-mode/options/utils.js index 5cb722a8f..b038913d5 100644 --- a/src/renderer/src/stories/pages/guided-mode/options/utils.js +++ b/src/renderer/src/stories/pages/guided-mode/options/utils.js @@ -51,15 +51,15 @@ export const run = async (url, payload, options = {}) => { const container = document.createElement("div"); container.append(loader); - const notDisplayed = element.style.display === 'none' - + const notDisplayed = element.style.display === "none"; + Object.assign(element.style, { - marginTop: notDisplayed ? '' : '0', - display: 'unset' - }) + marginTop: notDisplayed ? "" : "0", + display: "unset", + }); Object.assign(container.style, { - marginTop: notDisplayed ? '' : '25px', + marginTop: notDisplayed ? "" : "25px", display: "flex", flexDirection: "column", alignItems: "center", @@ -73,7 +73,7 @@ export const run = async (url, payload, options = {}) => { } if (!("base" in options)) options.base = "/neuroconv"; - if (options.base[0] !== '/') options.base = `/${options.base}` + if (options.base[0] !== "/") options.base = `/${options.base}`; // Clear private keys from being passed payload = sanitize(structuredClone(payload)); diff --git a/src/renderer/src/stories/pages/settings/SettingsPage.js b/src/renderer/src/stories/pages/settings/SettingsPage.js index a16ed08e4..485c474b3 100644 --- a/src/renderer/src/stories/pages/settings/SettingsPage.js +++ b/src/renderer/src/stories/pages/settings/SettingsPage.js @@ -17,7 +17,6 @@ import { SERVER_FILE_PATH, electron, path, port, fs } from "../../../electron/in const { shell } = electron; - import saveSVG from "../../assets/save.svg?raw"; import { header } from "../../forms/utils"; @@ -193,8 +192,8 @@ export class SettingsPage extends Page { testFolderInput.after(generatePipelineButton); }, 100); - const dataOutputPath = joinPath(testDataFolderPath, 'data') - const datasetOutputPath = joinPath(testDataFolderPath, 'dataset') + const dataOutputPath = joinPath(testDataFolderPath, "data"); + const datasetOutputPath = joinPath(testDataFolderPath, "dataset"); return html`
@@ -203,46 +202,54 @@ export class SettingsPage extends Page {

Server File Location: ${SERVER_FILE_PATH}

- ${fs.existsSync(datasetOutputPath) ? new Button({ - label: 'Delete Test Dataset', - onClick: async () => { - fs.rmSync(datasetOutputPath, { recursive: true }) - this.notify(`Test dataset successfully deleted from your system.`) - this.requestUpdate() - } - }) : new Button({ - label: 'Generate Test Dataset', - onClick: async () => { - - await run('generate', { - output_path: dataOutputPath - }, { - title: 'Generating test data', - html: 'This will take ~1min to complete.', - base: 'data' - }).catch((error) => { - this.notify(error.message, "error"); - throw error; - }); - - const { output_path } = await run('generate/dataset', { - input_path: dataOutputPath, - output_path: datasetOutputPath - }, { - title: 'Generating test dataset', - base: 'data' - }).catch((error) => { - this.notify(error.message, "error"); - throw error; - }); - - this.notify(`Test dataset successfully generated at ${output_path}!`); - if (shell) shell.showItemInFolder(output_path); - - this.requestUpdate() - } - }) - } + ${fs.existsSync(datasetOutputPath) + ? new Button({ + label: "Delete Test Dataset", + onClick: async () => { + fs.rmSync(datasetOutputPath, { recursive: true }); + this.notify(`Test dataset successfully deleted from your system.`); + this.requestUpdate(); + }, + }) + : new Button({ + label: "Generate Test Dataset", + onClick: async () => { + await run( + "generate", + { + output_path: dataOutputPath, + }, + { + title: "Generating test data", + html: "This will take ~1min to complete.", + base: "data", + } + ).catch((error) => { + this.notify(error.message, "error"); + throw error; + }); + + const { output_path } = await run( + "generate/dataset", + { + input_path: dataOutputPath, + output_path: datasetOutputPath, + }, + { + title: "Generating test dataset", + base: "data", + } + ).catch((error) => { + this.notify(error.message, "error"); + throw error; + }); + + this.notify(`Test dataset successfully generated at ${output_path}!`); + if (shell) shell.showItemInFolder(output_path); + + this.requestUpdate(); + }, + })}
From 29a45ec46385660bea0c009121efd132ae601052 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Thu, 1 Feb 2024 14:04:34 -0800 Subject: [PATCH 09/28] Add scikit-learn dependency --- environments/environment-Linux.yml | 1 + environments/environment-MAC-arm64.yml | 1 + environments/environment-MAC.yml | 1 + environments/environment-Windows.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/environments/environment-Linux.yml b/environments/environment-Linux.yml index 5829c5d04..af3a3d7ee 100644 --- a/environments/environment-Linux.yml +++ b/environments/environment-Linux.yml @@ -21,3 +21,4 @@ dependencies: - dandi >= 0.58.1 - pytest == 7.4.0 - pytest-cov == 4.1.0 + - scikit-learn == 1.4.0 diff --git a/environments/environment-MAC-arm64.yml b/environments/environment-MAC-arm64.yml index f3d239939..2c80ba4cb 100644 --- a/environments/environment-MAC-arm64.yml +++ b/environments/environment-MAC-arm64.yml @@ -25,3 +25,4 @@ dependencies: - dandi >= 0.58.1 - pytest == 7.4.0 - pytest-cov == 4.1.0 + - scikit-learn == 1.4.0 diff --git a/environments/environment-MAC.yml b/environments/environment-MAC.yml index 5829c5d04..af3a3d7ee 100644 --- a/environments/environment-MAC.yml +++ b/environments/environment-MAC.yml @@ -21,3 +21,4 @@ dependencies: - dandi >= 0.58.1 - pytest == 7.4.0 - pytest-cov == 4.1.0 + - scikit-learn == 1.4.0 diff --git a/environments/environment-Windows.yml b/environments/environment-Windows.yml index 11ee2fa6c..731b7bd67 100644 --- a/environments/environment-Windows.yml +++ b/environments/environment-Windows.yml @@ -21,3 +21,4 @@ dependencies: - dandi >= 0.58.1 - pytest == 7.2.2 - pytest-cov == 4.1.0 + - scikit-learn == 1.4.0 From 7f95eafd6dfa8cf41b79f68a4aedf2253b9e68d8 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Thu, 1 Feb 2024 14:17:05 -0800 Subject: [PATCH 10/28] Allow triggering and tracking the progress of the generation on the page itself --- .../stories/pages/settings/SettingsPage.js | 76 ++++++++++--------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/src/renderer/src/stories/pages/settings/SettingsPage.js b/src/renderer/src/stories/pages/settings/SettingsPage.js index 485c474b3..92edf768a 100644 --- a/src/renderer/src/stories/pages/settings/SettingsPage.js +++ b/src/renderer/src/stories/pages/settings/SettingsPage.js @@ -25,6 +25,9 @@ import testingSuiteYaml from "../../../../../../guide_testing_suite.yml"; import { run } from "../guided-mode/options/utils.js"; import { joinPath } from "../../../globals.js"; +const dataOutputPath = joinPath(testDataFolderPath, "data"); +const datasetOutputPath = joinPath(testDataFolderPath, "dataset"); + const propertiesToTransform = ["folder_path", "file_path"]; function saveNewPipelineFromYaml(name, sourceData, rootFolder) { @@ -138,6 +141,43 @@ export class SettingsPage extends Page { return (this.#notification = this.notify(message, type)); }; + generateTestData = async () => { + + await run( + "generate", + { + output_path: dataOutputPath, + }, + { + title: "Generating test data", + html: "This will take ~1min to complete.", + base: "data", + } + ).catch((error) => { + this.notify(error.message, "error"); + throw error; + }); + + const { output_path } = await run( + "generate/dataset", + { + input_path: dataOutputPath, + output_path: datasetOutputPath, + }, + { + title: "Generating test dataset", + base: "data", + } + ).catch((error) => { + this.notify(error.message, "error"); + throw error; + }); + + this.notify(`Test dataset successfully generated at ${output_path}!`); + + return output_path + } + beforeSave = async () => { const { resolved } = this.form; setUndefinedIfNotDeclared(schema.properties, resolved); @@ -192,9 +232,6 @@ export class SettingsPage extends Page { testFolderInput.after(generatePipelineButton); }, 100); - const dataOutputPath = joinPath(testDataFolderPath, "data"); - const datasetOutputPath = joinPath(testDataFolderPath, "dataset"); - return html`
@@ -214,39 +251,8 @@ export class SettingsPage extends Page { : new Button({ label: "Generate Test Dataset", onClick: async () => { - await run( - "generate", - { - output_path: dataOutputPath, - }, - { - title: "Generating test data", - html: "This will take ~1min to complete.", - base: "data", - } - ).catch((error) => { - this.notify(error.message, "error"); - throw error; - }); - - const { output_path } = await run( - "generate/dataset", - { - input_path: dataOutputPath, - output_path: datasetOutputPath, - }, - { - title: "Generating test dataset", - base: "data", - } - ).catch((error) => { - this.notify(error.message, "error"); - throw error; - }); - - this.notify(`Test dataset successfully generated at ${output_path}!`); + const output_path = await this.generateTestData() if (shell) shell.showItemInFolder(output_path); - this.requestUpdate(); }, })} From 4352274d38af5bfebecef05e753b5f542c09b123 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 22:17:37 +0000 Subject: [PATCH 11/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/renderer/src/stories/pages/settings/SettingsPage.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/renderer/src/stories/pages/settings/SettingsPage.js b/src/renderer/src/stories/pages/settings/SettingsPage.js index 92edf768a..de03d291c 100644 --- a/src/renderer/src/stories/pages/settings/SettingsPage.js +++ b/src/renderer/src/stories/pages/settings/SettingsPage.js @@ -142,7 +142,6 @@ export class SettingsPage extends Page { }; generateTestData = async () => { - await run( "generate", { @@ -175,8 +174,8 @@ export class SettingsPage extends Page { this.notify(`Test dataset successfully generated at ${output_path}!`); - return output_path - } + return output_path; + }; beforeSave = async () => { const { resolved } = this.form; @@ -251,7 +250,7 @@ export class SettingsPage extends Page { : new Button({ label: "Generate Test Dataset", onClick: async () => { - const output_path = await this.generateTestData() + const output_path = await this.generateTestData(); if (shell) shell.showItemInFolder(output_path); this.requestUpdate(); }, From f2b1e24517c3a9cf40baf5d660b70890608b8358 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 2 Feb 2024 11:13:04 -0500 Subject: [PATCH 12/28] Update src/renderer/src/stories/pages/settings/SettingsPage.js --- src/renderer/src/stories/pages/settings/SettingsPage.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/renderer/src/stories/pages/settings/SettingsPage.js b/src/renderer/src/stories/pages/settings/SettingsPage.js index de03d291c..b350484e8 100644 --- a/src/renderer/src/stories/pages/settings/SettingsPage.js +++ b/src/renderer/src/stories/pages/settings/SettingsPage.js @@ -149,7 +149,7 @@ export class SettingsPage extends Page { }, { title: "Generating test data", - html: "This will take ~1min to complete.", + html: "This will take several minutes to complete.", base: "data", } ).catch((error) => { From 134e04b933fca9f8a45f54e55432003264317874 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 2 Feb 2024 11:26:28 -0500 Subject: [PATCH 13/28] add sklearn to spec --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6d3fd9b9b..d98018c69 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "build:mac": "npm run build && npm run build:flask && npm run build:electron:mac", "build:linux": "npm run build && npm run build:flask && npm run build:electron:linux", "build:flask": "python -m PyInstaller nwb-guide.spec --log-level DEBUG --clean --noconfirm --distpath ./build/flask", - "build:flask:spec:base": "pyi-makespec --name nwb-guide --onedir --collect-data jsonschema_specifications --collect-all dandi --collect-all keyrings --collect-all unittest --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all hdmf_zarr --collect-all ndx_dandi_icephys --collect-all ci_info ./pyflask/app.py", + "build:flask:spec:base": "pyi-makespec --name nwb-guide --onedir --collect-data jsonschema_specifications --collect-all dandi --collect-all keyrings --collect-all unittest --collect-all nwbinspector --collect-all neuroconv --collect-all pynwb --collect-all hdmf --collect-all hdmf_zarr --collect-all ndx_dandi_icephys --collect-all sklearn --collect-all ci_info ./pyflask/app.py", "build:flask:spec": "npm run build:flask:spec:base && python prepare_pyinstaller_spec.py", "build:electron:win": "electron-builder build --win --publish never", "build:electron:mac": "electron-builder build --mac --publish never", From 591e7cf05eb02d70505b84ca229322135b8c3202 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 2 Feb 2024 11:26:57 -0500 Subject: [PATCH 14/28] add sklearn to spec --- nwb-guide.spec | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nwb-guide.spec b/nwb-guide.spec index e6623d43f..67ba79cb4 100644 --- a/nwb-guide.spec +++ b/nwb-guide.spec @@ -33,6 +33,8 @@ tmp_ret = collect_all('hdmf_zarr') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] tmp_ret = collect_all('ndx_dandi_icephys') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] +tmp_ret = collect_all('sklearn') +datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] tmp_ret = collect_all('ci_info') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] From cab8faeca11af7b6d6c17b9df1345586ff9dd81a Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Fri, 2 Feb 2024 13:08:11 -0800 Subject: [PATCH 15/28] Add open folder button and fix symlinks --- pyflask/manageNeuroconv/manage_neuroconv.py | 6 +-- src/renderer/src/stories/Button.js | 2 +- src/renderer/src/stories/assets/delete.svg | 1 + .../stories/pages/settings/SettingsPage.js | 40 ++++++++++++++----- 4 files changed, 35 insertions(+), 14 deletions(-) create mode 100644 src/renderer/src/stories/assets/delete.svg diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index c4d7cce06..c03d83a05 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -591,9 +591,7 @@ def generate_dataset(input_path: str, output_path: str): full_id = f"{subject}_{session}" session_output_directory = output_path / subject / full_id spikeglx_base_directory = base_path / "spikeglx" / f"{base_id}_g0" - phy_base_directory = base_path / "phy" / "phy_example_0" - - # phy_base_directory.symlink_to(session_output_directory / f'{full_id}_phy', True) + phy_base_directory = base_path / "phy" spikeglx_output_dir = session_output_directory / f"{full_id}_g0" phy_output_dir = session_output_directory / f"{full_id}_phy" @@ -614,7 +612,7 @@ def generate_dataset(input_path: str, output_path: str): phy_output_dir.symlink_to(phy_base_directory, True) - return {"output_path": str(output_path)} + return { "output_path": str(output_path) } def inspect_nwb_file(payload): diff --git a/src/renderer/src/stories/Button.js b/src/renderer/src/stories/Button.js index 850dc32dd..cc984e12b 100644 --- a/src/renderer/src/stories/Button.js +++ b/src/renderer/src/stories/Button.js @@ -34,7 +34,7 @@ export class Button extends LitElement { } .with-icon { - margin-left: 10px; + margin-left: 5px; } svg { diff --git a/src/renderer/src/stories/assets/delete.svg b/src/renderer/src/stories/assets/delete.svg new file mode 100644 index 000000000..e8530f4ac --- /dev/null +++ b/src/renderer/src/stories/assets/delete.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/renderer/src/stories/pages/settings/SettingsPage.js b/src/renderer/src/stories/pages/settings/SettingsPage.js index b350484e8..27d302946 100644 --- a/src/renderer/src/stories/pages/settings/SettingsPage.js +++ b/src/renderer/src/stories/pages/settings/SettingsPage.js @@ -18,6 +18,9 @@ import { SERVER_FILE_PATH, electron, path, port, fs } from "../../../electron/in const { shell } = electron; import saveSVG from "../../assets/save.svg?raw"; +import folderSVG from "../../assets/folder_open.svg?raw"; +import deleteSVG from "../../assets/delete.svg?raw"; +import generateSVG from "../../assets/restart.svg?raw"; import { header } from "../../forms/utils"; @@ -231,6 +234,7 @@ export class SettingsPage extends Page { testFolderInput.after(generatePipelineButton); }, 100); + return html`
@@ -238,23 +242,41 @@ export class SettingsPage extends Page {

Server File Location: ${SERVER_FILE_PATH}

+

Test Dataset

+
${fs.existsSync(datasetOutputPath) - ? new Button({ - label: "Delete Test Dataset", - onClick: async () => { - fs.rmSync(datasetOutputPath, { recursive: true }); - this.notify(`Test dataset successfully deleted from your system.`); - this.requestUpdate(); - }, - }) + ? [ + new Button({ + icon: deleteSVG, + label: "Delete", + size: 'small', + onClick: async () => { + fs.rmSync(datasetOutputPath, { recursive: true }); + this.notify(`Test dataset successfully deleted from your system.`); + this.requestUpdate(); + }, + }), + + new Button({ + icon: folderSVG, + label: "Open", + size: 'small', + onClick: async () => { + if (shell) shell.showItemInFolder(datasetOutputPath); + } + }) + ] : new Button({ - label: "Generate Test Dataset", + label: "Generate", + icon: generateSVG, + size: 'small', onClick: async () => { const output_path = await this.generateTestData(); if (shell) shell.showItemInFolder(output_path); this.requestUpdate(); }, })} +
From 3ed59ce5b81aca0397ab5c622e4de16d12c4011a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 2 Feb 2024 21:08:27 +0000 Subject: [PATCH 16/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 2 +- src/renderer/src/stories/assets/delete.svg | 2 +- .../stories/pages/settings/SettingsPage.js | 69 +++++++++---------- 3 files changed, 36 insertions(+), 37 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index c03d83a05..da720bf0d 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -612,7 +612,7 @@ def generate_dataset(input_path: str, output_path: str): phy_output_dir.symlink_to(phy_base_directory, True) - return { "output_path": str(output_path) } + return {"output_path": str(output_path)} def inspect_nwb_file(payload): diff --git a/src/renderer/src/stories/assets/delete.svg b/src/renderer/src/stories/assets/delete.svg index e8530f4ac..8cd29fc72 100644 --- a/src/renderer/src/stories/assets/delete.svg +++ b/src/renderer/src/stories/assets/delete.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/src/renderer/src/stories/pages/settings/SettingsPage.js b/src/renderer/src/stories/pages/settings/SettingsPage.js index 27d302946..5fa5f1820 100644 --- a/src/renderer/src/stories/pages/settings/SettingsPage.js +++ b/src/renderer/src/stories/pages/settings/SettingsPage.js @@ -234,7 +234,6 @@ export class SettingsPage extends Page { testFolderInput.after(generatePipelineButton); }, 100); - return html`
@@ -242,40 +241,40 @@ export class SettingsPage extends Page {

Server File Location: ${SERVER_FILE_PATH}

-

Test Dataset

-
- ${fs.existsSync(datasetOutputPath) - ? [ - new Button({ - icon: deleteSVG, - label: "Delete", - size: 'small', - onClick: async () => { - fs.rmSync(datasetOutputPath, { recursive: true }); - this.notify(`Test dataset successfully deleted from your system.`); - this.requestUpdate(); - }, - }), - - new Button({ - icon: folderSVG, - label: "Open", - size: 'small', - onClick: async () => { - if (shell) shell.showItemInFolder(datasetOutputPath); - } - }) - ] - : new Button({ - label: "Generate", - icon: generateSVG, - size: 'small', - onClick: async () => { - const output_path = await this.generateTestData(); - if (shell) shell.showItemInFolder(output_path); - this.requestUpdate(); - }, - })} +

Test Dataset

+
+ ${fs.existsSync(datasetOutputPath) + ? [ + new Button({ + icon: deleteSVG, + label: "Delete", + size: "small", + onClick: async () => { + fs.rmSync(datasetOutputPath, { recursive: true }); + this.notify(`Test dataset successfully deleted from your system.`); + this.requestUpdate(); + }, + }), + + new Button({ + icon: folderSVG, + label: "Open", + size: "small", + onClick: async () => { + if (shell) shell.showItemInFolder(datasetOutputPath); + }, + }), + ] + : new Button({ + label: "Generate", + icon: generateSVG, + size: "small", + onClick: async () => { + const output_path = await this.generateTestData(); + if (shell) shell.showItemInFolder(output_path); + this.requestUpdate(); + }, + })}
From 9b7615cd0690892ea097a69535a4d1ba8b3eedca Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Fri, 2 Feb 2024 17:46:22 -0500 Subject: [PATCH 17/28] fix channel length temporarily --- pyflask/manageNeuroconv/manage_neuroconv.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index da720bf0d..59f9da1b2 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -87,7 +87,9 @@ def coerce_schema_compliance_recursive(obj, schema): elif key in schema.get("properties", {}): prop_schema = schema["properties"][key] if prop_schema.get("type") == "number" and (value is None or value == "NaN"): - obj[key] = ( + obj[ + key + ] = ( math.nan ) # Turn None into NaN if a number is expected (JavaScript JSON.stringify turns NaN into None) elif prop_schema.get("type") == "number" and isinstance(value, int): @@ -573,7 +575,6 @@ def listen_to_neuroconv_events(): def generate_dataset(input_path: str, output_path: str): - base_path = Path(input_path) output_path = Path(output_path) @@ -707,7 +708,7 @@ def _format_spikeglx_meta_file(bin_file_path: str) -> str: with open(file=bin_file_path, mode="rb") as io: file_sha1 = hashlib.sha1(string=io.read()).hexdigest().upper() file_size = bin_file_path.stat().st_size - file_time_seconds = file_size / (384 * 2) # 384 channels with int16 itemsize + file_time_seconds = file_size / (385 * 2) # 385 channels with int16 itemsize meta_structure = f"""acqApLfSy=384,384,1 appVersion=20190327 @@ -776,7 +777,7 @@ def generate_test_data(output_path: str): phy_output_folder = base_path / "phy" recording, sorting = generate_ground_truth_recording( - sampling_frequency=30_000.0, num_channels=384, num_units=200, seed=0 + sampling_frequency=30_000.0, num_channels=385, num_units=200, seed=0 ) artificial_lf_band = bandpass_filter(recording=recording, freq_min=300, freq_max=6000) waveform_extractor = extract_waveforms(recording=recording, sorting=sorting, mode="memory") From 651203167bb38bee328ac7bdbbe0a61827571ab5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 2 Feb 2024 22:46:44 +0000 Subject: [PATCH 18/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 59f9da1b2..7bb55ef65 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -87,9 +87,7 @@ def coerce_schema_compliance_recursive(obj, schema): elif key in schema.get("properties", {}): prop_schema = schema["properties"][key] if prop_schema.get("type") == "number" and (value is None or value == "NaN"): - obj[ - key - ] = ( + obj[key] = ( math.nan ) # Turn None into NaN if a number is expected (JavaScript JSON.stringify turns NaN into None) elif prop_schema.get("type") == "number" and isinstance(value, int): From c0caab92bae00d40dc5972793cb151c9151d4b89 Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Fri, 2 Feb 2024 18:02:39 -0500 Subject: [PATCH 19/28] shorten length --- pyflask/manageNeuroconv/manage_neuroconv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 59f9da1b2..97a10382a 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -777,7 +777,7 @@ def generate_test_data(output_path: str): phy_output_folder = base_path / "phy" recording, sorting = generate_ground_truth_recording( - sampling_frequency=30_000.0, num_channels=385, num_units=200, seed=0 + durations=[5.0], sampling_frequency=30_000.0, num_channels=385, num_units=200, seed=0 ) artificial_lf_band = bandpass_filter(recording=recording, freq_min=300, freq_max=6000) waveform_extractor = extract_waveforms(recording=recording, sorting=sorting, mode="memory") From 97015858b65b28d025aa6473ad1495a05e7ab4a4 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Fri, 2 Feb 2024 15:19:41 -0800 Subject: [PATCH 20/28] Update showItemInFolder command --- src/main/main.ts | 4 ++ .../options/GuidedInspectorPage.js | 11 ++--- .../guided-mode/options/GuidedStubPreview.js | 11 ++--- .../stories/pages/settings/SettingsPage.js | 42 +++++++++---------- 4 files changed, 32 insertions(+), 36 deletions(-) diff --git a/src/main/main.ts b/src/main/main.ts index 0ee4524c6..84bf8ee7d 100755 --- a/src/main/main.ts +++ b/src/main/main.ts @@ -472,6 +472,10 @@ ipcMain.on("resize-window", (event, dir) => { globals.mainWindow.setSize(x, y); }); +ipcMain.on('showItemInFolder', function(event, fullPath) { + shell.showItemInFolder(fullPath); +}); + // autoUpdater.on("update-available", () => { // onWindowReady(win => send.call(win, "update_available")); // }); diff --git a/src/renderer/src/stories/pages/guided-mode/options/GuidedInspectorPage.js b/src/renderer/src/stories/pages/guided-mode/options/GuidedInspectorPage.js index b77177414..d3bc752c5 100644 --- a/src/renderer/src/stories/pages/guided-mode/options/GuidedInspectorPage.js +++ b/src/renderer/src/stories/pages/guided-mode/options/GuidedInspectorPage.js @@ -6,7 +6,7 @@ import folderOpenSVG from "../../../assets/folder_open.svg?raw"; import { electron } from "../../../../electron/index.js"; import { getSharedPath, removeFilePaths, truncateFilePaths } from "../../../preview/NWBFilePreview.js"; -const { shell } = electron; +const { ipcRenderer } = electron; import { until } from "lit/directives/until.js"; import { run } from "./utils.js"; import { InspectorList } from "../../../preview/inspector/InspectorList.js"; @@ -63,12 +63,9 @@ export class GuidedInspectorPage extends Page { ...this.headerButtons, html` - shell - ? shell.showItemInFolder( - getSharedPath(getStubArray(this.info.globalState.preview.stubs).map(({ file }) => file)) - ) - : ""} + @click=${() => { + if (ipcRenderer) ipcRenderer.send('showItemInFolder', getSharedPath(getStubArray(this.info.globalState.preview.stubs).map(({ file }) => file))) + }} >${unsafeSVG(folderOpenSVG)}`, ], diff --git a/src/renderer/src/stories/pages/guided-mode/options/GuidedStubPreview.js b/src/renderer/src/stories/pages/guided-mode/options/GuidedStubPreview.js index 093443d98..54917e076 100644 --- a/src/renderer/src/stories/pages/guided-mode/options/GuidedStubPreview.js +++ b/src/renderer/src/stories/pages/guided-mode/options/GuidedStubPreview.js @@ -6,7 +6,7 @@ import folderOpenSVG from "../../../assets/folder_open.svg?raw"; import { electron } from "../../../../electron/index.js"; import { NWBFilePreview, getSharedPath } from "../../../preview/NWBFilePreview.js"; -const { shell } = electron; +const { ipcRenderer } = electron; export const getStubArray = (stubs) => Object.values(stubs) @@ -24,12 +24,9 @@ export class GuidedStubPreviewPage extends Page { controls: () => html` - shell - ? shell.showItemInFolder( - getSharedPath(getStubArray(this.info.globalState.preview.stubs).map((item) => item.file)) - ) - : ""} + @click=${() => { + if (ipcRenderer) ipcRenderer.send('showItemInFolder', getSharedPath(getStubArray(this.info.globalState.preview.stubs).map((item) => item.file))) + }} >${unsafeSVG(folderOpenSVG)}`, }; diff --git a/src/renderer/src/stories/pages/settings/SettingsPage.js b/src/renderer/src/stories/pages/settings/SettingsPage.js index 5fa5f1820..fbad68c1e 100644 --- a/src/renderer/src/stories/pages/settings/SettingsPage.js +++ b/src/renderer/src/stories/pages/settings/SettingsPage.js @@ -12,11 +12,9 @@ import { Button } from "../../Button.js"; import { global, remove, save } from "../../../progress/index.js"; import { merge, setUndefinedIfNotDeclared } from "../utils.js"; -import { appDirectory, notyf, testDataFolderPath } from "../../../dependencies/globals.js"; +import { notyf, testDataFolderPath } from "../../../dependencies/globals.js"; import { SERVER_FILE_PATH, electron, path, port, fs } from "../../../electron/index.js"; -const { shell } = electron; - import saveSVG from "../../assets/save.svg?raw"; import folderSVG from "../../assets/folder_open.svg?raw"; import deleteSVG from "../../assets/delete.svg?raw"; @@ -256,25 +254,25 @@ export class SettingsPage extends Page { }, }), - new Button({ - icon: folderSVG, - label: "Open", - size: "small", - onClick: async () => { - if (shell) shell.showItemInFolder(datasetOutputPath); - }, - }), - ] - : new Button({ - label: "Generate", - icon: generateSVG, - size: "small", - onClick: async () => { - const output_path = await this.generateTestData(); - if (shell) shell.showItemInFolder(output_path); - this.requestUpdate(); - }, - })} + new Button({ + icon: folderSVG, + label: "Open", + size: 'small', + onClick: async () => { + if (electron.ipcRenderer) electron.ipcRenderer.send('showItemInFolder', datasetOutputPath); + } + }) + ] + : new Button({ + label: "Generate", + icon: generateSVG, + size: 'small', + onClick: async () => { + const output_path = await this.generateTestData(); + if (electron.ipcRenderer) electron.ipcRenderer.send('showItemInFolder', output_path); + this.requestUpdate(); + }, + })}
From 722c27dc318746dd88ace983f51b7d509426acd4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 2 Feb 2024 23:19:56 +0000 Subject: [PATCH 21/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../options/GuidedInspectorPage.js | 6 ++- .../guided-mode/options/GuidedStubPreview.js | 6 ++- .../stories/pages/settings/SettingsPage.js | 40 ++++++++++--------- 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/src/renderer/src/stories/pages/guided-mode/options/GuidedInspectorPage.js b/src/renderer/src/stories/pages/guided-mode/options/GuidedInspectorPage.js index d3bc752c5..d34489222 100644 --- a/src/renderer/src/stories/pages/guided-mode/options/GuidedInspectorPage.js +++ b/src/renderer/src/stories/pages/guided-mode/options/GuidedInspectorPage.js @@ -64,7 +64,11 @@ export class GuidedInspectorPage extends Page { html` { - if (ipcRenderer) ipcRenderer.send('showItemInFolder', getSharedPath(getStubArray(this.info.globalState.preview.stubs).map(({ file }) => file))) + if (ipcRenderer) + ipcRenderer.send( + "showItemInFolder", + getSharedPath(getStubArray(this.info.globalState.preview.stubs).map(({ file }) => file)) + ); }} >${unsafeSVG(folderOpenSVG)}`, diff --git a/src/renderer/src/stories/pages/guided-mode/options/GuidedStubPreview.js b/src/renderer/src/stories/pages/guided-mode/options/GuidedStubPreview.js index 54917e076..865a6ffb2 100644 --- a/src/renderer/src/stories/pages/guided-mode/options/GuidedStubPreview.js +++ b/src/renderer/src/stories/pages/guided-mode/options/GuidedStubPreview.js @@ -25,7 +25,11 @@ export class GuidedStubPreviewPage extends Page { html` { - if (ipcRenderer) ipcRenderer.send('showItemInFolder', getSharedPath(getStubArray(this.info.globalState.preview.stubs).map((item) => item.file))) + if (ipcRenderer) + ipcRenderer.send( + "showItemInFolder", + getSharedPath(getStubArray(this.info.globalState.preview.stubs).map((item) => item.file)) + ); }} >${unsafeSVG(folderOpenSVG)}`, diff --git a/src/renderer/src/stories/pages/settings/SettingsPage.js b/src/renderer/src/stories/pages/settings/SettingsPage.js index fbad68c1e..fcffeb13e 100644 --- a/src/renderer/src/stories/pages/settings/SettingsPage.js +++ b/src/renderer/src/stories/pages/settings/SettingsPage.js @@ -254,25 +254,27 @@ export class SettingsPage extends Page { }, }), - new Button({ - icon: folderSVG, - label: "Open", - size: 'small', - onClick: async () => { - if (electron.ipcRenderer) electron.ipcRenderer.send('showItemInFolder', datasetOutputPath); - } - }) - ] - : new Button({ - label: "Generate", - icon: generateSVG, - size: 'small', - onClick: async () => { - const output_path = await this.generateTestData(); - if (electron.ipcRenderer) electron.ipcRenderer.send('showItemInFolder', output_path); - this.requestUpdate(); - }, - })} + new Button({ + icon: folderSVG, + label: "Open", + size: "small", + onClick: async () => { + if (electron.ipcRenderer) + electron.ipcRenderer.send("showItemInFolder", datasetOutputPath); + }, + }), + ] + : new Button({ + label: "Generate", + icon: generateSVG, + size: "small", + onClick: async () => { + const output_path = await this.generateTestData(); + if (electron.ipcRenderer) + electron.ipcRenderer.send("showItemInFolder", output_path); + this.requestUpdate(); + }, + })} From 13a6a1ff5087d95133f4afa1b3c24894a1fd7cd2 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Fri, 2 Feb 2024 15:46:38 -0800 Subject: [PATCH 22/28] Account for manually deleted folders --- .../stories/pages/settings/SettingsPage.js | 55 ++++++++++++------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/src/renderer/src/stories/pages/settings/SettingsPage.js b/src/renderer/src/stories/pages/settings/SettingsPage.js index fbad68c1e..737a5ae82 100644 --- a/src/renderer/src/stories/pages/settings/SettingsPage.js +++ b/src/renderer/src/stories/pages/settings/SettingsPage.js @@ -143,22 +143,27 @@ export class SettingsPage extends Page { }; generateTestData = async () => { - await run( - "generate", - { - output_path: dataOutputPath, - }, - { - title: "Generating test data", - html: "This will take several minutes to complete.", - base: "data", - } - ).catch((error) => { - this.notify(error.message, "error"); - throw error; - }); - const { output_path } = await run( + if (!fs.existsSync(dataOutputPath)) { + await run( + "generate", + { + output_path: dataOutputPath, + }, + { + title: "Generating test data", + html: "This will take several minutes to complete.", + base: "data", + } + ).catch((error) => { + this.notify(error.message, "error"); + throw error; + }); + } + + + + await run( "generate/dataset", { input_path: dataOutputPath, @@ -173,9 +178,10 @@ export class SettingsPage extends Page { throw error; }); - this.notify(`Test dataset successfully generated at ${output_path}!`); - return output_path; + this.notify(`Test dataset successfully generated at ${datasetOutputPath}!`); + + return datasetOutputPath; }; beforeSave = async () => { @@ -232,6 +238,8 @@ export class SettingsPage extends Page { testFolderInput.after(generatePipelineButton); }, 100); + const deleteIfExists = (path) => fs.existsSync(path) ? fs.rmSync(path, { recursive: true }) : '' + return html`
@@ -241,14 +249,15 @@ export class SettingsPage extends Page {

Test Dataset

- ${fs.existsSync(datasetOutputPath) + ${fs.existsSync(datasetOutputPath) && fs.existsSync(dataOutputPath) ? [ new Button({ icon: deleteSVG, label: "Delete", size: "small", onClick: async () => { - fs.rmSync(datasetOutputPath, { recursive: true }); + deleteIfExists(dataOutputPath); + deleteIfExists(datasetOutputPath); this.notify(`Test dataset successfully deleted from your system.`); this.requestUpdate(); }, @@ -259,7 +268,13 @@ export class SettingsPage extends Page { label: "Open", size: 'small', onClick: async () => { - if (electron.ipcRenderer) electron.ipcRenderer.send('showItemInFolder', datasetOutputPath); + if (electron.ipcRenderer) { + if (fs.existsSync(datasetOutputPath)) electron.ipcRenderer.send('showItemInFolder', datasetOutputPath); + else { + this.notify('The test dataset no longer exists!', 'warning') + this.requestUpdate(); + } + } } }) ] From e86ee6fff4738e6c43ae50d73d332146abed0c1d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 2 Feb 2024 23:47:25 +0000 Subject: [PATCH 23/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../stories/pages/settings/SettingsPage.js | 62 +++++++++---------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/src/renderer/src/stories/pages/settings/SettingsPage.js b/src/renderer/src/stories/pages/settings/SettingsPage.js index 737a5ae82..56e413a58 100644 --- a/src/renderer/src/stories/pages/settings/SettingsPage.js +++ b/src/renderer/src/stories/pages/settings/SettingsPage.js @@ -143,7 +143,6 @@ export class SettingsPage extends Page { }; generateTestData = async () => { - if (!fs.existsSync(dataOutputPath)) { await run( "generate", @@ -161,8 +160,6 @@ export class SettingsPage extends Page { }); } - - await run( "generate/dataset", { @@ -178,7 +175,6 @@ export class SettingsPage extends Page { throw error; }); - this.notify(`Test dataset successfully generated at ${datasetOutputPath}!`); return datasetOutputPath; @@ -238,7 +234,7 @@ export class SettingsPage extends Page { testFolderInput.after(generatePipelineButton); }, 100); - const deleteIfExists = (path) => fs.existsSync(path) ? fs.rmSync(path, { recursive: true }) : '' + const deleteIfExists = (path) => (fs.existsSync(path) ? fs.rmSync(path, { recursive: true }) : ""); return html`
@@ -256,38 +252,40 @@ export class SettingsPage extends Page { label: "Delete", size: "small", onClick: async () => { - deleteIfExists(dataOutputPath); - deleteIfExists(datasetOutputPath); + deleteIfExists(dataOutputPath); + deleteIfExists(datasetOutputPath); this.notify(`Test dataset successfully deleted from your system.`); this.requestUpdate(); }, }), - new Button({ - icon: folderSVG, - label: "Open", - size: 'small', - onClick: async () => { - if (electron.ipcRenderer) { - if (fs.existsSync(datasetOutputPath)) electron.ipcRenderer.send('showItemInFolder', datasetOutputPath); - else { - this.notify('The test dataset no longer exists!', 'warning') - this.requestUpdate(); - } - } - } - }) - ] - : new Button({ - label: "Generate", - icon: generateSVG, - size: 'small', - onClick: async () => { - const output_path = await this.generateTestData(); - if (electron.ipcRenderer) electron.ipcRenderer.send('showItemInFolder', output_path); - this.requestUpdate(); - }, - })} + new Button({ + icon: folderSVG, + label: "Open", + size: "small", + onClick: async () => { + if (electron.ipcRenderer) { + if (fs.existsSync(datasetOutputPath)) + electron.ipcRenderer.send("showItemInFolder", datasetOutputPath); + else { + this.notify("The test dataset no longer exists!", "warning"); + this.requestUpdate(); + } + } + }, + }), + ] + : new Button({ + label: "Generate", + icon: generateSVG, + size: "small", + onClick: async () => { + const output_path = await this.generateTestData(); + if (electron.ipcRenderer) + electron.ipcRenderer.send("showItemInFolder", output_path); + this.requestUpdate(); + }, + })}
From 53e451333356a8f0dbfe264d3d637c043a93276f Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Fri, 2 Feb 2024 18:55:18 -0500 Subject: [PATCH 24/28] additional coercions --- pyflask/manageNeuroconv/manage_neuroconv.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index bf06077cf..d66fb4749 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -87,7 +87,9 @@ def coerce_schema_compliance_recursive(obj, schema): elif key in schema.get("properties", {}): prop_schema = schema["properties"][key] if prop_schema.get("type") == "number" and (value is None or value == "NaN"): - obj[key] = ( + obj[ + key + ] = ( math.nan ) # Turn None into NaN if a number is expected (JavaScript JSON.stringify turns NaN into None) elif prop_schema.get("type") == "number" and isinstance(value, int): @@ -775,8 +777,9 @@ def generate_test_data(output_path: str): phy_output_folder = base_path / "phy" recording, sorting = generate_ground_truth_recording( - durations=[5.0], sampling_frequency=30_000.0, num_channels=385, num_units=200, seed=0 + durations=[5.0], sampling_frequency=30_000.0, num_channels=385, dtype="float32", num_units=200, seed=0 ) + recording = (recording / 2.34375).astype(dtype="int16") artificial_lf_band = bandpass_filter(recording=recording, freq_min=300, freq_max=6000) waveform_extractor = extract_waveforms(recording=recording, sorting=sorting, mode="memory") From 3ac7553be6df9efdc0a3ba759cef5f980ecd0e8f Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Sun, 4 Feb 2024 12:34:40 -0500 Subject: [PATCH 25/28] make it even faster --- pyflask/manageNeuroconv/manage_neuroconv.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index d66fb4749..a2c9465f9 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -777,9 +777,10 @@ def generate_test_data(output_path: str): phy_output_folder = base_path / "phy" recording, sorting = generate_ground_truth_recording( - durations=[5.0], sampling_frequency=30_000.0, num_channels=385, dtype="float32", num_units=200, seed=0 + durations=[5.0], sampling_frequency=30_000.0, num_channels=385, dtype="float32", num_units=100, seed=0 ) - recording = (recording / 2.34375).astype(dtype="int16") + recording.set_channel_gains(gains=2.34375) + recording = recording.astype(dtype="int16") artificial_lf_band = bandpass_filter(recording=recording, freq_min=300, freq_max=6000) waveform_extractor = extract_waveforms(recording=recording, sorting=sorting, mode="memory") From e72241f7eef71b374bff0b01cd798cdfc7e8a076 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 4 Feb 2024 17:35:00 +0000 Subject: [PATCH 26/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index a2c9465f9..12723b87d 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -87,9 +87,7 @@ def coerce_schema_compliance_recursive(obj, schema): elif key in schema.get("properties", {}): prop_schema = schema["properties"][key] if prop_schema.get("type") == "number" and (value is None or value == "NaN"): - obj[ - key - ] = ( + obj[key] = ( math.nan ) # Turn None into NaN if a number is expected (JavaScript JSON.stringify turns NaN into None) elif prop_schema.get("type") == "number" and isinstance(value, int): From 550c91928012088922f14aa34a0de3c320340b1f Mon Sep 17 00:00:00 2001 From: codycbakerphd Date: Sun, 4 Feb 2024 13:52:46 -0500 Subject: [PATCH 27/28] enhance readability; make even faster --- pyflask/manageNeuroconv/manage_neuroconv.py | 52 ++++++++++++++++----- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 12723b87d..1a5856c0e 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -87,7 +87,9 @@ def coerce_schema_compliance_recursive(obj, schema): elif key in schema.get("properties", {}): prop_schema = schema["properties"][key] if prop_schema.get("type") == "number" and (value is None or value == "NaN"): - obj[key] = ( + obj[ + key + ] = ( math.nan ) # Turn None into NaN if a number is expected (JavaScript JSON.stringify turns NaN into None) elif prop_schema.get("type") == "number" and isinstance(value, int): @@ -766,21 +768,45 @@ def generate_test_data(output_path: str): Consists of a single-probe single-segment SpikeGLX recording (both AP and LF bands) as well as Phy spiking data. """ - from spikeinterface import generate_ground_truth_recording, write_binary_recording, extract_waveforms - from spikeinterface.preprocessing import bandpass_filter + import spikeinterface + from spikeinterface.extractors import NumpyRecording from spikeinterface.exporters import export_to_phy base_path = Path(output_path) spikeglx_output_folder = base_path / "spikeglx" phy_output_folder = base_path / "phy" - recording, sorting = generate_ground_truth_recording( - durations=[5.0], sampling_frequency=30_000.0, num_channels=385, dtype="float32", num_units=100, seed=0 + # Define NeuroPixel-like values for sampling rates and conversion factors + duration_in_s = 3.0 + number_of_units = 50 + number_of_channels = 385 # Have to include 'sync' channel to be proper SpikeGLX. TODO: artificiate sync pulses + ap_conversion_factor_to_uV = 2.34375 + ap_sampling_frequency = 30_000.0 + lf_sampling_frequency = 2_500.0 + downsample_factor = int(ap_sampling_frequency / lf_sampling_frequency) + + # Generate synthetic spiking and voltage traces with waveforms around them + artificial_ap_band, spiking = spikeinterface.generate_ground_truth_recording( + durations=[duration_in_s], + sampling_frequency=ap_sampling_frequency, + num_channels=number_of_channels, + dtype="float32", + num_units=number_of_units, + seed=0, # Fixed seed for reproducibility + ) + artificial_ap_band.set_channel_gains(gains=ap_conversion_factor_to_uV) + waveform_extractor = spikeinterface.extract_waveforms(recording=artificial_ap_band, sorting=spiking, mode="memory") + int16_artificial_ap_band = artificial_ap_band.astype(dtype="int16") + + # Approximate behavior of LF band with filter and downsampling + # TODO: currently looks a little out of scale? + artificial_lf_filter = spikeinterface.preprocessing.bandpass_filter( + recording=artificial_ap_band, freq_min=10, freq_max=300 + ) + int16_artificial_lf_band = NumpyRecording( + traces_list=artificial_lf_filter.get_traces()[::downsample_factor], + sampling_frequency=lf_sampling_frequency, ) - recording.set_channel_gains(gains=2.34375) - recording = recording.astype(dtype="int16") - artificial_lf_band = bandpass_filter(recording=recording, freq_min=300, freq_max=6000) - waveform_extractor = extract_waveforms(recording=recording, sorting=sorting, mode="memory") ap_file_path = spikeglx_output_folder / "Session1_g0" / "Session1_g0_imec0" / "Session1_g0_t0.imec0.ap.bin" ap_meta_file_path = spikeglx_output_folder / "Session1_g0" / "Session1_g0_imec0" / "Session1_g0_t0.imec0.ap.meta" @@ -789,8 +815,8 @@ def generate_test_data(output_path: str): # Make .bin files ap_file_path.parent.mkdir(parents=True, exist_ok=True) - write_binary_recording(recording=recording, file_paths=[ap_file_path]) - write_binary_recording(recording=artificial_lf_band, file_paths=[lf_file_path]) + spikeinterface.write_binary_recording(recording=int16_artificial_ap_band, file_paths=[ap_file_path]) + spikeinterface.write_binary_recording(recording=int16_artificial_lf_band, file_paths=[lf_file_path]) # Make .meta files ap_meta_content = _format_spikeglx_meta_file(bin_file_path=ap_file_path) @@ -802,4 +828,6 @@ def generate_test_data(output_path: str): io.write(lf_meta_content) # Make Phy folder - export_to_phy(waveform_extractor=waveform_extractor, output_folder=phy_output_folder, remove_if_exists=True) + export_to_phy( + waveform_extractor=waveform_extractor, output_folder=phy_output_folder, remove_if_exists=True, copy_binary=False + ) From 0b26dfd6f4ae3edea3bbee86206b5e14d9ac426e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 4 Feb 2024 18:53:05 +0000 Subject: [PATCH 28/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyflask/manageNeuroconv/manage_neuroconv.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyflask/manageNeuroconv/manage_neuroconv.py b/pyflask/manageNeuroconv/manage_neuroconv.py index 1a5856c0e..79e7caf58 100644 --- a/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/pyflask/manageNeuroconv/manage_neuroconv.py @@ -87,9 +87,7 @@ def coerce_schema_compliance_recursive(obj, schema): elif key in schema.get("properties", {}): prop_schema = schema["properties"][key] if prop_schema.get("type") == "number" and (value is None or value == "NaN"): - obj[ - key - ] = ( + obj[key] = ( math.nan ) # Turn None into NaN if a number is expected (JavaScript JSON.stringify turns NaN into None) elif prop_schema.get("type") == "number" and isinstance(value, int):