diff --git a/.gitignore b/.gitignore index e6aca5b2ff4..bb1b073d818 100755 --- a/.gitignore +++ b/.gitignore @@ -52,6 +52,7 @@ end-to-end-test/results-0-0..xml end-to-end-test/customReport.json end-to-end-test/customReportJSONP.js end-to-end-test/shared/results +api-e2e/json/merged-tests.json wdio/browserstack.txt env/custom.sh image-compare/errors.js @@ -65,3 +66,4 @@ e2e-localdb-workspace/ junit.xml .nvim.lua .luarc.json +api-e2e/validation.js diff --git a/api-e2e/json/nothing.json b/api-e2e/json/nothing.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/api-e2e/logValidator.js b/api-e2e/logValidator.js new file mode 100644 index 00000000000..eea07ce1355 --- /dev/null +++ b/api-e2e/logValidator.js @@ -0,0 +1,239 @@ +const csv = require('csvtojson'); +const _ = require('lodash'); + +const suppressors = require('./suppressors'); + +// passing 11/11 +//let csvFilePath = 'public/extract-2024-10-11T11_47_07.957Z.csv'; + +let hashes = ''; + +//csvFilePath = 'public/extract-2024-10-17T11_47_33.789Z.csv'; // 100k public + +// this is from a run 11/5 with some important failures to analyze +//hashes = `195248191,525365431,1791582317,1051626596,-489658843,352274093,-1357626028,776162948,1438895549,-1205163969,833980087,963616745,-1316865142,-1290515694,666661408,-60179780,588131830,1175958638`; + +// // these are for public 100k +//hashes = `776162948,1791582317,1051626596,1090390722,1634096744,-1357626028,-517670758,-297044990,-723148861,-887269158,-157223238,-1609018360,688132288,-1012605014,663602026,1144436202,2080416722,2102636334,1616351403,116697458,-1145140218,440961037,1788201866,64435906,1526226372,954198833,-333224080,-77262384,-969783118,-2092699827,1122708088,705191799,-910997525,369597899,-589249907,-1733269776,532408117,793803824,2005451592,1946034863,1348899627,1736153850,2004861981,1069675380,-2104618878,-1375780045,-1436966818,-1498011539,-1840476236,1636322635,128823282,712950665,-1807144066,-760096379,1806024553,-1843047838,-1380244121,1908080601,2049138719,75325645,-1564433337,214093972,1584368526,739417518,788541298,-1388038023,-476428894,-1214473123,1798846884,-1336448229,-1479102524,1188628176,1211735112,267198007,-1042782005,1595526137,966120768,-1318775387,-770140034,458149073,-1320441691,1768280517,795362073,440551502,-1083874499,860411865,-1922049418,799514642,2103810746,344484572,-22512484,2075871805,1492814359,2086203791,1046746847,780773665,-1592145833,-1575752643,1528265113,-1374458187,-1474139020,582526768,-2074481817,-1628958227,-2005572217,701108624,-793717747,2078439121,903680661,604898834,1159409033,1376551148,-1025123895,406726936,-1451816705,844353247,-1086596952,-524277403,-1173492647,-263714217,-1961112625,173857178,810614460,776625487,-31077333,-407077673,1152625827,-1204029497,-89652514,950808923,1051361409,655075270,1268317673,-429080735,-180799065,-1918582159,1007754487,445434938,-925144313,-753993222,-828029157,80215185,422205512,-418276888,286347826,1427361749,1841103889,195201275,-278082425,808983848,-558350133,-1982884570,1793886130,104465905,2020005969,-707477776,-1978778776,-146997675,-1605436757,-1288011598,1892578715,-421733597,-2058737510,80797926,2011748125,643117475,656010203,-1838584313,-1379158149,751580838,-300376426,651701037,-284421827,1414588548,1578030439,-1507068659,-552803979,2003360028,1012087380,1119495128,-1018724775,721889439,-1386470488,264752805,-1135621441,53685031,1749320410,-936382059,-271726321,-162436654,-967499791,-924355432,-1967807980,790912582,104911974,105129966,-1015598432,-1547063844,766139610,186806395,1609106547,1137458771,-79793169,-366866963,403522477,1692625915,1918102498,1948188379,1638022655,-7628346,859917518,984138719,-1727165229,-909568546,-1384377258,-1102903634,651279273,-621859020,50108050,721556658,-971350762,55114441,710343427,-1000493817,2065459736,-2010309650,-642014706,1673252949,1288568484,-673406526,-2005371191,1763472544,-1839313216,584288761,899056493,1853556760,-1017785218,-375634338,-217771303,332837231,519954103,-1260921458,1770567231,-848340841,-1575059881,1702190015,1767257846,-90449809,617363688,2134508661,-1353337156,-553248058,1465520871,-2088091542,441238581,263322615,-1243420049,-90200341,533957831,1315006521,1178063322,166833031,-557402244,1128260157,-2090759291,1960991180,-2143298382,2061460930,540705850,-891152750,551129118,364406009,1221598853,788481188,305474268,1341878576,-1625668352,-778412359,-1155472560,463536766,1955600881,-1339515224,1010291232,-1923309873,1182161716,-1303867461,20329773,1116578757,1408885855,1550380971,-1325200994,-941248117,1182607929,-475796764,-1396246057,-93501061,317029493,-960086759,-614052829,-106281559,-1543588030,-490981905,-583187386,-1698981871,452337345,-1946046494,726060739,2076629381`; +// the final 63 +//hashes = '1144436202,2080416722,2102636334,116697458,1616351403,440961037,-1145140218,1788201866,-2092699827,705191799,-1843047838,1806024553,75325645,1908080601,2049138719,-1388038023,788541298,-1214473123,-1564433337,-476428894,1595526137,966120768,-1318775387,-1320441691,-1083874499,2075871805,1159409033,1376551148,-936382059,-271726321,-924355432,-967499791,1948188379,-7628346,1638022655,-621859020,-673406526,710343427,-1000493817,2065459736,-2005371191,617363688,-90449809,2134508661,-1353337156,551129118,2061460930,-2090759291,1960991180,-2143298382,540705850,-891152750,364406009,-1625668352,305474268,1955600881,-1923309873,-106281559,-614052829,-960086759,-1698981871,452337345,-583187386'; +// 10 failures recent +//hashes = '-1934910867,1389452442,525365431,833980087,-1879923626,963616745,2000016239,1386926521,1695497181,-1316865142,799650787,-60179780,666661408,978744191,1175958638,-1840392882'; +//63 that i think have to do with legacy issue of one study not being profiled if it doesn't have any mutations +//hashes = `963616745,525365431,195248191,-1205163969,-489658843,352274093,833980087,588131830,666661408,-60179780,1175958638,-1290515694,-1316865142,1946317495,-459512179,209538309,-1702588459,-375315257,-1096903105,1550369929,-222092833,-1155320233,-1918427737,-1386500079,2016456345,1768219760,1290520296,-933755525,721663558,-349944428,-284954334,-255238155,198928237,-381698904,-2068343766,-593828993,2026738066,303583770,407843497,-639848193,-1435761929,-73533065,-1108056071,-2146896587,1195659709,-2140059509,830374389,-1087705987,-1918570841,-925412627,1032724409,-1364349579,1689544399,-716285275,-47234787,1333012639,-1515321993,418921433,-437389743,1585389039,-283518115,1070204552,-206015797,800445779` +/// failures from 60,000 range +//hashes = '525365431,195248191,-489658843,352274093,-1205163969,833980087,963616745,-1290515694,-1316865142,666661408,-60179780,588131830,1175958638,-1702588459,-1096903105,-375315257,1946317495,-459512179,209538309,2016456345,1768219760,1290520296,-933755525,-349944428,721663558,-255238155,-284954334,-381698904,2026738066,303583770,-1435761929,-593828993,407843497,-639848193,-73533065,-1108056071,-2146896587,1195659709,-925412627,-1364349579,-1087705987,830374389,-2140059509,-716285275,418921433,1585389039,-437389743,-1515321993,1333012639,-206015797,800445779,1753790976,1070204552,1299134243,1426940153,-828883087,1129786076,-1796177052,-1233216071,2037112369,625124883,-1630698357,2106175226,-1346597328,1942465147,-582774632,650014218,-191918718,-391489277,701134203,1015101777,1962031419,276915836,-209644718,1286715598,-1813995434,376853481,1751516243,1955987519,580686336,1597658602,-1502925390,-1956400996,257231649,-909356149,-1369667810,-17479096,885286553,-1531987729,363500082,605285128,-1122921293,-1817676926,-109162004,-904387127,-1233539587,-691963395,-2008149145,1657404911,-1538075689,68897453,-1061532379,1977869468,-512418540,-229443002,-1033104882,-1519665436,1905310044,-1946461845,1909568499,-1549745691,1814467917,-1980131529,-1786905665,1898767734,-1802401273,1220300810,-1282253689,-1657476259,-612577835,-1601749797,-793284027,642059469,-250799439,1439107491,-1318235365,-1947513833,-1490792801,-1920666338,-115987050,-105295877,809969161,1507364211,-1001894295,-1488454849,1936520631,280728358,-786707568,1970635288,1457775849,1500347132,1013786578,-1210913317,-670883245,1315778516,1645895756,-693510682,528166254,316255512,2084147070,-1556616998,-714684062'; + +//genie 28k end ing 10/29 +csvFilePath = 'genie/extract-2024-10-30T01_18_49.413Z.csv'; +//hashes = '449223846,272110920,-1025259712,621573671,1580564797,1430699486,38956759,563047391,1557088045,-36163512,-679048346,-1680780076,69536088,1375412477,-2056887121,-745242224,-456848485,-856335056,1065125809,-1668076144,225366637,500987904,-767115642'; + +// these are genie +//csvFilePath = 'genie/extract-2024-10-17T05_29_43.435Z.csv'; // passing as of 10/18 + +//csvFilePath = 'genie/extract-2024-10-14T16_35_15.134Z.csv'; //passing as of 10/18 +//csvFilePath = `genie/extract-2024-10-14T20_29_07.301Z.csv`; //passing as of 10/18 (20 supressed) +//'genie/extract-2024-10-14T20_29_07.301Z.csv' passing as of 10/18 (20 supressed) +//csvFilePath = 'genie/extract-2024-10-17T18_26_53.439Z.csv'; //2 failures 10/18 -1413667366,767115642 (one is violin, other 1 missing mutation) + +//pulic 50k from 10/24 +csvFilePath = 'public/extract-2024-10-25T00_15_06.092Z.csv'; +// all of the failures in this list involve one single study prad_msk_mdanderson_2023 +hashes = + '-209504909,-531562904,-1368856638,1849796680,-177009809,-1929920957,-1122995550,508841848,-1156904464,2118457351,253120514,1968211789,609196251,25878611,1765601863,431381695,1299594822,-1261047362,-1453203342,-403755087,953159045,220425913,-394456947,1125924226,1038987770,-1861172660,-208108949,2130671075,-1281975003,360646137,645987230,-1024047954,-1732607194,-363417317,-1884406877,-402637663,1979786357,1367728874,164801087,-1800445908,-249776498,-1184006841,-1882674601,-1192795682,-1250501534,987805391'; + +// recent genie +//csvFilePath = 'genie/extract-2024-11-18T01_20_46.023Z.csv'; +//hashes='500987904,1296737223,1974157372,1949632925,1135892867'; + +var axios = require('axios'); +var { runSpecs } = require('./validation'); + +var exclusions = []; + +const cliArgs = parseArgs(); + +const filters = []; + +// these are for genie +//hashes = '1359960927,1531749719,-430446928,-1530985417,-767115642'; + +// convert hashes to RegExps +hashes = hashes.length ? hashes.split(',').map(s => new RegExp(s)) : []; + +if (cliArgs.h) { + hashes = cliArgs.h.split(',').map(h => new RegExp(h)); +} + +const START = cliArgs.s || 0; +const LIMIT = cliArgs.l || 10000000; + +async function main() { + const files = await csv() + .fromFile(csvFilePath) + .then(async jsonObj => { + // clean out errant leading single quote + jsonObj.forEach(d => (d['@hash'] = d['@hash'].replace(/^'/, ''))); + + let uniq = _.uniqBy(jsonObj, '@hash') + .filter(d => { + return _.every( + exclusions.map(re => re.test(d['@url']) === false) + ); + }) + .filter(d => { + return ( + filters.length === 0 || + _.every(filters.map(re => re.test(d['@url']) === true)) + ); + }) + .filter(d => { + return ( + hashes.length === 0 || + _.some(hashes.map(re => re.test(d['@hash']) === true)) + ); + }); + // .filter(d => { + // const data = JSON.parse(d['@data']); + // + // console.log(int); + // return int.length === 0; + // }); + + const tests = uniq.slice(START, START + LIMIT).reduce((aggr, d) => { + //delete data.genomicProfiles; + //delete data.genomicDataFilters; + + try { + let data = JSON.parse(d['@data']); + + const url = d['@url'] + .replace(/^"|"$/g, '') + .replace(/^\/\/[^\/]*/, '') + .replace(/\/api\//, '/api/column-store/'); + + const label = d['@url'] + .match(/\/api\/[^\/]*/i)[0] + .replace(/\/api\//, '') + .split('-') + .map(s => s.replace(/^./, ss => ss.toUpperCase())) + .join(''); + + aggr.push({ + hash: d['@hash'], + label, + data, + url, + }); + } catch (err) { + console.log(err); + } + return aggr; + }, []); + + // tests.forEach((t)=>{ + // console.log(t.data.studyIds || t.data.studyViewFilter.studyIds); + // }) + + const fakeFiles = [ + { + file: 'fake', + suites: [ + { + tests, + }, + ], + }, + ]; + + return fakeFiles; + }); + + runSpecs( + files, + axios, + 'http://localhost:8082', + cliArgs.v || '', + onFail, + suppressors + ); +} + +main(); + +const onFail = (args, report) => { + //console.log(JSON.stringify(args.data, null, 5)); + try { + //console.log(report.clDataSorted[0]); + //console.log(report.legacyDataSorted[0]); + } catch (ex) {} + + // console.log( + // _.sum( + // JSON.stringify(report.clDataSorted) + // .split('') + // .map(c => c.charCodeAt(0)) + // ), + // _.sum( + // JSON.stringify(report.legacyDataSorted) + // .split('') + // .map(c => c.charCodeAt(0)) + // ) + // ); + + const url = 'http://localhost:8082' + args.url; + + const curl = ` + curl '${url}' + -H 'accept: application/json, text/plain, */*' + -H 'accept-language: en-US,en;q=0.9' + -H 'cache-control: no-cache' + -H 'content-type: application/json' + -H 'cookie: _ga_ET18FDC3P1=GS1.1.1727902893.87.0.1727902893.0.0.0; _gid=GA1.2.1570078648.1728481898; _ga_CKJ2CEEFD8=GS1.1.1728589307.172.1.1728589613.0.0.0; _ga_5260NDGD6Z=GS1.1.1728612388.318.1.1728612389.0.0.0; _gat_gtag_UA_17134933_2=1; _ga=GA1.1.1260093286.1710808634; _ga_334HHWHCPJ=GS1.1.1728647421.32.1.1728647514.0.0.0' + -H 'pragma: no-cache' + -H 'priority: u=1, i' + -H 'sec-ch-ua: "Google Chrome";v="129", "Not=A?Brand";v="8", "Chromium";v="129"' + -H 'sec-ch-ua-mobile: ?0' + -H 'sec-ch-ua-platform: "macOS"' + -H 'sec-fetch-dest: empty' + -H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36' + --data-raw '${JSON.stringify(args.data)}'; + `; + + cliArgs.c && + console.log( + curl + .trim() + .split('\n') + .join('') + ); + // + const studyIds = args.data.studyIds || args.data.studyViewFilter.studyIds; + cliArgs.u && + console.log( + `http://localhost:8082/study/summary?id=${studyIds.join( + ',' + )}#filterJson=${JSON.stringify(args.data)}` + ); +}; + +function parseArgs() { + const args = process.argv.slice(2); + + const pairs = args.filter(s => /=/.test(s)); + + const single = args.filter(s => !/=/.test(s)); + + const obj = {}; + + single.forEach(a => { + obj[a.replace(/^-/, '')] = true; + }); + + pairs.forEach(p => { + const tuple = p.split('='); + obj[tuple[0].replace(/^-/, '')] = tuple[1]; + }); + + return obj; +} diff --git a/api-e2e/merge-tests.js b/api-e2e/merge-tests.js new file mode 100644 index 00000000000..614fd272c27 --- /dev/null +++ b/api-e2e/merge-tests.js @@ -0,0 +1,33 @@ +const fs = require('fs/promises'); +const path = require('path'); + +// get rid of railing slash +const BACKEND_ROOT = (process.env.BACKEND_ROOT || '').replace(/\/$/, ''); + +const SPEC_ROOT = `${BACKEND_ROOT}/test/api-e2e/specs`; + +async function mergeTests() { + const files = (await fs.readdir(SPEC_ROOT)).map(fileName => { + return path.join(SPEC_ROOT, fileName); + }); + + const jsons = files.map(path => { + return fs.readFile(path).then(data => { + try { + const json = JSON.parse(data); + return { file: path, suites: json }; + } catch (ex) { + console.log('invalid apiTest json spec'); + return []; + } + }); + }); + + Promise.all(jsons) + .then(d => { + fs.writeFile('./api-e2e/json/merged-tests.json', JSON.stringify(d)); + }) + .then(r => console.log('merged-tests.json written')); +} + +mergeTests(); diff --git a/api-e2e/run.js b/api-e2e/run.js new file mode 100644 index 00000000000..f68a9952b50 --- /dev/null +++ b/api-e2e/run.js @@ -0,0 +1,26 @@ +var json = require('./json/merged-tests.json'); +var axios = require('axios'); +var { validate, reportValidationResult, runSpecs } = require('./validation'); +const test = json[1].suites[0].tests[0]; + +const host = process.env.API_TEST_HOST || 'http://localhost:8082'; + +console.log(`RUNNING TESTS AGAINST: ${host}`); + +async function main() { + const start = Date.now(); + + const fileFilter = process.env.API_TEST_FILTER || ''; + + const files = fileFilter?.trim().length + ? json.filter(f => new RegExp(fileFilter).test(f.file)) + : json; + + await axios.get(`${host}/api/info`).then(r => console.log(r.data)); + + await runSpecs(files, axios, host); + + console.log(`Elapsed: ${Date.now() - start}`); +} + +main(); diff --git a/api-e2e/suppressors.js b/api-e2e/suppressors.js new file mode 100644 index 00000000000..822f62e160e --- /dev/null +++ b/api-e2e/suppressors.js @@ -0,0 +1,339 @@ +const _ = require('lodash'); + +const suppressors = [ + // function(report) { + // return ( + // report.clDataSorted[0].counts.find(m => m.value == 'not_mutated') + // .count === + // report.legacyDataSorted[0].counts.find( + // m => m.value == 'not_profiled' + // ).count + // ); + // }, + // function(report) { + // const diff = _.difference( + // report.chResult.body[0].counts.map(c => c.value), + // report.legacyResult.body[0].counts.map(c => c.value) + // ); + // return ( + // diff.includes('not_profiled') && + // _.sumBy(report.legacyResult.body[0].counts, 'count') === + // _.sumBy(report.chResult.body[0].counts, 'count') + // ); + // }, + // function(report) { + // return ( + // report.test.data.studyViewFilter.clinicalDataFilters[0].values + // .length > 10 + // ); + // }, + + // function(report) { + // return report.test.data.clinicalDataFilters[0].values.length > 10; + // }, + + function(report) { + return ( + report.test.data.customDataFilters || + report.test.data.studyViewFilter.customDataFilters + ); + }, + + function(report) { + return ( + _.intersection( + report.test.data.studyIds || + report.test.data.studyViewFilter.studyIds, + [ + 'genie_private', + 'prad_organoids_msk_2022', // has data corruption (duplicate samples with casing) + ] + ).length > 0 + ); + }, + + function(report) { + // this is broke legacy endpoint that returns + return report.legacyResult.status == 501; + }, + + function(report) { + // some weird character that screws up sorting + return JSON.stringify(report.legacyResult.body).includes('ï¼'); + }, + + // function(report) { + // const txt = JSON.stringify(report.test); + // return badAttributes.some(attr => txt.includes(attr)); + // }, + + function(report) { + const txt = JSON.stringify(report.test); + return /unknown/i.test(txt); + }, + + // function(report) { + // return /showNA":false/.test(JSON.stringify(report.test)); + // }, + + function(report) { + // this study has generic assay data which is not properly handled + // by legacy. it cannot be validated + return /nsclc_public_genie_bpc/.test(JSON.stringify(report.test)); + }, + + // function(report) { + // return /MGMT_STATUS/.test(JSON.stringify(report.test)) && + // /glioma_mskcc_2019/.test(JSON.stringify(report.test)) + // }, + + function(report) { + return ( + report.chResult.body.length === 1 && + report.chResult.body[0].counts[0].count === 0 && + report.chResult.body[0].counts[0].value === 'na' && + report.legacyResult.body.length === 0 + ); + }, + + function(report) { + // some weird character that screws up sorting + return JSON.stringify(report.test.data).includes('Wilms'); + }, + // function(report) { + // const studs = 'acc_2019,ampca_bcm_2016,brain_cptac_2020,brca_igr_2015,breast_alpelisib_2020,brca_smc_2018,crc_eo_2020,ctcl_columbia_2015,dlbc_broad_2012,dlbcl_duke_2017,esca_broad,es_iocurie_2014,gct_msk_2016,gbm_mayo_pdx_sarkaria_2019,gbm_columbia_2019,hcc_inserm_fr_2015,histiocytosis_cobi_msk_2019,kirc_bgi,hnsc_mdanderson_2013,ihch_msk_2021,lihc_riken,luad_mskcc_2015,luad_broad,luad_cptac_2020,mbl_icgc,mds_tokyo_2011,mel_ucla_2016,lung_msk_pdx,mm_broad,nbl_amc_2012,mpn_cimr_2013,mnm_washu_2016,mixed_selpercatinib_2020,msk_ch_2020,msk_ch_ped_2021,nbl_target_2018_pub,paad_icgc,plmeso_nyu_2015,paad_qcmg_uq_2016,pediatric_dkfz_2017,pan_origimed_2020,error,prad_cpcg_2017,prad_mskcc_cheny1_organoids_2014,skcm_broad_dfarber,skcm_yale,stmyec_wcm_2022,um_qimr_2016,wt_target_2018_pub,laml_tcga,lgg_tcga,meso_tcga,mel_tsam_liang_2017,mbn_mdacc_2013,stad_tcga,rt_target_2018_pub,prostate_dkfz_2018,error,difg_glass,coadread_cass_2020,mbn_sfu_2023,cll_broad_2022,ucec_ccr_cfdna_msk_2022,msk_ch_2023,brca_tcga_gdc,esca_tcga_gdc,aml_tcga_gdc,difg_tcga_gdc,paad_tcga_gdc,prad_tcga_gdc,thpa_tcga_gdc,breast_cptac_gdc,luad_cptac_gdc,pancreas_cptac_gdc,pancan_pcawg_2020,hcc_clca_2024,brca_fuscc_2020'.split( + // ',' + // ); + // return ( + // _.intersection( + // report.test.data.studyIds || + // report.test.data.studyViewFilter.studyIds, + // studs + // ).length > 0 + // ); + // }, + + // function(report) { + // console.log("moo", report.chResult.body.filter(d=>d.numberOfAlteredCases===1).length) + // return false; + // }, + + // function(report) { + // return _.some(report.legacyResult.body, (val, i) => { + // return ( + // val.matchingGenePanelIds.length === 0 && + // (report.chResult.body[i].matchingGenePanelIds.length === 0 || + // report.chResult.body[i].matchingGenePanelIds[0] === 'WES') + // ); + // }); + // }, + + // function(report) { + // return ( + // report.legacyResult.body.length > 0 && + // report.chResult.body.length === 0 + // ); + // }, +]; + +module.exports = suppressors; + +const badAttributes = [ + 'HBV', + 'STAGE', + 'KPS', + 'SUBGROUP', + 'THERAPY', + 'ECOG', + 'NGS_TEST', + 'N_STAGE', + 'WGD', + 'LDH', + 'SOURCE', + 'OTHER', + 'COHORT', + 'DEATH', + 'SURGERY', + 'WBC', + 'HER2', + 'TNM', + 'TNMSTAGE', + 'STUDY', + 'HCV', + 'GROUP', + 'STATUS', + 'CENTER', + 'TMB', + 'MSI', + 'NECROSIS', + 'T_STAGE', + 'ICD_10', + 'PLOIDY', + 'HBSAG', + 'LNI', + 'FAB', + 'GRADE', + 'PSA', + 'IHC_HER2', + 'LINEAGE', + 'SUBTYPE', + 'COMMENTS', + 'LVSI', + 'IDH1_MUTATION', + 'STEMNESS_SCORE', + 'VIAL_NUMBER', + 'SAMPLE_NAME', + 'CYTOGENETICS', + 'TUMOR_GRADE', + 'AGE_AT_DX', + 'INSTITUTE', + 'ER_STATUS_BY_IHC', + 'IDH_STATUS', + 'CIRRHOSIS', + 'PRIOR_THERAPY', + 'RECURRENCE', + 'MULTIPLE_TUMORS', + 'TREATMENT_STATUS', + 'SMOKING_HISTORY', + 'BRAF_STATUS', + 'CELLULARITY', + 'KARYOTYPE', + 'SEQUENCING_TYPE', + 'SAMPLE_TYPE_ID', + 'ADJUVANT_THERAPY', + 'RESIDUAL_TUMOR', + '1P19Q_STATUS', + 'CHEMOTHERAPY', + 'TISSUE_SOURCE', + 'HISTOLOGY', + 'NTE_HER2_STATUS', + 'IMMUNE_STATUS', + 'PSTAGE_CATEGORY', + 'METASTASIS', + 'RADIATION', + 'GLEASON_SCORE', + 'MITOTIC_COUNT', + 'MUTATION_RATE', + 'HER2_FISH_STATUS', + 'AGE_AT_DIAGNOSIS', + 'DISEASE_TYPE', + 'MGMT_STATUS', + 'RNA_SEQ_ANALYSIS', + 'BRAF_MUTATION', + 'TUMOR_PURITY', + 'MUTATION_STATUS', + 'MORPHOLOGY', + 'HER2_IHC_SCORE', + 'LIVER_METS', + 'HEPATITIS', + 'RADIOTHERAPY', + 'NTE_ER_STATUS', + 'DISEASE_CODE', + 'TREATMENT', + 'HER2_FISH_METHOD', + 'PR_STATUS_BY_IHC', + 'TOTAL_MUTATIONS', + 'SAMPLE_SITE', + 'BRAIN_METS', + 'TUMOR_SIZE_CM', + 'IHC_SCORE', + 'LDH_ELEVATED', + 'PROJECT_CODE', + 'MUTATION_TYPE', + 'HEMOGLOBIN_LEVEL', + 'KRAS_MUTATION', + 'HER2_COPY_NUMBER', + 'TUMOR_SIZE', + 'TUMOR_LEVEL', + 'SURGERY_TYPE', + 'CSTAGE_CATEGORY', + 'BIOPSY_SITE', + 'TUMOR_STAGE', + 'HISTOPATHOLOGY', + 'PLATELET_COUNT', + 'BRESLOW_DEPTH', + 'METASTASECTOMY', + 'PROCEDURE_TYPE', + 'SMOKING_STATUS', + 'TUMOR_LOCATION', + 'PERCENTAGE_TUMOR_PURITY', + 'AMPLIFICATION_STATUS', + 'ABI_ENZA_EXPOSURE_STATUS', + 'NTE_CENT_17_HER2_RATIO', + 'ER_POSITIVITY_SCALE_USED', + 'PERINEURAL_INVASION', + 'TISSUE_SOURCE_SITE', + 'EXTRACAPSULAR_SPREAD', + 'PATIENT_DISPLAY_NAME', + 'PR_POSITIVITY_SCALE_USED', + 'EXTRATHYROIDAL_EXTENSION', + 'TUMOR_SAMPLE_HISTOLOGY', + 'AGE_AT_PROCUREMENT', + 'PERCENT_TUMOR_CELLS', + 'STAGING_SYSTEM_OTHER', + 'NTE_PR_STATUS_BY_IHC', + 'PB_BLAST_PERCENTAGE', + 'MARGIN_STATUS_REEXCISION', + 'BARRETTS_ESOPHAGUS', + 'PRIMARY_HISTOLOGY', + 'LYMPH_NODE_METASTASIS', + 'HER2_CENT17_RATIO', + 'METASTATIC_SITE_OTHER', + 'HYPERTENSION_DIAGNOSIS', + 'SURGICAL_PROCEDURE_FIRST', + 'TUMOR_TOTAL_DEPTH', + 'AFP_AT_PROCUREMENT', + 'MAPK_PATHWAY_ALTERATION', + 'IDH_CODEL_SUBTYPE', + 'TUMOR_CLASSIFICATION', + 'STAGE_AT_DIAGNOSIS', + 'ESTIMATE_TUMORPURITY', + 'TREATMENT_RESPONSE', + 'CENT17_COPY_NUMBER', + 'HISTOLOGICAL_SUBTYPE', + 'BM_BLAST_PERCENTAGE', + 'PATHOLOGY_REPORT_UUID', + 'ESTIMATE_STROMAL_SCORE', + 'LOCALIZED_VS_METS_AT_DX', + 'MOLECULAR_SUBTYPE', + 'NTE_HER2_FISH_STATUS', + 'ESTIMATE_IMMUNE_SCORE', + 'PERCENT_TUMOR_NUCLEI', + 'PRIMARY_SITE_OTHER', + 'PRIMARY_TUMOR_LOCATION', + 'VASCULAR_INVASION', + 'SYSTEMIC_TREATMENT', + 'HER2_AND_CENT17_SCALE_OTHER', + 'CUMULATIVE_TREATMENT_TYPE_COUNT', + 'ER_STATUS_IHC_PERCENT_POSITIVE', + 'ESOPHAGEAL_TUMOR_LOCATION_CENTERED', + 'OTHER_METHOD_OF_SAMPLE_PROCUREMENT', + 'NTE_PR_IHC_INTENSITY_SCORE', + 'FIRST_SURGICAL_PROCEDURE_OTHER', + 'SURGERY_FOR_POSITIVE_MARGINS_OTHER', + 'PR_POSITIVITY_IHC_INTENSITY_SCORE', + 'NTE_ER_IHC_INTENSITY_SCORE', + 'HER2_IHC_PERCENT_POSITIVE', + 'HER2_POSITIVITY_METHOD_TEXT', + 'PR_POSITIVITY_SCALE_OTHER', + 'SURGERY_FOR_POSITIVE_MARGINS', + 'SYNOVIAL_SS18SSX_FUSION_STATUS', + 'PERCENT_LYMPHOCYTE_INFILTRATION', + 'HER2_AND_CENT17_CELLS_COUNT', + 'AGE_AT_SEQ_REPORTED_YEARS', + 'METHOD_OF_SAMPLE_PROCUREMENT', + 'AJCC_PATHOLOGIC_TUMOR_STAGE', + 'NTE_HER2_POSITIVITY_IHC_SCORE', + 'PR_STATUS_IHC_PERCENT_POSITIVE', + 'HER2_POSITIVITY_SCALE_OTHER', + 'METASTATIC_TUMOR_INDICATOR', + 'PR_POSITIVITY_DEFINE_METHOD', + 'FOLLICULAR_COMPONENT_PERCENT', + 'CUMULATIVE_TREATMENT_TYPES', + 'MICROMET_DETECTION_BY_IHC', + 'CREATININE_LEVEL_PRERESECTION', + 'MOST_RECENT_TREATMENT_TYPE', + 'ER_POSITIVITY_SCALE_OTHER', + 'INCIDENTAL_PROSTATE_CANCER', + 'NEW_TUMOR_EVENT_AFTER_INITIAL_TREATMENT', +]; diff --git a/api-e2e/tsconfig.json b/api-e2e/tsconfig.json new file mode 100644 index 00000000000..1660a2a7230 --- /dev/null +++ b/api-e2e/tsconfig.json @@ -0,0 +1,8 @@ +{ + "compilerOptions": { + "skipLibCheck": true, + "outDir": "./" + }, + "types": [], + "files": ["./../src/shared/api/validation.ts"] +} \ No newline at end of file diff --git a/package.json b/package.json index a538ce59d69..8dc32e1138b 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,8 @@ "scripts": { "start": "lerna run watch --parallel", "startSSL": "lerna run watchSSL --parallel", + "mergeTests": "./scripts/api-test_env.sh && node api-e2e/merge-tests.js", + "apitests": "cd ./api-e2e && npx tsc && yarn run mergeTests && node ./run.js", "watch": "./scripts/env_vars.sh && eval \"$(./scripts/env_vars.sh)\" && cross-env NODE_ENV=development webpack-dev-server --compress", "watchSSL": "./scripts/env_vars.sh && eval \"$(./scripts/env_vars.sh)\" && cross-env NODE_ENV=development NODE_OPTIONS=--max-old-space-size=4096 webpack-dev-server --compress --https", "clean": "rimraf dist tsDist common-dist", @@ -31,7 +33,7 @@ "heroku-postbuild": "yarn run build && yarn add pushstate-server@3.0.1 -g", "updateAPI": "yarn run fetchAPI && yarn run buildAPI && yarn run updateOncoKbAPI && yarn run updateGenomeNexusAPI", "convertToSwagger2": "./scripts/convert_to_swagger2.sh && yarn run extendSwagger2Converter", - "fetchAPILocal": "export CBIOPORTAL_URL=http://localhost:8090 && curl -s -L -k ${CBIOPORTAL_URL}/api/v3/api-docs/public | json | grep -v basePath | grep -v termsOfService | grep -v host > packages/cbioportal-ts-api-client/src/generated/CBioPortalAPI-docs.json && curl -s -L -k ${CBIOPORTAL_URL}/api/v3/api-docs/internal | json | grep -v host | grep -v basePath | grep -v termsOfService > packages/cbioportal-ts-api-client/src/generated/CBioPortalAPIInternal-docs.json", + "fetchAPILocal": "export CBIOPORTAL_URL=http://localhost:8082 && curl -s -L -k ${CBIOPORTAL_URL}/api/v3/api-docs/public > packages/cbioportal-ts-api-client/src/generated/CBioPortalAPI-docs.json && curl -s -L -k ${CBIOPORTAL_URL}/api/v3/api-docs/internal | json > packages/cbioportal-ts-api-client/src/generated/CBioPortalAPIInternal-docs.json && yarn run convertToSwagger2", "fetchAPI": "./scripts/env_vars.sh && eval \"$(./scripts/env_vars.sh)\" && curl -s -L -k ${CBIOPORTAL_URL}/api/v3/api-docs/public > packages/cbioportal-ts-api-client/src/generated/CBioPortalAPI-docs.json && curl -s -L -k ${CBIOPORTAL_URL}/api/v3/api-docs/internal | json > packages/cbioportal-ts-api-client/src/generated/CBioPortalAPIInternal-docs.json && yarn run convertToSwagger2", "extendSwagger2Converter": "node scripts/extend_converter.js packages/cbioportal-ts-api-client/src/generated CBioPortalAPI CBioPortalAPIInternal", "buildAPI": "node scripts/generate-api.js packages/cbioportal-ts-api-client/src/generated CBioPortalAPI CBioPortalAPIInternal", @@ -148,6 +150,7 @@ "addthis-snippet": "^1.0.1", "autobind-decorator": "^2.1.0", "autoprefixer": "^6.7.0", + "axios": "^1.7.7", "babel-loader": "8.0.5", "babel-plugin-transform-es2015-modules-umd": "^6.22.0", "babel-polyfill": "^6.22.0", @@ -172,6 +175,7 @@ "cross-env": "^3.1.4", "css-loader": "^2.1.1", "cssnano": "^3.10.0", + "csvtojson": "^2.0.10", "d3": "3.5.6", "d3-dsv": "1.0.8", "d3-scale": "^2.0.0", diff --git a/packages/cbioportal-ts-api-client/src/generated/CBioPortalAPIInternal-docs.json b/packages/cbioportal-ts-api-client/src/generated/CBioPortalAPIInternal-docs.json index a8616dca66f..15f51e83d39 100644 --- a/packages/cbioportal-ts-api-client/src/generated/CBioPortalAPIInternal-docs.json +++ b/packages/cbioportal-ts-api-client/src/generated/CBioPortalAPIInternal-docs.json @@ -558,6 +558,41 @@ "operationId": "getClinicalEventTypeCountsUsingPOST" } }, + "/api/clinical-events-meta/fetch": { + "post": { + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "parameters": [ + { + "in": "body", + "name": "clinicalEventAttributeRequest", + "schema": { + "$ref": "#/definitions/ClinicalEventAttributeRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "items": { + "$ref": "#/definitions/ClinicalEvent" + }, + "type": "array" + } + } + }, + "tags": [ + "Clinical Events" + ], + "description": "Fetch clinical events meta", + "operationId": "fetchClinicalEventsMetaUsingPOST" + } + }, "/api/cna-genes/fetch": { "post": { "consumes": [ @@ -593,6 +628,93 @@ "operationId": "fetchCNAGenesUsingPOST" } }, + "/api/column-store/treatments/patient-counts/fetch": { + "post": { + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "parameters": [ + { + "default": "Agent", + "enum": [ + "Agent", + "AgentClass", + "AgentTarget" + ], + "in": "query", + "name": "tier", + "required": false, + "type": "string" + }, + { + "in": "body", + "name": "studyViewFilter", + "schema": { + "$ref": "#/definitions/StudyViewFilter" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/PatientTreatmentReport" + } + } + }, + "tags": [ + "study-view-column-store-controller" + ], + "description": "Get all patient level treatments", + "operationId": "fetchPatientTreatmentCountsUsing" + } + }, + "/api/column-store/treatments/sample-counts/fetch": { + "post": { + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "parameters": [ + { + "default": "Agent", + "enum": [ + "Agent", + "AgentClass", + "AgentTarget" + ], + "in": "query", + "name": "tier", + "required": false, + "type": "string" + }, + { + "in": "body", + "name": "studyViewFilter", + "schema": { + "$ref": "#/definitions/StudyViewFilter" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/SampleTreatmentReport" + } + } + }, + "tags": [ + "study-view-column-store-controller" + ], + "operationId": "fetchSampleTreatmentCountsUsing" + } + }, "/api/cosmic-counts/fetch": { "post": { "consumes": [ @@ -3219,6 +3341,41 @@ "operationId": "getSignificantlyMutatedGenesUsingGET" } }, + "/api/survival-data/fetch": { + "post": { + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "parameters": [ + { + "in": "body", + "name": "survivalRequest", + "schema": { + "$ref": "#/definitions/SurvivalRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "items": { + "$ref": "#/definitions/ClinicalData" + }, + "type": "array" + } + } + }, + "tags": [ + "Survival" + ], + "description": "Fetch survival data", + "operationId": "fetchSurvivalDataUsingPOST" + } + }, "/api/timestamps": { "get": { "produces": [ @@ -3958,6 +4115,27 @@ ], "type": "object" }, + "ClinicalEventAttributeRequest": { + "description": "clinical events Request", + "properties": { + "clinicalEventRequests": { + "items": { + "$ref": "#/definitions/ClinicalEventRequest" + }, + "type": "array", + "uniqueItems": true + }, + "patientIdentifiers": { + "items": { + "$ref": "#/definitions/PatientIdentifier" + }, + "maxItems": 10000000, + "minItems": 1, + "type": "array" + } + }, + "type": "object" + }, "ClinicalEventData": { "properties": { "key": { @@ -3973,6 +4151,47 @@ ], "type": "object" }, + "ClinicalEventRequest": { + "properties": { + "attributes": { + "items": { + "$ref": "#/definitions/ClinicalEventData" + }, + "type": "array" + }, + "eventType": { + "type": "string" + } + }, + "required": [ + "eventType" + ], + "type": "object" + }, + "ClinicalEventRequestIdentifier": { + "properties": { + "clinicalEventRequests": { + "items": { + "$ref": "#/definitions/ClinicalEventRequest" + }, + "maxItems": 10000000, + "minItems": 0, + "type": "array", + "uniqueItems": true + }, + "position": { + "enum": [ + "FIRST", + "LAST" + ], + "type": "string" + } + }, + "required": [ + "position" + ], + "type": "object" + }, "ClinicalEventSample": { "properties": { "patientId": { @@ -5464,6 +5683,29 @@ }, "type": "object" }, + "PatientIdentifier": { + "properties": { + "patientId": { + "type": "string" + }, + "studyId": { + "type": "string" + } + }, + "type": "object" + }, + "PatientTreatment": { + "properties": { + "count": { + "format": "int32", + "type": "integer" + }, + "treatment": { + "type": "string" + } + }, + "type": "object" + }, "PatientTreatmentFilter": { "properties": { "treatment": { @@ -5472,6 +5714,25 @@ }, "type": "object" }, + "PatientTreatmentReport": { + "properties": { + "patientTreatments": { + "items": { + "$ref": "#/definitions/PatientTreatment" + }, + "type": "array" + }, + "totalPatients": { + "format": "int32", + "type": "integer" + }, + "totalSamples": { + "format": "int32", + "type": "integer" + } + }, + "type": "object" + }, "PatientTreatmentRow": { "properties": { "count": { @@ -5690,6 +5951,21 @@ }, "type": "object" }, + "SampleTreatmentReport": { + "properties": { + "totalSamples": { + "format": "int32", + "type": "integer" + }, + "treatments": { + "items": { + "$ref": "#/definitions/SampleTreatmentRow" + }, + "type": "array" + } + }, + "type": "object" + }, "SampleTreatmentRow": { "properties": { "count": { @@ -6126,6 +6402,35 @@ }, "type": "object" }, + "SurvivalRequest": { + "description": "Survival Data Request", + "properties": { + "attributeIdPrefix": { + "type": "string" + }, + "censoredEventRequestIdentifier": { + "$ref": "#/definitions/ClinicalEventRequestIdentifier" + }, + "endEventRequestIdentifier": { + "$ref": "#/definitions/ClinicalEventRequestIdentifier" + }, + "patientIdentifiers": { + "items": { + "$ref": "#/definitions/PatientIdentifier" + }, + "maxItems": 10000000, + "minItems": 1, + "type": "array" + }, + "startEventRequestIdentifier": { + "$ref": "#/definitions/ClinicalEventRequestIdentifier" + } + }, + "required": [ + "attributeIdPrefix" + ], + "type": "object" + }, "VariantCount": { "properties": { "entrezGeneId": { diff --git a/packages/cbioportal-ts-api-client/src/generated/CBioPortalAPIInternal.ts b/packages/cbioportal-ts-api-client/src/generated/CBioPortalAPIInternal.ts index 02610206a86..ca9bc28d658 100644 --- a/packages/cbioportal-ts-api-client/src/generated/CBioPortalAPIInternal.ts +++ b/packages/cbioportal-ts-api-client/src/generated/CBioPortalAPIInternal.ts @@ -238,12 +238,30 @@ export type ClinicalEvent = { 'uniqueSampleKey': string +}; +export type ClinicalEventAttributeRequest = { + 'clinicalEventRequests': Array < ClinicalEventRequest > + + 'patientIdentifiers': Array < PatientIdentifier > + }; export type ClinicalEventData = { 'key': string 'value': string +}; +export type ClinicalEventRequest = { + 'attributes': Array < ClinicalEventData > + + 'eventType': string + +}; +export type ClinicalEventRequestIdentifier = { + 'clinicalEventRequests': Array < ClinicalEventRequest > + + 'position': "FIRST" | "LAST" + }; export type ClinicalEventSample = { 'patientId': string @@ -896,10 +914,30 @@ export type OredPatientTreatmentFilters = { export type OredSampleTreatmentFilters = { 'filters': Array < SampleTreatmentFilter > +}; +export type PatientIdentifier = { + 'patientId': string + + 'studyId': string + +}; +export type PatientTreatment = { + 'count': number + + 'treatment': string + }; export type PatientTreatmentFilter = { 'treatment': string +}; +export type PatientTreatmentReport = { + 'patientTreatments': Array < PatientTreatment > + + 'totalPatients': number + + 'totalSamples': number + }; export type PatientTreatmentRow = { 'count': number @@ -998,6 +1036,12 @@ export type SampleTreatmentFilter = { 'treatment': string +}; +export type SampleTreatmentReport = { + 'totalSamples': number + + 'treatments': Array < SampleTreatmentRow > + }; export type SampleTreatmentRow = { 'count': number @@ -1203,6 +1247,18 @@ export type StudyViewStructuralVariantFilter = { 'structVarQueries': Array < Array < StructuralVariantFilterQuery > > +}; +export type SurvivalRequest = { + 'attributeIdPrefix': string + + 'censoredEventRequestIdentifier': ClinicalEventRequestIdentifier + + 'endEventRequestIdentifier': ClinicalEventRequestIdentifier + + 'patientIdentifiers': Array < PatientIdentifier > + + 'startEventRequestIdentifier': ClinicalEventRequestIdentifier + }; export type VariantCount = { 'entrezGeneId': number @@ -2339,6 +2395,78 @@ export default class CBioPortalAPIInternal { return response.body; }); }; + fetchClinicalEventsMetaUsingPOSTURL(parameters: { + 'clinicalEventAttributeRequest' ? : ClinicalEventAttributeRequest, + $queryParameters ? : any + }): string { + let queryParameters: any = {}; + let path = '/api/clinical-events-meta/fetch'; + + if (parameters.$queryParameters) { + Object.keys(parameters.$queryParameters).forEach(function(parameterName) { + var parameter = parameters.$queryParameters[parameterName]; + queryParameters[parameterName] = parameter; + }); + } + let keys = Object.keys(queryParameters); + return this.domain + path + (keys.length > 0 ? '?' + (keys.map(key => key + '=' + encodeURIComponent(queryParameters[key])).join('&')) : ''); + }; + + /** + * Fetch clinical events meta + * @method + * @name CBioPortalAPIInternal#fetchClinicalEventsMetaUsingPOST + * @param {} clinicalEventAttributeRequest - A web service for supplying JSON formatted data to cBioPortal clients. Please note that this API is currently in beta and subject to change. + */ + fetchClinicalEventsMetaUsingPOSTWithHttpInfo(parameters: { + 'clinicalEventAttributeRequest' ? : ClinicalEventAttributeRequest, + $queryParameters ? : any, + $domain ? : string + }): Promise < request.Response > { + const domain = parameters.$domain ? parameters.$domain : this.domain; + const errorHandlers = this.errorHandlers; + const request = this.request; + let path = '/api/clinical-events-meta/fetch'; + let body: any; + let queryParameters: any = {}; + let headers: any = {}; + let form: any = {}; + return new Promise(function(resolve, reject) { + headers['Accept'] = 'application/json'; + headers['Content-Type'] = 'application/json'; + + if (parameters['clinicalEventAttributeRequest'] !== undefined) { + body = parameters['clinicalEventAttributeRequest']; + } + + if (parameters.$queryParameters) { + Object.keys(parameters.$queryParameters).forEach(function(parameterName) { + var parameter = parameters.$queryParameters[parameterName]; + queryParameters[parameterName] = parameter; + }); + } + + request('POST', domain + path, body, headers, queryParameters, form, reject, resolve, errorHandlers); + + }); + }; + + /** + * Fetch clinical events meta + * @method + * @name CBioPortalAPIInternal#fetchClinicalEventsMetaUsingPOST + * @param {} clinicalEventAttributeRequest - A web service for supplying JSON formatted data to cBioPortal clients. Please note that this API is currently in beta and subject to change. + */ + fetchClinicalEventsMetaUsingPOST(parameters: { + 'clinicalEventAttributeRequest' ? : ClinicalEventAttributeRequest, + $queryParameters ? : any, + $domain ? : string + }): Promise < Array < ClinicalEvent > + > { + return this.fetchClinicalEventsMetaUsingPOSTWithHttpInfo(parameters).then(function(response: request.Response) { + return response.body; + }); + }; fetchCNAGenesUsingPOSTURL(parameters: { 'studyViewFilter' ? : StudyViewFilter, $queryParameters ? : any @@ -2411,6 +2539,172 @@ export default class CBioPortalAPIInternal { return response.body; }); }; + fetchPatientTreatmentCountsUsingURL(parameters: { + 'tier' ? : "Agent" | "AgentClass" | "AgentTarget", + 'studyViewFilter' ? : StudyViewFilter, + $queryParameters ? : any + }): string { + let queryParameters: any = {}; + let path = '/api/column-store/treatments/patient-counts/fetch'; + if (parameters['tier'] !== undefined) { + queryParameters['tier'] = parameters['tier']; + } + + if (parameters.$queryParameters) { + Object.keys(parameters.$queryParameters).forEach(function(parameterName) { + var parameter = parameters.$queryParameters[parameterName]; + queryParameters[parameterName] = parameter; + }); + } + let keys = Object.keys(queryParameters); + return this.domain + path + (keys.length > 0 ? '?' + (keys.map(key => key + '=' + encodeURIComponent(queryParameters[key])).join('&')) : ''); + }; + + /** + * Get all patient level treatments + * @method + * @name CBioPortalAPIInternal#fetchPatientTreatmentCountsUsing + * @param {string} tier - A web service for supplying JSON formatted data to cBioPortal clients. Please note that this API is currently in beta and subject to change. + * @param {} studyViewFilter - A web service for supplying JSON formatted data to cBioPortal clients. Please note that this API is currently in beta and subject to change. + */ + fetchPatientTreatmentCountsUsingWithHttpInfo(parameters: { + 'tier' ? : "Agent" | "AgentClass" | "AgentTarget", + 'studyViewFilter' ? : StudyViewFilter, + $queryParameters ? : any, + $domain ? : string + }): Promise < request.Response > { + const domain = parameters.$domain ? parameters.$domain : this.domain; + const errorHandlers = this.errorHandlers; + const request = this.request; + let path = '/api/column-store/treatments/patient-counts/fetch'; + let body: any; + let queryParameters: any = {}; + let headers: any = {}; + let form: any = {}; + return new Promise(function(resolve, reject) { + headers['Accept'] = 'application/json'; + headers['Content-Type'] = 'application/json'; + + if (parameters['tier'] !== undefined) { + queryParameters['tier'] = parameters['tier']; + } + + if (parameters['studyViewFilter'] !== undefined) { + body = parameters['studyViewFilter']; + } + + if (parameters.$queryParameters) { + Object.keys(parameters.$queryParameters).forEach(function(parameterName) { + var parameter = parameters.$queryParameters[parameterName]; + queryParameters[parameterName] = parameter; + }); + } + + request('POST', domain + path, body, headers, queryParameters, form, reject, resolve, errorHandlers); + + }); + }; + + /** + * Get all patient level treatments + * @method + * @name CBioPortalAPIInternal#fetchPatientTreatmentCountsUsing + * @param {string} tier - A web service for supplying JSON formatted data to cBioPortal clients. Please note that this API is currently in beta and subject to change. + * @param {} studyViewFilter - A web service for supplying JSON formatted data to cBioPortal clients. Please note that this API is currently in beta and subject to change. + */ + fetchPatientTreatmentCountsUsing(parameters: { + 'tier' ? : "Agent" | "AgentClass" | "AgentTarget", + 'studyViewFilter' ? : StudyViewFilter, + $queryParameters ? : any, + $domain ? : string + }): Promise < PatientTreatmentReport > { + return this.fetchPatientTreatmentCountsUsingWithHttpInfo(parameters).then(function(response: request.Response) { + return response.body; + }); + }; + fetchSampleTreatmentCountsUsingURL(parameters: { + 'tier' ? : "Agent" | "AgentClass" | "AgentTarget", + 'studyViewFilter' ? : StudyViewFilter, + $queryParameters ? : any + }): string { + let queryParameters: any = {}; + let path = '/api/column-store/treatments/sample-counts/fetch'; + if (parameters['tier'] !== undefined) { + queryParameters['tier'] = parameters['tier']; + } + + if (parameters.$queryParameters) { + Object.keys(parameters.$queryParameters).forEach(function(parameterName) { + var parameter = parameters.$queryParameters[parameterName]; + queryParameters[parameterName] = parameter; + }); + } + let keys = Object.keys(queryParameters); + return this.domain + path + (keys.length > 0 ? '?' + (keys.map(key => key + '=' + encodeURIComponent(queryParameters[key])).join('&')) : ''); + }; + + /** + * + * @method + * @name CBioPortalAPIInternal#fetchSampleTreatmentCountsUsing + * @param {string} tier - A web service for supplying JSON formatted data to cBioPortal clients. Please note that this API is currently in beta and subject to change. + * @param {} studyViewFilter - A web service for supplying JSON formatted data to cBioPortal clients. Please note that this API is currently in beta and subject to change. + */ + fetchSampleTreatmentCountsUsingWithHttpInfo(parameters: { + 'tier' ? : "Agent" | "AgentClass" | "AgentTarget", + 'studyViewFilter' ? : StudyViewFilter, + $queryParameters ? : any, + $domain ? : string + }): Promise < request.Response > { + const domain = parameters.$domain ? parameters.$domain : this.domain; + const errorHandlers = this.errorHandlers; + const request = this.request; + let path = '/api/column-store/treatments/sample-counts/fetch'; + let body: any; + let queryParameters: any = {}; + let headers: any = {}; + let form: any = {}; + return new Promise(function(resolve, reject) { + headers['Accept'] = 'application/json'; + headers['Content-Type'] = 'application/json'; + + if (parameters['tier'] !== undefined) { + queryParameters['tier'] = parameters['tier']; + } + + if (parameters['studyViewFilter'] !== undefined) { + body = parameters['studyViewFilter']; + } + + if (parameters.$queryParameters) { + Object.keys(parameters.$queryParameters).forEach(function(parameterName) { + var parameter = parameters.$queryParameters[parameterName]; + queryParameters[parameterName] = parameter; + }); + } + + request('POST', domain + path, body, headers, queryParameters, form, reject, resolve, errorHandlers); + + }); + }; + + /** + * + * @method + * @name CBioPortalAPIInternal#fetchSampleTreatmentCountsUsing + * @param {string} tier - A web service for supplying JSON formatted data to cBioPortal clients. Please note that this API is currently in beta and subject to change. + * @param {} studyViewFilter - A web service for supplying JSON formatted data to cBioPortal clients. Please note that this API is currently in beta and subject to change. + */ + fetchSampleTreatmentCountsUsing(parameters: { + 'tier' ? : "Agent" | "AgentClass" | "AgentTarget", + 'studyViewFilter' ? : StudyViewFilter, + $queryParameters ? : any, + $domain ? : string + }): Promise < SampleTreatmentReport > { + return this.fetchSampleTreatmentCountsUsingWithHttpInfo(parameters).then(function(response: request.Response) { + return response.body; + }); + }; fetchCosmicCountsUsingPOSTURL(parameters: { 'keywords': Array < string > , $queryParameters ? : any @@ -7314,6 +7608,78 @@ export default class CBioPortalAPIInternal { return response.body; }); }; + fetchSurvivalDataUsingPOSTURL(parameters: { + 'survivalRequest' ? : SurvivalRequest, + $queryParameters ? : any + }): string { + let queryParameters: any = {}; + let path = '/api/survival-data/fetch'; + + if (parameters.$queryParameters) { + Object.keys(parameters.$queryParameters).forEach(function(parameterName) { + var parameter = parameters.$queryParameters[parameterName]; + queryParameters[parameterName] = parameter; + }); + } + let keys = Object.keys(queryParameters); + return this.domain + path + (keys.length > 0 ? '?' + (keys.map(key => key + '=' + encodeURIComponent(queryParameters[key])).join('&')) : ''); + }; + + /** + * Fetch survival data + * @method + * @name CBioPortalAPIInternal#fetchSurvivalDataUsingPOST + * @param {} survivalRequest - A web service for supplying JSON formatted data to cBioPortal clients. Please note that this API is currently in beta and subject to change. + */ + fetchSurvivalDataUsingPOSTWithHttpInfo(parameters: { + 'survivalRequest' ? : SurvivalRequest, + $queryParameters ? : any, + $domain ? : string + }): Promise < request.Response > { + const domain = parameters.$domain ? parameters.$domain : this.domain; + const errorHandlers = this.errorHandlers; + const request = this.request; + let path = '/api/survival-data/fetch'; + let body: any; + let queryParameters: any = {}; + let headers: any = {}; + let form: any = {}; + return new Promise(function(resolve, reject) { + headers['Accept'] = 'application/json'; + headers['Content-Type'] = 'application/json'; + + if (parameters['survivalRequest'] !== undefined) { + body = parameters['survivalRequest']; + } + + if (parameters.$queryParameters) { + Object.keys(parameters.$queryParameters).forEach(function(parameterName) { + var parameter = parameters.$queryParameters[parameterName]; + queryParameters[parameterName] = parameter; + }); + } + + request('POST', domain + path, body, headers, queryParameters, form, reject, resolve, errorHandlers); + + }); + }; + + /** + * Fetch survival data + * @method + * @name CBioPortalAPIInternal#fetchSurvivalDataUsingPOST + * @param {} survivalRequest - A web service for supplying JSON formatted data to cBioPortal clients. Please note that this API is currently in beta and subject to change. + */ + fetchSurvivalDataUsingPOST(parameters: { + 'survivalRequest' ? : SurvivalRequest, + $queryParameters ? : any, + $domain ? : string + }): Promise < Array < ClinicalData > + > { + return this.fetchSurvivalDataUsingPOSTWithHttpInfo(parameters).then(function(response: request.Response) { + return response.body; + }); + }; getAllTimestampsUsingGETURL(parameters: { $queryParameters ? : any }): string { diff --git a/packages/cbioportal-ts-api-client/src/index.tsx b/packages/cbioportal-ts-api-client/src/index.tsx index 5965c0a825d..7472a08a5f1 100644 --- a/packages/cbioportal-ts-api-client/src/index.tsx +++ b/packages/cbioportal-ts-api-client/src/index.tsx @@ -89,6 +89,9 @@ export { SampleTreatmentFilter, PatientTreatmentFilter, PatientTreatmentRow, + PatientTreatmentReport, + SampleTreatmentReport, + PatientTreatment, MutationDataFilter, GenericAssayDataFilter, AlterationFilter, diff --git a/packages/cbioportal-utils/src/api/apiUtils.ts b/packages/cbioportal-utils/src/api/apiUtils.ts index 8436dc5c5d7..dcae3792f0e 100644 --- a/packages/cbioportal-utils/src/api/apiUtils.ts +++ b/packages/cbioportal-utils/src/api/apiUtils.ts @@ -1,6 +1,8 @@ import { Buffer } from 'buffer'; import _ from 'lodash'; import * as request from 'superagent'; +// import {getServerConfig} from "../../../../src/config/config"; +// import {getBrowserWindow} from "cbioportal-frontend-commons"; const BASE64_ENCODING = 'base64'; const UTF8_ENCODING = 'utf8'; diff --git a/scripts/api-test_env.sh b/scripts/api-test_env.sh new file mode 100755 index 00000000000..46dcb1511c9 --- /dev/null +++ b/scripts/api-test_env.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# eval output of this file to get appropriate env variables e.g. eval "$(./env_vars.sh)" +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +RED='\033[0;31m' +NC='\033[0m' + + +if [[ "$BACKEND_ROOT" ]]; then + exit 0 +else + echo -e "${RED}No desired BACKEND_ROOT variable set${NC}" + echo -e "${RED}set with e.g. export BACKEND_ROOT=/path/to/my/backend/repo/${NC}" + exit 1 +fi diff --git a/src/appBootstrapper.tsx b/src/appBootstrapper.tsx index 901a49bff29..e9a35277002 100755 --- a/src/appBootstrapper.tsx +++ b/src/appBootstrapper.tsx @@ -143,8 +143,8 @@ browserWindow.postLoadForMskCIS = () => {}; // this is the only supported way to disable tracking for the $3Dmol.js (browserWindow as any).$3Dmol = { notrack: true }; -// make sure lodash doesn't overwrite (or set) global underscore -_.noConflict(); +// expose lodash on window +getBrowserWindow()._ = _; const routingStore = new ExtendedRoutingStore(); diff --git a/src/config/IAppConfig.ts b/src/config/IAppConfig.ts index 28a3a858e09..8167ac6f999 100644 --- a/src/config/IAppConfig.ts +++ b/src/config/IAppConfig.ts @@ -186,4 +186,5 @@ export interface IServerConfig { vaf_log_scale_default: boolean; // this has a default skin_study_view_show_sv_table: boolean; // this has a default enable_study_tags: boolean; + clickhouse_mode: boolean; } diff --git a/src/config/config.ts b/src/config/config.ts index 9e096df6b07..23c40a7fd7c 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -20,7 +20,9 @@ import internalGenomeNexusClient from '../shared/api/genomeNexusInternalClientIn import oncoKBClient from '../shared/api/oncokbClientInstance'; import genome2StructureClient from '../shared/api/g2sClientInstance'; import client from '../shared/api/cbioportalClientInstance'; -import internalClient from '../shared/api/cbioportalInternalClientInstance'; +import internalClient, { + getInternalClient, +} from '../shared/api/cbioportalInternalClientInstance'; import $ from 'jquery'; import { AppStore } from '../AppStore'; import { CBioPortalAPI, CBioPortalAPIInternal } from 'cbioportal-ts-api-client'; @@ -194,6 +196,7 @@ export function initializeAPIClients() { // we need to set the domain of our api clients (client as any).domain = getCbioPortalApiUrl(); (internalClient as any).domain = getCbioPortalApiUrl(); + (getInternalClient() as any).domain = getCbioPortalApiUrl(); (genomeNexusClient as any).domain = getGenomeNexusApiUrl(); (internalGenomeNexusClient as any).domain = getGenomeNexusApiUrl(); (oncoKBClient as any).domain = getOncoKbApiUrl(); @@ -388,3 +391,10 @@ export function initializeAppStore(appStore: AppStore) { appStore.authMethod = getServerConfig().authenticationMethod; appStore.userName = getServerConfig().user_display_name; } + +export function isClickhouseMode() { + return ( + !/legacy=1/.test(getBrowserWindow().location.search) && + getServerConfig().clickhouse_mode === true + ); +} diff --git a/src/config/serverConfigDefaults.ts b/src/config/serverConfigDefaults.ts index be24a0d9c55..6521b08a014 100644 --- a/src/config/serverConfigDefaults.ts +++ b/src/config/serverConfigDefaults.ts @@ -243,6 +243,8 @@ export const ServerConfigDefaults: Partial = { vaf_log_scale_default: false, skin_study_view_show_sv_table: false, + + clickhouse_mode: false, }; export default ServerConfigDefaults; diff --git a/src/pages/studyView/StudyViewPage.tsx b/src/pages/studyView/StudyViewPage.tsx index 508fe36e461..5a2b45c21e4 100644 --- a/src/pages/studyView/StudyViewPage.tsx +++ b/src/pages/studyView/StudyViewPage.tsx @@ -27,7 +27,7 @@ import IFrameLoader from '../../shared/components/iframeLoader/IFrameLoader'; import { StudySummaryTab } from 'pages/studyView/tabs/SummaryTab'; import StudyPageHeader from './studyPageHeader/StudyPageHeader'; import CNSegments from './tabs/CNSegments'; - +import { getInternalClient } from 'shared/api/cbioportalInternalClientInstance'; import AddChartButton from './addChartButton/AddChartButton'; import { sleep } from '../../shared/lib/TimeUtils'; import { Else, If, Then } from 'react-if'; @@ -79,6 +79,7 @@ import { } from 'shared/lib/customTabs/customTabHelpers'; import { VirtualStudyModal } from 'pages/studyView/virtualStudy/VirtualStudyModal'; import PlotsTab from 'shared/components/plots/PlotsTab'; +import { RFC80Test } from 'pages/studyView/rfc80Tester'; export interface IStudyViewPageProps { routing: any; @@ -145,7 +146,8 @@ export default class StudyViewPage extends React.Component< this.store = new StudyViewPageStore( this.props.appStore, ServerConfigHelpers.sessionServiceIsEnabled(), - this.urlWrapper + this.urlWrapper, + getInternalClient() ); // Expose store to window for use in custom tabs. @@ -171,7 +173,7 @@ export default class StudyViewPage extends React.Component< const query = props.routing.query; const hash = props.routing.location.hash; // clear hash if any - props.routing.location.hash = ''; + //props.routing.location.hash = ''; const newStudyViewFilter: StudyViewURLQuery = _.pick(query, [ 'id', 'studyId', @@ -1238,6 +1240,7 @@ export default class StudyViewPage extends React.Component< isLoggedIn={this.props.appStore.isLoggedIn} /> )} + ); } diff --git a/src/pages/studyView/StudyViewPageStore.ts b/src/pages/studyView/StudyViewPageStore.ts index 98d9da7d788..f8742cdb60a 100644 --- a/src/pages/studyView/StudyViewPageStore.ts +++ b/src/pages/studyView/StudyViewPageStore.ts @@ -1,5 +1,4 @@ import _ from 'lodash'; -import internalClient from 'shared/api/cbioportalInternalClientInstance'; import defaultClient from 'shared/api/cbioportalClientInstance'; import client from 'shared/api/cbioportalClientInstance'; import oncoKBClient from 'shared/api/oncokbClientInstance'; @@ -19,6 +18,7 @@ import { AndedSampleTreatmentFilters, BinsGeneratorConfig, CancerStudy, + CBioPortalAPIInternal, ClinicalAttribute, ClinicalAttributeCount, ClinicalAttributeCountFilter, @@ -64,7 +64,6 @@ import { OredPatientTreatmentFilters, OredSampleTreatmentFilters, Patient, - PatientTreatmentRow, ResourceData, Sample, SampleFilter, @@ -76,6 +75,10 @@ import { StructuralVariantFilterQuery, StudyViewFilter, StudyViewStructuralVariantFilter, + GenericAssayDataCountFilter, + GenericAssayDataCountItem, + SampleTreatmentReport, + PatientTreatmentReport, } from 'cbioportal-ts-api-client'; import { evaluatePutativeDriverInfo, @@ -216,6 +219,7 @@ import { } from '../../shared/api/urls'; import { DataType as DownloadDataType, + getBrowserWindow, MobxPromise, onMobxPromise, pluralize, @@ -299,7 +303,7 @@ import { CopyNumberEnrichmentEventType, MutationEnrichmentEventType, } from 'shared/lib/comparison/ComparisonStoreUtils'; -import { getServerConfig } from 'config/config'; +import { getServerConfig, isClickhouseMode } from 'config/config'; import { ChartUserSetting, CustomChart, @@ -373,7 +377,9 @@ import { PlotsColoringParam, PlotsSelectionParam, } from 'pages/resultsView/ResultsViewURLWrapper'; -import { SortDirection } from 'shared/components/lazyMobXTable/LazyMobXTable'; +import internalClient from 'shared/api/cbioportalInternalClientInstance'; + +const oldInternalClient = internalClient; export const STUDY_VIEW_FILTER_AUTOSUBMIT = 'study_view_filter_autosubmit'; @@ -602,7 +608,8 @@ export class StudyViewPageStore constructor( public appStore: AppStore, private sessionServiceIsEnabled: boolean, - private urlWrapper: StudyViewURLWrapper + private urlWrapper: StudyViewURLWrapper, + public internalClient: CBioPortalAPIInternal ) { makeObservable(this); @@ -4518,7 +4525,7 @@ export class StudyViewPageStore readonly unfilteredClinicalDataCount = remoteData({ invoke: () => { if (!_.isEmpty(this.unfilteredAttrsForNonNumerical)) { - return internalClient.fetchClinicalDataCountsUsingPOST({ + return this.internalClient.fetchClinicalDataCountsUsingPOST({ clinicalDataCountFilter: { attributes: this.unfilteredAttrsForNonNumerical, studyViewFilter: this.filters, @@ -4545,7 +4552,7 @@ export class StudyViewPageStore invoke: () => { //only invoke if there are filtered samples if (this.hasFilteredSamples) { - return internalClient.fetchCustomDataCountsUsingPOST({ + return this.internalClient.fetchCustomDataCountsUsingPOST({ clinicalDataCountFilter: { attributes: this.unfilteredCustomAttrsForNonNumerical, studyViewFilter: this.filters, @@ -4572,7 +4579,7 @@ export class StudyViewPageStore >({ invoke: () => { if (!_.isEmpty(this.newlyAddedUnfilteredAttrsForNonNumerical)) { - return internalClient.fetchClinicalDataCountsUsingPOST({ + return this.internalClient.fetchClinicalDataCountsUsingPOST({ clinicalDataCountFilter: { attributes: this .newlyAddedUnfilteredAttrsForNonNumerical, @@ -4601,7 +4608,7 @@ export class StudyViewPageStore this.hasSampleIdentifiersInFilter && this.newlyAddedUnfilteredAttrsForNumerical.length > 0 ) { - const clinicalDataBinCountData = await internalClient.fetchClinicalDataBinCountsUsingPOST( + const clinicalDataBinCountData = await this.internalClient.fetchClinicalDataBinCountsUsingPOST( { dataBinMethod: 'STATIC', clinicalDataBinCountFilter: { @@ -4642,7 +4649,7 @@ export class StudyViewPageStore return element !== undefined; } ); - const clinicalDataBinCountData = await internalClient.fetchClinicalDataBinCountsUsingPOST( + const clinicalDataBinCountData = await this.internalClient.fetchClinicalDataBinCountsUsingPOST( { dataBinMethod: 'STATIC', clinicalDataBinCountFilter: { @@ -4768,7 +4775,7 @@ export class StudyViewPageStore this._clinicalDataFilterSet.has(uniqueKey) || this.isInitialFilterState ) { - result = await internalClient.fetchClinicalDataCountsUsingPOST( + result = await this.internalClient.fetchClinicalDataCountsUsingPOST( { clinicalDataCountFilter: { attributes: [ @@ -4850,7 +4857,7 @@ export class StudyViewPageStore if (!this.hasFilteredSamples) { return []; } - result = await internalClient.fetchCustomDataCountsUsingPOST( + result = await this.internalClient.fetchCustomDataCountsUsingPOST( { clinicalDataCountFilter: { attributes: [ @@ -4898,7 +4905,7 @@ export class StudyViewPageStore return element !== undefined; } ); - const result2 = await internalClient.fetchCustomDataBinCountsUsingPOST( + const result2 = await this.internalClient.fetchCustomDataBinCountsUsingPOST( { dataBinMethod: 'STATIC', clinicalDataBinCountFilter: { @@ -4930,7 +4937,7 @@ export class StudyViewPageStore const attribute: ClinicalDataBinFilter = getDefaultClinicalDataBinFilter( chartMeta.clinicalAttribute! ); - const result = await internalClient.fetchCustomDataBinCountsUsingPOST( + const result = await this.internalClient.fetchCustomDataBinCountsUsingPOST( { dataBinMethod: 'STATIC', clinicalDataBinCountFilter: { @@ -4977,7 +4984,7 @@ export class StudyViewPageStore const attribute: ClinicalDataBinFilter = this._customDataBinFilterSet.get( uniqueKey )!; - const result = await internalClient.fetchCustomDataBinCountsUsingPOST( + const result = await this.internalClient.fetchCustomDataBinCountsUsingPOST( { dataBinMethod: 'STATIC', clinicalDataBinCountFilter: { @@ -5030,26 +5037,46 @@ export class StudyViewPageStore chartMeta.uniqueKey ); if (chartInfo) { - const result = await invokeGenericAssayDataCount( - chartInfo, - this.filters + let result: GenericAssayDataCountItem[] = []; + + result = await this.internalClient.fetchGenericAssayDataCountsUsingPOST( + { + genericAssayDataCountFilter: { + genericAssayDataFilters: [ + { + stableId: + chartInfo.genericAssayEntityId, + profileType: chartInfo.profileType, + } as GenericAssayDataFilter, + ], + studyViewFilter: this.filters, + } as GenericAssayDataCountFilter, + } ); if (_.isEmpty(result)) { return res; } - if (!this.chartToUsedColors.has(result!.stableId)) { - this.chartToUsedColors.set( - result!.stableId, - new Set() - ); + let data = result.find( + d => d.stableId === chartInfo.genericAssayEntityId + ); + let counts: ClinicalDataCount[] = []; + let stableId: string = ''; + if (data !== undefined) { + counts = data.counts.map(c => { + return { + count: c.count, + value: c.value, + } as ClinicalDataCount; + }); + stableId = data.stableId; + if (!this.chartToUsedColors.has(stableId)) { + this.chartToUsedColors.set(stableId, new Set()); + } } - return this.addColorToCategories( - result!.counts, - result!.stableId - ); + return this.addColorToCategories(counts, stableId); } return res; }, @@ -5209,7 +5236,7 @@ export class StudyViewPageStore if (!this.hasSampleIdentifiersInFilter) { return []; } - result = await internalClient.fetchClinicalDataBinCountsUsingPOST( + result = await this.internalClient.fetchClinicalDataBinCountsUsingPOST( { dataBinMethod, clinicalDataBinCountFilter: { @@ -5265,7 +5292,7 @@ export class StudyViewPageStore )!; //only invoke if there are filtered samples if (chartInfo && this.hasFilteredSamples) { - const genomicDataBins = await internalClient.fetchGenomicDataBinCountsUsingPOST( + const genomicDataBins = await this.internalClient.fetchGenomicDataBinCountsUsingPOST( { dataBinMethod: DataBinMethodConstants.STATIC, genomicDataBinCountFilter: { @@ -5315,7 +5342,7 @@ export class StudyViewPageStore chartMeta.uniqueKey )!; if (chartInfo) { - const gaDataBins = await internalClient.fetchGenericAssayDataBinCountsUsingPOST( + const gaDataBins = await this.internalClient.fetchGenericAssayDataBinCountsUsingPOST( { dataBinMethod: DataBinMethodConstants.STATIC, genericAssayDataBinCountFilter: { @@ -5746,7 +5773,7 @@ export class StudyViewPageStore readonly resourceDefinitions = remoteData({ await: () => [this.queriedPhysicalStudies], invoke: () => { - return internalClient.fetchResourceDefinitionsUsingPOST({ + return this.internalClient.fetchResourceDefinitionsUsingPOST({ studyIds: this.queriedPhysicalStudies.result.map( study => study.studyId ), @@ -5771,7 +5798,7 @@ export class StudyViewPageStore const promises = []; for (const resource of studyResourceDefinitions) { promises.push( - internalClient + this.internalClient .getAllStudyResourceDataInStudyUsingGET({ studyId: resource.studyId, resourceId: resource.resourceId, @@ -7723,7 +7750,7 @@ export class StudyViewPageStore attr => attr.attributeId ); - return internalClient.fetchClinicalDataCountsUsingPOST({ + return this.internalClient.fetchClinicalDataCountsUsingPOST({ clinicalDataCountFilter: { attributes, studyViewFilter: this.initialFilters, @@ -7754,7 +7781,7 @@ export class StudyViewPageStore >({ await: () => [this.initialVisibleAttributesClinicalDataBinAttributes], invoke: async () => { - const clinicalDataBinCountData = await internalClient.fetchClinicalDataBinCountsUsingPOST( + const clinicalDataBinCountData = await this.internalClient.fetchClinicalDataBinCountsUsingPOST( { dataBinMethod: 'STATIC', clinicalDataBinCountFilter: { @@ -8008,7 +8035,7 @@ export class StudyViewPageStore !_.isEmpty(studyViewFilter.sampleIdentifiers) || !_.isEmpty(studyViewFilter.studyIds) ) { - return internalClient.fetchFilteredSamplesUsingPOST({ + return this.internalClient.fetchFilteredSamplesUsingPOST({ studyViewFilter: studyViewFilter, }); } @@ -8128,7 +8155,7 @@ export class StudyViewPageStore )}` ); } - return internalClient.fetchFilteredSamplesUsingPOST({ + return this.internalClient.fetchFilteredSamplesUsingPOST({ studyViewFilter: this.filters, }); } else { @@ -8240,7 +8267,7 @@ export class StudyViewPageStore >( q => ({ invoke: async () => ({ - data: await internalClient.fetchClinicalDataViolinPlotsUsingPOST( + data: await this.internalClient.fetchClinicalDataViolinPlotsUsingPOST( { categoricalAttributeId: q.chartInfo.categoricalAttr.clinicalAttributeId, @@ -8332,7 +8359,7 @@ export class StudyViewPageStore ) { parameters.yAxisStart = 0; // mutation count always starts at 0 } - const result: any = await internalClient.fetchClinicalDataDensityPlotUsingPOST( + const result: any = await this.internalClient.fetchClinicalDataDensityPlotUsingPOST( parameters ); const bins = result.bins.filter( @@ -8392,7 +8419,7 @@ export class StudyViewPageStore : [this.mutationProfiles], invoke: async () => { if (!_.isEmpty(this.mutationProfiles.result)) { - let mutatedGenes = await internalClient.fetchMutatedGenesUsingPOST( + let mutatedGenes = await this.internalClient.fetchMutatedGenesUsingPOST( { studyViewFilter: this.filters, } @@ -8448,7 +8475,7 @@ export class StudyViewPageStore : [this.structuralVariantProfiles], invoke: async () => { if (!_.isEmpty(this.structuralVariantProfiles.result)) { - const structuralVariantGenes = await internalClient.fetchStructuralVariantGenesUsingPOST( + const structuralVariantGenes = await this.internalClient.fetchStructuralVariantGenesUsingPOST( { studyViewFilter: this.filters, } @@ -8504,7 +8531,7 @@ export class StudyViewPageStore : [this.structuralVariantProfiles], invoke: async () => { if (!_.isEmpty(this.structuralVariantProfiles.result)) { - const structuralVariantCounts = await internalClient.fetchStructuralVariantCountsUsingPOST( + const structuralVariantCounts = await this.internalClient.fetchStructuralVariantCountsUsingPOST( { studyViewFilter: this.filters, } @@ -8581,9 +8608,11 @@ export class StudyViewPageStore : [this.cnaProfiles], invoke: async () => { if (!_.isEmpty(this.cnaProfiles.result)) { - let cnaGenes = await internalClient.fetchCNAGenesUsingPOST({ - studyViewFilter: this.filters, - }); + let cnaGenes = await this.internalClient.fetchCNAGenesUsingPOST( + { + studyViewFilter: this.filters, + } + ); return cnaGenes.map(item => { return { ...item, @@ -8683,7 +8712,7 @@ export class StudyViewPageStore const molecularProfileIds = this.molecularProfiles.result.map( molecularProfile => molecularProfile.molecularProfileId ); - const report = await internalClient.fetchAlterationDriverAnnotationReportUsingPOST( + const report = await this.internalClient.fetchAlterationDriverAnnotationReportUsingPOST( { molecularProfileIds } ); return { @@ -9213,7 +9242,7 @@ export class StudyViewPageStore await: () => [this.molecularProfiles], invoke: async () => { const [counts, selectedSamples] = await Promise.all([ - internalClient.fetchMolecularProfileSampleCountsUsingPOST({ + this.internalClient.fetchMolecularProfileSampleCountsUsingPOST({ studyViewFilter: this.filters, }), toPromise(this.selectedSamples), @@ -9238,7 +9267,7 @@ export class StudyViewPageStore readonly caseListSampleCounts = remoteData({ invoke: async () => { const [counts, selectedSamples] = await Promise.all([ - internalClient.fetchCaseListCountsUsingPOST({ + this.internalClient.fetchCaseListCountsUsingPOST({ studyViewFilter: this.filters, }), toPromise(this.selectedSamples), @@ -9279,9 +9308,11 @@ export class StudyViewPageStore readonly initialMolecularProfileSampleCounts = remoteData({ invoke: async () => { - return internalClient.fetchMolecularProfileSampleCountsUsingPOST({ - studyViewFilter: this.initialFilters, - }); + return this.internalClient.fetchMolecularProfileSampleCountsUsingPOST( + { + studyViewFilter: this.initialFilters, + } + ); }, default: [], }); @@ -9342,7 +9373,7 @@ export class StudyViewPageStore let clinicalAttributeCountFilter = { sampleIdentifiers, } as ClinicalAttributeCountFilter; - return internalClient.getClinicalAttributeCountsUsingPOST({ + return this.internalClient.getClinicalAttributeCountsUsingPOST({ clinicalAttributeCountFilter, }); }, @@ -9421,9 +9452,7 @@ export class StudyViewPageStore } const calculateSampleCount = ( - result: - | (SampleTreatmentRow | PatientTreatmentRow)[] - | undefined + result: SampleTreatmentRow[] | undefined ) => { if (!result) { return 0; @@ -9439,34 +9468,34 @@ export class StudyViewPageStore }, new Set()).size; }; if (!_.isEmpty(this.sampleTreatments.result)) { - ret['SAMPLE_TREATMENTS'] = calculateSampleCount( - this.sampleTreatments.result - ); + ret[ + 'SAMPLE_TREATMENTS' + ] = this.sampleTreatments.result!.totalSamples; } if (!_.isEmpty(this.patientTreatments.result)) { - ret['PATIENT_TREATMENTS'] = calculateSampleCount( - this.patientTreatments.result - ); + ret[ + 'PATIENT_TREATMENTS' + ] = this.patientTreatments.result!.totalPatients; } if (!_.isEmpty(this.sampleTreatmentGroups.result)) { - ret['SAMPLE_TREATMENT_GROUPS'] = calculateSampleCount( - this.sampleTreatmentGroups.result - ); + ret[ + 'SAMPLE_TREATMENT_GROUPS' + ] = this.sampleTreatments.result!.totalSamples; } if (!_.isEmpty(this.patientTreatmentGroups.result)) { - ret['PATIENT_TREATMENT_GROUPS'] = calculateSampleCount( - this.patientTreatmentGroups.result - ); + ret[ + 'PATIENT_TREATMENT_GROUPS' + ] = this.patientTreatmentGroups.result!.totalPatients; } if (!_.isEmpty(this.sampleTreatmentTarget.result)) { - ret['SAMPLE_TREATMENT_TARGET'] = calculateSampleCount( - this.sampleTreatmentTarget.result - ); + ret[ + 'SAMPLE_TREATMENT_TARGET' + ] = this.sampleTreatments.result!.totalSamples; } if (!_.isEmpty(this.patientTreatmentTarget.result)) { - ret['PATIENT_TREATMENT_TARGET'] = calculateSampleCount( - this.patientTreatmentTarget.result - ); + ret[ + 'PATIENT_TREATMENT_TARGET' + ] = this.patientTreatmentTarget.result!.totalPatients; } if (!_.isEmpty(this.structuralVariantProfiles.result)) { const structVarGenesUniqueKey = getUniqueKeyFromMolecularProfileIds( @@ -10050,7 +10079,7 @@ export class StudyViewPageStore } public readonly clinicalEventTypeCounts = remoteData({ invoke: async () => { - return internalClient.getClinicalEventTypeCountsUsingPOST({ + return this.internalClient.getClinicalEventTypeCountsUsingPOST({ studyViewFilter: this.filters, }); }, @@ -10064,9 +10093,11 @@ export class StudyViewPageStore filters.studyIds = this.queriedPhysicalStudyIds.result; return Promise.resolve( ( - await internalClient.getClinicalEventTypeCountsUsingPOST({ - studyViewFilter: filters as StudyViewFilter, - }) + await this.internalClient.getClinicalEventTypeCountsUsingPOST( + { + studyViewFilter: filters as StudyViewFilter, + } + ) ).length > 0 ); }, @@ -10390,20 +10421,41 @@ export class StudyViewPageStore // a specific treatment public readonly sampleTreatments = remoteData({ await: () => [this.shouldDisplaySampleTreatments], - invoke: () => { + invoke: async () => { if (this.shouldDisplaySampleTreatments.result) { - return internalClient.getAllSampleTreatmentsUsingPOST({ - studyViewFilter: this.filters, - }); + if (isClickhouseMode()) { + return await this.internalClient.fetchSampleTreatmentCountsUsing( + { + studyViewFilter: this.filters, + } + ); + } else { + // we need to transform old response into new SampleTreatmentReport + const old = await oldInternalClient.getAllSampleTreatmentsUsingPOST( + { + studyViewFilter: this.filters, + } + ); + const resp: SampleTreatmentReport = { + treatments: old, + totalSamples: _(old) + .flatMap(r => r.samples) + .map(r => r.sampleId) + .uniq() + .value().length, + }; + return resp; + } + } else { + return Promise.resolve(undefined); } - return Promise.resolve([]); }, }); public readonly shouldDisplayPatientTreatments = remoteData({ await: () => [this.queriedPhysicalStudyIds], invoke: () => { - return internalClient.getContainsTreatmentDataUsingPOST({ + return this.internalClient.getContainsTreatmentDataUsingPOST({ studyIds: toJS(this.queriedPhysicalStudyIds.result), }); }, @@ -10412,7 +10464,7 @@ export class StudyViewPageStore public readonly shouldDisplaySampleTreatments = remoteData({ await: () => [this.queriedPhysicalStudyIds], invoke: () => { - return internalClient.getContainsSampleTreatmentDataUsingPOST({ + return this.internalClient.getContainsSampleTreatmentDataUsingPOST({ studyIds: toJS(this.queriedPhysicalStudyIds.result), }); }, @@ -10422,13 +10474,36 @@ export class StudyViewPageStore // a specific treatment public readonly patientTreatments = remoteData({ await: () => [this.shouldDisplayPatientTreatments], - invoke: () => { + invoke: async () => { if (this.shouldDisplayPatientTreatments.result) { - return internalClient.getAllPatientTreatmentsUsingPOST({ - studyViewFilter: this.filters, - }); + if (isClickhouseMode()) { + return await this.internalClient.fetchPatientTreatmentCountsUsing( + { + studyViewFilter: this.filters, + } + ); + } else { + // we need to transform pre-clickhouse response into new SampleTreatmentReport + const legacyData = await this.internalClient.getAllPatientTreatmentsUsingPOST( + { + studyViewFilter: this.filters, + } + ); + const totalPatients = _(legacyData) + .flatMap(r => r.samples) + .map(r => r.patientId) + .uniq() + .value().length; + const resp: PatientTreatmentReport = { + patientTreatments: legacyData, + totalSamples: 0, // this is always zero, should be cleaned up in backend and deleted + totalPatients, + }; + return resp; + } + } else { + return Promise.resolve(undefined); } - return Promise.resolve([]); }, }); @@ -10436,7 +10511,7 @@ export class StudyViewPageStore await: () => [this.shouldDisplaySampleTreatmentGroups], invoke: () => { if (this.shouldDisplaySampleTreatmentGroups.result) { - return internalClient.getAllSampleTreatmentsUsingPOST({ + return this.internalClient.getAllSampleTreatmentsUsingPOST({ studyViewFilter: this.filters, tier: 'AgentClass', }); @@ -10451,7 +10526,7 @@ export class StudyViewPageStore if (!getServerConfig().enable_treatment_groups) { return Promise.resolve(false); } - return internalClient.getContainsTreatmentDataUsingPOST({ + return this.internalClient.getContainsTreatmentDataUsingPOST({ studyIds: toJS(this.queriedPhysicalStudyIds.result), tier: 'AgentClass', }); @@ -10464,7 +10539,7 @@ export class StudyViewPageStore if (!getServerConfig().enable_treatment_groups) { return Promise.resolve(false); } - return internalClient.getContainsSampleTreatmentDataUsingPOST({ + return this.internalClient.getContainsSampleTreatmentDataUsingPOST({ studyIds: toJS(this.queriedPhysicalStudyIds.result), tier: 'AgentClass', }); @@ -10477,12 +10552,12 @@ export class StudyViewPageStore await: () => [this.shouldDisplayPatientTreatmentGroups], invoke: () => { if (this.shouldDisplayPatientTreatmentGroups.result) { - return internalClient.getAllPatientTreatmentsUsingPOST({ + return this.internalClient.fetchPatientTreatmentCountsUsing({ studyViewFilter: this.filters, tier: 'AgentClass', }); } - return Promise.resolve([]); + return Promise.resolve(undefined); }, }); @@ -10490,7 +10565,7 @@ export class StudyViewPageStore await: () => [this.shouldDisplaySampleTreatmentTarget], invoke: () => { if (this.shouldDisplaySampleTreatmentTarget.result) { - return internalClient.getAllSampleTreatmentsUsingPOST({ + return this.internalClient.getAllSampleTreatmentsUsingPOST({ studyViewFilter: this.filters, tier: 'AgentTarget', }); @@ -10505,7 +10580,7 @@ export class StudyViewPageStore if (!getServerConfig().enable_treatment_groups) { return Promise.resolve(false); } - return internalClient.getContainsTreatmentDataUsingPOST({ + return this.internalClient.getContainsTreatmentDataUsingPOST({ studyIds: toJS(this.queriedPhysicalStudyIds.result), tier: 'AgentTarget', }); @@ -10518,7 +10593,7 @@ export class StudyViewPageStore if (!getServerConfig().enable_treatment_groups) { return Promise.resolve(false); } - return internalClient.getContainsSampleTreatmentDataUsingPOST({ + return this.internalClient.getContainsSampleTreatmentDataUsingPOST({ studyIds: toJS(this.queriedPhysicalStudyIds.result), tier: 'AgentTarget', }); @@ -10530,13 +10605,10 @@ export class StudyViewPageStore public readonly patientTreatmentTarget = remoteData({ await: () => [this.shouldDisplayPatientTreatmentTarget], invoke: () => { - if (this.shouldDisplayPatientTreatmentTarget.result) { - return internalClient.getAllPatientTreatmentsUsingPOST({ - studyViewFilter: this.filters, - tier: 'AgentTarget', - }); - } - return Promise.resolve([]); + return this.internalClient.fetchPatientTreatmentCountsUsing({ + studyViewFilter: this.filters, + tier: 'AgentTarget', + }); }, }); @@ -11061,14 +11133,16 @@ export class StudyViewPageStore // createStructuralVariantQuery(sv, this.plotsSelectedGenes.result!) // ); - return await internalClient.fetchStructuralVariantsUsingPOST({ - structuralVariantFilter: { - entrezGeneIds, - structuralVariantQueries: [], - sampleMolecularIdentifiers, - molecularProfileIds: [], - }, - }); + return await this.internalClient.fetchStructuralVariantsUsingPOST( + { + structuralVariantFilter: { + entrezGeneIds, + structuralVariantQueries: [], + sampleMolecularIdentifiers, + molecularProfileIds: [], + }, + } + ); } }, }); @@ -11368,7 +11442,7 @@ export class StudyViewPageStore readonly genesets = remoteData({ invoke: () => { if (this.genesetIds && this.genesetIds.length > 0) { - return internalClient.fetchGenesetsUsingPOST({ + return this.internalClient.fetchGenesetsUsingPOST({ genesetIds: this.genesetIds.slice(), }); } else { @@ -11414,7 +11488,7 @@ export class StudyViewPageStore if (_.isEmpty(filters)) { return []; } else { - return internalClient.fetchStructuralVariantsUsingPOST({ + return this.internalClient.fetchStructuralVariantsUsingPOST({ structuralVariantFilter: { entrezGeneIds: [q.entrezGeneId], sampleMolecularIdentifiers: filters, diff --git a/src/pages/studyView/StudyViewUtils.tsx b/src/pages/studyView/StudyViewUtils.tsx index e2e8b78bc78..f2c548c3161 100644 --- a/src/pages/studyView/StudyViewUtils.tsx +++ b/src/pages/studyView/StudyViewUtils.tsx @@ -25,10 +25,12 @@ import { NumericGeneMolecularData, Patient, PatientIdentifier, + PatientTreatmentReport, PatientTreatmentRow, Sample, SampleClinicalDataCollection, SampleIdentifier, + SampleTreatmentReport, SampleTreatmentRow, StructuralVariantFilterQuery, StudyViewFilter, @@ -47,7 +49,9 @@ import { } from './StudyViewPageStore'; import { StudyViewPageTabKeyEnum } from 'pages/studyView/StudyViewPageTabs'; import { Layout } from 'react-grid-layout'; -import internalClient from 'shared/api/cbioportalInternalClientInstance'; +import internalClient, { + getInternalClient, +} from 'shared/api/cbioportalInternalClientInstance'; import defaultClient from 'shared/api/cbioportalClientInstance'; import client from 'shared/api/cbioportalClientInstance'; import { @@ -2539,7 +2543,7 @@ export function getSamplesByExcludingFiltersOnChart( updatedFilter.sampleIdentifiers = queriedSampleIdentifiers; } } - return internalClient.fetchFilteredSamplesUsingPOST({ + return getInternalClient().fetchFilteredSamplesUsingPOST({ studyViewFilter: updatedFilter, }); } @@ -3182,7 +3186,7 @@ export async function getAllClinicalDataByStudyViewFilter( const [remoteClinicalDataCollection, totalItems]: [ SampleClinicalDataCollection, number - ] = await internalClient + ] = await getInternalClient() .fetchClinicalDataClinicalTableUsingPOSTWithHttpInfo({ studyViewFilter, pageSize: pageSize | 500, @@ -4073,7 +4077,7 @@ export async function invokeGenericAssayDataCount( chartInfo: GenericAssayChart, filters: StudyViewFilter ) { - const result: GenericAssayDataCountItem[] = await internalClient.fetchGenericAssayDataCountsUsingPOST( + const result: GenericAssayDataCountItem[] = await getInternalClient().fetchGenericAssayDataCountsUsingPOST( { genericAssayDataCountFilter: { genericAssayDataFilters: [ @@ -4135,12 +4139,16 @@ export async function invokeGenomicDataCount( projection: 'SUMMARY', }, }; - result = await internalClient.fetchMutationDataCountsUsingPOST(params); + result = await getInternalClient().fetchMutationDataCountsUsingPOST( + params + ); getDisplayedValue = transformMutatedType; getDisplayedColor = (value: string) => getMutationColorByCategorization(transformMutatedType(value)); } else { - result = await internalClient.fetchGenomicDataCountsUsingPOST(params); + result = await getInternalClient().fetchGenomicDataCountsUsingPOST( + params + ); getDisplayedValue = getCNAByAlteration; getDisplayedColor = (value: string | number) => getCNAColorByAlteration(getCNAByAlteration(value)); @@ -4194,7 +4202,7 @@ export async function invokeMutationDataCount( }, } as any; - const result = await internalClient.fetchMutationDataCountsUsingPOST( + const result = await getInternalClient().fetchMutationDataCountsUsingPOST( params ); @@ -4536,26 +4544,29 @@ export async function getGenesCNADownloadData( } export async function getPatientTreatmentDownloadData( - promise: MobxPromise + promise: MobxPromise ): Promise { if (promise.result) { const header = ['Treatment', '#']; let data = [header.join('\t')]; - _.each(promise.result, (record: PatientTreatmentRow) => { - let rowData = [record.treatment, record.count]; - data.push(rowData.join('\t')); - }); + _.each( + promise.result.patientTreatments, + (record: PatientTreatmentRow) => { + let rowData = [record.treatment, record.count]; + data.push(rowData.join('\t')); + } + ); return data.join('\n'); } else return ''; } export async function getSampleTreatmentDownloadData( - promise: MobxPromise + promise: MobxPromise ): Promise { if (promise.result) { const header = ['Treatment', 'Pre/Post', '#']; let data = [header.join('\t')]; - _.each(promise.result, (record: SampleTreatmentRow) => { + _.each(promise.result.treatments, (record: SampleTreatmentRow) => { let rowData = [record.treatment, record.time, record.count]; data.push(rowData.join('\t')); }); diff --git a/src/pages/studyView/rfc80Tester.tsx b/src/pages/studyView/rfc80Tester.tsx new file mode 100644 index 00000000000..5e3f4951515 --- /dev/null +++ b/src/pages/studyView/rfc80Tester.tsx @@ -0,0 +1,175 @@ +import * as React from 'react'; +import _ from 'lodash'; +import { useCallback, useEffect } from 'react'; +import axios from 'axios'; +import { + reportValidationResult, + runSpecs, + validate, +} from 'shared/api/validation.ts'; +import { getBrowserWindow } from 'cbioportal-frontend-commons'; +import { observer } from 'mobx-react'; +import { useLocalObservable } from 'mobx-react-lite'; +import { SAVE_TEST_KEY } from 'shared/api/testMaker'; + +const CACHE_KEY: string = 'testCache'; + +const RFC_TEST_SHOW: string = 'RFC_TEST_SHOW'; + +const LIVE_VALIDATE_KEY: string = 'LIVE_VALIDATE_KEY'; + +function getCache() { + return getBrowserWindow()[CACHE_KEY] || {}; + //return localStorage.getItem(CACHE_KEY); +} + +function clearCache() { + getBrowserWindow()[CACHE_KEY] = {}; +} + +export const RFC80Test = observer(function() { + const store = useLocalObservable(() => ({ + tests: [], + show: !!localStorage.getItem(RFC_TEST_SHOW), + listening: !!localStorage.getItem(SAVE_TEST_KEY), + validate: !!localStorage.getItem(LIVE_VALIDATE_KEY), + })); + + const clearCacheCallback = useCallback(() => { + clearCache(); + }, []); + + const toggleListener = useCallback(() => { + store.listening = !store.listening; + if (getBrowserWindow().localStorage.getItem(SAVE_TEST_KEY)) { + getBrowserWindow().localStorage.removeItem(SAVE_TEST_KEY); + } else { + getBrowserWindow().localStorage.setItem(SAVE_TEST_KEY, 'true'); + } + }, []); + + const toggleShow = useCallback(() => { + !!localStorage.getItem(RFC_TEST_SHOW) + ? localStorage.removeItem(RFC_TEST_SHOW) + : localStorage.setItem(RFC_TEST_SHOW, 'true'); + store.show = !store.show; + }, []); + + const toggleLiveValidate = useCallback(() => { + !!localStorage.getItem(LIVE_VALIDATE_KEY) + ? localStorage.removeItem(LIVE_VALIDATE_KEY) + : localStorage.setItem(LIVE_VALIDATE_KEY, 'true'); + store.validate = !store.validate; + }, []); + + const runTests = useCallback(async () => { + let json = []; + + try { + json = await $.getJSON( + 'https://localhost:3000/common/merged-tests.json' + ); + } catch (ex) { + alert('merged-tests.json not found'); + } + + const fileFilter = $('#apiTestFilter') + .val() + ?.toString(); + + const files: any[] = fileFilter?.trim().length + ? json.filter((f: any) => new RegExp(fileFilter).test(f.file)) + : json; + + await runSpecs(files, axios, '', 'verbose'); + }, []); + + useEffect(() => { + if (getCache()) { + const tests = getCache(); + const parsed = _.values(tests).map((j: any) => j); + store.tests = parsed; + } + + const checker = setInterval(() => { + if (getCache()) { + const tests = getCache(); + const parsed = _.values(tests); + store.tests = parsed; + } else { + store.tests = []; + } + }, 1000); + + return () => { + clearInterval(checker); + }; + }, []); + + const txt = ` + { + "name":"", + "note":"", + "studies":[], + "tests":[ + ${store.tests.map((t: any) => JSON.stringify(t)).join(',\n\n')} + ] + }`; + + if (!store.show) { + return ( +
+ +
+ ); + } + + return ( +
+ + + + + + + { + + } +
+ ); +}); diff --git a/src/pages/studyView/table/treatments/PatientTreatmentsTable.tsx b/src/pages/studyView/table/treatments/PatientTreatmentsTable.tsx index 8a5a335c95a..4c9cc9c1f2e 100644 --- a/src/pages/studyView/table/treatments/PatientTreatmentsTable.tsx +++ b/src/pages/studyView/table/treatments/PatientTreatmentsTable.tsx @@ -7,7 +7,10 @@ import { Column, SortDirection, } from '../../../../shared/components/lazyMobXTable/LazyMobXTable'; -import { PatientTreatmentRow } from 'cbioportal-ts-api-client'; +import { + PatientTreatmentReport, + PatientTreatment, +} from 'cbioportal-ts-api-client'; import { correctColumnWidth } from 'pages/studyView/StudyViewUtils'; import LabeledCheckbox from 'shared/components/labeledCheckbox/LabeledCheckbox'; import styles from 'pages/studyView/table/tables.module.scss'; @@ -36,7 +39,7 @@ export type PatientTreatmentsTableColumn = { export type PatientTreatmentsTableProps = { tableType: TreatmentTableType; - promise: MobxPromise; + promise: MobxPromise; width: number; height: number; filters: string[][]; @@ -59,9 +62,7 @@ const DEFAULT_COLUMN_WIDTH_RATIO: { [PatientTreatmentsTableColumnKey.COUNT]: 0.2, }; -class MultiSelectionTableComponent extends FixedHeaderTable< - PatientTreatmentRow -> {} +class MultiSelectionTableComponent extends FixedHeaderTable {} @observer export class PatientTreatmentsTable extends TreatmentsTable< @@ -80,7 +81,7 @@ export class PatientTreatmentsTable extends TreatmentsTable< } createNubmerColumnCell( - row: PatientTreatmentRow, + row: PatientTreatment, cellMargin: number ): JSX.Element { return ( @@ -111,9 +112,7 @@ export class PatientTreatmentsTable extends TreatmentsTable< cellMargin: number ) => { const defaults: { - [key in PatientTreatmentsTableColumnKey]: Column< - PatientTreatmentRow - >; + [key in PatientTreatmentsTableColumnKey]: Column; } = { [PatientTreatmentsTableColumnKey.TREATMENT]: { name: columnKey, @@ -123,10 +122,10 @@ export class PatientTreatmentsTable extends TreatmentsTable< headerName={columnKey} /> ), - render: (data: PatientTreatmentRow) => ( + render: (data: PatientTreatment) => ( ), - sortBy: (data: PatientTreatmentRow) => data.treatment, + sortBy: (data: PatientTreatment) => data.treatment, defaultSortDirection: 'asc' as 'asc', filter: filterTreatmentCell, width: columnWidth, @@ -140,9 +139,9 @@ export class PatientTreatmentsTable extends TreatmentsTable< headerName={columnKey} /> ), - render: (data: PatientTreatmentRow) => + render: (data: PatientTreatment) => this.createNubmerColumnCell(data, 28), - sortBy: (data: PatientTreatmentRow) => + sortBy: (data: PatientTreatment) => data.count + toNumericValue(data.treatment), defaultSortDirection: 'desc' as 'desc', filter: filterTreatmentCell, @@ -181,8 +180,8 @@ export class PatientTreatmentsTable extends TreatmentsTable< ); } - @computed get tableData(): PatientTreatmentRow[] { - return this.props.promise.result || []; + @computed get tableData(): PatientTreatment[] { + return this.props.promise.result?.patientTreatments || []; } @computed @@ -206,7 +205,7 @@ export class PatientTreatmentsTable extends TreatmentsTable< .filter(data => this.flattenedFilters.includes(treatmentUniqueKey(data)) ) - .sortBy(data => + .sortBy(data => ifNotDefined( order[treatmentUniqueKey(data)], Number.POSITIVE_INFINITY diff --git a/src/pages/studyView/table/treatments/SampleTreatmentsTable.tsx b/src/pages/studyView/table/treatments/SampleTreatmentsTable.tsx index 43070757afe..025fff9d7b8 100644 --- a/src/pages/studyView/table/treatments/SampleTreatmentsTable.tsx +++ b/src/pages/studyView/table/treatments/SampleTreatmentsTable.tsx @@ -7,7 +7,10 @@ import { Column, SortDirection, } from '../../../../shared/components/lazyMobXTable/LazyMobXTable'; -import { SampleTreatmentRow } from 'cbioportal-ts-api-client'; +import { + SampleTreatmentReport, + SampleTreatmentRow, +} from 'cbioportal-ts-api-client'; import { correctColumnWidth } from 'pages/studyView/StudyViewUtils'; import LabeledCheckbox from 'shared/components/labeledCheckbox/LabeledCheckbox'; import styles from 'pages/studyView/table/tables.module.scss'; @@ -37,7 +40,7 @@ export type SampleTreatmentsTableColumn = { export type SampleTreatmentsTableProps = { tableType: TreatmentTableType; - promise: MobxPromise; + promise: MobxPromise; width: number; height: number; filters: string[][]; @@ -214,7 +217,7 @@ export class SampleTreatmentsTable extends TreatmentsTable< } @computed get tableData(): SampleTreatmentRow[] { - return this.props.promise.result || []; + return this.props.promise.result?.treatments || []; } @computed get selectableTableData() { diff --git a/src/pages/studyView/table/treatments/treatmentsTableUtil.tsx b/src/pages/studyView/table/treatments/treatmentsTableUtil.tsx index d18e2694bf4..af38c602b92 100644 --- a/src/pages/studyView/table/treatments/treatmentsTableUtil.tsx +++ b/src/pages/studyView/table/treatments/treatmentsTableUtil.tsx @@ -3,6 +3,7 @@ import { SampleTreatmentFilter, PatientTreatmentFilter, PatientTreatmentRow, + PatientTreatment, } from 'cbioportal-ts-api-client'; import { ChartMeta } from 'pages/studyView/StudyViewUtils'; import styles from 'pages/studyView/table/tables.module.scss'; @@ -90,7 +91,7 @@ export const TreatmentGenericColumnHeader = class GenericColumnHeader extends Re }; export const TreatmentColumnCell = class TreatmentColumnCell extends React.Component< - { row: PatientTreatmentRow | SampleTreatmentRow }, + { row: PatientTreatment | SampleTreatmentRow }, {} > { render() { @@ -99,7 +100,7 @@ export const TreatmentColumnCell = class TreatmentColumnCell extends React.Compo }; export function filterTreatmentCell( - cell: PatientTreatmentRow | SampleTreatmentRow, + cell: PatientTreatment | SampleTreatmentRow, filter: string ): boolean { return cell.treatment.toUpperCase().includes(filter.toUpperCase()); diff --git a/src/shared/api/cbioportalInternalClientInstance.ts b/src/shared/api/cbioportalInternalClientInstance.ts index 0aa9e2f3c57..7af697dc7d4 100644 --- a/src/shared/api/cbioportalInternalClientInstance.ts +++ b/src/shared/api/cbioportalInternalClientInstance.ts @@ -1,5 +1,157 @@ import { CBioPortalAPIInternal } from 'cbioportal-ts-api-client'; +import { + getLoadConfig, + getServerConfig, + isClickhouseMode, +} from 'config/config'; +import { getBrowserWindow, hashString } from 'cbioportal-frontend-commons'; +import { toJS } from 'mobx'; +import { reportValidationResult, validate } from 'shared/api/validation'; +import _ from 'lodash'; +import { makeTest, urlChopper } from 'shared/api/testMaker'; +import axios from 'axios'; +import pako from 'pako'; + +function proxyColumnStore(client: any, endpoint: string) { + if (getBrowserWindow().location.search.includes('legacy')) { + return; + } + + const method = endpoint.match( + new RegExp('fetchPatientTreatmentCounts|fetchSampleTreatmentCounts') + ) + ? `${endpoint}UsingWithHttpInfo` + : `${endpoint}UsingPOSTWithHttpInfo`; + const old = client[method]; + + client[method] = function(params: any) { + const host = getLoadConfig().baseUrl; + + const oldRequest = this.request; + + const endpoints = [ + 'ClinicalDataCounts', + 'MutatedGenes', + 'CaseList', + 'ClinicalDataBin', + 'MolecularProfileSample', + 'CNAGenes', + 'StructuralVariantGenes', + 'FilteredSamples', + 'ClinicalDataDensity', + 'MutationDataCounts', + 'PatientTreatmentCounts', + 'SampleTreatmentCounts', + 'GenomicData', + 'GenericAssay', + 'ViolinPlots', + 'ClinicalEventTypeCounts', + ]; + + const matchedMethod = method.match(new RegExp(endpoints.join('|'))); + if (localStorage.getItem('LIVE_VALIDATE_KEY') && matchedMethod) { + this.request = function(...origArgs: any[]) { + const params = toJS(arguments[2]); + const oldSuccess = arguments[7]; + + arguments[7] = function(response: any) { + const url = + origArgs[1].replace( + /column-store\/api/, + 'column-store' + ) + + '?' + + _.map(origArgs[4], (v, k) => `${k}=${v}&`).join(''); + + setTimeout(() => { + makeTest(params, urlChopper(url), matchedMethod[0]); + }, 1000); + + const hash = hashString( + JSON.stringify({ data: params, url: urlChopper(url) }) + ); + + validate( + axios, + url, + params, + matchedMethod[0], + hash, + response.body, + response.headers['elapsed-time'] + ).then((result: any) => { + reportValidationResult(result, 'LIVE', 'verbose'); + }); + + return oldSuccess.apply(this, arguments); + }; + + oldRequest.apply(this, arguments); + }; + } + + params.$domain = method.match( + new RegExp('PatientTreatmentCounts|SampleTreatmentCounts') + ) + ? `//${host}` + : `//${host}/api/column-store`; + const url = old.apply(this, [params]); + + this.request = oldRequest; + + return url; + }; +} const internalClient = new CBioPortalAPIInternal(); +const internalClientColumnStore = new CBioPortalAPIInternal(); + +const oldRequest = (internalClientColumnStore as any).request; +(internalClientColumnStore as any).request = function(...args: any) { + args[1] = args[1].replace(/column-store\/api/, 'column-store'); + // Compress request body if enabled + if (getServerConfig().enable_request_body_gzip_compression) { + if (args[0] === 'POST' && !_.isEmpty(args[2])) { + let bodyString = JSON.stringify(args[2]); + if (bodyString.length > 10000) { + args[3]['Content-Encoding'] = 'gzip'; + args[2] = pako.gzip(bodyString).buffer; + } else { + // Store stringified body, so that stringify only runs once. + args[2] = bodyString; + } + } + } + return oldRequest.apply(this, args); +}; + +proxyColumnStore(internalClientColumnStore, 'fetchCNAGenes'); +proxyColumnStore(internalClientColumnStore, 'fetchStructuralVariantGenes'); +proxyColumnStore(internalClientColumnStore, 'fetchCaseListCounts'); +proxyColumnStore( + internalClientColumnStore, + 'fetchMolecularProfileSampleCounts' +); +proxyColumnStore(internalClientColumnStore, 'fetchMutatedGenes'); +proxyColumnStore(internalClientColumnStore, 'fetchFilteredSamples'); +proxyColumnStore(internalClientColumnStore, 'fetchClinicalDataCounts'); +proxyColumnStore(internalClientColumnStore, 'fetchClinicalDataBinCounts'); +proxyColumnStore(internalClientColumnStore, 'fetchClinicalDataDensityPlot'); +proxyColumnStore(internalClientColumnStore, 'fetchMutationDataCounts'); +proxyColumnStore(internalClientColumnStore, 'fetchPatientTreatmentCounts'); +proxyColumnStore(internalClientColumnStore, 'fetchSampleTreatmentCounts'); +proxyColumnStore(internalClientColumnStore, 'fetchClinicalDataDensityPlot'); +proxyColumnStore(internalClientColumnStore, 'getClinicalEventTypeCounts'); +proxyColumnStore(internalClientColumnStore, 'fetchMutationDataCounts'); +proxyColumnStore(internalClientColumnStore, 'fetchGenomicDataCounts'); +proxyColumnStore(internalClientColumnStore, 'fetchGenomicDataBinCounts'); +proxyColumnStore(internalClientColumnStore, 'fetchGenericAssayDataBinCounts'); +proxyColumnStore(internalClientColumnStore, 'fetchGenericAssayDataCounts'); +proxyColumnStore(internalClientColumnStore, 'fetchClinicalDataViolinPlots'); + export default internalClient; + +export function getInternalClient() { + return isClickhouseMode() ? internalClientColumnStore : internalClient; +} diff --git a/src/shared/api/testMaker.ts b/src/shared/api/testMaker.ts new file mode 100644 index 00000000000..758b157808d --- /dev/null +++ b/src/shared/api/testMaker.ts @@ -0,0 +1,80 @@ +import { getBrowserWindow, hashString } from 'cbioportal-frontend-commons'; +import { toJS } from 'mobx'; +import _ from 'lodash'; + +export const SAVE_TEST_KEY = 'save_test_enabled'; + +export function urlChopper(url: string) { + try { + if (typeof url === 'string') { + return url.match(/[^\/]*\/\/[^\/]*(\/.*)/)![1]; + } else { + return url; + } + } catch (ex) { + return url; + } +} + +export async function makeTest(data: any, url: string, label: string) { + const hash = hashString(JSON.stringify({ data, url: urlChopper(url) })); + + const filterString = $('.userSelections') + .find('*') + .contents() + .filter(function() { + return this.nodeType === 3; + }) + .toArray() + .map(n => n.textContent) + .slice(0, -1) + .reduce((acc, s) => { + switch (s) { + case null: + acc += ''; + break; + case '(': + acc += ' ('; + break; + case ')': + acc += ') '; + break; + case 'or': + acc += ' OR '; + break; + case 'and': + acc += ' AND '; + break; + default: + acc += s || ''; + break; + } + return acc; + }, ''); + + const entry = { + hash, + filterString, + data, + url, + label, + studies: toJS(getBrowserWindow().studyViewPageStore.studyIds), + filterUrl: urlChopper( + getBrowserWindow().studyPage.studyViewFullUrlWithFilter + ), + }; + + if (getBrowserWindow().localStorage.getItem(SAVE_TEST_KEY)) + saveTest(hash, entry); + + return entry; +} + +function saveTest(hash: number, entry: any) { + const testCache = getBrowserWindow().testCache || {}; + + if (!(hash in testCache)) { + testCache[hash] = entry; + getBrowserWindow().testCache = testCache; + } +} diff --git a/src/shared/api/validation.ts b/src/shared/api/validation.ts new file mode 100644 index 00000000000..80168655be6 --- /dev/null +++ b/src/shared/api/validation.ts @@ -0,0 +1,691 @@ +export const isObject = (value: any) => { + return ( + typeof value === 'object' && + value !== null && + !Array.isArray(value) && + !(value instanceof RegExp) && + !(value instanceof Date) && + !(value instanceof Set) && + !(value instanceof Map) + ); +}; + +function getLogLevel(level: string) { + if (typeof window != 'undefined') { + return 'verbose-all'; + } else { + return level; + } +} + +export function dynamicSortSingle(property: string) { + var sortOrder = 1; + if (property[0] === '-') { + sortOrder = -1; + property = property.substr(1); + } + return function(a: any, b: any) { + /* next line works with strings and numbers, + * and you may want to customize it to your needs + */ + var result = + a[property] < b[property] ? -1 : a[property] > b[property] ? 1 : 0; + return result * sortOrder; + }; +} + +export function dynamicSort(property: string[]) { + if (property.length === 1) { + return dynamicSortSingle(property[0]); + } else { + const prop1 = property[0]; + const prop2 = property[1]; + return function(a: any, b: any) { + /* next line works with strings and numbers, + * and you may want to customize it to your needs + */ + let af = a[prop1]; + let bf = b[prop1]; + let as = a[prop2]; + let bs = b[prop2]; + + // If first value is same + if (af == bf) { + return as < bs ? -1 : as > bs ? 1 : 0; + } else { + return af < bf ? -1 : 1; + } + }; + } +} + +export function getArrays(inp: any, output: Array) { + if (inp instanceof Array) { + output.push(inp); + + inp.forEach(n => getArrays(n, output)); + } else if (isObject(inp)) { + for (const k in inp) { + if (/\d\.\d{10,}$/.test(inp[k])) { + try { + inp[k] = inp[k].toFixed(5); + } catch (ex) {} + } + } + + // if (inp.counts) { + // inp.counts = inp.counts.filter((n: any) => { + // return n.label != 'NA'; + // }); + // } + + // this is get rid if extraneouys properties that conflict + delete inp.numberOfProfiledCases; + delete inp.matchingGenePanelIds; + delete inp.cytoband; + // delete inp.numberOfProfiledCases; + + Object.values(inp).forEach(nn => getArrays(nn, output)); + } + return output; +} + +const deleteFields: Record = { + MolecularProfileSample: ['label'], + MolecularProfileSampleCounts: ['label'], + CaseList: ['label'], + SampleListsCounts: ['label'], + CnaGenes: ['qValue', 'entrezGeneId', 'entrezGeneIds'], + MutatedGenes: ['qValue', 'entrezGeneId', 'entrezGeneIds'], + ClinicalDataViolinPlots: [], + StructuralVariantGenes: ['entrezGeneId', 'entrezGeneIds'], +}; + +const sortFields: Record = { + ClinicalDataBinCounts: 'attributeId,specialValue', + ClinicalDataBin: 'attributeId,specialValue', + FilteredSamples: 'uniqueSampleKey', + SampleTreatmentCounts: 'treatment,time', + PatientTreatmentCounts: 'treatment', + ClinicalDataCounts: 'attributeId,value', + ClinicalDataTypeCounts: 'eventType', + ClinicalEventTypeCounts: 'eventType', + ClinicalDataViolinPlots: 'sampleId,numSamples,category', +}; + +function getLegacyPatientTreatmentCountUrl(url: string) { + return url.replace( + /api\/treatments\/patient-counts\/fetch?/, + 'api/treatments/patient' + ); +} + +function getLegacySampleTreatmentCountUrl(url: string) { + return url.replace( + /api\/treatments\/sample-counts\/fetch?/, + 'api/treatments/sample' + ); +} + +const treatmentLegacyUrl: Record string> = { + PatientTreatmentCounts: getLegacyPatientTreatmentCountUrl, + SampleTreatmentCounts: getLegacySampleTreatmentCountUrl, +}; + +const treatmentConverter: Record any> = { + PatientTreatmentCounts: convertLegacyPatientTreatmentCountsToCh, + SampleTreatmentCounts: convertLegacySampleTreatmentCountsToCh, +}; + +function convertLegacySampleTreatmentCountsToCh(legacyData: any) { + const sampleIdSet = new Set(); + const treatments: Array<{ + time: string; + treatment: string; + count: number; + samples: Array; + }> = []; + + legacyData.forEach((legacySampleTreatment: any) => { + let treatment = { + count: legacySampleTreatment['count'], + samples: new Array(), + time: legacySampleTreatment['time'], + treatment: legacySampleTreatment['treatment'], + }; + + treatments.push(treatment); + const samples = legacySampleTreatment['samples']; + if (samples instanceof Array) { + samples.forEach(sample => { + sampleIdSet.add(sample['sampleId']); + }); + } + }); + return { + totalSamples: sampleIdSet.size, + treatments: treatments, + }; +} + +function convertLegacyPatientTreatmentCountsToCh(legacyData: any) { + const patientIdSet = new Set(); + const treatments: Array<{ treatment: string; count: number }> = []; + + legacyData.forEach((legacyTreatment: any) => { + let treatment = { + count: legacyTreatment['count'], + treatment: legacyTreatment['treatment'], + }; + treatments.push(treatment); + + const samples = legacyTreatment['samples']; + if (samples instanceof Array) { + samples.forEach(sample => { + patientIdSet.add(sample['patientId']); + }); + } + }); + + return { + totalPatients: patientIdSet.size, + totalSamples: 0, + patientTreatments: treatments, + }; +} + +export function deepSort(inp: any, label: string) { + const arrs = getArrays(inp, []); + + arrs.forEach(arr => { + if (label in deleteFields) { + arr.forEach((m: any) => { + deleteFields[label].forEach(l => { + delete m[l]; + }); + }); + } + + arr.forEach((m: any) => { + if (m.value && m.value.toLowerCase) m.value = m.value.toLowerCase(); + }); + + arr.forEach((m: any) => { + if (m.specialValue && m.specialValue.toLowerCase) + m.specialValue = m.specialValue.toLowerCase(); + }); + + if (!arr.length) return; + if (!isObject(arr[0])) { + arr.sort(); + } else { + // it's an array of objects + + // this is going to make sure the keys in the objects + // are in a sorted order + arr.forEach((o: any) => { + Object.keys(o) + .sort() + .forEach(k => { + const val = o[k]; + delete o[k]; + o[k] = val; + }); + }); + + if (sortFields[label]) { + //console.log("SORTING BY", arr); + attemptSort(sortFields[label].split(','), arr); + } else { + const fields = [ + 'attributeId', + 'value', + 'hugoGeneSymbol', + 'uniqueSampleKey', + 'alteration', + ]; + fields.forEach(f => attemptSort([f], arr)); + } + } + }); + + return inp; +} + +function attemptSort(keys: string[], arr: any) { + arr.sort(dynamicSort(keys)); +} + +let win: any; + +try { + win = window; +} catch (ex) { + win = {}; +} + +function removeElement(nums: any[], val: any) { + for (let i = 0; i < nums.length; i++) { + if (nums[i] === val) { + nums.splice(i, 1); + i--; + } + } +} + +export function compareCounts(clData: any, legacyData: any, label: string) { + // @ts-ignore + let clDataClone = win.structuredClone ? structuredClone(clData) : clData; + + let legacyDataClone = win.structuredClone + ? // @ts-ignore + structuredClone(legacyData) + : legacyData; + + // get trid of duplicates + //clDataClone = filterDuplicates(clDataClone); + + var clDataSorted = deepSort(clDataClone, label); + var legacyDataSorted = deepSort(legacyDataClone, label); + + // correct for messed up spearmanCorr + if (clDataSorted && clDataSorted.spearmanCorr) { + clDataSorted.spearmanCorr = parseFloat( + clDataSorted.spearmanCorr.toString() + ).toFixed(5); + legacyDataSorted.spearmanCorr = parseFloat( + legacyDataSorted.spearmanCorr.toString() + ).toFixed(5); + } + + // getArrays(clDataSorted, []).forEach((arr: any) => { + // arr.filter((n: any) => /NA/i.test(n.value)).forEach((val: any) => { + // removeElement(arr, val); + // }); + // }); + + // getArrays(legacyDataSorted, []).forEach((arr: any) => { + // arr.filter((n: any) => /NA/i.test(n.value)).forEach((val: any) => { + // removeElement(arr, val); + // }); + // }); + + // getArrays(legacyDataSorted, []).forEach((arr: any) => { + // arr.filter((n: any) => /NA/i.test(n.specialValue)&&n.count===0 ).forEach((val: any) => { + // removeElement(arr, val); + // }); + // }); + + // get rid of these little guys + if (clDataSorted && clDataSorted.filter) + clDataSorted = clDataSorted.filter((n: any) => n.specialValue != 'NA'); + + if (legacyDataSorted && legacyDataSorted.filter) + legacyDataSorted = legacyDataSorted.filter( + (n: any) => n.specialValue != 'NA' + ); + + if (treatmentConverter[label]) { + legacyDataSorted = treatmentConverter[label](legacyDataSorted); + } + const result = + JSON.stringify(clDataSorted) === JSON.stringify(legacyDataSorted); + + return { + clDataSorted, + legacyDataSorted, + status: result, + label, + }; +} + +export async function validate( + ajax: any, + url: string, + params: any, + label: string, + hash: number, + body?: any, + elapsedTime: any = 0, + assertResponse: any[] | undefined = undefined, + onFail: (...args: any[]) => void = () => {} +) { + let chXHR: any; + + let chResult; + let legacyResult; + + if (body) { + chResult = { body, elapsedTime, status: 200 }; + } else { + chResult = await ajax + .post(url, params) + .then(function(response: any) { + return { + status: response.status, + body: response.data, + elapsedTime: response.headers['elapsed-time'], + }; + }) + .catch(function(error: any) { + return { + body: null, + error, + elapsedTime: null, + status: error.status, + }; + }); + } + + if (assertResponse) { + legacyResult = { + body: assertResponse, + elapsedTime: null, + }; + } else { + let legacyUrl = url.replace(/column-store\//, ''); + + if (treatmentLegacyUrl[label]) { + legacyUrl = treatmentLegacyUrl[label](legacyUrl); + } + + legacyResult = await ajax + .post(legacyUrl, params) + .then(function(response: any) { + return { + status: response.status, + body: response.data, + elapsedTime: response.headers['elapsed-time'], + }; + }) + .catch(function(error: any) { + return { + body: null, + error, + elapsedTime: null, + status: error.status, + }; + }); + } + + const result: any = compareCounts(chResult.body, legacyResult.body, label); + result.url = url; + result.hash = hash; + result.data = params; + result.chDuration = chResult.elapsedTime; + result.legacyDuration = legacyResult.elapsedTime; + result.chError = chResult.error; + result.legacyResult = legacyResult; + result.chResult = chResult; + + if (!result.status) { + onFail(url); + } + + return result; +} + +const red = '\x1b[31m'; +const green = '\x1b[32m'; +const blue = '\x1b[36m'; +const reset = '\x1b[0m'; + +export function reportValidationResult( + result: any, + prefix = '', + logLevel = '' +) { + const skipMessage = + result.test && result.test.skip ? `(SKIPPED ${result.test.skip})` : ''; + + const errorStatus = result.chError ? `(${result.chError.status})` : ''; + + const data = result.data || result?.test.data; + + const studies = ( + data?.studyIds || + data?.studyViewFilter?.studyIds || + [] + ).join(','); + + !result.status && + !result.supressed && + console.groupCollapsed( + `${red} ${prefix} ${result.label} (${result.hash}) ${skipMessage} failed (${errorStatus}) ${studies} :( ${reset}` + ); + + if (result.supressed) { + console.log( + `${blue} ${prefix} ${result.label} (${result.hash}) ${skipMessage} SUPPRESSED :( ${reset}` + ); + } + + if (getLogLevel(logLevel) === 'verbose' && !result.status) { + console.log('failed test', { + url: result.url, + test: result.test, + studies: result?.test?.studies, + legacyDuration: result.legacyDuration, + chDuration: result.chDuration, + equal: result.status, + httpError: result.httpError, + }); + } + + if (result.status) { + const studies = result.test + ? result.test.data.studyIds || + result.test.data.studyViewFilter.studyIds + : []; + console.log( + `${prefix} ${result.label} (${ + result.hash + }) passed :) (${studies.join(',')}) ch: ${ + result.chDuration + } legacy: ${result.legacyDuration && result.legacyDuration}` + ); + // @ts-ignore + // process.stdout.clearLine(0) + // // @ts-ignore + // process.stdout.cursorTo(0) + // // @ts-ignore + // process.stdout.write(`passed`) + } + + if (!result.status && getLogLevel(logLevel).includes('verbose')) { + // violin plot response has a single node of rows + const chSubject = result?.clDataSorted.rows || result?.clDataSorted; + const legacySubject = + result?.legacyDataSorted.rows || result?.legacyDataSorted; + + if (chSubject?.length && legacySubject?.length) { + for (var i = 0; i < chSubject?.length; i++) { + const cl = chSubject[i]; + if (JSON.stringify(cl) !== JSON.stringify(legacySubject[i])) { + console.groupCollapsed( + `First invalid item (${result.label})` + ); + console.log('Clickhouse:', cl); + console.log('Legacy:', legacySubject[i]); + console.groupEnd(); + break; + } + } + } + if (getLogLevel(logLevel).includes('all')) { + console.groupCollapsed('All Data'); + console.log( + `CH: ${chSubject?.length}, Legacy:${legacySubject.length}` + ); + console.log('legacy', result.legacyDataSorted); + console.log('CH', result.clDataSorted); + console.groupEnd(); + } + } + + !result.status && console.groupEnd(); +} + +export async function runSpecs( + files: any, + axios: any, + host: string = '', + logLevel = '', + onFail: any = () => {}, + suppressors: any = [] +) { + // @ts-ignore + let allTests = files + // @ts-ignore + .flatMap((n: any) => n.suites) + // @ts-ignore + .flatMap((n: any) => n.tests); + + const totalCount = allTests.length; + + const onlyDetected = allTests.some((t: any) => t.only === true); + + console.log(`Running specs (${files.length} of ${totalCount})`); + + if (getLogLevel(logLevel).includes('verbose')) { + console.groupCollapsed('specs'); + //console.log('raw', json); + console.log('filtered', files); + console.groupEnd(); + } + + let place = 0; + let errors: any[] = []; + let skips: any[] = []; + let passed: any[] = []; + let httpErrors: any[] = []; + let supressed: any[] = []; + + // + // let filterFunc = (test:any)=>{ + // return [2087302238,-1234322951,-341016470,388930409].includes(test.hash); + // } + let filterFunc = (test: any) => true; + + const invokers: (() => Promise)[] = [] as any; + files + .map((f: any) => f.suites) + .forEach((suite: any) => { + suite.forEach((col: any) => + col.tests.filter(filterFunc).forEach((test: any) => { + test.url = test.url.replace( + /column-store\/api/, + 'column-store' + ); + + if (!onlyDetected || test.only) { + invokers.push( + // @ts-ignore + () => { + return validate( + axios, + host + test.url, + test.data, + test.label, + test.hash, + undefined, + undefined, + test.assertResponse + ).then((report: any) => { + if (!report.status) { + onFail(test, report); + } + + report.test = test; + place = place + 1; + const prefix = `${place} of ${totalCount}`; + if (report instanceof Promise) { + report.then((report: any) => { + if (test?.skip) { + skips.push(test.hash); + } else if (!report.status) { + report.httpError + ? httpErrors.push(test.hash) + : errors.push(test.hash); + } else if (report.status) + passed.push(test.hash); + + reportValidationResult( + report, + prefix, + logLevel + ); + }); + } else { + if (test?.skip) { + skips.push(test.hash); + } else if (!report.status) { + let supress = []; + + supress = suppressors + .map((f: any) => { + try { + return f(report); + } catch (exc) { + return false; + } + }) + .filter((r: any) => r); + + if (supress.length) { + supressed.push(test.hash); + report.supressed = true; + } else { + report.httpError + ? httpErrors.push(test.hash) + : errors.push(test.hash); + } + } else if (report.status) + passed.push(test.hash); + + reportValidationResult( + report, + prefix, + logLevel + ); + } + }); + } + ); + } + }) + ); + }); + + const concurrent = 5; + const batches = Math.ceil(invokers.length / concurrent); + + for (var i = 0; i < batches; i++) { + const proms = []; + for (const inv of invokers.slice( + i * concurrent, + (i + 1) * concurrent + )) { + proms.push(inv()); + } + await Promise.all(proms); + } + + console.group('FINAL REPORT'); + console.log(`PASSED: ${passed.length} of ${totalCount}`); + console.log(`FAILED: ${errors.length} (${errors.join(',')})`); + console.log(`HTTP ERRORS: ${httpErrors.length} (${httpErrors.join(',')})`); + console.log(`SKIPPED: ${skips.length} (${skips.join(',')})`); + console.log(`SUPRESSED: ${supressed.length} (${supressed.join(',')})`); + console.groupEnd(); + + try { + if (errors.length > 0) { + process.exit(1); + } else { + process.exit(0); + } + } catch (ex) { + // fail silently we're in browser + } +} diff --git a/webpack.config.js b/webpack.config.js index 3580b03db73..60e2697c3a2 100755 --- a/webpack.config.js +++ b/webpack.config.js @@ -35,13 +35,10 @@ function cleanAndValidateUrl(url) { const NODE_ENV = process.env.NODE_ENV || 'development'; -var jsonFN = require('json-fn'); - const dotenv = require('dotenv'); const webpack = require('webpack'); const path = require('path'); - const join = path.join; const resolve = path.resolve; @@ -170,6 +167,7 @@ var config = { { from: './common-dist', to: 'reactapp' }, { from: './src/rootImages', to: 'images' }, { from: './src/common', to: 'common' }, + { from: './api-e2e/json', to: 'common' }, { from: './src/globalStyles/prefixed-bootstrap.min.css', to: 'reactapp/prefixed-bootstrap.min.css', diff --git a/yarn.lock b/yarn.lock index 840732ddb91..fcaa483e411 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5411,6 +5411,15 @@ aws4@^1.2.1, aws4@^1.8.0: resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f" integrity sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ== +axios@^1.7.7: + version "1.7.7" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.7.7.tgz#2f554296f9892a72ac8d8e4c5b79c14a91d0a47f" + integrity sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q== + dependencies: + follow-redirects "^1.15.6" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + babel-cli@^6.4.5: version "6.26.0" resolved "https://registry.yarnpkg.com/babel-cli/-/babel-cli-6.26.0.tgz#502ab54874d7db88ad00b887a06383ce03d002f1" @@ -8941,6 +8950,15 @@ csstype@^3.0.2: resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.0.9.tgz#6410af31b26bd0520933d02cbc64fce9ce3fbf0b" integrity sha512-rpw6JPxK6Rfg1zLOYCSwle2GFOOsnjmDYDaBwEcwoOg4qlsIVCN789VkBZDJAGi4T07gI4YSutR43t9Zz4Lzuw== +csvtojson@^2.0.10: + version "2.0.10" + resolved "https://registry.yarnpkg.com/csvtojson/-/csvtojson-2.0.10.tgz#11e7242cc630da54efce7958a45f443210357574" + integrity sha512-lUWFxGKyhraKCW8Qghz6Z0f2l/PqB1W3AO0HKJzGIQ5JRSlR651ekJDiGJbBT4sRNNv5ddnSGVEnsxP9XRCVpQ== + dependencies: + bluebird "^3.5.1" + lodash "^4.17.3" + strip-bom "^2.0.0" + cubic-hermite@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/cubic-hermite/-/cubic-hermite-1.0.0.tgz#84e3b2f272b31454e8393b99bb6aed45168c14e5" @@ -11211,6 +11229,11 @@ follow-redirects@^1.0.0: dependencies: debug "^3.2.6" +follow-redirects@^1.15.6: + version "1.15.9" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1" + integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ== + font-atlas-sdf@^1.3.3: version "1.3.3" resolved "https://registry.yarnpkg.com/font-atlas-sdf/-/font-atlas-sdf-1.3.3.tgz#8323f136c69d73a235aa8c6ada640e58f180b8c0" @@ -11313,6 +11336,15 @@ form-data@^3.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" +form-data@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.1.tgz#ba1076daaaa5bfd7e99c1a6cb02aa0a5cff90d48" + integrity sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + form-data@~2.1.1: version "2.1.4" resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.1.4.tgz#33c183acf193276ecaa98143a69e94bfee1750d1" @@ -15510,7 +15542,7 @@ lodash.words@^3.0.0: dependencies: lodash._root "^3.0.0" -lodash@4.x, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.7.0: +lodash@4.x, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.3, lodash@^4.7.0: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -19663,6 +19695,11 @@ proxy-addr@~2.0.4, proxy-addr@~2.0.5: forwarded "~0.1.2" ipaddr.js "1.9.0" +proxy-from-env@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" + integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== + prr@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476"