diff --git a/.gitignore b/.gitignore index f7a096c4dc..589207c6c0 100644 --- a/.gitignore +++ b/.gitignore @@ -81,6 +81,9 @@ coverage cypress-coverage reports +# clinic.js performance profiling data +.clinic + # Downloaded artifacts from Nexus / Github / etc via download-artifact script artifacts **/zap-*.zip diff --git a/package-lock.json b/package-lock.json index fe9ee23578..ade69309ae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@quasar/extras": "^1.15.6", "@types/yargs": "^15.0.5", "axios": "^0.26.1", + "axios-cookiejar-support": "^4.0.6", "body-parser": "^1.19.0", "bufferutil": "^4.0.3", "bytebuffer": "^5.0.1", @@ -24,10 +25,12 @@ "express-session": "^1.17.2", "fast-glob": "^3.2.11", "file-loader": "^6.2.0", + "find": "^0.3.0", "glob": "^8.0.3", "handlebars": "^4.7.7", "http-status-codes": "^2.1.4", "lodash": "^4.17.21", + "node-cache": "^5.1.2", "node-ipc": "^9.2.1", "pino": "^7.9.2", "promised-handlebars": "^2.0.1", @@ -2305,6 +2308,19 @@ "node": ">=0.6" } }, + "node_modules/@cypress/request/node_modules/tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "dev": true, + "dependencies": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/@cypress/vite-dev-server": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/@cypress/vite-dev-server/-/vite-dev-server-2.2.3.tgz", @@ -8403,6 +8419,24 @@ "follow-redirects": "^1.14.8" } }, + "node_modules/axios-cookiejar-support": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/axios-cookiejar-support/-/axios-cookiejar-support-4.0.6.tgz", + "integrity": "sha512-lWDhgM6bc2xYAsHkXEhceLpTu9ytAeIz1VSuL5FoUgGx2lqcMNbNxTD9Hm4x5c8JF5Me0HfNrb06fhEGMC30mQ==", + "dependencies": { + "http-cookie-agent": "^5.0.2" + }, + "engines": { + "node": ">=14.18.0 <15.0.0 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/3846masa" + }, + "peerDependencies": { + "axios": ">=0.20.0", + "tough-cookie": ">=4.0.0" + } + }, "node_modules/babel-core": { "version": "7.0.0-bridge.0", "resolved": "https://registry.npmjs.org/babel-core/-/babel-core-7.0.0-bridge.0.tgz", @@ -15878,6 +15912,14 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "node_modules/find": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/find/-/find-0.3.0.tgz", + "integrity": "sha512-iSd+O4OEYV/I36Zl8MdYJO0xD82wH528SaCieTVHhclgiYNe9y+yPKSwK+A7/WsmHL1EZ+pYUJBXWTL5qofksw==", + "dependencies": { + "traverse-chain": "~0.1.0" + } + }, "node_modules/find-babel-config": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/find-babel-config/-/find-babel-config-1.2.0.tgz", @@ -17583,6 +17625,33 @@ "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==", "devOptional": true }, + "node_modules/http-cookie-agent": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/http-cookie-agent/-/http-cookie-agent-5.0.2.tgz", + "integrity": "sha512-BiBmZyIMGl5mLKmY7KH2uCVlcNUl1jexjdtWXFCUF4DFOrNZg1c5iPPTzWDzU7Ngfb6fB03DPpJQ80KQWmycsg==", + "dependencies": { + "agent-base": "^6.0.2" + }, + "engines": { + "node": ">=14.18.0 <15.0.0 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/3846masa" + }, + "peerDependencies": { + "deasync": "^0.1.26", + "tough-cookie": "^4.0.0", + "undici": "^5.11.0" + }, + "peerDependenciesMeta": { + "deasync": { + "optional": true + }, + "undici": { + "optional": true + } + } + }, "node_modules/http-deceiver": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz", @@ -19922,21 +19991,6 @@ "node": ">=v12.22.7" } }, - "node_modules/jest-environment-jsdom/node_modules/tough-cookie": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.2.tgz", - "integrity": "sha512-G9fqXWoYFZgTc2z8Q5zaHy/vJMjm+WV0AkAeHxVCQiEB1b+dGvWzFW6QV07cY5jQ5gRkeid2qIkzkxUnmoQZUQ==", - "dev": true, - "dependencies": { - "psl": "^1.1.33", - "punycode": "^2.1.1", - "universalify": "^0.2.0", - "url-parse": "^1.5.3" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/jest-environment-jsdom/node_modules/tr46": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", @@ -19949,15 +20003,6 @@ "node": ">=12" } }, - "node_modules/jest-environment-jsdom/node_modules/universalify": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", - "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", - "dev": true, - "engines": { - "node": ">= 4.0.0" - } - }, "node_modules/jest-environment-jsdom/node_modules/w3c-xmlserializer": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-4.0.0.tgz", @@ -21687,30 +21732,6 @@ "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==", "dev": true }, - "node_modules/jsdom/node_modules/tough-cookie": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.2.tgz", - "integrity": "sha512-G9fqXWoYFZgTc2z8Q5zaHy/vJMjm+WV0AkAeHxVCQiEB1b+dGvWzFW6QV07cY5jQ5gRkeid2qIkzkxUnmoQZUQ==", - "dev": true, - "dependencies": { - "psl": "^1.1.33", - "punycode": "^2.1.1", - "universalify": "^0.2.0", - "url-parse": "^1.5.3" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/jsdom/node_modules/universalify": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", - "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", - "dev": true, - "engines": { - "node": ">= 4.0.0" - } - }, "node_modules/jsdom/node_modules/ws": { "version": "7.5.9", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", @@ -23901,6 +23922,25 @@ "integrity": "sha512-ibPK3iA+vaY1eEjESkQkM0BbCqFOaZMiXRTtdB0u7b4djtY6JnsjvPdUHVMg6xQt3B8fpTTWHI9A+ADjM9frzg==", "dev": true }, + "node_modules/node-cache": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/node-cache/-/node-cache-5.1.2.tgz", + "integrity": "sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==", + "dependencies": { + "clone": "2.x" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/node-cache/node_modules/clone": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", + "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/node-dir": { "version": "0.1.17", "resolved": "https://registry.npmjs.org/node-dir/-/node-dir-0.1.17.tgz", @@ -27363,8 +27403,7 @@ "node_modules/psl": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", - "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==", - "dev": true + "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==" }, "node_modules/pump": { "version": "3.0.0", @@ -27380,7 +27419,6 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.2.0.tgz", "integrity": "sha512-LN6QV1IJ9ZhxWTNdktaPClrNfp8xdSAYS0Zk2ddX7XsXZAxckMHPCBcHRo0cTcEIgYPRiGEkmji3Idkh2yFtYw==", - "dev": true, "engines": { "node": ">=6" } @@ -27438,8 +27476,7 @@ "node_modules/querystringify": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", - "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", - "dev": true + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==" }, "node_modules/queue-microtask": { "version": "1.2.3", @@ -28255,6 +28292,19 @@ "node": ">=0.6" } }, + "node_modules/request/node_modules/tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "dev": true, + "dependencies": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/request/node_modules/uuid": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", @@ -28314,8 +28364,7 @@ "node_modules/requires-port": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", - "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", - "dev": true + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==" }, "node_modules/requizzle": { "version": "0.2.4", @@ -31100,16 +31149,25 @@ } }, "node_modules/tough-cookie": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", - "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", - "dev": true, + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.2.tgz", + "integrity": "sha512-G9fqXWoYFZgTc2z8Q5zaHy/vJMjm+WV0AkAeHxVCQiEB1b+dGvWzFW6QV07cY5jQ5gRkeid2qIkzkxUnmoQZUQ==", "dependencies": { - "psl": "^1.1.28", - "punycode": "^2.1.1" + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.2.0", + "url-parse": "^1.5.3" }, "engines": { - "node": ">=0.8" + "node": ">=6" + } + }, + "node_modules/tough-cookie/node_modules/universalify": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", + "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", + "engines": { + "node": ">= 4.0.0" } }, "node_modules/tr46": { @@ -31124,6 +31182,11 @@ "node": ">=8" } }, + "node_modules/traverse-chain": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/traverse-chain/-/traverse-chain-0.1.0.tgz", + "integrity": "sha512-up6Yvai4PYKhpNp5PkYtx50m3KbwQrqDwbuZP/ItyL64YEWHAvH6Md83LFLV/GRSk/BoUVwwgUzX6SOQSbsfAg==" + }, "node_modules/treeify": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/treeify/-/treeify-1.1.0.tgz", @@ -32106,7 +32169,6 @@ "version": "1.5.10", "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", - "dev": true, "dependencies": { "querystringify": "^2.1.1", "requires-port": "^1.0.0" @@ -35836,6 +35898,16 @@ "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.3.tgz", "integrity": "sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==", "dev": true + }, + "tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "dev": true, + "requires": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + } } } }, @@ -40636,6 +40708,14 @@ "follow-redirects": "^1.14.8" } }, + "axios-cookiejar-support": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/axios-cookiejar-support/-/axios-cookiejar-support-4.0.6.tgz", + "integrity": "sha512-lWDhgM6bc2xYAsHkXEhceLpTu9ytAeIz1VSuL5FoUgGx2lqcMNbNxTD9Hm4x5c8JF5Me0HfNrb06fhEGMC30mQ==", + "requires": { + "http-cookie-agent": "^5.0.2" + } + }, "babel-core": { "version": "7.0.0-bridge.0", "resolved": "https://registry.npmjs.org/babel-core/-/babel-core-7.0.0-bridge.0.tgz", @@ -46287,6 +46367,14 @@ } } }, + "find": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/find/-/find-0.3.0.tgz", + "integrity": "sha512-iSd+O4OEYV/I36Zl8MdYJO0xD82wH528SaCieTVHhclgiYNe9y+yPKSwK+A7/WsmHL1EZ+pYUJBXWTL5qofksw==", + "requires": { + "traverse-chain": "~0.1.0" + } + }, "find-babel-config": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/find-babel-config/-/find-babel-config-1.2.0.tgz", @@ -47593,6 +47681,14 @@ "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==", "devOptional": true }, + "http-cookie-agent": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/http-cookie-agent/-/http-cookie-agent-5.0.2.tgz", + "integrity": "sha512-BiBmZyIMGl5mLKmY7KH2uCVlcNUl1jexjdtWXFCUF4DFOrNZg1c5iPPTzWDzU7Ngfb6fB03DPpJQ80KQWmycsg==", + "requires": { + "agent-base": "^6.0.2" + } + }, "http-deceiver": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz", @@ -49275,18 +49371,6 @@ "xmlchars": "^2.2.0" } }, - "tough-cookie": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.2.tgz", - "integrity": "sha512-G9fqXWoYFZgTc2z8Q5zaHy/vJMjm+WV0AkAeHxVCQiEB1b+dGvWzFW6QV07cY5jQ5gRkeid2qIkzkxUnmoQZUQ==", - "dev": true, - "requires": { - "psl": "^1.1.33", - "punycode": "^2.1.1", - "universalify": "^0.2.0", - "url-parse": "^1.5.3" - } - }, "tr46": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", @@ -49296,12 +49380,6 @@ "punycode": "^2.1.1" } }, - "universalify": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", - "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", - "dev": true - }, "w3c-xmlserializer": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-4.0.0.tgz", @@ -50637,24 +50715,6 @@ "integrity": "sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==", "dev": true }, - "tough-cookie": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.2.tgz", - "integrity": "sha512-G9fqXWoYFZgTc2z8Q5zaHy/vJMjm+WV0AkAeHxVCQiEB1b+dGvWzFW6QV07cY5jQ5gRkeid2qIkzkxUnmoQZUQ==", - "dev": true, - "requires": { - "psl": "^1.1.33", - "punycode": "^2.1.1", - "universalify": "^0.2.0", - "url-parse": "^1.5.3" - } - }, - "universalify": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", - "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", - "dev": true - }, "ws": { "version": "7.5.9", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", @@ -52478,6 +52538,21 @@ "integrity": "sha512-ibPK3iA+vaY1eEjESkQkM0BbCqFOaZMiXRTtdB0u7b4djtY6JnsjvPdUHVMg6xQt3B8fpTTWHI9A+ADjM9frzg==", "dev": true }, + "node-cache": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/node-cache/-/node-cache-5.1.2.tgz", + "integrity": "sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==", + "requires": { + "clone": "2.x" + }, + "dependencies": { + "clone": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", + "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==" + } + } + }, "node-dir": { "version": "0.1.17", "resolved": "https://registry.npmjs.org/node-dir/-/node-dir-0.1.17.tgz", @@ -55095,8 +55170,7 @@ "psl": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", - "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==", - "dev": true + "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==" }, "pump": { "version": "3.0.0", @@ -55111,8 +55185,7 @@ "punycode": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.2.0.tgz", - "integrity": "sha512-LN6QV1IJ9ZhxWTNdktaPClrNfp8xdSAYS0Zk2ddX7XsXZAxckMHPCBcHRo0cTcEIgYPRiGEkmji3Idkh2yFtYw==", - "dev": true + "integrity": "sha512-LN6QV1IJ9ZhxWTNdktaPClrNfp8xdSAYS0Zk2ddX7XsXZAxckMHPCBcHRo0cTcEIgYPRiGEkmji3Idkh2yFtYw==" }, "pupa": { "version": "2.1.1", @@ -55145,8 +55218,7 @@ "querystringify": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", - "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", - "dev": true + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==" }, "queue-microtask": { "version": "1.2.3", @@ -55789,6 +55861,16 @@ "integrity": "sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==", "dev": true }, + "tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "dev": true, + "requires": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + } + }, "uuid": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", @@ -55843,8 +55925,7 @@ "requires-port": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", - "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", - "dev": true + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==" }, "requizzle": { "version": "0.2.4", @@ -58035,13 +58116,21 @@ "dev": true }, "tough-cookie": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", - "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", - "dev": true, + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.2.tgz", + "integrity": "sha512-G9fqXWoYFZgTc2z8Q5zaHy/vJMjm+WV0AkAeHxVCQiEB1b+dGvWzFW6QV07cY5jQ5gRkeid2qIkzkxUnmoQZUQ==", "requires": { - "psl": "^1.1.28", - "punycode": "^2.1.1" + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.2.0", + "url-parse": "^1.5.3" + }, + "dependencies": { + "universalify": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", + "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==" + } } }, "tr46": { @@ -58053,6 +58142,11 @@ "punycode": "^2.1.1" } }, + "traverse-chain": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/traverse-chain/-/traverse-chain-0.1.0.tgz", + "integrity": "sha512-up6Yvai4PYKhpNp5PkYtx50m3KbwQrqDwbuZP/ItyL64YEWHAvH6Md83LFLV/GRSk/BoUVwwgUzX6SOQSbsfAg==" + }, "treeify": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/treeify/-/treeify-1.1.0.tgz", @@ -58791,7 +58885,6 @@ "version": "1.5.10", "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", - "dev": true, "requires": { "querystringify": "^2.1.1", "requires-port": "^1.0.0" diff --git a/package.json b/package.json index 9f72d937a3..24bb59c38e 100644 --- a/package.json +++ b/package.json @@ -118,6 +118,7 @@ "@quasar/extras": "^1.15.6", "@types/yargs": "^15.0.5", "axios": "^0.26.1", + "axios-cookiejar-support": "^4.0.6", "body-parser": "^1.19.0", "bufferutil": "^4.0.3", "bytebuffer": "^5.0.1", @@ -130,10 +131,12 @@ "express-session": "^1.17.2", "fast-glob": "^3.2.11", "file-loader": "^6.2.0", + "find": "^0.3.0", "glob": "^8.0.3", "handlebars": "^4.7.7", "http-status-codes": "^2.1.4", "lodash": "^4.17.21", + "node-cache": "^5.1.2", "node-ipc": "^9.2.1", "pino": "^7.9.2", "promised-handlebars": "^2.0.1", diff --git a/src-electron/db/db-cache.js b/src-electron/db/db-cache.js index e53db1fba2..786111d752 100644 --- a/src-electron/db/db-cache.js +++ b/src-electron/db/db-cache.js @@ -22,15 +22,15 @@ */ const dbApi = require('./db-api.js') const dbMapping = require('./db-mapping.js') - -const cacheEnabled = true -let cache = {} +const NodeCache = require('node-cache') +const cache = new NodeCache({ useClones: false }) +let cacheEnabled = true /** * Clears the entire cache. */ function clear() { - cache = {} + cache.flushAll() } /** @@ -38,11 +38,10 @@ function clear() { * @param {*} key * @param {*} packageId * @param {*} data + * @returns Returns true on success. */ function put(key, packageId, data) { - const keyCache = {} - keyCache[packageId] = data - cache[key] = keyCache + return cache.set(JSON.stringify([key, packageId]), data) } /** @@ -50,15 +49,10 @@ function put(key, packageId, data) { * * @param {*} key * @param {*} packageId - * @returns cached object or null if none is present + * @returns cached object or undefined if none is present or expired. */ function get(key, packageId) { - const keyCache = cache[key] - if (keyCache != null) { - return keyCache[packageId] - } else { - return null - } + return cache.get(JSON.stringify([key, packageId])) } /** @@ -69,13 +63,70 @@ function get(key, packageId) { * @returns true or false, depending on whether the cache is present. */ function isCached(key, packageId) { - const keyCache = cache[key] - return keyCache != null && keyCache[packageId] != null + return cache.has(JSON.stringify([key, packageId])) +} + +/** + * Cache input / output of provided queryFunction + * The queryFunction is assumed to have the following signature: + * + * async function queryFunction(db, ...) {...} + * + * The DB handle is ignored and the remaining arguments are used as the cache key. + * + * @param {*} key + * @param {*} packageId + * @returns true or false, depending on whether the cache is present. + */ +function cacheQuery(queryFunction) { + return function () { + if (arguments.length && cacheEnabled) { + // assume queryFunction's first argument is always the DB connection handler + let key = JSON.stringify([ + queryFunction.name, + Array.prototype.slice.call(arguments, 1), + ]) + + // check cache + let value = cache.get(key) + if (value == undefined) { + value = queryFunction.apply(this, arguments) + cache.set(key, value) + } + return value + } else { + return queryFunction.apply(this, arguments) + } + } +} + +/** + * Returns the cache statistics. + * + */ +function cacheStats() { + return cache.getStats() +} + +/** + * Enable the Database Query cache + */ +function enable() { + cacheEnabled = true +} + +/** + * Disable the database cache + */ +function disable() { + cacheEnabled = false } exports.clear = clear exports.put = put exports.get = get exports.isCached = isCached - -exports.cacheEnabled = cacheEnabled +exports.cacheQuery = cacheQuery +exports.cacheStats = cacheStats +exports.enable = enable +exports.disable = disable diff --git a/src-electron/db/query-atomic.js b/src-electron/db/query-atomic.js index f70d6c6cee..61774d4636 100644 --- a/src-electron/db/query-atomic.js +++ b/src-electron/db/query-atomic.js @@ -21,6 +21,7 @@ * @module DB API: zcl database access */ const dbApi = require('./db-api.js') +const dbCache = require('./db-cache.js') const dbMapping = require('./db-mapping.js') const ATOMIC_QUERY = ` @@ -82,5 +83,5 @@ async function selectAtomicById(db, id) { } exports.selectAllAtomics = selectAllAtomics -exports.selectAtomicType = selectAtomicType +exports.selectAtomicType = dbCache.cacheQuery(selectAtomicType) exports.selectAtomicById = selectAtomicById diff --git a/src-electron/db/query-bitmap.js b/src-electron/db/query-bitmap.js index 02c5e312f1..c9220f4a86 100644 --- a/src-electron/db/query-bitmap.js +++ b/src-electron/db/query-bitmap.js @@ -21,6 +21,7 @@ * @module DB API: zcl database access */ const dbApi = require('./db-api') +const dbCache = require('./db-cache') const dbMapping = require('./db-mapping') /** @@ -86,4 +87,4 @@ WHERE BITMAP_ID = ?`, exports.selectBitmapById = selectBitmapById exports.selectAllBitmaps = selectAllBitmaps -exports.selectBitmapByName = selectBitmapByName +exports.selectBitmapByName = dbCache.cacheQuery(selectBitmapByName) diff --git a/src-electron/db/query-data-type.js b/src-electron/db/query-data-type.js index 3c634c3ad1..926bc2384a 100644 --- a/src-electron/db/query-data-type.js +++ b/src-electron/db/query-data-type.js @@ -24,6 +24,7 @@ const dbMapping = require('./db-mapping') const queryZcl = require('./query-zcl') const envConfig = require('../util/env') const dbEnum = require('../../src-shared/db-enum.js') +const dbCache = require('./db-cache') /** * Gathers the data type information of an entry based on data type id along @@ -182,6 +183,6 @@ async function selectSizeFromType(db, packageIds, value) { } exports.selectDataTypeById = selectDataTypeById -exports.selectDataTypeByName = selectDataTypeByName +exports.selectDataTypeByName = dbCache.cacheQuery(selectDataTypeByName) exports.selectAllDataTypes = selectAllDataTypes exports.selectSizeFromType = selectSizeFromType diff --git a/src-electron/db/query-enum.js b/src-electron/db/query-enum.js index 26beb07ad0..c1304d34e6 100644 --- a/src-electron/db/query-enum.js +++ b/src-electron/db/query-enum.js @@ -21,6 +21,7 @@ * @module DB API: zcl database access */ const dbApi = require('./db-api') +const dbCache = require('./db-cache') const dbMapping = require('./db-mapping') /** @@ -211,8 +212,7 @@ ORDER BY NAME`, // exports exports.selectAllEnums = selectAllEnums -exports.selectEnumByName = selectEnumByName - +exports.selectEnumByName = dbCache.cacheQuery(selectEnumByName) exports.selectEnumById = selectEnumById exports.selectClusterEnums = selectClusterEnums exports.selectAllEnumItemsById = selectAllEnumItemsById diff --git a/src-electron/db/query-number.js b/src-electron/db/query-number.js index 3159d71689..fa3a1a7898 100644 --- a/src-electron/db/query-number.js +++ b/src-electron/db/query-number.js @@ -18,9 +18,9 @@ /** * This module provides queries for numbers */ - const dbApi = require('./db-api') const dbMapping = require('./db-mapping') +const dbCache = require('./db-cache') /** * Select an number matched by name. @@ -98,6 +98,6 @@ async function selectAllNumbers(db, packageId) { .then((rows) => rows.map(dbMapping.map.number)) } -exports.selectNumberByName = selectNumberByName +exports.selectNumberByName = dbCache.cacheQuery(selectNumberByName) exports.selectAllNumbers = selectAllNumbers exports.selectNumberById = selectNumberById diff --git a/src-electron/db/query-struct.js b/src-electron/db/query-struct.js index 3ab18686eb..6bb12b7c90 100644 --- a/src-electron/db/query-struct.js +++ b/src-electron/db/query-struct.js @@ -21,6 +21,7 @@ * @module DB API: zcl database access */ const dbApi = require('./db-api') +const dbCache = require('./db-cache') const dbMapping = require('./db-mapping') async function selectAllStructs(db, packageId) { @@ -142,6 +143,6 @@ WHERE exports.selectStructById = selectStructById exports.selectAllStructs = selectAllStructs -exports.selectStructByName = selectStructByName +exports.selectStructByName = dbCache.cacheQuery(selectStructByName) exports.selectStructsWithClusterAssociation = selectStructsWithClusterAssociation diff --git a/src-electron/main-process/startup.js b/src-electron/main-process/startup.js index d5bba7ea46..fac5b0d917 100644 --- a/src-electron/main-process/startup.js +++ b/src-electron/main-process/startup.js @@ -22,6 +22,7 @@ const _ = require('lodash') const YAML = require('yaml') const dbApi = require('../db/db-api.js') +const dbCache = require('../db/db-cache.js') const dbEnum = require('../../src-shared/db-enum.js') const env = require('../util/env') const zclLoader = require('../zcl/zcl-loader.js') @@ -631,6 +632,8 @@ async function startGeneration(argv, options) { ) ) + await dbApi.closeDatabase(mainDb) + if (options.quitFunction != null) options.quitFunction() } /** @@ -744,6 +747,11 @@ async function startUpMainInstance(argv, callbacks) { env.logInitLogFile() } + if (argv.disableDbCaching) { + console.log('⛔ Dabatase caching is disabled.') + dbCache.disable() + } + // For now delete the DB file. There is some weird constraint we run into. if (argv.clearDb != null) { clearDatabaseFile(env.sqliteFile()) diff --git a/src-electron/rest/admin.js b/src-electron/rest/admin.js index 8a41069ceb..edba7d52cf 100644 --- a/src-electron/rest/admin.js +++ b/src-electron/rest/admin.js @@ -16,6 +16,7 @@ */ const dbApi = require('../db/db-api.js') +const dbCache = require('../db/db-cache.js') const restApi = require('../../src-shared/rest-api.js') const env = require('../util/env') const { StatusCodes } = require('http-status-codes') @@ -84,6 +85,27 @@ function httpGetVersion(db) { } } +/** + * API: /cache + * Response JSON: + *
+ * 	 {
+ *      keys: 0,    // global key count
+ *      hits: 0,    // global hit count
+ *      misses: 0,  // global miss count
+ *      ksize: 0,   // global key size count in approximately bytes
+ *      vsize: 0    // global value size count in approximately bytes
+ * 	 }
+ * 
+ * + * @param {*} db + */ +function httpGetCache(db) { + return (request, response) => { + response.status(StatusCodes.OK).json(dbCache.cacheStats()) + } +} + exports.post = [ { uri: restApi.uri.sql, @@ -96,4 +118,8 @@ exports.get = [ uri: restApi.uri.version, callback: httpGetVersion, }, + { + uri: restApi.uri.cache, + callback: httpGetCache, + }, ] diff --git a/src-electron/rest/file-ops.js b/src-electron/rest/file-ops.js index edf0614a5e..f34ebb8bf4 100644 --- a/src-electron/rest/file-ops.js +++ b/src-electron/rest/file-ops.js @@ -65,7 +65,7 @@ function httpPostFileOpen(db) { let response = { sessionId: importResult.sessionId, - sessionKey: req.session.id, + sessionKey: req.zapSessionId, } env.logInfo( `Loaded project(${name}) into database. RESP: ${JSON.stringify( diff --git a/src-electron/util/args.ts b/src-electron/util/args.ts index eba7c5e6a1..6a3dbf4938 100644 --- a/src-electron/util/args.ts +++ b/src-electron/util/args.ts @@ -233,6 +233,10 @@ export function processCommandLineArguments(argv: string[]) { desc: 'Specifying the output YAML file to capture convert results.', type: 'string', }) + .option('disableDbCaching', { + desc: 'Disable query caching when accessing database', + type: 'boolean', + }) .usage('Usage: $0 [options] ... [file.zap] ...') .version( `Version: ${zapVersion.version}\nFeature level: ${ diff --git a/src-script/gen-test-runner.js b/src-script/gen-test-runner.js new file mode 100755 index 0000000000..c7efb3ea0b --- /dev/null +++ b/src-script/gen-test-runner.js @@ -0,0 +1,162 @@ +#!/usr/bin/env node +/** + * + * Copyright (c) 2020 Silicon Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This test runner profiles the ZAP backend generation speed by loading / calling generate through its RESTful APIs. + * + * Usage: + * $ ./gen-test-runner.js --port $ZAP_SERVER_PORT --apps $ZAP_SAMPLE_APP --out $GENERATED_DIR [--noWarmup] [--run $NUMBER] + */ + +const yargs = require('yargs/yargs') +const { hideBin } = require('yargs/helpers') +const argv = yargs(hideBin(process.argv)).argv +const axios = require('axios').default +const { CookieJar } = require('tough-cookie') +const { wrapper } = require('axios-cookiejar-support') +const restApi = require('../src-shared/rest-api') +const fs = require('fs') +const path = require('path') +const find = require('find') +let port = argv.port +const baseURL = `http://localhost:${port}` +const OUTPUT_DIR = argv.out + +let sessionId = '' +let TEST_RUN_COUNT = argv.run +let sampleApps = [] + +if (argv.run == undefined) { + TEST_RUN_COUNT = 20 +} + +function getSampleAppNamefromPath(sampleAppPath) { + const layers = path.dirname(sampleAppPath).split('/') + return layers[layers.length - 3] +} + +async function main() { + // grab sample apps from CLI + if (fs.existsSync(argv.apps) && fs.lstatSync(argv.apps).isDirectory()) { + let sampleAppsPaths = find.fileSync(/\.zap$/, argv.apps) + let sampleAppsNames = sampleAppsPaths.map((x) => + getSampleAppNamefromPath(x) + ) + sampleAppsNames.forEach((name, index) => { + sampleApps.push({ name, path: sampleAppsPaths[index] }) + }) + } else { + sampleApps.push({ + name: getSampleAppNamefromPath(argv.apps), + path: argv.apps, + }) + } + console.log(`ZAP port: ${baseURL}`) + console.log(`Output: ${OUTPUT_DIR}`) + + // warm up cache / db / etc + if (argv.noWarmup == undefined) { + await runTest('Cache warmup: ', sampleApps[0].path) + } + + let totalGenerationTimeSec = 0 + + for (let index = 0; index < sampleApps.length; index++) { + let { name, path } = sampleApps[index] + console.log(`App: ${name}`) + + for (let i = 1; i <= TEST_RUN_COUNT; i++) { + const LOG_PREFIX = `Run [${i}/${TEST_RUN_COUNT}]: ` + let res = await runTest(LOG_PREFIX, path) + totalGenerationTimeSec += res.elapsedSec + } + } + + if (TEST_RUN_COUNT) + console.log( + `Average generation time: ${totalGenerationTimeSec / TEST_RUN_COUNT}s` + ) +} + +async function runTest(LOG_PREFIX, zapFilePath) { + console.log(LOG_PREFIX + `Generation started.`) + let start = Date.now() + await generate(LOG_PREFIX, zapFilePath) + let finish = Date.now() + let elapsedSec = (finish - start) / 1000 + console.log(LOG_PREFIX + `Generation time: ${elapsedSec}s`) + return { elapsedSec } +} + +async function generate(LOG_PREFIX, zapFilePath) { + // share cookie / session across HTTP requests + const jar = new CookieJar() + axios.defaults.withCredentials = true + const client = wrapper(axios.create({ baseURL, jar })) + wrapper(axios) + + // load page + await client.get('') + + await client + .post(restApi.ide.open, { + zapFilePath, + }) + .then(function (response) { + console.log(LOG_PREFIX + `SessionId: ${response.data?.sessionId}`) + sessionId = response.data?.sessionId + }) + .catch(function (error) { + console.log(error) + }) + + // await client + // .put(restApi.uri.generate, { + // generationDirectory: '/Users/jiteng/Downloads/zap_generation', + // params: { + // 'sessionId': sessionId + // } + // }) + // .then(function (response) { + // console.log(`File generated to /Users/jiteng/Downloads/zap_generation`) + // }) + // .catch(function (error) { + // console.log(error) + // }) + + await axios({ + method: 'put', + baseURL, + url: restApi.uri.generate, + jar, + params: { + sessionId: sessionId, + }, + data: { + generationDirectory: OUTPUT_DIR, + }, + }) + .then(function (response) { + console.log(LOG_PREFIX + `Generation finished.`) + }) + .catch(function (error) { + console.log(LOG_PREFIX + `Generation failed!`) + }) +} + +main() diff --git a/src-shared/rest-api.js b/src-shared/rest-api.js index be50258c6c..5ff9632b77 100644 --- a/src-shared/rest-api.js +++ b/src-shared/rest-api.js @@ -39,6 +39,7 @@ const uri = { previewNameIndex: '/preview/:name/:index', sql: '/sql', version: '/version', + cache: '/cache', packages: `/packages`, getAllPackages: `/allpackages`, addNewPackage: `/packages/add`, diff --git a/tsconfig.json b/tsconfig.json index 5bfadf91c6..c2e66c2100 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,7 @@ "outDir": "./dist", "noImplicitAny": true, "module": "commonjs", - "sourceMap": true, + "sourceMap": false, "target": "es2019", "allowJs": true, "moduleResolution": "node",