diff --git a/package-lock.json b/package-lock.json index 37f10037..488743ae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -98,7 +98,8 @@ "style-loader": "^3.3.1", "svg-url-loader": "^8.0.0", "ts-loader": "^9.4.1", - "typescript": "~4.8.4" + "typescript": "~4.8.4", + "worker-loader": "^3.0.8" } }, "node_modules/@ampproject/remapping": { @@ -24119,6 +24120,44 @@ "workbox-core": "6.5.4" } }, + "node_modules/worker-loader": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/worker-loader/-/worker-loader-3.0.8.tgz", + "integrity": "sha512-XQyQkIFeRVC7f7uRhFdNMe/iJOdO6zxAaR3EWbDp45v3mDhrTi+++oswKNxShUNjPC/1xUp5DB29YKLhFo129g==", + "dev": true, + "dependencies": { + "loader-utils": "^2.0.0", + "schema-utils": "^3.0.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^4.0.0 || ^5.0.0" + } + }, + "node_modules/worker-loader/node_modules/schema-utils": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.1.2.tgz", + "integrity": "sha512-pvjEHOgWc9OWA/f/DE3ohBWTD6EleVLf7iFUkoSwAxttdBhB9QUebQgxER2kWueOvRJXPHNnyrvvh9eZINB8Eg==", + "dev": true, + "dependencies": { + "@types/json-schema": "^7.0.8", + "ajv": "^6.12.5", + "ajv-keywords": "^3.5.2" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", diff --git a/package.json b/package.json index 30e28e6a..9772b3d0 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,8 @@ "style-loader": "^3.3.1", "svg-url-loader": "^8.0.0", "ts-loader": "^9.4.1", - "typescript": "~4.8.4" + "typescript": "~4.8.4", + "worker-loader": "^3.0.8" }, "dependencies": { "@electron/remote": "^2.0.8", diff --git a/src/components/FileTree/FileTree.tsx b/src/components/FileTree/FileTree.tsx index f64372b2..1bfa4afc 100644 --- a/src/components/FileTree/FileTree.tsx +++ b/src/components/FileTree/FileTree.tsx @@ -1,13 +1,12 @@ import RcTree from "rc-tree"; import { DataNode, Key } from "rc-tree/lib/interface"; import React, { useEffect, useState } from "react"; -import { Element, scroller } from "react-scroll"; +import { Element } from "react-scroll"; import EllipticLoader from "../EllipticLoader"; import { PathType, useWorkbenchDB } from "../../contexts/dbContext"; import SwitcherIcon from "./SwitcherIcon"; -import { scrollToDomElement } from "../../utils/dom"; import "./FileTree.css"; @@ -56,19 +55,15 @@ const FileTree = (props: React.HTMLProps) => { useEffect(() => { if (!initialized || !db || !importedSqliteFilePath) return; - db.sync.then(() => { - db.findAllJSTree().then((treeData) => { + db.findAllJSTree() + .then((treeData) => { console.log("Filetree data", treeData); // Wrap with react-scroll wrapper function wrapNode(node: DataNode) { const key = String(node.key); node.title = ( - + {String(node.title)} {/* {String(node.title)} */} @@ -77,8 +72,7 @@ const FileTree = (props: React.HTMLProps) => { } treeData.forEach(wrapNode); setTreeData(treeData as unknown as DataNode[]); - }); - }); + }) }, [importedSqliteFilePath]); function selectPath(path: string, pathType: PathType) { diff --git a/src/components/Layout/Layout.tsx b/src/components/Layout/Layout.tsx index d60650e7..d142d947 100644 --- a/src/components/Layout/Layout.tsx +++ b/src/components/Layout/Layout.tsx @@ -1,31 +1,43 @@ -import React from 'react' -import { useLocation } from 'react-router-dom'; +import React, { useEffect } from "react"; +import { useLocation } from "react-router-dom"; // eslint-disable-next-line import/namespace -import { Allotment } from 'allotment'; +import { Allotment } from "allotment"; -import Navbar from '../Navbar/Navbar'; -import FileTree from '../FileTree/FileTree' -import ImportFallback from '../ImportFallback/ImportFallback'; +import Navbar from "../Navbar/Navbar"; +import FileTree from "../FileTree/FileTree"; +import ImportFallback from "../ImportFallback/ImportFallback"; -import { useWorkbenchDB } from '../../contexts/dbContext'; -import { FILE_TREE_ROUTES, IMPORT_FALLBACK_ROUTES } from '../../constants/routes'; -import ProgressLoader from '../ProgressLoader/ProgressLoader'; +import { useWorkbenchDB } from "../../contexts/dbContext"; +import { + FILE_TREE_ROUTES, + IMPORT_FALLBACK_ROUTES, +} from "../../constants/routes"; +import ProgressLoader from "../ProgressLoader/ProgressLoader"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { faGear } from "@fortawesome/free-solid-svg-icons"; import "allotment/dist/style.css"; -import './layout.css'; +import "./layout.css"; const Layout = (props: React.PropsWithChildren) => { const { pathname } = useLocation(); - const { initialized, loadingStatus } = useWorkbenchDB(); - - const isImportFallbackRoute = IMPORT_FALLBACK_ROUTES.find(route => pathname.includes(route)) !== undefined; - const showFileTree = FILE_TREE_ROUTES.find(route => pathname.includes(route)) !== undefined; - + const { initialized, loadingStatus, processingQuery } = useWorkbenchDB(); + + const isImportFallbackRoute = + IMPORT_FALLBACK_ROUTES.find((route) => pathname.includes(route)) !== + undefined; + const showFileTree = + FILE_TREE_ROUTES.find((route) => pathname.includes(route)) !== undefined; + + // useEffect(() => { + // console.log("Loader status", processingQuery ? "Showing" : "Hiding"); + // }, [processingQuery]); + return ( -
+
- + { > - -
- { - isImportFallbackRoute && !initialized ? ( - loadingStatus !== null ? - - : - - ) : props.children - } + +
+ {isImportFallbackRoute && !initialized ? ( + loadingStatus !== null ? ( + + ) : ( + + ) + ) : ( + props.children + )} + {processingQuery && ( +
+ Processing +
+ )}
- ) -} + ); +}; -export default Layout \ No newline at end of file +export default Layout; diff --git a/src/components/Layout/layout.css b/src/components/Layout/layout.css index 19c4e718..472e64df 100644 --- a/src/components/Layout/layout.css +++ b/src/components/Layout/layout.css @@ -8,12 +8,37 @@ box-shadow: rgb(177 177 177) 1px 0px 18px -3px; overflow: overlay !important; } + .content-pane { overflow-y: overlay !important; } + .content-container { padding: 10px; padding-top: 5px; padding-bottom: 5px; min-height: 100%; +} + +.query-processing-indicator { + position: fixed; + background: #f3f9ff; + padding: 10px; + right: 20px; + z-index: 999; + border-radius: 8px; + -webkit-box-shadow: 0px 3px 15px -2px rgba(0, 0, 0, 0.67); + box-shadow: 0px 3px 15px -2px rgba(0, 0, 0, 0.67); + + /* Animate */ + bottom: -100%; + opacity: 0; + animation: smooth-appear 0.5s ease forwards; +} + +@keyframes smooth-appear { + to { + bottom: 20px; + opacity: 1; + } } \ No newline at end of file diff --git a/src/contexts/dbContext.tsx b/src/contexts/dbContext.tsx index 646db164..5884b1fd 100644 --- a/src/contexts/dbContext.tsx +++ b/src/contexts/dbContext.tsx @@ -36,9 +36,12 @@ interface BasicValueState { interface WorkbenchContextProperties extends BasicValueState { currentPath: string | null; currentPathType: PathType; + loadingStatus: null | number; + processingQuery: boolean; startImport: () => void; abortImport: () => void; - loadingStatus: null | number; + startProcessing: () => void; + endProcessing: () => void; sqliteParser: (sqliteFilePath: string, preventNavigation?: boolean) => void; jsonParser: ( jsonFilePath: string, @@ -55,16 +58,19 @@ interface WorkbenchContextProperties extends BasicValueState { export const defaultWorkbenchContextValue: WorkbenchContextProperties = { db: null, initialized: false, - importedSqliteFilePath: null, - loadingStatus: null, currentPath: null, currentPathType: "directory", + importedSqliteFilePath: null, + loadingStatus: null, + processingQuery: false, jsonParser: () => null, sqliteParser: () => null, importJsonFile: () => null, updateLoadingStatus: () => null, startImport: () => null, abortImport: () => null, + startProcessing: () => null, + endProcessing: () => null, updateCurrentPath: () => null, goToFileInTableView: () => null, updateWorkbenchDB: () => null, @@ -80,6 +86,7 @@ export const WorkbenchDBProvider = ( const navigate = useNavigate(); const [loadingStatus, updateLoadingStatus] = useState(null); + const [processingQuery, setProcessingQuery] = useState(null); const [value, setValue] = useState({ db: null, initialized: false, @@ -92,13 +99,13 @@ export const WorkbenchDBProvider = ( setCurrentPath(path); setCurrentPathType(pathType); } - - function changeRouteOnImport(){ + + function changeRouteOnImport() { navigate(DEFAULT_ROUTE_ON_IMPORT); } - function goToFileInTableView(path: string){ - updateCurrentPath(path, 'file'); + function goToFileInTableView(path: string) { + updateCurrentPath(path, "file"); navigate("/" + ROUTES.TABLE_VIEW); } @@ -456,15 +463,18 @@ export const WorkbenchDBProvider = ( setProcessingQuery(true), + endProcessing: () => setProcessingQuery(false), updateCurrentPath, goToFileInTableView, updateWorkbenchDB, @@ -475,4 +485,4 @@ export const WorkbenchDBProvider = ( ); }; -export const useWorkbenchDB = () => useContext(WorkbenchContext); \ No newline at end of file +export const useWorkbenchDB = () => useContext(WorkbenchContext); diff --git a/src/pages/About/About.tsx b/src/pages/About/About.tsx index 386cd2b8..e6d7d6ec 100644 --- a/src/pages/About/About.tsx +++ b/src/pages/About/About.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { version } from "../../../package.json"; +import packageInfo from "../../../package.json"; import "./about.css"; @@ -8,7 +8,7 @@ const About = () => {

About ScanCode Workbench - v{version} + v{packageInfo.version}


Overview

diff --git a/src/pages/FileInfoDash/FileInfoDash.tsx b/src/pages/FileInfoDash/FileInfoDash.tsx index 19c333c1..037d3039 100644 --- a/src/pages/FileInfoDash/FileInfoDash.tsx +++ b/src/pages/FileInfoDash/FileInfoDash.tsx @@ -17,6 +17,8 @@ import "./FileInfoDash.css"; const FileInfoDash = () => { const workbenchDB = useWorkbenchDB(); + const { db, initialized, currentPath, startProcessing, endProcessing } = + workbenchDB; const [progLangsData, setProgLangsData] = useState(null); const [mimeTypesData, setMimeTypesData] = useState(null); @@ -27,10 +29,10 @@ const FileInfoDash = () => { }); useEffect(() => { - const { db, initialized, currentPath } = workbenchDB; - if (!initialized || !db || !currentPath) return; + startProcessing(); + db.sync .then((db) => db.File.findOne({ where: { path: currentPath } })) .then((root) => { @@ -88,7 +90,8 @@ const FileInfoDash = () => { const { chartData: mimeTypesChartData } = formatChartData(fileMimeTypes); setMimeTypesData(mimeTypesChartData); - }); + }) + .then(endProcessing); }, [workbenchDB]); return ( diff --git a/src/pages/Home/Home.tsx b/src/pages/Home/Home.tsx index d311fc78..3909ae5a 100644 --- a/src/pages/Home/Home.tsx +++ b/src/pages/Home/Home.tsx @@ -2,7 +2,6 @@ import moment from "moment"; import electron from "electron"; import * as electronFs from "fs"; import { toast } from "react-toastify"; -import { useNavigate } from "react-router-dom"; import React, { useMemo, useState } from "react"; import { diff --git a/src/pages/LicenseInfoDash/LicenseInfoDash.tsx b/src/pages/LicenseInfoDash/LicenseInfoDash.tsx index 0fdc0da2..a86e4f6e 100644 --- a/src/pages/LicenseInfoDash/LicenseInfoDash.tsx +++ b/src/pages/LicenseInfoDash/LicenseInfoDash.tsx @@ -19,6 +19,8 @@ import "./licenseInfoDash.css"; const LicenseInfoDash = () => { const workbenchDB = useWorkbenchDB(); + const { db, initialized, currentPath, startProcessing, endProcessing } = + workbenchDB; const [licenseExpressionData, setLicenseExpressionData] = useState(null); const [licenseKeyData, setLicenseKeyData] = useState(null); @@ -30,10 +32,10 @@ const LicenseInfoDash = () => { }); useEffect(() => { - const { db, initialized, currentPath } = workbenchDB; - if (!initialized || !db || !currentPath) return; + startProcessing(); + db.sync .then((db) => db.FlatFile.findAll({ @@ -48,6 +50,8 @@ const LicenseInfoDash = () => { attributes: ["fileId", "license_detections"], }) ) + // @REMOVE_THIS + // .then((flatFiles) => new Promise(resolve => setTimeout(()=>resolve(flatFiles), 4000))) .then((flatFiles) => { const fileIDs = flatFiles.map((flatFile) => flatFile.getDataValue("fileId") @@ -66,7 +70,7 @@ const LicenseInfoDash = () => { })); // Query and prepare chart for license expression - db.sync + const ExpressionPromise = db.sync .then((db) => db.LicenseExpression.findAll({ where: { fileId: fileIDs } }) ) @@ -110,7 +114,7 @@ const LicenseInfoDash = () => { }); // Query and prepare chart for license policy - db.sync + const PolicyPromise = db.sync .then((db) => db.LicensePolicy.findAll({ where: { fileId: fileIDs } }) ) @@ -124,8 +128,11 @@ const LicenseInfoDash = () => { const { chartData } = formatChartData(labels); setLicensePolicyData(chartData); }); - }); - }, [workbenchDB]); + + return Promise.all([ExpressionPromise, PolicyPromise]); + }) + .then(endProcessing); + }, [currentPath]); return (
diff --git a/src/pages/Licenses/Licenses.tsx b/src/pages/Licenses/Licenses.tsx index 0f597667..ee061289 100644 --- a/src/pages/Licenses/Licenses.tsx +++ b/src/pages/Licenses/Licenses.tsx @@ -42,10 +42,12 @@ const LicenseDetections = () => { const [licenseClues, setLicenseClues] = useState( null ); - const { db } = useWorkbenchDB(); + const { db, startProcessing, endProcessing } = useWorkbenchDB(); useEffect(() => { - db.sync.then(async () => { + startProcessing(); + + (async () => { const newLicenseDetections: LicenseDetectionDetails[] = ( await db.getAllLicenseDetections() ).map((detection) => ({ @@ -104,12 +106,6 @@ const LicenseDetections = () => { const queriedClueFileIdx = Number( searchParams.get(QUERY_KEYS.LICENSE_CLUE_FILE_CLUE_IDX) || 0 ); - // console.log("All license queries", { - // queriedDetectionIdentifier, - // queriedClueExpression, - // queriedClueFilePath, - // queriedClueFileIdx, - // }); if (queriedDetectionIdentifier) { const queriedDetection: LicenseDetectionDetails | null = @@ -157,7 +153,7 @@ const LicenseDetections = () => { activateLicenseClue(newLicenseClues[0]); } } - }); + })().then(endProcessing); }, []); if (!licenseDetections || !licenseClues) { diff --git a/src/pages/PackageInfoDash/PackageInfoDash.tsx b/src/pages/PackageInfoDash/PackageInfoDash.tsx index f7a340c9..4a1dcc01 100644 --- a/src/pages/PackageInfoDash/PackageInfoDash.tsx +++ b/src/pages/PackageInfoDash/PackageInfoDash.tsx @@ -15,6 +15,8 @@ import { NO_VALUE_DETECTED_LABEL } from "../../constants/data"; const PackageInfoDash = () => { const workbenchDB = useWorkbenchDB(); + const { db, initialized, currentPath, startProcessing, endProcessing } = + workbenchDB; const [packageTypeData, setPackageTypeData] = useState(null); const [packageLangData, setPackageLangData] = useState(null); const [packageLicenseData, setPackageLicenseData] = useState(null); @@ -23,10 +25,10 @@ const PackageInfoDash = () => { }); useEffect(() => { - const { db, initialized, currentPath } = workbenchDB; - if (!initialized || !db || !currentPath) return; + startProcessing(); + const where: WhereOptions = { path: { [Op.or]: [ @@ -43,18 +45,18 @@ const PackageInfoDash = () => { attributes: ["id"], }) ) + // @REMOVE_THIS + // .then((flatFiles) => new Promise(resolve => setTimeout(()=>resolve(flatFiles), 2000))) .then((files) => { const fileIDs = files.map((file) => file.getDataValue("id")); // Query and prepare chart for package types - db.sync + const PackageDataPromise = db.sync .then((db) => db.PackageData.findAll({ where: { fileId: fileIDs } })) .then((packageData) => { // Prepare count of packages setScanData({ totalPackages: packageData.length }); - return packageData; - }) - .then((packageData) => { + // Prepare chart for package types const packageTypes = packageData.map( (packageEntry) => @@ -85,8 +87,10 @@ const PackageInfoDash = () => { setPackageLicenseData(packageLicenseExpChartData); }); - }); - }, [workbenchDB]); + return [PackageDataPromise]; + }) + .then(endProcessing); + }, [currentPath]); return (
diff --git a/src/pages/Packages/Packages.tsx b/src/pages/Packages/Packages.tsx index 7684dc2a..09905f47 100644 --- a/src/pages/Packages/Packages.tsx +++ b/src/pages/Packages/Packages.tsx @@ -23,6 +23,8 @@ import "./packages.css"; const Packages = () => { const workbenchDB = useWorkbenchDB(); + const { db, initialized, currentPath, startProcessing, endProcessing } = + workbenchDB; const [searchParams] = useSearchParams(); const [expandedPackages, setExpandedPackages] = useState([]); const [packagesWithDeps, setPackagesWithDeps] = useState< @@ -74,244 +76,253 @@ const Packages = () => { }; useEffect(() => { - const { db, initialized, currentPath } = workbenchDB; - if (!initialized || !db || !currentPath) return; - db.sync.then(async () => { - const packages = await db.getAllPackages(); - const deps = await db.getAllDependencies(); - console.log("Raw Packages & deps", packages, deps); - if (!packages.length && !deps.length) { - console.log("No package or deps available"); - setPackageGroups([]); - return; - } + startProcessing(); + db.sync + .then(async () => { + const packages = await db.getAllPackages(); + const deps = await db.getAllDependencies(); + console.log("Raw Packages & deps", packages, deps); + if (!packages.length && !deps.length) { + console.log("No package or deps available"); + setPackageGroups([]); + return; + } - // const type_other = 'type-other'; - const packageMapping = new Map( - packages.map((packageInfo): [string, PackageDetails] => [ - packageInfo.getDataValue("package_uid").toString({}), - { - package_uid: packageInfo.getDataValue("package_uid").toString({}), - name: packageInfo.getDataValue("name").toString({}), - type: packageInfo.getDataValue("type").toString({}), - dependencies: [], - namespace: - packageInfo.getDataValue("namespace")?.toString({}) || null, - version: packageInfo.getDataValue("version")?.toString({}) || null, - qualifiers: JSON.parse( - packageInfo.getDataValue("qualifiers").toString({}) - ), - subpath: packageInfo.getDataValue("subpath")?.toString({}) || null, - primary_language: - packageInfo.getDataValue("primary_language")?.toString({}) || - null, - description: - packageInfo.getDataValue("description")?.toString({}) || null, - release_date: - packageInfo.getDataValue("release_date")?.toString({}) || null, - parties: JSON.parse( - packageInfo.getDataValue("parties").toString({}) - ), - keywords: JSON.parse( - packageInfo.getDataValue("keywords").toString({}) + // const type_other = 'type-other'; + const packageMapping = new Map( + packages.map((packageInfo): [string, PackageDetails] => [ + packageInfo.getDataValue("package_uid").toString({}), + { + package_uid: packageInfo.getDataValue("package_uid").toString({}), + name: packageInfo.getDataValue("name").toString({}), + type: packageInfo.getDataValue("type").toString({}), + dependencies: [], + namespace: + packageInfo.getDataValue("namespace")?.toString({}) || null, + version: + packageInfo.getDataValue("version")?.toString({}) || null, + qualifiers: JSON.parse( + packageInfo.getDataValue("qualifiers").toString({}) + ), + subpath: + packageInfo.getDataValue("subpath")?.toString({}) || null, + primary_language: + packageInfo.getDataValue("primary_language")?.toString({}) || + null, + description: + packageInfo.getDataValue("description")?.toString({}) || null, + release_date: + packageInfo.getDataValue("release_date")?.toString({}) || null, + parties: JSON.parse( + packageInfo.getDataValue("parties").toString({}) + ), + keywords: JSON.parse( + packageInfo.getDataValue("keywords").toString({}) + ), + homepage_url: + packageInfo.getDataValue("homepage_url")?.toString({}) || null, + download_url: + packageInfo.getDataValue("download_url")?.toString({}) || null, + size: packageInfo.getDataValue("size")?.toString({}) || null, + sha1: packageInfo.getDataValue("sha1")?.toString({}) || null, + md5: packageInfo.getDataValue("md5")?.toString({}) || null, + sha256: packageInfo.getDataValue("sha256")?.toString({}) || null, + sha512: packageInfo.getDataValue("sha512")?.toString({}) || null, + bug_tracking_url: + packageInfo.getDataValue("bug_tracking_url")?.toString({}) || + null, + code_view_url: + packageInfo.getDataValue("code_view_url")?.toString({}) || null, + vcs_url: + packageInfo.getDataValue("vcs_url")?.toString({}) || null, + copyright: + packageInfo.getDataValue("copyright")?.toString({}) || null, + declared_license_expression: + packageInfo + .getDataValue("declared_license_expression") + ?.toString({}) || null, + declared_license_expression_spdx: + packageInfo + .getDataValue("declared_license_expression_spdx") + ?.toString({}) || null, + other_license_expression: + packageInfo + .getDataValue("other_license_expression") + ?.toString({}) || null, + other_license_expression_spdx: + packageInfo + .getDataValue("other_license_expression_spdx") + ?.toString({}) || null, + extracted_license_statement: + packageInfo + .getDataValue("extracted_license_statement") + ?.toString({}) + .replace(/(^"|"$)/g, "") || null, + notice_text: + packageInfo.getDataValue("notice_text")?.toString({}) || null, + source_packages: JSON.parse( + packageInfo.getDataValue("source_packages").toString({}) + ), + extra_data: JSON.parse( + packageInfo.getDataValue("extra_data").toString({}) + ), + repository_homepage_url: + packageInfo + .getDataValue("repository_homepage_url") + ?.toString({}) || null, + repository_download_url: + packageInfo + .getDataValue("repository_download_url") + ?.toString({}) || null, + api_data_url: + packageInfo.getDataValue("api_data_url")?.toString({}) || null, + datafile_paths: + JSON.parse( + packageInfo.getDataValue("datafile_paths")?.toString({}) || + "[]" + ) || [], + datasource_ids: + JSON.parse( + packageInfo.getDataValue("datasource_ids")?.toString({}) || + "[]" + ) || [], + purl: packageInfo.getDataValue("purl").toString({}), + }, + ]) + ); + const MISC_DEPS = "others"; + const MISC_PACKAGE: PackageDetails = { + package_uid: "misc", + name: "Misc dependencies", + type: "Other", + dependencies: [], + namespace: "", + version: null, + qualifiers: {}, + subpath: null, + primary_language: null, + description: null, + release_date: null, + parties: {}, + keywords: {}, + homepage_url: null, + download_url: null, + size: null, + sha1: null, + md5: null, + sha256: null, + sha512: null, + bug_tracking_url: null, + code_view_url: null, + vcs_url: null, + copyright: null, + declared_license_expression: null, + declared_license_expression_spdx: null, + other_license_expression: null, + other_license_expression_spdx: null, + extracted_license_statement: null, + notice_text: null, + source_packages: {}, + extra_data: {}, + repository_homepage_url: null, + repository_download_url: null, + api_data_url: null, + datafile_paths: [], + datasource_ids: [], + purl: null, + }; + packageMapping.set(MISC_DEPS, MISC_PACKAGE); + + // Group dependencies in their respective packages + deps.forEach((dependencyInfo) => { + const targetPackageUid: string | null = dependencyInfo + .getDataValue("for_package_uid") + ?.toString({}); + packageMapping.get(targetPackageUid || MISC_DEPS).dependencies.push({ + // ...dependencyInfo, // For debugging + purl: dependencyInfo.getDataValue("purl").toString({}), + extracted_requirement: + dependencyInfo + .getDataValue("extracted_requirement") + ?.toString({}) || "", + scope: dependencyInfo + .getDataValue("scope") + .toString({}) as DEPENDENCY_SCOPES, + is_runtime: dependencyInfo.getDataValue("is_runtime"), + is_optional: dependencyInfo.getDataValue("is_optional"), + is_resolved: dependencyInfo.getDataValue("is_resolved"), + resolved_package: JSON.parse( + dependencyInfo.getDataValue("resolved_package").toString({}) ), - homepage_url: - packageInfo.getDataValue("homepage_url")?.toString({}) || null, - download_url: - packageInfo.getDataValue("download_url")?.toString({}) || null, - size: packageInfo.getDataValue("size")?.toString({}) || null, - sha1: packageInfo.getDataValue("sha1")?.toString({}) || null, - md5: packageInfo.getDataValue("md5")?.toString({}) || null, - sha256: packageInfo.getDataValue("sha256")?.toString({}) || null, - sha512: packageInfo.getDataValue("sha512")?.toString({}) || null, - bug_tracking_url: - packageInfo.getDataValue("bug_tracking_url")?.toString({}) || + dependency_uid: dependencyInfo + .getDataValue("dependency_uid") + .toString({}), + for_package_uid: + dependencyInfo.getDataValue("for_package_uid")?.toString({}) || null, - code_view_url: - packageInfo.getDataValue("code_view_url")?.toString({}) || null, - vcs_url: packageInfo.getDataValue("vcs_url")?.toString({}) || null, - copyright: - packageInfo.getDataValue("copyright")?.toString({}) || null, - declared_license_expression: - packageInfo - .getDataValue("declared_license_expression") - ?.toString({}) || null, - declared_license_expression_spdx: - packageInfo - .getDataValue("declared_license_expression_spdx") - ?.toString({}) || null, - other_license_expression: - packageInfo - .getDataValue("other_license_expression") - ?.toString({}) || null, - other_license_expression_spdx: - packageInfo - .getDataValue("other_license_expression_spdx") - ?.toString({}) || null, - extracted_license_statement: - packageInfo - .getDataValue("extracted_license_statement") - ?.toString({}) - .replace(/(^"|"$)/g, "") || null, - notice_text: - packageInfo.getDataValue("notice_text")?.toString({}) || null, - source_packages: JSON.parse( - packageInfo.getDataValue("source_packages").toString({}) - ), - extra_data: JSON.parse( - packageInfo.getDataValue("extra_data").toString({}) - ), - repository_homepage_url: - packageInfo - .getDataValue("repository_homepage_url") - ?.toString({}) || null, - repository_download_url: - packageInfo - .getDataValue("repository_download_url") - ?.toString({}) || null, - api_data_url: - packageInfo.getDataValue("api_data_url")?.toString({}) || null, - datafile_paths: - JSON.parse( - packageInfo.getDataValue("datafile_paths")?.toString({}) || "[]" - ) || [], - datasource_ids: - JSON.parse( - packageInfo.getDataValue("datasource_ids")?.toString({}) || "[]" - ) || [], - purl: packageInfo.getDataValue("purl").toString({}), - }, - ]) - ); - const MISC_DEPS = "others"; - const MISC_PACKAGE: PackageDetails = { - package_uid: "misc", - name: "Misc dependencies", - type: "Other", - dependencies: [], - namespace: "", - version: null, - qualifiers: {}, - subpath: null, - primary_language: null, - description: null, - release_date: null, - parties: {}, - keywords: {}, - homepage_url: null, - download_url: null, - size: null, - sha1: null, - md5: null, - sha256: null, - sha512: null, - bug_tracking_url: null, - code_view_url: null, - vcs_url: null, - copyright: null, - declared_license_expression: null, - declared_license_expression_spdx: null, - other_license_expression: null, - other_license_expression_spdx: null, - extracted_license_statement: null, - notice_text: null, - source_packages: {}, - extra_data: {}, - repository_homepage_url: null, - repository_download_url: null, - api_data_url: null, - datafile_paths: [], - datasource_ids: [], - purl: null, - }; - packageMapping.set(MISC_DEPS, MISC_PACKAGE); - - // Group dependencies in their respective packages - deps.forEach((dependencyInfo) => { - const targetPackageUid: string | null = dependencyInfo - .getDataValue("for_package_uid") - ?.toString({}); - packageMapping.get(targetPackageUid || MISC_DEPS).dependencies.push({ - // ...dependencyInfo, // For debugging - purl: dependencyInfo.getDataValue("purl").toString({}), - extracted_requirement: - dependencyInfo - .getDataValue("extracted_requirement") - ?.toString({}) || "", - scope: dependencyInfo - .getDataValue("scope") - .toString({}) as DEPENDENCY_SCOPES, - is_runtime: dependencyInfo.getDataValue("is_runtime"), - is_optional: dependencyInfo.getDataValue("is_optional"), - is_resolved: dependencyInfo.getDataValue("is_resolved"), - resolved_package: JSON.parse( - dependencyInfo.getDataValue("resolved_package").toString({}) - ), - dependency_uid: dependencyInfo - .getDataValue("dependency_uid") - .toString({}), - for_package_uid: - dependencyInfo.getDataValue("for_package_uid")?.toString({}) || - null, - datafile_path: dependencyInfo - .getDataValue("datafile_path") - .toString({}), - datasource_id: dependencyInfo - .getDataValue("datasource_id") - .toString({}), + datafile_path: dependencyInfo + .getDataValue("datafile_path") + .toString({}), + datasource_id: dependencyInfo + .getDataValue("datasource_id") + .toString({}), + }); }); - }); - // Ignore misc package if no misc dependencies found - if (!MISC_PACKAGE.dependencies.length) { - packageMapping.delete(MISC_DEPS); - } + // Ignore misc package if no misc dependencies found + if (!MISC_PACKAGE.dependencies.length) { + packageMapping.delete(MISC_DEPS); + } - const parsedPackageWithDeps = Array.from(packageMapping.values()); - // @TODO - What are qualifiers ? - parsedPackageWithDeps.forEach((pkg) => { - if (Object.keys(pkg.qualifiers).length) console.log("Qualifying:", pkg); - }); - // console.log("Packages with deps:", parsedPackageWithDeps); - setPackagesWithDeps(parsedPackageWithDeps); + const parsedPackageWithDeps = Array.from(packageMapping.values()); + // @TODO - What are qualifiers ? + parsedPackageWithDeps.forEach((pkg) => { + if (Object.keys(pkg.qualifiers).length) + console.log("Qualifying:", pkg); + }); + // console.log("Packages with deps:", parsedPackageWithDeps); + setPackagesWithDeps(parsedPackageWithDeps); - // Group packages in their respective package type group - const packageGroupMapping = new Map(); - parsedPackageWithDeps.forEach((packageDetails) => { - if (!packageGroupMapping.has(packageDetails.type)) { - packageGroupMapping.set(packageDetails.type, []); - } - packageGroupMapping.get(packageDetails.type).push(packageDetails); - }); - const parsedPackageGroups = Array.from(packageGroupMapping.entries()).map( - ([type, packages]): PackageTypeGroupDetails => ({ - type, - packages, - }) - ); - setPackageGroups(parsedPackageGroups); - // console.log("Package groups", parsedPackageGroups); + // Group packages in their respective package type group + const packageGroupMapping = new Map(); + parsedPackageWithDeps.forEach((packageDetails) => { + if (!packageGroupMapping.has(packageDetails.type)) { + packageGroupMapping.set(packageDetails.type, []); + } + packageGroupMapping.get(packageDetails.type).push(packageDetails); + }); + const parsedPackageGroups = Array.from( + packageGroupMapping.entries() + ).map( + ([type, packages]): PackageTypeGroupDetails => ({ + type, + packages, + }) + ); + setPackageGroups(parsedPackageGroups); + // console.log("Package groups", parsedPackageGroups); - setExpandedPackages([]); + setExpandedPackages([]); - // Select package based on query or default - const queriedPackageUid = searchParams.get(QUERY_KEYS.PACKAGE); - const queriedPackage = parsedPackageWithDeps.find( - (packageInfo) => packageInfo.package_uid === queriedPackageUid - ); - if (queriedPackage) { - activatePackage(queriedPackage); - console.log( - `Activate queried package(${queriedPackageUid}): `, - queriedPackage + // Select package based on query or default + const queriedPackageUid = searchParams.get(QUERY_KEYS.PACKAGE); + const queriedPackage = parsedPackageWithDeps.find( + (packageInfo) => packageInfo.package_uid === queriedPackageUid ); - } else { - activatePackage(parsedPackageWithDeps[0]); - } - }); - }, [workbenchDB]); + if (queriedPackage) { + activatePackage(queriedPackage); + console.log( + `Activate queried package(${queriedPackageUid}): `, + queriedPackage + ); + } else { + activatePackage(parsedPackageWithDeps[0]); + } + }) + .then(endProcessing); + }, [currentPath]); function collapsePackage(target_package_uid: string, e?: React.MouseEvent) { setExpandedPackages((prevPackages) => diff --git a/src/pages/ScanInfo/ScanInfo.tsx b/src/pages/ScanInfo/ScanInfo.tsx index 45bd573f..32b599cc 100644 --- a/src/pages/ScanInfo/ScanInfo.tsx +++ b/src/pages/ScanInfo/ScanInfo.tsx @@ -42,55 +42,54 @@ function parseIfValidJson(str: unknown) { const ScanInfo = () => { const workbenchDB = useWorkbenchDB(); + const { db, initialized, currentPath, startProcessing, endProcessing } = workbenchDB; const [parsedScanInfo, setParsedScanInfo] = useState(null); useEffect(() => { - const { db, initialized, currentPath } = workbenchDB; - if (!initialized || !db || !currentPath) return; - db.sync.then(() => { - db.getScanInfo().then((rawInfo) => { - console.log("Raw scan info:", rawInfo); - const newParsedScanInfo: ScanInfo = { - tool_name: rawInfo.getDataValue("tool_name").toString({}) || "", - tool_version: rawInfo.getDataValue("tool_version").toString({}) || "", - notice: rawInfo.getDataValue("notice").toString({}) || "", - duration: Number(rawInfo.getDataValue("duration")), - options: - Object.entries( - parseIfValidJson(rawInfo.getDataValue("options")?.toString({})) || - [] - ) || [], - input: - parseIfValidJson(rawInfo.getDataValue("input")?.toString({})) || [], - files_count: Number(rawInfo.getDataValue("files_count")), - output_format_version: - rawInfo.getDataValue("output_format_version")?.toString({}) || "", - spdx_license_list_version: - rawInfo.getDataValue("spdx_license_list_version")?.toString({}) || - "", - operating_system: - rawInfo.getDataValue("operating_system")?.toString({}) || "", - cpu_architecture: - rawInfo.getDataValue("cpu_architecture")?.toString({}) || "", - platform: rawInfo.getDataValue("platform")?.toString({}) || "", - platform_version: - rawInfo.getDataValue("platform_version")?.toString({}) || "", - python_version: - rawInfo.getDataValue("python_version")?.toString({}) || "", - workbench_version: - rawInfo.getDataValue("workbench_version")?.toString({}) || "", - workbench_notice: - rawInfo.getDataValue("workbench_notice")?.toString({}) || "", - raw_header_content: - rawInfo.getDataValue("header_content")?.toString({}) || "", - }; - console.log("Parsed scan info:", newParsedScanInfo); - setParsedScanInfo(newParsedScanInfo); - }); - }); - }, [workbenchDB]); + startProcessing(); + db.getScanInfo().then((rawInfo) => { + console.log("Raw scan info:", rawInfo); + const newParsedScanInfo: ScanInfo = { + tool_name: rawInfo.getDataValue("tool_name").toString({}) || "", + tool_version: rawInfo.getDataValue("tool_version").toString({}) || "", + notice: rawInfo.getDataValue("notice").toString({}) || "", + duration: Number(rawInfo.getDataValue("duration")), + options: + Object.entries( + parseIfValidJson(rawInfo.getDataValue("options")?.toString({})) || + [] + ) || [], + input: + parseIfValidJson(rawInfo.getDataValue("input")?.toString({})) || [], + files_count: Number(rawInfo.getDataValue("files_count")), + output_format_version: + rawInfo.getDataValue("output_format_version")?.toString({}) || "", + spdx_license_list_version: + rawInfo.getDataValue("spdx_license_list_version")?.toString({}) || + "", + operating_system: + rawInfo.getDataValue("operating_system")?.toString({}) || "", + cpu_architecture: + rawInfo.getDataValue("cpu_architecture")?.toString({}) || "", + platform: rawInfo.getDataValue("platform")?.toString({}) || "", + platform_version: + rawInfo.getDataValue("platform_version")?.toString({}) || "", + python_version: + rawInfo.getDataValue("python_version")?.toString({}) || "", + workbench_version: + rawInfo.getDataValue("workbench_version")?.toString({}) || "", + workbench_notice: + rawInfo.getDataValue("workbench_notice")?.toString({}) || "", + raw_header_content: + rawInfo.getDataValue("header_content")?.toString({}) || "", + }; + console.log("Parsed scan info:", newParsedScanInfo); + setParsedScanInfo(newParsedScanInfo); + }) + .then(endProcessing); + }, [currentPath]); return (
diff --git a/src/pages/TableView/TableView.tsx b/src/pages/TableView/TableView.tsx index 674ab82e..d086ac9c 100644 --- a/src/pages/TableView/TableView.tsx +++ b/src/pages/TableView/TableView.tsx @@ -24,7 +24,8 @@ import { useWorkbenchState } from "../../contexts/stateContext"; import "./TableView.css"; const TableView = () => { - const { db, initialized, currentPath } = useWorkbenchDB(); + const { db, initialized, currentPath, startProcessing, endProcessing } = + useWorkbenchDB(); const { columnDefs, columnState, setColumnDefs, updateColState } = useWorkbenchState(); @@ -55,11 +56,13 @@ const TableView = () => { } } - // Update table data whenever new db is loaded or path is changed useEffect(() => { if (!initialized || !db || !currentPath) return; - db.sync + startProcessing(); + + // Update table data whenever new db is loaded or path is changed + const DataProcessorPromise = db.sync .then((db) => db.FlatFile.findAll({ where: { @@ -94,13 +97,9 @@ const TableView = () => { return [...COLUMN_GROUPS.DEFAULT]; }); }); - }, [currentPath]); - // Update set filters whenever new db is loaded - useEffect(() => { - if (!initialized || !db || !currentPath) return; - - db.sync.then((db) => { + // Update set filters whenever new db is loaded or path is changed + const FilterProcessorPromise = db.sync.then((db) => { const filterPromises: Promise[] = []; Object.values(ALL_COLUMNS).forEach((columnDef) => { @@ -148,7 +147,7 @@ const TableView = () => { filterPromises.push(filterPromise); }); - Promise.all(filterPromises).then(() => { + return Promise.all(filterPromises).then(() => { // console.log( // "Generated unique set filters:", // columnDefs.map(coldef => coldef.filterParams) @@ -157,9 +156,12 @@ const TableView = () => { if (prevColDefs.length) return [...prevColDefs]; return [...COLUMN_GROUPS.DEFAULT]; }); - // setDefaultColumnGroup(); }); }); + + Promise.all([DataProcessorPromise, FilterProcessorPromise]).then( + endProcessing + ); }, [db, currentPath]); useEffect(() => { diff --git a/src/services/workbenchDB.ts b/src/services/workbenchDB.ts index dfdcacdc..9b02d44e 100644 --- a/src/services/workbenchDB.ts +++ b/src/services/workbenchDB.ts @@ -108,6 +108,7 @@ export class WorkbenchDB { sequelize: Sequelize; db: DatabaseStructure; sync: Promise; + config: WorkbenchDbConfig; constructor(config: WorkbenchDbConfig) { // Constructor returns an object which effectively represents a connection @@ -116,6 +117,12 @@ export class WorkbenchDB { const user = config && config.dbUser ? config.dbUser : null; const password = config && config.dbPassword ? config.dbPassword : null; const storage = config && config.dbStorage ? config.dbStorage : ":memory:"; + this.config = { + dbName: name, + dbStorage: storage, + dbUser: user, + dbPassword: password, + } // console.log("Sequelize DB details", { // name, diff --git a/tsconfig.json b/tsconfig.json index f9557528..25317f45 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { "jsx": "react", "allowJs": true, - "module": "commonjs", + "module": "ES2020", "skipLibCheck": true, "esModuleInterop": true, "noImplicitAny": true, diff --git a/webpack.rules.js b/webpack.rules.js index b3f154ee..d8789fb5 100644 --- a/webpack.rules.js +++ b/webpack.rules.js @@ -4,36 +4,40 @@ module.exports = [ // We're specifying native_modules in the test because the asset relocator loader generates a // "fake" .node file which is really a cjs file. test: /native_modules\/.+\.node$/, - use: 'node-loader', + use: "node-loader", + }, + { + test: /\.worker\.js$/, + use: { loader: "worker-loader" }, }, { test: /\.(png|jpg|jpeg|eps|gif)$/i, use: [ { - loader: 'url-loader', + loader: "url-loader", options: { limit: 8192, - } + }, }, ], - type: 'javascript/auto' + type: "javascript/auto", }, { test: /\.(m?js|node)$/, parser: { amd: false }, use: { - loader: '@vercel/webpack-asset-relocator-loader', + loader: "@vercel/webpack-asset-relocator-loader", options: { - outputAssetBase: 'native_modules', + outputAssetBase: "native_modules", }, }, }, - + { test: /\.tsx?$/, exclude: /(node_modules|\.webpack)/, use: { - loader: 'ts-loader', + loader: "ts-loader", options: { transpileOnly: true, }, @@ -41,6 +45,6 @@ module.exports = [ }, { test: /\.svg$/, - use: ['@svgr/webpack'], + use: ["@svgr/webpack"], }, ];