refactor: decouple tree-walker & scanner by removing Dependency class

workspaces/scanner/package.json

@@ -15,6 +15,7 @@
     "prepublishOnly": "npm run build && pkg-ok",
     "test": "npm run test-only",
     "test-only": "glob -c \"tsx --test\" \"./test/**/*.spec.ts\"",
+    "test:walker": "glob -c \"tsx --test\" \"./test/depWalker.spec.ts\"",
     "coverage": "c8 -r html npm run test-only"
   },
   "files": [

workspaces/scanner/src/depWalker.ts

@@ -18,7 +18,46 @@ import {
 } from "./utils/index.js";
 import { packageMetadata, manifestMetadata } from "./npmRegistry.js";
 import { Logger, ScannerLoggerEvents } from "./class/logger.class.js";
-import type { DependencyVersion, Options, Payload } from "./types.js";
+import type {
+  Dependency,
+  DependencyVersion,
+  Options,
+  Payload
+} from "./types.js";
+
+// CONSTANTS
+const kDefaultDependencyVersionFields = {
+  description: "",
+  size: 0,
+  author: null,
+  engines: {},
+  scripts: {},
+  license: [],
+  composition: {
+    extensions: [],
+    files: [],
+    minified: [],
+    unused: [],
+    missing: [],
+    required_files: [],
+    required_nodejs: [],
+    required_thirdparty: [],
+    required_subpath: []
+  }
+};
+const kDefaultDependencyMetadata: Dependency["metadata"] = {
+  publishedCount: 0,
+  lastUpdateAt: new Date(),
+  lastVersion: "N/A",
+  hasChangedAuthor: false,
+  hasManyPublishers: false,
+  hasReceivedUpdateInOneYear: true,
+  homepage: null,
+  author: null,
+  publishers: [],
+  maintainers: [],
+  integrity: {}
+};
 
 const { version: packageVersion } = JSON.parse(
   readFileSync(
@@ -60,7 +99,7 @@ export async function depWalker(
 
   // We are dealing with an exclude Map to avoid checking a package more than one time in searchDeepDependencies
   const exclude: Map<string, Set<string>> = new Map();
-  const dependencies: Map<string, any> = new Map();
+  const dependencies: Map<string, Dependency> = new Map();
 
   {
     logger
@@ -80,59 +119,66 @@ export async function depWalker(
       registry,
       packageLock
     };
-    for await (const currentDep of treeWalker.npm.walk(manifest, rootDepsOptions)) {
-      const { name, version, dev } = currentDep;
-
-      const current = currentDep.exportAsPlainObject(name === manifest.name ? 0 : void 0);
-      let proceedDependencyAnalysis = true;
-
+    for await (const current of treeWalker.npm.walk(manifest, rootDepsOptions)) {
+      const { name, version, ...currentVersion } = current;
+      const dependency: Dependency = {
+        versions: {
+          [version]: {
+            ...currentVersion,
+            ...structuredClone(kDefaultDependencyVersionFields)
+          }
+        },
+        vulnerabilities: [],
+        metadata: structuredClone(kDefaultDependencyMetadata)
+      };
+
+      let proceedDependencyScan = true;
       if (dependencies.has(name)) {
         const dep = dependencies.get(name)!;
-        promisesToWait.push(manifestMetadata(name, version, dep));
+        promisesToWait.push(
+          manifestMetadata(name, version, dep)
+        );
 
-        const currVersion = Object.keys(current.versions)[0]!;
-        if (currVersion in dep.versions) {
+        if (version in dep.versions) {
           // The dependency has already entered the analysis
           // This happens if the package is used by multiple packages in the tree
-          proceedDependencyAnalysis = false;
+          proceedDependencyScan = false;
         }
         else {
-          dep.versions[currVersion] = current.versions[currVersion];
+          dep.versions[version] = dependency.versions[version];
         }
       }
       else {
-        dependencies.set(name, current);
+        dependencies.set(name, dependency);
       }
 
       // If the dependency is a DevDependencies we ignore it.
-      if (dev) {
+      if (current.isDevDependency || !proceedDependencyScan) {
         continue;
       }
 
-      if (proceedDependencyAnalysis) {
-        logger.tick(ScannerLoggerEvents.analysis.tree);
+      logger.tick(ScannerLoggerEvents.analysis.tree);
 
-        // There is no need to fetch 'N' times the npm metadata for the same package.
-        if (fetchedMetadataPackages.has(name) || !current.versions[version]!.existOnRemoteRegistry) {
-          logger.tick(ScannerLoggerEvents.analysis.registry);
-        }
-        else {
-          fetchedMetadataPackages.add(name);
-          promisesToWait.push(packageMetadata(name, version, {
-            ref: current as any,
-            logger
-          }));
-        }
-
-        // TODO: re-abstract and fix ref type
-        promisesToWait.push(scanDirOrArchive(name, version, {
-          ref: current.versions[version] as any,
-          location,
-          tmpLocation: scanRootNode && name === manifest.name ? null : tmpLocation,
-          locker: tarballLocker,
-          registry
+      // There is no need to fetch 'N' times the npm metadata for the same package.
+      if (fetchedMetadataPackages.has(name) || !current.existOnRemoteRegistry) {
+        logger.tick(ScannerLoggerEvents.analysis.registry);
+      }
+      else {
+        fetchedMetadataPackages.add(name);
+        promisesToWait.push(packageMetadata(name, version, {
+          dependency,
+          logger
         }));
       }
+
+      // TODO: re-abstract and fix ref type
+      promisesToWait.push(scanDirOrArchive(name, version, {
+        ref: dependency.versions[version] as any,
+        location,
+        tmpLocation: scanRootNode && name === manifest.name ? null : tmpLocation,
+        locker: tarballLocker,
+        registry
+      }));
     }
 
     logger.end(ScannerLoggerEvents.analysis.tree);
@@ -141,7 +187,9 @@ export async function depWalker(
     await Promise.allSettled(promisesToWait);
     await timers.setImmediate();
 
-    logger.end(ScannerLoggerEvents.analysis.tarball).end(ScannerLoggerEvents.analysis.registry);
+    logger
+      .end(ScannerLoggerEvents.analysis.tarball)
+      .end(ScannerLoggerEvents.analysis.registry);
   }
 
   const { hydratePayloadDependencies, strategy } = await vuln.setStrategy(vulnerabilityStrategy);
@@ -177,14 +225,16 @@ export async function depWalker(
     }
     for (const version of Object.entries(dependency.versions)) {
       const [verStr, verDescriptor] = version as [string, DependencyVersion];
-      verDescriptor.flags.push(...addMissingVersionFlags(new Set(verDescriptor.flags), dependency));
+      verDescriptor.flags.push(
+        ...addMissingVersionFlags(new Set(verDescriptor.flags), dependency)
+      );
 
       const usedDeps = exclude.get(`${packageName}@${verStr}`) || new Set();
       if (usedDeps.size === 0) {
         continue;
       }
 
-      const usedBy = Object.create(null);
+      const usedBy: Record<string, string> = Object.create(null);
       for (const [name, version] of [...usedDeps].map((name) => name.split(" ") as [string, string])) {
         usedBy[name] = version;
       }

workspaces/scanner/src/npmRegistry.ts

@@ -14,7 +14,12 @@ import {
 // Import Internal Dependencies
 import { getLinks } from "./utils/index.js";
 import { Logger } from "./class/logger.class.js";
-import type { Author, Maintainer, Dependency, Publisher } from "./types.js";
+import type {
+  Author,
+  Maintainer,
+  Publisher,
+  Dependency
+} from "./types.js";
 
 export async function manifestMetadata(
   name: string,
@@ -41,17 +46,15 @@ export async function manifestMetadata(
 
 export interface PackageMetadataOptions {
   logger: Logger;
-  ref: Exclude<Dependency, "metadata"> & {
-    metadata: Partial<Dependency["metadata"]>
-  };
+  dependency: Dependency;
 }
 
 export async function packageMetadata(
   name: string,
   version: string,
   options: PackageMetadataOptions
 ): Promise<void> {
-  const { ref, logger } = options;
+  const { dependency, logger } = options;
   const packageSpec = `${name}:${version}`;
 
   try {
@@ -74,12 +77,11 @@ export async function packageMetadata(
       maintainers: pkg.maintainers
         .map((maintainer) => parseAuthor(maintainer)!) ?? [],
       publishers: [],
-      integrity: {},
-      dependencyCount: 0
+      integrity: {}
     };
 
     const isOutdated = semver.neq(version, lastVersion);
-    const flags = ref.versions[version]!.flags;
+    const flags = dependency.versions[version]!.flags;
     if (isOutdated) {
       flags.push("isOutdated");
     }
@@ -132,8 +134,11 @@ export async function packageMetadata(
     }
 
     await addNpmAvatar(metadata);
-    Object.assign(ref.versions[version]!, { links: getLinks(pkg.versions[version]!) });
-    Object.assign(ref.metadata, metadata);
+    Object.assign(
+      dependency.versions[version]!,
+      { links: getLinks(pkg.versions[version]!) }
+    );
+    dependency.metadata = metadata;
   }
   catch {
     // ignore

workspaces/scanner/src/types.ts

@@ -76,27 +76,29 @@ export interface DependencyVersion {
   usedBy: Record<string, string>;
   /** Size on disk of the extracted tarball (in bytes) */
   size: number;
+  /** Count of dependencies */
+  dependencyCount: number;
   /** Package description */
   description: string;
   /** Author of the package. This information is not trustable and can be empty. */
   author: Author | null;
   engines: Engines;
-  repository: Repository;
+  repository?: Repository;
   scripts: Record<string, string>;
   /**
    * JS-X-Ray warnings
    *
    * @see https://github.com/NodeSecure/js-x-ray/blob/master/WARNINGS.md
    */
   warnings: JSXRay.Warning<JSXRay.WarningDefault>[];
+  alias: Record<string, string>;
   /** Tarball composition (files and dependencies) */
   composition: {
     /** Files extensions (.js, .md, .exe etc..) */
     extensions: string[];
     files: string[];
     /** Minified files (foo.min.js etc..) */
     minified: string[];
-    alias: Record<string, string>;
     required_files: string[];
     required_thirdparty: string[];
     required_nodejs: string[];
@@ -134,8 +136,6 @@ export interface DependencyVersion {
 export interface Dependency {
   /** NPM Registry metadata */
   metadata: {
-    /** Count of dependencies */
-    dependencyCount: number;
     /** Number of releases published on npm */
     publishedCount: number;
     lastUpdateAt: Date;

workspaces/scanner/test/fixtures/depWalker/slimio.is-result.json

@@ -28,7 +28,9 @@
           "coverage": "nyc npm test",
           "report": "nyc report --reporter=html"
         },
+        "dependencyCount": 0,
         "warnings": [],
+        "alias": {},
         "composition": {
           "extensions": [
             "",
@@ -41,7 +43,6 @@
           "minified": [],
           "unused": [],
           "missing": [],
-          "alias": {},
           "required_nodejs": [],
           "required_thirdparty": [],
           "required_subpath": {}
@@ -96,7 +97,6 @@
     },
     "vulnerabilities": [],
     "metadata": {
-      "dependencyCount": 0,
       "publishedCount": 8,
       "lastUpdateAt": "2023-01-23T02:15:37.203Z",
       "lastVersion": "2.0.0",