feat(tarball): implement new EntryFilesAnalyser API

Author: fraxken

.changeset/easy-bananas-carry.md

@@ -0,0 +1,6 @@
+---
+"@nodesecure/tarball": major
+"@nodesecure/scanner": minor
+---
+
+Implement new major JS-X-Ray API and completely refactor tarball package

package-lock.json

@@ -0,0 +1,37 @@
+// Import Node.js Dependencies
+import fs from "node:fs/promises";
+import path from "node:path";
+import os from "node:os";
+
+export class TempDirectory {
+  location: string;
+  id: string;
+
+  constructor(
+    location: string,
+    id: string
+  ) {
+    this.location = location;
+    this.id = id;
+  }
+
+  static async create() {
+    const location = await fs.mkdtemp(
+      path.join(os.tmpdir(), "/")
+    );
+
+    return new TempDirectory(
+      location,
+      location.slice(-6)
+    );
+  }
+
+  async clear() {
+    await fs.rm(
+      this.location,
+      { recursive: true, force: true }
+    );
+
+    return this;
+  }
+}

workspaces/scanner/src/class/TempDirectory.class.ts

@@ -1,15 +1,18 @@
 // Import Node.js Dependencies
 import path from "node:path";
-import { readFileSync, promises as fs } from "node:fs";
+import { readFileSync } from "node:fs";
 import timers from "node:timers/promises";
-import os from "node:os";
 
 // Import Third-party Dependencies
 import { Mutex, MutexRelease } from "@openally/mutex";
-import { scanDirOrArchive, type ScanDirOrArchiveOptions } from "@nodesecure/tarball";
+import {
+  extractAndResolve,
+  scanDirOrArchive
+} from "@nodesecure/tarball";
 import * as Vulnera from "@nodesecure/vulnera";
 import { npm } from "@nodesecure/tree-walker";
 import { parseAuthor } from "@nodesecure/utils";
+import { ManifestManager } from "@nodesecure/mama";
 import type { ManifestVersion, PackageJSON } from "@nodesecure/npm-types";
 
 // Import Internal Dependencies
@@ -20,6 +23,7 @@ import {
   getManifestLinks
 } from "./utils/index.js";
 import { packageMetadata, manifestMetadata } from "./npmRegistry.js";
+import { TempDirectory } from "./class/TempDirectory.class.js";
 import { Logger, ScannerLoggerEvents } from "./class/logger.class.js";
 import type {
   Dependency,
@@ -90,11 +94,10 @@ export async function depWalker(
     registry
   } = options;
 
-  // Create TMP directory
-  const tmpLocation = await fs.mkdtemp(path.join(os.tmpdir(), "/"));
+  const tempDir = await TempDirectory.create();
 
   const payload: Partial<Payload> = {
-    id: tmpLocation.slice(-6),
+    id: tempDir.id,
     rootDependencyName: manifest.name,
     scannerVersion: packageVersion,
     vulnerabilityStrategy,
@@ -179,10 +182,12 @@ export async function depWalker(
       const scanDirOptions = {
         ref: dependency.versions[version] as any,
         location,
-        tmpLocation: scanRootNode && name === manifest.name ? null : tmpLocation,
+        isRootNode: scanRootNode && name === manifest.name,
         registry
       };
-      operationsQueue.push(scanDirOrArchiveEx(name, version, locker, scanDirOptions));
+      operationsQueue.push(
+        scanDirOrArchiveEx(name, version, locker, tempDir, scanDirOptions)
+      );
     }
 
     logger.end(ScannerLoggerEvents.analysis.tree);
@@ -279,7 +284,7 @@ export async function depWalker(
   }
   finally {
     await timers.setImmediate();
-    await fs.rm(tmpLocation, { recursive: true, force: true });
+    await tempDir.clear();
 
     logger.emit(ScannerLoggerEvents.done);
   }
@@ -290,12 +295,33 @@ async function scanDirOrArchiveEx(
   name: string,
   version: string,
   locker: Mutex,
-  options: ScanDirOrArchiveOptions
+  tempDir: TempDirectory,
+  options: {
+    registry?: string;
+    isRootNode: boolean;
+    location: string | undefined;
+    ref: any;
+  }
 ) {
   const free = await locker.acquire();
 
   try {
-    await scanDirOrArchive(name, version, options);
+    const {
+      registry,
+      location = process.cwd(),
+      isRootNode,
+      ref
+    } = options;
+
+    const mama = await (isRootNode ?
+      ManifestManager.fromPackageJSON(location) :
+      extractAndResolve(tempDir.location, {
+        spec: `${name}@${version}`,
+        registry
+      })
+    );
+
+    await scanDirOrArchive(mama, ref);
   }
   catch {
     // ignore

workspaces/scanner/src/depWalker.ts

@@ -14,6 +14,7 @@ import type { PackageJSON } from "@nodesecure/npm-types";
 import { depWalker } from "./depWalker.js";
 import { NPM_TOKEN, urlToString } from "./utils/index.js";
 import { Logger, ScannerLoggerEvents } from "./class/logger.class.js";
+import { TempDirectory } from "./class/TempDirectory.class.js";
 import { comparePayloads } from "./comparePayloads.js";
 import type { Options } from "./types.js";
 
@@ -87,23 +88,20 @@ export async function verify(
     return tarball.scanPackage(process.cwd());
   }
 
-  const tmpLocation = await fs.mkdtemp(
-    path.join(os.tmpdir(), "nsecure-")
-  );
-  const dest = path.join(tmpLocation, packageName);
+  const tempDir = await TempDirectory.create();
 
   try {
-    await pacote.extract(packageName, dest, {
-      ...NPM_TOKEN, registry: getLocalRegistryURL(), cache: `${os.homedir()}/.npm`
+    const mama = await tarball.extractAndResolve(tempDir.location, {
+      spec: packageName,
+      registry: getLocalRegistryURL()
     });
-
-    const scanResult = await tarball.scanPackage(dest, packageName);
+    const scanResult = await tarball.scanPackage(mama);
 
     return scanResult;
   }
   finally {
     await timers.setImmediate();
-    await fs.rm(tmpLocation, { recursive: true, force: true });
+    await tempDir.clear();
   }
 }
 

workspaces/scanner/src/index.ts

@@ -229,7 +229,10 @@ describe("scanner.cwd()", () => {
     );
     const pkg = dependencies["random-package"];
 
-    assert.strictEqual(pkg.metadata.author, null);
+    assert.deepEqual(pkg.metadata.author, {
+      email: "[email protected]",
+      name: "John Doe"
+    });
   });
 });
 

workspaces/scanner/test/depWalker.spec.ts

@@ -35,23 +35,21 @@ console.log(scanResult);
 
 ## API
 
-### scanDirOrArchive
+- [SourceCode](./docs/SourceCode.md)
+- [NpmTarball](./docs/NpmTarball.md)
 
-Method created for Scanner (to be refactored soon)
+---
 
-```ts
-export interface ScanDirOrArchiveOptions {
-  ref: DependencyRef;
-  location?: string;
-  tmpLocation?: null | string;
-  locker: Locker;
-  registry: string;
-}
-```
+> [!CAUTION]
+> The following APIs are considered legacy and are waiting for deprecation in future releases.
+
+### scanDirOrArchive(locationOrManifest: string | ManifestManager, ref: DependencyRef): Promise< void >
 
-### scanPackage(dest: string, packageName?: string): Promise< ScannedPackageResult > 
+Scan a given local project or tarball (by providing the path or directly the ManifestManager instance).
 
-Scan a given tarball archive or a local project.
+### scanPackage(manifestOrLocation: string | ManifestManager): Promise< ScannedPackageResult > 
+
+Scan a given local project containing a Manifest (package.json).
 
 ```ts
 interface ScannedPackageResult {
@@ -68,13 +66,17 @@ interface ScannedPackageResult {
   /** Unique license contained in the tarball (MIT, ISC ..) */
   uniqueLicenseIds: string[];
   /** All licenses with their SPDX */
-  licenses: ntlp.SpdxLicenseConformance[];
+  licenses: conformance.SpdxFileLicenseConformance[];
   ast: {
     dependencies: Record<string, Record<string, Dependency>>;
     warnings: Warning[];
   };
 }
 ```
 
+### extractAndResolve(location: string, options: TarballResolutionOptions): Promise< ManifestManager >
+
+Extract a given remote package.
+
 ## License
 MIT

workspaces/tarball/README.md

@@ -0,0 +1,40 @@
+# NpmTarball
+
+## Usage example
+
+```ts
+import { ManifestManager } from "@nodesecure/mama";
+import { NpmTarball } from "@nodesecure/tarball";
+
+const mama = await ManifestManager.fromPackageJSON(
+  location
+);
+const extractor = new NpmTarball(mama);
+
+const {
+  composition,
+  conformance,
+  code
+} = await extractor.scanFiles();
+```
+
+## API
+
+### constructor(manifest: ManifestManager)
+
+Create a new NpmTarball instance.
+
+> [!CAUTION]
+> ManifestManager instance must have a location defined
+
+### scanFiles(): Promise< ScannedFilesResult >
+
+Scan all the files contained in the tarball and obtain a complete report, including detection of JavaScript threats.
+
+```ts
+interface ScannedFilesResult {
+  composition: TarballComposition;
+  conformance: SpdxExtractedResult;
+  code: SourceCodeReport;
+}
+```