Skip to content

Commit 2112de1

Browse files
committed
feat(tarball): implement new EntryFilesAnalyser API
1 parent ee413ab commit 2112de1

File tree

23 files changed

+984
-358
lines changed

23 files changed

+984
-358
lines changed

.changeset/easy-bananas-carry.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
"@nodesecure/tarball": major
3+
"@nodesecure/scanner": minor
4+
---
5+
6+
Implement new major JS-X-Ray API and completely refactor tarball package

package-lock.json

Lines changed: 26 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
// Import Node.js Dependencies
2+
import fs from "node:fs/promises";
3+
import path from "node:path";
4+
import os from "node:os";
5+
6+
export class TempDirectory {
7+
location: string;
8+
id: string;
9+
10+
constructor(
11+
location: string,
12+
id: string
13+
) {
14+
this.location = location;
15+
this.id = id;
16+
}
17+
18+
static async create() {
19+
const location = await fs.mkdtemp(
20+
path.join(os.tmpdir(), "/")
21+
);
22+
23+
return new TempDirectory(
24+
location,
25+
location.slice(-6)
26+
);
27+
}
28+
29+
async clear() {
30+
await fs.rm(
31+
this.location,
32+
{ recursive: true, force: true }
33+
);
34+
35+
return this;
36+
}
37+
}

workspaces/scanner/src/depWalker.ts

Lines changed: 37 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
11
// Import Node.js Dependencies
22
import path from "node:path";
3-
import { readFileSync, promises as fs } from "node:fs";
3+
import { readFileSync } from "node:fs";
44
import timers from "node:timers/promises";
5-
import os from "node:os";
65

76
// Import Third-party Dependencies
87
import { Mutex, MutexRelease } from "@openally/mutex";
9-
import { scanDirOrArchive, type ScanDirOrArchiveOptions } from "@nodesecure/tarball";
8+
import {
9+
extractAndResolve,
10+
scanDirOrArchive
11+
} from "@nodesecure/tarball";
1012
import * as Vulnera from "@nodesecure/vulnera";
1113
import { npm } from "@nodesecure/tree-walker";
1214
import { parseAuthor } from "@nodesecure/utils";
15+
import { ManifestManager } from "@nodesecure/mama";
1316
import type { ManifestVersion, PackageJSON } from "@nodesecure/npm-types";
1417

1518
// Import Internal Dependencies
@@ -20,6 +23,7 @@ import {
2023
getManifestLinks
2124
} from "./utils/index.js";
2225
import { packageMetadata, manifestMetadata } from "./npmRegistry.js";
26+
import { TempDirectory } from "./class/TempDirectory.class.js";
2327
import { Logger, ScannerLoggerEvents } from "./class/logger.class.js";
2428
import type {
2529
Dependency,
@@ -90,11 +94,10 @@ export async function depWalker(
9094
registry
9195
} = options;
9296

93-
// Create TMP directory
94-
const tmpLocation = await fs.mkdtemp(path.join(os.tmpdir(), "/"));
97+
const tempDir = await TempDirectory.create();
9598

9699
const payload: Partial<Payload> = {
97-
id: tmpLocation.slice(-6),
100+
id: tempDir.id,
98101
rootDependencyName: manifest.name,
99102
scannerVersion: packageVersion,
100103
vulnerabilityStrategy,
@@ -179,10 +182,12 @@ export async function depWalker(
179182
const scanDirOptions = {
180183
ref: dependency.versions[version] as any,
181184
location,
182-
tmpLocation: scanRootNode && name === manifest.name ? null : tmpLocation,
185+
isRootNode: scanRootNode && name === manifest.name,
183186
registry
184187
};
185-
operationsQueue.push(scanDirOrArchiveEx(name, version, locker, scanDirOptions));
188+
operationsQueue.push(
189+
scanDirOrArchiveEx(name, version, locker, tempDir, scanDirOptions)
190+
);
186191
}
187192

188193
logger.end(ScannerLoggerEvents.analysis.tree);
@@ -279,7 +284,7 @@ export async function depWalker(
279284
}
280285
finally {
281286
await timers.setImmediate();
282-
await fs.rm(tmpLocation, { recursive: true, force: true });
287+
await tempDir.clear();
283288

284289
logger.emit(ScannerLoggerEvents.done);
285290
}
@@ -290,12 +295,33 @@ async function scanDirOrArchiveEx(
290295
name: string,
291296
version: string,
292297
locker: Mutex,
293-
options: ScanDirOrArchiveOptions
298+
tempDir: TempDirectory,
299+
options: {
300+
registry?: string;
301+
isRootNode: boolean;
302+
location: string | undefined;
303+
ref: any;
304+
}
294305
) {
295306
const free = await locker.acquire();
296307

297308
try {
298-
await scanDirOrArchive(name, version, options);
309+
const {
310+
registry,
311+
location = process.cwd(),
312+
isRootNode,
313+
ref
314+
} = options;
315+
316+
const mama = await (isRootNode ?
317+
ManifestManager.fromPackageJSON(location!) :
318+
extractAndResolve(tempDir.location, {
319+
spec: `${name}@${version}`,
320+
registry
321+
})
322+
);
323+
324+
await scanDirOrArchive(mama, ref);
299325
}
300326
catch {
301327
// ignore

workspaces/scanner/src/index.ts

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import type { PackageJSON } from "@nodesecure/npm-types";
1414
import { depWalker } from "./depWalker.js";
1515
import { NPM_TOKEN, urlToString } from "./utils/index.js";
1616
import { Logger, ScannerLoggerEvents } from "./class/logger.class.js";
17+
import { TempDirectory } from "./class/TempDirectory.class.js";
1718
import { comparePayloads } from "./comparePayloads.js";
1819
import type { Options } from "./types.js";
1920

@@ -87,23 +88,20 @@ export async function verify(
8788
return tarball.scanPackage(process.cwd());
8889
}
8990

90-
const tmpLocation = await fs.mkdtemp(
91-
path.join(os.tmpdir(), "nsecure-")
92-
);
93-
const dest = path.join(tmpLocation, packageName);
91+
const tempDir = await TempDirectory.create();
9492

9593
try {
96-
await pacote.extract(packageName, dest, {
97-
...NPM_TOKEN, registry: getLocalRegistryURL(), cache: `${os.homedir()}/.npm`
94+
const mama = await tarball.extractAndResolve(tempDir.location, {
95+
spec: packageName,
96+
registry: getLocalRegistryURL()
9897
});
99-
100-
const scanResult = await tarball.scanPackage(dest, packageName);
98+
const scanResult = await tarball.scanPackage(mama);
10199

102100
return scanResult;
103101
}
104102
finally {
105103
await timers.setImmediate();
106-
await fs.rm(tmpLocation, { recursive: true, force: true });
104+
await tempDir.clear();
107105
}
108106
}
109107

workspaces/tarball/README.md

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -35,23 +35,21 @@ console.log(scanResult);
3535
3636
## API
3737

38-
### scanDirOrArchive
38+
- [SourceCode](./docs/SourceCode.md)
39+
- [NpmTarball](./docs/NpmTarball.md)
3940

40-
Method created for Scanner (to be refactored soon)
41+
---
4142

42-
```ts
43-
export interface ScanDirOrArchiveOptions {
44-
ref: DependencyRef;
45-
location?: string;
46-
tmpLocation?: null | string;
47-
locker: Locker;
48-
registry: string;
49-
}
50-
```
43+
> [!CAUTION]
44+
> The following APIs are considered legacy and are waiting for deprecation in future releases.
45+
46+
### scanDirOrArchive(locationOrManifest: string | ManifestManager, ref: DependencyRef): Promise< void >
5147

52-
### scanPackage(dest: string, packageName?: string): Promise< ScannedPackageResult >
48+
Scan a given local project or tarball (by providing the path or directly the ManifestManager instance).
5349

54-
Scan a given tarball archive or a local project.
50+
### scanPackage(manifestOrLocation: string | ManifestManager): Promise< ScannedPackageResult >
51+
52+
Scan a given local project containing a Manifest (package.json).
5553

5654
```ts
5755
interface ScannedPackageResult {
@@ -68,13 +66,17 @@ interface ScannedPackageResult {
6866
/** Unique license contained in the tarball (MIT, ISC ..) */
6967
uniqueLicenseIds: string[];
7068
/** All licenses with their SPDX */
71-
licenses: ntlp.SpdxLicenseConformance[];
69+
licenses: conformance.SpdxFileLicenseConformance[];
7270
ast: {
7371
dependencies: Record<string, Record<string, Dependency>>;
7472
warnings: Warning[];
7573
};
7674
}
7775
```
7876

77+
### extractAndResolve(location: string, options: TarballResolutionOptions): Promise< ManifestManager >
78+
79+
Extract a given remote package.
80+
7981
## License
8082
MIT

workspaces/tarball/docs/NpmTarball.md

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# NpmTarball
2+
3+
## Usage example
4+
5+
```ts
6+
import { ManifestManager } from "@nodesecure/mama";
7+
import { NpmTarball } from "@nodesecure/tarball";
8+
9+
const mama = await ManifestManager.fromPackageJSON(
10+
location
11+
);
12+
const extractor = new NpmTarball(mama);
13+
14+
const {
15+
composition,
16+
conformance,
17+
code
18+
} = await extractor.scanFiles();
19+
```
20+
21+
## API
22+
23+
### constructor(manifest: ManifestManager)
24+
25+
Create a new NpmTarball instance.
26+
27+
> [!CAUTION]
28+
> ManifestManager instance must have a location defined
29+
30+
### scanFiles(): Promise< ScannedFilesResult >
31+
32+
Scan all the files contained in the tarball and obtain a complete report, including detection of JavaScript threats.
33+
34+
```ts
35+
interface ScannedFilesResult {
36+
composition: TarballComposition;
37+
conformance: SpdxExtractedResult;
38+
code: SourceCodeReport;
39+
}
40+
```

0 commit comments

Comments
 (0)