feat: support specifying what parser to use in --lockfile (google#94)

Resolves google#67
Resolves google#124

Author: G-Rath

README.md

@@ -119,7 +119,11 @@ If you want to check for known vulnerabilities in specific lockfiles, you can us
 osv-scanner --lockfile=/path/to/your/package-lock.json --lockfile=/path/to/another/Cargo.lock
 ```
 
-It is possible to specify more than one lockfile at a time. 
+It is possible to specify more than one lockfile at a time; you can also specify how to parse an arbitrary file:
+
+```console
+osv-scanner --lockfile 'requirements.txt:/path/to/your/extra-requirements.txt'
+```
 
 A wide range of lockfiles are supported by utilizing this [lockfile package](https://github.com/google/osv-scanner/tree/main/pkg/lockfile). This is the current list of supported lockfiles:
 
@@ -140,7 +144,21 @@ A wide range of lockfiles are supported by utilizing this [lockfile package](htt
 - `pubspec.lock`
 - `requirements.txt`[\*](https://github.com/google/osv-scanner/issues/34)
 - `yarn.lock`
-- `/lib/apk/db/installed` (Alpine)
+
+The scanner also supports `installed` files used by the Alpine Package Keeper (apk) that typically live at `/lib/apk/db/installed`,
+however you must specify this explicitly using the `--lockfile` flag:
+
+```console
+osv-scanner --lockfile 'apk-installed:/lib/apk/db/installed'
+```
+
+If the file you are scanning is located in a directory that has a colon in its name,
+you can prefix the path to just a colon to explicitly signal to the scanner that
+it should infer the parser based on the filename:
+
+```bash
+$ osv-scanner --lockfile ':/path/to/my:projects/package-lock.json'
+```
 
 ### Scanning a Debian based docker image packages (preview)
 

cmd/osv-scanner/fixtures/locks-empty/Gemfile.lock

@@ -0,0 +1,3 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+

cmd/osv-scanner/fixtures/locks-empty/composer.lock

@@ -0,0 +1,32 @@
+C:Q1Ef3iwt+cMdGngEgaFr2URIJhKzQ=
+P:apk-tools
+V:2.12.10-r1
+A:x86_64
+S:120973
+I:307200
+T:Alpine Package Keeper - package manager for alpine
+U:https://gitlab.alpinelinux.org/alpine/apk-tools
+L:GPL-2.0-only
+o:apk-tools
+m:Natanael Copa <[email protected]>
+t:1666552494
+c:0188f510baadbae393472103427b9c1875117136
+D:musl>=1.2 ca-certificates-bundle so:libc.musl-x86_64.so.1 so:libcrypto.so.3 so:libssl.so.3 so:libz.so.1
+p:so:libapk.so.3.12.0=3.12.0 cmd:apk=2.12.10-r1
+F:etc
+F:etc/apk
+F:etc/apk/keys
+F:etc/apk/protected_paths.d
+F:lib
+R:libapk.so.3.12.0
+a:0:0:755
+Z:Q1opjpYqXgzmOVo7EbNe8l5Xol08g=
+F:lib/apk
+F:lib/apk/exec
+F:sbin
+R:apk
+a:0:0:755
+Z:Q1/4bmOPe/H1YhHRzlrj27oufThMw=
+F:var
+F:var/lib
+F:var/lib/apk

cmd/osv-scanner/fixtures/locks-empty/yarn.lock

@@ -4,6 +4,7 @@ package main
 import (
 	"bytes"
 	"fmt"
+	"path/filepath"
 	"regexp"
 	"strings"
 	"testing"
@@ -243,3 +244,135 @@ func TestRun(t *testing.T) {
 		})
 	}
 }
+
+func TestRun_LockfileWithExplicitParseAs(t *testing.T) {
+	t.Parallel()
+
+	tests := []cliTestCase{
+		// unsupported parse-as
+		{
+			name:         "",
+			args:         []string{"", "-L", "my-file:my-file"},
+			wantExitCode: 127,
+			wantStdout:   "",
+			wantStderr: `
+				could not determine parser, requested my-file
+			`,
+		},
+		// empty is default
+		{
+			name: "",
+			args: []string{
+				"",
+				"-L",
+				":" + filepath.FromSlash("./fixtures/locks-many/composer.lock"),
+			},
+			wantExitCode: 0,
+			wantStdout: `
+				Scanned %%/fixtures/locks-many/composer.lock file and found 1 packages
+			`,
+			wantStderr: "",
+		},
+		// empty works as an escape (no fixture because it's not valid on Windows)
+		{
+			name: "",
+			args: []string{
+				"",
+				"-L",
+				":" + filepath.FromSlash("./path/to/my:file"),
+			},
+			wantExitCode: 127,
+			wantStdout:   "",
+			wantStderr: `
+				could not determine parser for %%/path/to/my:file
+			`,
+		},
+		{
+			name: "",
+			args: []string{
+				"",
+				"-L",
+				":" + filepath.FromSlash("./path/to/my:project/package-lock.json"),
+			},
+			wantExitCode: 127,
+			wantStdout:   "",
+			wantStderr: `
+				could not read %%/path/to/my:project/package-lock.json: open %%/path/to/my:project/package-lock.json: no such file or directory
+			`,
+		},
+		// when an explicit parse-as is given, it's applied to that file
+		{
+			name: "",
+			args: []string{
+				"",
+				"-L",
+				"package-lock.json:" + filepath.FromSlash("./fixtures/locks-insecure/my-package-lock.json"),
+				filepath.FromSlash("./fixtures/locks-insecure"),
+			},
+			wantExitCode: 1,
+			wantStdout: `
+				Scanned %%/fixtures/locks-insecure/my-package-lock.json file as a package-lock.json and found 1 packages
+				Scanning dir ./fixtures/locks-insecure
+				Scanned %%/fixtures/locks-insecure/composer.lock file and found 0 packages
+				+-------------------------------------+-----------+-----------+---------+----------------------------------------------+
+				| OSV URL (ID IN BOLD)                | ECOSYSTEM | PACKAGE   | VERSION | SOURCE                                       |
+				+-------------------------------------+-----------+-----------+---------+----------------------------------------------+
+				| https://osv.dev/GHSA-whgm-jr23-g3j9 | npm       | ansi-html | 0.0.1   | fixtures/locks-insecure/my-package-lock.json |
+				+-------------------------------------+-----------+-----------+---------+----------------------------------------------+
+			`,
+			wantStderr: "",
+		},
+		// files that error on parsing stop parsable files from being checked
+		{
+			name: "",
+			args: []string{
+				"",
+				"-L",
+				"Cargo.lock:" + filepath.FromSlash("./fixtures/locks-insecure/my-package-lock.json"),
+				filepath.FromSlash("./fixtures/locks-insecure"),
+				filepath.FromSlash("./fixtures/locks-many"),
+			},
+			wantExitCode: 127,
+			wantStdout:   "",
+			wantStderr: `
+				(parsing as Cargo.lock) could not parse %%/fixtures/locks-insecure/my-package-lock.json: toml: line 1: expected '.' or '=', but got '{' instead
+			`,
+		},
+		// parse-as takes priority, even if it's wrong
+		{
+			name: "",
+			args: []string{
+				"",
+				"-L",
+				"package-lock.json:" + filepath.FromSlash("./fixtures/locks-many/yarn.lock"),
+			},
+			wantExitCode: 127,
+			wantStdout:   "",
+			wantStderr: `
+				(parsing as package-lock.json) could not parse %%/fixtures/locks-many/yarn.lock: invalid character '#' looking for beginning of value
+			`,
+		},
+		// "apk-installed" is supported
+		{
+			name: "",
+			args: []string{
+				"",
+				"-L",
+				"apk-installed:" + filepath.FromSlash("./fixtures/locks-many/installed"),
+			},
+			wantExitCode: 0,
+			wantStdout: `
+				Scanned %%/fixtures/locks-many/installed file as a apk-installed and found 1 packages
+			`,
+			wantStderr: "",
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			testCli(t, tt)
+		})
+	}
+}

cmd/osv-scanner/fixtures/locks-insecure/composer.lock

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"fmt"
 	"os"
+	"sort"
 	"strings"
 )
 
@@ -104,3 +105,23 @@ func ParseApkInstalled(pathToLockfile string) ([]PackageDetails, error) {
 
 	return packages, nil
 }
+
+// FromApkInstalled attempts to parse the given file as an "apk-installed" lockfile
+// used by the Alpine Package Keeper (apk) to record installed packages.
+func FromApkInstalled(pathToInstalled string) (Lockfile, error) {
+	packages, err := ParseApkInstalled(pathToInstalled)
+
+	sort.Slice(packages, func(i, j int) bool {
+		if packages[i].Name == packages[j].Name {
+			return packages[i].Version < packages[j].Version
+		}
+
+		return packages[i].Name < packages[j].Name
+	})
+
+	return Lockfile{
+		FilePath: pathToInstalled,
+		ParsedAs: "apk-installed",
+		Packages: packages,
+	}, err
+}

cmd/osv-scanner/fixtures/locks-insecure/my-package-lock.json

@@ -126,11 +126,19 @@ func Parse(pathToLockfile string, parseAs string) (Lockfile, error) {
 	parser, parsedAs := FindParser(pathToLockfile, parseAs)
 
 	if parser == nil {
+		if parseAs != "" {
+			return Lockfile{}, fmt.Errorf("%w, requested %s", ErrParserNotFound, parseAs)
+		}
+
 		return Lockfile{}, fmt.Errorf("%w for %s", ErrParserNotFound, pathToLockfile)
 	}
 
 	packages, err := parser(pathToLockfile)
 
+	if err != nil && parseAs != "" {
+		err = fmt.Errorf("(parsing as %s) %w", parsedAs, err)
+	}
+
 	sort.Slice(packages, func(i, j int) bool {
 		if packages[i].Name == packages[j].Name {
 			return packages[i].Version < packages[j].Version

cmd/osv-scanner/fixtures/locks-many/installed

@@ -67,7 +67,7 @@ func scanDir(r *output.Reporter, query *osv.BatchedQuery, dir string, skipGit bo
 
 		if !info.IsDir() {
 			if parser, _ := lockfile.FindParser(path, ""); parser != nil {
-				err := scanLockfile(r, query, path)
+				err := scanLockfile(r, query, path, "")
 				if err != nil {
 					r.PrintError(fmt.Sprintf("Attempted to scan lockfile but failed: %s\n", path))
 				}
@@ -89,12 +89,29 @@ func scanDir(r *output.Reporter, query *osv.BatchedQuery, dir string, skipGit bo
 
 // scanLockfile will load, identify, and parse the lockfile path passed in, and add the dependencies specified
 // within to `query`
-func scanLockfile(r *output.Reporter, query *osv.BatchedQuery, path string) error {
-	parsedLockfile, err := lockfile.Parse(path, "")
+func scanLockfile(r *output.Reporter, query *osv.BatchedQuery, path string, parseAs string) error {
+	var err error
+	var parsedLockfile lockfile.Lockfile
+
+	// special case for the APK parser because it has a very generic name while
+	// living at a specific location, so it's not included in the map of parsers
+	// used by lockfile.Parse to avoid false-positives when scanning projects
+	if parseAs == "apk-installed" {
+		parsedLockfile, err = lockfile.FromApkInstalled(path)
+	} else {
+		parsedLockfile, err = lockfile.Parse(path, parseAs)
+	}
+
 	if err != nil {
 		return err
 	}
-	r.PrintText(fmt.Sprintf("Scanned %s file and found %d packages\n", path, len(parsedLockfile.Packages)))
+	parsedAsComment := ""
+
+	if parseAs != "" {
+		parsedAsComment = fmt.Sprintf("as a %s ", parseAs)
+	}
+
+	r.PrintText(fmt.Sprintf("Scanned %s file %sand found %d packages\n", path, parsedAsComment, len(parsedLockfile.Packages)))
 
 	for _, pkgDetail := range parsedLockfile.Packages {
 		pkgDetailQuery := osv.MakePkgRequest(pkgDetail)
@@ -266,6 +283,16 @@ func filterResponse(r *output.Reporter, query osv.BatchedQuery, resp *osv.Batche
 	return len(hiddenVulns)
 }
 
+func parseLockfilePath(lockfileElem string) (string, string) {
+	if !strings.Contains(lockfileElem, ":") {
+		lockfileElem = ":" + lockfileElem
+	}
+
+	splits := strings.SplitN(lockfileElem, ":", 2)
+
+	return splits[0], splits[1]
+}
+
 // Perform osv scanner action, with optional reporter to output information
 func DoScan(actions ScannerActions, r *output.Reporter) (models.VulnerabilityResults, error) {
 	if r == nil {
@@ -294,12 +321,13 @@ func DoScan(actions ScannerActions, r *output.Reporter) (models.VulnerabilityRes
 	}
 
 	for _, lockfileElem := range actions.LockfilePaths {
-		lockfileElem, err := filepath.Abs(lockfileElem)
+		parseAs, lockfilePath := parseLockfilePath(lockfileElem)
+		lockfilePath, err := filepath.Abs(lockfilePath)
 		if err != nil {
 			r.PrintError(fmt.Sprintf("Failed to resolved path with error %s\n", err))
 			return models.VulnerabilityResults{}, err
 		}
-		err = scanLockfile(r, &query, lockfileElem)
+		err = scanLockfile(r, &query, lockfilePath, parseAs)
 		if err != nil {
 			return models.VulnerabilityResults{}, err
 		}