Skip to content

Commit c050cea

Browse files
authored
Add pdm lockfile support (#776)
Add support for parsing package information from `pdm.lock` -files used by `pdm`, package and dependency manager for Python (https://pdm-project.org/latest/)
1 parent 96b1e4e commit c050cea

14 files changed

+473
-15
lines changed

docs/supported_languages_and_lockfiles.md

+13-13
Original file line numberDiff line numberDiff line change
@@ -22,19 +22,19 @@ nav_order: 2
2222

2323
A wide range of lockfiles are supported by utilizing this [lockfile package](https://github.com/google/osv-scanner/tree/main/pkg/lockfile).
2424

25-
| Language | Compatible Lockfile(s) |
26-
| :--------- | :------------------------------------------------------------------------------------------------------------------- |
27-
| C/C++ | `conan.lock`<br>[C/C++ commit scanning](#cc-scanning) |
28-
| Dart | `pubspec.lock` |
29-
| Elixir | `mix.lock` |
30-
| Go | `go.mod` |
31-
| Java | `buildscript-gradle.lockfile`<br>`gradle.lockfile`<br>`pom.xml`[\*](https://github.com/google/osv-scanner/issues/35) |
32-
| Javascript | `package-lock.json`<br>`pnpm-lock.yaml`<br>`yarn.lock` |
33-
| PHP | `composer.lock` |
34-
| Python | `Pipfile.lock`<br>`poetry.lock`<br>`requirements.txt`[\*](https://github.com/google/osv-scanner/issues/34) |
35-
| R | `renv.lock` |
36-
| Ruby | `Gemfile.lock` |
37-
| Rust | `Cargo.lock` |
25+
| Language | Compatible Lockfile(s) |
26+
| :--------- | :----------------------------------------------------------------------------------------------------------------------- |
27+
| C/C++ | `conan.lock`<br>[C/C++ commit scanning](#cc-scanning) |
28+
| Dart | `pubspec.lock` |
29+
| Elixir | `mix.lock` |
30+
| Go | `go.mod` |
31+
| Java | `buildscript-gradle.lockfile`<br>`gradle.lockfile`<br>`pom.xml`[\*](https://github.com/google/osv-scanner/issues/35) |
32+
| Javascript | `package-lock.json`<br>`pnpm-lock.yaml`<br>`yarn.lock` |
33+
| PHP | `composer.lock` |
34+
| Python | `Pipfile.lock`<br>`poetry.lock`<br>`requirements.txt`<br>`pdm.lock`[\*](https://github.com/google/osv-scanner/issues/34) |
35+
| R | `renv.lock` |
36+
| Ruby | `Gemfile.lock` |
37+
| Rust | `Cargo.lock` |
3838

3939
## Alpine Package Keeper and Debian Package Keeper
4040

pkg/lockfile/ecosystems_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,10 @@ func TestKnownEcosystems(t *testing.T) {
3535
expectedCount := numberOfLockfileParsers(t)
3636

3737
// - npm, yarn, and pnpm,
38-
// - pip, poetry, and pipenv,
38+
// - pip, poetry, pdm and pipenv,
3939
// - maven and gradle,
4040
// all use the same ecosystem so "ignore" those parsers in the count
41-
expectedCount -= 5
41+
expectedCount -= 6
4242

4343
ecosystems := lockfile.KnownEcosystems()
4444

pkg/lockfile/extract_test.go

+2
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ func TestFindExtractor(t *testing.T) {
4141
"go.mod": "go.mod",
4242
"gradle.lockfile": "gradle.lockfile",
4343
"mix.lock": "mix.lock",
44+
"pdm.lock": "pdm.lock",
4445
"Pipfile.lock": "Pipfile.lock",
4546
"package-lock.json": "package-lock.json",
4647
"packages.lock.json": "packages.lock.json",
@@ -92,6 +93,7 @@ func TestExtractDeps_FindsExpectedExtractor(t *testing.T) {
9293
"go.mod",
9394
"gradle.lockfile",
9495
"mix.lock",
96+
"pdm.lock",
9597
"Pipfile.lock",
9698
"package-lock.json",
9799
"packages.lock.json",
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# This file is @generated by PDM.
2+
# It is not intended for manual editing.
3+
4+
[metadata]
5+
groups = ["default", "dev"]
6+
strategy = ["cross_platform", "inherit_metadata"]
7+
lock_version = "4.4.1"
8+
content_hash = "sha256:5a543e4cbf50fa2fae6c9180c8c4b4031bbaf7e95c26484384109782ebdfd647"
9+
10+
[[package]]
11+
name = "pyroute2"
12+
version = "0.7.11"
13+
summary = "Python Netlink library"
14+
groups = ["dev"]
15+
dependencies = [
16+
"win-inet-pton; platform_system == \"Windows\"",
17+
]
18+
files = [
19+
{file = "pyroute2-0.7.11-py3-none-any.whl", hash = "sha256:95852e702149b3d6abc8484d3291c38c45660168e8db76e5566a60ef0e133d5b"},
20+
]
21+
22+
[[package]]
23+
name = "toml"
24+
version = "0.10.2"
25+
requires_python = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
26+
summary = "Python Library for Tom's Obvious, Minimal Language"
27+
groups = ["default"]
28+
files = [
29+
{file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"},
30+
{file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"},
31+
]
32+
33+
[[package]]
34+
name = "win-inet-pton"
35+
version = "1.1.0"
36+
summary = "Native inet_pton and inet_ntop implementation for Python on Windows (with ctypes)."
37+
groups = ["dev"]
38+
marker = "platform_system == \"Windows\""
39+
files = [
40+
{file = "win_inet_pton-1.1.0-py2.py3-none-any.whl", hash = "sha256:eaf0193cbe7152ac313598a0da7313fb479f769343c0c16c5308f64887dc885b"},
41+
{file = "win_inet_pton-1.1.0.tar.gz", hash = "sha256:dd03d942c0d3e2b1cf8bab511844546dfa5f74cb61b241699fa379ad707dea4f"},
42+
]

pkg/lockfile/fixtures/pdm/empty.toml

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# This file is @generated by PDM.
2+
# It is not intended for manual editing.
3+
4+
[metadata]
5+
groups = ["default"]
6+
strategy = ["cross_platform", "inherit_metadata"]
7+
lock_version = "4.4.1"
8+
content_hash = "sha256:ebb844511b46d2da311c9adf39499a26883e110605ef99ca9c6762905dcb3e56"
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# This file is @generated by PDM.
2+
# It is not intended for manual editing.
3+
4+
[metadata]
5+
groups = ["default"]
6+
strategy = ["cross_platform", "inherit_metadata"]
7+
lock_version = "4.4.1"
8+
content_hash = "sha256:93fc3209615f0fa4a29b2bd263bb5e72c2343ebc334df8c56c5d9f2cacfc0241"
9+
10+
[[package]]
11+
name = "toml"
12+
version = "0.10.2"
13+
requires_python = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
14+
git = "https://github.com/uiri/toml.git"
15+
revision = "65bab7582ce14c55cdeec2244c65ea23039c9e6f"
16+
summary = "Python Library for Tom's Obvious, Minimal Language"
17+
groups = ["default"]
+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
this is not valid toml! (I think)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# This file is @generated by PDM.
2+
# It is not intended for manual editing.
3+
4+
[metadata]
5+
groups = ["default", "tmp"]
6+
strategy = ["cross_platform", "inherit_metadata"]
7+
lock_version = "4.4.1"
8+
content_hash = "sha256:dfa3dd060fb1217f183be92d3a42fc9b77ca2ca340a7ac15c6786de5bbced943"
9+
10+
[[package]]
11+
name = "pyroute2"
12+
version = "0.7.11"
13+
summary = "Python Netlink library"
14+
groups = ["tmp"]
15+
dependencies = [
16+
"win-inet-pton; platform_system == \"Windows\"",
17+
]
18+
files = [
19+
{file = "pyroute2-0.7.11-py3-none-any.whl", hash = "sha256:95852e702149b3d6abc8484d3291c38c45660168e8db76e5566a60ef0e133d5b"},
20+
]
21+
22+
[[package]]
23+
name = "toml"
24+
version = "0.10.2"
25+
requires_python = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
26+
summary = "Python Library for Tom's Obvious, Minimal Language"
27+
groups = ["default"]
28+
files = [
29+
{file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"},
30+
{file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"},
31+
]
32+
33+
[[package]]
34+
name = "win-inet-pton"
35+
version = "1.1.0"
36+
summary = "Native inet_pton and inet_ntop implementation for Python on Windows (with ctypes)."
37+
groups = ["tmp"]
38+
marker = "platform_system == \"Windows\""
39+
files = [
40+
{file = "win_inet_pton-1.1.0-py2.py3-none-any.whl", hash = "sha256:eaf0193cbe7152ac313598a0da7313fb479f769343c0c16c5308f64887dc885b"},
41+
{file = "win_inet_pton-1.1.0.tar.gz", hash = "sha256:dd03d942c0d3e2b1cf8bab511844546dfa5f74cb61b241699fa379ad707dea4f"},
42+
]
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# This file is @generated by PDM.
2+
# It is not intended for manual editing.
3+
4+
[metadata]
5+
groups = ["default"]
6+
strategy = ["cross_platform", "inherit_metadata"]
7+
lock_version = "4.4.1"
8+
content_hash = "sha256:0cee617a22cf58c87c4b154a4a31e08351b4e38f471f6c82edbb1ee185bda2cf"
9+
10+
[[package]]
11+
name = "toml"
12+
version = "0.10.2"
13+
requires_python = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
14+
summary = "Python Library for Tom's Obvious, Minimal Language"
15+
groups = ["default"]
16+
files = [
17+
{file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"},
18+
{file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"},
19+
]
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# This file is @generated by PDM.
2+
# It is not intended for manual editing.
3+
4+
[metadata]
5+
groups = ["default"]
6+
strategy = ["cross_platform", "inherit_metadata"]
7+
lock_version = "4.4.1"
8+
content_hash = "sha256:0acb7cdc3e805d9bec1f3347b79b69d92ba257d2cd82b5ef4355010930d46deb"
9+
10+
[[package]]
11+
name = "six"
12+
version = "1.16.0"
13+
requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
14+
summary = "Python 2 and 3 compatibility utilities"
15+
groups = ["default"]
16+
files = [
17+
{file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
18+
{file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
19+
]
20+
21+
[[package]]
22+
name = "toml"
23+
version = "0.10.2"
24+
requires_python = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
25+
summary = "Python Library for Tom's Obvious, Minimal Language"
26+
groups = ["default"]
27+
files = [
28+
{file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"},
29+
{file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"},
30+
]

pkg/lockfile/parse-pdm-lock.go

+79
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
package lockfile
2+
3+
import (
4+
"fmt"
5+
"path/filepath"
6+
7+
"github.com/BurntSushi/toml"
8+
)
9+
10+
type PdmLockPackage struct {
11+
Name string `toml:"name"`
12+
Version string `toml:"version"`
13+
Groups []string `toml:"groups"`
14+
Revision string `toml:"revision"`
15+
}
16+
17+
type PdmLockFile struct {
18+
Version string `toml:"lock-version"`
19+
Packages []PdmLockPackage `toml:"package"`
20+
}
21+
22+
const PdmEcosystem = PipEcosystem
23+
24+
type PdmLockExtractor struct{}
25+
26+
func (p PdmLockExtractor) ShouldExtract(path string) bool {
27+
return filepath.Base(path) == "pdm.lock"
28+
}
29+
30+
func (p PdmLockExtractor) Extract(f DepFile) ([]PackageDetails, error) {
31+
var parsedLockFile *PdmLockFile
32+
33+
_, err := toml.NewDecoder(f).Decode(&parsedLockFile)
34+
if err != nil {
35+
return []PackageDetails{}, fmt.Errorf("could not extract from %s: %w", f.Path(), err)
36+
}
37+
packages := make([]PackageDetails, 0, len(parsedLockFile.Packages))
38+
39+
for _, pkg := range parsedLockFile.Packages {
40+
details := PackageDetails{
41+
Name: pkg.Name,
42+
Version: pkg.Version,
43+
Ecosystem: PdmEcosystem,
44+
CompareAs: PdmEcosystem,
45+
}
46+
47+
var optional = true
48+
for _, gr := range pkg.Groups {
49+
if gr == "dev" {
50+
details.DepGroups = append(details.DepGroups, "dev")
51+
optional = false
52+
} else if gr == "default" {
53+
optional = false
54+
}
55+
}
56+
if optional {
57+
details.DepGroups = append(details.DepGroups, "optional")
58+
}
59+
60+
if pkg.Revision != "" {
61+
details.Commit = pkg.Revision
62+
}
63+
64+
packages = append(packages, details)
65+
}
66+
67+
return packages, nil
68+
}
69+
70+
var _ Extractor = PdmLockExtractor{}
71+
72+
//nolint:gochecknoinits
73+
func init() {
74+
registerExtractor("pdm.lock", PdmLockExtractor{})
75+
}
76+
77+
func ParsePdmLock(pathToLockfile string) ([]PackageDetails, error) {
78+
return extractFromFile(pathToLockfile, PdmLockExtractor{})
79+
}

0 commit comments

Comments
 (0)