Skip to content

Commit d50e4af

Browse files
epolackepolack
committed
regression tests & benchmark
1 parent 00c9c48 commit d50e4af

File tree

13 files changed

+413
-6
lines changed

13 files changed

+413
-6
lines changed

‎.github/workflows/ci.yaml

+5-5
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,11 @@ jobs:
2121
fail-fast: false
2222
matrix:
2323
include:
24-
- {mode: stable, os: ubuntu-latest, payload: noslow-example }
25-
- {mode: stable, os: macOS-latest, payload: noslow }
26-
- {mode: stable, os: windows-latest, payload: noslow }
27-
- {mode: stable, os: ubuntu-latest, payload: noslow-mpi }
28-
- {mode: nightly, os: ubuntu-latest, payload: noslow }
24+
- {mode: stable, os: ubuntu-latest, payload: example-noslow-noregression }
25+
- {mode: stable, os: macOS-latest, payload: noslow-noregression }
26+
- {mode: stable, os: windows-latest, payload: noslow-noregression }
27+
- {mode: stable, os: ubuntu-latest, payload: mpi-noslow-noregression }
28+
- {mode: nightly, os: ubuntu-latest, payload: noslow-noregression }
2929
env:
3030
GKS_ENCODING: utf8
3131
GKSwstype: 100 # Needed for Plots-related tests

‎.github/workflows/regression.yaml

+98
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
name: Regression
2+
on:
3+
push:
4+
branches:
5+
- master
6+
tags: ['*']
7+
pull_request:
8+
schedule:
9+
- cron: '0 4 * * 6' # Run every Saturday
10+
concurrency:
11+
# Skip intermediate builds: always.
12+
# Cancel intermediate builds: only if it is a pull request build.
13+
group: ${{ github.workflow }}-${{ github.ref }}
14+
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
15+
16+
jobs:
17+
test:
18+
name: Benchmarking ${{ matrix.description }}
19+
runs-on: ubuntu-latest
20+
strategy:
21+
fail-fast: false
22+
matrix:
23+
include:
24+
- {description: run, payload: benchmarks.jl }
25+
- {description: load, payload: load.jl }
26+
steps:
27+
# Remove older benchmark comment
28+
- name: pr-deleter
29+
uses: maheshrayas/[email protected]
30+
with:
31+
github_token: ${{ secrets.GITHUB_TOKEN }}
32+
org: <orgname>
33+
repo: <repo>
34+
user: github-actions[bot]
35+
issue: ${{github.event.number}}
36+
37+
- uses: actions/checkout@v4
38+
39+
- name: Setup Julia stable
40+
uses: julia-actions/setup-julia@v1
41+
with:
42+
version: '1.9'
43+
arch: x64
44+
45+
- uses: julia-actions/cache@v1
46+
with:
47+
include-matrix: false
48+
- uses: julia-actions/julia-buildpkg@v1
49+
50+
- name: Install dependencies
51+
run: |
52+
julia --project=benchmark -e '
53+
using Pkg
54+
Pkg.develop(PackageSpec(; path=pwd()))
55+
Pkg.instantiate()'
56+
57+
- name: Run benchmarks against master
58+
# Remove baseline once merged. Regression tests will only work after this is merged
59+
# in master.
60+
run: |
61+
julia --project=benchmark -e "
62+
using BenchmarkCI
63+
baseline = \"HEAD\"
64+
script = \"\$(pwd())/benchmark/${{ matrix.payload }}\"
65+
BenchmarkCI.judge(; baseline, script, retune=true)"
66+
if: ${{ github.event_name == 'pull_request' }}
67+
68+
- name: Run benchmarks against last release
69+
run: |
70+
julia --project=benchmark -e "
71+
import Pkg
72+
baseline = \"v\" * Pkg.TOML.parsefile(\"Project.toml\")[\"version\"]
73+
script = \"\$(pwd())/benchmark/${{ matrix.payload }}\"
74+
using BenchmarkCI
75+
BenchmarkCI.judge(; baseline, script, retune=true)"
76+
if: ${{ github.event_name == 'schedule' ||
77+
github.event.push.ref == 'refs/heads/master' }}
78+
79+
- name: Print judgement
80+
run: |
81+
julia --project=benchmark -e '
82+
using BenchmarkCI
83+
BenchmarkCI.displayjudgement()'
84+
85+
- name: Post results
86+
run: |
87+
julia --project=benchmark -e '
88+
using BenchmarkCI
89+
BenchmarkCI.postjudge()'
90+
env:
91+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
92+
93+
- name: Is report successful
94+
run: |
95+
res=$(julia --project=benchmark -e '
96+
using BenchmarkCI
97+
BenchmarkCI.displayjudgement()' | grep --count ':x:')
98+
if [[ $res -gt 1 ]]; then exit 1; fi

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,5 @@ Manifest.toml
1010
/LocalPreferences.toml
1111
.vscode
1212
.CondaPkg
13+
/.benchmarkci
14+
/benchmark/**/*.json

‎Project.toml

+2-1
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@ ASEconvert = "3da9722f-58c2-4165-81be-b4d7253e8fd2"
128128
Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
129129
AtomsIO = "1692102d-eeb4-4df9-807b-c9517f998d44"
130130
AtomsIOPython = "9e4c859b-2281-48ef-8059-f50fe53c37b0"
131+
BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
131132
CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
132133
CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2"
133134
ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66"
@@ -150,4 +151,4 @@ WriteVTK = "64499a7a-5c06-52f2-abe2-ccb03c286192"
150151
wannier90_jll = "c5400fa0-8d08-52c2-913f-1e3f656c1ce9"
151152

152153
[targets]
153-
test = ["Test", "TestItemRunner", "ASEconvert", "Aqua", "AtomsIO", "AtomsIOPython", "CUDA", "CUDA_Runtime_jll", "ComponentArrays", "DoubleFloats", "FiniteDiff", "FiniteDifferences", "GenericLinearAlgebra", "IntervalArithmetic", "JLD2", "JSON3", "Logging", "Plots", "QuadGK", "Random", "KrylovKit", "Wannier", "WriteVTK", "wannier90_jll"]
154+
test = ["Test", "TestItemRunner", "ASEconvert", "Aqua", "AtomsIO", "AtomsIOPython", "BenchmarkTools", "CUDA", "CUDA_Runtime_jll", "ComponentArrays", "DoubleFloats", "FiniteDiff", "FiniteDifferences", "GenericLinearAlgebra", "IntervalArithmetic", "JLD2", "JSON3", "Logging", "Plots", "QuadGK", "Random", "KrylovKit", "Wannier", "WriteVTK", "wannier90_jll"]

‎benchmark/Project.toml

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
[deps]
2+
AtomsBase = "a963bdd2-2df7-4f54-a1ee-49d51e6be12a"
3+
BenchmarkCI = "20533458-34a3-403d-a444-e18f38190b5b"
4+
BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
5+
DFTK = "acf6eb54-70d9-11e9-0013-234b7a5f5337"
6+
ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
7+
PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
8+
TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
9+
Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d"
10+
UnitfulAtomic = "a7773ee8-282e-5fa2-be4e-bd808c38a91a"

‎benchmark/benchmarks.jl

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
using BenchmarkTools
2+
using TestItemRunner
3+
4+
const SUITE = BenchmarkGroup()
5+
6+
@run_package_tests filter=ti->(:regression ∈ ti.tags)

‎benchmark/humongous/Project.toml

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
[deps]
2+
AtomsIO = "1692102d-eeb4-4df9-807b-c9517f998d44"
3+
BenchmarkCI = "20533458-34a3-403d-a444-e18f38190b5b"
4+
BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
5+
DFTK = "acf6eb54-70d9-11e9-0013-234b7a5f5337"
6+
LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
7+
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
8+
MKL = "33e6dc65-8f57-5167-99aa-e5a354878fb2"
9+
PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
10+
TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"

‎benchmark/humongous/benchmarks.jl

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
using BenchmarkTools
2+
using TestItemRunner
3+
4+
function run_scenario(scenario, complexity)
5+
scenario_filter(i) = occursin(string(scenario), i.filename) && complexity ∈ i.tags
6+
@run_package_tests filter=scenario_filter
7+
end
8+
9+
all_scenarios() = [:AlSiO2H, :Cr19, :Fe2MnAl, :Mn2RuGa, :WFe]
10+
function make_suite(; scenarios=all_scenarios(), complexity=:debug)
11+
@assert complexity ∈ [:debug, :small, :full]
12+
@assert all(scenarios .∈ Ref(all_scenarios()))
13+
14+
suite = BenchmarkGroup()
15+
for scenario in scenarios
16+
suite[scenario] = @benchmarkable run_scenario($scenario, $complexity)
17+
end
18+
suite
19+
end
20+
21+
const SUITE = make_suite(; scenarios=[:AlSiO2H])

‎benchmark/humongous/run.jl

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
ROOTPATH = abspath(joinpath(@__DIR__, "../.."))
2+
import Pkg
3+
Pkg.activate(@__DIR__)
4+
if !isfile(joinpath(@__DIR__, "Manifest.toml"))
5+
Pkg.develop(Pkg.PackageSpec(; path=ROOTPATH))
6+
Pkg.instantiate()
7+
end
8+
9+
import BenchmarkCI
10+
import LibGit2
11+
12+
"""
13+
Launch with
14+
```julia
15+
julia --project=benchmark/humongous -e '
16+
include("benchmark/humongous/run.jl")
17+
run_benchmark()'
18+
```
19+
"""
20+
function run_benchmark(; retune=false, baseline="origin/master", target="HEAD",
21+
script=nothing)
22+
mktempdir(mktempdir()) do repo_dir # TestItemRunner needs access to parent directory as well.
23+
project = joinpath(ROOTPATH, "benchmark", "humongous")
24+
# Workaround to be able to benchmark releases before the use of PkgBenchmark.
25+
# WARN: In this case, we need PkgBenchmark to be installed globally.
26+
if isnothing(script)
27+
# We run the default benchmark.
28+
script = joinpath(project, "benchmarks.jl")
29+
else
30+
occursin(ROOTPATH, abspath(script)) &&
31+
error("Script should be outside the repository.")
32+
end
33+
script_copy = joinpath(repo_dir, "benchmarks.jl")
34+
cp(script, script_copy)
35+
36+
LibGit2.clone("https://github.com/epolack/DFTK-testproblems",
37+
joinpath(repo_dir, "test"))
38+
39+
BenchmarkCI.judge(; baseline, target, retune, script=script_copy, project)
40+
41+
BenchmarkCI.displayjudgement()
42+
end
43+
end

‎benchmark/load.jl

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
using BenchmarkTools
2+
3+
const SUITE = BenchmarkGroup()
4+
5+
julia_cmd = unsafe_string(Base.JLOptions().julia_bin)
6+
SUITE["load"] = @benchmarkable run(`$julia_cmd \
7+
--startup-file=no \
8+
--project=$(Base.active_project()) \
9+
-e 'using DFTK'`)
10+
SUITE["pecompilation"] =
11+
@benchmarkable run(`$julia_cmd \
12+
--startup-file=no \
13+
--project=$(Base.active_project()) \
14+
-e 'Base.compilecache(Base.identify_package("DFTK"))'`)

0 commit comments

Comments
 (0)