Skip to content

Commit 62f97a5

Browse files
committed
regression tests & benchmark
1 parent fd1196b commit 62f97a5

File tree

11 files changed

+320
-6
lines changed

11 files changed

+320
-6
lines changed

.github/workflows/ci.yaml

+5-5
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,11 @@ jobs:
2121
fail-fast: false
2222
matrix:
2323
include:
24-
- {mode: stable, os: ubuntu-latest, payload: noslow-example }
25-
- {mode: stable, os: macOS-latest, payload: noslow }
26-
- {mode: stable, os: windows-latest, payload: noslow }
27-
- {mode: stable, os: ubuntu-latest, payload: noslow-mpi }
28-
- {mode: nightly, os: ubuntu-latest, payload: noslow }
24+
- {mode: stable, os: ubuntu-latest, payload: example-noslow-noregression }
25+
- {mode: stable, os: macOS-latest, payload: noslow-noregression }
26+
- {mode: stable, os: windows-latest, payload: noslow-noregression }
27+
- {mode: stable, os: ubuntu-latest, payload: mpi-noslow-noregression }
28+
- {mode: nightly, os: ubuntu-latest, payload: noslow-noregression }
2929
env:
3030
GKS_ENCODING: utf8
3131
GKSwstype: 100 # Needed for Plots-related tests

.github/workflows/regression.yaml

+74
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
name: Regression tests
2+
on:
3+
push:
4+
branches:
5+
- master
6+
tags: ['*']
7+
pull_request:
8+
schedule:
9+
- cron: '0 4 * * 6' # Run every Saturday
10+
concurrency:
11+
# Skip intermediate builds: always.
12+
# Cancel intermediate builds: only if it is a pull request build.
13+
group: ${{ github.workflow }}-${{ github.ref }}
14+
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
15+
16+
jobs:
17+
Benchmark:
18+
runs-on: ubuntu-latest
19+
steps:
20+
# Remove older benchmark comment
21+
- name: pr-deleter
22+
uses: maheshrayas/[email protected]
23+
with:
24+
github_token: ${{ secrets.GITHUB_TOKEN }}
25+
org: <orgname>
26+
repo: <repo>
27+
user: github-actions[bot]
28+
issue: ${{github.event.number}}
29+
- uses: actions/checkout@v2
30+
- uses: julia-actions/setup-julia@latest
31+
with:
32+
version: 1
33+
- uses: julia-actions/julia-buildpkg@latest
34+
- name: Install dependencies
35+
run: |
36+
julia --project=benchmark -e '
37+
using Pkg
38+
Pkg.develop(PackageSpec(; path=pwd()))
39+
Pkg.instantiate()'
40+
- name: Run benchmarks against master
41+
# Remove baseline once merged. Regression tests will only work after this is merged
42+
# in master.
43+
run: |
44+
julia --project=benchmark -e '
45+
using BenchmarkCI
46+
BenchmarkCI.judge(; baseline="HEAD", retune=true)'
47+
if: ${{ github.event_name == 'pull_request' }}
48+
- name: Run benchmarks against last release
49+
run: |
50+
julia --project=benchmark -e '
51+
import Pkg
52+
baseline="v"*Pkg.TOML.parsefile("Project.toml")["version"]
53+
using BenchmarkCI
54+
BenchmarkCI.judge(; baseline, retune=true)'
55+
if: ${{ github.event_name == 'schedule' ||
56+
github.event.push.ref == 'refs/heads/master' }}
57+
- name: Print judgement
58+
run: |
59+
julia --project=benchmark -e '
60+
using BenchmarkCI
61+
BenchmarkCI.displayjudgement()'
62+
- name: Post results
63+
run: |
64+
julia --project=benchmark -e '
65+
using BenchmarkCI
66+
BenchmarkCI.postjudge()'
67+
env:
68+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
69+
- name: Is report successful
70+
run: |
71+
res=$(julia --project=benchmark -e '
72+
using BenchmarkCI
73+
BenchmarkCI.displayjudgement()' | grep --count ':x:')
74+
if [[ $res -gt 1 ]]; then exit 1; fi

.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,5 @@ Manifest.toml
1010
/LocalPreferences.toml
1111
.vscode
1212
.CondaPkg
13+
/.benchmarkci
14+
/benchmark/**/*.json

Project.toml

+2-1
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@ ASEconvert = "3da9722f-58c2-4165-81be-b4d7253e8fd2"
128128
Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
129129
AtomsIO = "1692102d-eeb4-4df9-807b-c9517f998d44"
130130
AtomsIOPython = "9e4c859b-2281-48ef-8059-f50fe53c37b0"
131+
BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
131132
CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
132133
CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2"
133134
ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66"
@@ -150,4 +151,4 @@ WriteVTK = "64499a7a-5c06-52f2-abe2-ccb03c286192"
150151
wannier90_jll = "c5400fa0-8d08-52c2-913f-1e3f656c1ce9"
151152

152153
[targets]
153-
test = ["Test", "TestItemRunner", "ASEconvert", "Aqua", "AtomsIO", "AtomsIOPython", "CUDA", "CUDA_Runtime_jll", "ComponentArrays", "DoubleFloats", "FiniteDiff", "FiniteDifferences", "GenericLinearAlgebra", "IntervalArithmetic", "JLD2", "JSON3", "Logging", "Plots", "QuadGK", "Random", "KrylovKit", "Wannier", "WriteVTK", "wannier90_jll"]
154+
test = ["Test", "TestItemRunner", "ASEconvert", "Aqua", "AtomsIO", "AtomsIOPython", "BenchmarkTools", "CUDA", "CUDA_Runtime_jll", "ComponentArrays", "DoubleFloats", "FiniteDiff", "FiniteDifferences", "GenericLinearAlgebra", "IntervalArithmetic", "JLD2", "JSON3", "Logging", "Plots", "QuadGK", "Random", "KrylovKit", "Wannier", "WriteVTK", "wannier90_jll"]

benchmark/Project.toml

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
[deps]
2+
AtomsBase = "a963bdd2-2df7-4f54-a1ee-49d51e6be12a"
3+
BenchmarkCI = "20533458-34a3-403d-a444-e18f38190b5b"
4+
BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
5+
DFTK = "acf6eb54-70d9-11e9-0013-234b7a5f5337"
6+
ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
7+
PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
8+
TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
9+
Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d"
10+
UnitfulAtomic = "a7773ee8-282e-5fa2-be4e-bd808c38a91a"

benchmark/benchmarks.jl

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
using BenchmarkTools
2+
using TestItemRunner
3+
4+
const SUITE = BenchmarkGroup()
5+
6+
julia_cmd = unsafe_string(Base.JLOptions().julia_bin)
7+
SUITE["load"] = @benchmarkable run(`$julia_cmd \
8+
--startup-file=no \
9+
--project=$(Base.active_project()) \
10+
-e 'using DFTK'`)
11+
SUITE["pecompilation"] =
12+
@benchmarkable run(`$julia_cmd \
13+
--startup-file=no \
14+
--project=$(Base.active_project()) \
15+
-e 'Base.compilecache(Base.identify_package("DFTK"))'`)
16+
@run_package_tests filter=ti->(:regression ti.tags)

benchmark/humongous/Project.toml

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
[deps]
2+
AtomsIOPython = "9e4c859b-2281-48ef-8059-f50fe53c37b0"
3+
BenchmarkCI = "20533458-34a3-403d-a444-e18f38190b5b"
4+
BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
5+
DFTK = "acf6eb54-70d9-11e9-0013-234b7a5f5337"
6+
LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
7+
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
8+
MKL = "33e6dc65-8f57-5167-99aa-e5a354878fb2"
9+
PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
10+
TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"

benchmark/humongous/benchmarks.jl

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
using BenchmarkTools
2+
using TestItemRunner
3+
4+
const SUITE = BenchmarkGroup()
5+
6+
SUITE["AlSiO2H"] = BenchmarkGroup()
7+
SUITE["AlSiO2H"] = @benchmarkable @run_package_tests filter=(i->occursin("AlSiO2H.jl", i.filename) &&
8+
:debug i.tags)

benchmark/humongous/run.jl

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
ROOTPATH = joinpath(@__DIR__, "../..")
2+
import Pkg
3+
Pkg.activate(@__DIR__)
4+
if !isfile(joinpath(@__DIR__, "Manifest.toml"))
5+
Pkg.develop(Pkg.PackageSpec(; path=ROOTPATH))
6+
Pkg.instantiate()
7+
end
8+
9+
using BenchmarkCI
10+
11+
import LibGit2
12+
repo = mktempdir(mktempdir()) # TestItemRunner needs access to parent directory as well.
13+
LibGit2.clone("https://github.com/epolack/DFTK-testproblems", repo)
14+
15+
# Workaround to be able to benchmark releases before the use of PkgBenchmark.
16+
script = tempname(repo) * ".jl"
17+
benchpath = joinpath(ROOTPATH, "benchmark", "humongous", "benchmarks.jl")
18+
project = dirname(benchpath)
19+
cp(benchpath, script)
20+
21+
BenchmarkCI.judge(; retune=true, script, project)
22+
23+
BenchmarkCI.displayjudgement()

benchmark/regression/testcases.jl

+156
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
@testsetup module Regression
2+
using DFTK
3+
using Unitful
4+
using UnitfulAtomic
5+
using AtomsBase
6+
using ..TestCases: magnesium
7+
8+
high_symmetry = let
9+
a = 4.474
10+
lattice = [[0, a, a], [a, 0, a], [a, a, 0]]u"bohr"
11+
x = 6.711
12+
y = 2.237
13+
atoms = [
14+
Atom(:Cu, [0, 0, 0]u"bohr", magnetic_moment=0),
15+
Atom(:O, [x, y, x]u"bohr", magnetic_moment=0),
16+
Atom(:O, [x, y, y]u"bohr", magnetic_moment=0),
17+
]
18+
system = periodic_system(atoms, lattice)
19+
merge(DFTK.parse_system(system), (; temperature=0.03, Ecut=10, kgrid=[4,4,4],
20+
n_electrons=45))
21+
end
22+
high_kpoints = merge(magnesium, (; kgrid=[13,13,13], Ecut=10))
23+
high_Ecut = merge(magnesium, (; kgrid=[4,4,4], Ecut=60))
24+
testcases = (; high_symmetry, high_kpoints, high_Ecut)
25+
end
26+
27+
@testitem "Hamiltonian application" tags=[:regression] setup=[TestCases, Regression] begin
28+
using DFTK
29+
using LinearAlgebra
30+
using BenchmarkTools
31+
using .Main: SUITE
32+
33+
for testcase in Regression.testcases
34+
model = Model(testcase.lattice, testcase.atoms, testcase.positions;
35+
testcase.temperature, terms=[Kinetic()])
36+
basis = PlaneWaveBasis(model; testcase.Ecut, testcase.kgrid)
37+
38+
n_electrons = testcase.n_electrons
39+
n_bands = div(n_electrons, 2, RoundUp)
40+
ψ = [Matrix(qr(randn(ComplexF64, length(G_vectors(basis, kpt)), n_bands)).Q)
41+
for kpt in basis.kpoints]
42+
filled_occ = DFTK.filled_occupation(model)
43+
occupation = [filled_occ * rand(n_bands) for _ = 1:length(basis.kpoints)]
44+
occ_scaling = n_electrons / sum(sum(occupation))
45+
occupation = [occ * occ_scaling for occ in occupation]
46+
47+
(; ham) = energy_hamiltonian(basis, ψ, occupation)
48+
49+
SUITE["ham"] = BenchmarkGroup()
50+
SUITE["ham"] = @benchmarkable for ik = 1:length($(basis.kpoints))
51+
$(ham.blocks)[ik]*$ψ[ik]
52+
end
53+
end
54+
end
55+
56+
@testitem "Single SCF step" tags=[:regression] setup=[TestCases, Regression] begin
57+
using DFTK
58+
using BenchmarkTools
59+
using .Main: SUITE
60+
61+
for testcase in Regression.testcases
62+
model = model_LDA(testcase.lattice, testcase.atoms, testcase.positions;
63+
testcase.temperature)
64+
basis = PlaneWaveBasis(model; testcase.Ecut, testcase.kgrid)
65+
SUITE["scf"] = BenchmarkGroup()
66+
SUITE["scf"] = @benchmarkable self_consistent_field($basis; tol=1e5)
67+
end
68+
end
69+
70+
@testitem "Density + symmetrization" tags=[:regression] setup=[TestCases, Regression] begin
71+
using DFTK
72+
using BenchmarkTools
73+
using .Main: SUITE
74+
75+
for testcase in Regression.testcases
76+
model = model_LDA(testcase.lattice, testcase.atoms, testcase.positions;
77+
testcase.temperature)
78+
basis = PlaneWaveBasis(model; testcase.Ecut, testcase.kgrid)
79+
scfres = self_consistent_field(basis; tol=10)
80+
81+
ψ, occupation = DFTK.select_occupied_orbitals(basis, scfres.ψ, scfres.occupation;
82+
threshold=1e-6)
83+
84+
SUITE["density", "ρ", "sym"] = BenchmarkGroup()
85+
SUITE["density", "ρ"] = @benchmarkable compute_density($basis, $ψ, $occupation)
86+
SUITE["density", "sym"] = @benchmarkable DFTK.symmetrize_ρ($basis, $(scfres.ρ))
87+
end
88+
end
89+
90+
@testitem "Basis construction" tags=[:regression] setup=[TestCases, Regression] begin
91+
using DFTK
92+
using BenchmarkTools
93+
using .Main: SUITE
94+
95+
for testcase in Regression.testcases
96+
model = model_LDA(testcase.lattice, testcase.atoms, testcase.positions;
97+
testcase.temperature)
98+
SUITE["basis"] = BenchmarkGroup()
99+
SUITE["basis"] = @benchmarkable PlaneWaveBasis($model; Ecut=$(testcase.Ecut),
100+
kgrid=$(testcase.kgrid))
101+
end
102+
end
103+
104+
@testitem "Sternheimer" tags=[:regression] setup=[TestCases, Regression] begin
105+
using DFTK
106+
using BenchmarkTools
107+
using .Main: SUITE
108+
109+
for testcase in Regression.testcases
110+
model = model_LDA(testcase.lattice, testcase.atoms, testcase.positions;
111+
testcase.temperature)
112+
basis = PlaneWaveBasis(model; testcase.Ecut, testcase.kgrid)
113+
scfres = self_consistent_field(basis; tol=10)
114+
115+
rhs = DFTK.compute_projected_gradient(basis, scfres.ψ, scfres.occupation)
116+
SUITE["sternheimer"] = BenchmarkGroup()
117+
SUITE["sternheimer"] = @benchmarkable DFTK.solve_ΩplusK_split($scfres, $rhs)
118+
end
119+
end
120+
121+
@testitem "Response with AD" tags=[:regression] setup=[TestCases, Regression] begin
122+
using DFTK
123+
using BenchmarkTools
124+
using LinearAlgebra
125+
using ForwardDiff
126+
using .Main: SUITE
127+
128+
function make_basis::T; a=10., Ecut=30) where {T}
129+
lattice=T(a) * I(3) # lattice is a cube of ``a`` Bohrs
130+
# Helium at the center of the box
131+
atoms = [ElementPsp(:He; psp=load_psp("hgh/lda/He-q2"))]
132+
positions = [[1/2, 1/2, 1/2]]
133+
134+
model = model_DFT(lattice, atoms, positions, [:lda_x, :lda_c_vwn];
135+
extra_terms=[ExternalFromReal(r -> -ε * (r[1] - a/2))],
136+
symmetries=false)
137+
PlaneWaveBasis(model; Ecut, kgrid=[1, 1, 1]) # No k-point sampling on isolated system
138+
end
139+
140+
# dipole moment of a given density (assuming the current geometry)
141+
function dipole(basis, ρ)
142+
@assert isdiag(basis.model.lattice)
143+
a = basis.model.lattice[1, 1]
144+
rr = [a * (r[1] - 1/2) for r in r_vectors(basis)]
145+
sum(rr .* ρ) * basis.dvol
146+
end
147+
148+
# Function to compute the dipole for a given field strength
149+
function compute_dipole(ε; tol=1e-8, kwargs...)
150+
scfres = self_consistent_field(make_basis(ε; kwargs...); tol)
151+
dipole(scfres.basis, scfres.ρ)
152+
end
153+
154+
SUITE["response", "ad"] = BenchmarkGroup()
155+
SUITE["response", "ad"] = @benchmarkable ForwardDiff.derivative($compute_dipole, 0.0)
156+
end

benchmark/run.jl

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
ROOTPATH = joinpath(@__DIR__, "..")
2+
import Pkg
3+
Pkg.activate(@__DIR__)
4+
if !isfile(joinpath(@__DIR__, "Manifest.toml"))
5+
Pkg.develop(Pkg.PackageSpec(; path=ROOTPATH))
6+
Pkg.instantiate()
7+
end
8+
9+
using BenchmarkCI
10+
11+
# Remove target once merged. Regression tests will only work after this is merged in master.
12+
BenchmarkCI.judge(; baseline="HEAD")
13+
14+
BenchmarkCI.displayjudgement()

0 commit comments

Comments
 (0)