Merge remote-tracking branch 'origin/master' into 34268-show-on-sparse-adjoints

Author: simeonschaub

.buildkite/0_webui.yml

@@ -0,0 +1,24 @@
+# This file represents what is put into the webUI.
+# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect.
+# We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps.
+agents:
+  queue: "julia"
+  sandbox.jl: "true"
+
+steps:
+  - label: ":unlock: Unlock secrets, launch pipelines"
+    plugins:
+      - staticfloat/cryptic:
+          # Our list of pipelines that should be launched (but don't require a signature)
+          # These pipelines can be modified by any contributor and CI will still run.
+          # Build secrets will not be available in these pipelines (or their children)
+          # but some of our signed pipelines can wait upon the completion of these unsigned
+          # pipelines.
+          unsigned_pipelines:
+            - .buildkite/pipeline.yml
+
+          # Our signed pipelines must have a `signature` or `signature_file` parameter that
+          # verifies the treehash of the pipeline itself and the inputs listed in `inputs`
+          signed_pipelines:
+            - pipeline: .buildkite/signed_pipeline_test.yml
+              signature: "U2FsdGVkX18aZgryp6AJTArgD2uOnVWyFFGVOP5qsY4WbGQ/LVAcYiMEp9cweV+2iht+vmEF949CuuGTeQPA1fKlhPwkG3nZ688752DUB6en9oM2nuL31NoDKWHhpygZ"

.buildkite/coverage-linux64/0_webui.yml

@@ -0,0 +1,16 @@
+# This file represents what is put into the webUI.
+# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect.
+# We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps.
+agents:
+  queue: "julia"
+  sandbox.jl: "true"
+
+steps:
+  - label: ":unlock: Unlock secrets, launch pipelines"
+    plugins:
+      - staticfloat/cryptic:
+          # Our signed pipelines must have a `signature` or `signature_file` parameter that
+          # verifies the treehash of the pipeline itself and the inputs listed in `inputs`
+          signed_pipelines:
+            - pipeline: .buildkite/coverage-linux64/pipeline.yml
+              signature: "U2FsdGVkX18eQWpd3hMYLO5Kd+6K+oBoLk1I6J3qIw7lc6g5/jaeWyq/wralosZCfTzyjS4NstNKFvhQf3KDPEBVElipNvTxoWOjVLRVOrfBqqvTkQN4xVosY/r026Gy"

.buildkite/coverage-linux64/README.md

@@ -0,0 +1,6 @@
+# Coverage pipeline
+
+We run coverage on a separate pipeline, that uses a scheduled build rather than webhooks.
+The pipeline is here: https://buildkite.com/julialang/julia-coverage-linux64
+
+It contains [its own webui steps](0_webuiy.ml) (listed here in this repository for clarity) and its own [pipeline.yml](pipeline.yml).

.buildkite/coverage-linux64/pipeline.yml

@@ -0,0 +1,44 @@
+# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones
+# since we need nestable sandboxing.  The rootfs images being used here are built from
+# the `.buildkite/rootfs_images/llvm-passes.jl` file.
+agents:
+  queue: "julia"
+  # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
+  sandbox.jl: "true"
+  os: "linux"
+
+steps:
+  - label: ":unlock: :coverage: Run coverage test"
+    plugins:
+      - staticfloat/cryptic:
+          variables:
+            - CODECOV_TOKEN="U2FsdGVkX19l0fhdBabbuiEdysyEabkJLRHfxm7CNRkuGbnwPV365sxxC7Czs/CVcws0N1oB4pVwALRRMe36oA=="
+            - COVERALLS_TOKEN="U2FsdGVkX19zopI0hMNzzi2UUOvNVFD8Y0iisFnO/ryVxU7Tit8ZEaeN+gxodRx4CosUUh192F1+q3dTMWRIvw=="
+      - JuliaCI/julia#v1:
+          version: 1.6
+      - staticfloat/sandbox#v1:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz
+          rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8"
+          uid: 1000
+          gid: 1000
+    commands: |
+      echo "--- Build Julia from source"
+      make -j 6
+
+      echo "--- Print Julia version info"
+      ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
+      ./julia -e '@info "" Sys.CPU_THREADS'
+      # this is necessary to make sure that the LibGit2 tests passes
+      git config --global init.defaultBranch master
+
+      echo "--- Run Julia tests with code coverage enabled"
+      # Run the actual tests
+      ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_base.jl
+
+      echo "--- Process and upload coverage information"
+      ./julia .buildkite/coverage-linux64/upload_coverage.jl
+    timeout_in_minutes: 600 # 600 minutes = 10 hours
+
+# We must accept the signed job id secret in order to propagate secrets
+env:
+  BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?}

.buildkite/coverage-linux64/run_tests_base.jl

@@ -0,0 +1,36 @@
+# When running this file, make sure to set the `--code-coverage=all` command-line flag.
+
+# Important note: even if one or more tests fail, we will still exit with status code 0.
+
+# The reason for this is that we always want to upload code coverage, even if some of the
+# tests fail. Therefore, even if the `coverage-linux64` pipeline passes, you should not
+# assume that all of the tests passed. If you want to know if all of the tests are passing,
+# please look at the status of the `tester_linux64` pipeline.
+
+const include_tests = String[]
+
+const exclude_tests = String[]
+
+empty!(Base.DEPOT_PATH)
+push!(Base.DEPOT_PATH, mktempdir(; cleanup = true))
+
+module ChooseTests
+    include(joinpath(dirname(dirname(@__DIR__)), "test", "choosetests.jl"))
+end
+
+const tests = ChooseTests.choosetests() |>
+              first |>
+              x -> setdiff(x, exclude_tests) |>
+              x -> vcat(x, include_tests) |>
+              unique |>
+              sort
+
+const ncores = min(Sys.CPU_THREADS, Threads.nthreads())
+
+@info "" ncores Sys.CPU_THREADS Threads.nthreads()
+
+try
+    Base.runtests(tests; ncores)
+catch ex
+    @error "" exception=(ex, catch_backtrace())
+end

.buildkite/coverage-linux64/upload_coverage.jl

@@ -0,0 +1,219 @@
+empty!(Base.DEPOT_PATH)
+push!(Base.DEPOT_PATH, mktempdir(; cleanup = true))
+
+import Pkg
+import Logging
+import TOML
+
+Pkg.add(; name = "Coverage", uuid = "a2441757-f6aa-5fb2-8edb-039e3f45d037", version = "1")
+Pkg.precompile()
+
+import Coverage
+
+function process_folders()
+    # `Coverage.process_folder` will have a LOT of `@info` statements that will make the log
+    # way too long. So before we run `Coverage.process_folder`, we disable logging for `@info`
+    # statements. After we run `Coverage.process_folder`, we re-enable logging for `@info`
+    # statements.
+    Logging.disable_logging(Logging.Info)
+    fcs_base   = Coverage.process_folder("base");
+    fcs_stdlib = Coverage.process_folder("stdlib");
+    Logging.disable_logging(Logging.Debug)
+
+    fcs = Coverage.merge_coverage_counts(
+        fcs_base,
+        fcs_stdlib,
+    );
+
+    return fcs
+end
+
+function get_external_stdlib_names(stdlib_dir::AbstractString)
+    filename_list = filter(x -> isfile(joinpath(stdlib_dir, x)), readdir(stdlib_dir))
+    # find all of the files like `Pkg.version`, `Statistics.version`, etc.
+    regex_matches_or_nothing = match.(Ref(r"^([\w].*?)\.version$"), filename_list)
+    regex_matches = filter(x -> x !== nothing, regex_matches_or_nothing)
+    # get the names of the external stdlibs, like `Pkg`, `Statistics`, etc.
+    external_stdlib_names = only.(regex_matches)
+    unique!(external_stdlib_names)
+    sort!(external_stdlib_names)
+    @info "# Begin list of external stdlibs"
+    for (i, x) in enumerate(external_stdlib_names)
+        @info "$(i). $(x)"
+    end
+    @info "# End list of external stdlibs"
+    return external_stdlib_names
+end
+
+function get_external_stdlib_prefixes(stdlib_dir::AbstractString)
+    external_stdlib_names = get_external_stdlib_names(stdlib_dir)
+    prefixes_1 = joinpath.(Ref(stdlib_dir), external_stdlib_names, Ref(""))
+    prefixes_2 = joinpath.(Ref(stdlib_dir), string.(external_stdlib_names, Ref("-")))
+    prefixes = vcat(prefixes_1, prefixes_2)
+    unique!(prefixes)
+    sort!(prefixes)
+    # example of what `prefixes` might look like:
+    # 4-element Vector{String}:
+    # "stdlib/Pkg-"
+    # "stdlib/Pkg/"
+    # "stdlib/Statistics-"
+    # "stdlib/Statistics/"
+    return prefixes
+end
+
+function print_coverage_summary(fc::Coverage.FileCoverage)
+    cov_lines, tot_lines = Coverage.get_summary(fc)
+    if cov_lines == tot_lines == 0
+        cov_pct = 0
+    else
+        cov_pct = floor(Int, cov_lines/tot_lines * 100)
+    end
+    pad_1 = 71
+    pad_2 = 15
+    pad_3 = 15
+    col_1 = rpad(fc.filename, pad_1)
+    col_2 = rpad(string(cov_pct, " %"), pad_2)
+    col_3 = string(
+        rpad(string(cov_lines), pad_3),
+        string(tot_lines),
+    )
+    @info "$(col_1) $(col_2) $(col_3)"
+    return nothing
+end
+
+function print_coverage_summary(
+        fcs::Vector{Coverage.FileCoverage}, description::AbstractString,
+    )
+    cov_lines, tot_lines = Coverage.get_summary(fcs)
+    if cov_lines == tot_lines == 0
+        cov_pct = 0
+    else
+        cov_pct = floor(Int, cov_lines/tot_lines * 100)
+    end
+    @info "$(description): $(cov_pct)% ($(cov_lines)/$(tot_lines))"
+    return nothing
+end
+
+function buildkite_env(name::String)
+    value = String(strip(ENV[name]))
+    if isempty(value)
+        throw(ErrorException("environment variable $(name) is empty"))
+    end
+    return value
+end
+
+function buildkite_env(name_1::String, name_2::String, default::String)
+    value_1 = String(strip(ENV[name_1]))
+    value_2 = String(strip(ENV[name_2]))
+    !isempty(value_1) && return value_1
+    !isempty(value_2) && return value_2
+    return default
+end
+
+function buildkite_branch_and_commit()
+    branch = buildkite_env("BUILDKITE_BRANCH")
+    commit = buildkite_env("BUILDKITE_COMMIT")
+    head_rev_parse = String(strip(read(`git rev-parse HEAD`, String)))
+    if strip(commit) == "HEAD"
+        commit = head_rev_parse
+    end
+    if commit !== head_rev_parse
+        msg = "mismatch"
+        @error msg commit head_rev_parse
+        throw(ErrorException(msg))
+    end
+    if !occursin(r"^[a-f0-9]{40}$", commit)
+        msg = "BUILDKITE_COMMIT does not look like a long commit SHA"
+        @error msg commit
+        throw(ErrorException(msg))
+    end
+    return (; branch, commit)
+end
+
+function codecov_buildkite_add_local_to_kwargs()
+    branch, commit = buildkite_branch_and_commit()
+    kwargs = Coverage.Codecov.set_defaults(
+        Dict();
+        branch,
+        commit,
+    )
+    return kwargs
+end
+
+function coveralls_buildkite_query_git_info()
+    branch, commit = buildkite_branch_and_commit()
+    remote_name  = "origin"
+    remote       = buildkite_env("BUILDKITE_REPO")
+    message      = buildkite_env("BUILDKITE_MESSAGE")
+    author_name  = buildkite_env(
+        "BUILDKITE_BUILD_AUTHOR",
+        "BUILDKITE_BUILD_CREATOR",
+        "",
+    )
+    author_email = buildkite_env(
+        "BUILDKITE_BUILD_AUTHOR_EMAIL",
+        "BUILDKITE_BUILD_CREATOR_EMAIL",
+        "",
+    )
+    remotes = [
+        Dict(
+            "name"  => remote_name,
+            "url"   => remote,
+        )
+    ]
+    head = Dict(
+        "id"                => commit,
+        "author_name"       => author_name,
+        "author_email"      => author_email,
+        "committer_name"    => author_name,
+        "committer_email"   => author_email,
+        "message"           => message,
+    )
+    git_info = Dict(
+        "branch"  => branch,
+        "remotes" => remotes,
+        "head"    => head,
+    )
+    return git_info
+end
+
+const fcs = process_folders()
+
+# Only include source code files. Exclude test files, benchmarking files, etc.
+filter!(fcs) do fc
+    occursin(r"^base\/", fc.filename) || occursin("/src/", fc.filename)
+end;
+
+# Exclude all external stdlibs (stdlibs that live in external repos).
+const external_stdlib_prefixes = get_external_stdlib_prefixes("stdlib")
+filter!(fcs) do fc
+    all(x -> !startswith(fc.filename, x), external_stdlib_prefixes)
+end;
+
+# Exclude all stdlib JLLs (stdlibs of the form `stdlib/*_jll/`).
+filter!(fcs) do fc
+    !occursin(r"^stdlib\/[A-Za-z0-9]*?_jll\/", fc.filename)
+end;
+
+sort!(fcs; by = fc -> fc.filename);
+
+print_coverage_summary.(fcs);
+print_coverage_summary(fcs, "Total")
+
+let
+    git_info = coveralls_buildkite_query_git_info()
+    @info "" git_info
+    @info "" git_info["branch"]
+    @info "" git_info["head"]
+
+    # In order to upload to Coveralls, you need to have the `COVERALLS_TOKEN` environment variable defined.
+    Coverage.Coveralls.submit_local(fcs, git_info)
+end
+
+let
+    kwargs = codecov_buildkite_add_local_to_kwargs()
+    @info "" kwargs
+
+    # In order to upload to Codecov, you need to have the `CODECOV_TOKEN` environment variable defined.
+    Coverage.Codecov.submit_generic(fcs, kwargs)
+end

.buildkite/cryptic_repo_keys/README.md

@@ -0,0 +1,6 @@
+## Cryptic repository keys
+
+This folder contains RSA-encrypted symmetric AES keys.
+These are used by buildkite agents to decrypt the secrets embedded within this repository.
+Each buildkite agent contains an RSA secret key that is used to unlock the symmetric AES key that was used to encrypt the secrets within this repository.
+For more information, see the [`cryptic` buildkite plugin repository](https://github.com/staticfloat/cryptic-buildkite-plugin).

.buildkite/cryptic_repo_keys/repo_key.2297e5e7

@@ -0,0 +1,40 @@
+# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones
+# since we need nestable sandboxing.  The rootfs images being used here are built from
+# the `.buildkite/rootfs_images/llvm-passes.jl` file.
+agents:
+  queue: "julia"
+  # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
+  sandbox.jl: "true"
+  os: "linux"
+
+steps:
+  - label: "analyzegc"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: 1.6
+      - staticfloat/sandbox#v1:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz
+          rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8"
+    commands: |
+      echo "--- Install in-tree LLVM dependencies"
+      make -j 6 -C deps install-llvm install-clang install-llvm-tools install-libuv install-utf8proc install-unwind
+      echo "+++ run clangsa/analyzegc"
+      make -j 6 -C test/clangsa
+      make -j 6 -C src analyzegc
+    timeout_in_minutes: 60
+
+  - label: "llvmpasses"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: 1.6
+      - staticfloat/sandbox#v1:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz
+          rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8"
+          uid: 1000
+          gid: 1000
+    commands: |
+      echo "+++ run llvmpasses"
+      make -j 6 release
+      make -j 6 -C src install-analysis-deps
+      make -j 6 -C test/llvmpasses
+    timeout_in_minutes: 60