Merge pull request #43735 from JuliaLang/backports-release-1.6

release-1.6: Backports for 1.6.6

Author: KristofferC

Makefile

@@ -48,7 +48,7 @@ $(BUILDROOT)/doc/_build/html/en/index.html: $(shell find $(BUILDROOT)/base $(BUI
 
 julia-symlink: julia-cli-$(JULIA_BUILD_MODE)
 ifeq ($(OS),WINNT)
-	@echo '@"%~dp0\'"$$(echo $(call rel_path,$(BUILDROOT),$(JULIA_EXECUTABLE)) | tr / '\\')"\" '%*' > $(BUILDROOT)/julia.bat
+	echo '@"%~dp0/'"$$(echo '$(call rel_path,$(BUILDROOT),$(JULIA_EXECUTABLE))')"'" %*' | tr / '\\' > $(BUILDROOT)/julia.bat
 	chmod a+x $(BUILDROOT)/julia.bat
 else
 ifndef JULIA_VAGRANT_BUILD

base/Base.jl

@@ -244,9 +244,6 @@ include("weakkeydict.jl")
 include("logging.jl")
 using .CoreLogging
 
-# BinaryPlatforms, used by Artifacts
-include("binaryplatforms.jl")
-
 # functions defined in Random
 function rand end
 function randn end
@@ -298,6 +295,9 @@ using .Order
 include("sort.jl")
 using .Sort
 
+# BinaryPlatforms, used by Artifacts.  Needs `Sort`.
+include("binaryplatforms.jl")
+
 # Fast math
 include("fastmath.jl")
 using .FastMath

base/abstractdict.jl

@@ -155,7 +155,10 @@ empty(a::AbstractDict) = empty(a, keytype(a), valtype(a))
 empty(a::AbstractDict, ::Type{V}) where {V} = empty(a, keytype(a), V) # Note: this is the form which makes sense for `Vector`.
 
 copy(a::AbstractDict) = merge!(empty(a), a)
-copy!(dst::AbstractDict, src::AbstractDict) = merge!(empty!(dst), src)
+function copy!(dst::AbstractDict, src::AbstractDict)
+    dst === src && return dst
+    merge!(empty!(dst), src)
+end
 
 """
     merge!(d::AbstractDict, others::AbstractDict...)

base/abstractset.jl

@@ -3,7 +3,10 @@
 eltype(::Type{<:AbstractSet{T}}) where {T} = @isdefined(T) ? T : Any
 sizehint!(s::AbstractSet, n) = nothing
 
-copy!(dst::AbstractSet, src::AbstractSet) = union!(empty!(dst), src)
+function copy!(dst::AbstractSet, src::AbstractSet)
+    dst === src && return dst
+    union!(empty!(dst), src)
+end
 
 ## set operations (union, intersection, symmetric difference)
 

base/asyncmap.jl

@@ -15,7 +15,7 @@ up to 100 tasks will be used for concurrent mapping.
 
 `ntasks` can also be specified as a zero-arg function. In this case, the
 number of tasks to run in parallel is checked before processing every element and a new
-task started if the value of `ntasks_func` is less than the current number
+task started if the value of `ntasks_func` is greater than the current number
 of tasks.
 
 If `batch_size` is specified, the collection is processed in batch mode. `f` must

base/binaryplatforms.jl

@@ -608,7 +608,8 @@ const arch_march_isa_mapping = let
             "armv8_0" => get_set("aarch64", "armv8.0-a"),
             "armv8_1" => get_set("aarch64", "armv8.1-a"),
             "armv8_2_crypto" => get_set("aarch64", "armv8.2-a+crypto"),
-            "armv8_4_crypto_sve" => get_set("aarch64", "armv8.4-a+crypto+sve"),
+            "a64fx" => get_set("aarch64", "a64fx"),
+            "apple_m1" => get_set("aarch64", "apple_m1"),
         ],
         "powerpc64le" => [
             "power8" => get_set("powerpc64le", "power8"),

base/cmd.jl

@@ -230,7 +230,7 @@ byteenv(env::Union{AbstractVector{Pair{T,V}}, Tuple{Vararg{Pair{T,V}}}}) where {
     String[cstr(k*"="*string(v)) for (k,v) in env]
 
 """
-    setenv(command::Cmd, env; dir="")
+    setenv(command::Cmd, env; dir)
 
 Set environment variables to use when running the given `command`. `env` is either a
 dictionary mapping strings to strings, an array of strings of the form `"var=val"`, or
@@ -239,11 +239,22 @@ existing environment, create `env` through `copy(ENV)` and then setting `env["va
 as desired, or use `addenv`.
 
 The `dir` keyword argument can be used to specify a working directory for the command.
+`dir` defaults to the currently set `dir` for `command` (which is the current working
+directory if not specified already).
 """
-setenv(cmd::Cmd, env; dir="") = Cmd(cmd; env=byteenv(env), dir=dir)
-setenv(cmd::Cmd, env::Pair{<:AbstractString}...; dir="") =
+setenv(cmd::Cmd, env; dir=cmd.dir) = Cmd(cmd; env=byteenv(env), dir=dir)
+setenv(cmd::Cmd, env::Pair{<:AbstractString}...; dir=cmd.dir) =
     setenv(cmd, env; dir=dir)
-setenv(cmd::Cmd; dir="") = Cmd(cmd; dir=dir)
+setenv(cmd::Cmd; dir=cmd.dir) = Cmd(cmd; dir=dir)
+
+# split environment entry string into before and after first `=` (key and value)
+function splitenv(e::String)
+    i = findnext('=', e, 2)
+    if i === nothing
+        throw(ArgumentError("malformed environment entry"))
+    end
+    e[1:prevind(e, i)], e[nextind(e, i):end]
+end
 
 """
     addenv(command::Cmd, env...; inherit::Bool = true)
@@ -262,7 +273,7 @@ function addenv(cmd::Cmd, env::Dict; inherit::Bool = true)
             merge!(new_env, ENV)
         end
     else
-        for (k, v) in split.(cmd.env, "=")
+        for (k, v) in splitenv.(cmd.env)
             new_env[string(k)::String] = string(v)::String
         end
     end
@@ -277,7 +288,7 @@ function addenv(cmd::Cmd, pairs::Pair{<:AbstractString}...; inherit::Bool = true
 end
 
 function addenv(cmd::Cmd, env::Vector{<:AbstractString}; inherit::Bool = true)
-    return addenv(cmd, Dict(k => v for (k, v) in split.(env, "=")); inherit)
+    return addenv(cmd, Dict(k => v for (k, v) in splitenv.(env)); inherit)
 end
 
 (&)(left::AbstractCmd, right::AbstractCmd) = AndCmds(left, right)

base/cpuid.jl

@@ -56,9 +56,10 @@ const ISAs_by_family = Dict(
     "aarch64" => [
         # Implicit in all sets, because always required: fp, asimd
         "armv8.0-a" => ISA(Set{UInt32}()),
-        "armv8.1-a" => ISA(Set((JL_AArch64_lse, JL_AArch64_crc, JL_AArch64_rdm))),
-        "armv8.2-a+crypto" => ISA(Set((JL_AArch64_lse, JL_AArch64_crc, JL_AArch64_rdm, JL_AArch64_aes, JL_AArch64_sha2))),
-        "armv8.4-a+crypto+sve" => ISA(Set((JL_AArch64_lse, JL_AArch64_crc, JL_AArch64_rdm, JL_AArch64_fp16fml, JL_AArch64_aes, JL_AArch64_sha2, JL_AArch64_dotprod, JL_AArch64_sve))),
+        "armv8.1-a" => ISA(Set((JL_AArch64_v8_1a, JL_AArch64_lse, JL_AArch64_crc, JL_AArch64_rdm))),
+        "armv8.2-a+crypto" => ISA(Set((JL_AArch64_v8_2a, JL_AArch64_lse, JL_AArch64_crc, JL_AArch64_rdm, JL_AArch64_aes, JL_AArch64_sha2))),
+        "a64fx" => ISA(Set((JL_AArch64_v8_2a, JL_AArch64_lse, JL_AArch64_crc, JL_AArch64_rdm, JL_AArch64_sha2, JL_AArch64_ccpp, JL_AArch64_complxnum, JL_AArch64_fullfp16, JL_AArch64_sve))),
+        "apple_m1" => ISA(Set((JL_AArch64_v8_5a, JL_AArch64_lse, JL_AArch64_crc, JL_AArch64_rdm, JL_AArch64_aes, JL_AArch64_sha2, JL_AArch64_sha3, JL_AArch64_ccpp, JL_AArch64_complxnum, JL_AArch64_fp16fml, JL_AArch64_fullfp16, JL_AArch64_dotprod, JL_AArch64_rcpc, JL_AArch64_altnzcv))),
     ],
     "powerpc64le" => [
         # We have no way to test powerpc64le features yet, so we're only going to declare the lowest ISA:
@@ -88,14 +89,27 @@ function normalize_arch(arch::String)
     return arch
 end
 
+let
+    # Collect all relevant features for the current architecture, if any.
+    FEATURES = UInt32[]
+    arch = normalize_arch(String(Sys.ARCH))
+    if arch in keys(ISAs_by_family)
+        for isa in ISAs_by_family[arch]
+            unique!(append!(FEATURES, last(isa).features))
+        end
+    end
+
+    # Use `@eval` to inline the list of features.
+    @eval function cpu_isa()
+        return ISA(Set{UInt32}(feat for feat in $(FEATURES) if test_cpu_feature(feat)))
+    end
+end
+
 """
     cpu_isa()
 
 Return the [`ISA`](@ref) (instruction set architecture) of the current CPU.
 """
-function cpu_isa()
-    all_features = last(last(get(ISAs_by_family, normalize_arch(String(Sys.ARCH)), "" => [ISA(Set{UInt32}())]))).features
-    return ISA(Set{UInt32}(feat for feat in all_features if test_cpu_feature(feat)))
-end
+cpu_isa
 
 end # module CPUID

base/expr.jl

@@ -422,15 +422,16 @@ macro generated(f)
     if isa(f, Expr) && (f.head === :function || is_short_function_def(f))
         body = f.args[2]
         lno = body.args[1]
+        tmp = gensym("tmp")
         return Expr(:escape,
                     Expr(f.head, f.args[1],
                          Expr(:block,
                               lno,
                               Expr(:if, Expr(:generated),
                                    # https://github.com/JuliaLang/julia/issues/25678
                                    Expr(:block,
-                                        :(local tmp = $body),
-                                        :(if tmp isa Core.CodeInfo; return tmp; else tmp; end)),
+                                        :(local $tmp = $body),
+                                        :(if $tmp isa $(GlobalRef(Core, :CodeInfo)); return $tmp; else $tmp; end)),
                                    Expr(:block,
                                         Expr(:meta, :generated_only),
                                         Expr(:return, nothing))))))

base/io.jl

@@ -20,7 +20,7 @@ struct SystemError <: Exception
     extrainfo
     SystemError(p::AbstractString, e::Integer, extrainfo) = new(p, e, extrainfo)
     SystemError(p::AbstractString, e::Integer) = new(p, e, nothing)
-    SystemError(p::AbstractString) = new(p, Libc.errno())
+    SystemError(p::AbstractString) = new(p, Libc.errno(), nothing)
 end
 
 lock(::IO) = nothing

base/loading.jl

@@ -1571,7 +1571,8 @@ function srctext_files(f::IO, srctextpos::Int64)
 end
 
 # Test to see if this UUID is mentioned in this `Project.toml`; either as
-# the top-level UUID (e.g. that of the project itself) or as a dependency.
+# the top-level UUID (e.g. that of the project itself), as a dependency,
+# or as a extra for Preferences.
 function get_uuid_name(project::Dict{String, Any}, uuid::UUID)
     uuid_p = get(project, "uuid", nothing)::Union{Nothing, String}
     name = get(project, "name", nothing)::Union{Nothing, String}
@@ -1586,6 +1587,16 @@ function get_uuid_name(project::Dict{String, Any}, uuid::UUID)
             end
         end
     end
+    for subkey in ("deps", "extras")
+        subsection = get(project, subkey, nothing)::Union{Nothing, Dict{String, Any}}
+        if subsection !== nothing
+            for (k, v) in subsection
+                if uuid == UUID(v::String)
+                    return k
+                end
+            end
+        end
+    end
     return nothing
 end
 

base/process.jl

@@ -389,13 +389,26 @@ Returns the value returned by `f`.
 """
 function open(f::Function, cmds::AbstractCmd, args...; kwargs...)
     P = open(cmds, args...; kwargs...)
+    function waitkill(P::Process)
+        close(P)
+        # 0.1 seconds after we hope it dies (from closing stdio),
+        # we kill the process with SIGTERM (15)
+        local t = Timer(0.1) do t
+            process_running(P) && kill(P)
+        end
+        wait(P)
+        close(t)
+    end
     ret = try
         f(P)
     catch
-        kill(P)
+        waitkill(P)
         rethrow()
-    finally
-        close(P.in)
+    end
+    close(P.in)
+    if !eof(P.out)
+        waitkill(P)
+        throw(_UVError("open(do)", UV_EPIPE))
     end
     success(P) || pipeline_error(P)
     return ret

base/sort.jl

@@ -651,6 +651,8 @@ end
 
 defalg(v::AbstractArray) = DEFAULT_STABLE
 defalg(v::AbstractArray{<:Union{Number, Missing}}) = DEFAULT_UNSTABLE
+defalg(v::AbstractArray{Missing}) = DEFAULT_UNSTABLE
+defalg(v::AbstractArray{Union{}}) = DEFAULT_UNSTABLE
 
 function sort!(v::AbstractVector, alg::Algorithm, order::Ordering)
     inds = axes(v,1)

base/util.jl

@@ -571,6 +571,9 @@ function runtests(tests = ["all"]; ncores::Int = ceil(Int, Sys.CPU_THREADS::Int
     seed !== nothing && push!(tests, "--seed=0x$(string(seed % UInt128, base=16))") # cast to UInt128 to avoid a minus sign
     ENV2 = copy(ENV)
     ENV2["JULIA_CPU_THREADS"] = "$ncores"
+    delete!(ENV2, "JULIA_LOAD_PATH")
+    delete!(ENV2, "JULIA_PROJECT")
+    ENV2["JULIA_DEPOT_PATH"] = mktempdir(; cleanup = true)
     try
         run(setenv(`$(julia_cmd()) $(joinpath(Sys.BINDIR::String,
             Base.DATAROOTDIR, "julia", "test", "runtests.jl")) $tests`, ENV2))

contrib/generate_precompile.jl

@@ -178,6 +178,11 @@ if Test !== nothing
     """
 end
 
+const JULIA_PROMPT = "julia> "
+const PKG_PROMPT = "pkg> "
+const SHELL_PROMPT = "shell> "
+const HELP_PROMPT = "help?> "
+
 function generate_precompile_statements()
     start_time = time_ns()
     debug_output = devnull # or stdout
@@ -261,7 +266,7 @@ function generate_precompile_statements()
             end
         end
         # wait for the definitive prompt before start writing to the TTY
-        readuntil(output_copy, "julia>")
+        readuntil(output_copy, JULIA_PROMPT)
         sleep(0.1)
         readavailable(output_copy)
         # Input our script
@@ -279,9 +284,16 @@ function generate_precompile_statements()
                 write(ptm, l, "\n")
                 readuntil(output_copy, "\n")
                 # wait for the next prompt-like to appear
-                # NOTE: this is rather inaccurate because the Pkg REPL mode is a special flower
                 readuntil(output_copy, "\n")
-                readuntil(output_copy, "> ")
+                strbuf = ""
+                while true
+                    strbuf *= String(readavailable(output_copy))
+                    occursin(JULIA_PROMPT, strbuf) && break
+                    occursin(PKG_PROMPT, strbuf) && break
+                    occursin(SHELL_PROMPT, strbuf) && break
+                    occursin(HELP_PROMPT, strbuf) && break
+                    sleep(0.1)
+                end
             end
             println()
         end
@@ -344,6 +356,7 @@ generate_precompile_statements()
 
 # As a last step in system image generation,
 # remove some references to build time environment for a more reproducible build.
+Base.Filesystem.temp_cleanup_purge(force=true)
 @eval Base PROGRAM_FILE = ""
 @eval Sys begin
     BINDIR = ""

contrib/new-stdlib.sh

@@ -50,14 +50,14 @@ UNAME=$(echo "$NAME" | tr [a-z] [A-Z])
 
 sed -e "/^STDLIBS_EXT =/,/^\$/s!^\$!\\
 STDLIBS_EXT += $NAME\\
-${UNAME}_GIT_URL := git://github.com/$USER/$NAME.jl.git\\
-${UNAME}_TAR_URL = https://api.github.com/repos/$USER/$NAME.jl/tarball/\$1\\
 !" "$ROOT/Makefile" >"$ROOT/Makefile.tmp"
 mv "$ROOT/Makefile.tmp" "$ROOT/Makefile"
 
 cat >"$ROOT/$NAME.version" <<EOF
 ${UNAME}_BRANCH = master
 ${UNAME}_SHA1 = $SHA1
+${UNAME}_GIT_URL := https://github.com/$USER/$NAME.jl.git
+${UNAME}_TAR_URL = https://api.github.com/repos/$USER/$NAME.jl/tarball/\$1
 EOF
 
 git add "$ROOT/$NAME.version"

deps/blas.mk

@@ -1,7 +1,7 @@
 ## OpenBLAS ##
 ifneq ($(USE_BINARYBUILDER_OPENBLAS), 1)
 # LAPACK is built into OpenBLAS by default
-OPENBLAS_GIT_URL := git://github.com/xianyi/OpenBLAS.git
+OPENBLAS_GIT_URL := https://github.com/xianyi/OpenBLAS.git
 OPENBLAS_TAR_URL = https://api.github.com/repos/xianyi/OpenBLAS/tarball/$1
 $(eval $(call git-external,openblas,OPENBLAS,,,$(BUILDDIR)))
 

deps/checksums/Pkg-30a31381d06d845999314d39005ee8457bd08924.tar.gz/md5

@@ -0,0 +1 @@
+945e57c4f3c20aa69069b89eb050a3e5

deps/checksums/Pkg-30a31381d06d845999314d39005ee8457bd08924.tar.gz/sha512

@@ -0,0 +1 @@
+5b026a801e60023598b51ee4d24bb6fd19eb811f7f7840fb88ece2335bbe1048f43fa94e7dcd65239c2b50540feab4028fabe4cb4fa5eafdb6075fe2b336bca8

deps/checksums/Pkg-3d41ca188d72e1dc1862e780f31153d629d6a3dd.tar.gz/md5

@@ -1,7 +1,7 @@
 ## libgit2
 ifneq ($(USE_BINARYBUILDER_LIBGIT2),1)
 
-LIBGIT2_GIT_URL := git://github.com/libgit2/libgit2.git
+LIBGIT2_GIT_URL := https://github.com/libgit2/libgit2.git
 LIBGIT2_TAR_URL = https://api.github.com/repos/libgit2/libgit2/tarball/$1
 $(eval $(call git-external,libgit2,LIBGIT2,CMakeLists.txt,,$(SRCCACHE)))
 

deps/checksums/Pkg-3d41ca188d72e1dc1862e780f31153d629d6a3dd.tar.gz/sha512

@@ -1,6 +1,6 @@
 ## libssh2
 ifneq ($(USE_BINARYBUILDER_LIBSSH2), 1)
-LIBSSH2_GIT_URL := git://github.com/libssh2/libssh2.git
+LIBSSH2_GIT_URL := https://github.com/libssh2/libssh2.git
 LIBSSH2_TAR_URL = https://api.github.com/repos/libssh2/libssh2/tarball/$1
 $(eval $(call git-external,libssh2,LIBSSH2,CMakeLists.txt,,$(SRCCACHE)))
 

deps/libgit2.mk

@@ -1,6 +1,6 @@
 ## LIBUV ##
 ifneq ($(USE_BINARYBUILDER_LIBUV),1)
-LIBUV_GIT_URL:=git://github.com/JuliaLang/libuv.git
+LIBUV_GIT_URL:=https://github.com/JuliaLang/libuv.git
 LIBUV_TAR_URL=https://api.github.com/repos/JuliaLang/libuv/tarball/$1
 $(eval $(call git-external,libuv,LIBUV,configure,,$(SRCCACHE)))
 

deps/libssh2.mk

@@ -1,5 +1,5 @@
 ## LIBWHICH ##
-LIBWHICH_GIT_URL := git://github.com/vtjnash/libwhich.git
+LIBWHICH_GIT_URL := https://github.com/vtjnash/libwhich.git
 LIBWHICH_TAR_URL = https://api.github.com/repos/vtjnash/libwhich/tarball/$1
 $(eval $(call git-external,libwhich,LIBWHICH,,,$(BUILDDIR)))
 

deps/libuv.mk

@@ -1,6 +1,6 @@
 ## openlibm ##
 ifneq ($(USE_BINARYBUILDER_OPENLIBM), 1)
-OPENLIBM_GIT_URL := git://github.com/JuliaMath/openlibm.git
+OPENLIBM_GIT_URL := https://github.com/JuliaMath/openlibm.git
 OPENLIBM_TAR_URL = https://api.github.com/repos/JuliaMath/openlibm/tarball/$1
 $(eval $(call git-external,openlibm,OPENLIBM,,,$(BUILDDIR)))