Merge pull request #44324 from JuliaLang/backports-release-1.8

Backports for julia 1.8.0-beta2

Author: KristofferC

.buildkite/pipelines/main/misc/sanitizers.yml

@@ -21,7 +21,6 @@ steps:
     timeout_in_minutes: 120
     if: | # We only run the `asan` job on Julia 1.8 and later.
       (pipeline.slug != "julia-release-1-dot-6") && (pipeline.slug != "julia-release-1-dot-7")
-    soft_fail: true # TODO: delete this line (and thus disallow failures) once JuliaLang/julia#42540 is fixed
     commands: |
       echo "--- Build julia-debug with ASAN"
       contrib/asan/build.sh ./tmp/test-asan -j$${JULIA_CPU_THREADS:?} debug

Make.inc

@@ -1,4 +1,4 @@
-# -*- mode: makefile-gmake -*-
+# -*- mode: makefile -*-
 # vi:syntax=make
 
 ## Note:
@@ -483,9 +483,6 @@ endif
 endif
 
 ifeq ($(USEGCC),1)
-ifeq ($(SANITIZE),1)
-$(error Sanitizers are only supported with clang. Try setting SANITIZE=0)
-endif
 CC := $(CROSS_COMPILE)gcc
 CXX := $(CROSS_COMPILE)g++
 JCFLAGS := -std=gnu99 -pipe $(fPIC) -fno-strict-aliasing -D_FILE_OFFSET_BITS=64
@@ -519,6 +516,8 @@ JCPPFLAGS += -D_LARGEFILE_SOURCE -D_DARWIN_USE_64_BIT_INODE=1
 endif
 endif
 
+JLDFLAGS :=
+
 ifeq ($(USECCACHE), 1)
 # Expand CC, CXX and FC here already because we want the original definition and not the ccache version.
 CC_ARG   := $(CC)
@@ -1237,15 +1236,11 @@ IFUNC_DETECT_SRC := 'void (*f0(void))(void) { return (void(*)(void))0L; }; void
 ifeq (supported, $(shell echo $(IFUNC_DETECT_SRC) | $(CC) -Werror -x c - -S -o /dev/null > /dev/null 2>&1 && echo supported))
 JCPPFLAGS += -DJULIA_HAS_IFUNC_SUPPORT=1
 endif
-JLDFLAGS := -Wl,-Bdynamic
-ifneq ($(SANITIZE),1)
-ifneq ($(SANITIZE_MEMORY),1)
-ifneq ($(LLVM_SANITIZE),1)
+JLDFLAGS += -Wl,-Bdynamic
 OSLIBS += -Wl,--version-script=$(JULIAHOME)/src/julia.expmap
+ifneq ($(SANITIZE),1)
 JLDFLAGS += -Wl,-no-undefined
 endif
-endif
-endif
 ifeq (-Bsymbolic-functions, $(shell $(LD) --help | grep -o -e "-Bsymbolic-functions"))
 JLIBLDFLAGS := -Wl,-Bsymbolic-functions
 else
@@ -1256,7 +1251,7 @@ JLIBLDFLAGS :=
 endif
 
 ifeq ($(OS), FreeBSD)
-JLDFLAGS := -Wl,-Bdynamic
+JLDFLAGS += -Wl,-Bdynamic
 OSLIBS += -lelf -lkvm -lrt -lpthread -latomic
 
 # Tweak order of libgcc_s in DT_NEEDED,
@@ -1274,7 +1269,6 @@ SHLIB_EXT := dylib
 OSLIBS += -framework CoreFoundation
 WHOLE_ARCHIVE := -Xlinker -all_load
 NO_WHOLE_ARCHIVE :=
-JLDFLAGS :=
 HAVE_SSP := 1
 JLIBLDFLAGS := -Wl,-compatibility_version,$(SOMAJOR) -Wl,-current_version,$(JULIA_MAJOR_VERSION).$(JULIA_MINOR_VERSION).$(JULIA_PATCH_VERSION)
 endif
@@ -1283,7 +1277,7 @@ ifeq ($(OS), WINNT)
 HAVE_SSP := 1
 OSLIBS += -Wl,--export-all-symbols -Wl,--version-script=$(JULIAHOME)/src/julia.expmap \
 	$(NO_WHOLE_ARCHIVE) -lpsapi -lkernel32 -lws2_32 -liphlpapi -lwinmm -ldbghelp -luserenv -lsecur32 -latomic
-JLDFLAGS := -Wl,--stack,8388608
+JLDFLAGS += -Wl,--stack,8388608
 ifeq ($(ARCH),i686)
 JLDFLAGS += -Wl,--large-address-aware
 endif
@@ -1478,7 +1472,7 @@ endif
 endif
 ifeq ($(OS),Darwin)
 ifeq ($(ARCH),aarch64)
-LIBGCC_NAME := libgcc_s.2.$(SHLIB_EXT)
+LIBGCC_NAME := libgcc_s.1.1.$(SHLIB_EXT)
 else
 LIBGCC_NAME := libgcc_s.1.$(SHLIB_EXT)
 endif

Makefile

@@ -427,9 +427,21 @@ ifeq ($(OS), Linux)
 endif
 ifeq ($(OS), WINNT)
 	cd $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin && rm -f llvm* llc.exe lli.exe opt.exe LTO.dll bugpoint.exe macho-dump.exe
+endif
+	# If we're on macOS, and we have a codesigning identity, then codesign the binary-dist tarball!
+ifeq ($(OS),Darwin)
+ifneq ($(MACOS_CODESIGN_IDENTITY),)
+	echo "Codesigning with identity $(MACOS_CODESIGN_IDENTITY)"; \
+	MACHO_FILES=$$(find "$(BUILDROOT)/julia-$(JULIA_COMMIT)" -type f -perm -0111 | cut -d: -f1); \
+	for f in $${MACHO_FILES}; do \
+		echo "Codesigning $${f}..."; \
+		codesign -s "$(MACOS_CODESIGN_IDENTITY)" --option=runtime --entitlements $(JULIAHOME)/contrib/mac/app/Entitlements.plist -vvv --timestamp --deep --force "$${f}"; \
+	done
+endif
 endif
 	cd $(BUILDROOT) && $(TAR) zcvf $(JULIA_BINARYDIST_FILENAME).tar.gz julia-$(JULIA_COMMIT)
 
+
 exe:
 	# run Inno Setup to compile installer
 	$(call spawn,$(JULIAHOME)/dist-extras/inno/iscc.exe /DAppVersion=$(JULIA_VERSION) /DSourceDir="$(call cygpath_w,$(BUILDROOT)/julia-$(JULIA_COMMIT))" /DRepoDir="$(call cygpath_w,$(JULIAHOME))" /F"$(JULIA_BINARYDIST_FILENAME)" /O"$(call cygpath_w,$(BUILDROOT))" $(INNO_ARGS) $(call cygpath_w,$(JULIAHOME)/contrib/windows/build-installer.iss))

NEWS.md

@@ -13,11 +13,11 @@ New language features
   no errors were thrown ([#42211]).
 * `@inline` and `@noinline` annotations can now be placed within a function body ([#41312]).
 * `@inline` and `@noinline` annotations can now be applied to a function call site or block
-  to enforce the involved function calls to be (or not to be) inlined ([#41312]).
+  to enforce the involved function calls to be (or not to be) inlined ([#41328]).
 * `∀`, `∃`, and `∄` are now allowed as identifier characters ([#42314]).
 * Support for Unicode 14.0.0 ([#43443]).
 * `Module(:name, false, false)` can be used to create a `module` that contains no names
-  (it does not import `Base` or `Core` and does not contain a reference to itself) ([#40110, #42154]).
+  (it does not import `Base` or `Core` and does not contain a reference to itself) ([#40110], [#42154]).
 
 Language changes
 ----------------
@@ -54,6 +54,12 @@ Compiler/Runtime improvements
   Code heavily dependent on constant propagation should see significant compile-time performance improvements and
   certain cases (e.g. calls to uninlinable functions that are nevertheless effect free) should see runtime performance
   improvements. Effects may be overwritten manually with the `@Base.assume_effects` macro ([#43852]).
+* Precompilation (with explicit `precompile` directives or representative workloads) now saves more type-inferred code,
+  resulting in reduced time-to-first task for packages that use precompilation.  This change also eliminates the
+  runtime performance degradation occasionally triggered by precompilation on older Julia versions. More specifically,
+  any newly-inferred method/type combinations needed by your package--regardless of where those methods were
+  defined--can now be cached in the precompile file, as long as they are inferrably called by a method owned by
+  your package ([#43990]).
 
 Command-line option changes
 ---------------------------
@@ -111,6 +117,26 @@ Standard library changes
 * Intersect returns a result with the eltype of the type-promoted eltypes of the two inputs ([#41769]).
 * Iterating an `Iterators.Reverse` now falls back on reversing the eachindex iterator, if possible ([#43110]).
 
+#### Package Manager
+
+* New `⌃` and `⌅` indicators beside packages in `pkg> status` that have new versions available.
+  `⌅` indicates when new versions cannot be installed ([Pkg#2906]).
+* New `outdated::Bool` kwarg to `Pkg.status` (`--outdated` or `-o` in the REPL mode) to show
+  information about packages not at the latest version ([Pkg#2284]).
+* New `compat::Bool` kwarg to `Pkg.status` (`--compat` or `-c` in the REPL mode) to show any [compat]
+  entries in the Project.toml ([Pkg#2702]).
+* New `pkg> compat` (and `Pkg.compat`) mode for setting Project compat entries. Provides an interactive editor
+  via `pkg> compat`, or direct entry manipulation via `pkg> Foo 0.4,0.5` which can load current entries via tab-completion.
+  i.e. `pkg> compat Fo<TAB>` autocompletes to `pkg> Foo 0.4,0.5` so that the existing entry can be edited ([Pkg#2702]).
+* Pkg now only tries to download packages from the package server in case the server tracks a registry that contains
+  the package ([Pkg#2689]).
+* `Pkg.instantiate` will now warn when a Project.toml is out of sync with a Manifest.toml. It does this by storing a hash
+  of the project deps and compat entries (other fields are ignored) in the manifest when it is resolved, so that any change
+  to the Project.toml deps or compat entries without a re-resolve can be detected ([Pkg#2815]).
+* If `pkg> add` cannot find a package with the provided name it will now suggest similarly named packages that can be added ([Pkg#2985]).
+* The julia version stored in the manifest no longer includes the build number i.e. master will now record as `1.9.0-DEV` ([Pkg#2995]).
+* Interrupting a `pkg> test` will now be caught more reliably and exit back to the REPL gracefully ([Pkg#2933]).
+
 #### InteractiveUtils
 
 * New macro `@time_imports` for reporting any time spent importing packages and their dependencies ([#41612]).
@@ -212,6 +238,16 @@ Tooling Improvements
 * `GC.enable_logging(true)` can be used to log each garbage collection, with the
   time it took and the amount of memory that was collected ([#43511]).
 
+<!-- manually added -->
+[Pkg#2284]: https://github.com/JuliaLang/Pkg.jl/issues/2284
+[Pkg#2689]: https://github.com/JuliaLang/Pkg.jl/issues/2689
+[Pkg#2702]: https://github.com/JuliaLang/Pkg.jl/issues/2702
+[Pkg#2815]: https://github.com/JuliaLang/Pkg.jl/issues/2815
+[Pkg#2906]: https://github.com/JuliaLang/Pkg.jl/issues/2906
+[Pkg#2933]: https://github.com/JuliaLang/Pkg.jl/issues/2933
+[Pkg#2985]: https://github.com/JuliaLang/Pkg.jl/issues/2985
+[Pkg#2995]: https://github.com/JuliaLang/Pkg.jl/issues/2995
+
 <!--- generated by NEWS-update.jl: -->
 [#33711]: https://github.com/JuliaLang/julia/issues/33711
 [#36265]: https://github.com/JuliaLang/julia/issues/36265
@@ -220,12 +256,14 @@ Tooling Improvements
 [#38791]: https://github.com/JuliaLang/julia/issues/38791
 [#39241]: https://github.com/JuliaLang/julia/issues/39241
 [#39245]: https://github.com/JuliaLang/julia/issues/39245
+[#40110]: https://github.com/JuliaLang/julia/issues/40110
 [#40382]: https://github.com/JuliaLang/julia/issues/40382
 [#40642]: https://github.com/JuliaLang/julia/issues/40642
 [#40783]: https://github.com/JuliaLang/julia/issues/40783
 [#40980]: https://github.com/JuliaLang/julia/issues/40980
 [#41085]: https://github.com/JuliaLang/julia/issues/41085
 [#41312]: https://github.com/JuliaLang/julia/issues/41312
+[#41328]: https://github.com/JuliaLang/julia/issues/41328
 [#41449]: https://github.com/JuliaLang/julia/issues/41449
 [#41551]: https://github.com/JuliaLang/julia/issues/41551
 [#41576]: https://github.com/JuliaLang/julia/issues/41576
@@ -238,6 +276,7 @@ Tooling Improvements
 [#41794]: https://github.com/JuliaLang/julia/issues/41794
 [#41888]: https://github.com/JuliaLang/julia/issues/41888
 [#41936]: https://github.com/JuliaLang/julia/issues/41936
+[#42154]: https://github.com/JuliaLang/julia/issues/42154
 [#42211]: https://github.com/JuliaLang/julia/issues/42211
 [#42225]: https://github.com/JuliaLang/julia/issues/42225
 [#42248]: https://github.com/JuliaLang/julia/issues/42248
@@ -274,5 +313,6 @@ Tooling Improvements
 [#43852]: https://github.com/JuliaLang/julia/issues/43852
 [#43865]: https://github.com/JuliaLang/julia/issues/43865
 [#43919]: https://github.com/JuliaLang/julia/issues/43919
+[#43990]: https://github.com/JuliaLang/julia/issues/43990
 [#44080]: https://github.com/JuliaLang/julia/issues/44080
 [#44136]: https://github.com/JuliaLang/julia/issues/44136

base/Base.jl

@@ -427,7 +427,7 @@ end_base_include = time_ns()
 const _sysimage_modules = PkgId[]
 in_sysimage(pkgid::PkgId) = pkgid in _sysimage_modules
 
-# Precompiles for Revise
+# Precompiles for Revise and other packages
 # TODO: move these to contrib/generate_precompile.jl
 # The problem is they don't work there
 for match = _methods(+, (Int, Int), -1, get_world_counter())
@@ -461,6 +461,23 @@ for match = _methods(+, (Int, Int), -1, get_world_counter())
 
     # Code loading uses this
     sortperm(mtime.(readdir(".")), rev=true)
+    # JLLWrappers uses these
+    Dict{UUID,Set{String}}()[UUID("692b3bcd-3c85-4b1f-b108-f13ce0eb3210")] = Set{String}()
+    get!(Set{String}, Dict{UUID,Set{String}}(), UUID("692b3bcd-3c85-4b1f-b108-f13ce0eb3210"))
+    eachindex(IndexLinear(), Expr[])
+    push!(Expr[], Expr(:return, false))
+    vcat(String[], String[])
+    k, v = (:hello => nothing)
+    precompile(indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int))
+    precompile(indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int, Int))
+    # Preferences uses these
+    precompile(get_preferences, (UUID,))
+    precompile(record_compiletime_preference, (UUID, String))
+    get(Dict{String,Any}(), "missing", nothing)
+    delete!(Dict{String,Any}(), "missing")
+    for (k, v) in Dict{String,Any}()
+        println(k)
+    end
 
     break   # only actually need to do this once
 end

base/Makefile

@@ -206,7 +206,11 @@ else ifneq ($(USE_SYSTEM_OPENLIBM),0)
 $(eval $(call symlink_system_library,OPENLIBM,$(LIBMNAME)))
 endif
 
+ifeq ($(APPLE_ARCH),arm64)
+$(eval $(call symlink_system_library,CSL,libgcc_s,1.1))
+else
 $(eval $(call symlink_system_library,CSL,libgcc_s,1))
+endif
 ifneq (,$(LIBGFORTRAN_VERSION))
 $(eval $(call symlink_system_library,CSL,libgfortran,$(LIBGFORTRAN_VERSION)))
 endif

base/binaryplatforms.jl

@@ -40,10 +40,10 @@ struct Platform <: AbstractPlatform
     # The "compare strategy" allows selective overriding on how a tag is compared
     compare_strategies::Dict{String,Function}
 
-    function Platform(arch::String, os::String;
+    # Passing `tags` as a `Dict` avoids the need to infer different NamedTuple specializations
+    function Platform(arch::String, os::String, _tags::Dict{String};
                       validate_strict::Bool = false,
-                      compare_strategies::Dict{String,<:Function} = Dict{String,Function}(),
-                      kwargs...)
+                      compare_strategies::Dict{String,<:Function} = Dict{String,Function}())
         # A wee bit of normalization
         os = lowercase(os)
         arch = CPUID.normalize_arch(arch)
@@ -52,8 +52,9 @@ struct Platform <: AbstractPlatform
             "arch" => arch,
             "os" => os,
         )
-        for (tag, value) in kwargs
-            tag = lowercase(string(tag::Symbol))
+        for (tag, value) in _tags
+            value = value::Union{String,VersionNumber,Nothing}
+            tag = lowercase(tag)
             if tag ∈ ("arch", "os")
                 throw(ArgumentError("Cannot double-pass key $(tag)"))
             end
@@ -70,8 +71,8 @@ struct Platform <: AbstractPlatform
             if tag ∈ ("libgfortran_version", "libstdcxx_version", "os_version")
                 if isa(value, VersionNumber)
                     value = string(value)
-                elseif isa(value, AbstractString)
-                    v = tryparse(VersionNumber, String(value)::String)
+                elseif isa(value, String)
+                    v = tryparse(VersionNumber, value)
                     if isa(v, VersionNumber)
                         value = string(v)
                     end
@@ -110,6 +111,19 @@ struct Platform <: AbstractPlatform
     end
 end
 
+# Keyword interface (to avoid inference of specialized NamedTuple methods, use the Dict interface for `tags`)
+function Platform(arch::String, os::String;
+                  validate_strict::Bool = false,
+                  compare_strategies::Dict{String,<:Function} = Dict{String,Function}(),
+                  kwargs...)
+    tags = Dict{String,Any}(String(tag)::String=>tagvalue(value) for (tag, value) in kwargs)
+    return Platform(arch, os, tags; validate_strict, compare_strategies)
+end
+
+tagvalue(v::Union{String,VersionNumber,Nothing}) = v
+tagvalue(v::Symbol) = String(v)
+tagvalue(v::AbstractString) = convert(String, v)::String
+
 # Simple tag insertion that performs a little bit of validation
 function add_tag!(tags::Dict{String,String}, tag::String, value::String)
     # I know we said only alphanumeric and dots, but let's be generous so that we can expand
@@ -699,21 +713,22 @@ function Base.parse(::Type{Platform}, triplet::AbstractString; validate_strict::
         end
 
         # Extract the information we're interested in:
+        tags = Dict{String,Any}()
         arch = get_field(m, arch_mapping)
         os = get_field(m, os_mapping)
-        libc = get_field(m, libc_mapping)
-        call_abi = get_field(m, call_abi_mapping)
-        libgfortran_version = get_field(m, libgfortran_version_mapping)
-        libstdcxx_version = get_field(m, libstdcxx_version_mapping)
-        cxxstring_abi = get_field(m, cxxstring_abi_mapping)
+        tags["libc"] = get_field(m, libc_mapping)
+        tags["call_abi"] = get_field(m, call_abi_mapping)
+        tags["libgfortran_version"] = get_field(m, libgfortran_version_mapping)
+        tags["libstdcxx_version"] = get_field(m, libstdcxx_version_mapping)
+        tags["cxxstring_abi"] = get_field(m, cxxstring_abi_mapping)
         function split_tags(tagstr)
             tag_fields = split(tagstr, "-"; keepempty=false)
             if isempty(tag_fields)
                 return Pair{String,String}[]
             end
-            return map(v -> Symbol(v[1]) => v[2], split.(tag_fields, "+"))
+            return map(v -> String(v[1]) => String(v[2]), split.(tag_fields, "+"))
         end
-        tags = split_tags(m["tags"])
+        merge!(tags, Dict(split_tags(m["tags"])))
 
         # Special parsing of os version number, if any exists
         function extract_os_version(os_name, pattern)
@@ -730,18 +745,9 @@ function Base.parse(::Type{Platform}, triplet::AbstractString; validate_strict::
         if os == "freebsd"
             os_version = extract_os_version("freebsd", r".*freebsd([\d.]+)")
         end
+        tags["os_version"] = os_version
 
-        return Platform(
-            arch, os;
-            validate_strict,
-            libc,
-            call_abi,
-            libgfortran_version,
-            cxxstring_abi,
-            libstdcxx_version,
-            os_version,
-            tags...,
-        )
+        return Platform(arch, os, tags; validate_strict)
     end
     throw(ArgumentError("Platform `$(triplet)` is not an officially supported platform"))
 end
@@ -1068,4 +1074,9 @@ function select_platform(download_info::Dict, platform::AbstractPlatform = HostP
     return download_info[p]
 end
 
+# precompiles to reduce latency (see https://github.com/JuliaLang/julia/pull/43990#issuecomment-1025692379)
+Dict{Platform,String}()[HostPlatform()] = ""
+Platform("x86_64", "linux", Dict{String,Any}(); validate_strict=true)
+Platform("x86_64", "linux", Dict{String,String}(); validate_strict=false)  # called this way from Artifacts.unpack_platform
+
 end # module